diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index fad2c1f026..acf0abeed1 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -233,7 +233,6 @@ OctreeServer::OctreeServer(ReceivedMessage& message) : _argc(0), _argv(NULL), _parsedArgV(NULL), - _httpManager(NULL), _statusPort(0), _packetsPerClientPerInterval(10), _packetsTotalPerInterval(DEFAULT_PACKETS_PER_INTERVAL), @@ -285,7 +284,7 @@ void OctreeServer::initHTTPManager(int port) { QString documentRoot = QString("%1/web").arg(PathUtils::getAppDataPath()); // setup an httpManager with us as the request handler and the parent - _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this); + _httpManager.reset(new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this)); } bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) { diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index b25e537d70..87145dd46e 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -183,7 +183,7 @@ protected: bool _isShuttingDown = false; - HTTPManager* _httpManager; + std::unique_ptr _httpManager; int _statusPort; QString _statusHost; diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index 47b55bb5c2..b4685b907f 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -16,6 +16,8 @@ #include #include +#include + #include #include diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 4e65df495c..42a74aa2fc 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -149,7 +149,6 @@ DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this), - _httpsManager(NULL), _allAssignments(), _unfulfilledAssignments(), _isUsingDTLS(false), @@ -385,6 +384,8 @@ DomainServer::~DomainServer() { _contentManager->terminate(); } + DependencyManager::destroy(); + // cleanup the AssetClient thread DependencyManager::destroy(); _assetClientThread.quit(); @@ -439,7 +440,7 @@ bool DomainServer::optionallyReadX509KeyAndCertificate() { QSslCertificate sslCertificate(&certFile); QSslKey privateKey(&keyFile, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey, keyPassphraseString.toUtf8()); - _httpsManager = new HTTPSManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTPS_PORT, sslCertificate, privateKey, QString(), this, this); + _httpsManager.reset(new HTTPSManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTPS_PORT, sslCertificate, privateKey, QString(), this)); qDebug() << "TCP server listening for HTTPS connections on" << DOMAIN_SERVER_HTTPS_PORT; diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 01adbd99a9..3462f14e3c 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -220,7 +220,7 @@ private: DomainGatekeeper _gatekeeper; HTTPManager _httpManager; - HTTPSManager* _httpsManager; + std::unique_ptr _httpsManager; QHash _allAssignments; QQueue _unfulfilledAssignments; diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index 3cf1e1450e..6896c7a9c9 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -13,6 +13,7 @@ #include +#include #include #include #include diff --git a/interface/src/SecondaryCamera.cpp b/interface/src/SecondaryCamera.cpp index db51cf99c8..b9a767f700 100644 --- a/interface/src/SecondaryCamera.cpp +++ b/interface/src/SecondaryCamera.cpp @@ -9,11 +9,13 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "Application.h" #include "SecondaryCamera.h" -#include -#include + #include +#include +#include + +#include "Application.h" using RenderArgsPointer = std::shared_ptr; diff --git a/libraries/embedded-webserver/src/HTTPConnection.cpp b/libraries/embedded-webserver/src/HTTPConnection.cpp index 12da599575..f65cd87f6e 100644 --- a/libraries/embedded-webserver/src/HTTPConnection.cpp +++ b/libraries/embedded-webserver/src/HTTPConnection.cpp @@ -11,6 +11,8 @@ #include "HTTPConnection.h" +#include + #include #include #include @@ -29,11 +31,92 @@ const char* HTTPConnection::StatusCode404 = "404 Not Found"; const char* HTTPConnection::StatusCode500 = "500 Internal server error"; const char* HTTPConnection::DefaultContentType = "text/plain; charset=ISO-8859-1"; -HTTPConnection::HTTPConnection (QTcpSocket* socket, HTTPManager* parentManager) : + +class MemoryStorage : public HTTPConnection::Storage { +public: + static std::unique_ptr make(qint64 size); + virtual ~MemoryStorage() = default; + + const QByteArray& content() const override { return _array; } + qint64 bytesLeftToWrite() const override { return _array.size() - _bytesWritten; } + void write(const QByteArray& data) override; + +private: + MemoryStorage(qint64 size) { _array.resize(size); } + + QByteArray _array; + qint64 _bytesWritten { 0 }; +}; + +std::unique_ptr MemoryStorage::make(qint64 size) { + return std::unique_ptr(new MemoryStorage(size)); +} + +void MemoryStorage::write(const QByteArray& data) { + assert(data.size() <= bytesLeftToWrite()); + memcpy(_array.data() + _bytesWritten, data.data(), data.size()); + _bytesWritten += data.size(); +} + + +class FileStorage : public HTTPConnection::Storage { +public: + static std::unique_ptr make(qint64 size); + virtual ~FileStorage(); + + const QByteArray& content() const override { return _wrapperArray; }; + qint64 bytesLeftToWrite() const override { return _mappedMemorySize - _bytesWritten; } + void write(const QByteArray& data) override; + +private: + FileStorage(std::unique_ptr file, uchar* mapped, qint64 size); + + // Byte array is const because any edit will trigger a deep copy + // and pull all the data we want to keep on disk in memory. + const QByteArray _wrapperArray; + std::unique_ptr _file; + + uchar* const _mappedMemoryAddress { nullptr }; + const qint64 _mappedMemorySize { 0 }; + qint64 _bytesWritten { 0 }; +}; + +std::unique_ptr FileStorage::make(qint64 size) { + auto file = std::unique_ptr(new QTemporaryFile()); + file->open(); // Open for resize + file->resize(size); + auto mapped = file->map(0, size); // map the entire file + + return std::unique_ptr(new FileStorage(std::move(file), mapped, size)); +} + +// Use QByteArray::fromRawData to avoid a new allocation and access the already existing +// memory directly as long as all operations on the array are const. +FileStorage::FileStorage(std::unique_ptr file, uchar* mapped, qint64 size) : + _wrapperArray(QByteArray::fromRawData(reinterpret_cast(mapped), size)), + _file(std::move(file)), + _mappedMemoryAddress(mapped), + _mappedMemorySize(size) +{ +} + +FileStorage::~FileStorage() { + _file->unmap(_mappedMemoryAddress); + _file->close(); +} + +void FileStorage::write(const QByteArray& data) { + assert(data.size() <= bytesLeftToWrite()); + // We write directly to the mapped memory + memcpy(_mappedMemoryAddress + _bytesWritten, data.data(), data.size()); + _bytesWritten += data.size(); +} + + +HTTPConnection::HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager) : QObject(parentManager), _parentManager(parentManager), _socket(socket), - _stream(socket), _address(socket->peerAddress()) { // take over ownership of the socket @@ -62,7 +145,7 @@ QHash HTTPConnection::parseUrlEncodedForm() { return QHash(); } - QUrlQuery form { _requestContent }; + QUrlQuery form { _requestContent->content() }; QHash pairs; for (auto pair : form.queryItems()) { auto key = QUrl::fromPercentEncoding(pair.first.toLatin1().replace('+', ' ')); @@ -97,7 +180,7 @@ QList HTTPConnection::parseFormData() const { QByteArray end = "\r\n--" + boundary + "--\r\n"; QList data; - QBuffer buffer(const_cast(&_requestContent)); + QBuffer buffer(const_cast(&_requestContent->content())); buffer.open(QIODevice::ReadOnly); while (buffer.canReadLine()) { QByteArray line = buffer.readLine().trimmed(); @@ -107,12 +190,13 @@ QList HTTPConnection::parseFormData() const { QByteArray line = buffer.readLine().trimmed(); if (line.isEmpty()) { // content starts after this line - int idx = _requestContent.indexOf(end, buffer.pos()); + int idx = _requestContent->content().indexOf(end, buffer.pos()); if (idx == -1) { qWarning() << "Missing end boundary." << _address; return data; } - datum.second = _requestContent.mid(buffer.pos(), idx - buffer.pos()); + datum.second = QByteArray::fromRawData(_requestContent->content().data() + buffer.pos(), + idx - buffer.pos()); data.append(datum); buffer.seek(idx + end.length()); @@ -256,7 +340,24 @@ void HTTPConnection::readHeaders() { _parentManager->handleHTTPRequest(this, _requestUrl); } else { - _requestContent.resize(clength.toInt()); + bool success = false; + auto length = clength.toInt(&success); + if (!success) { + qWarning() << "Invalid header." << _address << trimmed; + respond("400 Bad Request", "The header was malformed."); + return; + } + + // Storing big requests in memory gets expensive, especially on servers + // with limited memory. So we store big requests in a temporary file on disk + // and map it to faster read/write access. + static const int MAX_CONTENT_SIZE_IN_MEMORY = 10 * 1000 * 1000; + if (length < MAX_CONTENT_SIZE_IN_MEMORY) { + _requestContent = MemoryStorage::make(length); + } else { + _requestContent = FileStorage::make(length); + } + connect(_socket, SIGNAL(readyRead()), SLOT(readContent())); // read any content immediately available @@ -285,12 +386,13 @@ void HTTPConnection::readHeaders() { } void HTTPConnection::readContent() { - int size = _requestContent.size(); - if (_socket->bytesAvailable() < size) { - return; - } - _socket->read(_requestContent.data(), size); - _socket->disconnect(this, SLOT(readContent())); + auto size = std::min(_socket->bytesAvailable(), _requestContent->bytesLeftToWrite()); - _parentManager->handleHTTPRequest(this, _requestUrl.path()); + _requestContent->write(_socket->read(size)); + + if (_requestContent->bytesLeftToWrite() == 0) { + _socket->disconnect(this, SLOT(readContent())); + + _parentManager->handleHTTPRequest(this, _requestUrl.path()); + } } diff --git a/libraries/embedded-webserver/src/HTTPConnection.h b/libraries/embedded-webserver/src/HTTPConnection.h index e4d23e3c90..4b42acf296 100644 --- a/libraries/embedded-webserver/src/HTTPConnection.h +++ b/libraries/embedded-webserver/src/HTTPConnection.h @@ -16,14 +16,14 @@ #ifndef hifi_HTTPConnection_h #define hifi_HTTPConnection_h -#include #include -#include #include #include +#include #include #include #include +#include #include #include @@ -57,52 +57,63 @@ public: /// WebSocket close status codes. enum ReasonCode { NoReason = 0, NormalClosure = 1000, GoingAway = 1001 }; + class Storage { + public: + Storage() = default; + virtual ~Storage() = default; + + virtual const QByteArray& content() const = 0; + + virtual qint64 bytesLeftToWrite() const = 0; + virtual void write(const QByteArray& data) = 0; + }; + /// Initializes the connection. - HTTPConnection (QTcpSocket* socket, HTTPManager* parentManager); + HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager); /// Destroys the connection. - virtual ~HTTPConnection (); + virtual ~HTTPConnection(); /// Returns a pointer to the underlying socket, to which WebSocket message bodies should be written. - QTcpSocket* socket () const { return _socket; } + QTcpSocket* socket() const { return _socket; } /// Returns the request operation. - QNetworkAccessManager::Operation requestOperation () const { return _requestOperation; } + QNetworkAccessManager::Operation requestOperation() const { return _requestOperation; } /// Returns a reference to the request URL. - const QUrl& requestUrl () const { return _requestUrl; } + const QUrl& requestUrl() const { return _requestUrl; } /// Returns a copy of the request header value. If it does not exist, it will return a default constructed QByteArray. QByteArray requestHeader(const QString& key) const { return _requestHeaders.value(key.toLower().toLocal8Bit()); } /// Returns a reference to the request content. - const QByteArray& requestContent () const { return _requestContent; } + const QByteArray& requestContent() const { return _requestContent->content(); } /// Parses the request content as form data, returning a list of header/content pairs. - QList parseFormData () const; + QList parseFormData() const; /// Parses the request content as a url encoded form, returning a hash of key/value pairs. /// Duplicate keys are not supported. QHash parseUrlEncodedForm(); /// Sends a response and closes the connection. - void respond (const char* code, const QByteArray& content = QByteArray(), + void respond(const char* code, const QByteArray& content = QByteArray(), const char* contentType = DefaultContentType, const Headers& headers = Headers()); - void respond (const char* code, std::unique_ptr device, + void respond(const char* code, std::unique_ptr device, const char* contentType = DefaultContentType, const Headers& headers = Headers()); protected slots: /// Reads the request line. - void readRequest (); + void readRequest(); /// Reads the headers. - void readHeaders (); + void readHeaders(); /// Reads the content. - void readContent (); + void readContent(); protected: void respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 size); @@ -112,9 +123,6 @@ protected: /// The underlying socket. QTcpSocket* _socket; - - /// The data stream for writing to the socket. - QDataStream _stream; /// The stored address. QHostAddress _address; @@ -132,7 +140,7 @@ protected: QByteArray _lastRequestHeader; /// The content of the request. - QByteArray _requestContent; + std::unique_ptr _requestContent; /// Response content std::unique_ptr _responseDevice; diff --git a/libraries/embedded-webserver/src/HTTPManager.cpp b/libraries/embedded-webserver/src/HTTPManager.cpp index db6fc8e084..4a75994e12 100644 --- a/libraries/embedded-webserver/src/HTTPManager.cpp +++ b/libraries/embedded-webserver/src/HTTPManager.cpp @@ -24,8 +24,7 @@ const int SOCKET_ERROR_EXIT_CODE = 2; const int SOCKET_CHECK_INTERVAL_IN_MS = 30000; -HTTPManager::HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler, QObject* parent) : - QTcpServer(parent), +HTTPManager::HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler) : _listenAddress(listenAddress), _documentRoot(documentRoot), _requestHandler(requestHandler), diff --git a/libraries/embedded-webserver/src/HTTPManager.h b/libraries/embedded-webserver/src/HTTPManager.h index cb76eed9f2..597f6921cc 100644 --- a/libraries/embedded-webserver/src/HTTPManager.h +++ b/libraries/embedded-webserver/src/HTTPManager.h @@ -33,7 +33,7 @@ class HTTPManager : public QTcpServer, public HTTPRequestHandler { Q_OBJECT public: /// Initializes the manager. - HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler = NULL, QObject* parent = 0); + HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler = nullptr); bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override; diff --git a/libraries/embedded-webserver/src/HTTPSConnection.h b/libraries/embedded-webserver/src/HTTPSConnection.h index 7b53dc0063..3e6c752a08 100644 --- a/libraries/embedded-webserver/src/HTTPSConnection.h +++ b/libraries/embedded-webserver/src/HTTPSConnection.h @@ -23,4 +23,4 @@ protected slots: void handleSSLErrors(const QList& errors); }; -#endif // hifi_HTTPSConnection_h \ No newline at end of file +#endif // hifi_HTTPSConnection_h diff --git a/libraries/embedded-webserver/src/HTTPSManager.cpp b/libraries/embedded-webserver/src/HTTPSManager.cpp index 8ba44f98ac..95339a0dd6 100644 --- a/libraries/embedded-webserver/src/HTTPSManager.cpp +++ b/libraries/embedded-webserver/src/HTTPSManager.cpp @@ -16,8 +16,8 @@ #include "HTTPSConnection.h" HTTPSManager::HTTPSManager(QHostAddress listenAddress, quint16 port, const QSslCertificate& certificate, const QSslKey& privateKey, - const QString& documentRoot, HTTPSRequestHandler* requestHandler, QObject* parent) : - HTTPManager(listenAddress, port, documentRoot, requestHandler, parent), + const QString& documentRoot, HTTPSRequestHandler* requestHandler) : + HTTPManager(listenAddress, port, documentRoot, requestHandler), _certificate(certificate), _privateKey(privateKey), _sslRequestHandler(requestHandler) diff --git a/libraries/embedded-webserver/src/HTTPSManager.h b/libraries/embedded-webserver/src/HTTPSManager.h index 2d3cc9ed62..9cea32862c 100644 --- a/libraries/embedded-webserver/src/HTTPSManager.h +++ b/libraries/embedded-webserver/src/HTTPSManager.h @@ -31,7 +31,7 @@ public: const QSslCertificate& certificate, const QSslKey& privateKey, const QString& documentRoot, - HTTPSRequestHandler* requestHandler = NULL, QObject* parent = 0); + HTTPSRequestHandler* requestHandler = nullptr); void setCertificate(const QSslCertificate& certificate) { _certificate = certificate; } void setPrivateKey(const QSslKey& privateKey) { _privateKey = privateKey; } @@ -48,4 +48,4 @@ private: HTTPSRequestHandler* _sslRequestHandler; }; -#endif // hifi_HTTPSManager_h \ No newline at end of file +#endif // hifi_HTTPSManager_h