diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index 47b55bb5c2..b4685b907f 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -16,6 +16,8 @@ #include #include +#include + #include #include diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 991b8627d0..42a74aa2fc 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -384,6 +384,8 @@ DomainServer::~DomainServer() { _contentManager->terminate(); } + DependencyManager::destroy(); + // cleanup the AssetClient thread DependencyManager::destroy(); _assetClientThread.quit(); diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index 3cf1e1450e..6896c7a9c9 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -13,6 +13,7 @@ #include +#include #include #include #include diff --git a/libraries/embedded-webserver/src/HTTPConnection.cpp b/libraries/embedded-webserver/src/HTTPConnection.cpp index 12230e5e1d..f65cd87f6e 100644 --- a/libraries/embedded-webserver/src/HTTPConnection.cpp +++ b/libraries/embedded-webserver/src/HTTPConnection.cpp @@ -11,6 +11,8 @@ #include "HTTPConnection.h" +#include + #include #include #include @@ -29,6 +31,88 @@ const char* HTTPConnection::StatusCode404 = "404 Not Found"; const char* HTTPConnection::StatusCode500 = "500 Internal server error"; const char* HTTPConnection::DefaultContentType = "text/plain; charset=ISO-8859-1"; + +class MemoryStorage : public HTTPConnection::Storage { +public: + static std::unique_ptr make(qint64 size); + virtual ~MemoryStorage() = default; + + const QByteArray& content() const override { return _array; } + qint64 bytesLeftToWrite() const override { return _array.size() - _bytesWritten; } + void write(const QByteArray& data) override; + +private: + MemoryStorage(qint64 size) { _array.resize(size); } + + QByteArray _array; + qint64 _bytesWritten { 0 }; +}; + +std::unique_ptr MemoryStorage::make(qint64 size) { + return std::unique_ptr(new MemoryStorage(size)); +} + +void MemoryStorage::write(const QByteArray& data) { + assert(data.size() <= bytesLeftToWrite()); + memcpy(_array.data() + _bytesWritten, data.data(), data.size()); + _bytesWritten += data.size(); +} + + +class FileStorage : public HTTPConnection::Storage { +public: + static std::unique_ptr make(qint64 size); + virtual ~FileStorage(); + + const QByteArray& content() const override { return _wrapperArray; }; + qint64 bytesLeftToWrite() const override { return _mappedMemorySize - _bytesWritten; } + void write(const QByteArray& data) override; + +private: + FileStorage(std::unique_ptr file, uchar* mapped, qint64 size); + + // Byte array is const because any edit will trigger a deep copy + // and pull all the data we want to keep on disk in memory. + const QByteArray _wrapperArray; + std::unique_ptr _file; + + uchar* const _mappedMemoryAddress { nullptr }; + const qint64 _mappedMemorySize { 0 }; + qint64 _bytesWritten { 0 }; +}; + +std::unique_ptr FileStorage::make(qint64 size) { + auto file = std::unique_ptr(new QTemporaryFile()); + file->open(); // Open for resize + file->resize(size); + auto mapped = file->map(0, size); // map the entire file + + return std::unique_ptr(new FileStorage(std::move(file), mapped, size)); +} + +// Use QByteArray::fromRawData to avoid a new allocation and access the already existing +// memory directly as long as all operations on the array are const. +FileStorage::FileStorage(std::unique_ptr file, uchar* mapped, qint64 size) : + _wrapperArray(QByteArray::fromRawData(reinterpret_cast(mapped), size)), + _file(std::move(file)), + _mappedMemoryAddress(mapped), + _mappedMemorySize(size) +{ +} + +FileStorage::~FileStorage() { + _file->unmap(_mappedMemoryAddress); + _file->close(); +} + +void FileStorage::write(const QByteArray& data) { + assert(data.size() <= bytesLeftToWrite()); + // We write directly to the mapped memory + memcpy(_mappedMemoryAddress + _bytesWritten, data.data(), data.size()); + _bytesWritten += data.size(); +} + + HTTPConnection::HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager) : QObject(parentManager), _parentManager(parentManager), @@ -61,7 +145,7 @@ QHash HTTPConnection::parseUrlEncodedForm() { return QHash(); } - QUrlQuery form { _requestContent }; + QUrlQuery form { _requestContent->content() }; QHash pairs; for (auto pair : form.queryItems()) { auto key = QUrl::fromPercentEncoding(pair.first.toLatin1().replace('+', ' ')); @@ -96,7 +180,7 @@ QList HTTPConnection::parseFormData() const { QByteArray end = "\r\n--" + boundary + "--\r\n"; QList data; - QBuffer buffer(const_cast(&_requestContent)); + QBuffer buffer(const_cast(&_requestContent->content())); buffer.open(QIODevice::ReadOnly); while (buffer.canReadLine()) { QByteArray line = buffer.readLine().trimmed(); @@ -106,12 +190,13 @@ QList HTTPConnection::parseFormData() const { QByteArray line = buffer.readLine().trimmed(); if (line.isEmpty()) { // content starts after this line - int idx = _requestContent.indexOf(end, buffer.pos()); + int idx = _requestContent->content().indexOf(end, buffer.pos()); if (idx == -1) { qWarning() << "Missing end boundary." << _address; return data; } - datum.second = _requestContent.mid(buffer.pos(), idx - buffer.pos()); + datum.second = QByteArray::fromRawData(_requestContent->content().data() + buffer.pos(), + idx - buffer.pos()); data.append(datum); buffer.seek(idx + end.length()); @@ -255,7 +340,24 @@ void HTTPConnection::readHeaders() { _parentManager->handleHTTPRequest(this, _requestUrl); } else { - _requestContent.resize(clength.toInt()); + bool success = false; + auto length = clength.toInt(&success); + if (!success) { + qWarning() << "Invalid header." << _address << trimmed; + respond("400 Bad Request", "The header was malformed."); + return; + } + + // Storing big requests in memory gets expensive, especially on servers + // with limited memory. So we store big requests in a temporary file on disk + // and map it to faster read/write access. + static const int MAX_CONTENT_SIZE_IN_MEMORY = 10 * 1000 * 1000; + if (length < MAX_CONTENT_SIZE_IN_MEMORY) { + _requestContent = MemoryStorage::make(length); + } else { + _requestContent = FileStorage::make(length); + } + connect(_socket, SIGNAL(readyRead()), SLOT(readContent())); // read any content immediately available @@ -284,12 +386,13 @@ void HTTPConnection::readHeaders() { } void HTTPConnection::readContent() { - int size = _requestContent.size(); - if (_socket->bytesAvailable() < size) { - return; - } - _socket->read(_requestContent.data(), size); - _socket->disconnect(this, SLOT(readContent())); + auto size = std::min(_socket->bytesAvailable(), _requestContent->bytesLeftToWrite()); - _parentManager->handleHTTPRequest(this, _requestUrl.path()); + _requestContent->write(_socket->read(size)); + + if (_requestContent->bytesLeftToWrite() == 0) { + _socket->disconnect(this, SLOT(readContent())); + + _parentManager->handleHTTPRequest(this, _requestUrl.path()); + } } diff --git a/libraries/embedded-webserver/src/HTTPConnection.h b/libraries/embedded-webserver/src/HTTPConnection.h index 44a523423a..4b42acf296 100644 --- a/libraries/embedded-webserver/src/HTTPConnection.h +++ b/libraries/embedded-webserver/src/HTTPConnection.h @@ -16,14 +16,14 @@ #ifndef hifi_HTTPConnection_h #define hifi_HTTPConnection_h -#include #include -#include #include #include +#include #include #include #include +#include #include #include @@ -57,6 +57,17 @@ public: /// WebSocket close status codes. enum ReasonCode { NoReason = 0, NormalClosure = 1000, GoingAway = 1001 }; + class Storage { + public: + Storage() = default; + virtual ~Storage() = default; + + virtual const QByteArray& content() const = 0; + + virtual qint64 bytesLeftToWrite() const = 0; + virtual void write(const QByteArray& data) = 0; + }; + /// Initializes the connection. HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager); @@ -76,7 +87,7 @@ public: QByteArray requestHeader(const QString& key) const { return _requestHeaders.value(key.toLower().toLocal8Bit()); } /// Returns a reference to the request content. - const QByteArray& requestContent() const { return _requestContent; } + const QByteArray& requestContent() const { return _requestContent->content(); } /// Parses the request content as form data, returning a list of header/content pairs. QList parseFormData() const; @@ -129,7 +140,7 @@ protected: QByteArray _lastRequestHeader; /// The content of the request. - QByteArray _requestContent; + std::unique_ptr _requestContent; /// Response content std::unique_ptr _responseDevice;