Merge pull request #13217 from Atlante45/fix/ds-memory

Fix uploaded content archives taking too much memory

assignment-client/src/octree/OctreeServer.cpp

@@ -233,7 +233,6 @@ OctreeServer::OctreeServer(ReceivedMessage& message) :
     _argc(0),
     _argv(NULL),
     _parsedArgV(NULL),
-    _httpManager(NULL),
     _statusPort(0),
     _packetsPerClientPerInterval(10),
     _packetsTotalPerInterval(DEFAULT_PACKETS_PER_INTERVAL),
@@ -285,7 +284,7 @@ void OctreeServer::initHTTPManager(int port) {
     QString documentRoot = QString("%1/web").arg(PathUtils::getAppDataPath());
 
     // setup an httpManager with us as the request handler and the parent
-    _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
+    _httpManager.reset(new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this));
 }
 
 bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {

assignment-client/src/octree/OctreeServer.h

@@ -183,7 +183,7 @@ protected:
 
     bool _isShuttingDown = false;
 
-    HTTPManager* _httpManager;
+    std::unique_ptr<HTTPManager> _httpManager;
     int _statusPort;
     QString _statusHost;
 

domain-server/src/DomainGatekeeper.cpp

@@ -16,6 +16,8 @@
 #include <openssl/x509.h>
 #include <random>
 
+#include <QDataStream>
+
 #include <AccountManager.h>
 #include <Assignment.h>
 

domain-server/src/DomainServer.cpp

@@ -149,7 +149,6 @@ DomainServer::DomainServer(int argc, char* argv[]) :
     QCoreApplication(argc, argv),
     _gatekeeper(this),
     _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this),
-    _httpsManager(NULL),
     _allAssignments(),
     _unfulfilledAssignments(),
     _isUsingDTLS(false),
@@ -385,6 +384,8 @@ DomainServer::~DomainServer() {
         _contentManager->terminate();
     }
 
+    DependencyManager::destroy<AccountManager>();
+
     // cleanup the AssetClient thread
     DependencyManager::destroy<AssetClient>();
     _assetClientThread.quit();
@@ -439,7 +440,7 @@ bool DomainServer::optionallyReadX509KeyAndCertificate() {
         QSslCertificate sslCertificate(&certFile);
         QSslKey privateKey(&keyFile, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey, keyPassphraseString.toUtf8());
 
-        _httpsManager = new HTTPSManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTPS_PORT, sslCertificate, privateKey, QString(), this, this);
+        _httpsManager.reset(new HTTPSManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTPS_PORT, sslCertificate, privateKey, QString(), this));
 
         qDebug() << "TCP server listening for HTTPS connections on" << DOMAIN_SERVER_HTTPS_PORT;
 

domain-server/src/DomainServer.h

@@ -220,7 +220,7 @@ private:
     DomainGatekeeper _gatekeeper;
 
     HTTPManager _httpManager;
-    HTTPSManager* _httpsManager;
+    std::unique_ptr<HTTPSManager> _httpsManager;
 
     QHash<QUuid, SharedAssignmentPointer> _allAssignments;
     QQueue<SharedAssignmentPointer> _unfulfilledAssignments;

ice-server/src/IceServer.cpp

@@ -13,6 +13,7 @@
 
 #include <openssl/x509.h>
 
+#include <QtCore/QDataStream>
 #include <QtCore/QJsonDocument>
 #include <QtCore/QTimer>
 #include <QtNetwork/QNetworkReply>

interface/src/SecondaryCamera.cpp

@@ -9,11 +9,13 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-#include "Application.h"
 #include "SecondaryCamera.h"
-#include <TextureCache.h>
+
-#include <gpu/Context.h>
 #include <glm/gtx/transform.hpp>
+#include <gpu/Context.h>
+#include <TextureCache.h>
+
+#include "Application.h"
 
 using RenderArgsPointer = std::shared_ptr<RenderArgs>;
 

libraries/embedded-webserver/src/HTTPConnection.cpp

@@ -11,6 +11,8 @@
 
 #include "HTTPConnection.h"
 
+#include <assert.h>
+
 #include <QBuffer>
 #include <QCryptographicHash>
 #include <QTcpSocket>
@@ -29,11 +31,92 @@ const char* HTTPConnection::StatusCode404 = "404 Not Found";
 const char* HTTPConnection::StatusCode500 = "500 Internal server error";
 const char* HTTPConnection::DefaultContentType = "text/plain; charset=ISO-8859-1";
 
-HTTPConnection::HTTPConnection (QTcpSocket* socket, HTTPManager* parentManager) :
+
+class MemoryStorage : public HTTPConnection::Storage {
+public:
+    static std::unique_ptr<MemoryStorage> make(qint64 size);
+    virtual ~MemoryStorage() = default;
+
+    const QByteArray& content() const override { return _array; }
+    qint64 bytesLeftToWrite() const override { return _array.size() - _bytesWritten; }
+    void write(const QByteArray& data) override;
+
+private:
+    MemoryStorage(qint64 size) { _array.resize(size); }
+
+    QByteArray _array;
+    qint64 _bytesWritten { 0 };
+};
+
+std::unique_ptr<MemoryStorage> MemoryStorage::make(qint64 size) {
+    return std::unique_ptr<MemoryStorage>(new MemoryStorage(size));
+}
+
+void MemoryStorage::write(const QByteArray& data) {
+    assert(data.size() <= bytesLeftToWrite());
+    memcpy(_array.data() + _bytesWritten, data.data(), data.size());
+    _bytesWritten += data.size();
+}
+
+
+class FileStorage : public HTTPConnection::Storage {
+public:
+    static std::unique_ptr<FileStorage> make(qint64 size);
+    virtual ~FileStorage();
+
+    const QByteArray& content() const override { return _wrapperArray; };
+    qint64 bytesLeftToWrite() const override { return _mappedMemorySize - _bytesWritten; }
+    void write(const QByteArray& data) override;
+
+private:
+    FileStorage(std::unique_ptr<QTemporaryFile> file, uchar* mapped, qint64 size);
+
+    // Byte array is const because any edit will trigger a deep copy
+    // and pull all the data we want to keep on disk in memory.
+    const QByteArray _wrapperArray;
+    std::unique_ptr<QTemporaryFile> _file;
+
+    uchar* const _mappedMemoryAddress { nullptr };
+    const qint64 _mappedMemorySize { 0 };
+    qint64 _bytesWritten { 0 };
+};
+
+std::unique_ptr<FileStorage> FileStorage::make(qint64 size) {
+    auto file = std::unique_ptr<QTemporaryFile>(new QTemporaryFile());
+    file->open(); // Open for resize
+    file->resize(size);
+    auto mapped = file->map(0, size); // map the entire file
+
+    return std::unique_ptr<FileStorage>(new FileStorage(std::move(file), mapped, size));
+}
+
+// Use QByteArray::fromRawData to avoid a new allocation and access the already existing
+// memory directly as long as all operations on the array are const.
+FileStorage::FileStorage(std::unique_ptr<QTemporaryFile> file, uchar* mapped, qint64 size) :
+    _wrapperArray(QByteArray::fromRawData(reinterpret_cast<char*>(mapped), size)),
+    _file(std::move(file)),
+    _mappedMemoryAddress(mapped),
+    _mappedMemorySize(size)
+{
+}
+
+FileStorage::~FileStorage() {
+    _file->unmap(_mappedMemoryAddress);
+    _file->close();
+}
+
+void FileStorage::write(const QByteArray& data) {
+    assert(data.size() <= bytesLeftToWrite());
+    // We write directly to the mapped memory
+    memcpy(_mappedMemoryAddress + _bytesWritten, data.data(), data.size());
+    _bytesWritten += data.size();
+}
+
+
+HTTPConnection::HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager) :
     QObject(parentManager),
     _parentManager(parentManager),
     _socket(socket),
-    _stream(socket),
     _address(socket->peerAddress())
 {
     // take over ownership of the socket
@@ -62,7 +145,7 @@ QHash<QString, QString> HTTPConnection::parseUrlEncodedForm() {
         return QHash<QString, QString>();
     }
 
-    QUrlQuery form { _requestContent };
+    QUrlQuery form { _requestContent->content() };
     QHash<QString, QString> pairs;
     for (auto pair : form.queryItems()) {
         auto key = QUrl::fromPercentEncoding(pair.first.toLatin1().replace('+', ' '));
@@ -97,7 +180,7 @@ QList<FormData> HTTPConnection::parseFormData() const {
     QByteArray end = "\r\n--" + boundary + "--\r\n";
 
     QList<FormData> data;
-    QBuffer buffer(const_cast<QByteArray*>(&_requestContent));
+    QBuffer buffer(const_cast<QByteArray*>(&_requestContent->content()));
     buffer.open(QIODevice::ReadOnly);
     while (buffer.canReadLine()) {
         QByteArray line = buffer.readLine().trimmed();
@@ -107,12 +190,13 @@ QList<FormData> HTTPConnection::parseFormData() const {
                 QByteArray line = buffer.readLine().trimmed();
                 if (line.isEmpty()) {
                     // content starts after this line
-                    int idx = _requestContent.indexOf(end, buffer.pos());
+                    int idx = _requestContent->content().indexOf(end, buffer.pos());
                     if (idx == -1) {
                         qWarning() << "Missing end boundary." << _address;
                         return data;
                     }
-                    datum.second = _requestContent.mid(buffer.pos(), idx - buffer.pos());
+                    datum.second = QByteArray::fromRawData(_requestContent->content().data() + buffer.pos(),
+                                                           idx - buffer.pos());
                     data.append(datum);
                     buffer.seek(idx + end.length());
 
@@ -256,7 +340,24 @@ void HTTPConnection::readHeaders() {
                 _parentManager->handleHTTPRequest(this, _requestUrl);
 
             } else {
-                _requestContent.resize(clength.toInt());
+                bool success = false;
+                auto length = clength.toInt(&success);
+                if (!success) {
+                    qWarning() << "Invalid header." << _address << trimmed;
+                    respond("400 Bad Request", "The header was malformed.");
+                    return;
+                }
+
+                // Storing big requests in memory gets expensive, especially on servers
+                // with limited memory. So we store big requests in a temporary file on disk
+                // and map it to faster read/write access.
+                static const int MAX_CONTENT_SIZE_IN_MEMORY = 10 * 1000 * 1000;
+                if (length < MAX_CONTENT_SIZE_IN_MEMORY) {
+                    _requestContent = MemoryStorage::make(length);
+                } else {
+                    _requestContent = FileStorage::make(length);
+                }
+
                 connect(_socket, SIGNAL(readyRead()), SLOT(readContent()));
 
                 // read any content immediately available
@@ -285,12 +386,13 @@ void HTTPConnection::readHeaders() {
 }
 
 void HTTPConnection::readContent() {
-    int size = _requestContent.size();
+    auto size = std::min(_socket->bytesAvailable(), _requestContent->bytesLeftToWrite());
-    if (_socket->bytesAvailable() < size) {
-        return;
-    }
-    _socket->read(_requestContent.data(), size);
-    _socket->disconnect(this, SLOT(readContent()));
 
-    _parentManager->handleHTTPRequest(this, _requestUrl.path());
+    _requestContent->write(_socket->read(size));
+
+    if (_requestContent->bytesLeftToWrite() == 0) {
+        _socket->disconnect(this, SLOT(readContent()));
+
+        _parentManager->handleHTTPRequest(this, _requestUrl.path());
+    }
 }

libraries/embedded-webserver/src/HTTPConnection.h

@@ -16,14 +16,14 @@
 #ifndef hifi_HTTPConnection_h
 #define hifi_HTTPConnection_h
 
-#include <QDataStream>
 #include <QHash>
-#include <QtNetwork/QHostAddress>
 #include <QIODevice>
 #include <QList>
+#include <QtNetwork/QHostAddress>
 #include <QtNetwork/QNetworkAccessManager>
 #include <QObject>
 #include <QPair>
+#include <QTemporaryFile>
 #include <QUrl>
 
 #include <memory>
@@ -57,52 +57,63 @@ public:
     /// WebSocket close status codes.
     enum ReasonCode { NoReason = 0, NormalClosure = 1000, GoingAway = 1001 };
 
+    class Storage {
+    public:
+        Storage() = default;
+        virtual ~Storage() = default;
+
+        virtual const QByteArray& content() const = 0;
+
+        virtual qint64 bytesLeftToWrite() const = 0;
+        virtual void write(const QByteArray& data) = 0;
+    };
+
     /// Initializes the connection.
-    HTTPConnection (QTcpSocket* socket, HTTPManager* parentManager);
+    HTTPConnection(QTcpSocket* socket, HTTPManager* parentManager);
 
     /// Destroys the connection.
-    virtual ~HTTPConnection ();
+    virtual ~HTTPConnection();
 
     /// Returns a pointer to the underlying socket, to which WebSocket message bodies should be written.
-    QTcpSocket* socket () const { return _socket; }
+    QTcpSocket* socket() const { return _socket; }
 
     /// Returns the request operation.
-    QNetworkAccessManager::Operation requestOperation () const { return _requestOperation; }
+    QNetworkAccessManager::Operation requestOperation() const { return _requestOperation; }
 
     /// Returns a reference to the request URL.
-    const QUrl& requestUrl () const { return _requestUrl; }
+    const QUrl& requestUrl() const { return _requestUrl; }
 
     /// Returns a copy of the request header value. If it does not exist, it will return a default constructed QByteArray.
     QByteArray requestHeader(const QString& key) const { return _requestHeaders.value(key.toLower().toLocal8Bit()); }
 
     /// Returns a reference to the request content.
-    const QByteArray& requestContent () const { return _requestContent; }
+    const QByteArray& requestContent() const { return _requestContent->content(); }
 
     /// Parses the request content as form data, returning a list of header/content pairs.
-    QList<FormData> parseFormData () const;
+    QList<FormData> parseFormData() const;
 
     /// Parses the request content as a url encoded form, returning a hash of key/value pairs.
     /// Duplicate keys are not supported.
     QHash<QString, QString> parseUrlEncodedForm();
 
     /// Sends a response and closes the connection.
-    void respond (const char* code, const QByteArray& content = QByteArray(),
+    void respond(const char* code, const QByteArray& content = QByteArray(),
         const char* contentType = DefaultContentType,
         const Headers& headers = Headers());
-    void respond (const char* code, std::unique_ptr<QIODevice> device,
+    void respond(const char* code, std::unique_ptr<QIODevice> device,
         const char* contentType = DefaultContentType,
         const Headers& headers = Headers());
 
 protected slots:
 
     /// Reads the request line.
-    void readRequest ();
+    void readRequest();
 
     /// Reads the headers.
-    void readHeaders ();
+    void readHeaders();
 
     /// Reads the content.
-    void readContent ();
+    void readContent();
 
 protected:
     void respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 size);
@@ -112,9 +123,6 @@ protected:
 
     /// The underlying socket.
     QTcpSocket* _socket;
-    
-    /// The data stream for writing to the socket.
-    QDataStream _stream;
 
     /// The stored address.
     QHostAddress _address;
@@ -132,7 +140,7 @@ protected:
     QByteArray _lastRequestHeader;
 
     /// The content of the request.
-    QByteArray _requestContent;
+    std::unique_ptr<Storage> _requestContent;
 
     /// Response content
     std::unique_ptr<QIODevice> _responseDevice;

libraries/embedded-webserver/src/HTTPManager.cpp

@@ -24,8 +24,7 @@
 const int SOCKET_ERROR_EXIT_CODE = 2;
 const int SOCKET_CHECK_INTERVAL_IN_MS = 30000;
 
-HTTPManager::HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler, QObject* parent) :
+HTTPManager::HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler) :
-    QTcpServer(parent),
     _listenAddress(listenAddress),
     _documentRoot(documentRoot),
     _requestHandler(requestHandler),

libraries/embedded-webserver/src/HTTPManager.h

@@ -33,7 +33,7 @@ class HTTPManager : public QTcpServer, public HTTPRequestHandler {
    Q_OBJECT
 public:
     /// Initializes the manager.
-    HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler = NULL, QObject* parent = 0);
+    HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler = nullptr);
     
     bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
 

libraries/embedded-webserver/src/HTTPSConnection.h

@@ -23,4 +23,4 @@ protected slots:
     void handleSSLErrors(const QList<QSslError>& errors);
 };
 
-#endif // hifi_HTTPSConnection_h
+#endif // hifi_HTTPSConnection_h

libraries/embedded-webserver/src/HTTPSManager.cpp

@@ -16,8 +16,8 @@
 #include "HTTPSConnection.h"
 
 HTTPSManager::HTTPSManager(QHostAddress listenAddress, quint16 port, const QSslCertificate& certificate, const QSslKey& privateKey,
-                           const QString& documentRoot, HTTPSRequestHandler* requestHandler, QObject* parent) :
+                           const QString& documentRoot, HTTPSRequestHandler* requestHandler) :
-    HTTPManager(listenAddress, port, documentRoot, requestHandler, parent),
+    HTTPManager(listenAddress, port, documentRoot, requestHandler),
     _certificate(certificate),
     _privateKey(privateKey),
     _sslRequestHandler(requestHandler)

libraries/embedded-webserver/src/HTTPSManager.h

@@ -31,7 +31,7 @@ public:
                  const QSslCertificate& certificate,
                  const QSslKey& privateKey,
                  const QString& documentRoot,
-                 HTTPSRequestHandler* requestHandler = NULL, QObject* parent = 0);
+                 HTTPSRequestHandler* requestHandler = nullptr);
     
     void setCertificate(const QSslCertificate& certificate) { _certificate = certificate; }
     void setPrivateKey(const QSslKey& privateKey) { _privateKey = privateKey; }
@@ -48,4 +48,4 @@ private:
     HTTPSRequestHandler* _sslRequestHandler;
 };
 
-#endif // hifi_HTTPSManager_h
+#endif // hifi_HTTPSManager_h