Merge pull request #12406 from huffman/feat/ds-backup-apis

Backup APIs

domain-server/src/BackupHandler.h

@@ -54,19 +54,19 @@ private:
     struct Model : Concept {
         Model(T* x) : data(x) {}
 
-        void loadBackup(QuaZip& zip) {
+        void loadBackup(QuaZip& zip) override {
             data->loadBackup(zip);
         }
-        void createBackup(QuaZip& zip) {
+        void createBackup(QuaZip& zip) override {
             data->createBackup(zip);
         }
-        void recoverBackup(QuaZip& zip) {
+        void recoverBackup(QuaZip& zip) override {
             data->recoverBackup(zip);
         }
-        void deleteBackup(QuaZip& zip) {
+        void deleteBackup(QuaZip& zip) override {
             data->deleteBackup(zip);
         }
-        void consolidateBackup(QuaZip& zip) {
+        void consolidateBackup(QuaZip& zip) override {
             data->consolidateBackup(zip);
         }
 
@@ -79,11 +79,14 @@ private:
 #include <quazip5/quazipfile.h>
 class EntitiesBackupHandler {
 public:
-    EntitiesBackupHandler(QString entitiesFilePath) : _entitiesFilePath(entitiesFilePath) {}
+    EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) :
+        _entitiesFilePath(entitiesFilePath),
+        _entitiesReplacementFilePath(entitiesReplacementFilePath) {}
 
     void loadBackup(QuaZip& zip) {}
 
-    void createBackup(QuaZip& zip) const {
+    // Create a skeleton backup
+    void createBackup(QuaZip& zip) {
         QFile entitiesFile { _entitiesFilePath };
 
         if (entitiesFile.open(QIODevice::ReadOnly)) {
@@ -92,17 +95,48 @@ public:
             zipFile.write(entitiesFile.readAll());
             zipFile.close();
             if (zipFile.getZipError() != UNZ_OK) {
-                qDebug() << "testCreate(): outFile.close(): " << zipFile.getZipError();
+                qCritical() << "Failed to zip models.json.gz: " << zipFile.getZipError();
             }
         }
     }
 
-    void recoverBackup(QuaZip& zip) const {}
+    // Recover from a full backup
-    void deleteBackup(QuaZip& zip) {}
+    void recoverBackup(QuaZip& zip) {
-    void consolidateBackup(QuaZip& zip) const {}
+        if (!zip.setCurrentFile("models.json.gz")) {
+            qWarning() << "Failed to find models.json.gz while recovering backup";
+            return;
+        }
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::ReadOnly)) {
+            qCritical() << "Failed to open models.json.gz in backup";
+            return;
+        }
+        auto data = zipFile.readAll();
+
+        QFile entitiesFile { _entitiesReplacementFilePath };
+
+        if (entitiesFile.open(QIODevice::WriteOnly)) {
+            entitiesFile.write(data);
+        }
+
+        zipFile.close();
+
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical() << "Failed to zip models.json.gz: " << zipFile.getZipError();
+        }
+    }
+
+    // Delete a skeleton backup
+    void deleteBackup(QuaZip& zip) {
+    }
+
+    // Create a full backup
+    void consolidateBackup(QuaZip& zip) {
+    }
 
 private:
     QString _entitiesFilePath;
+    QString _entitiesReplacementFilePath;
 };
 
 #endif /* hifi_BackupHandler_h */

domain-server/src/DomainContentBackupManager.cpp

@@ -28,15 +28,17 @@
 #include <NumericalConstants.h>
 #include <PerfStat.h>
 #include <PathUtils.h>
+#include <shared/QtHelpers.h>
 
 #include "DomainServer.h"
 #include "DomainContentBackupManager.h"
 const int DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL = 1000 * 30;  // every 30 seconds
 
 // Backup format looks like: daily_backup-TIMESTAMP.zip
-const static QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
+static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
-const static QString DATETIME_FORMAT_RE("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}");
+static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
-
+static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
+static const QString MANUAL_BACKUP_PREFIX { "backup-" };
 void DomainContentBackupManager::addBackupHandler(BackupHandler handler) {
     _backupHandlers.push_back(std::move(handler));
 }
@@ -83,7 +85,7 @@ void DomainContentBackupManager::parseSettings(const QJsonObject& settings) {
 
             auto name = obj["Name"].toString();
             auto format = obj["format"].toString();
-            format = name.replace(" ", "_").toLower() + "-";
+            format = name.replace(" ", "_").toLower();
 
             qCDebug(domain_server) << "    Name:" << name;
             qCDebug(domain_server) << "        format:" << format;
@@ -140,7 +142,7 @@ bool DomainContentBackupManager::process() {
 
         if (sinceLastSave > intervalToCheck) {
             _lastCheck = now;
-            persist();
+            backup();
         }
     }
 
@@ -149,32 +151,18 @@ bool DomainContentBackupManager::process() {
 
 void DomainContentBackupManager::aboutToFinish() {
     qCDebug(domain_server) << "Persist thread about to finish...";
-    persist();
+    backup();
-}
+    qCDebug(domain_server) << "Persist thread done with about to finish...";
-
+    _stopThread = true;
-void DomainContentBackupManager::persist() {
-    QDir backupDir { _backupDirectory };
-    backupDir.mkpath(".");
-
-    // create our "lock" file to indicate we're saving.
-    QString lockFileName = _backupDirectory + "/running.lock";
-
-    std::ofstream lockFile(qPrintable(lockFileName), std::ios::out | std::ios::binary);
-    if (lockFile.is_open()) {
-        backup();
-
-        lockFile.close();
-        remove(qPrintable(lockFileName));
-    }
 }
 
 bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
                                                      QString& mostRecentBackupFileName,
                                                      QDateTime& mostRecentBackupTime) {
-    QRegExp formatRE { QRegExp::escape(format) + "(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
+    QRegExp formatRE { AUTOMATIC_BACKUP_PREFIX + QRegExp::escape(format) + "\\-(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
 
     QStringList filters;
-    filters << format + "*.zip";
+    filters << AUTOMATIC_BACKUP_PREFIX + format + "*.zip";
 
     bool bestBackupFound = false;
     QString bestBackupFile;
@@ -216,15 +204,98 @@ bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
     return bestBackupFound;
 }
 
+void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "deleteBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    QFile backupFile { backupDir.filePath(backupName) };
+    auto success = backupFile.remove();
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    qDebug() << "Recovering from" << backupName;
+
+    bool success { false };
+    QDir backupDir { _backupDirectory };
+    QFile backupFile { backupDir.filePath(backupName) };
+    if (backupFile.open(QIODevice::ReadOnly)) {
+        QuaZip zip { &backupFile };
+        if (!zip.open(QuaZip::Mode::mdUnzip)) {
+            qWarning() << "Failed to unzip file: " << backupName;
+            success = false;
+        } else {
+            for (auto& handler : _backupHandlers) {
+                handler.recoverBackup(zip);
+            }
+            
+            qDebug() << "Successfully recovered from " << backupName;
+            success = true;
+        }
+        backupFile.close();
+    } else {
+        success = false;
+        qWarning() << "Invalid id: " << backupName;
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
+    std::vector<BackupItemInfo> backups;
+
+    QDir backupDir { _backupDirectory };
+    auto matchingFiles =
+            backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
+                                    QDir::Files | QDir::NoSymLinks, QDir::Name);
+    QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
+    QString nameFormat = "(.+)";
+    QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
+    QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
+
+    for (const auto& fileInfo : matchingFiles) {
+        auto fileName = fileInfo.fileName();
+        if (backupNameFormat.exactMatch(fileName)) {
+            auto type = backupNameFormat.cap(1);
+            auto name = backupNameFormat.cap(2);
+            auto dateTime = backupNameFormat.cap(3);
+            auto createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
+            if (!createdAt.isValid()) {
+                continue;
+            }
+
+            BackupItemInfo backup { fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt, type == MANUAL_BACKUP_PREFIX };
+            backups.push_back(backup);
+        }
+    }
+
+    return backups;
+}
+
 void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
     QDir backupDir { _backupDirectory };
     if (backupDir.exists() && rule.maxBackupVersions > 0) {
-        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name << "...";
+        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
 
         auto matchingFiles =
-                backupDir.entryInfoList({ rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
+                backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
 
         int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
+        qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
         for (int i = 0; i < backupsToDelete; ++i) {
             auto fileInfo = matchingFiles[i].absoluteFilePath();
             QFile backupFile(fileInfo);
@@ -235,11 +306,11 @@ void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule)
             }
         }
 
-        qCDebug(domain_server) << "Done rolling old backup versions...";
+        qCDebug(domain_server) << "Done removing old backup versions";
     } else {
         qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
                                 << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
-                                << " No need to roll backups...";
+                                << " No need to roll backups";
     }
 }
 
@@ -288,21 +359,15 @@ void DomainContentBackupManager::backup() {
             qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
                                     << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
 
-            auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+            bool success;
-            auto fileName = "backup-" + rule.extensionFormat + timestamp + ".zip";
+            QString path;
-            QuaZip zip(_backupDirectory + "/" + fileName);
+            std::tie(success, path) =  createBackup(AUTOMATIC_BACKUP_PREFIX, rule.extensionFormat);
-            if (!zip.open(QuaZip::mdAdd)) {
+            if (!success) {
-                qDebug() << "Could not open backup archive:" << zip.getZipName();
+                qCWarning(domain_server) << "Failed to create backup for" << rule.name << "at" << path;
-                qDebug() << "    ERROR:" << zip.getZipError();
+                continue;
             }
 
-            for (auto& handler : _backupHandlers) {
+            qDebug() << "Created backup: " << path;
-                handler.createBackup(zip);
-            }
-
-            zip.close();
-
-            qDebug() << "Created backup: " << fileName;
 
             rule.lastBackupSeconds = nowSeconds;
 
@@ -340,3 +405,27 @@ void DomainContentBackupManager::consolidate(QString fileName) {
         zip.close();
     }
 }
+
+void DomainContentBackupManager::createManualBackup(const QString& name) {
+    createBackup(MANUAL_BACKUP_PREFIX, name);
+}
+
+std::pair<bool, QString> DomainContentBackupManager::createBackup(const QString& prefix, const QString& name) {
+    auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+    auto fileName = prefix + name + "-" + timestamp + ".zip";
+    auto path = _backupDirectory + "/" + fileName;
+    QuaZip zip(path);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCWarning(domain_server) << "Failed to open zip file at " << path;
+        qCWarning(domain_server) << "    ERROR:" << zip.getZipError();
+        return { false, path };
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler.createBackup(zip);
+    }
+
+    zip.close();
+
+    return { true, path };
+}

domain-server/src/DomainContentBackupManager.h

@@ -21,6 +21,16 @@
 
 #include "BackupHandler.h"
 
+#include <shared/MiniPromises.h>
+
+struct BackupItemInfo {
+    QString id;
+    QString name;
+    QString absolutePath;
+    QDateTime createdAt;
+    bool isManualBackup;
+};
+
 class DomainContentBackupManager : public GenericThread {
     Q_OBJECT
 public:
@@ -41,17 +51,26 @@ public:
                                bool debugTimestampNow = false);
 
     void addBackupHandler(BackupHandler handler);
+    std::vector<BackupItemInfo> getAllBackups();
 
     void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
 
     void replaceData(QByteArray data);
 
+    void createManualBackup(const QString& name);
+
+public slots:
+    void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
+    void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
+
+signals:
+    void loadCompleted();
+
 protected:
     /// Implements generic processing behavior for this thread.
     virtual void setup() override;
     virtual bool process() override;
 
-    void persist();
     void load();
     void backup();
     void consolidate(QString fileName);
@@ -60,8 +79,10 @@ protected:
     int64_t getMostRecentBackupTimeInSecs(const QString& format);
     void parseSettings(const QJsonObject& settings);
 
+    std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
+
 private:
-    QString _backupDirectory;
+    const QString _backupDirectory;
     std::vector<BackupHandler> _backupHandlers;
     int _persistInterval { 0 };
 

domain-server/src/DomainServer.cpp

@@ -296,9 +296,14 @@ DomainServer::DomainServer(int argc, char* argv[]) :
     maybeHandleReplacementEntityFile();
 
     _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager.settingsResponseObjectForType("6")["entity_server_settings"].toObject()));
-    _contentManager->addBackupHandler(new EntitiesBackupHandler(getEntitiesFilePath()));
+    _contentManager->addBackupHandler(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath()));
     _contentManager->addBackupHandler(new BackupSupervisor(getContentBackupDir()));
     _contentManager->initialize(true);
+
+    qDebug() << "Existing backups:";
+    for (auto& backup : _contentManager->getAllBackups()) {
+        qDebug() << "  Backup: " << backup.name << backup.createdAt;
+    }
 }
 
 void DomainServer::parseCommandLine() {
@@ -1734,6 +1739,12 @@ void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessag
     auto data = message->readAll();
     auto filePath = getEntitiesFilePath();
 
+    QDir dir(getEntitiesDirPath());
+    if (!dir.exists()) {
+        qCDebug(domain_server) << "Creating entities content directory:" << dir.absolutePath();
+        dir.mkpath(".");
+    }
+
     QFile f(filePath);
     if (f.open(QIODevice::WriteOnly)) {
         f.write(data);
@@ -1744,12 +1755,12 @@ void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessag
             qCDebug(domain_server) << "Failed to read new octree data info";
         }
     } else {
-        qCDebug(domain_server) << "Failed to write new entities file";
+        qCDebug(domain_server) << "Failed to write new entities file:" << filePath;
     }
 }
 
 QString DomainServer::getContentBackupDir() {
-    return PathUtils::getAppDataFilePath("backup");
+    return PathUtils::getAppDataFilePath("backups");
 }
 
 QString DomainServer::getEntitiesDirPath() {
@@ -1922,6 +1933,9 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     const QString URI_API_PLACES = "/api/places";
     const QString URI_API_DOMAINS = "/api/domains";
     const QString URI_API_DOMAINS_ID = "/api/domains/";
+    const QString URI_API_BACKUPS = "/api/backups";
+    const QString URI_API_BACKUPS_ID = "/api/backups/";
+    const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
 
     const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
@@ -2106,6 +2120,26 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             // send the response
             connection->respond(HTTPConnection::StatusCode200, nodesDocument.toJson(), qPrintable(JSON_MIME_TYPE));
 
+            return true;
+        } else if (url.path() == URI_API_BACKUPS) {
+            QJsonObject rootJSON;
+            QJsonArray backupsJSON;
+
+            auto backups = _contentManager->getAllBackups();
+
+            for (const auto& backup : backups) {
+                QJsonObject obj;
+                obj["id"] = backup.id;
+                obj["name"] = backup.name;
+                obj["createdAtMillis"] = backup.createdAt.toMSecsSinceEpoch();
+                obj["isManualBackup"] = backup.isManualBackup;
+                backupsJSON.push_back(obj);
+            }
+
+            rootJSON["backups"] = backupsJSON;
+            QJsonDocument docJSON(rootJSON);
+
+            connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
             return true;
         } else if (url.path() == URI_RESTART) {
             connection->respond(HTTPConnection::StatusCode200);
@@ -2211,6 +2245,19 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             return true;
 
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto params = connection->parseUrlEncodedForm();
+            auto it = params.find("name");
+            if (it == params.end()) {
+                connection->respond(HTTPConnection::StatusCode400, "Bad request, missing `name`");
+                return true;
+            }
+
+            _contentManager->createManualBackup(it.value());
+
+            connection->respond(HTTPConnection::StatusCode200);
+            return true;
+
         } else if (url.path() == "/domain_settings") {
             auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
             if (!accessTokenVariant) {
@@ -2219,8 +2266,21 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             }
 
         } else if (url.path() == URI_API_DOMAINS) {
-
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains", "domain", { "label" });
+
+        } else if (url.path().startsWith(URI_API_BACKUPS_RECOVER)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_RECOVER).length());
+            auto deferred = makePromise("recoverFromBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->recoverFromBackup(deferred, id);
+            return true;
         }
     } else if (connection->requestOperation() == QNetworkAccessManager::PutOperation) {
         if (url.path() == URI_API_DOMAINS) {
@@ -2309,7 +2369,22 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
         QRegExp allNodesDeleteRegex(ALL_NODE_DELETE_REGEX_STRING);
         QRegExp nodeDeleteRegex(NODE_DELETE_REGEX_STRING);
 
-        if (nodeDeleteRegex.indexIn(url.path()) != -1) {
+        if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("deleteBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->deleteBackup(deferred, id);
+
+            return true;
+
+        } else if (nodeDeleteRegex.indexIn(url.path()) != -1) {
             // this is a request to DELETE one node by UUID
 
             // pull the captured string, if it exists
@@ -3244,8 +3319,6 @@ void DomainServer::maybeHandleReplacementEntityFile() {
 }
 
 void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    // enumerate the nodes and find any octree type servers with active sockets
-
     //Assume we have compressed data
     auto compressedOctree = octreeFile;
     QByteArray jsonOctree;

libraries/embedded-webserver/src/HTTPConnection.cpp

@@ -133,12 +133,33 @@ QList<FormData> HTTPConnection::parseFormData() const {
 }
 
 void HTTPConnection::respond(const char* code, const QByteArray& content, const char* contentType, const Headers& headers) {
+    QByteArray data(content);
+    auto device { std::unique_ptr<QBuffer>(new QBuffer()) };
+    device->setBuffer(new QByteArray(content));
+    if (device->open(QIODevice::ReadOnly)) {
+        respond(code, std::move(device), contentType, headers);
+    } else {
+        qCritical() << "Error opening QBuffer to respond to " << _requestUrl.path();
+    }
+}
+
+void HTTPConnection::respond(const char* code, std::unique_ptr<QIODevice> device, const char* contentType, const Headers& headers) {
+    _responseDevice = std::move(device);
+
     _socket->write("HTTP/1.1 ");
+
+    if (_responseDevice->isSequential()) {
+        qWarning() << "Error responding to HTTPConnection: sequential IO devices not supported";
+        _socket->write(StatusCode500);
+        _socket->write("\r\n");
+        _socket->disconnect(SIGNAL(readyRead()), this);
+        _socket->disconnectFromHost();
+        return;
+    }
+
     _socket->write(code);
     _socket->write("\r\n");
 
-    int csize = content.size();
-
     for (Headers::const_iterator it = headers.constBegin(), end = headers.constEnd();
             it != end; it++) {
         _socket->write(it.key());
@@ -146,6 +167,8 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
         _socket->write(it.value());
         _socket->write("\r\n");
     }
+
+    int csize = _responseDevice->size();
     if (csize > 0) {
         _socket->write("Content-Length: ");
         _socket->write(QByteArray::number(csize));
@@ -157,20 +180,35 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
     }
     _socket->write("Connection: close\r\n\r\n");
 
-    if (csize > 0) {
+    if (_responseDevice->atEnd()) {
-        _socket->write(content);
+        _socket->disconnectFromHost();
+    } else {
+        int totalToBeWritten = csize;
+        connect(_socket, &QTcpSocket::bytesWritten, this, [this, totalToBeWritten](size_t bytes) mutable {
+            constexpr size_t HTTP_RESPONSE_CHUNK_SIZE = 1024 * 10;
+            if (!_responseDevice->atEnd()) {
+                totalToBeWritten -= _socket->write(_responseDevice->read(HTTP_RESPONSE_CHUNK_SIZE));
+                if (_responseDevice->atEnd()) {
+                    _socket->disconnectFromHost();
+                    disconnect(_socket, &QTcpSocket::bytesWritten, this, nullptr);
+                }
+            }
+        });
+
     }
 
     // make sure we receive no further read notifications
-    _socket->disconnect(SIGNAL(readyRead()), this);
+    disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
-
-    _socket->disconnectFromHost();
 }
 
 void HTTPConnection::readRequest() {
     if (!_socket->canReadLine()) {
         return;
     }
+    if (!_requestUrl.isEmpty()) {
+        qDebug() << "Request URL was already set";
+        return;
+    }
     // parse out the method and resource
     QByteArray line = _socket->readLine().trimmed();
     if (line.startsWith("HEAD")) {

libraries/embedded-webserver/src/HTTPConnection.h

@@ -26,6 +26,8 @@
 #include <QPair>
 #include <QUrl>
 
+#include <memory>
+
 class QTcpSocket;
 class HTTPManager;
 class MaskFilter;
@@ -87,6 +89,9 @@ public:
     void respond (const char* code, const QByteArray& content = QByteArray(),
         const char* contentType = DefaultContentType,
         const Headers& headers = Headers());
+    void respond (const char* code, std::unique_ptr<QIODevice> device,
+        const char* contentType = DefaultContentType,
+        const Headers& headers = Headers());
 
 protected slots:
 
@@ -127,6 +132,9 @@ protected:
 
     /// The content of the request.
     QByteArray _requestContent;
+
+    /// Response content
+    std::unique_ptr<QIODevice> _responseDevice;
 };
 
 #endif // hifi_HTTPConnection_h

libraries/embedded-webserver/src/HTTPManager.cpp

@@ -98,13 +98,14 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
             // file exists, serve it
             static QMimeDatabase mimeDatabase;
             
-            QFile localFile(filePath);
+            auto localFile = std::unique_ptr<QFile>(new QFile(filePath));
-            localFile.open(QIODevice::ReadOnly);
+            localFile->open(QIODevice::ReadOnly);
-            QByteArray localFileData = localFile.readAll();
+            QByteArray localFileData;
             
             QFileInfo localFileInfo(filePath);
             
             if (localFileInfo.completeSuffix() == "shtml") {
+                localFileData = localFile->readAll();
                 // this is a file that may have some SSI statements
                 // the only thing we support is the include directive, but check the contents for that
                 
@@ -153,8 +154,12 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
                 ? QString { "text/html" }
                 : mimeDatabase.mimeTypeForFile(filePath).name();
 
-            connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
+            if (localFileData.isNull()) {
-            
+                connection->respond(HTTPConnection::StatusCode200, std::move(localFile), qPrintable(mimeType));
+            } else {
+                connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
+            }
+
             return true;
         }
     }