Merge pull request #160 from ctrlaltdavid/fix/http-serverless

Add support for serverless domains via HTTP/HTTPS

interface/src/Application.cpp

@@ -512,27 +512,6 @@ std::atomic<uint64_t> DeadlockWatchdogThread::_maxElapsed;
 std::atomic<int> DeadlockWatchdogThread::_maxElapsedAverage;
 ThreadSafeMovingAverage<int, DeadlockWatchdogThread::HEARTBEAT_SAMPLES> DeadlockWatchdogThread::_movingAverage;
 
-bool isDomainURL(QUrl url) {
-    if (!url.isValid()) {
-        return false;
-    }
-    if (url.scheme() == URL_SCHEME_HIFI) {
-        return true;
-    }
-    if (url.scheme() != HIFI_URL_SCHEME_FILE) {
-        // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can
-        // be loaded over http(s)
-        // && url.scheme() != HIFI_URL_SCHEME_HTTP &&
-        // url.scheme() != HIFI_URL_SCHEME_HTTPS
-        return false;
-    }
-    if (url.path().endsWith(".json", Qt::CaseInsensitive) ||
-        url.path().endsWith(".json.gz", Qt::CaseInsensitive)) {
-        return true;
-    }
-    return false;
-}
-
 #ifdef Q_OS_WIN
 static const UINT UWM_IDENTIFY_INSTANCES =
     RegisterWindowMessage("UWM_IDENTIFY_INSTANCES_{8AB82783-B74A-4258-955B-8188C22AA0D6}_" + qgetenv("USERNAME"));
@@ -564,14 +543,6 @@ public:
                 return true;
             }
 
-            if (message->message == WM_COPYDATA) {
-                COPYDATASTRUCT* pcds = (COPYDATASTRUCT*)(message->lParam);
-                QUrl url = QUrl((const char*)(pcds->lpData));
-                if (isDomainURL(url)) {
-                    DependencyManager::get<AddressManager>()->handleLookupString(url.toString());
-                    return true;
-                }
-            }
             // Attempting to close MIDI interfaces of a hot-unplugged device can result in audio-driver deadlock.
             // Detecting MIDI devices that have been added/removed after starting Inteface has been disabled.
             // https://support.microsoft.com/en-us/help/4460006/midi-device-app-hangs-when-former-midi-api-is-used
@@ -4059,7 +4030,7 @@ void Application::setIsServerlessMode(bool serverlessDomain) {
     }
 }
 
-std::map<QString, QString> Application::prepareServerlessDomainContents(QUrl domainURL) {
+std::map<QString, QString> Application::prepareServerlessDomainContents(QUrl domainURL, QByteArray data) {
     QUuid serverlessSessionID = QUuid::createUuid();
     getMyAvatar()->setSessionUUID(serverlessSessionID);
     auto nodeList = DependencyManager::get<NodeList>();
@@ -4070,14 +4041,13 @@ std::map<QString, QString> Application::prepareServerlessDomainContents(QUrl dom
     permissions.setAll(true);
     nodeList->setPermissions(permissions);
 
-    // we can't import directly into the main tree because we would need to lock it, and
+    // FIXME: Lock the main tree and import directly into it.
-    // Octree::readFromURL calls loop.exec which can run code which will also attempt to lock the tree.
     EntityTreePointer tmpTree(new EntityTree());
     tmpTree->setIsServerlessMode(true);
     tmpTree->createRootElement();
     auto myAvatar = getMyAvatar();
     tmpTree->setMyAvatar(myAvatar);
-    bool success = tmpTree->readFromURL(domainURL.toString());
+    bool success = tmpTree->readFromByteArray(domainURL.toString(), data);
     if (success) {
         tmpTree->reaverageOctreeElements();
         tmpTree->sendEntities(&_entityEditSender, getEntities()->getTree(), 0, 0, 0);
@@ -4100,12 +4070,26 @@ void Application::loadServerlessDomain(QUrl domainURL) {
         return;
     }
 
-    auto namedPaths = prepareServerlessDomainContents(domainURL);
+    QString trimmedUrl = domainURL.toString().trimmed();
-    auto nodeList = DependencyManager::get<NodeList>();
+    bool DEFAULT_IS_OBSERVABLE = true;
+    const qint64 DEFAULT_CALLER_ID = -1;
+    auto request = DependencyManager::get<ResourceManager>()->createResourceRequest(
+        this, trimmedUrl, DEFAULT_IS_OBSERVABLE, DEFAULT_CALLER_ID, "Application::loadServerlessDomain");
 
-    nodeList->getDomainHandler().connectedToServerless(namedPaths);
+    if (!request) {
+        return;
+    }
 
-    _fullSceneReceivedCounter++;
+    connect(request, &ResourceRequest::finished, this, [=]() {
+        if (request->getResult() == ResourceRequest::Success) {
+            auto namedPaths = prepareServerlessDomainContents(domainURL, request->getData());
+            auto nodeList = DependencyManager::get<NodeList>();
+            nodeList->getDomainHandler().connectedToServerless(namedPaths);
+            _fullSceneReceivedCounter++;
+        }
+        request->deleteLater();
+    });
+    request->send();
 }
 
 void Application::loadErrorDomain(QUrl domainURL) {
@@ -4114,16 +4098,7 @@ void Application::loadErrorDomain(QUrl domainURL) {
         return;
     }
 
-    if (domainURL.isEmpty()) {
+    loadServerlessDomain(domainURL);
-        return;
-    }
-
-    auto namedPaths = prepareServerlessDomainContents(domainURL);
-    auto nodeList = DependencyManager::get<NodeList>();
-
-    nodeList->getDomainHandler().loadedErrorDomain(namedPaths);
-
-    _fullSceneReceivedCounter++;
 }
 
 bool Application::importImage(const QString& urlString) {
@@ -5542,6 +5517,8 @@ bool Application::importEntities(const QString& urlOrFilename, const bool isObse
     _entityClipboard->withWriteLock([&] {
         _entityClipboard->eraseAllOctreeElements();
 
+        // FIXME: readFromURL() can take over the main event loop which may cause problems, especially if downloading the JSON 
+        // from the Web.
         success = _entityClipboard->readFromURL(urlOrFilename, isObservable, callerId);
         if (success) {
             _entityClipboard->reaverageOctreeElements();
@@ -9259,7 +9236,7 @@ void Application::readArgumentsFromLocalSocket() const {
 
     // If we received a message, try to open it as a URL
     if (message.length() > 0) {
-        DependencyManager::get<WindowScriptingInterface>()->openUrl(QString::fromUtf8(message));
+        DependencyManager::get<AddressManager>()->handleLookupString(QString::fromUtf8(message));
     }
 }
 

interface/src/Application.h

@@ -464,7 +464,7 @@ public slots:
     void setPreferredCursor(const QString& cursor);
 
     void setIsServerlessMode(bool serverlessDomain);
-    std::map<QString, QString> prepareServerlessDomainContents(QUrl domainURL);
+    std::map<QString, QString> prepareServerlessDomainContents(QUrl domainURL, QByteArray data);
 
     void loadServerlessDomain(QUrl domainURL);
     void loadErrorDomain(QUrl domainURL);

interface/src/main.cpp

@@ -262,7 +262,9 @@ int main(int argc, const char* argv[]) {
         if (socket.waitForConnected(LOCAL_SERVER_TIMEOUT_MS)) {
             if (parser.isSet(urlOption)) {
                 QUrl url = QUrl(parser.value(urlOption));
-                if (url.isValid() && (url.scheme() == URL_SCHEME_HIFI || url.scheme() == URL_SCHEME_HIFIAPP)) {
+                if (url.isValid() && (url.scheme() == URL_SCHEME_HIFI || url.scheme() == URL_SCHEME_HIFIAPP
+                        || url.scheme() == HIFI_URL_SCHEME_HTTP || url.scheme() == HIFI_URL_SCHEME_HTTPS
+                        || url.scheme() == HIFI_URL_SCHEME_FILE)) {
                     qDebug() << "Writing URL to local socket";
                     socket.write(url.toString().toUtf8());
                     if (!socket.waitForBytesWritten(5000)) {

libraries/networking/src/AddressManager.cpp

@@ -157,6 +157,7 @@ void AddressManager::storeCurrentAddress() {
     auto url = currentAddress();
 
     if (url.scheme() == HIFI_URL_SCHEME_FILE ||
+        url.scheme() == HIFI_URL_SCHEME_HTTP || url.scheme() == HIFI_URL_SCHEME_HTTPS ||
         (url.scheme() == URL_SCHEME_HIFI && !url.host().isEmpty())) {
         // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can
         // be loaded over http(s)
@@ -358,13 +359,11 @@ bool AddressManager::handleUrl(const QUrl& lookupUrlIn, LookupTrigger trigger) {
         emit lookupResultsFinished();
 
         return true;
-    } else if (lookupUrl.scheme() == HIFI_URL_SCHEME_FILE) {
+    } else if (lookupUrl.scheme() == HIFI_URL_SCHEME_FILE || lookupUrl.scheme() == HIFI_URL_SCHEME_HTTPS
-        // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can
+            || lookupUrl.scheme() == HIFI_URL_SCHEME_HTTP) {
-        // be loaded over http(s)
+               
-        // lookupUrl.scheme() == URL_SCHEME_HTTP ||
+        // Save the last visited domain URL.
-        // lookupUrl.scheme() == HIFI_URL_SCHEME_HTTPS ||
+        _lastVisitedURL = lookupUrl;
-        // TODO once a file can return a connection refusal if there were to be some kind of load error, we'd
-        // need to store the previous domain tried in _lastVisitedURL. For now , do not store it.
 
         _previousAPILookup.clear();
         _shareablePlaceName.clear();

libraries/networking/src/DomainHandler.cpp

@@ -209,7 +209,9 @@ void DomainHandler::setURLAndID(QUrl domainURL, QUuid domainID) {
     }
 
     // if it's in the error state, reset and try again.
-    if ((_domainURL != domainURL || _sockAddr.getPort() != domainPort) || _isInErrorState) {
+    if (_domainURL != domainURL 
+        || (_sockAddr.getPort() != domainPort && domainURL.scheme() == URL_SCHEME_HIFI)
+        || _isInErrorState) {
         // re-set the domain info so that auth information is reloaded
         hardReset("Changing domain URL");
 

libraries/octree/src/Octree.cpp

@@ -738,7 +738,6 @@ bool Octree::readFromURL(
 ) {
     QString trimmedUrl = urlString.trimmed();
     QString marketplaceID = getMarketplaceID(trimmedUrl);
-    qDebug() << "!!!!! going to createResourceRequest " << callerId;
     auto request = std::unique_ptr<ResourceRequest>(
         DependencyManager::get<ResourceManager>()->createResourceRequest(
             this, trimmedUrl, isObservable, callerId, "Octree::readFromURL"));
@@ -770,6 +769,24 @@ bool Octree::readFromURL(
     return readFromStream(data.size(), inputStream, marketplaceID);
 }
 
+bool Octree::readFromByteArray(
+    const QString& urlString,
+    const QByteArray& data
+) {
+    QString trimmedUrl = urlString.trimmed();
+    QString marketplaceID = getMarketplaceID(trimmedUrl);
+
+    QByteArray uncompressedJsonData;
+    bool wasCompressed = gunzip(data, uncompressedJsonData);
+
+    if (wasCompressed) {
+        QDataStream inputStream(uncompressedJsonData);
+        return readFromStream(uncompressedJsonData.size(), inputStream, marketplaceID);
+    }
+
+    QDataStream inputStream(data);
+    return readFromStream(data.size(), inputStream, marketplaceID);
+}
 
 bool Octree::readFromStream(
     uint64_t streamLength,

libraries/octree/src/Octree.h

@@ -217,6 +217,7 @@ public:
     // Octree importers
     bool readFromFile(const char* filename);
     bool readFromURL(const QString& url, const bool isObservable = true, const qint64 callerId = -1); // will support file urls as well...
+    bool readFromByteArray(const QString& url, const QByteArray& byteArray);
     bool readFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID="");
     bool readSVOFromStream(uint64_t streamLength, QDataStream& inputStream);
     bool readJSONFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID="");