Merge pull request #4535 from ZappoMan/scriptCaching

Script caching & Fix to Stack Overflow Crash on Windows with lots of entity scripts

interface/src/Application.cpp

@@ -76,7 +76,7 @@
 #include <PhysicsEngine.h>
 #include <ProgramObject.h>
 #include <ResourceCache.h>
-//#include <ScriptCache.h>
+#include <ScriptCache.h>
 #include <SettingHandle.h>
 #include <SoundCache.h>
 #include <TextRenderer.h>
@@ -221,7 +221,7 @@ bool setupEssentials(int& argc, char** argv) {
     auto addressManager = DependencyManager::set<AddressManager>();
     auto nodeList = DependencyManager::set<NodeList>(NodeType::Agent, listenPort);
     auto geometryCache = DependencyManager::set<GeometryCache>();
-    //auto scriptCache = DependencyManager::set<ScriptCache>();
+    auto scriptCache = DependencyManager::set<ScriptCache>();
     auto soundCache = DependencyManager::set<SoundCache>();
     auto glowEffect = DependencyManager::set<GlowEffect>();
     auto faceshift = DependencyManager::set<Faceshift>();
@@ -620,7 +620,7 @@ Application::~Application() {
     DependencyManager::destroy<AnimationCache>();
     DependencyManager::destroy<TextureCache>();
     DependencyManager::destroy<GeometryCache>();
-    //DependencyManager::destroy<ScriptCache>();
+    DependencyManager::destroy<ScriptCache>();
     DependencyManager::destroy<SoundCache>();
     
     QThread* nodeThread = DependencyManager::get<NodeList>()->thread();

libraries/entities-renderer/src/EntityTreeRenderer.cpp

@@ -110,14 +110,30 @@ void EntityTreeRenderer::shutdown() {
     _shuttingDown = true;
 }
 
+void EntityTreeRenderer::scriptContentsAvailable(const QUrl& url, const QString& scriptContents) {
+    if (_waitingOnPreload.contains(url)) {
+        QList<EntityItemID> entityIDs = _waitingOnPreload.values(url);
+        _waitingOnPreload.remove(url);
+        foreach(EntityItemID entityID, entityIDs) {
+            checkAndCallPreload(entityID);
+        } 
+    }
+}
 
-QScriptValue EntityTreeRenderer::loadEntityScript(const EntityItemID& entityItemID) {
+void EntityTreeRenderer::errorInLoadingScript(const QUrl& url) {
+    if (_waitingOnPreload.contains(url)) {
+        _waitingOnPreload.remove(url);
+    }
+}
+
+QScriptValue EntityTreeRenderer::loadEntityScript(const EntityItemID& entityItemID, bool isPreload) {
     EntityItem* entity = static_cast<EntityTree*>(_tree)->findEntityByEntityItemID(entityItemID);
-    return loadEntityScript(entity);
+    return loadEntityScript(entity, isPreload);
 }
 
 
-QString EntityTreeRenderer::loadScriptContents(const QString& scriptMaybeURLorText, bool& isURL) {
+QString EntityTreeRenderer::loadScriptContents(const QString& scriptMaybeURLorText, bool& isURL, bool& isPending, QUrl& urlOut) {
+    isPending = false;
     QUrl url(scriptMaybeURLorText);
     
     // If the url is not valid, this must be script text...
@@ -126,6 +142,7 @@ QString EntityTreeRenderer::loadScriptContents(const QString& scriptMaybeURLorTe
         return scriptMaybeURLorText;
     }
     isURL = true;
+    urlOut = url;
 
     QString scriptContents; // assume empty
     
@@ -148,20 +165,8 @@ QString EntityTreeRenderer::loadScriptContents(const QString& scriptMaybeURLorTe
                 qDebug() << "ERROR Loading file:" << fileName;
             }
         } else {
-            QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+            auto scriptCache = DependencyManager::get<ScriptCache>();
-            QNetworkRequest networkRequest = QNetworkRequest(url);
+            scriptContents = scriptCache->getScript(url, this, isPending);
-            networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
-            QNetworkReply* reply = networkAccessManager.get(networkRequest);
-            qDebug() << "Downloading script at" << url;
-            QEventLoop loop;
-            QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
-            loop.exec();
-            if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) {
-                scriptContents = reply->readAll();
-            } else {
-                qDebug() << "ERROR Loading file:" << url.toString();
-            }
-            delete reply;
         }
     }
     
@@ -169,7 +174,7 @@ QString EntityTreeRenderer::loadScriptContents(const QString& scriptMaybeURLorTe
 }
 
 
-QScriptValue EntityTreeRenderer::loadEntityScript(EntityItem* entity) {
+QScriptValue EntityTreeRenderer::loadEntityScript(EntityItem* entity, bool isPreload) {
     if (_shuttingDown) {
         return QScriptValue(); // since we're shutting down, we don't load any more scripts
     }
@@ -203,7 +208,18 @@ QScriptValue EntityTreeRenderer::loadEntityScript(EntityItem* entity) {
     }
     
     bool isURL = false; // loadScriptContents() will tell us if this is a URL or just text.
-    QString scriptContents = loadScriptContents(entityScript, isURL);
+    bool isPending = false;
+    QUrl url;
+    QString scriptContents = loadScriptContents(entityScript, isURL, isPending, url);
+    
+    if (isPending && isPreload && isURL) {
+        _waitingOnPreload.insert(url, entityID);
+        
+    }
+    
+    if (scriptContents.isEmpty()) {
+        return QScriptValue(); // no script contents...
+    }
     
     QScriptSyntaxCheckResult syntaxCheck = QScriptEngine::checkSyntax(scriptContents);
     if (syntaxCheck.state() != QScriptSyntaxCheckResult::Valid) {
@@ -920,7 +936,7 @@ void EntityTreeRenderer::entitySciptChanging(const EntityItemID& entityID) {
 void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID) {
     if (_tree && !_shuttingDown) {
         // load the entity script if needed...
-        QScriptValue entityScript = loadEntityScript(entityID);
+        QScriptValue entityScript = loadEntityScript(entityID, true); // is preload!
         if (entityScript.property("preload").isValid()) {
             QScriptValueList entityArgs = createEntityArgs(entityID);
             entityScript.property("preload").call(entityScript, entityArgs);

libraries/entities-renderer/src/EntityTreeRenderer.h

@@ -16,6 +16,7 @@
 #include <EntityScriptingInterface.h> // for RayToEntityIntersectionResult
 #include <MouseEvent.h>
 #include <OctreeRenderer.h>
+#include <ScriptCache.h>
 
 class Model;
 class ScriptEngine;
@@ -31,7 +32,7 @@ public:
 };
 
 // Generic client side Octree renderer class.
-class EntityTreeRenderer : public OctreeRenderer, public EntityItemFBXService {
+class EntityTreeRenderer : public OctreeRenderer, public EntityItemFBXService, public ScriptUser {
     Q_OBJECT
 public:
     EntityTreeRenderer(bool wantScripts, AbstractViewStateInterface* viewState, 
@@ -84,6 +85,9 @@ public:
     /// hovering over, and entering entities
     void connectSignalsToSlots(EntityScriptingInterface* entityScriptingInterface);
 
+    virtual void scriptContentsAvailable(const QUrl& url, const QString& scriptContents);
+    virtual void errorInLoadingScript(const QUrl& url);
+
 signals:
     void mousePressOnEntity(const EntityItemID& entityItemID, const MouseEvent& event);
     void mouseMoveOnEntity(const EntityItemID& entityItemID, const MouseEvent& event);
@@ -138,10 +142,10 @@ private:
     ScriptEngine* _entitiesScriptEngine;
     ScriptEngine* _sandboxScriptEngine;
 
-    QScriptValue loadEntityScript(EntityItem* entity);
+    QScriptValue loadEntityScript(EntityItem* entity, bool isPreload = false);
-    QScriptValue loadEntityScript(const EntityItemID& entityItemID);
+    QScriptValue loadEntityScript(const EntityItemID& entityItemID, bool isPreload = false);
     QScriptValue getPreviouslyLoadedEntityScript(const EntityItemID& entityItemID);
-    QString loadScriptContents(const QString& scriptMaybeURLorText, bool& isURL);
+    QString loadScriptContents(const QString& scriptMaybeURLorText, bool& isURL, bool& isPending, QUrl& url);
     QScriptValueList createMouseEventArgs(const EntityItemID& entityID, QMouseEvent* event, unsigned int deviceID);
     QScriptValueList createMouseEventArgs(const EntityItemID& entityID, const MouseEvent& mouseEvent);
     
@@ -157,6 +161,8 @@ private:
     bool _dontDoPrecisionPicking;
     
     bool _shuttingDown = false;
+
+    QMultiMap<QUrl, EntityItemID> _waitingOnPreload;
     
 };
 

libraries/script-engine/src/ScriptCache.cpp

@@ -0,0 +1,76 @@
+//
+//  ScriptCache.cpp
+//  libraries/script-engine/src
+//
+//  Created by Brad Hefta-Gaub on 2015-03-30
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <QCoreApplication>
+#include <QEventLoop>
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QObject>
+
+#include <NetworkAccessManager.h>
+#include <SharedUtil.h>
+
+#include "ScriptCache.h"
+
+ScriptCache::ScriptCache(QObject* parent) {
+    // nothing to do here...
+}
+
+QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending) {
+    QString scriptContents;
+    if (_scriptCache.contains(url)) {
+        qDebug() << "Found script in cache:" << url.toString();
+        scriptContents = _scriptCache[url];
+        scriptUser->scriptContentsAvailable(url, scriptContents);
+        isPending = false;
+    } else {
+        isPending = true;
+        bool alreadyWaiting = _scriptUsers.contains(url);
+        _scriptUsers.insert(url, scriptUser);
+        
+        if (alreadyWaiting) {
+            qDebug() << "Already downloading script at:" << url.toString();
+        } else {
+            QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+            QNetworkRequest networkRequest = QNetworkRequest(url);
+            networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
+
+            qDebug() << "Downloading script at:" << url.toString();
+            QNetworkReply* reply = networkAccessManager.get(networkRequest);
+            connect(reply, &QNetworkReply::finished, this, &ScriptCache::scriptDownloaded);
+        }
+    }
+    return scriptContents;
+}
+
+void ScriptCache::scriptDownloaded() {
+    QNetworkReply* reply = qobject_cast<QNetworkReply*>(sender());
+    QUrl url = reply->url();
+    QList<ScriptUser*> scriptUsers = _scriptUsers.values(url);
+    _scriptUsers.remove(url);
+    
+    if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) {
+        _scriptCache[url] = reply->readAll();
+        qDebug() << "Done downloading script at:" << url.toString();
+
+        foreach(ScriptUser* user, scriptUsers) {
+            user->scriptContentsAvailable(url, _scriptCache[url]);
+        }
+    } else {
+        qDebug() << "ERROR Loading file:" << reply->url().toString();
+        foreach(ScriptUser* user, scriptUsers) {
+            user->errorInLoadingScript(url);
+        }
+    }
+    reply->deleteLater();
+}
+
+    

libraries/script-engine/src/ScriptCache.h

@@ -0,0 +1,41 @@
+//
+//  ScriptCache.h
+//  libraries/script-engine/src
+//
+//  Created by Brad Hefta-Gaub on 2015-03-30
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_ScriptCache_h
+#define hifi_ScriptCache_h
+
+#include <ResourceCache.h>
+
+class ScriptUser {
+public:
+    virtual void scriptContentsAvailable(const QUrl& url, const QString& scriptContents) = 0;
+    virtual void errorInLoadingScript(const QUrl& url) = 0;
+};
+
+/// Interface for loading scripts
+class ScriptCache : public QObject, public Dependency {
+    Q_OBJECT
+    SINGLETON_DEPENDENCY
+
+public:
+    QString getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending);
+    
+private slots:
+    void scriptDownloaded();
+    
+private:
+    ScriptCache(QObject* parent = NULL);
+    
+    QHash<QUrl, QString> _scriptCache;
+    QMultiMap<QUrl, ScriptUser*> _scriptUsers;
+};
+
+#endif // hifi_ScriptCache_h

libraries/script-engine/src/ScriptEngine.cpp

@@ -34,6 +34,7 @@
 #include "EventTypes.h"
 #include "MenuItemProperties.h"
 #include "ScriptAudioInjector.h"
+#include "ScriptCache.h"
 #include "ScriptEngine.h"
 #include "TypedArrays.h"
 #include "XMLHttpRequestClass.h"
@@ -279,27 +280,22 @@ void ScriptEngine::loadURL(const QUrl& scriptURL) {
                 emit errorLoadingScript(_fileNameString);
             }
         } else {
-            QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+            bool isPending;
-            QNetworkRequest networkRequest = QNetworkRequest(url);
+            auto scriptCache = DependencyManager::get<ScriptCache>();
-            networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
+            scriptCache->getScript(url, this, isPending);
-            QNetworkReply* reply = networkAccessManager.get(networkRequest);
+            
-            connect(reply, &QNetworkReply::finished, this, &ScriptEngine::handleScriptDownload);
         }
     }
 }
 
-void ScriptEngine::handleScriptDownload() {
+void ScriptEngine::scriptContentsAvailable(const QUrl& url, const QString& scriptContents) {
-    QNetworkReply* reply = qobject_cast<QNetworkReply*>(sender());
+    _scriptContents = scriptContents;
-    
+    emit scriptLoaded(_fileNameString);
-    if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) {
+}
-        _scriptContents = reply->readAll();
+
-        emit scriptLoaded(_fileNameString);
+void ScriptEngine::errorInLoadingScript(const QUrl& url) {
-    } else {
+    qDebug() << "ERROR Loading file:" << url.toString();
-        qDebug() << "ERROR Loading file:" << reply->url().toString();
+    emit errorLoadingScript(_fileNameString); // ??
-        emit errorLoadingScript(_fileNameString);
-    }
-    
-    reply->deleteLater();
 }
 
 void ScriptEngine::init() {

libraries/script-engine/src/ScriptEngine.h

@@ -28,6 +28,7 @@
 #include "ArrayBufferClass.h"
 #include "AudioScriptingInterface.h"
 #include "Quat.h"
+#include "ScriptCache.h"
 #include "ScriptUUID.h"
 #include "Vec3.h"
 
@@ -35,7 +36,7 @@ const QString NO_SCRIPT("");
 
 const unsigned int SCRIPT_DATA_CALLBACK_USECS = floor(((1.0 / 60.0f) * 1000 * 1000) + 0.5);
 
-class ScriptEngine : public QScriptEngine {
+class ScriptEngine : public QScriptEngine, public ScriptUser {
     Q_OBJECT
 public:
     ScriptEngine(const QString& scriptContents = NO_SCRIPT,
@@ -94,6 +95,9 @@ public:
     
     void waitTillDoneRunning();
 
+    virtual void scriptContentsAvailable(const QUrl& url, const QString& scriptContents);
+    virtual void errorInLoadingScript(const QUrl& url);
+
 public slots:
     void loadURL(const QUrl& scriptURL);
     void stop();
@@ -160,8 +164,6 @@ private:
     ArrayBufferClass* _arrayBufferClass;
 
     QHash<QUuid, quint16> _outgoingScriptAudioSequenceNumbers;
-private slots:
-    void handleScriptDownload();
 
 private:
     static QSet<ScriptEngine*> _allKnownScriptEngines;