From 9ac783a88d65d8acbe1192606453e40e577a2179 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 9 May 2016 14:56:25 -0700 Subject: [PATCH 1/2] Black box asynch resource retrieval --- interface/src/Application.cpp | 9 --------- interface/src/Application.h | 2 -- libraries/networking/src/ResourceCache.cpp | 22 ++++------------------ libraries/networking/src/ResourceCache.h | 17 +++++++---------- 4 files changed, 11 insertions(+), 39 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5b0d5c65ce..6b1cb48508 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -584,12 +584,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : // put the NodeList and datagram processing on the node thread nodeList->moveToThread(nodeThread); - // Model background downloads need to happen on the Datagram Processor Thread. The idle loop will - // emit checkBackgroundDownloads to cause the ModelCache to check it's queue for requested background - // downloads. - auto modelCache = DependencyManager::get(); - connect(this, &Application::checkBackgroundDownloads, modelCache.data(), &ModelCache::checkAsynchronousGets); - // put the audio processing on a separate thread QThread* audioThread = new QThread(); audioThread->setObjectName("Audio Thread"); @@ -2746,9 +2740,6 @@ void Application::idle(uint64_t now) { } _overlayConductor.update(secondsSinceLastUpdate); - - // check for any requested background downloads. - emit checkBackgroundDownloads(); } void Application::setLowVelocityFilter(bool lowVelocityFilter) { diff --git a/interface/src/Application.h b/interface/src/Application.h index 2911d42b65..965ad183fe 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -224,8 +224,6 @@ public: signals: void svoImportRequested(const QString& url); - void checkBackgroundDownloads(); - void fullAvatarURLChanged(const QString& newValue, const QString& modelName); void beforeAboutToQuit(); diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index d6a2fa7dba..e3b333aae9 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -316,23 +316,6 @@ void ResourceCache::setRequestLimit(int limit) { } } -void ResourceCache::getResourceAsynchronously(const QUrl& url) { - qCDebug(networking) << "ResourceCache::getResourceAsynchronously" << url.toString(); - QWriteLocker locker(&_resourcesToBeGottenLock); - _resourcesToBeGotten.enqueue(QUrl(url)); -} - -void ResourceCache::checkAsynchronousGets() { - assert(QThread::currentThread() == thread()); - QWriteLocker locker(&_resourcesToBeGottenLock); - if (!_resourcesToBeGotten.isEmpty()) { - QUrl url = _resourcesToBeGotten.dequeue(); - - locker.unlock(); - getResource(url); - } -} - QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, bool delayLoad, void* extra) { QSharedPointer resource; @@ -346,8 +329,11 @@ QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& } if (QThread::currentThread() != thread()) { + qCDebug(networking) << "Fetching asynchronously:" << url; assert(delayLoad); - getResourceAsynchronously(url); + QMetaObject::invokeMethod(this, "getResource", + Q_ARG(QUrl, url), Q_ARG(QUrl, fallback), Q_ARG(bool, delayLoad)); + // Cannot use extra parameter as it might be freed before the invocation return QSharedPointer(); } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 64e3f4dc34..6678c3157a 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -179,9 +179,6 @@ public: signals: void dirty(); -public slots: - void checkAsynchronousGets(); - protected slots: void updateTotalSize(const qint64& deltaSize); @@ -190,6 +187,13 @@ protected slots: // and delegate to it (see TextureCache::prefetch(const QUrl&, int). ScriptableResource* prefetch(const QUrl& url, void* extra); + /// Loads a resource from the specified URL. + /// \param fallback a fallback URL to load if the desired one is unavailable + /// \param delayLoad if true, don't load the resource immediately; wait until load is first requested + /// \param extra extra data to pass to the creator, if appropriate + QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), + bool delayLoad = false, void* extra = NULL); + private slots: void clearATPAssets(); @@ -200,13 +204,6 @@ protected: // the QScriptEngine will delete the pointer when it is garbage collected. Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); } - /// Loads a resource from the specified URL. - /// \param fallback a fallback URL to load if the desired one is unavailable - /// \param delayLoad if true, don't load the resource immediately; wait until load is first requested - /// \param extra extra data to pass to the creator, if appropriate - QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), - bool delayLoad = false, void* extra = NULL); - /// Creates a new resource. virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) = 0; From 09d879e19f4140ac8899f02b45e37c0ca02c03d8 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 9 May 2016 18:01:21 -0700 Subject: [PATCH 2/2] Remove delayLoad --- libraries/animation/src/AnimationCache.cpp | 2 +- libraries/animation/src/AnimationCache.h | 4 ++-- libraries/audio/src/SoundCache.cpp | 2 +- libraries/audio/src/SoundCache.h | 4 ++-- .../src/model-networking/ModelCache.cpp | 6 +++--- .../src/model-networking/ModelCache.h | 4 ++-- .../src/model-networking/ShaderCache.cpp | 14 ++++++-------- .../src/model-networking/ShaderCache.h | 5 +++-- .../src/model-networking/TextureCache.cpp | 8 ++++---- .../src/model-networking/TextureCache.h | 4 ++-- libraries/networking/src/ResourceCache.cpp | 18 +++++++++--------- libraries/networking/src/ResourceCache.h | 13 +++++++------ .../recording/src/recording/ClipCache.cpp | 15 ++++++--------- libraries/recording/src/recording/ClipCache.h | 4 ++-- 14 files changed, 50 insertions(+), 53 deletions(-) diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index 9364045857..482c4211cb 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -36,7 +36,7 @@ AnimationPointer AnimationCache::getAnimation(const QUrl& url) { } QSharedPointer AnimationCache::createResource(const QUrl& url, const QSharedPointer& fallback, - bool delayLoad, const void* extra) { + const void* extra) { return QSharedPointer(new Animation(url), &Resource::deleter); } diff --git a/libraries/animation/src/AnimationCache.h b/libraries/animation/src/AnimationCache.h index 59a4ad0498..9da649e66e 100644 --- a/libraries/animation/src/AnimationCache.h +++ b/libraries/animation/src/AnimationCache.h @@ -35,8 +35,8 @@ public: protected: - virtual QSharedPointer createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra); + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra); private: explicit AnimationCache(QObject* parent = NULL); virtual ~AnimationCache() { } diff --git a/libraries/audio/src/SoundCache.cpp b/libraries/audio/src/SoundCache.cpp index 2752c6669f..96a2cee204 100644 --- a/libraries/audio/src/SoundCache.cpp +++ b/libraries/audio/src/SoundCache.cpp @@ -35,7 +35,7 @@ SharedSoundPointer SoundCache::getSound(const QUrl& url) { } QSharedPointer SoundCache::createResource(const QUrl& url, const QSharedPointer& fallback, - bool delayLoad, const void* extra) { + const void* extra) { qCDebug(audio) << "Requesting sound at" << url.toString(); return QSharedPointer(new Sound(url), &Resource::deleter); } diff --git a/libraries/audio/src/SoundCache.h b/libraries/audio/src/SoundCache.h index 7942e16010..59a25dd847 100644 --- a/libraries/audio/src/SoundCache.h +++ b/libraries/audio/src/SoundCache.h @@ -25,8 +25,8 @@ public: Q_INVOKABLE SharedSoundPointer getSound(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra); + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra); private: SoundCache(QObject* parent = NULL); }; diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 24834967d6..8cd6d9b65e 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -71,7 +71,7 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) { GeometryExtra extra{ mapping, _textureBaseUrl }; // Get the raw GeometryResource, not the wrapped NetworkGeometry - _geometryResource = modelCache->getResource(url, QUrl(), false, &extra).staticCast(); + _geometryResource = modelCache->getResource(url, QUrl(), &extra).staticCast(); // Avoid caching nested resources - their references will be held by the parent _geometryResource->_isCacheable = false; @@ -236,7 +236,7 @@ ModelCache::ModelCache() { } QSharedPointer ModelCache::createResource(const QUrl& url, const QSharedPointer& fallback, - bool delayLoad, const void* extra) { + const void* extra) { Resource* resource = nullptr; if (url.path().toLower().endsWith(".fst")) { resource = new GeometryMappingResource(url); @@ -252,7 +252,7 @@ QSharedPointer ModelCache::createResource(const QUrl& url, const QShar std::shared_ptr ModelCache::getGeometry(const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl) { GeometryExtra geometryExtra = { mapping, textureBaseUrl }; - GeometryResource::Pointer resource = getResource(url, QUrl(), true, &geometryExtra).staticCast(); + GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast(); if (resource) { if (resource->isLoaded() && resource->shouldSetTextures()) { resource->setTextures(); diff --git a/libraries/model-networking/src/model-networking/ModelCache.h b/libraries/model-networking/src/model-networking/ModelCache.h index bf47f293e8..f15e1106e2 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.h +++ b/libraries/model-networking/src/model-networking/ModelCache.h @@ -44,8 +44,8 @@ public: protected: friend class GeometryMappingResource; - virtual QSharedPointer createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra); + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra); private: ModelCache(); diff --git a/libraries/model-networking/src/model-networking/ShaderCache.cpp b/libraries/model-networking/src/model-networking/ShaderCache.cpp index 3e14a99f87..bf7ade07f7 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.cpp +++ b/libraries/model-networking/src/model-networking/ShaderCache.cpp @@ -7,11 +7,8 @@ // #include "ShaderCache.h" -NetworkShader::NetworkShader(const QUrl& url, bool delayLoad) - : Resource(url, delayLoad) -{ - -} +NetworkShader::NetworkShader(const QUrl& url) : + Resource(url) {} void NetworkShader::downloadFinished(const QByteArray& data) { _source = QString::fromUtf8(data); @@ -24,10 +21,11 @@ ShaderCache& ShaderCache::instance() { } NetworkShaderPointer ShaderCache::getShader(const QUrl& url) { - return ResourceCache::getResource(url, QUrl(), false, nullptr).staticCast(); + return ResourceCache::getResource(url, QUrl(), nullptr).staticCast(); } -QSharedPointer ShaderCache::createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) { - return QSharedPointer(new NetworkShader(url, delayLoad), &Resource::deleter); +QSharedPointer ShaderCache::createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra) { + return QSharedPointer(new NetworkShader(url), &Resource::deleter); } diff --git a/libraries/model-networking/src/model-networking/ShaderCache.h b/libraries/model-networking/src/model-networking/ShaderCache.h index ffe8d437ce..4af12fdbfa 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.h +++ b/libraries/model-networking/src/model-networking/ShaderCache.h @@ -13,7 +13,7 @@ class NetworkShader : public Resource { public: - NetworkShader(const QUrl& url, bool delayLoad); + NetworkShader(const QUrl& url); virtual void downloadFinished(const QByteArray& data) override; QString _source; @@ -28,7 +28,8 @@ public: NetworkShaderPointer getShader(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra) override; }; #endif diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 2aaddace88..000ca67989 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -167,7 +167,7 @@ ScriptableResource* TextureCache::prefetch(const QUrl& url, int type) { NetworkTexturePointer TextureCache::getTexture(const QUrl& url, Type type, const QByteArray& content) { TextureExtra extra = { type, content }; - return ResourceCache::getResource(url, QUrl(), content.isEmpty(), &extra).staticCast(); + return ResourceCache::getResource(url, QUrl(), &extra).staticCast(); } @@ -231,8 +231,8 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, Type type return gpu::TexturePointer(loader(image, QUrl::fromLocalFile(path).fileName().toStdString())); } -QSharedPointer TextureCache::createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra) { +QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra) { const TextureExtra* textureExtra = static_cast(extra); auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; auto content = textureExtra ? textureExtra->content : QByteArray(); @@ -241,7 +241,7 @@ QSharedPointer TextureCache::createResource(const QUrl& url, } NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content) : - Resource(url, !content.isEmpty()), + Resource(url), _type(type) { _textureSource = std::make_shared(); diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 8fd0b12369..f3c6a86b49 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -131,8 +131,8 @@ protected: // Overload ResourceCache::prefetch to allow specifying texture type for loads Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type); - virtual QSharedPointer createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra); + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra); private: TextureCache(); diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index e3b333aae9..36828b3992 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -179,7 +179,7 @@ ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) { result = new ScriptableResource(url); - auto resource = getResource(url, QUrl(), false, extra); + auto resource = getResource(url, QUrl(), extra); result->_resource = resource; result->setObjectName(url.toString()); @@ -316,8 +316,7 @@ void ResourceCache::setRequestLimit(int limit) { } } -QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, - bool delayLoad, void* extra) { +QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra) { QSharedPointer resource; { QReadLocker locker(&_resourcesLock); @@ -330,19 +329,20 @@ QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& if (QThread::currentThread() != thread()) { qCDebug(networking) << "Fetching asynchronously:" << url; - assert(delayLoad); QMetaObject::invokeMethod(this, "getResource", - Q_ARG(QUrl, url), Q_ARG(QUrl, fallback), Q_ARG(bool, delayLoad)); + Q_ARG(QUrl, url), Q_ARG(QUrl, fallback)); // Cannot use extra parameter as it might be freed before the invocation return QSharedPointer(); } if (!url.isValid() && !url.isEmpty() && fallback.isValid()) { - return getResource(fallback, QUrl(), delayLoad); + return getResource(fallback, QUrl()); } - resource = createResource(url, fallback.isValid() ? - getResource(fallback, QUrl(), true) : QSharedPointer(), delayLoad, extra); + resource = createResource( + url, + fallback.isValid() ? getResource(fallback, QUrl()) : QSharedPointer(), + extra); resource->setSelf(resource); resource->setCache(this); connect(resource.data(), &Resource::updateSize, this, &ResourceCache::updateTotalSize); @@ -494,7 +494,7 @@ const int DEFAULT_REQUEST_LIMIT = 10; int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT; int ResourceCache::_requestsActive = 0; -Resource::Resource(const QUrl& url, bool delayLoad) : +Resource::Resource(const QUrl& url) : _url(url), _activeUrl(url), _request(nullptr) { diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 6678c3157a..11b80ca349 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -187,12 +187,13 @@ protected slots: // and delegate to it (see TextureCache::prefetch(const QUrl&, int). ScriptableResource* prefetch(const QUrl& url, void* extra); - /// Loads a resource from the specified URL. + /// Loads a resource from the specified URL and returns it. + /// If the caller is on a different thread than the ResourceCache, + /// returns an empty smart pointer and loads its asynchronously. /// \param fallback a fallback URL to load if the desired one is unavailable - /// \param delayLoad if true, don't load the resource immediately; wait until load is first requested /// \param extra extra data to pass to the creator, if appropriate QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), - bool delayLoad = false, void* extra = NULL); + void* extra = NULL); private slots: void clearATPAssets(); @@ -205,8 +206,8 @@ protected: Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); } /// Creates a new resource. - virtual QSharedPointer createResource(const QUrl& url, - const QSharedPointer& fallback, bool delayLoad, const void* extra) = 0; + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, + const void* extra) = 0; void addUnusedResource(const QSharedPointer& resource); void removeUnusedResource(const QSharedPointer& resource); @@ -257,7 +258,7 @@ class Resource : public QObject { public: - Resource(const QUrl& url, bool delayLoad = false); + Resource(const QUrl& url); ~Resource(); /// Returns the key last used to identify this resource in the unused map. diff --git a/libraries/recording/src/recording/ClipCache.cpp b/libraries/recording/src/recording/ClipCache.cpp index 145ecfc7b9..66d5fb0d88 100644 --- a/libraries/recording/src/recording/ClipCache.cpp +++ b/libraries/recording/src/recording/ClipCache.cpp @@ -9,12 +9,9 @@ #include "impl/PointerClip.h" using namespace recording; -NetworkClipLoader::NetworkClipLoader(const QUrl& url, bool delayLoad) - : Resource(url, delayLoad), _clip(std::make_shared(url)) -{ - -} - +NetworkClipLoader::NetworkClipLoader(const QUrl& url) : + Resource(url), + _clip(std::make_shared(url)) {} void NetworkClip::init(const QByteArray& clipData) { _clipData = clipData; @@ -32,10 +29,10 @@ ClipCache& ClipCache::instance() { } NetworkClipLoaderPointer ClipCache::getClipLoader(const QUrl& url) { - return ResourceCache::getResource(url, QUrl(), false, nullptr).staticCast(); + return ResourceCache::getResource(url, QUrl(), nullptr).staticCast(); } -QSharedPointer ClipCache::createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) { - return QSharedPointer(new NetworkClipLoader(url, delayLoad), &Resource::deleter); +QSharedPointer ClipCache::createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) { + return QSharedPointer(new NetworkClipLoader(url), &Resource::deleter); } diff --git a/libraries/recording/src/recording/ClipCache.h b/libraries/recording/src/recording/ClipCache.h index c72d45648d..7df83efdbf 100644 --- a/libraries/recording/src/recording/ClipCache.h +++ b/libraries/recording/src/recording/ClipCache.h @@ -31,7 +31,7 @@ private: class NetworkClipLoader : public Resource { public: - NetworkClipLoader(const QUrl& url, bool delayLoad); + NetworkClipLoader(const QUrl& url); virtual void downloadFinished(const QByteArray& data) override; ClipPointer getClip() { return _clip; } bool completed() { return _failedToLoad || isLoaded(); } @@ -49,7 +49,7 @@ public: NetworkClipLoaderPointer getClipLoader(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, bool delayLoad, const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) override; }; }