From 91937041731f2908df99afffccace23db729189b Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Tue, 5 Feb 2019 18:27:27 -0800 Subject: [PATCH] trying to get hashes working --- .../src/model-networking/MaterialCache.cpp | 2 +- .../src/model-networking/ModelCache.cpp | 50 +++++++++++++++++-- .../src/model-networking/ShaderCache.cpp | 2 +- .../src/model-networking/TextureCache.cpp | 22 +++++++- libraries/networking/src/ResourceCache.cpp | 24 ++++----- libraries/networking/src/ResourceCache.h | 19 +++---- 6 files changed, 89 insertions(+), 30 deletions(-) diff --git a/libraries/model-networking/src/model-networking/MaterialCache.cpp b/libraries/model-networking/src/model-networking/MaterialCache.cpp index 7dcd7b8a61..aaa9767397 100644 --- a/libraries/model-networking/src/model-networking/MaterialCache.cpp +++ b/libraries/model-networking/src/model-networking/MaterialCache.cpp @@ -417,7 +417,7 @@ MaterialCache& MaterialCache::instance() { } NetworkMaterialResourcePointer MaterialCache::getMaterial(const QUrl& url) { - return ResourceCache::getResource(url, QUrl()).staticCast(); + return ResourceCache::getResource(url).staticCast(); } QSharedPointer MaterialCache::createResource(const QUrl& url) { diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 7515dad256..1535f5cfad 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -40,6 +40,50 @@ public: bool combineParts; }; +// From: https://stackoverflow.com/questions/41145012/how-to-hash-qvariant +class QVariantHasher { +public: + QVariantHasher() : buff(&bb), ds(&buff) { + bb.reserve(1000); + buff.open(QIODevice::WriteOnly); + } + uint hash(const QVariant& v) { + buff.seek(0); + ds << v; + return qHashBits(bb.constData(), buff.pos()); + } +private: + QByteArray bb; + QBuffer buff; + QDataStream ds; +}; + +namespace std { + template <> + struct hash { + size_t operator()(const QVariantHash& a) const { + QVariantHasher hasher; + return hasher.hash(a); + } + }; + + template <> + struct hash { + size_t operator()(const QUrl& a) const { + return qHash(a); + } + }; + + template <> + struct hash { + size_t operator()(const GeometryExtra& a) const { + size_t result = 0; + hash_combine(result, a.mapping, a.textureBaseUrl, a.combineParts); + return result; + } + }; +} + QUrl resolveTextureBaseUrl(const QUrl& url, const QUrl& textureBaseUrl) { return textureBaseUrl.isValid() ? textureBaseUrl : url; } @@ -110,7 +154,7 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) { GeometryExtra extra { _mapping, _textureBaseUrl, false }; // Get the raw GeometryResource - _geometryResource = modelCache->getResource(url, QUrl(), &extra).staticCast(); + _geometryResource = modelCache->getResource(url, QUrl(), &extra, std::hash()(extra)).staticCast(); // Avoid caching nested resources - their references will be held by the parent _geometryResource->_isCacheable = false; @@ -355,7 +399,7 @@ GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl) { bool combineParts = true; GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts }; - GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast(); + GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash()(geometryExtra)).staticCast(); if (resource) { if (resource->isLoaded() && resource->shouldSetTextures()) { resource->setTextures(); @@ -368,7 +412,7 @@ GeometryResource::Pointer ModelCache::getCollisionGeometryResource(const QUrl& u const QVariantHash& mapping, const QUrl& textureBaseUrl) { bool combineParts = false; GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts }; - GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast(); + GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash()(geometryExtra)).staticCast(); if (resource) { if (resource->isLoaded() && resource->shouldSetTextures()) { resource->setTextures(); diff --git a/libraries/model-networking/src/model-networking/ShaderCache.cpp b/libraries/model-networking/src/model-networking/ShaderCache.cpp index b774ca36c5..8d060c42f2 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.cpp +++ b/libraries/model-networking/src/model-networking/ShaderCache.cpp @@ -21,7 +21,7 @@ ShaderCache& ShaderCache::instance() { } NetworkShaderPointer ShaderCache::getShader(const QUrl& url) { - return ResourceCache::getResource(url, QUrl()).staticCast(); + return ResourceCache::getResource(url).staticCast(); } QSharedPointer ShaderCache::createResource(const QUrl& url) { diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 2f29eaf385..910de258f9 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -194,10 +194,28 @@ public: int maxNumPixels; }; +namespace std { + template <> + struct hash { + size_t operator()(const QByteArray& a) const { + return qHash(a); + } + }; + + template <> + struct hash { + size_t operator()(const TextureExtra& a) const { + size_t result = 0; + hash_combine(result, (int)a.type, a.content, a.maxNumPixels); + return result; + } + }; +} + ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxNumPixels) { auto byteArray = QByteArray(); TextureExtra extra = { (image::TextureUsage::Type)type, byteArray, maxNumPixels }; - return ResourceCache::prefetch(url, &extra); + return ResourceCache::prefetch(url, &extra, std::hash()(extra)); } NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) { @@ -211,7 +229,7 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs modifiedUrl.setQuery(query.toString()); } TextureExtra extra = { type, content, maxNumPixels }; - return ResourceCache::getResource(modifiedUrl, QUrl(), &extra).staticCast(); + return ResourceCache::getResource(modifiedUrl, QUrl(), &extra, std::hash()(extra)).staticCast(); } gpu::TexturePointer TextureCache::getTextureByHash(const std::string& hash) { diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 43fc93ffc5..cb7b8c7c82 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -158,8 +158,8 @@ void ScriptableResourceCache::updateTotalSize(const qint64& deltaSize) { _resourceCache->updateTotalSize(deltaSize); } -ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra) { - return _resourceCache->prefetch(url, extra); +ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) { + return _resourceCache->prefetch(url, extra, extraHash); } @@ -211,20 +211,20 @@ void ScriptableResource::disconnectHelper() { } } -ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) { +ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) { ScriptableResource* result = nullptr; if (QThread::currentThread() != thread()) { // Must be called in thread to ensure getResource returns a valid pointer BLOCKING_INVOKE_METHOD(this, "prefetch", Q_RETURN_ARG(ScriptableResource*, result), - Q_ARG(QUrl, url), Q_ARG(void*, extra)); + Q_ARG(QUrl, url), Q_ARG(void*, extra), Q_ARG(size_t, extraHash)); return result; } result = new ScriptableResource(url); - auto resource = getResource(url, QUrl(), extra); + auto resource = getResource(url, QUrl(), extra, extraHash); result->_resource = resource; result->setObjectName(url.toString()); @@ -298,7 +298,7 @@ void ResourceCache::refreshAll() { clearUnusedResources(); resetUnusedResourceCounter(); - QHash>> allResources; + QHash>> allResources; { QReadLocker locker(&_resourcesLock); allResources = _resources; @@ -343,13 +343,8 @@ void ResourceCache::setRequestLimit(uint32_t limit) { } } -QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, int extraHash) { +QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash) { QSharedPointer resource; - if (extra && extraHash < 0) { - qDebug() << "ResourceCache::getResource: ERROR! Non-null extra, but invalid extraHash"; - return resource; - } - { QReadLocker locker(&_resourcesLock); auto& resourcesWithExtraHash = _resources[url]; @@ -553,8 +548,8 @@ bool ResourceCache::attemptHighestPriorityRequest() { static int requestID = 0; Resource::Resource(const Resource& other) : + QObject(), _url(other._url), - _extraHash(other._extraHash), _effectiveBaseURL(other._effectiveBaseURL), _activeUrl(other._activeUrl), _requestByteRange(other._requestByteRange), @@ -566,7 +561,8 @@ Resource::Resource(const Resource& other) : _bytesReceived(other._bytesReceived), _bytesTotal(other._bytesTotal), _bytes(other._bytes), - _requestID(++requestID) { + _requestID(++requestID), + _extraHash(other._extraHash) { if (!other._loaded) { _startedLoading = false; } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index dd60e3c110..740bdadc48 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -231,15 +231,16 @@ protected slots: // Prefetches a resource to be held by the QScriptEngine. // Left as a protected member so subclasses can overload prefetch // and delegate to it (see TextureCache::prefetch(const QUrl&, int). - ScriptableResource* prefetch(const QUrl& url, void* extra); + ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash); // FIXME: The return type is not recognized by JavaScript. /// Loads a resource from the specified URL and returns it. /// If the caller is on a different thread than the ResourceCache, /// returns an empty smart pointer and loads its asynchronously. /// \param fallback a fallback URL to load if the desired one is unavailable - /// \param extra extra data to pass to the creator, if appropriate - QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), void* extra = NULL, int extraHash = -1); + // FIXME: std::numeric_limits::max() could be a valid extraHash + QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl()) { return getResource(url, fallback, nullptr, std::numeric_limits::max()); } + QSharedPointer getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash); private slots: void clearATPAssets(); @@ -250,7 +251,7 @@ protected: // which should be a QScriptEngine with ScriptableResource registered, so that // the QScriptEngine will delete the pointer when it is garbage collected. // JSDoc is provided on more general function signature. - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); } + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits::max()); } /// Creates a new resource. virtual QSharedPointer createResource(const QUrl& url) = 0; @@ -277,7 +278,7 @@ private: void resetResourceCounters(); // Resources - QHash>> _resources; + QHash>> _resources; QReadWriteLock _resourcesLock { QReadWriteLock::Recursive }; int _lastLRUKey = 0; @@ -331,10 +332,10 @@ public: * Prefetches a resource. * @function ResourceCache.prefetch * @param {string} url - URL of the resource to prefetch. - * @param {object} [extra=null] * @returns {ResourceObject} */ - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra = nullptr); + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits::max()); } + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash); signals: @@ -416,7 +417,7 @@ public: unsigned int getDownloadAttemptsRemaining() { return _attemptsRemaining; } virtual void setExtra(void* extra) {}; - void setExtraHash(int extraHash) { _extraHash = extraHash; } + void setExtraHash(size_t extraHash) { _extraHash = extraHash; } signals: /// Fired when the resource begins downloading. @@ -495,7 +496,7 @@ protected: int _requestID; ResourceRequest* _request{ nullptr }; - int _extraHash { -1 }; + size_t _extraHash; public slots: void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal);