From 3ee448a89a91b3737b316ad693964fa789342d72 Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Tue, 5 Feb 2019 17:32:15 -0800 Subject: [PATCH 01/11] fix scattering and working on resource cache bug --- libraries/animation/src/AnimationCache.cpp | 7 +- libraries/animation/src/AnimationCache.h | 7 +- libraries/audio/src/SoundCache.cpp | 7 +- libraries/audio/src/SoundCache.h | 5 +- .../src/RenderableModelEntityItem.cpp | 6 +- libraries/fbx/src/FBXSerializer_Material.cpp | 13 ++- libraries/fbx/src/FSTReader.cpp | 4 +- libraries/graphics/src/graphics/Material.cpp | 2 +- .../src/model-networking/MaterialCache.cpp | 8 +- .../src/model-networking/MaterialCache.h | 3 +- .../src/model-networking/ModelCache.cpp | 32 +++++-- .../src/model-networking/ModelCache.h | 12 ++- .../src/model-networking/ShaderCache.cpp | 8 +- .../src/model-networking/ShaderCache.h | 5 +- .../src/model-networking/TextureCache.cpp | 51 ++++++---- .../src/model-networking/TextureCache.h | 8 +- libraries/networking/src/ResourceCache.cpp | 95 ++++++++++++++----- libraries/networking/src/ResourceCache.h | 19 ++-- .../recording/src/recording/ClipCache.cpp | 5 +- libraries/recording/src/recording/ClipCache.h | 5 +- .../render-utils/src/RenderPipelines.cpp | 2 +- 21 files changed, 199 insertions(+), 105 deletions(-) diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index f7a7dd861a..4e988334f9 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -36,12 +36,13 @@ AnimationPointer AnimationCache::getAnimation(const QUrl& url) { return getResource(url).staticCast(); } -QSharedPointer AnimationCache::createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) { +QSharedPointer AnimationCache::createResource(const QUrl& url) { return QSharedPointer(new Animation(url), &Resource::deleter); } -Animation::Animation(const QUrl& url) : Resource(url) {} +QSharedPointer AnimationCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new Animation(*resource.staticCast().data()), &Resource::deleter); +} AnimationReader::AnimationReader(const QUrl& url, const QByteArray& data) : _url(url), diff --git a/libraries/animation/src/AnimationCache.h b/libraries/animation/src/AnimationCache.h index 2f8168625e..eea64475df 100644 --- a/libraries/animation/src/AnimationCache.h +++ b/libraries/animation/src/AnimationCache.h @@ -34,9 +34,9 @@ public: Q_INVOKABLE AnimationPointer getAnimation(const QUrl& url); protected: + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) override; private: explicit AnimationCache(QObject* parent = NULL); virtual ~AnimationCache() { } @@ -62,7 +62,8 @@ class Animation : public Resource { public: - explicit Animation(const QUrl& url); + Animation(const Animation& other) : Resource(other), _hfmModel(other._hfmModel) {} + Animation(const QUrl& url) : Resource(url) {} QString getType() const override { return "Animation"; } diff --git a/libraries/audio/src/SoundCache.cpp b/libraries/audio/src/SoundCache.cpp index 845fd6ab4f..343de46e9a 100644 --- a/libraries/audio/src/SoundCache.cpp +++ b/libraries/audio/src/SoundCache.cpp @@ -33,9 +33,12 @@ SharedSoundPointer SoundCache::getSound(const QUrl& url) { return getResource(url).staticCast(); } -QSharedPointer SoundCache::createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) { +QSharedPointer SoundCache::createResource(const QUrl& url) { auto resource = QSharedPointer(new Sound(url), &Resource::deleter); resource->setLoadPriority(this, SOUNDS_LOADING_PRIORITY); return resource; } + +QSharedPointer SoundCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new Sound(*resource.staticCast().data()), &Resource::deleter); +} \ No newline at end of file diff --git a/libraries/audio/src/SoundCache.h b/libraries/audio/src/SoundCache.h index 64d392a41d..48c3354877 100644 --- a/libraries/audio/src/SoundCache.h +++ b/libraries/audio/src/SoundCache.h @@ -24,8 +24,9 @@ public: Q_INVOKABLE SharedSoundPointer getSound(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; + private: SoundCache(QObject* parent = NULL); }; diff --git a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp index 7e01af04dd..9515ef94b5 100644 --- a/libraries/entities-renderer/src/RenderableModelEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableModelEntityItem.cpp @@ -280,11 +280,7 @@ bool RenderableModelEntityItem::findDetailedParabolaIntersection(const glm::vec3 } void RenderableModelEntityItem::fetchCollisionGeometryResource() { - QUrl hullURL(getCollisionShapeURL()); - QUrlQuery queryArgs(hullURL); - queryArgs.addQueryItem("collision-hull", ""); - hullURL.setQuery(queryArgs); - _compoundShapeResource = DependencyManager::get()->getCollisionGeometryResource(hullURL); + _compoundShapeResource = DependencyManager::get()->getCollisionGeometryResource(getCollisionShapeURL()); } bool RenderableModelEntityItem::computeShapeFailedToLoad() { diff --git a/libraries/fbx/src/FBXSerializer_Material.cpp b/libraries/fbx/src/FBXSerializer_Material.cpp index 0a1d15b72a..9caf713e75 100644 --- a/libraries/fbx/src/FBXSerializer_Material.cpp +++ b/libraries/fbx/src/FBXSerializer_Material.cpp @@ -76,13 +76,12 @@ HFMTexture FBXSerializer::getTexture(const QString& textureID, const QString& ma } void FBXSerializer::consolidateHFMMaterials(const QVariantHash& mapping) { - - QString materialMapString = mapping.value("materialMap").toString(); - QJsonDocument materialMapDocument = QJsonDocument::fromJson(materialMapString.toUtf8()); - QJsonObject materialMap = materialMapDocument.object(); - if (!materialMapString.isEmpty()) { - if (materialMapDocument.isEmpty() || materialMap.isEmpty()) { - qCDebug(modelformat) << "fbx Material Map found but did not produce valid JSON:" << materialMapString; + QJsonObject materialMap; + if (mapping.contains("materialMap")) { + QByteArray materialMapValue = mapping.value("materialMap").toByteArray(); + materialMap = QJsonDocument::fromJson(materialMapValue).object(); + if (materialMap.isEmpty()) { + qCDebug(modelformat) << "fbx Material Map found but did not produce valid JSON:" << materialMapValue; } } for (QHash::iterator it = _hfmMaterials.begin(); it != _hfmMaterials.end(); it++) { diff --git a/libraries/fbx/src/FSTReader.cpp b/libraries/fbx/src/FSTReader.cpp index 43806560dc..0ca1c6c9e6 100644 --- a/libraries/fbx/src/FSTReader.cpp +++ b/libraries/fbx/src/FSTReader.cpp @@ -88,7 +88,7 @@ QByteArray FSTReader::writeMapping(const QVariantHash& mapping) { << BLENDSHAPE_FIELD << JOINT_INDEX_FIELD; QBuffer buffer; buffer.open(QIODevice::WriteOnly); - + for (auto key : PREFERED_ORDER) { auto it = mapping.find(key); if (it != mapping.constEnd()) { @@ -104,7 +104,7 @@ QByteArray FSTReader::writeMapping(const QVariantHash& mapping) { } } } - + for (auto it = mapping.constBegin(); it != mapping.constEnd(); it++) { if (!PREFERED_ORDER.contains(it.key())) { writeVariant(buffer, it); diff --git a/libraries/graphics/src/graphics/Material.cpp b/libraries/graphics/src/graphics/Material.cpp index 7befb7e053..c19b139936 100755 --- a/libraries/graphics/src/graphics/Material.cpp +++ b/libraries/graphics/src/graphics/Material.cpp @@ -105,7 +105,7 @@ void Material::setMetallic(float metallic) { void Material::setScattering(float scattering) { scattering = glm::clamp(scattering, 0.0f, 1.0f); - _key.setMetallic(scattering > 0.0f); + _key.setScattering(scattering > 0.0f); _scattering = scattering; } diff --git a/libraries/model-networking/src/model-networking/MaterialCache.cpp b/libraries/model-networking/src/model-networking/MaterialCache.cpp index b6550a5e9e..7dcd7b8a61 100644 --- a/libraries/model-networking/src/model-networking/MaterialCache.cpp +++ b/libraries/model-networking/src/model-networking/MaterialCache.cpp @@ -417,9 +417,13 @@ MaterialCache& MaterialCache::instance() { } NetworkMaterialResourcePointer MaterialCache::getMaterial(const QUrl& url) { - return ResourceCache::getResource(url, QUrl(), nullptr).staticCast(); + return ResourceCache::getResource(url, QUrl()).staticCast(); } -QSharedPointer MaterialCache::createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) { +QSharedPointer MaterialCache::createResource(const QUrl& url) { return QSharedPointer(new NetworkMaterialResource(url), &Resource::deleter); +} + +QSharedPointer MaterialCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new NetworkMaterialResource(*resource.staticCast().data()), &Resource::deleter); } \ No newline at end of file diff --git a/libraries/model-networking/src/model-networking/MaterialCache.h b/libraries/model-networking/src/model-networking/MaterialCache.h index 074cd6c98d..6abadfc030 100644 --- a/libraries/model-networking/src/model-networking/MaterialCache.h +++ b/libraries/model-networking/src/model-networking/MaterialCache.h @@ -53,7 +53,8 @@ public: NetworkMaterialResourcePointer getMaterial(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; }; #endif diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 8c541040a7..7515dad256 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -107,7 +107,7 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) { } auto modelCache = DependencyManager::get(); - GeometryExtra extra{ _mapping, _textureBaseUrl, false }; + GeometryExtra extra { _mapping, _textureBaseUrl, false }; // Get the raw GeometryResource _geometryResource = modelCache->getResource(url, QUrl(), &extra).staticCast(); @@ -253,13 +253,19 @@ void GeometryReader::run() { class GeometryDefinitionResource : public GeometryResource { Q_OBJECT public: - GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl, bool combineParts) : - GeometryResource(url, resolveTextureBaseUrl(url, textureBaseUrl)), _modelLoader(modelLoader), _mapping(mapping), _combineParts(combineParts) {} + GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url) : GeometryResource(url), _modelLoader(modelLoader) {} + GeometryDefinitionResource(const GeometryDefinitionResource& other) : + GeometryResource(other), + _modelLoader(other._modelLoader), + _mapping(other._mapping), + _combineParts(other._combineParts) {} QString getType() const override { return "GeometryDefinition"; } virtual void downloadFinished(const QByteArray& data) override; + void setExtra(void* extra) override; + protected: Q_INVOKABLE void setGeometryDefinition(HFMModel::Pointer hfmModel); @@ -269,6 +275,13 @@ private: bool _combineParts; }; +void GeometryDefinitionResource::setExtra(void* extra) { + const GeometryExtra* geometryExtra = static_cast(extra); + _mapping = geometryExtra ? geometryExtra->mapping : QVariantHash(); + _textureBaseUrl = resolveTextureBaseUrl(_url, geometryExtra ? geometryExtra->textureBaseUrl : QUrl()); + _combineParts = geometryExtra ? geometryExtra->combineParts : true; +} + void GeometryDefinitionResource::downloadFinished(const QByteArray& data) { if (_url != _effectiveBaseURL) { _url = _effectiveBaseURL; @@ -323,22 +336,21 @@ ModelCache::ModelCache() { modelFormatRegistry->addFormat(GLTFSerializer()); } -QSharedPointer ModelCache::createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) { +QSharedPointer ModelCache::createResource(const QUrl& url) { Resource* resource = nullptr; if (url.path().toLower().endsWith(".fst")) { resource = new GeometryMappingResource(url); } else { - const GeometryExtra* geometryExtra = static_cast(extra); - auto mapping = geometryExtra ? geometryExtra->mapping : QVariantHash(); - auto textureBaseUrl = geometryExtra ? geometryExtra->textureBaseUrl : QUrl(); - bool combineParts = geometryExtra ? geometryExtra->combineParts : true; - resource = new GeometryDefinitionResource(_modelLoader, url, mapping, textureBaseUrl, combineParts); + resource = new GeometryDefinitionResource(_modelLoader, url); } return QSharedPointer(resource, &Resource::deleter); } +QSharedPointer ModelCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new GeometryDefinitionResource(*resource.staticCast().data()), &Resource::deleter); +} + GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl) { bool combineParts = true; diff --git a/libraries/model-networking/src/model-networking/ModelCache.h b/libraries/model-networking/src/model-networking/ModelCache.h index 1018bdecd5..497cae86a3 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.h +++ b/libraries/model-networking/src/model-networking/ModelCache.h @@ -82,8 +82,12 @@ class GeometryResource : public Resource, public Geometry { public: using Pointer = QSharedPointer; - GeometryResource(const QUrl& url, const QUrl& textureBaseUrl = QUrl()) : - Resource(url), _textureBaseUrl(textureBaseUrl) {} + GeometryResource(const QUrl& url) : Resource(url) {} + GeometryResource(const GeometryResource& other) : + Resource(other), + Geometry(other), + _textureBaseUrl(other._textureBaseUrl), + _isCacheable(other._isCacheable) {} virtual bool areTexturesLoaded() const override { return isLoaded() && Geometry::areTexturesLoaded(); } @@ -153,8 +157,8 @@ public: protected: friend class GeometryMappingResource; - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; private: ModelCache(); diff --git a/libraries/model-networking/src/model-networking/ShaderCache.cpp b/libraries/model-networking/src/model-networking/ShaderCache.cpp index bf7ade07f7..b774ca36c5 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.cpp +++ b/libraries/model-networking/src/model-networking/ShaderCache.cpp @@ -21,11 +21,13 @@ ShaderCache& ShaderCache::instance() { } NetworkShaderPointer ShaderCache::getShader(const QUrl& url) { - return ResourceCache::getResource(url, QUrl(), nullptr).staticCast(); + return ResourceCache::getResource(url, QUrl()).staticCast(); } -QSharedPointer ShaderCache::createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) { +QSharedPointer ShaderCache::createResource(const QUrl& url) { return QSharedPointer(new NetworkShader(url), &Resource::deleter); } +QSharedPointer ShaderCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new NetworkShader(*resource.staticCast().data()), &Resource::deleter); +} diff --git a/libraries/model-networking/src/model-networking/ShaderCache.h b/libraries/model-networking/src/model-networking/ShaderCache.h index bd78e6e7e3..fe9edd7ddf 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.h +++ b/libraries/model-networking/src/model-networking/ShaderCache.h @@ -14,6 +14,7 @@ class NetworkShader : public Resource { public: NetworkShader(const QUrl& url); + NetworkShader(const NetworkShader& other) : Resource(other), _source(other._source) {} QString getType() const override { return "NetworkShader"; } @@ -31,8 +32,8 @@ public: NetworkShaderPointer getShader(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; }; #endif diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 4c30dc6d93..2f29eaf385 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -305,42 +305,52 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::Te return gpu::TexturePointer(loader(std::move(image), path.toStdString(), shouldCompress, target, false)); } -QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) { - const TextureExtra* textureExtra = static_cast(extra); - auto type = textureExtra ? textureExtra->type : image::TextureUsage::DEFAULT_TEXTURE; - auto content = textureExtra ? textureExtra->content : QByteArray(); - auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; - NetworkTexture* texture = new NetworkTexture(url, type, content, maxNumPixels); - return QSharedPointer(texture, &Resource::deleter); +QSharedPointer TextureCache::createResource(const QUrl& url) { + return QSharedPointer(new NetworkTexture(url), &Resource::deleter); +} + +QSharedPointer TextureCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new NetworkTexture(*resource.staticCast().data()), &Resource::deleter); } int networkTexturePointerMetaTypeId = qRegisterMetaType>(); NetworkTexture::NetworkTexture(const QUrl& url) : -Resource(url), -_type(), -_maxNumPixels(100) + Resource(url), + _type(), + _maxNumPixels(100) { _textureSource = std::make_shared(url); _lowestRequestedMipLevel = 0; _loaded = true; } +NetworkTexture::NetworkTexture(const NetworkTexture& other) : + Resource(other), + _type(other._type), + _currentlyLoadingResourceType(other._currentlyLoadingResourceType), + _originalWidth(other._originalWidth), + _originalHeight(other._originalHeight), + _width(other._width), + _height(other._height), + _maxNumPixels(other._maxNumPixels) +{ +} + static bool isLocalUrl(const QUrl& url) { auto scheme = url.scheme(); return (scheme == HIFI_URL_SCHEME_FILE || scheme == URL_SCHEME_QRC || scheme == RESOURCE_SCHEME); } -NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) : - Resource(url), - _type(type), - _maxNumPixels(maxNumPixels) -{ - _textureSource = std::make_shared(url, (int)type); +void NetworkTexture::setExtra(void* extra) { + const TextureExtra* textureExtra = static_cast(extra); + _type = textureExtra ? textureExtra->type : image::TextureUsage::DEFAULT_TEXTURE; + _maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; + + _textureSource = std::make_shared(_url, (int)_type); _lowestRequestedMipLevel = 0; - auto fileNameLowercase = url.fileName().toLower(); + auto fileNameLowercase = _url.fileName().toLower(); if (fileNameLowercase.endsWith(TEXTURE_META_EXTENSION)) { _currentlyLoadingResourceType = ResourceType::META; } else if (fileNameLowercase.endsWith(".ktx")) { @@ -351,17 +361,18 @@ NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, _shouldFailOnRedirect = _currentlyLoadingResourceType != ResourceType::KTX; - if (type == image::TextureUsage::CUBE_TEXTURE) { + if (_type == image::TextureUsage::CUBE_TEXTURE) { setLoadPriority(this, SKYBOX_LOAD_PRIORITY); } else if (_currentlyLoadingResourceType == ResourceType::KTX) { setLoadPriority(this, HIGH_MIPS_LOAD_PRIORITY); } - if (!url.isValid()) { + if (!_url.isValid()) { _loaded = true; } // if we have content, load it after we have our self pointer + auto content = textureExtra ? textureExtra->content : QByteArray(); if (!content.isEmpty()) { _startedLoading = true; QMetaObject::invokeMethod(this, "downloadFinished", Qt::QueuedConnection, Q_ARG(const QByteArray&, content)); diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 3933e3ae56..d744d060b6 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -46,7 +46,7 @@ class NetworkTexture : public Resource, public Texture { public: NetworkTexture(const QUrl& url); - NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels); + NetworkTexture(const NetworkTexture& other); ~NetworkTexture() override; QString getType() const override { return "NetworkTexture"; } @@ -63,6 +63,8 @@ public: Q_INVOKABLE void setOriginalDescriptor(ktx::KTXDescriptor* descriptor) { _originalKtxDescriptor.reset(descriptor); } + void setExtra(void* extra) override; + signals: void networkTextureCreated(const QWeakPointer& self); @@ -201,8 +203,8 @@ protected: // Overload ResourceCache::prefetch to allow specifying texture type for loads Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; private: friend class ImageReader; diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 20fe05e7b8..43fc93ffc5 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -265,16 +265,17 @@ ResourceCache::~ResourceCache() { void ResourceCache::clearATPAssets() { { QWriteLocker locker(&_resourcesLock); - for (auto& url : _resources.keys()) { + QList urls = _resources.keys(); + for (auto& url : urls) { // If this is an ATP resource if (url.scheme() == URL_SCHEME_ATP) { - - // Remove it from the resource hash - auto resource = _resources.take(url); - if (auto strongRef = resource.lock()) { - // Make sure the resource won't reinsert itself - strongRef->setCache(nullptr); - _totalResourcesSize -= strongRef->getBytes(); + auto resourcesWithExtraHash = _resources.take(url); + for (auto& resource : resourcesWithExtraHash) { + if (auto strongRef = resource.lock()) { + // Make sure the resource won't reinsert itself + strongRef->setCache(nullptr); + _totalResourcesSize -= strongRef->getBytes(); + } } } } @@ -297,16 +298,20 @@ void ResourceCache::refreshAll() { clearUnusedResources(); resetUnusedResourceCounter(); - QHash> resources; + QHash>> allResources; { QReadLocker locker(&_resourcesLock); - resources = _resources; + allResources = _resources; } // Refresh all remaining resources in use - foreach (QSharedPointer resource, resources) { - if (resource) { - resource->refresh(); + // FIXME: this will trigger multiple refreshes for the same resource if they have different hashes + for (auto& resourcesWithExtraHash : allResources) { + for (auto& resourceWeak : resourcesWithExtraHash) { + auto resource = resourceWeak.lock(); + if (resource) { + resource->refresh(); + } } } } @@ -338,39 +343,59 @@ void ResourceCache::setRequestLimit(uint32_t limit) { } } -QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra) { +QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, int extraHash) { QSharedPointer resource; + if (extra && extraHash < 0) { + qDebug() << "ResourceCache::getResource: ERROR! Non-null extra, but invalid extraHash"; + return resource; + } + { QReadLocker locker(&_resourcesLock); - resource = _resources.value(url).lock(); + auto& resourcesWithExtraHash = _resources[url]; + auto resourcesWithExtraHashIter = resourcesWithExtraHash.find(extraHash); + if (resourcesWithExtraHashIter != resourcesWithExtraHash.end()) { + // We've seen this extra info before + resource = resourcesWithExtraHashIter.value().lock(); + } else if (resourcesWithExtraHash.size() > 0.0f) { + // We haven't seen this extra info before, but we've already downloaded the resource. We need a new copy of this object (with any old hash). + resource = createResourceCopy(resourcesWithExtraHash.begin().value().lock()); + resource->setExtra(extra); + resource->setExtraHash(extraHash); + resource->setSelf(resource); + resource->setCache(this); + resource->moveToThread(qApp->thread()); + connect(resource.data(), &Resource::updateSize, this, &ResourceCache::updateTotalSize); + resourcesWithExtraHash.insert(extraHash, resource); + removeUnusedResource(resource); + resource->ensureLoading(); + } } if (resource) { removeUnusedResource(resource); } - if (!resource && !url.isValid() && !url.isEmpty() && fallback.isValid()) { - resource = getResource(fallback, QUrl()); + if (!resource && (!url.isValid() || url.isEmpty()) && fallback.isValid()) { + resource = getResource(fallback, QUrl(), extra, extraHash); } if (!resource) { - resource = createResource( - url, - fallback.isValid() ? getResource(fallback, QUrl()) : QSharedPointer(), - extra); + resource = createResource(url); + resource->setExtra(extra); + resource->setExtraHash(extraHash); resource->setSelf(resource); resource->setCache(this); resource->moveToThread(qApp->thread()); connect(resource.data(), &Resource::updateSize, this, &ResourceCache::updateTotalSize); { QWriteLocker locker(&_resourcesLock); - _resources.insert(url, resource); + _resources[url].insert(extraHash, resource); } removeUnusedResource(resource); resource->ensureLoading(); } - DependencyManager::get()->update( - resource->getURL(), -1, "ResourceCache::getResource"); + DependencyManager::get()->update(resource->getURL(), -1, "ResourceCache::getResource"); return resource; } @@ -527,6 +552,26 @@ bool ResourceCache::attemptHighestPriorityRequest() { static int requestID = 0; +Resource::Resource(const Resource& other) : + _url(other._url), + _extraHash(other._extraHash), + _effectiveBaseURL(other._effectiveBaseURL), + _activeUrl(other._activeUrl), + _requestByteRange(other._requestByteRange), + _shouldFailOnRedirect(other._shouldFailOnRedirect), + _startedLoading(other._startedLoading), + _failedToLoad(other._failedToLoad), + _loaded(other._loaded), + _loadPriorities(other._loadPriorities), + _bytesReceived(other._bytesReceived), + _bytesTotal(other._bytesTotal), + _bytes(other._bytes), + _requestID(++requestID) { + if (!other._loaded) { + _startedLoading = false; + } +} + Resource::Resource(const QUrl& url) : _url(url), _activeUrl(url), @@ -678,7 +723,7 @@ void Resource::setSize(const qint64& bytes) { void Resource::reinsert() { QWriteLocker locker(&_cache->_resourcesLock); - _cache->_resources.insert(_url, _self); + _cache->_resources[_url].insert(_extraHash, _self); } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 275684f73e..dd60e3c110 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -239,8 +239,7 @@ protected slots: /// returns an empty smart pointer and loads its asynchronously. /// \param fallback a fallback URL to load if the desired one is unavailable /// \param extra extra data to pass to the creator, if appropriate - QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), - void* extra = NULL); + QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), void* extra = NULL, int extraHash = -1); private slots: void clearATPAssets(); @@ -254,8 +253,8 @@ protected: Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); } /// Creates a new resource. - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, - const void* extra) = 0; + virtual QSharedPointer createResource(const QUrl& url) = 0; + virtual QSharedPointer createResourceCopy(const QSharedPointer& resource) = 0; void addUnusedResource(const QSharedPointer& resource); void removeUnusedResource(const QSharedPointer& resource); @@ -278,7 +277,7 @@ private: void resetResourceCounters(); // Resources - QHash> _resources; + QHash>> _resources; QReadWriteLock _resourcesLock { QReadWriteLock::Recursive }; int _lastLRUKey = 0; @@ -359,7 +358,8 @@ class Resource : public QObject { Q_OBJECT public: - + + Resource(const Resource& other); Resource(const QUrl& url); virtual ~Resource(); @@ -415,6 +415,9 @@ public: unsigned int getDownloadAttempts() { return _attempts; } unsigned int getDownloadAttemptsRemaining() { return _attemptsRemaining; } + virtual void setExtra(void* extra) {}; + void setExtraHash(int extraHash) { _extraHash = extraHash; } + signals: /// Fired when the resource begins downloading. void loading(); @@ -469,7 +472,7 @@ protected: virtual bool handleFailedRequest(ResourceRequest::Result result); QUrl _url; - QUrl _effectiveBaseURL{ _url }; + QUrl _effectiveBaseURL { _url }; QUrl _activeUrl; ByteRange _requestByteRange; bool _shouldFailOnRedirect { false }; @@ -492,6 +495,8 @@ protected: int _requestID; ResourceRequest* _request{ nullptr }; + int _extraHash { -1 }; + public slots: void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal); void handleReplyFinished(); diff --git a/libraries/recording/src/recording/ClipCache.cpp b/libraries/recording/src/recording/ClipCache.cpp index c63350de7f..c08dd40ad8 100644 --- a/libraries/recording/src/recording/ClipCache.cpp +++ b/libraries/recording/src/recording/ClipCache.cpp @@ -48,8 +48,11 @@ NetworkClipLoaderPointer ClipCache::getClipLoader(const QUrl& url) { return getResource(url).staticCast(); } -QSharedPointer ClipCache::createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) { +QSharedPointer ClipCache::createResource(const QUrl& url) { qCDebug(recordingLog) << "Loading recording at" << url; return QSharedPointer(new NetworkClipLoader(url), &Resource::deleter); } +QSharedPointer ClipCache::createResourceCopy(const QSharedPointer& resource) { + return QSharedPointer(new NetworkClipLoader(*resource.staticCast().data()), &Resource::deleter); +} \ No newline at end of file diff --git a/libraries/recording/src/recording/ClipCache.h b/libraries/recording/src/recording/ClipCache.h index 2c3465e725..202cd2f00e 100644 --- a/libraries/recording/src/recording/ClipCache.h +++ b/libraries/recording/src/recording/ClipCache.h @@ -33,6 +33,8 @@ class NetworkClipLoader : public Resource { Q_OBJECT public: NetworkClipLoader(const QUrl& url); + NetworkClipLoader(const NetworkClipLoader& other) : Resource(other), _clip(other._clip) {} + virtual void downloadFinished(const QByteArray& data) override; ClipPointer getClip() { return _clip; } bool completed() { return _failedToLoad || isLoaded(); } @@ -54,7 +56,8 @@ public slots: NetworkClipLoaderPointer getClipLoader(const QUrl& url); protected: - virtual QSharedPointer createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) override; + virtual QSharedPointer createResource(const QUrl& url) override; + QSharedPointer createResourceCopy(const QSharedPointer& resource) override; private: ClipCache(QObject* parent = nullptr); diff --git a/libraries/render-utils/src/RenderPipelines.cpp b/libraries/render-utils/src/RenderPipelines.cpp index 85bdf0fadc..5f3763ac2a 100644 --- a/libraries/render-utils/src/RenderPipelines.cpp +++ b/libraries/render-utils/src/RenderPipelines.cpp @@ -579,7 +579,7 @@ void RenderPipelines::updateMultiMaterial(graphics::MultiMaterial& multiMaterial } else { forceDefault = true; } - schemaKey.setScattering(true); + schemaKey.setScatteringMap(true); } break; case graphics::MaterialKey::EMISSIVE_MAP_BIT: From 91937041731f2908df99afffccace23db729189b Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Tue, 5 Feb 2019 18:27:27 -0800 Subject: [PATCH 02/11] trying to get hashes working --- .../src/model-networking/MaterialCache.cpp | 2 +- .../src/model-networking/ModelCache.cpp | 50 +++++++++++++++++-- .../src/model-networking/ShaderCache.cpp | 2 +- .../src/model-networking/TextureCache.cpp | 22 +++++++- libraries/networking/src/ResourceCache.cpp | 24 ++++----- libraries/networking/src/ResourceCache.h | 19 +++---- 6 files changed, 89 insertions(+), 30 deletions(-) diff --git a/libraries/model-networking/src/model-networking/MaterialCache.cpp b/libraries/model-networking/src/model-networking/MaterialCache.cpp index 7dcd7b8a61..aaa9767397 100644 --- a/libraries/model-networking/src/model-networking/MaterialCache.cpp +++ b/libraries/model-networking/src/model-networking/MaterialCache.cpp @@ -417,7 +417,7 @@ MaterialCache& MaterialCache::instance() { } NetworkMaterialResourcePointer MaterialCache::getMaterial(const QUrl& url) { - return ResourceCache::getResource(url, QUrl()).staticCast(); + return ResourceCache::getResource(url).staticCast(); } QSharedPointer MaterialCache::createResource(const QUrl& url) { diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 7515dad256..1535f5cfad 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -40,6 +40,50 @@ public: bool combineParts; }; +// From: https://stackoverflow.com/questions/41145012/how-to-hash-qvariant +class QVariantHasher { +public: + QVariantHasher() : buff(&bb), ds(&buff) { + bb.reserve(1000); + buff.open(QIODevice::WriteOnly); + } + uint hash(const QVariant& v) { + buff.seek(0); + ds << v; + return qHashBits(bb.constData(), buff.pos()); + } +private: + QByteArray bb; + QBuffer buff; + QDataStream ds; +}; + +namespace std { + template <> + struct hash { + size_t operator()(const QVariantHash& a) const { + QVariantHasher hasher; + return hasher.hash(a); + } + }; + + template <> + struct hash { + size_t operator()(const QUrl& a) const { + return qHash(a); + } + }; + + template <> + struct hash { + size_t operator()(const GeometryExtra& a) const { + size_t result = 0; + hash_combine(result, a.mapping, a.textureBaseUrl, a.combineParts); + return result; + } + }; +} + QUrl resolveTextureBaseUrl(const QUrl& url, const QUrl& textureBaseUrl) { return textureBaseUrl.isValid() ? textureBaseUrl : url; } @@ -110,7 +154,7 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) { GeometryExtra extra { _mapping, _textureBaseUrl, false }; // Get the raw GeometryResource - _geometryResource = modelCache->getResource(url, QUrl(), &extra).staticCast(); + _geometryResource = modelCache->getResource(url, QUrl(), &extra, std::hash()(extra)).staticCast(); // Avoid caching nested resources - their references will be held by the parent _geometryResource->_isCacheable = false; @@ -355,7 +399,7 @@ GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl) { bool combineParts = true; GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts }; - GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast(); + GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash()(geometryExtra)).staticCast(); if (resource) { if (resource->isLoaded() && resource->shouldSetTextures()) { resource->setTextures(); @@ -368,7 +412,7 @@ GeometryResource::Pointer ModelCache::getCollisionGeometryResource(const QUrl& u const QVariantHash& mapping, const QUrl& textureBaseUrl) { bool combineParts = false; GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts }; - GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast(); + GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash()(geometryExtra)).staticCast(); if (resource) { if (resource->isLoaded() && resource->shouldSetTextures()) { resource->setTextures(); diff --git a/libraries/model-networking/src/model-networking/ShaderCache.cpp b/libraries/model-networking/src/model-networking/ShaderCache.cpp index b774ca36c5..8d060c42f2 100644 --- a/libraries/model-networking/src/model-networking/ShaderCache.cpp +++ b/libraries/model-networking/src/model-networking/ShaderCache.cpp @@ -21,7 +21,7 @@ ShaderCache& ShaderCache::instance() { } NetworkShaderPointer ShaderCache::getShader(const QUrl& url) { - return ResourceCache::getResource(url, QUrl()).staticCast(); + return ResourceCache::getResource(url).staticCast(); } QSharedPointer ShaderCache::createResource(const QUrl& url) { diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 2f29eaf385..910de258f9 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -194,10 +194,28 @@ public: int maxNumPixels; }; +namespace std { + template <> + struct hash { + size_t operator()(const QByteArray& a) const { + return qHash(a); + } + }; + + template <> + struct hash { + size_t operator()(const TextureExtra& a) const { + size_t result = 0; + hash_combine(result, (int)a.type, a.content, a.maxNumPixels); + return result; + } + }; +} + ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxNumPixels) { auto byteArray = QByteArray(); TextureExtra extra = { (image::TextureUsage::Type)type, byteArray, maxNumPixels }; - return ResourceCache::prefetch(url, &extra); + return ResourceCache::prefetch(url, &extra, std::hash()(extra)); } NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) { @@ -211,7 +229,7 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs modifiedUrl.setQuery(query.toString()); } TextureExtra extra = { type, content, maxNumPixels }; - return ResourceCache::getResource(modifiedUrl, QUrl(), &extra).staticCast(); + return ResourceCache::getResource(modifiedUrl, QUrl(), &extra, std::hash()(extra)).staticCast(); } gpu::TexturePointer TextureCache::getTextureByHash(const std::string& hash) { diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 43fc93ffc5..cb7b8c7c82 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -158,8 +158,8 @@ void ScriptableResourceCache::updateTotalSize(const qint64& deltaSize) { _resourceCache->updateTotalSize(deltaSize); } -ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra) { - return _resourceCache->prefetch(url, extra); +ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) { + return _resourceCache->prefetch(url, extra, extraHash); } @@ -211,20 +211,20 @@ void ScriptableResource::disconnectHelper() { } } -ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) { +ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) { ScriptableResource* result = nullptr; if (QThread::currentThread() != thread()) { // Must be called in thread to ensure getResource returns a valid pointer BLOCKING_INVOKE_METHOD(this, "prefetch", Q_RETURN_ARG(ScriptableResource*, result), - Q_ARG(QUrl, url), Q_ARG(void*, extra)); + Q_ARG(QUrl, url), Q_ARG(void*, extra), Q_ARG(size_t, extraHash)); return result; } result = new ScriptableResource(url); - auto resource = getResource(url, QUrl(), extra); + auto resource = getResource(url, QUrl(), extra, extraHash); result->_resource = resource; result->setObjectName(url.toString()); @@ -298,7 +298,7 @@ void ResourceCache::refreshAll() { clearUnusedResources(); resetUnusedResourceCounter(); - QHash>> allResources; + QHash>> allResources; { QReadLocker locker(&_resourcesLock); allResources = _resources; @@ -343,13 +343,8 @@ void ResourceCache::setRequestLimit(uint32_t limit) { } } -QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, int extraHash) { +QSharedPointer ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash) { QSharedPointer resource; - if (extra && extraHash < 0) { - qDebug() << "ResourceCache::getResource: ERROR! Non-null extra, but invalid extraHash"; - return resource; - } - { QReadLocker locker(&_resourcesLock); auto& resourcesWithExtraHash = _resources[url]; @@ -553,8 +548,8 @@ bool ResourceCache::attemptHighestPriorityRequest() { static int requestID = 0; Resource::Resource(const Resource& other) : + QObject(), _url(other._url), - _extraHash(other._extraHash), _effectiveBaseURL(other._effectiveBaseURL), _activeUrl(other._activeUrl), _requestByteRange(other._requestByteRange), @@ -566,7 +561,8 @@ Resource::Resource(const Resource& other) : _bytesReceived(other._bytesReceived), _bytesTotal(other._bytesTotal), _bytes(other._bytes), - _requestID(++requestID) { + _requestID(++requestID), + _extraHash(other._extraHash) { if (!other._loaded) { _startedLoading = false; } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index dd60e3c110..740bdadc48 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -231,15 +231,16 @@ protected slots: // Prefetches a resource to be held by the QScriptEngine. // Left as a protected member so subclasses can overload prefetch // and delegate to it (see TextureCache::prefetch(const QUrl&, int). - ScriptableResource* prefetch(const QUrl& url, void* extra); + ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash); // FIXME: The return type is not recognized by JavaScript. /// Loads a resource from the specified URL and returns it. /// If the caller is on a different thread than the ResourceCache, /// returns an empty smart pointer and loads its asynchronously. /// \param fallback a fallback URL to load if the desired one is unavailable - /// \param extra extra data to pass to the creator, if appropriate - QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl(), void* extra = NULL, int extraHash = -1); + // FIXME: std::numeric_limits::max() could be a valid extraHash + QSharedPointer getResource(const QUrl& url, const QUrl& fallback = QUrl()) { return getResource(url, fallback, nullptr, std::numeric_limits::max()); } + QSharedPointer getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash); private slots: void clearATPAssets(); @@ -250,7 +251,7 @@ protected: // which should be a QScriptEngine with ScriptableResource registered, so that // the QScriptEngine will delete the pointer when it is garbage collected. // JSDoc is provided on more general function signature. - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); } + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits::max()); } /// Creates a new resource. virtual QSharedPointer createResource(const QUrl& url) = 0; @@ -277,7 +278,7 @@ private: void resetResourceCounters(); // Resources - QHash>> _resources; + QHash>> _resources; QReadWriteLock _resourcesLock { QReadWriteLock::Recursive }; int _lastLRUKey = 0; @@ -331,10 +332,10 @@ public: * Prefetches a resource. * @function ResourceCache.prefetch * @param {string} url - URL of the resource to prefetch. - * @param {object} [extra=null] * @returns {ResourceObject} */ - Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra = nullptr); + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits::max()); } + Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash); signals: @@ -416,7 +417,7 @@ public: unsigned int getDownloadAttemptsRemaining() { return _attemptsRemaining; } virtual void setExtra(void* extra) {}; - void setExtraHash(int extraHash) { _extraHash = extraHash; } + void setExtraHash(size_t extraHash) { _extraHash = extraHash; } signals: /// Fired when the resource begins downloading. @@ -495,7 +496,7 @@ protected: int _requestID; ResourceRequest* _request{ nullptr }; - int _extraHash { -1 }; + size_t _extraHash; public slots: void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal); From 446176d3a516d81dc100789c04cccc021c2d6aa1 Mon Sep 17 00:00:00 2001 From: raveenajain Date: Thu, 7 Feb 2019 15:08:04 -0800 Subject: [PATCH 03/11] update to parse glb files --- libraries/fbx/src/GLTFSerializer.cpp | 67 +++++++++++++++++++++++++--- libraries/fbx/src/GLTFSerializer.h | 3 ++ 2 files changed, 64 insertions(+), 6 deletions(-) mode change 100644 => 100755 libraries/fbx/src/GLTFSerializer.cpp mode change 100644 => 100755 libraries/fbx/src/GLTFSerializer.h diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp old mode 100644 new mode 100755 index 96c236f703..f2ea16aa31 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -124,6 +124,31 @@ bool GLTFSerializer::getObjectArrayVal(const QJsonObject& object, const QString& return _defined; } +QByteArray GLTFSerializer::setGLBChunks(const QByteArray& data) { + int byte = 4; + int jsonStart = data.indexOf("JSON", Qt::CaseSensitive); + int binStart = data.indexOf("BIN", Qt::CaseSensitive); + int jsonLength, binLength; + QByteArray jsonLengthChunk, binLengthChunk; + + jsonLengthChunk = data.mid(jsonStart - byte, byte); + QDataStream tempJsonLen(jsonLengthChunk); + tempJsonLen.setByteOrder(QDataStream::LittleEndian); + tempJsonLen >> jsonLength; + QByteArray jsonChunk = data.mid(jsonStart + byte, jsonLength); + + if (binStart != -1) { + binLengthChunk = data.mid(binStart - byte, byte); + + QDataStream tempBinLen(binLengthChunk); + tempBinLen.setByteOrder(QDataStream::LittleEndian); + tempBinLen >> binLength; + + _glbBinary = data.mid(binStart + byte, binLength); + } + return jsonChunk; +} + int GLTFSerializer::getMeshPrimitiveRenderingMode(const QString& type) { if (type == "POINTS") { @@ -309,6 +334,14 @@ bool GLTFSerializer::addBuffer(const QJsonObject& object) { GLTFBuffer buffer; getIntVal(object, "byteLength", buffer.byteLength, buffer.defined); + + if (_url.toString().endsWith("glb")) { + if (!_glbBinary.isEmpty()) { + buffer.blob = _glbBinary; + } else { + return false; + } + } if (getStringVal(object, "uri", buffer.uri, buffer.defined)) { if (!readBinary(buffer.uri, buffer.blob)) { return false; @@ -530,9 +563,16 @@ bool GLTFSerializer::addTexture(const QJsonObject& object) { bool GLTFSerializer::parseGLTF(const QByteArray& data) { PROFILE_RANGE_EX(resource_parse, __FUNCTION__, 0xffff0000, nullptr); - - QJsonDocument d = QJsonDocument::fromJson(data); + + QByteArray jsonChunk = data; + + if (_url.toString().endsWith("glb") && data.indexOf("glTF") == 0 && data.contains("JSON")) { + jsonChunk = setGLBChunks(data); + } + + QJsonDocument d = QJsonDocument::fromJson(jsonChunk); QJsonObject jsFile = d.object(); + bool success = setAsset(jsFile); if (success) { QJsonArray accessors; @@ -904,6 +944,10 @@ MediaType GLTFSerializer::getMediaType() const { MediaType mediaType("gltf"); mediaType.extensions.push_back("gltf"); mediaType.webMediaTypes.push_back("model/gltf+json"); + + mediaType.extensions.push_back("glb"); + mediaType.webMediaTypes.push_back("model/gltf-binary"); + return mediaType; } @@ -912,9 +956,9 @@ std::unique_ptr GLTFSerializer::getFactory() const { } HFMModel::Pointer GLTFSerializer::read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url) { - - _url = url; + _url = url; + // Normalize url for local files QUrl normalizeUrl = DependencyManager::get()->normalizeURL(_url); if (normalizeUrl.scheme().isEmpty() || (normalizeUrl.scheme() == "file")) { @@ -943,7 +987,7 @@ bool GLTFSerializer::readBinary(const QString& url, QByteArray& outdata) { bool success; std::tie(success, outdata) = requestData(binaryUrl); - + return success; } @@ -1003,7 +1047,7 @@ QNetworkReply* GLTFSerializer::request(QUrl& url, bool isTest) { HFMTexture GLTFSerializer::getHFMTexture(const GLTFTexture& texture) { HFMTexture fbxtex = HFMTexture(); fbxtex.texcoordSet = 0; - + if (texture.defined["source"]) { QString url = _file.images[texture.source].uri; QString fname = QUrl(url).fileName(); @@ -1011,6 +1055,17 @@ HFMTexture GLTFSerializer::getHFMTexture(const GLTFTexture& texture) { qCDebug(modelformat) << "fname: " << fname; fbxtex.name = fname; fbxtex.filename = textureUrl.toEncoded(); + + if (_url.toString().endsWith("glb") && !_glbBinary.isEmpty()) { + int bufferView = _file.images[texture.source].bufferView; + + GLTFBufferView& imagesBufferview = _file.bufferviews[bufferView]; + int offset = imagesBufferview.byteOffset; + int length = imagesBufferview.byteLength; + + fbxtex.content = _glbBinary.mid(offset, length); + fbxtex.filename = textureUrl.toEncoded().append(texture.source); + } } return fbxtex; } diff --git a/libraries/fbx/src/GLTFSerializer.h b/libraries/fbx/src/GLTFSerializer.h old mode 100644 new mode 100755 index 5fca77c4fd..fe1dea49e0 --- a/libraries/fbx/src/GLTFSerializer.h +++ b/libraries/fbx/src/GLTFSerializer.h @@ -709,6 +709,7 @@ public: private: GLTFFile _file; QUrl _url; + QByteArray _glbBinary; glm::mat4 getModelTransform(const GLTFNode& node); @@ -731,6 +732,8 @@ private: QVector& values, QMap& defined); bool getObjectArrayVal(const QJsonObject& object, const QString& fieldname, QJsonArray& objects, QMap& defined); + + QByteArray setGLBChunks(const QByteArray& data); int getMaterialAlphaMode(const QString& type); int getAccessorType(const QString& type); From 79bf26ee27332aa8a94af93b9fc2ff39a9be423d Mon Sep 17 00:00:00 2001 From: Matt Hardcastle Date: Thu, 7 Feb 2019 15:57:14 -0800 Subject: [PATCH 04/11] Add prebuild metrics logging for CI system Metric about the times various parts of the build take are great to have. This change takes us a step in that direction by adding metrics to the prebuild step of the build process. These metrics are off by default; use the `--ci-build` option of `prebuild.py` to enable them. --- prebuild.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/prebuild.py b/prebuild.py index fb54b8d6fe..060e1fd3b0 100644 --- a/prebuild.py +++ b/prebuild.py @@ -35,9 +35,50 @@ import re import tempfile import time import functools +import subprocess +import logging + +from uuid import uuid4 +from contextlib import contextmanager print = functools.partial(print, flush=True) +class TrackableLogger(logging.Logger): + guid = str(uuid4()) + + def _log(self, msg, *args, **kwargs): + x = {'guid': self.guid} + if 'extra' in kwargs: + kwargs['extra'].update(x) + else: + kwargs['extra'] = x + super()._log(msg, *args, **kwargs) + +logging.setLoggerClass(TrackableLogger) +logger = logging.getLogger('prebuild') + +def headSha(): + repo_dir = os.path.dirname(os.path.abspath(__file__)) + git = subprocess.Popen( + 'git rev-parse --short HEAD', + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, cwd=repo_dir, universal_newlines=True, + ) + stdout, _ = git.communicate() + sha = stdout.split('\n')[0] + if not sha: + raise RuntimeError("couldn't find git sha") + return sha + +@contextmanager +def timer(name): + ''' Print the elapsed time a context's execution takes to execute ''' + start = time.time() + yield + # Please take care when modifiying this print statement. + # Log parsing logic may depend on it. + logger.info('%s took %.3f secs' % (name, time.time() - start)) + def parse_args(): # our custom ports, relative to the script location defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports') @@ -50,6 +91,7 @@ def parse_args(): parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') parser.add_argument('--ports-path', type=str, default=defaultPortsPath) + parser.add_argument('--ci-build', action='store_true') if True: args = parser.parse_args() else: @@ -66,11 +108,19 @@ def main(): del os.environ[var] args = parse_args() + + if args.ci_build: + logging.basicConfig(datefmt='%s', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO) + + logger.info('sha=%s' % headSha()) + logger.info('start') + # Only allow one instance of the program to run at a time pm = hifi_vcpkg.VcpkgRepo(args) with hifi_singleton.Singleton(pm.lockFile) as lock: - if not pm.upToDate(): - pm.bootstrap() + with timer('Bootstraping'): + if not pm.upToDate(): + pm.bootstrap() # Always write the tag, even if we changed nothing. This # allows vcpkg to reclaim disk space by identifying directories with @@ -80,11 +130,13 @@ def main(): # Grab our required dependencies: # * build host tools, like spirv-cross and scribe # * build client dependencies like openssl and nvtt - pm.setupDependencies() + with timer('Setting up dependencies'): + pm.setupDependencies() # wipe out the build directories (after writing the tag, since failure # here shouldn't invalidte the vcpkg install) - pm.cleanBuilds() + with timer('Cleaning builds'): + pm.cleanBuilds() # If we're running in android mode, we also need to grab a bunch of additional binaries # (this logic is all migrated from the old setupDependencies tasks in gradle) @@ -98,7 +150,10 @@ def main(): hifi_android.QtPackager(appPath, qtPath).bundle() # Write the vcpkg config to the build directory last - pm.writeConfig() + with timer('Writing configuration'): + pm.writeConfig() + + logger.info('end') print(sys.argv) main() From 709f784feacdf5d8d8abbce276b6c9f519d94ddc Mon Sep 17 00:00:00 2001 From: raveenajain Date: Fri, 8 Feb 2019 16:51:18 -0800 Subject: [PATCH 05/11] :} --- libraries/fbx/src/GLTFSerializer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index 9a23a9428f..d6de3d4b25 100755 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -1095,6 +1095,7 @@ HFMTexture GLTFSerializer::getHFMTexture(const GLTFTexture& texture) { fbxtex.content = _glbBinary.mid(offset, length); fbxtex.filename = textureUrl.toEncoded().append(texture.source); + } if (url.contains("data:image/jpeg;base64,") || url.contains("data:image/png;base64,")) { fbxtex.content = requestEmbeddedData(url); From dff98f462f621493fa31c61df033cdb13b3fe79c Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sat, 9 Feb 2019 13:13:13 -0800 Subject: [PATCH 06/11] Removed unnecessary of spaces. --- tools/nitpick/ui/Nitpick.ui | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/nitpick/ui/Nitpick.ui b/tools/nitpick/ui/Nitpick.ui index 319452233f..16aaa9594d 100644 --- a/tools/nitpick/ui/Nitpick.ui +++ b/tools/nitpick/ui/Nitpick.ui @@ -43,7 +43,7 @@ - 3 + 0 @@ -85,7 +85,7 @@ - Create all MD files + Create all MD files @@ -124,7 +124,7 @@ - Create all Recursive Scripts + Create all Recursive Scripts From b2fb7a737be139b4a53f586d6e2550b8187c03bf Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sat, 9 Feb 2019 13:21:36 -0800 Subject: [PATCH 07/11] Ready for testing. --- tools/nitpick/src/Nitpick.cpp | 2 +- tools/nitpick/src/Test.cpp | 154 +++++++++++++--------------------- tools/nitpick/src/Test.h | 9 +- 3 files changed, 65 insertions(+), 100 deletions(-) diff --git a/tools/nitpick/src/Nitpick.cpp b/tools/nitpick/src/Nitpick.cpp index 78ed0ca0af..fa53730ce0 100644 --- a/tools/nitpick/src/Nitpick.cpp +++ b/tools/nitpick/src/Nitpick.cpp @@ -40,7 +40,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Nitpick - v2.0.1"); + setWindowTitle("Nitpick - v2.1.0"); } Nitpick::~Nitpick() { diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp index 2e62296146..59583cda8b 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/Test.cpp @@ -758,47 +758,66 @@ void Test::createAllRecursiveScripts() { return; } + createAllRecursiveScripts(_testsRootDirectory); createRecursiveScript(_testsRootDirectory, false); - - QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - // Only process directories - QDir dir; - if (!isAValidDirectory(directory)) { - continue; - } - - // Only process directories that have sub-directories - bool hasNoSubDirectories{ true }; - QDirIterator it2(directory, QDirIterator::Subdirectories); - while (it2.hasNext()) { - QString directory2 = it2.next(); - - // Only process directories - QDir dir; - if (isAValidDirectory(directory2)) { - hasNoSubDirectories = false; - break; - } - } - - if (!hasNoSubDirectories) { - createRecursiveScript(directory, false); - } - } - QMessageBox::information(0, "Success", "Scripts have been created"); } -void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) { - const QString recursiveTestsScriptName("testRecursive.js"); - const QString recursiveTestsFilename(topLevelDirectory + "/" + recursiveTestsScriptName); +void Test::createAllRecursiveScripts(const QString& directory) { + QDirIterator it(directory, QDirIterator::Subdirectories); + + while (it.hasNext()) { + QString nextDirectory = it.next(); + if (isAValidDirectory(nextDirectory)) { + createAllRecursiveScripts(nextDirectory); + createRecursiveScript(nextDirectory, false); + } + } +} + +void Test::createRecursiveScript(const QString& directory, bool interactiveMode) { + // If folder contains a test, then we are at a leaf + const QString testPathname{ directory + "/" + TEST_FILENAME }; + if (QFileInfo(testPathname).exists()) { + return; + } + + // Directories are included in reverse order. The nitpick scripts use a stack mechanism, + // so this ensures that the tests run in alphabetical order (a convenience when debugging) + QStringList directories; + QDirIterator it(directory); + while (it.hasNext()) { + QString subDirectory = it.next(); + + // Only process directories + if (!isAValidDirectory(subDirectory)) { + continue; + } + + const QString testPathname{ subDirectory + "/" + TEST_FILENAME }; + if (QFileInfo(testPathname).exists()) { + // Current folder contains a test script + directories.push_front(testPathname); + } + + const QString testRecursivePathname{ subDirectory + "/" + TEST_RECURSIVE_FILENAME }; + if (QFileInfo(testRecursivePathname).exists()) { + // Current folder contains a recursive script + directories.push_front(testRecursivePathname); + } + } + + // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant + if (directories.length() == 0) { + return; + } + + // Open the recursive script file + const QString recursiveTestsFilename(directory + "/" + TEST_RECURSIVE_FILENAME); QFile recursiveTestsFile(recursiveTestsFilename); if (!recursiveTestsFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create \"" + recursiveTestsScriptName + "\" in directory \"" + topLevelDirectory + "\""); + "Failed to create \"" + TEST_RECURSIVE_FILENAME + "\" in directory \"" + directory + "\""); exit(-1); } @@ -812,72 +831,16 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact QString user = nitpick->getSelectedUser(); textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch + - "/tests/utils/branchUtils.js\";" - << endl; - textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl; - textStream << "var nitpick = createNitpick(Script.resolvePath(\".\"));" << endl << endl; + "/tests/utils/branchUtils.js\";" + << endl; + textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl << endl; - textStream << "var testsRootPath = nitpick.getTestsRootPath();" << endl << endl; - - // Wait 10 seconds before starting - textStream << "if (typeof Test !== 'undefined') {" << endl; - textStream << " Test.wait(10000);" << endl; - textStream << "};" << endl << endl; + textStream << "if (typeof nitpick === 'undefined') var nitpick = createNitpick(Script.resolvePath(\".\"));" << endl; + textStream << "if (typeof testsRootPath === 'undefined') var testsRootPath = nitpick.getTestsRootPath();" << endl << endl; textStream << "nitpick.enableRecursive();" << endl; textStream << "nitpick.enableAuto();" << endl << endl; - // This is used to verify that the recursive test contains at least one test - bool testFound{ false }; - - // Directories are included in reverse order. The nitpick scripts use a stack mechanism, - // so this ensures that the tests run in alphabetical order (a convenience when debugging) - QStringList directories; - - // First test if top-level folder has a test.js file - const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - // Current folder contains a test - directories.push_front(testPathname); - - testFound = true; - } - - QDirIterator it(topLevelDirectory, QDirIterator::Subdirectories); - while (it.hasNext()) { - QString directory = it.next(); - - // Only process directories - QDir dir(directory); - if (!isAValidDirectory(directory)) { - continue; - } - - const QString testPathname{ directory + "/" + TEST_FILENAME }; - QFileInfo fileInfo(testPathname); - if (fileInfo.exists()) { - // Current folder contains a test - directories.push_front(testPathname); - - testFound = true; - } - } - - if (interactiveMode && !testFound) { - QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); - recursiveTestsFile.close(); - return; - } - - // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant - // The script will be closed and deleted - if (directories.length() == 0) { - recursiveTestsFile.close(); - QFile::remove(recursiveTestsFilename); - return; - } - // Now include the test scripts for (int i = 0; i < directories.length(); ++i) { includeTest(textStream, directories.at(i)); @@ -928,7 +891,6 @@ void Test::createTestsOutline() { QString directory = it.next(); // Only process directories - QDir dir; if (!isAValidDirectory(directory)) { continue; } diff --git a/tools/nitpick/src/Test.h b/tools/nitpick/src/Test.h index aafd2f5711..842e4bdb48 100644 --- a/tools/nitpick/src/Test.h +++ b/tools/nitpick/src/Test.h @@ -72,9 +72,11 @@ public: void updateTestRailRunResult(); - void createRecursiveScript(); void createAllRecursiveScripts(); - void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); + void createAllRecursiveScripts(const QString& directory); + + void createRecursiveScript(); + void createRecursiveScript(const QString& directory, bool interactiveMode); int compareImageLists(); int checkTextResults(); @@ -109,7 +111,8 @@ private: bool _isRunningFromCommandLine{ false }; bool _isRunningInAutomaticTestRun{ false }; - const QString TEST_FILENAME { "test.js" }; + const QString TEST_FILENAME{ "test.js" }; + const QString TEST_RECURSIVE_FILENAME{ "testRecursive.js" }; const QString TEST_RESULTS_FOLDER { "TestResults" }; const QString TEST_RESULTS_FILENAME { "TestResults.txt" }; From 20841a0eea1237dc8278f506ec18150e8245547a Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Sat, 9 Feb 2019 16:42:34 -0800 Subject: [PATCH 08/11] fix skybox loading I think and pull in sabrina's ktx caching fix --- .../src/model-networking/TextureCache.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 910de258f9..c7235337c2 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -335,12 +335,8 @@ int networkTexturePointerMetaTypeId = qRegisterMetaType(url); - _lowestRequestedMipLevel = 0; - _loaded = true; } NetworkTexture::NetworkTexture(const NetworkTexture& other) : @@ -425,7 +421,7 @@ gpu::TexturePointer NetworkTexture::getFallbackTexture() const { class ImageReader : public QRunnable { public: ImageReader(const QWeakPointer& resource, const QUrl& url, - const QByteArray& data, int maxNumPixels); + const QByteArray& data, size_t extraHash, int maxNumPixels); void run() override final; void read(); @@ -435,6 +431,7 @@ private: QWeakPointer _resource; QUrl _url; QByteArray _content; + size_t _extraHash; int _maxNumPixels; }; @@ -1068,7 +1065,7 @@ void NetworkTexture::loadTextureContent(const QByteArray& content) { return; } - QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _extraHash, _maxNumPixels)); } void NetworkTexture::refresh() { @@ -1093,10 +1090,11 @@ void NetworkTexture::refresh() { Resource::refresh(); } -ImageReader::ImageReader(const QWeakPointer& resource, const QUrl& url, const QByteArray& data, int maxNumPixels) : +ImageReader::ImageReader(const QWeakPointer& resource, const QUrl& url, const QByteArray& data, size_t extraHash, int maxNumPixels) : _resource(resource), _url(url), _content(data), + _extraHash(extraHash), _maxNumPixels(maxNumPixels) { DependencyManager::get()->incrementStat("PendingProcessing"); @@ -1152,11 +1150,12 @@ void ImageReader::read() { } auto networkTexture = resource.staticCast(); - // Hash the source image to for KTX caching + // Hash the source image and extraHash for KTX caching std::string hash; { QCryptographicHash hasher(QCryptographicHash::Md5); hasher.addData(_content); + hasher.addData(std::to_string(_extraHash).c_str()); hash = hasher.result().toHex().toStdString(); } From d3e0aa5d8cd2cb69afb1afadadb17b29a9187a5b Mon Sep 17 00:00:00 2001 From: NissimHadar Date: Sun, 10 Feb 2019 20:32:38 -0800 Subject: [PATCH 09/11] Make sure only top-level recursive script runs recursively. --- tools/nitpick/src/Test.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp index 59583cda8b..20d5ad89a6 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/Test.cpp @@ -835,8 +835,8 @@ void Test::createRecursiveScript(const QString& directory, bool interactiveMode) << endl; textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl << endl; - textStream << "if (typeof nitpick === 'undefined') var nitpick = createNitpick(Script.resolvePath(\".\"));" << endl; - textStream << "if (typeof testsRootPath === 'undefined') var testsRootPath = nitpick.getTestsRootPath();" << endl << endl; + textStream << "if (typeof nitpick === 'undefined') nitpick = createNitpick(Script.resolvePath(\".\"));" << endl; + textStream << "if (typeof testsRootPath === 'undefined') testsRootPath = nitpick.getTestsRootPath();" << endl << endl; textStream << "nitpick.enableRecursive();" << endl; textStream << "nitpick.enableAuto();" << endl << endl; @@ -847,7 +847,10 @@ void Test::createRecursiveScript(const QString& directory, bool interactiveMode) } textStream << endl; - textStream << "nitpick.runRecursive();" << endl; + textStream << "if (typeof runningRecursive === 'undefined') {" << endl; + textStream << " runningRecursive = true;" << endl; + textStream << " nitpick.runRecursive();" << endl; + textStream << "}" << endl << endl; recursiveTestsFile.close(); } From 16b48046c64c60466f116d5d8976b7df34db2321 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 11 Feb 2019 15:27:30 -0800 Subject: [PATCH 10/11] Fix not being able to type text into particle url field --- scripts/system/html/js/entityProperties.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/system/html/js/entityProperties.js b/scripts/system/html/js/entityProperties.js index ee95312fa4..14a0031f1f 100644 --- a/scripts/system/html/js/entityProperties.js +++ b/scripts/system/html/js/entityProperties.js @@ -2328,7 +2328,7 @@ function createTextureProperty(property, elProperty) { elInput.setAttribute("id", elementID); elInput.setAttribute("type", "text"); - let imageLoad = _.debounce(function (url) { + let imageLoad = function(url) { if (url.slice(0, 5).toLowerCase() === "atp:/") { elImage.src = ""; elImage.style.display = "none"; @@ -2348,15 +2348,12 @@ function createTextureProperty(property, elProperty) { elDiv.classList.remove("no-preview"); elDiv.classList.add("no-texture"); } - }, IMAGE_DEBOUNCE_TIMEOUT); - elInput.imageLoad = imageLoad; - elInput.oninput = function (event) { - // Add throttle - let url = event.target.value; - imageLoad(url); - updateProperty(property.name, url, property.isParticleProperty) }; - elInput.onchange = elInput.oninput; + elInput.imageLoad = imageLoad; + elInput.addEventListener('change', createEmitTextPropertyUpdateFunction(property)); + elInput.addEventListener('change', function(ev) { + imageLoad(ev.target.value); + }); elProperty.appendChild(elInput); elProperty.appendChild(elDiv); From d96b0534ab539aab1be085f2edf8c06d6b0e3480 Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Mon, 11 Feb 2019 16:12:13 -0800 Subject: [PATCH 11/11] fix resource texture crash --- .../src/model-networking/TextureCache.cpp | 12 ++++++++---- .../src/model-networking/TextureCache.h | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index c7235337c2..d4cf7e6ce9 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -333,10 +333,14 @@ QSharedPointer TextureCache::createResourceCopy(const QSharedPointer>(); -NetworkTexture::NetworkTexture(const QUrl& url) : +NetworkTexture::NetworkTexture(const QUrl& url, bool resourceTexture) : Resource(url), _maxNumPixels(100) { + if (resourceTexture) { + _textureSource = std::make_shared(url); + _loaded = true; + } } NetworkTexture::NetworkTexture(const NetworkTexture& other) : @@ -1244,11 +1248,11 @@ void ImageReader::read() { Q_ARG(int, texture->getHeight())); } -NetworkTexturePointer TextureCache::getResourceTexture(QUrl resourceTextureUrl) { +NetworkTexturePointer TextureCache::getResourceTexture(const QUrl& resourceTextureUrl) { gpu::TexturePointer texture; if (resourceTextureUrl == SPECTATOR_CAMERA_FRAME_URL) { if (!_spectatorCameraNetworkTexture) { - _spectatorCameraNetworkTexture.reset(new NetworkTexture(resourceTextureUrl)); + _spectatorCameraNetworkTexture.reset(new NetworkTexture(resourceTextureUrl, true)); } if (!_spectatorCameraFramebuffer) { getSpectatorCameraFramebuffer(); // initialize frame buffer @@ -1259,7 +1263,7 @@ NetworkTexturePointer TextureCache::getResourceTexture(QUrl resourceTextureUrl) // FIXME: Generalize this, DRY up this code if (resourceTextureUrl == HMD_PREVIEW_FRAME_URL) { if (!_hmdPreviewNetworkTexture) { - _hmdPreviewNetworkTexture.reset(new NetworkTexture(resourceTextureUrl)); + _hmdPreviewNetworkTexture.reset(new NetworkTexture(resourceTextureUrl, true)); } if (_hmdPreviewFramebuffer) { texture = _hmdPreviewFramebuffer->getRenderBuffer(0); diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index d744d060b6..cdedc64ea5 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -45,7 +45,7 @@ class NetworkTexture : public Resource, public Texture { Q_OBJECT public: - NetworkTexture(const QUrl& url); + NetworkTexture(const QUrl& url, bool resourceTexture = false); NetworkTexture(const NetworkTexture& other); ~NetworkTexture() override; @@ -183,7 +183,7 @@ public: gpu::TexturePointer getTextureByHash(const std::string& hash); gpu::TexturePointer cacheTextureByHash(const std::string& hash, const gpu::TexturePointer& texture); - NetworkTexturePointer getResourceTexture(QUrl resourceTextureUrl); + NetworkTexturePointer getResourceTexture(const QUrl& resourceTextureUrl); const gpu::FramebufferPointer& getHmdPreviewFramebuffer(int width, int height); const gpu::FramebufferPointer& getSpectatorCameraFramebuffer(); const gpu::FramebufferPointer& getSpectatorCameraFramebuffer(int width, int height);