Merge branch 'master' of https://github.com/highfidelity/hifi into QmlMarketplace

libraries/animation/src/AnimationCache.cpp

@@ -36,12 +36,13 @@ AnimationPointer AnimationCache::getAnimation(const QUrl& url) {
     return getResource(url).staticCast<Animation>();
 }
 
-QSharedPointer<Resource> AnimationCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+QSharedPointer<Resource> AnimationCache::createResource(const QUrl& url) {
-    const void* extra) {
     return QSharedPointer<Resource>(new Animation(url), &Resource::deleter);
 }
 
-Animation::Animation(const QUrl& url) : Resource(url) {}
+QSharedPointer<Resource> AnimationCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new Animation(*resource.staticCast<Animation>().data()), &Resource::deleter);
+}
 
 AnimationReader::AnimationReader(const QUrl& url, const QByteArray& data) :
     _url(url),

libraries/animation/src/AnimationCache.h

@@ -34,9 +34,9 @@ public:
     Q_INVOKABLE AnimationPointer getAnimation(const QUrl& url);
 
 protected:
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
-        const void* extra) override;
 private:
     explicit AnimationCache(QObject* parent = NULL);
     virtual ~AnimationCache() { }
@@ -62,7 +62,8 @@ class Animation : public Resource {
 
 public:
 
-    explicit Animation(const QUrl& url);
+    Animation(const Animation& other) : Resource(other), _hfmModel(other._hfmModel) {}
+    Animation(const QUrl& url) : Resource(url) {}
 
     QString getType() const override { return "Animation"; }
 

libraries/audio/src/SoundCache.cpp

@@ -33,9 +33,12 @@ SharedSoundPointer SoundCache::getSound(const QUrl& url) {
     return getResource(url).staticCast<Sound>();
 }
 
-QSharedPointer<Resource> SoundCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+QSharedPointer<Resource> SoundCache::createResource(const QUrl& url) {
-    const void* extra) {
     auto resource = QSharedPointer<Resource>(new Sound(url), &Resource::deleter);
     resource->setLoadPriority(this, SOUNDS_LOADING_PRIORITY);
     return resource;
 }
+
+QSharedPointer<Resource> SoundCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new Sound(*resource.staticCast<Sound>().data()), &Resource::deleter);
+}

libraries/audio/src/SoundCache.h

@@ -24,8 +24,9 @@ public:
     Q_INVOKABLE SharedSoundPointer getSound(const QUrl& url);
 
 protected:
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
-        const void* extra) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
+
 private:
     SoundCache(QObject* parent = NULL);
 };

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -280,11 +280,7 @@ bool RenderableModelEntityItem::findDetailedParabolaIntersection(const glm::vec3
 }
 
 void RenderableModelEntityItem::fetchCollisionGeometryResource() {
-    QUrl hullURL(getCollisionShapeURL());
+    _compoundShapeResource = DependencyManager::get<ModelCache>()->getCollisionGeometryResource(getCollisionShapeURL());
-    QUrlQuery queryArgs(hullURL);
-    queryArgs.addQueryItem("collision-hull", "");
-    hullURL.setQuery(queryArgs);
-    _compoundShapeResource = DependencyManager::get<ModelCache>()->getCollisionGeometryResource(hullURL);
 }
 
 bool RenderableModelEntityItem::computeShapeFailedToLoad() {

libraries/fbx/src/FBXSerializer_Material.cpp

@@ -76,13 +76,12 @@ HFMTexture FBXSerializer::getTexture(const QString& textureID, const QString& ma
 }
 
 void FBXSerializer::consolidateHFMMaterials(const QVariantHash& mapping) {
-
+    QJsonObject materialMap;
-    QString materialMapString = mapping.value("materialMap").toString();
+    if (mapping.contains("materialMap")) {
-    QJsonDocument materialMapDocument = QJsonDocument::fromJson(materialMapString.toUtf8());
+        QByteArray materialMapValue = mapping.value("materialMap").toByteArray();
-    QJsonObject materialMap = materialMapDocument.object();
+        materialMap = QJsonDocument::fromJson(materialMapValue).object();
-    if (!materialMapString.isEmpty()) {
+        if (materialMap.isEmpty()) {
-        if (materialMapDocument.isEmpty() || materialMap.isEmpty()) {
+            qCDebug(modelformat) << "fbx Material Map found but did not produce valid JSON:" << materialMapValue;
-            qCDebug(modelformat) << "fbx Material Map found but did not produce valid JSON:" << materialMapString;
         }
     }
     for (QHash<QString, HFMMaterial>::iterator it = _hfmMaterials.begin(); it != _hfmMaterials.end(); it++) {

libraries/fbx/src/FSTReader.cpp

@@ -88,7 +88,7 @@ QByteArray FSTReader::writeMapping(const QVariantHash& mapping) {
     << BLENDSHAPE_FIELD << JOINT_INDEX_FIELD;
     QBuffer buffer;
     buffer.open(QIODevice::WriteOnly);
-    
+
     for (auto key : PREFERED_ORDER) {
         auto it = mapping.find(key);
         if (it != mapping.constEnd()) {
@@ -104,7 +104,7 @@ QByteArray FSTReader::writeMapping(const QVariantHash& mapping) {
             }
         }
     }
-    
+
     for (auto it = mapping.constBegin(); it != mapping.constEnd(); it++) {
         if (!PREFERED_ORDER.contains(it.key())) {
             writeVariant(buffer, it);

libraries/fbx/src/GLTFSerializer.cpp

@@ -124,6 +124,31 @@ bool GLTFSerializer::getObjectArrayVal(const QJsonObject& object, const QString&
     return _defined;
 }
 
+QByteArray GLTFSerializer::setGLBChunks(const QByteArray& data) {
+    int byte = 4; 
+    int jsonStart = data.indexOf("JSON", Qt::CaseSensitive);
+    int binStart = data.indexOf("BIN", Qt::CaseSensitive);
+    int jsonLength, binLength;
+    QByteArray jsonLengthChunk, binLengthChunk;
+
+    jsonLengthChunk = data.mid(jsonStart - byte, byte);
+    QDataStream tempJsonLen(jsonLengthChunk);
+    tempJsonLen.setByteOrder(QDataStream::LittleEndian);
+    tempJsonLen >> jsonLength;
+    QByteArray jsonChunk = data.mid(jsonStart + byte, jsonLength);
+
+    if (binStart != -1) {
+        binLengthChunk = data.mid(binStart - byte, byte);
+
+        QDataStream tempBinLen(binLengthChunk);
+        tempBinLen.setByteOrder(QDataStream::LittleEndian);
+        tempBinLen >> binLength;
+
+        _glbBinary = data.mid(binStart + byte, binLength);
+    }
+    return jsonChunk;
+}
+
 int GLTFSerializer::getMeshPrimitiveRenderingMode(const QString& type)
 {
     if (type == "POINTS") {
@@ -309,6 +334,14 @@ bool GLTFSerializer::addBuffer(const QJsonObject& object) {
     GLTFBuffer buffer;
    
     getIntVal(object, "byteLength", buffer.byteLength, buffer.defined);
+
+    if (_url.toString().endsWith("glb")) {
+        if (!_glbBinary.isEmpty()) {
+            buffer.blob = _glbBinary;
+        } else {
+            return false;
+        }
+    }
     if (getStringVal(object, "uri", buffer.uri, buffer.defined)) {
         if (!readBinary(buffer.uri, buffer.blob)) {
             return false;
@@ -535,9 +568,16 @@ bool GLTFSerializer::addTexture(const QJsonObject& object) {
 
 bool GLTFSerializer::parseGLTF(const QByteArray& data) {
     PROFILE_RANGE_EX(resource_parse, __FUNCTION__, 0xffff0000, nullptr);
-    
+
-    QJsonDocument d = QJsonDocument::fromJson(data);
+    QByteArray jsonChunk = data;
+
+    if (_url.toString().endsWith("glb") && data.indexOf("glTF") == 0 && data.contains("JSON")) {
+        jsonChunk = setGLBChunks(data);
+    }    
+   
+    QJsonDocument d = QJsonDocument::fromJson(jsonChunk);
     QJsonObject jsFile = d.object();
+
     bool success = setAsset(jsFile);
     if (success) {
         QJsonArray accessors;
@@ -924,6 +964,10 @@ MediaType GLTFSerializer::getMediaType() const {
     MediaType mediaType("gltf");
     mediaType.extensions.push_back("gltf");
     mediaType.webMediaTypes.push_back("model/gltf+json");
+
+    mediaType.extensions.push_back("glb");
+    mediaType.webMediaTypes.push_back("model/gltf-binary");
+
     return mediaType;
 }
 
@@ -932,9 +976,9 @@ std::unique_ptr<hfm::Serializer::Factory> GLTFSerializer::getFactory() const {
 }
 
 HFMModel::Pointer GLTFSerializer::read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url) {
-    
-    _url = url;
 
+    _url = url;
+    
     // Normalize url for local files
     QUrl normalizeUrl = DependencyManager::get<ResourceManager>()->normalizeURL(_url);
     if (normalizeUrl.scheme().isEmpty() || (normalizeUrl.scheme() == "file")) {
@@ -1032,7 +1076,7 @@ QNetworkReply* GLTFSerializer::request(QUrl& url, bool isTest) {
 HFMTexture GLTFSerializer::getHFMTexture(const GLTFTexture& texture) {
     HFMTexture fbxtex = HFMTexture();
     fbxtex.texcoordSet = 0;
-    
+  
     if (texture.defined["source"]) {
         QString url = _file.images[texture.source].uri;
 
@@ -1041,6 +1085,17 @@ HFMTexture GLTFSerializer::getHFMTexture(const GLTFTexture& texture) {
         qCDebug(modelformat) << "fname: " << fname;
         fbxtex.name = fname;
         fbxtex.filename = textureUrl.toEncoded();
+        
+        if (_url.toString().endsWith("glb") && !_glbBinary.isEmpty()) {
+            int bufferView = _file.images[texture.source].bufferView;
+       
+            GLTFBufferView& imagesBufferview = _file.bufferviews[bufferView];
+            int offset = imagesBufferview.byteOffset;
+            int length = imagesBufferview.byteLength;
+
+            fbxtex.content = _glbBinary.mid(offset, length);
+            fbxtex.filename = textureUrl.toEncoded().append(texture.source);
+        }
 
         if (url.contains("data:image/jpeg;base64,") || url.contains("data:image/png;base64,")) {
             fbxtex.content = requestEmbeddedData(url); 

libraries/fbx/src/GLTFSerializer.h

@@ -709,6 +709,7 @@ public:
 private:
     GLTFFile _file;
     QUrl _url;
+    QByteArray _glbBinary;
 
     glm::mat4 getModelTransform(const GLTFNode& node);
 
@@ -731,6 +732,8 @@ private:
                            QVector<double>& values, QMap<QString, bool>&  defined);
     bool getObjectArrayVal(const QJsonObject& object, const QString& fieldname, 
                            QJsonArray& objects, QMap<QString, bool>& defined);
+
+    QByteArray setGLBChunks(const QByteArray& data);
     
     int getMaterialAlphaMode(const QString& type);
     int getAccessorType(const QString& type);

libraries/graphics/src/graphics/Material.cpp

@@ -105,7 +105,7 @@ void Material::setMetallic(float metallic) {
 
 void Material::setScattering(float scattering) {
     scattering = glm::clamp(scattering, 0.0f, 1.0f);
-    _key.setMetallic(scattering > 0.0f);
+    _key.setScattering(scattering > 0.0f);
     _scattering = scattering;
 }
 

libraries/model-networking/src/model-networking/MaterialCache.cpp

@@ -417,9 +417,13 @@ MaterialCache& MaterialCache::instance() {
 }
 
 NetworkMaterialResourcePointer MaterialCache::getMaterial(const QUrl& url) {
-    return ResourceCache::getResource(url, QUrl(), nullptr).staticCast<NetworkMaterialResource>();
+    return ResourceCache::getResource(url).staticCast<NetworkMaterialResource>();
 }
 
-QSharedPointer<Resource> MaterialCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback, const void* extra) {
+QSharedPointer<Resource> MaterialCache::createResource(const QUrl& url) {
     return QSharedPointer<Resource>(new NetworkMaterialResource(url), &Resource::deleter);
+}
+
+QSharedPointer<Resource> MaterialCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new NetworkMaterialResource(*resource.staticCast<NetworkMaterialResource>().data()), &Resource::deleter);
 }

libraries/model-networking/src/model-networking/MaterialCache.h

@@ -53,7 +53,8 @@ public:
     NetworkMaterialResourcePointer getMaterial(const QUrl& url);
 
 protected:
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback, const void* extra) override;
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 };
 
 #endif

libraries/model-networking/src/model-networking/ModelCache.cpp

@@ -40,6 +40,50 @@ public:
     bool combineParts;
 };
 
+// From: https://stackoverflow.com/questions/41145012/how-to-hash-qvariant
+class QVariantHasher {
+public:
+    QVariantHasher() : buff(&bb), ds(&buff) {
+        bb.reserve(1000);
+        buff.open(QIODevice::WriteOnly);
+    }
+    uint hash(const QVariant& v) {
+        buff.seek(0);
+        ds << v;
+        return qHashBits(bb.constData(), buff.pos());
+    }
+private:
+    QByteArray bb;
+    QBuffer buff;
+    QDataStream ds;
+};
+
+namespace std {
+    template <>
+    struct hash<QVariantHash> {
+        size_t operator()(const QVariantHash& a) const {
+            QVariantHasher hasher;
+            return hasher.hash(a);
+        }
+    };
+
+    template <>
+    struct hash<QUrl> {
+        size_t operator()(const QUrl& a) const {
+            return qHash(a);
+        }
+    };
+
+    template <>
+    struct hash<GeometryExtra> {
+        size_t operator()(const GeometryExtra& a) const {
+            size_t result = 0;
+            hash_combine(result, a.mapping, a.textureBaseUrl, a.combineParts);
+            return result;
+        }
+    };
+}
+
 QUrl resolveTextureBaseUrl(const QUrl& url, const QUrl& textureBaseUrl) {
     return textureBaseUrl.isValid() ? textureBaseUrl : url;
 }
@@ -107,10 +151,10 @@ void GeometryMappingResource::downloadFinished(const QByteArray& data) {
         }
 
         auto modelCache = DependencyManager::get<ModelCache>();
-        GeometryExtra extra{ _mapping, _textureBaseUrl, false };
+        GeometryExtra extra { _mapping, _textureBaseUrl, false };
 
         // Get the raw GeometryResource
-        _geometryResource = modelCache->getResource(url, QUrl(), &extra).staticCast<GeometryResource>();
+        _geometryResource = modelCache->getResource(url, QUrl(), &extra, std::hash<GeometryExtra>()(extra)).staticCast<GeometryResource>();
         // Avoid caching nested resources - their references will be held by the parent
         _geometryResource->_isCacheable = false;
 
@@ -253,13 +297,19 @@ void GeometryReader::run() {
 class GeometryDefinitionResource : public GeometryResource {
     Q_OBJECT
 public:
-    GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl, bool combineParts) :
+    GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url) : GeometryResource(url), _modelLoader(modelLoader) {}
-        GeometryResource(url, resolveTextureBaseUrl(url, textureBaseUrl)), _modelLoader(modelLoader), _mapping(mapping), _combineParts(combineParts) {}
+    GeometryDefinitionResource(const GeometryDefinitionResource& other) :
+        GeometryResource(other),
+        _modelLoader(other._modelLoader),
+        _mapping(other._mapping),
+        _combineParts(other._combineParts) {}
 
     QString getType() const override { return "GeometryDefinition"; }
 
     virtual void downloadFinished(const QByteArray& data) override;
 
+    void setExtra(void* extra) override;
+
 protected:
     Q_INVOKABLE void setGeometryDefinition(HFMModel::Pointer hfmModel, QVariantHash mapping);
 
@@ -269,6 +319,13 @@ private:
     bool _combineParts;
 };
 
+void GeometryDefinitionResource::setExtra(void* extra) {
+    const GeometryExtra* geometryExtra = static_cast<const GeometryExtra*>(extra);
+    _mapping = geometryExtra ? geometryExtra->mapping : QVariantHash();
+    _textureBaseUrl = resolveTextureBaseUrl(_url, geometryExtra ? geometryExtra->textureBaseUrl : QUrl());
+    _combineParts = geometryExtra ? geometryExtra->combineParts : true;
+}
+
 void GeometryDefinitionResource::downloadFinished(const QByteArray& data) {
     if (_url != _effectiveBaseURL) {
         _url = _effectiveBaseURL;
@@ -323,27 +380,26 @@ ModelCache::ModelCache() {
     modelFormatRegistry->addFormat(GLTFSerializer());
 }
 
-QSharedPointer<Resource> ModelCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+QSharedPointer<Resource> ModelCache::createResource(const QUrl& url) {
-                                                    const void* extra) {
     Resource* resource = nullptr;
     if (url.path().toLower().endsWith(".fst")) {
         resource = new GeometryMappingResource(url);
     } else {
-        const GeometryExtra* geometryExtra = static_cast<const GeometryExtra*>(extra);
+        resource = new GeometryDefinitionResource(_modelLoader, url);
-        auto mapping = geometryExtra ? geometryExtra->mapping : QVariantHash();
-        auto textureBaseUrl = geometryExtra ? geometryExtra->textureBaseUrl : QUrl();
-        bool combineParts = geometryExtra ? geometryExtra->combineParts : true;
-        resource = new GeometryDefinitionResource(_modelLoader, url, mapping, textureBaseUrl, combineParts);
     }
 
     return QSharedPointer<Resource>(resource, &Resource::deleter);
 }
 
+QSharedPointer<Resource> ModelCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new GeometryDefinitionResource(*resource.staticCast<GeometryDefinitionResource>().data()), &Resource::deleter);
+}
+
 GeometryResource::Pointer ModelCache::getGeometryResource(const QUrl& url,
                                                           const QVariantHash& mapping, const QUrl& textureBaseUrl) {
     bool combineParts = true;
     GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts };
-    GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast<GeometryResource>();
+    GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash<GeometryExtra>()(geometryExtra)).staticCast<GeometryResource>();
     if (resource) {
         if (resource->isLoaded() && resource->shouldSetTextures()) {
             resource->setTextures();
@@ -356,7 +412,7 @@ GeometryResource::Pointer ModelCache::getCollisionGeometryResource(const QUrl& u
                                                                    const QVariantHash& mapping, const QUrl& textureBaseUrl) {
     bool combineParts = false;
     GeometryExtra geometryExtra = { mapping, textureBaseUrl, combineParts };
-    GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra).staticCast<GeometryResource>();
+    GeometryResource::Pointer resource = getResource(url, QUrl(), &geometryExtra, std::hash<GeometryExtra>()(geometryExtra)).staticCast<GeometryResource>();
     if (resource) {
         if (resource->isLoaded() && resource->shouldSetTextures()) {
             resource->setTextures();

libraries/model-networking/src/model-networking/ModelCache.h

@@ -82,8 +82,12 @@ class GeometryResource : public Resource, public Geometry {
 public:
     using Pointer = QSharedPointer<GeometryResource>;
 
-    GeometryResource(const QUrl& url, const QUrl& textureBaseUrl = QUrl()) :
+    GeometryResource(const QUrl& url) : Resource(url) {}
-        Resource(url), _textureBaseUrl(textureBaseUrl) {}
+    GeometryResource(const GeometryResource& other) :
+        Resource(other),
+        Geometry(other),
+        _textureBaseUrl(other._textureBaseUrl),
+        _isCacheable(other._isCacheable) {}
 
     virtual bool areTexturesLoaded() const override { return isLoaded() && Geometry::areTexturesLoaded(); }
 
@@ -153,8 +157,8 @@ public:
 protected:
     friend class GeometryMappingResource;
 
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
-                                                    const void* extra) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 
 private:
     ModelCache();

libraries/model-networking/src/model-networking/ShaderCache.cpp

@@ -21,11 +21,13 @@ ShaderCache& ShaderCache::instance() {
 }
 
 NetworkShaderPointer ShaderCache::getShader(const QUrl& url) {
-    return ResourceCache::getResource(url, QUrl(), nullptr).staticCast<NetworkShader>();
+    return ResourceCache::getResource(url).staticCast<NetworkShader>();
 }
 
-QSharedPointer<Resource> ShaderCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+QSharedPointer<Resource> ShaderCache::createResource(const QUrl& url) {
-    const void* extra) {
     return QSharedPointer<Resource>(new NetworkShader(url), &Resource::deleter);
 }
 
+QSharedPointer<Resource> ShaderCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new NetworkShader(*resource.staticCast<NetworkShader>().data()), &Resource::deleter);
+}

libraries/model-networking/src/model-networking/ShaderCache.h

@@ -14,6 +14,7 @@
 class NetworkShader : public Resource {
 public:
     NetworkShader(const QUrl& url);
+    NetworkShader(const NetworkShader& other) : Resource(other), _source(other._source) {}
 
     QString getType() const override { return "NetworkShader"; }
 
@@ -31,8 +32,8 @@ public:
     NetworkShaderPointer getShader(const QUrl& url);
 
 protected:
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
-        const void* extra) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 };
 
 #endif

libraries/model-networking/src/model-networking/TextureCache.cpp

@@ -194,10 +194,28 @@ public:
     int maxNumPixels;
 };
 
+namespace std {
+    template <>
+    struct hash<QByteArray> {
+        size_t operator()(const QByteArray& a) const {
+            return qHash(a);
+        }
+    };
+
+    template <>
+    struct hash<TextureExtra> {
+        size_t operator()(const TextureExtra& a) const {
+            size_t result = 0;
+            hash_combine(result, (int)a.type, a.content, a.maxNumPixels);
+            return result;
+        }
+    };
+}
+
 ScriptableResource* TextureCache::prefetch(const QUrl& url, int type, int maxNumPixels) {
     auto byteArray = QByteArray();
     TextureExtra extra = { (image::TextureUsage::Type)type, byteArray, maxNumPixels };
-    return ResourceCache::prefetch(url, &extra);
+    return ResourceCache::prefetch(url, &extra, std::hash<TextureExtra>()(extra));
 }
 
 NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) {
@@ -211,7 +229,7 @@ NetworkTexturePointer TextureCache::getTexture(const QUrl& url, image::TextureUs
         modifiedUrl.setQuery(query.toString());
     }
     TextureExtra extra = { type, content, maxNumPixels };
-    return ResourceCache::getResource(modifiedUrl, QUrl(), &extra).staticCast<NetworkTexture>();
+    return ResourceCache::getResource(modifiedUrl, QUrl(), &extra, std::hash<TextureExtra>()(extra)).staticCast<NetworkTexture>();
 }
 
 gpu::TexturePointer TextureCache::getTextureByHash(const std::string& hash) {
@@ -305,26 +323,36 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::Te
     return gpu::TexturePointer(loader(std::move(image), path.toStdString(), shouldCompress, target, false));
 }
 
-QSharedPointer<Resource> TextureCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+QSharedPointer<Resource> TextureCache::createResource(const QUrl& url) {
-    const void* extra) {
+    return QSharedPointer<Resource>(new NetworkTexture(url), &Resource::deleter);
-    const TextureExtra* textureExtra = static_cast<const TextureExtra*>(extra);
+}
-    auto type = textureExtra ? textureExtra->type : image::TextureUsage::DEFAULT_TEXTURE;
+
-    auto content = textureExtra ? textureExtra->content : QByteArray();
+QSharedPointer<Resource> TextureCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
-    auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
+    return QSharedPointer<Resource>(new NetworkTexture(*resource.staticCast<NetworkTexture>().data()), &Resource::deleter);
-    NetworkTexture* texture = new NetworkTexture(url, type, content, maxNumPixels);
-    return QSharedPointer<Resource>(texture, &Resource::deleter);
 }
 
 int networkTexturePointerMetaTypeId = qRegisterMetaType<QWeakPointer<NetworkTexture>>();
 
-NetworkTexture::NetworkTexture(const QUrl& url) :
+NetworkTexture::NetworkTexture(const QUrl& url, bool resourceTexture) :
-Resource(url),
+    Resource(url),
-_type(),
+    _maxNumPixels(100)
-_maxNumPixels(100)
+{
+    if (resourceTexture) {
+        _textureSource = std::make_shared<gpu::TextureSource>(url);
+        _loaded = true;
+    }
+}
+
+NetworkTexture::NetworkTexture(const NetworkTexture& other) :
+    Resource(other),
+    _type(other._type),
+    _currentlyLoadingResourceType(other._currentlyLoadingResourceType),
+    _originalWidth(other._originalWidth),
+    _originalHeight(other._originalHeight),
+    _width(other._width),
+    _height(other._height),
+    _maxNumPixels(other._maxNumPixels)
 {
-    _textureSource = std::make_shared<gpu::TextureSource>(url);
-    _lowestRequestedMipLevel = 0;
-    _loaded = true;
 }
 
 static bool isLocalUrl(const QUrl& url) {
@@ -332,15 +360,15 @@ static bool isLocalUrl(const QUrl& url) {
     return (scheme == HIFI_URL_SCHEME_FILE || scheme == URL_SCHEME_QRC || scheme == RESOURCE_SCHEME);
 }
 
-NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) :
+void NetworkTexture::setExtra(void* extra) {
-    Resource(url),
+    const TextureExtra* textureExtra = static_cast<const TextureExtra*>(extra);
-    _type(type),
+    _type = textureExtra ? textureExtra->type : image::TextureUsage::DEFAULT_TEXTURE;
-    _maxNumPixels(maxNumPixels)
+    _maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
-{
+
-    _textureSource = std::make_shared<gpu::TextureSource>(url, (int)type);
+    _textureSource = std::make_shared<gpu::TextureSource>(_url, (int)_type);
     _lowestRequestedMipLevel = 0;
 
-    auto fileNameLowercase = url.fileName().toLower();
+    auto fileNameLowercase = _url.fileName().toLower();
     if (fileNameLowercase.endsWith(TEXTURE_META_EXTENSION)) {
         _currentlyLoadingResourceType = ResourceType::META;
     } else if (fileNameLowercase.endsWith(".ktx")) {
@@ -351,17 +379,18 @@ NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type,
 
     _shouldFailOnRedirect = _currentlyLoadingResourceType != ResourceType::KTX;
 
-    if (type == image::TextureUsage::CUBE_TEXTURE) {
+    if (_type == image::TextureUsage::CUBE_TEXTURE) {
         setLoadPriority(this, SKYBOX_LOAD_PRIORITY);
     } else if (_currentlyLoadingResourceType == ResourceType::KTX) {
         setLoadPriority(this, HIGH_MIPS_LOAD_PRIORITY);
     }
 
-    if (!url.isValid()) {
+    if (!_url.isValid()) {
         _loaded = true;
     }
 
     // if we have content, load it after we have our self pointer
+    auto content = textureExtra ? textureExtra->content : QByteArray();
     if (!content.isEmpty()) {
         _startedLoading = true;
         QMetaObject::invokeMethod(this, "downloadFinished", Qt::QueuedConnection, Q_ARG(const QByteArray&, content));
@@ -396,7 +425,7 @@ gpu::TexturePointer NetworkTexture::getFallbackTexture() const {
 class ImageReader : public QRunnable {
 public:
     ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url,
-                const QByteArray& data, int maxNumPixels);
+                const QByteArray& data, size_t extraHash, int maxNumPixels);
     void run() override final;
     void read();
 
@@ -406,6 +435,7 @@ private:
     QWeakPointer<Resource> _resource;
     QUrl _url;
     QByteArray _content;
+    size_t _extraHash;
     int _maxNumPixels;
 };
 
@@ -1039,7 +1069,7 @@ void NetworkTexture::loadTextureContent(const QByteArray& content) {
         return;
     }
 
-    QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels));
+    QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _extraHash, _maxNumPixels));
 }
 
 void NetworkTexture::refresh() {
@@ -1064,10 +1094,11 @@ void NetworkTexture::refresh() {
     Resource::refresh();
 }
 
-ImageReader::ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url, const QByteArray& data, int maxNumPixels) :
+ImageReader::ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url, const QByteArray& data, size_t extraHash, int maxNumPixels) :
     _resource(resource),
     _url(url),
     _content(data),
+    _extraHash(extraHash),
     _maxNumPixels(maxNumPixels)
 {
     DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
@@ -1123,11 +1154,12 @@ void ImageReader::read() {
     }
     auto networkTexture = resource.staticCast<NetworkTexture>();
 
-    // Hash the source image to for KTX caching
+    // Hash the source image and extraHash for KTX caching
     std::string hash;
     {
         QCryptographicHash hasher(QCryptographicHash::Md5);
         hasher.addData(_content);
+        hasher.addData(std::to_string(_extraHash).c_str());
         hash = hasher.result().toHex().toStdString();
     }
 
@@ -1216,11 +1248,11 @@ void ImageReader::read() {
                                 Q_ARG(int, texture->getHeight()));
 }
 
-NetworkTexturePointer TextureCache::getResourceTexture(QUrl resourceTextureUrl) {
+NetworkTexturePointer TextureCache::getResourceTexture(const QUrl& resourceTextureUrl) {
     gpu::TexturePointer texture;
     if (resourceTextureUrl == SPECTATOR_CAMERA_FRAME_URL) {
         if (!_spectatorCameraNetworkTexture) {
-            _spectatorCameraNetworkTexture.reset(new NetworkTexture(resourceTextureUrl));
+            _spectatorCameraNetworkTexture.reset(new NetworkTexture(resourceTextureUrl, true));
         }
         if (!_spectatorCameraFramebuffer) {
             getSpectatorCameraFramebuffer(); // initialize frame buffer
@@ -1231,7 +1263,7 @@ NetworkTexturePointer TextureCache::getResourceTexture(QUrl resourceTextureUrl)
     // FIXME: Generalize this, DRY up this code
     if (resourceTextureUrl == HMD_PREVIEW_FRAME_URL) {
         if (!_hmdPreviewNetworkTexture) {
-            _hmdPreviewNetworkTexture.reset(new NetworkTexture(resourceTextureUrl));
+            _hmdPreviewNetworkTexture.reset(new NetworkTexture(resourceTextureUrl, true));
         }
         if (_hmdPreviewFramebuffer) {
             texture = _hmdPreviewFramebuffer->getRenderBuffer(0);

libraries/model-networking/src/model-networking/TextureCache.h

@@ -45,8 +45,8 @@ class NetworkTexture : public Resource, public Texture {
     Q_OBJECT
 
 public:
-    NetworkTexture(const QUrl& url);
+    NetworkTexture(const QUrl& url, bool resourceTexture = false);
-    NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels);
+    NetworkTexture(const NetworkTexture& other);
     ~NetworkTexture() override;
 
     QString getType() const override { return "NetworkTexture"; }
@@ -63,6 +63,8 @@ public:
 
     Q_INVOKABLE void setOriginalDescriptor(ktx::KTXDescriptor* descriptor) { _originalKtxDescriptor.reset(descriptor); }
 
+    void setExtra(void* extra) override;
+
 signals:
     void networkTextureCreated(const QWeakPointer<NetworkTexture>& self);
 
@@ -181,7 +183,7 @@ public:
     gpu::TexturePointer getTextureByHash(const std::string& hash);
     gpu::TexturePointer cacheTextureByHash(const std::string& hash, const gpu::TexturePointer& texture);
 
-    NetworkTexturePointer getResourceTexture(QUrl resourceTextureUrl);
+    NetworkTexturePointer getResourceTexture(const QUrl& resourceTextureUrl);
     const gpu::FramebufferPointer& getHmdPreviewFramebuffer(int width, int height);
     const gpu::FramebufferPointer& getSpectatorCameraFramebuffer();
     const gpu::FramebufferPointer& getSpectatorCameraFramebuffer(int width, int height);
@@ -201,8 +203,8 @@ protected:
     // Overload ResourceCache::prefetch to allow specifying texture type for loads
     Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, int type, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS);
 
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
-        const void* extra) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 
 private:
     friend class ImageReader;

libraries/networking/src/ResourceCache.cpp

@@ -158,8 +158,8 @@ void ScriptableResourceCache::updateTotalSize(const qint64& deltaSize) {
     _resourceCache->updateTotalSize(deltaSize);
 }
 
-ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra) {
+ScriptableResource* ScriptableResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) {
-    return _resourceCache->prefetch(url, extra);
+    return _resourceCache->prefetch(url, extra, extraHash);
 }
 
 
@@ -211,20 +211,20 @@ void ScriptableResource::disconnectHelper() {
     }
 }
 
-ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra) {
+ScriptableResource* ResourceCache::prefetch(const QUrl& url, void* extra, size_t extraHash) {
     ScriptableResource* result = nullptr;
 
     if (QThread::currentThread() != thread()) {
         // Must be called in thread to ensure getResource returns a valid pointer
         BLOCKING_INVOKE_METHOD(this, "prefetch",
             Q_RETURN_ARG(ScriptableResource*, result),
-            Q_ARG(QUrl, url), Q_ARG(void*, extra));
+            Q_ARG(QUrl, url), Q_ARG(void*, extra), Q_ARG(size_t, extraHash));
         return result;
     }
 
     result = new ScriptableResource(url);
 
-    auto resource = getResource(url, QUrl(), extra);
+    auto resource = getResource(url, QUrl(), extra, extraHash);
     result->_resource = resource;
     result->setObjectName(url.toString());
 
@@ -265,16 +265,17 @@ ResourceCache::~ResourceCache() {
 void ResourceCache::clearATPAssets() {
     {
         QWriteLocker locker(&_resourcesLock);
-        for (auto& url : _resources.keys()) {
+        QList<QUrl> urls = _resources.keys();
+        for (auto& url : urls) {
             // If this is an ATP resource
             if (url.scheme() == URL_SCHEME_ATP) {
-
+                auto resourcesWithExtraHash = _resources.take(url);
-                // Remove it from the resource hash
+                for (auto& resource : resourcesWithExtraHash) {
-                auto resource = _resources.take(url);
+                    if (auto strongRef = resource.lock()) {
-                if (auto strongRef = resource.lock()) {
+                        // Make sure the resource won't reinsert itself
-                    // Make sure the resource won't reinsert itself
+                        strongRef->setCache(nullptr);
-                    strongRef->setCache(nullptr);
+                        _totalResourcesSize -= strongRef->getBytes();
-                    _totalResourcesSize -= strongRef->getBytes();
+                    }
                 }
             }
         }
@@ -297,16 +298,20 @@ void ResourceCache::refreshAll() {
     clearUnusedResources();
     resetUnusedResourceCounter();
 
-    QHash<QUrl, QWeakPointer<Resource>> resources;
+    QHash<QUrl, QHash<size_t, QWeakPointer<Resource>>> allResources;
     {
         QReadLocker locker(&_resourcesLock);
-        resources = _resources;
+        allResources = _resources;
     }
 
     // Refresh all remaining resources in use
-    foreach (QSharedPointer<Resource> resource, resources) {
+    // FIXME: this will trigger multiple refreshes for the same resource if they have different hashes
-        if (resource) {
+    for (auto& resourcesWithExtraHash : allResources) {
-            resource->refresh();
+        for (auto& resourceWeak : resourcesWithExtraHash) {
+            auto resource = resourceWeak.lock();
+            if (resource) {
+                resource->refresh();
+            }
         }
     }
 }
@@ -338,39 +343,54 @@ void ResourceCache::setRequestLimit(uint32_t limit) {
     }
 }
 
-QSharedPointer<Resource> ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra) {
+QSharedPointer<Resource> ResourceCache::getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash) {
     QSharedPointer<Resource> resource;
     {
         QReadLocker locker(&_resourcesLock);
-        resource = _resources.value(url).lock();
+        auto& resourcesWithExtraHash = _resources[url];
+        auto resourcesWithExtraHashIter = resourcesWithExtraHash.find(extraHash);
+        if (resourcesWithExtraHashIter != resourcesWithExtraHash.end()) {
+            // We've seen this extra info before
+            resource = resourcesWithExtraHashIter.value().lock();
+        } else if (resourcesWithExtraHash.size() > 0.0f) {
+            // We haven't seen this extra info before, but we've already downloaded the resource.  We need a new copy of this object (with any old hash).
+            resource = createResourceCopy(resourcesWithExtraHash.begin().value().lock());
+            resource->setExtra(extra);
+            resource->setExtraHash(extraHash);
+            resource->setSelf(resource);
+            resource->setCache(this);
+            resource->moveToThread(qApp->thread());
+            connect(resource.data(), &Resource::updateSize, this, &ResourceCache::updateTotalSize);
+            resourcesWithExtraHash.insert(extraHash, resource);
+            removeUnusedResource(resource);
+            resource->ensureLoading();
+        }
     }
     if (resource) {
         removeUnusedResource(resource);
     }
 
-    if (!resource && !url.isValid() && !url.isEmpty() && fallback.isValid()) {
+    if (!resource && (!url.isValid() || url.isEmpty()) && fallback.isValid()) {
-        resource = getResource(fallback, QUrl());
+        resource = getResource(fallback, QUrl(), extra, extraHash);
     }
 
     if (!resource) {
-        resource = createResource(
+        resource = createResource(url);
-            url,
+        resource->setExtra(extra);
-            fallback.isValid() ?  getResource(fallback, QUrl()) : QSharedPointer<Resource>(),
+        resource->setExtraHash(extraHash);
-            extra);
         resource->setSelf(resource);
         resource->setCache(this);
         resource->moveToThread(qApp->thread());
         connect(resource.data(), &Resource::updateSize, this, &ResourceCache::updateTotalSize);
         {
             QWriteLocker locker(&_resourcesLock);
-            _resources.insert(url, resource);
+            _resources[url].insert(extraHash, resource);
         }
         removeUnusedResource(resource);
         resource->ensureLoading();
     }
 
-    DependencyManager::get<ResourceRequestObserver>()->update(
+    DependencyManager::get<ResourceRequestObserver>()->update(resource->getURL(), -1, "ResourceCache::getResource");
-        resource->getURL(), -1, "ResourceCache::getResource");
     return resource;
 }
 
@@ -527,6 +547,27 @@ bool ResourceCache::attemptHighestPriorityRequest() {
 
 static int requestID = 0;
 
+Resource::Resource(const Resource& other) :
+    QObject(),
+    _url(other._url),
+    _effectiveBaseURL(other._effectiveBaseURL),
+    _activeUrl(other._activeUrl),
+    _requestByteRange(other._requestByteRange),
+    _shouldFailOnRedirect(other._shouldFailOnRedirect),
+    _startedLoading(other._startedLoading),
+    _failedToLoad(other._failedToLoad),
+    _loaded(other._loaded),
+    _loadPriorities(other._loadPriorities),
+    _bytesReceived(other._bytesReceived),
+    _bytesTotal(other._bytesTotal),
+    _bytes(other._bytes),
+    _requestID(++requestID),
+    _extraHash(other._extraHash) {
+    if (!other._loaded) {
+        _startedLoading = false;
+    }
+}
+
 Resource::Resource(const QUrl& url) :
     _url(url),
     _activeUrl(url),
@@ -678,7 +719,7 @@ void Resource::setSize(const qint64& bytes) {
 
 void Resource::reinsert() {
     QWriteLocker locker(&_cache->_resourcesLock);
-    _cache->_resources.insert(_url, _self);
+    _cache->_resources[_url].insert(_extraHash, _self);
 }
 
 

libraries/networking/src/ResourceCache.h

@@ -231,16 +231,16 @@ protected slots:
     // Prefetches a resource to be held by the QScriptEngine.
     // Left as a protected member so subclasses can overload prefetch
     // and delegate to it (see TextureCache::prefetch(const QUrl&, int).
-    ScriptableResource* prefetch(const QUrl& url, void* extra);
+    ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash);
 
     // FIXME: The return type is not recognized by JavaScript.
     /// Loads a resource from the specified URL and returns it.
     /// If the caller is on a different thread than the ResourceCache,
     /// returns an empty smart pointer and loads its asynchronously.
     /// \param fallback a fallback URL to load if the desired one is unavailable
-    /// \param extra extra data to pass to the creator, if appropriate
+    // FIXME: std::numeric_limits<size_t>::max() could be a valid extraHash
-    QSharedPointer<Resource> getResource(const QUrl& url, const QUrl& fallback = QUrl(),
+    QSharedPointer<Resource> getResource(const QUrl& url, const QUrl& fallback = QUrl()) { return getResource(url, fallback, nullptr, std::numeric_limits<size_t>::max()); }
-        void* extra = NULL);
+    QSharedPointer<Resource> getResource(const QUrl& url, const QUrl& fallback, void* extra, size_t extraHash);
 
 private slots:
     void clearATPAssets();
@@ -251,11 +251,11 @@ protected:
     // which should be a QScriptEngine with ScriptableResource registered, so that
     // the QScriptEngine will delete the pointer when it is garbage collected.
     // JSDoc is provided on more general function signature.
-    Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr); }
+    Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits<size_t>::max()); }
 
     /// Creates a new resource.
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) = 0;
-                                                    const void* extra) = 0;
+    virtual QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) = 0;
 
     void addUnusedResource(const QSharedPointer<Resource>& resource);
     void removeUnusedResource(const QSharedPointer<Resource>& resource);
@@ -278,7 +278,7 @@ private:
     void resetResourceCounters();
 
     // Resources
-    QHash<QUrl, QWeakPointer<Resource>> _resources;
+    QHash<QUrl, QHash<size_t, QWeakPointer<Resource>>> _resources;
     QReadWriteLock _resourcesLock { QReadWriteLock::Recursive };
     int _lastLRUKey = 0;
 
@@ -332,10 +332,10 @@ public:
      * Prefetches a resource.
      * @function ResourceCache.prefetch
      * @param {string} url - URL of the resource to prefetch.
-     * @param {object} [extra=null]
      * @returns {ResourceObject}
      */
-    Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra = nullptr);
+    Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url) { return prefetch(url, nullptr, std::numeric_limits<size_t>::max()); }
+    Q_INVOKABLE ScriptableResource* prefetch(const QUrl& url, void* extra, size_t extraHash);
 
 signals:
 
@@ -359,7 +359,8 @@ class Resource : public QObject {
     Q_OBJECT
 
 public:
-    
+
+    Resource(const Resource& other);
     Resource(const QUrl& url);
     virtual ~Resource();
 
@@ -415,6 +416,9 @@ public:
     unsigned int getDownloadAttempts() { return _attempts; }
     unsigned int getDownloadAttemptsRemaining() { return _attemptsRemaining; }
 
+    virtual void setExtra(void* extra) {};
+    void setExtraHash(size_t extraHash) { _extraHash = extraHash; }
+
 signals:
     /// Fired when the resource begins downloading.
     void loading();
@@ -469,7 +473,7 @@ protected:
     virtual bool handleFailedRequest(ResourceRequest::Result result);
 
     QUrl _url;
-    QUrl _effectiveBaseURL{ _url };
+    QUrl _effectiveBaseURL { _url };
     QUrl _activeUrl;
     ByteRange _requestByteRange;
     bool _shouldFailOnRedirect { false };
@@ -492,6 +496,8 @@ protected:
     int _requestID;
     ResourceRequest* _request{ nullptr };
 
+    size_t _extraHash;
+
 public slots:
     void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal);
     void handleReplyFinished();

libraries/recording/src/recording/ClipCache.cpp

@@ -48,8 +48,11 @@ NetworkClipLoaderPointer ClipCache::getClipLoader(const QUrl& url) {
     return getResource(url).staticCast<NetworkClipLoader>();
 }
 
-QSharedPointer<Resource> ClipCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback, const void* extra) {
+QSharedPointer<Resource> ClipCache::createResource(const QUrl& url) {
     qCDebug(recordingLog) << "Loading recording at" << url;
     return QSharedPointer<Resource>(new NetworkClipLoader(url), &Resource::deleter);
 }
 
+QSharedPointer<Resource> ClipCache::createResourceCopy(const QSharedPointer<Resource>& resource) {
+    return QSharedPointer<Resource>(new NetworkClipLoader(*resource.staticCast<NetworkClipLoader>().data()), &Resource::deleter);
+}

libraries/recording/src/recording/ClipCache.h

@@ -33,6 +33,8 @@ class NetworkClipLoader : public Resource {
     Q_OBJECT
 public:
     NetworkClipLoader(const QUrl& url);
+    NetworkClipLoader(const NetworkClipLoader& other) : Resource(other), _clip(other._clip) {}
+
     virtual void downloadFinished(const QByteArray& data) override;
     ClipPointer getClip() { return _clip; }
     bool completed() { return _failedToLoad || isLoaded(); }
@@ -54,7 +56,8 @@ public slots:
     NetworkClipLoaderPointer getClipLoader(const QUrl& url);
 
 protected:
-    virtual QSharedPointer<Resource> createResource(const QUrl& url, const QSharedPointer<Resource>& fallback, const void* extra) override;
+    virtual QSharedPointer<Resource> createResource(const QUrl& url) override;
+    QSharedPointer<Resource> createResourceCopy(const QSharedPointer<Resource>& resource) override;
 
 private:
     ClipCache(QObject* parent = nullptr);

libraries/render-utils/src/RenderPipelines.cpp

@@ -579,7 +579,7 @@ void RenderPipelines::updateMultiMaterial(graphics::MultiMaterial& multiMaterial
                         } else {
                             forceDefault = true;
                         }
-                        schemaKey.setScattering(true);
+                        schemaKey.setScatteringMap(true);
                     }
                     break;
                 case graphics::MaterialKey::EMISSIVE_MAP_BIT:

prebuild.py

@@ -35,9 +35,50 @@ import re
 import tempfile
 import time
 import functools
+import subprocess
+import logging
+
+from uuid import uuid4
+from contextlib import contextmanager
 
 print = functools.partial(print, flush=True)
 
+class TrackableLogger(logging.Logger):
+    guid = str(uuid4())
+
+    def _log(self, msg, *args, **kwargs):
+        x = {'guid': self.guid}
+        if 'extra' in kwargs:
+            kwargs['extra'].update(x)
+        else:
+            kwargs['extra'] = x
+        super()._log(msg, *args, **kwargs)
+
+logging.setLoggerClass(TrackableLogger)
+logger = logging.getLogger('prebuild')
+
+def headSha():
+    repo_dir = os.path.dirname(os.path.abspath(__file__))
+    git = subprocess.Popen(
+        'git rev-parse --short HEAD',
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        shell=True, cwd=repo_dir, universal_newlines=True,
+    )
+    stdout, _ = git.communicate()
+    sha = stdout.split('\n')[0]
+    if not sha:
+        raise RuntimeError("couldn't find git sha")
+    return sha
+
+@contextmanager
+def timer(name):
+    ''' Print the elapsed time a context's execution takes to execute '''
+    start = time.time()
+    yield
+    # Please take care when modifiying this print statement.
+    # Log parsing logic may depend on it.
+    logger.info('%s took %.3f secs' % (name, time.time() - start))
+
 def parse_args():
     # our custom ports, relative to the script location
     defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports')
@@ -50,6 +91,7 @@ def parse_args():
     parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
     parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
     parser.add_argument('--ports-path', type=str, default=defaultPortsPath)
+    parser.add_argument('--ci-build', action='store_true')
     if True:
         args = parser.parse_args()
     else:
@@ -66,11 +108,19 @@ def main():
             del os.environ[var]
 
     args = parse_args()
+
+    if args.ci_build:
+        logging.basicConfig(datefmt='%s', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO)
+
+    logger.info('sha=%s' % headSha())
+    logger.info('start')
+
     # Only allow one instance of the program to run at a time
     pm = hifi_vcpkg.VcpkgRepo(args)
     with hifi_singleton.Singleton(pm.lockFile) as lock:
-        if not pm.upToDate():
+        with timer('Bootstraping'):
-            pm.bootstrap()
+            if not pm.upToDate():
+                pm.bootstrap()
 
         # Always write the tag, even if we changed nothing.  This 
         # allows vcpkg to reclaim disk space by identifying directories with
@@ -80,11 +130,13 @@ def main():
         # Grab our required dependencies:
         #  * build host tools, like spirv-cross and scribe
         #  * build client dependencies like openssl and nvtt
-        pm.setupDependencies()
+        with timer('Setting up dependencies'):
+            pm.setupDependencies()
 
         # wipe out the build directories (after writing the tag, since failure 
         # here shouldn't invalidte the vcpkg install)
-        pm.cleanBuilds()
+        with timer('Cleaning builds'):
+            pm.cleanBuilds()
 
         # If we're running in android mode, we also need to grab a bunch of additional binaries
         # (this logic is all migrated from the old setupDependencies tasks in gradle)
@@ -98,7 +150,10 @@ def main():
             hifi_android.QtPackager(appPath, qtPath).bundle()
 
         # Write the vcpkg config to the build directory last
-        pm.writeConfig()
+        with timer('Writing configuration'):
+            pm.writeConfig()
+
+    logger.info('end')
 
 print(sys.argv)
 main()

scripts/system/html/js/entityProperties.js

@@ -2328,7 +2328,7 @@ function createTextureProperty(property, elProperty) {
     elInput.setAttribute("id", elementID);
     elInput.setAttribute("type", "text"); 
     
-    let imageLoad = _.debounce(function (url) {
+    let imageLoad = function(url) {
         if (url.slice(0, 5).toLowerCase() === "atp:/") {
             elImage.src = "";
             elImage.style.display = "none";
@@ -2348,15 +2348,12 @@ function createTextureProperty(property, elProperty) {
             elDiv.classList.remove("no-preview");
             elDiv.classList.add("no-texture");
         }
-    }, IMAGE_DEBOUNCE_TIMEOUT);
-    elInput.imageLoad = imageLoad;
-    elInput.oninput = function (event) {
-        // Add throttle
-        let url = event.target.value;
-        imageLoad(url);
-        updateProperty(property.name, url, property.isParticleProperty)
     };
-    elInput.onchange = elInput.oninput;
+    elInput.imageLoad = imageLoad;
+    elInput.addEventListener('change', createEmitTextPropertyUpdateFunction(property));
+    elInput.addEventListener('change', function(ev) {
+        imageLoad(ev.target.value);
+    });
 
     elProperty.appendChild(elInput);
     elProperty.appendChild(elDiv);

tools/nitpick/src/Nitpick.cpp

@@ -40,7 +40,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) {
  
     _ui.plainTextEdit->setReadOnly(true);
 
-   setWindowTitle("Nitpick - v2.0.1");
+   setWindowTitle("Nitpick - v2.1.0");
 }
 
 Nitpick::~Nitpick() {

tools/nitpick/src/Test.cpp

@@ -758,47 +758,66 @@ void Test::createAllRecursiveScripts() {
         return;
     }
 
+    createAllRecursiveScripts(_testsRootDirectory);
     createRecursiveScript(_testsRootDirectory, false);
-
-    QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-        QString directory = it.next();
-
-        // Only process directories
-        QDir dir;
-        if (!isAValidDirectory(directory)) {
-            continue;
-        }
-
-        // Only process directories that have sub-directories
-        bool hasNoSubDirectories{ true };
-        QDirIterator it2(directory, QDirIterator::Subdirectories);
-        while (it2.hasNext()) {
-            QString directory2 = it2.next();
-
-            // Only process directories
-            QDir dir;
-            if (isAValidDirectory(directory2)) {
-                hasNoSubDirectories = false;
-                break;
-            }
-        }
-
-        if (!hasNoSubDirectories) {
-            createRecursiveScript(directory, false);
-        }
-    }
-
     QMessageBox::information(0, "Success", "Scripts have been created");
 }
 
-void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) {
+void Test::createAllRecursiveScripts(const QString& directory) {
-    const QString recursiveTestsScriptName("testRecursive.js");
+    QDirIterator it(directory, QDirIterator::Subdirectories);
-    const QString recursiveTestsFilename(topLevelDirectory + "/" + recursiveTestsScriptName);
+
+    while (it.hasNext()) {
+        QString nextDirectory = it.next();
+        if (isAValidDirectory(nextDirectory)) {
+            createAllRecursiveScripts(nextDirectory);
+            createRecursiveScript(nextDirectory, false);
+        }
+    }
+}
+
+void Test::createRecursiveScript(const QString& directory, bool interactiveMode) {
+    // If folder contains a test, then we are at a leaf
+    const QString testPathname{ directory + "/" + TEST_FILENAME };
+    if (QFileInfo(testPathname).exists()) {
+        return;
+    }
+
+    // Directories are included in reverse order.  The nitpick scripts use a stack mechanism,
+    // so this ensures that the tests run in alphabetical order (a convenience when debugging)
+    QStringList directories;
+    QDirIterator it(directory);
+    while (it.hasNext()) {
+        QString subDirectory = it.next();
+
+        // Only process directories
+        if (!isAValidDirectory(subDirectory)) {
+            continue;
+        }
+
+        const QString testPathname{ subDirectory + "/" + TEST_FILENAME };
+        if (QFileInfo(testPathname).exists()) {
+            // Current folder contains a test script
+            directories.push_front(testPathname);
+        }
+
+        const QString testRecursivePathname{ subDirectory + "/" + TEST_RECURSIVE_FILENAME };
+        if (QFileInfo(testRecursivePathname).exists()) {
+            // Current folder contains a recursive script
+            directories.push_front(testRecursivePathname);
+        }
+    }
+
+    // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant
+    if (directories.length() == 0) {
+        return;
+    }
+
+    // Open the recursive script file
+    const QString recursiveTestsFilename(directory + "/" + TEST_RECURSIVE_FILENAME);
     QFile recursiveTestsFile(recursiveTestsFilename);
     if (!recursiveTestsFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
         QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__),
-                              "Failed to create \"" + recursiveTestsScriptName + "\" in directory \"" + topLevelDirectory + "\"");
+            "Failed to create \"" + TEST_RECURSIVE_FILENAME + "\" in directory \"" + directory + "\"");
 
         exit(-1);
     }
@@ -812,79 +831,26 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact
     QString user = nitpick->getSelectedUser();
 
     textStream << "PATH_TO_THE_REPO_PATH_UTILS_FILE = \"https://raw.githubusercontent.com/" + user + "/hifi_tests/" + branch +
-                      "/tests/utils/branchUtils.js\";"
+        "/tests/utils/branchUtils.js\";"
-               << endl;
+        << endl;
-    textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl;
+    textStream << "Script.include(PATH_TO_THE_REPO_PATH_UTILS_FILE);" << endl << endl;
-    textStream << "var nitpick = createNitpick(Script.resolvePath(\".\"));" << endl << endl;
 
-    textStream << "var testsRootPath = nitpick.getTestsRootPath();" << endl << endl;
+    textStream << "if (typeof nitpick === 'undefined') nitpick = createNitpick(Script.resolvePath(\".\"));" << endl;
-
+    textStream << "if (typeof testsRootPath === 'undefined') testsRootPath = nitpick.getTestsRootPath();" << endl << endl;
-    // Wait 10 seconds before starting
-    textStream << "if (typeof Test !== 'undefined') {" << endl;
-    textStream << "    Test.wait(10000);" << endl;
-    textStream << "};" << endl << endl;
 
     textStream << "nitpick.enableRecursive();" << endl;
     textStream << "nitpick.enableAuto();" << endl << endl;
 
-    // This is used to verify that the recursive test contains at least one test
-    bool testFound{ false };
-
-    // Directories are included in reverse order.  The nitpick scripts use a stack mechanism,
-    // so this ensures that the tests run in alphabetical order (a convenience when debugging)
-    QStringList directories;
-
-    // First test if top-level folder has a test.js file
-    const QString testPathname{ topLevelDirectory + "/" + TEST_FILENAME };
-    QFileInfo fileInfo(testPathname);
-    if (fileInfo.exists()) {
-        // Current folder contains a test
-        directories.push_front(testPathname);
-
-        testFound = true;
-    }
-
-    QDirIterator it(topLevelDirectory, QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-        QString directory = it.next();
-
-        // Only process directories
-        QDir dir(directory);
-        if (!isAValidDirectory(directory)) {
-            continue;
-        }
-
-        const QString testPathname{ directory + "/" + TEST_FILENAME };
-        QFileInfo fileInfo(testPathname);
-        if (fileInfo.exists()) {
-            // Current folder contains a test
-            directories.push_front(testPathname);
-
-            testFound = true;
-        }
-    }
-
-    if (interactiveMode && !testFound) {
-        QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found");
-        recursiveTestsFile.close();
-        return;
-    }
-
-    // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant
-    // The script will be closed and deleted
-    if (directories.length() == 0) {
-        recursiveTestsFile.close();
-        QFile::remove(recursiveTestsFilename);
-        return;
-    }
-
     // Now include the test scripts
     for (int i = 0; i < directories.length(); ++i) {
         includeTest(textStream, directories.at(i));
     }
 
     textStream << endl;
-    textStream << "nitpick.runRecursive();" << endl;
+    textStream << "if (typeof runningRecursive === 'undefined') {" << endl; 
+    textStream << "   runningRecursive = true;" << endl;
+    textStream << "   nitpick.runRecursive();" << endl;
+    textStream << "}" << endl << endl;
 
     recursiveTestsFile.close();
 }
@@ -928,7 +894,6 @@ void Test::createTestsOutline() {
         QString directory = it.next();
 
         // Only process directories
-        QDir dir;
         if (!isAValidDirectory(directory)) {
             continue;
         }

tools/nitpick/src/Test.h

@@ -72,9 +72,11 @@ public:
 
     void updateTestRailRunResult();
 
-    void createRecursiveScript();
     void createAllRecursiveScripts();
-    void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode);
+    void createAllRecursiveScripts(const QString& directory);
+
+    void createRecursiveScript();
+    void createRecursiveScript(const QString& directory, bool interactiveMode);
 
     int compareImageLists();
     int checkTextResults();
@@ -109,7 +111,8 @@ private:
     bool _isRunningFromCommandLine{ false };
     bool _isRunningInAutomaticTestRun{ false };
 
-    const QString TEST_FILENAME { "test.js" };
+    const QString TEST_FILENAME{ "test.js" };
+    const QString TEST_RECURSIVE_FILENAME{ "testRecursive.js" };
     const QString TEST_RESULTS_FOLDER { "TestResults" };
     const QString TEST_RESULTS_FILENAME { "TestResults.txt" };
 

tools/nitpick/ui/Nitpick.ui

@@ -43,7 +43,7 @@
      </rect>
     </property>
     <property name="currentIndex">
-     <number>3</number>
+     <number>0</number>
     </property>
     <widget class="QWidget" name="tab_1">
      <attribute name="title">
@@ -85,7 +85,7 @@
        </rect>
       </property>
       <property name="text">
-       <string>Create all  MD files</string>
+       <string>Create all MD files</string>
       </property>
      </widget>
      <widget class="QPushButton" name="createTestsOutlinePushbutton">
@@ -124,7 +124,7 @@
        </rect>
       </property>
       <property name="text">
-       <string>Create all  Recursive Scripts</string>
+       <string>Create all Recursive Scripts</string>
       </property>
      </widget>
      <widget class="QPushButton" name="createTestAutoScriptPushbutton">