From a0c56618b36ef9032ef90e920b407ce53f473e53 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 10 Mar 2017 03:57:36 +0000 Subject: [PATCH 1/2] update FileCache to rm manifest --- libraries/networking/src/FileCache.cpp | 145 ++++++++----------------- libraries/networking/src/FileCache.h | 43 ++++---- 2 files changed, 66 insertions(+), 122 deletions(-) diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index b851539b1b..78008562ed 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -60,45 +60,17 @@ void FileCache::initialize() { QDir dir(_dirpath.c_str()); if (dir.exists()) { - std::unordered_map> persistedEntries; - if (dir.exists(MANIFEST_NAME.c_str())) { - std::ifstream manifest; - manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString()); - while (manifest.good()) { - std::string key, metadata; - std::getline(manifest, key, '\t'); - std::getline(manifest, metadata, '\n'); - if (!key.empty()) { - qCDebug(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str()); - auto filename = key + '.' + _ext; - persistedEntries[filename] = { key, metadata }; - } - } - } else { - qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str()); - } + auto nameFilters = QStringList(("*." + _ext).c_str()); + auto filters = QDir::Filters(QDir::NoDotAndDotDot | QDir::Files); + auto sort = QDir::SortFlags(QDir::Time); + auto files = dir.entryList(nameFilters, filters, sort); - std::unordered_map entries; - - foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) { - const auto& it = persistedEntries.find(filename.toStdString()); - if (it == persistedEntries.cend()) { - // unlink extra files - dir.remove(filename); - qCDebug(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); - } else { - // load existing files - const Key& key = it->second.first; - const std::string& metadata = it->second.second; - const std::string filepath = dir.filePath(filename).toStdString(); - const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg(); - - FilePointer file(loadFile(key, filepath, length, metadata).release(), &fileDeleter); - file->_cache = this; - _files[key] = file; - _numTotalFiles += 1; - _totalFilesSize += length; - } + // load persisted files + foreach(QString filename, files) { + const Key key = filename.section('.', 0, 1).toStdString(); + const std::string filepath = dir.filePath(filename).toStdString(); + const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg(); + addFile(Metadata(key, length), filepath); } qCDebug(file_cache, "[%s] Initialized %s", _dirname.c_str(), _dirpath.c_str()); @@ -110,32 +82,40 @@ void FileCache::initialize() { _initialized = true; } -FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length, void* extra) { +FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath) { + FilePointer file(createFile(std::move(metadata), filepath).release(), &fileDeleter); + if (file) { + _numTotalFiles += 1; + _totalFilesSize += file->getLength(); + file->_cache = this; + emit dirty(); + + Lock lock(_filesMutex); + _files[file->getKey()] = file; + } + return file; +} + +FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata) { assert(_initialized); - std::string filepath = getFilepath(key); + std::string filepath = getFilepath(metadata.key); Lock lock(_filesMutex); // if file already exists, return it - FilePointer file = getFile(key); + FilePointer file = getFile(metadata.key); if (file) { - qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), key.c_str()); + qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), metadata.key.c_str()); return file; } // write the new file FILE* saveFile = fopen(filepath.c_str(), "wb"); - if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { - file.reset(createFile(key, filepath, length, extra).release(), &fileDeleter); - file->_cache = this; - _files[key] = file; - _numTotalFiles += 1; - _totalFilesSize += length; - - emit dirty(); + if (saveFile != nullptr && fwrite(data, metadata.length, 1, saveFile) && fclose(saveFile) == 0) { + file = addFile(std::move(metadata), filepath); } else { - qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), key.c_str(), strerror(errno)); + qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), metadata.key.c_str(), strerror(errno)); errno = 0; } @@ -149,7 +129,7 @@ FilePointer FileCache::getFile(const Key& key) { Lock lock(_filesMutex); - // check if file already exists + // check if file exists const auto it = _files.find(key); if (it != _files.cend()) { file = it->second.lock(); @@ -221,58 +201,22 @@ void FileCache::reserve(size_t length) { _numUnusedFiles -= 1; _totalFilesSize -= length; _unusedFilesSize -= length; - - unusedLock.unlock(); - evictedFile(file); - unusedLock.lock(); } } void FileCache::clear() { - auto forAllFiles = [&](std::function functor) { - Lock unusedFilesLock(_unusedFilesMutex); - for (const auto& pair : _unusedFiles) { - functor(pair.second); + Lock unusedFilesLock(_unusedFilesMutex); + for (const auto& pair : _unusedFiles) { + auto& file = pair.second; + file->_cache = nullptr; + + if (_totalFilesSize > _offlineFilesMaxSize) { + _totalFilesSize -= file->getLength(); + } else { + file->_shouldPersist = true; + qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str()); } - // clear files so they are not reiterated from _files - _unusedFiles.clear(); - unusedFilesLock.unlock(); - - Lock filesLock(_filesMutex); - for (const auto& pair : _files) { - FilePointer file; - if ((file = pair.second.lock())) { - functor(file); - } - } - }; - - try { - std::string manifestPath= _dirpath + '/' + MANIFEST_NAME; - std::ofstream manifest(manifestPath); - - forAllFiles([&](const FilePointer& file) { - file->_cache = nullptr; - - if (_totalFilesSize > _offlineFilesMaxSize) { - _totalFilesSize -= file->getLength(); - } else { - manifest << file->getKey() << '\t' << file->getMetadata() << '\n'; - file->_shouldPersist = true; - qCDebug(file_cache, "[%s] Persisting %s (%s)", - _dirname.c_str(), file->getKey().c_str(), file->getMetadata().c_str()); - } - }); - } catch (std::exception& e) { - qCWarning(file_cache, "[%s] Failed to write manifest (%s)", _dirname.c_str(), e.what()); - - forAllFiles([](const FilePointer& file) { - file->_cache = nullptr; - file->_shouldPersist = false; - }); } - - Lock lock(_unusedFilesMutex); _unusedFiles.clear(); } @@ -285,6 +229,11 @@ void File::deleter() { } } +File::File(Metadata&& metadata, const std::string& filepath) : + _key(std::move(metadata.key)), + _length(metadata.length), + _filepath(filepath) {} + File::~File() { QFile file(getFilepath().c_str()); if (file.exists() && !_shouldPersist) { diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index 945d43b224..f77db555bc 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -52,21 +52,24 @@ public: // initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg") FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr); - // precondition: there should be no references to Files when FileCache is destroyed virtual ~FileCache(); - // derived classes are left to implement hashing of the files on their own using Key = std::string; + struct Metadata { + Metadata(const Key& key, size_t length) : + key(key), length(length) {} + Key key; + size_t length; + }; // derived classes should implement a setter/getter, for example, for a FileCache backing a network cache: // - // DerivedFilePointer writeFile(const DerivedData& data) { - // return writeFile(data->key, data->data, data->length, &data); + // DerivedFilePointer writeFile(const char* data, DerivedMetadata&& metadata) { + // return writeFile(data, std::forward(metadata)); // } // // DerivedFilePointer getFile(const QUrl& url) { - // // assuming storage/removal of url->hash in createFile/evictedFile overrides - // auto key = lookup_hash_for(url); + // auto key = lookup_hash_for(url); // assuming hashing url in create/evictedFile overrides // return getFile(key); // } @@ -77,15 +80,11 @@ protected: /// must be called after construction to create the cache on the fs and restore persisted files void initialize(); - FilePointer writeFile(const Key& key, const char* data, size_t length, void* extra); + FilePointer writeFile(const char* data, Metadata&& metadata); FilePointer getFile(const Key& key); - /// create a file (ex.: create a class derived from File and store it in a secondary map with extra->url) - virtual std::unique_ptr createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0; - /// load a file - virtual std::unique_ptr loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0; - /// take action when a file is evicted from the cache (ex.: evict it from a secondary map) - virtual void evictedFile(const FilePointer& file) = 0; + /// create a file + virtual std::unique_ptr createFile(Metadata&& metadata, const std::string& filepath) = 0; private: using Mutex = std::recursive_mutex; @@ -95,6 +94,7 @@ private: std::string getFilepath(const Key& key); + FilePointer addFile(Metadata&& metadata, const std::string& filepath); void addUnusedFile(const FilePointer file); void removeUnusedFile(const FilePointer file); void reserve(size_t length); @@ -126,31 +126,26 @@ class File : public QObject { public: using Key = FileCache::Key; + using Metadata = FileCache::Metadata; - std::string getFilepath() const { return _filepath; } Key getKey() const { return _key; } size_t getLength() const { return _length; } + std::string getFilepath() const { return _filepath; } - // the destructor should handle unlinking of the actual filepath virtual ~File(); - // overrides should call File::deleter to maintain caching behavior + /// overrides should call File::deleter to maintain caching behavior virtual void deleter(); protected: - // when constructed, the file has already been created/written - File(const Key& key, const std::string& filepath, size_t length) : - _filepath(filepath), _key(key), _length(length) {} - - /// get metadata to store with a file between instances (ex.: return the url of a hash) - virtual std::string getMetadata() const = 0; - - const std::string _filepath; + /// when constructed, the file has already been created/written + File(Metadata&& metadata, const std::string& filepath); private: friend class FileCache; const Key _key; const size_t _length; + const std::string _filepath; FileCache* _cache; int _LRUKey { 0 }; From a27d88da1a3acafc35f9abf5765815d3c7743a3d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Sun, 12 Mar 2017 20:14:38 -0400 Subject: [PATCH 2/2] update TextureCache to use FileCache --- .../src/model-networking/KTXCache.cpp | 54 ++--- .../src/model-networking/KTXCache.h | 38 +-- .../src/model-networking/TextureCache.cpp | 220 ++++++++---------- .../src/model-networking/TextureCache.h | 6 +- 4 files changed, 120 insertions(+), 198 deletions(-) diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp index d0380c7635..908d3245ce 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.cpp +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -16,55 +16,27 @@ using File = cache::File; using FilePointer = cache::FilePointer; -KTXFilePointer KTXCache::writeFile(Data data) { - return std::static_pointer_cast(FileCache::writeFile(data.key, data.data, data.length, (void*)&data)); +KTXCache::KTXCache(const std::string& dir, const std::string& ext) : + FileCache(dir, ext) { + initialize(); } -KTXFilePointer KTXCache::getFile(const QUrl& url) { - Key key; - { - Lock lock(_urlMutex); - const auto it = _urlMap.find(url); - if (it != _urlMap.cend()) { - key = it->second; - } - } - - KTXFilePointer file; - if (!key.empty()) { - file = std::static_pointer_cast(FileCache::getFile(key)); - } - - return file; +KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) { + FilePointer file = FileCache::writeFile(data, std::move(metadata)); + return std::static_pointer_cast(file); } -std::unique_ptr KTXCache::createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) { - Lock lock(_urlMutex); - _urlMap[url] = key; - return std::unique_ptr(new KTXFile(key, filepath, length, url)); +KTXFilePointer KTXCache::getFile(const Key& key) { + return std::static_pointer_cast(FileCache::getFile(key)); } -std::unique_ptr KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { - const QUrl& url = reinterpret_cast(extra)->url; - qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url; - return createKTXFile(key, filepath, length, url); +std::unique_ptr KTXCache::createFile(Metadata&& metadata, const std::string& filepath) { + qCInfo(file_cache) << "Wrote KTX" << metadata.key.c_str(); + return std::unique_ptr(new KTXFile(std::move(metadata), filepath)); } -std::unique_ptr KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) { - const QUrl url = QString(metadata.c_str()); - qCInfo(file_cache) << "Loaded KTX" << key.c_str() << url; - return createKTXFile(key, filepath, length, url); -} - -void KTXCache::evictedFile(const FilePointer& file) { - const QUrl url = std::static_pointer_cast(file)->getUrl(); - Lock lock(_urlMutex); - _urlMap.erase(url); -} - -std::string KTXFile::getMetadata() const { - return _url.toString().toStdString(); -} +KTXFile::KTXFile(Metadata&& metadata, const std::string& filepath) : + cache::File(std::move(metadata), filepath) {} std::unique_ptr KTXFile::getKTX() const { ktx::StoragePointer storage = std::make_shared(getFilepath().c_str()); diff --git a/libraries/model-networking/src/model-networking/KTXCache.h b/libraries/model-networking/src/model-networking/KTXCache.h index 84dda48ee2..4ef5e52721 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.h +++ b/libraries/model-networking/src/model-networking/KTXCache.h @@ -27,53 +27,25 @@ class KTXCache : public cache::FileCache { Q_OBJECT public: - KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) { initialize(); } + KTXCache(const std::string& dir, const std::string& ext); - struct Data { - Data(const QUrl& url, const Key& key, const char* data, size_t length) : - url(url), key(key), data(data), length(length) {} - const QUrl url; - const Key key; - const char* data; - size_t length; - }; - - KTXFilePointer writeFile(Data data); - KTXFilePointer getFile(const QUrl& url); + KTXFilePointer writeFile(const char* data, Metadata&& metadata); + KTXFilePointer getFile(const Key& key); protected: - std::unique_ptr createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; - std::unique_ptr loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final; - void evictedFile(const cache::FilePointer& file) override final; - -private: - std::unique_ptr createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url); - - using Mutex = std::mutex; - using Lock = std::lock_guard; - struct QUrlHasher { std::size_t operator()(QUrl const& url) const { return qHash(url); } }; - - std::unordered_map _urlMap; - Mutex _urlMutex; + std::unique_ptr createFile(Metadata&& metadata, const std::string& filepath) override final; }; class KTXFile : public cache::File { Q_OBJECT public: - QUrl getUrl() const { return _url; } std::unique_ptr getKTX() const; protected: - KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) : - File(key, filepath, length), _url(url) {} - - std::string getMetadata() const override final; - -private: friend class KTXCache; - const QUrl _url; + KTXFile(Metadata&& metadata, const std::string& filepath); }; #endif // hifi_KTXCache_h diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 16d84b69df..b9977e33d8 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -43,8 +43,8 @@ #include Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image") -Q_LOGGING_CATEGORY(trace_resource_parse_ktx, "trace.resource.parse.ktx") -Q_LOGGING_CATEGORY(trace_resource_cache_ktx, "trace.resource.cache.ktx") +Q_LOGGING_CATEGORY(trace_resource_parse_image_raw, "trace.resource.parse.image.raw") +Q_LOGGING_CATEGORY(trace_resource_parse_image_ktx, "trace.resource.parse.image.ktx") const std::string TextureCache::KTX_DIRNAME { "ktx_cache" }; const std::string TextureCache::KTX_EXT { "ktx" }; @@ -296,34 +296,14 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, Type type QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) { - KTXFilePointer file = _ktxCache.getFile(url); const TextureExtra* textureExtra = static_cast(extra); auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; - - NetworkTexture* texture; - if (file) { - texture = new NetworkTexture(url, type, file); - } else { - auto content = textureExtra ? textureExtra->content : QByteArray(); - auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; - texture = new NetworkTexture(url, type, content, maxNumPixels); - } - + auto content = textureExtra ? textureExtra->content : QByteArray(); + auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; + NetworkTexture* texture = new NetworkTexture(url, type, content, maxNumPixels); return QSharedPointer(texture, &Resource::deleter); } -NetworkTexture::NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file) : - Resource(url), - _type(type), - _file(file) { - _textureSource = std::make_shared(); - - if (file) { - _startedLoading = true; - QMetaObject::invokeMethod(this, "loadFile", Qt::QueuedConnection); - } -} - NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) : Resource(url), _type(type), @@ -342,12 +322,6 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con } } -NetworkTexture::NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content) : - NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_NUM_PIXELS) -{ - _textureLoader = textureLoader; -} - NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const { if (_type == CUSTOM_TEXTURE) { return _textureLoader; @@ -387,30 +361,8 @@ gpu::TexturePointer NetworkTexture::getFallbackTexture() const { class Reader : public QRunnable { public: - Reader(const QWeakPointer& resource, const QUrl& url) : _resource(resource), _url(url) { - DependencyManager::get()->incrementStat("PendingProcessing"); - } - void run() override final { - DependencyManager::get()->decrementStat("PendingProcessing"); - CounterStat counter("Processing"); - - // Run this with low priority, then restore thread priority - auto originalPriority = QThread::currentThread()->priority(); - if (originalPriority == QThread::InheritPriority) { - originalPriority = QThread::NormalPriority; - } - QThread::currentThread()->setPriority(QThread::LowPriority); - Finally restorePriority([originalPriority]{ - QThread::currentThread()->setPriority(originalPriority); - }); - - if (!_resource.lock()) { // to ensure the resource is still needed - qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; - return; - } - - read(); - } + Reader(const QWeakPointer& resource, const QUrl& url); + void run() override final; virtual void read() = 0; protected: @@ -418,42 +370,83 @@ protected: QUrl _url; }; -class FileReader : public Reader { -public: - FileReader(const QWeakPointer& resource, const QUrl& url) : Reader(resource, url) {} - void read() override final; -}; - class ImageReader : public Reader { public: ImageReader(const QWeakPointer& resource, const QUrl& url, - const QByteArray& data, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); + const QByteArray& data, const std::string& hash, int maxNumPixels); void read() override final; private: static void listSupportedImageFormats(); QByteArray _content; + std::string _hash; int _maxNumPixels; }; -void NetworkTexture::downloadFinished(const QByteArray& data) { - // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new ImageReader(_self, _url, data)); -} +class KTXReader : public Reader { +public: + KTXReader(const QWeakPointer& resource, const QUrl& url, const KTXFilePointer& ktxFile); + void read() override final; -void NetworkTexture::loadFile() { - QThreadPool::globalInstance()->start(new FileReader(_self, _url)); +private: + KTXFilePointer _file; +}; + +void NetworkTexture::downloadFinished(const QByteArray& data) { + loadContent(data); } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels)); + // Hash the source image to for KTX caching + std::string hash; + { + QCryptographicHash hasher(QCryptographicHash::Md5); + hasher.addData(content); + hash = hasher.result().toHex().toStdString(); + } + + std::unique_ptr reader; + KTXFilePointer ktxFile; + if (!_cache.isNull() && (ktxFile = static_cast(_cache.data())->_ktxCache.getFile(hash))) { + reader.reset(new KTXReader(_self, _url, ktxFile)); + } else { + reader.reset(new ImageReader(_self, _url, content, hash, _maxNumPixels)); + } + + QThreadPool::globalInstance()->start(reader.release()); +} + +Reader::Reader(const QWeakPointer& resource, const QUrl& url) : + _resource(resource), _url(url) { + DependencyManager::get()->incrementStat("PendingProcessing"); +} + +void Reader::run() { + PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); + DependencyManager::get()->decrementStat("PendingProcessing"); + CounterStat counter("Processing"); + + auto originalPriority = QThread::currentThread()->priority(); + if (originalPriority == QThread::InheritPriority) { + originalPriority = QThread::NormalPriority; + } + QThread::currentThread()->setPriority(QThread::LowPriority); + Finally restorePriority([originalPriority]{ QThread::currentThread()->setPriority(originalPriority); }); + + if (!_resource.data()) { + qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; + return; + } + + read(); } ImageReader::ImageReader(const QWeakPointer& resource, const QUrl& url, - const QByteArray& data, int maxNumPixels) : - Reader(resource, url), _content(data), _maxNumPixels(maxNumPixels) { + const QByteArray& data, const std::string& hash, int maxNumPixels) : + Reader(resource, url), _content(data), _hash(hash), _maxNumPixels(maxNumPixels) { listSupportedImageFormats(); + #if DEBUG_DUMP_TEXTURE_LOADS static auto start = usecTimestampNow() / USECS_PER_MSEC; auto now = usecTimestampNow() / USECS_PER_MSEC - start; @@ -482,36 +475,7 @@ void ImageReader::listSupportedImageFormats() { }); } -void FileReader::read() { - PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); - - gpu::TexturePointer texture; - { - auto resource = _resource.lock(); // to ensure the resource is still needed - if (!resource) { - qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; - return; - } - - auto ktx = resource.staticCast()->_file->getKTX(); - texture.reset(gpu::Texture::unserialize(ktx)); - texture->setKtxBacking(ktx); - } - - auto resource = _resource.lock(); // to ensure the resource is still needed - if (resource) { - QMetaObject::invokeMethod(resource.data(), "setImage", - Q_ARG(gpu::TexturePointer, texture), - Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight())); - } else { - qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; - } - -} - void ImageReader::read() { - PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); - // Help the QImage loader by extracting the image file format from the url filename ext. // Some tga are not created properly without it. auto filename = _url.fileName().toStdString(); @@ -541,7 +505,6 @@ void ImageReader::read() { QSize(imageWidth, imageHeight) << ")"; } - // Load the image into a gpu::Texture gpu::TexturePointer texture = nullptr; { auto resource = _resource.lock(); // to ensure the resource is still needed @@ -552,37 +515,22 @@ void ImageReader::read() { auto url = _url.toString().toStdString(); - PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0); + PROFILE_RANGE_EX(resource_parse_image_raw, __FUNCTION__, 0xffff0000, 0); + // Load the image into a gpu::Texture auto networkTexture = resource.staticCast(); texture.reset(networkTexture->getTextureLoader()(image, url)); texture->setSource(url); if (texture) { texture->setFallbackTexture(networkTexture->getFallbackTexture()); } - } - // Hash the source image to use as a filename for on-disk caching - std::string hash; - { - QCryptographicHash hasher(QCryptographicHash::Md5); - hasher.addData((const char*)image.bits(), image.byteCount() * sizeof(char)); - hash = hasher.result().toHex().toStdString(); - } - - { - auto resource = _resource.lock(); // to ensure the resource is still needed - if (!resource) { - qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; - return; - } - - PROFILE_RANGE_EX(resource_cache_ktx, __FUNCTION__, 0xffffff00, 0); + // Save the image into a KTXFile auto ktx = gpu::Texture::serialize(*texture); const char* data = reinterpret_cast(ktx->_storage->data()); size_t length = ktx->_storage->size(); KTXFilePointer file; auto& ktxCache = DependencyManager::get()->_ktxCache; - if (!ktx || !(file = ktxCache.writeFile({ _url, hash, data, length }))) { + if (!ktx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(_hash, length)))) { qCWarning(modelnetworking) << _url << "file cache failed"; } else { resource.staticCast()->_file = file; @@ -600,3 +548,35 @@ void ImageReader::read() { qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; } } + +KTXReader::KTXReader(const QWeakPointer& resource, const QUrl& url, + const KTXFilePointer& ktxFile) : + Reader(resource, url), _file(ktxFile) {} + +void KTXReader::read() { + PROFILE_RANGE_EX(resource_parse_image_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); + + gpu::TexturePointer texture; + { + auto resource = _resource.lock(); // to ensure the resource is still needed + if (!resource) { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + return; + } + + resource.staticCast()->_file = _file; + auto ktx = _file->getKTX(); + texture.reset(gpu::Texture::unserialize(ktx)); + texture->setKtxBacking(ktx); + } + + auto resource = _resource.lock(); // to ensure the resource is still needed + if (resource) { + QMetaObject::invokeMethod(resource.data(), "setImage", + Q_ARG(gpu::TexturePointer, texture), + Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight())); + } else { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + } + +} diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index ad628c6f2c..8ee3f59416 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -65,9 +65,7 @@ public: typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName); using TextureLoaderFunc = std::function; - NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file); NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels); - NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content); QString getType() const override { return "NetworkTexture"; } @@ -89,11 +87,10 @@ protected: virtual void downloadFinished(const QByteArray& data) override; Q_INVOKABLE void loadContent(const QByteArray& content); - Q_INVOKABLE void loadFile(); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); private: - friend class FileReader; + friend class KTXReader; friend class ImageReader; Type _type; @@ -149,6 +146,7 @@ protected: private: friend class ImageReader; + friend class NetworkTexture; friend class DilatableNetworkTexture; TextureCache();