From 8969d7df1d9cd61718e45043b2c172b7d4684b43 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 23 Feb 2017 16:26:52 -0500 Subject: [PATCH 01/11] FileCache --- libraries/networking/src/FileCache.cpp | 246 +++++++++++++++++++++++++ libraries/networking/src/FileCache.h | 149 +++++++++++++++ 2 files changed, 395 insertions(+) create mode 100644 libraries/networking/src/FileCache.cpp create mode 100644 libraries/networking/src/FileCache.h diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp new file mode 100644 index 0000000000..a717546de4 --- /dev/null +++ b/libraries/networking/src/FileCache.cpp @@ -0,0 +1,246 @@ +// +// FileCache.cpp +// libraries/model-networking/src +// +// Created by Zach Pomerantz on 2/21/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "FileCache.h" + +#include +#include +#include + +#include + +#include + +Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache") + +static const std::string MANIFEST_NAME = "manifest"; + +void FileCache::setUnusedFileCacheSize(size_t unusedFilesMaxSize) { + _unusedFilesMaxSize = std::min(unusedFilesMaxSize, MAX_UNUSED_MAX_SIZE); + reserve(0); + emit dirty(); +} + +void FileCache::setOfflineFileCacheSize(size_t offlineFilesMaxSize) { + _offlineFilesMaxSize = std::min(offlineFilesMaxSize, MAX_UNUSED_MAX_SIZE); +} + +FileCache::FileCache(const std::string& dirname, const std::string& ext, QObject* parent) : + QObject(parent), + _dir(createDir(dirname)), + _ext(ext) {} + +FileCache::~FileCache() { + clear(); +} + +void fileDeleter(File* file) { + file->deleter(); +} + +FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length, void* extra) { + std::string filepath = getFilepath(key); + + Lock lock(_filesMutex); + + // if file already exists, return it + FilePointer file = getFile(key); + if (file) { + qCWarning(file_cache) << "Attempted to overwrite" << filepath.c_str(); + return file; + } + + // write the new file + FILE* saveFile = fopen(filepath.c_str(), "wb"); + if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { + file.reset(createFile(key, filepath, length, extra), &fileDeleter); + fclose(saveFile); + file->_cache = this; + _files[key] = file; + _numTotalFiles += 1; + _totalFilesSize += length; + + emit dirty(); + } else { + qCWarning(file_cache, "Failed to write %s (%s)", filepath.c_str(), strerror(errno)); + errno = 0; + } + + return file; +} + +FilePointer FileCache::getFile(const Key& key) { + FilePointer file; + + Lock lock(_filesMutex); + + // check if file already exists + const auto it = _files.find(key); + if (it != _files.cend()) { + file = it->second.lock(); + if (file) { + // if it exists, it is active - remove it from the cache + removeUnusedFile(file); + emit dirty(); + } else { + // if not, remove the weak_ptr + _files.erase(it); + } + } + + return file; +} + +File* FileCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { + return new File(key, filepath, length); +} + +std::string FileCache::createDir(const std::string& dirname) { + QString dirpath = ServerPathUtils::getDataFilePath(dirname.c_str()); + QDir dir(dirpath); + + if (dir.exists()) { + std::unordered_set persistedEntries; + if (dir.exists(MANIFEST_NAME.c_str())) { + std::ifstream manifest; + manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString()); + while (manifest.good()) { + std::string entry; + manifest >> entry; + persistedEntries.insert(entry); + + // ZZMP: rm + for (const auto& entry : persistedEntries) + qDebug() << "ZZMP" << entry.c_str(); + qDebug() << "ZZMP" << "---"; + } + } + + foreach(QString filename, dir.entryList()) { + if (persistedEntries.find(filename.toStdString()) == persistedEntries.cend()) { + dir.remove(filename); + } + } + } else { + dir.mkpath(dirpath); + } + + return dirpath.toStdString(); +} + +std::string FileCache::getFilepath(const Key& key) { + return _dir + key + '.' + _ext; +} + +void FileCache::addUnusedFile(const FilePointer file) { + { + Lock lock(_filesMutex); + _files[file->getKey()] = file; + } + + reserve(file->getLength()); + file->_LRUKey = ++_lastLRUKey; + + { + Lock lock(_unusedFilesMutex); + _unusedFiles.insert({ file->_LRUKey, file }); + _numUnusedFiles += 1; + _unusedFilesSize += file->getLength(); + } + + emit dirty(); +} + +void FileCache::removeUnusedFile(const FilePointer file) { + Lock lock(_unusedFilesMutex); + const auto it = _unusedFiles.find(file->_LRUKey); + if (it != _unusedFiles.cend()) { + _unusedFiles.erase(it); + _numUnusedFiles -= 1; + _unusedFilesSize -= file->getLength(); + } +} + +void FileCache::reserve(size_t length) { + Lock unusedLock(_unusedFilesMutex); + while (!_unusedFiles.empty() && + _unusedFilesSize + length > _unusedFilesMaxSize) { + auto it = _unusedFiles.begin(); + auto file = it->second; + auto length = file->getLength(); + + unusedLock.unlock(); + { + file->_cache = nullptr; + Lock lock(_filesMutex); + _files.erase(file->getKey()); + } + unusedLock.lock(); + + _unusedFiles.erase(it); + _numTotalFiles -= 1; + _numUnusedFiles -= 1; + _totalFilesSize -= length; + _unusedFilesSize -= length; + + unusedLock.unlock(); + evictedFile(file); + unusedLock.lock(); + } +} + +void FileCache::clear() { + std::string manifestPath= _dir + MANIFEST_NAME; + FILE* manifest = fopen(manifestPath.c_str(), "wb"); + + Lock lock(_unusedFilesMutex); + for (const auto& val : _unusedFiles) { + const FilePointer& file = val.second; + file->_cache = nullptr; + + if (_unusedFilesSize > _offlineFilesMaxSize) { + _unusedFilesSize -= file->getLength(); + } else { + std::string key = file->getKey() + '.' + _ext + '\n'; + if (manifest != nullptr && !fwrite(key.c_str(), key.length(), 1, manifest)) { + manifest = nullptr; // to prevent future writes + } + file->_shouldPersist = true; + } + } + + if (manifest == nullptr || fclose(manifest) != 0) { + for (const auto& val : _unusedFiles) { + val.second->_shouldPersist = false; + } + + qCWarning(file_cache, "Failed to write %s (%s)", manifestPath.c_str(), strerror(errno)); + errno = 0; + } + + _unusedFiles.clear(); +} + +void File::deleter() { + if (_cache) { + FilePointer self(this, &fileDeleter); + _cache->addUnusedFile(self); + } else { + deleteLater(); + } +} + +File::~File() { + QFile file(getFilepath().c_str()); + if (file.exists() && !_shouldPersist) { + file.remove(); + } +} diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h new file mode 100644 index 0000000000..f068f6e7d5 --- /dev/null +++ b/libraries/networking/src/FileCache.h @@ -0,0 +1,149 @@ +// +// FileCache.h +// libraries/networking/src +// +// Created by Zach Pomerantz on 2/21/2017. +// Copyright 2017 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FileCache_h +#define hifi_FileCache_h + +#include +#include +#include +#include +#include +#include + +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(file_cache) + +class File; +using FilePointer = std::shared_ptr; + +class FileCache : public QObject { + Q_OBJECT + Q_PROPERTY(size_t numTotal READ getNumTotalFiles NOTIFY dirty) + Q_PROPERTY(size_t numCached READ getNumCachedFiles NOTIFY dirty) + Q_PROPERTY(size_t sizeTotal READ getSizeTotalFiles NOTIFY dirty) + Q_PROPERTY(size_t sizeCached READ getSizeCachedFiles NOTIFY dirty) + + static const size_t BYTES_PER_MEGABYTES = 1024 * 1024; + static const size_t BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES; + static const size_t DEFAULT_UNUSED_MAX_SIZE = 5 * BYTES_PER_GIGABYTES; // 5GB + static const size_t MAX_UNUSED_MAX_SIZE = 100 * BYTES_PER_GIGABYTES; // 100GB + static const size_t DEFAULT_OFFLINE_MAX_SIZE = 2 * BYTES_PER_GIGABYTES; // 2GB + +public: + size_t getNumTotalFiles() const { return _numTotalFiles; } + size_t getNumCachedFiles() const { return _numUnusedFiles; } + size_t getSizeTotalFiles() const { return _totalFilesSize; } + size_t getSizeCachedFiles() const { return _unusedFilesSize; } + + void setUnusedFileCacheSize(size_t unusedFilesMaxSize); + size_t getUnusedFileCacheSize() const { return _unusedFilesSize; } + + void setOfflineFileCacheSize(size_t offlineFilesMaxSize); + + // initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg") + FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr); + // precondition: there should be no references to Files when FileCache is destroyed + virtual ~FileCache(); + + // derived classes are left to implement hashing of the files on their own + using Key = std::string; + + // derived classes should implement a setter/getter, for example, for a FileCache backing a network cache: + // + // DerivedFilePointer writeFile(const DerivedData& data) { + // return writeFile(data->key, data->data, data->length, &data); + // } + // + // DerivedFilePointer getFile(const QUrl& url) { + // // assuming storage/removal of url->hash in createFile/evictedFile overrides + // auto key = lookup_hash_for(url); + // return getFile(key); + // } + +signals: + void dirty(); + +protected: + FilePointer writeFile(const Key& key, const char* data, size_t length, void* extra); + FilePointer getFile(const Key& key); + + virtual File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra); + virtual void evictedFile(const FilePointer& file) = 0; + +private: + using Mutex = std::recursive_mutex; + using Lock = std::unique_lock; + + friend class File; + + std::string createDir(const std::string& dirname); + std::string getFilepath(const Key& key); + + void addUnusedFile(const FilePointer file); + void removeUnusedFile(const FilePointer file); + void reserve(size_t length); + void clear(); + + std::atomic _numTotalFiles { 0 }; + std::atomic _numUnusedFiles { 0 }; + std::atomic _totalFilesSize { 0 }; + std::atomic _unusedFilesSize { 0 }; + + std::string _dir; + std::string _ext; + + std::unordered_map> _files; + Mutex _filesMutex; + + std::map _unusedFiles; + Mutex _unusedFilesMutex; + size_t _unusedFilesMaxSize { DEFAULT_UNUSED_MAX_SIZE }; + int _lastLRUKey { 0 }; + + size_t _offlineFilesMaxSize { DEFAULT_OFFLINE_MAX_SIZE }; +}; + +class File : public QObject { + Q_OBJECT + +public: + using Key = FileCache::Key; + + std::string getFilepath() const { return _filepath; } + Key getKey() const { return _key; } + size_t getLength() const { return _length; } + + // overrides should call File::deleter to maintain caching behavior + virtual void deleter(); + +protected: + // when constructed, the file has already been created/written + File(const Key& key, const std::string& filepath, size_t length) : + _filepath(filepath), _key(key), _length(length) {} + // the destructor should handle unlinking of the actual filepath + virtual ~File(); + + const std::string _filepath; + +private: + friend class FileCache; + + const Key _key; + const size_t _length; + + FileCache* _cache; + int _LRUKey { 0 }; + + bool _shouldPersist { false }; +}; + +#endif // hifi_FileCache_h From cfe14518a1ba08884a8c1399b3ac78051cab9815 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 23 Feb 2017 16:27:08 -0500 Subject: [PATCH 02/11] KTXCache --- .../src/model-networking/KTXCache.cpp | 49 +++++++++++++ .../src/model-networking/KTXCache.h | 69 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 libraries/model-networking/src/model-networking/KTXCache.cpp create mode 100644 libraries/model-networking/src/model-networking/KTXCache.h diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp new file mode 100644 index 0000000000..1ab32698b6 --- /dev/null +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -0,0 +1,49 @@ +// +// KTXCache.cpp +// libraries/model-networking/src +// +// Created by Zach Pomerantz on 2/22/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "KTXCache.h" + +#include + +KTXFilePointer KTXCache::writeFile(Data data) { + return std::static_pointer_cast(FileCache::writeFile(data.key, data.data, data.length, (void*)&data)); +} + +KTXFilePointer KTXCache::getFile(const QUrl& url) { + Key key; + { + Lock lock(_urlMutex); + const auto it = _urlMap.find(url); + if (it != _urlMap.cend()) { + key = it->second; + } + } + + KTXFilePointer file; + if (!key.empty()) { + file = std::static_pointer_cast(FileCache::getFile(key)); + } + + return file; +} + +File* KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { + const QUrl& url = reinterpret_cast(extra)->url; + Lock lock(_urlMutex); + _urlMap[url] = key; + return new KTXFile(key, filepath, length, url); +} + +void KTXCache::evictedFile(const FilePointer& file) { + const QUrl url = std::static_pointer_cast(file)->getUrl(); + Lock lock(_urlMutex); + _urlMap.erase(url); +} diff --git a/libraries/model-networking/src/model-networking/KTXCache.h b/libraries/model-networking/src/model-networking/KTXCache.h new file mode 100644 index 0000000000..5b9cb04061 --- /dev/null +++ b/libraries/model-networking/src/model-networking/KTXCache.h @@ -0,0 +1,69 @@ +// +// KTXCache.h +// libraries/model-networking/src +// +// Created by Zach Pomerantz 2/22/2017. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_KTXCache_h +#define hifi_KTXCache_h + +#include + +#include + +class KTXFile; +using KTXFilePointer = std::shared_ptr; + +class KTXCache : public FileCache { + Q_OBJECT + +public: + KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) {} + + struct Data { + Data(const QUrl& url, const Key& key, const char* data, size_t length) : + url(url), key(key), data(data), length(length) {} + const QUrl url; + const Key key; + const char* data; + size_t length; + }; + + KTXFilePointer writeFile(Data data); + KTXFilePointer getFile(const QUrl& url); + +protected: + File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; + void evictedFile(const FilePointer& file) override final; + +private: + using Mutex = std::mutex; + using Lock = std::lock_guard; + struct QUrlHasher { std::size_t operator()(QUrl const& url) const { return qHash(url); } }; + + std::unordered_map _urlMap; + Mutex _urlMutex; +}; + +class KTXFile : public File { + Q_OBJECT + +public: + QUrl getUrl() const { return _url; } + +protected: + KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) : + File(key, filepath, length), _url(url) {} + +private: + friend class KTXCache; + + const QUrl _url; +}; + +#endif // hifi_KTXCache_h From c044daf7b22953211197e7cef6d30102b47f0e0f Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 23 Feb 2017 16:27:28 -0500 Subject: [PATCH 03/11] TextureCache --- libraries/model-networking/CMakeLists.txt | 2 +- .../src/model-networking/TextureCache.cpp | 383 +++++++++++------- .../src/model-networking/TextureCache.h | 16 +- libraries/model/src/model/TextureMap.cpp | 88 ---- 4 files changed, 247 insertions(+), 242 deletions(-) diff --git a/libraries/model-networking/CMakeLists.txt b/libraries/model-networking/CMakeLists.txt index ed8cd7b5f9..00aa17ff57 100644 --- a/libraries/model-networking/CMakeLists.txt +++ b/libraries/model-networking/CMakeLists.txt @@ -1,4 +1,4 @@ set(TARGET_NAME model-networking) setup_hifi_library() -link_hifi_libraries(shared networking model fbx) +link_hifi_libraries(shared networking model fbx ktx) diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 1f21e9e78d..4224cf076c 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -18,27 +18,39 @@ #include #include #include + +#if DEBUG_DUMP_TEXTURE_LOADS #include #include +#endif #include #include #include +#include + #include #include #include #include +#include #include "ModelNetworkingLogging.h" #include #include Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image") +Q_LOGGING_CATEGORY(trace_resource_parse_ktx, "trace.resource.parse.ktx") +Q_LOGGING_CATEGORY(trace_resource_cache_ktx, "trace.resource.cache.ktx") -TextureCache::TextureCache() { +const std::string TextureCache::KTX_DIRNAME { "ktx_cache" }; +const std::string TextureCache::KTX_EXT { "ktx" }; + +TextureCache::TextureCache() : + _ktxCache(KTX_DIRNAME, KTX_EXT) { setUnusedResourceCacheSize(0); setObjectName("TextureCache"); @@ -61,7 +73,7 @@ TextureCache::~TextureCache() { // this list taken from Ken Perlin's Improved Noise reference implementation (orig. in Java) at // http://mrl.nyu.edu/~perlin/noise/ -const int permutation[256] = +const int permutation[256] = { 151, 160, 137, 91, 90, 15, 131, 13, 201, 95, 96, 53, 194, 233, 7, 225, 140, 36, 103, 30, 69, 142, 8, 99, 37, 240, 21, 10, 23, 190, 6, 148, @@ -241,7 +253,7 @@ NetworkTexture::TextureLoaderFunc getTextureLoaderForType(NetworkTexture::Type t return NetworkTexture::TextureLoaderFunc(); break; } - + case Type::DEFAULT_TEXTURE: default: { return model::TextureUsage::create2DTextureFromImage; @@ -259,12 +271,32 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, Type type QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, const void* extra) { + KTXFilePointer file = _ktxCache.getFile(url); const TextureExtra* textureExtra = static_cast(extra); auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; - auto content = textureExtra ? textureExtra->content : QByteArray(); - auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; - return QSharedPointer(new NetworkTexture(url, type, content, maxNumPixels), - &Resource::deleter); + + NetworkTexture* texture; + if (file) { + texture = new NetworkTexture(url, type, file); + } else { + auto content = textureExtra ? textureExtra->content : QByteArray(); + auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS; + texture = new NetworkTexture(url, type, content, maxNumPixels); + } + + return QSharedPointer(texture, &Resource::deleter); +} + +NetworkTexture::NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file) : + Resource(url), + _type(type), + _file(file) { + _textureSource = std::make_shared(); + + if (file) { + _startedLoading = true; + QMetaObject::invokeMethod(this, "loadFile", Qt::QueuedConnection); + } } NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) : @@ -278,7 +310,6 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con _loaded = true; } - std::string theName = url.toString().toStdString(); // if we have content, load it after we have our self pointer if (!content.isEmpty()) { _startedLoading = true; @@ -299,149 +330,6 @@ NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const { return getTextureLoaderForType(_type); } - -class ImageReader : public QRunnable { -public: - - ImageReader(const QWeakPointer& resource, const QByteArray& data, - const QUrl& url = QUrl(), int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); - - virtual void run() override; - -private: - static void listSupportedImageFormats(); - - QWeakPointer _resource; - QUrl _url; - QByteArray _content; - int _maxNumPixels; -}; - -void NetworkTexture::downloadFinished(const QByteArray& data) { - // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new ImageReader(_self, data, _url)); -} - -void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, content, _url, _maxNumPixels)); -} - -ImageReader::ImageReader(const QWeakPointer& resource, const QByteArray& data, - const QUrl& url, int maxNumPixels) : - _resource(resource), - _url(url), - _content(data), - _maxNumPixels(maxNumPixels) -{ -#if DEBUG_DUMP_TEXTURE_LOADS - static auto start = usecTimestampNow() / USECS_PER_MSEC; - auto now = usecTimestampNow() / USECS_PER_MSEC - start; - QString urlStr = _url.toString(); - auto dot = urlStr.lastIndexOf("."); - QString outFileName = QString(QCryptographicHash::hash(urlStr.toLocal8Bit(), QCryptographicHash::Md5).toHex()) + urlStr.right(urlStr.length() - dot); - QFile loadRecord("h:/textures/loads.txt"); - loadRecord.open(QFile::Text | QFile::Append | QFile::ReadWrite); - loadRecord.write(QString("%1 %2\n").arg(now).arg(outFileName).toLocal8Bit()); - outFileName = "h:/textures/" + outFileName; - QFileInfo outInfo(outFileName); - if (!outInfo.exists()) { - QFile outFile(outFileName); - outFile.open(QFile::WriteOnly | QFile::Truncate); - outFile.write(data); - outFile.close(); - } -#endif - DependencyManager::get()->incrementStat("PendingProcessing"); -} - -void ImageReader::listSupportedImageFormats() { - static std::once_flag once; - std::call_once(once, []{ - auto supportedFormats = QImageReader::supportedImageFormats(); - qCDebug(modelnetworking) << "List of supported Image formats:" << supportedFormats.join(", "); - }); -} - -void ImageReader::run() { - DependencyManager::get()->decrementStat("PendingProcessing"); - - CounterStat counter("Processing"); - - PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); - auto originalPriority = QThread::currentThread()->priority(); - if (originalPriority == QThread::InheritPriority) { - originalPriority = QThread::NormalPriority; - } - QThread::currentThread()->setPriority(QThread::LowPriority); - Finally restorePriority([originalPriority]{ - QThread::currentThread()->setPriority(originalPriority); - }); - - if (!_resource.data()) { - qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; - return; - } - listSupportedImageFormats(); - - // Help the QImage loader by extracting the image file format from the url filename ext. - // Some tga are not created properly without it. - auto filename = _url.fileName().toStdString(); - auto filenameExtension = filename.substr(filename.find_last_of('.') + 1); - QImage image = QImage::fromData(_content, filenameExtension.c_str()); - - // Note that QImage.format is the pixel format which is different from the "format" of the image file... - auto imageFormat = image.format(); - int imageWidth = image.width(); - int imageHeight = image.height(); - - if (imageWidth == 0 || imageHeight == 0 || imageFormat == QImage::Format_Invalid) { - if (filenameExtension.empty()) { - qCDebug(modelnetworking) << "QImage failed to create from content, no file extension:" << _url; - } else { - qCDebug(modelnetworking) << "QImage failed to create from content" << _url; - } - return; - } - - if (imageWidth * imageHeight > _maxNumPixels) { - float scaleFactor = sqrtf(_maxNumPixels / (float)(imageWidth * imageHeight)); - int originalWidth = imageWidth; - int originalHeight = imageHeight; - imageWidth = (int)(scaleFactor * (float)imageWidth + 0.5f); - imageHeight = (int)(scaleFactor * (float)imageHeight + 0.5f); - QImage newImage = image.scaled(QSize(imageWidth, imageHeight), Qt::IgnoreAspectRatio, Qt::SmoothTransformation); - image.swap(newImage); - qCDebug(modelnetworking) << "Downscale image" << _url - << "from" << originalWidth << "x" << originalHeight - << "to" << imageWidth << "x" << imageHeight; - } - - gpu::TexturePointer texture = nullptr; - { - // Double-check the resource still exists between long operations. - auto resource = _resource.toStrongRef(); - if (!resource) { - qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; - return; - } - - auto url = _url.toString().toStdString(); - - PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffffff00, 0); - texture.reset(resource.dynamicCast()->getTextureLoader()(image, url)); - } - - // Ensure the resource has not been deleted - auto resource = _resource.toStrongRef(); - if (!resource) { - qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; - } else { - QMetaObject::invokeMethod(resource.data(), "setImage", - Q_ARG(gpu::TexturePointer, texture), - Q_ARG(int, imageWidth), Q_ARG(int, imageHeight)); - } -} - void NetworkTexture::setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight) { _originalWidth = originalWidth; @@ -464,3 +352,196 @@ void NetworkTexture::setImage(gpu::TexturePointer texture, int originalWidth, emit networkTextureCreated(qWeakPointerCast (_self)); } + +class Reader : public QRunnable { +public: + Reader(const QWeakPointer& resource, const QUrl& url) : _resource(resource), _url(url) { + DependencyManager::get()->incrementStat("PendingProcessing"); + } + void run() override final { + DependencyManager::get()->decrementStat("PendingProcessing"); + CounterStat counter("Processing"); + + // Run this with low priority, then restore thread priority + auto originalPriority = QThread::currentThread()->priority(); + if (originalPriority == QThread::InheritPriority) { + originalPriority = QThread::NormalPriority; + } + QThread::currentThread()->setPriority(QThread::LowPriority); + Finally restorePriority([originalPriority]{ + QThread::currentThread()->setPriority(originalPriority); + }); + + if (!_resource.lock()) { // to ensure the resource is still needed + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + return; + } + + read(); + } + virtual void read() = 0; + +protected: + QWeakPointer _resource; + QUrl _url; +}; + +class FileReader : public Reader { +public: + FileReader(const QWeakPointer& resource, const QUrl& url) : Reader(resource, url) {} + void read() override final; +}; + +class ImageReader : public Reader { +public: + ImageReader(const QWeakPointer& resource, const QUrl& url, + const QByteArray& data, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); + void read() override final; + +private: + static void listSupportedImageFormats(); + + QByteArray _content; + int _maxNumPixels; +}; + +void NetworkTexture::downloadFinished(const QByteArray& data) { + // send the reader off to the thread pool + QThreadPool::globalInstance()->start(new ImageReader(_self, _url, data)); +} + +void NetworkTexture::loadFile() { + QThreadPool::globalInstance()->start(new FileReader(_self, _url)); +} + +void NetworkTexture::loadContent(const QByteArray& content) { + QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels)); +} + +ImageReader::ImageReader(const QWeakPointer& resource, const QUrl& url, + const QByteArray& data, int maxNumPixels) : + Reader(resource, url), _content(data), _maxNumPixels(maxNumPixels) { + listSupportedImageFormats(); +#if DEBUG_DUMP_TEXTURE_LOADS + static auto start = usecTimestampNow() / USECS_PER_MSEC; + auto now = usecTimestampNow() / USECS_PER_MSEC - start; + QString urlStr = _url.toString(); + auto dot = urlStr.lastIndexOf("."); + QString outFileName = QString(QCryptographicHash::hash(urlStr.toLocal8Bit(), QCryptographicHash::Md5).toHex()) + urlStr.right(urlStr.length() - dot); + QFile loadRecord("h:/textures/loads.txt"); + loadRecord.open(QFile::Text | QFile::Append | QFile::ReadWrite); + loadRecord.write(QString("%1 %2\n").arg(now).arg(outFileName).toLocal8Bit()); + outFileName = "h:/textures/" + outFileName; + QFileInfo outInfo(outFileName); + if (!outInfo.exists()) { + QFile outFile(outFileName); + outFile.open(QFile::WriteOnly | QFile::Truncate); + outFile.write(data); + outFile.close(); + } +#endif +} + +void ImageReader::listSupportedImageFormats() { + static std::once_flag once; + std::call_once(once, []{ + auto supportedFormats = QImageReader::supportedImageFormats(); + qCDebug(modelnetworking) << "List of supported Image formats:" << supportedFormats.join(", "); + }); +} + +void FileReader::read() { + PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0); + + // TODO: + // auto ktx = ktx::KTX::create(); + // auto texture = gpu::Texture::unserialize(getUsage(), getUsageType(), ktx, getSampler()); + // FIXME: do I need to set the file as a backing file here? +} + +void ImageReader::read() { + PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); + + // Help the QImage loader by extracting the image file format from the url filename ext. + // Some tga are not created properly without it. + auto filename = _url.fileName().toStdString(); + auto filenameExtension = filename.substr(filename.find_last_of('.') + 1); + QImage image = QImage::fromData(_content, filenameExtension.c_str()); + int imageWidth = image.width(); + int imageHeight = image.height(); + + // Validate that the image loaded + if (imageWidth == 0 || imageHeight == 0 || image.format() == QImage::Format_Invalid) { + QString reason(filenameExtension.empty() ? "" : "(no file extension)"); + qCWarning(modelnetworking) << "Failed to load" << _url << reason; + return; + } + + // Validate the image is less than _maxNumPixels, and downscale if necessary + if (imageWidth * imageHeight > _maxNumPixels) { + float scaleFactor = sqrtf(_maxNumPixels / (float)(imageWidth * imageHeight)); + int originalWidth = imageWidth; + int originalHeight = imageHeight; + imageWidth = (int)(scaleFactor * (float)imageWidth + 0.5f); + imageHeight = (int)(scaleFactor * (float)imageHeight + 0.5f); + QImage newImage = image.scaled(QSize(imageWidth, imageHeight), Qt::IgnoreAspectRatio, Qt::SmoothTransformation); + image.swap(newImage); + qCDebug(modelnetworking).nospace() << "Downscaled " << _url << " (" << + QSize(originalWidth, originalHeight) << " to " << + QSize(imageWidth, imageHeight) << ")"; + } + + // Load the image into a gpu::Texture + gpu::TexturePointer texture = nullptr; + { + auto resource = _resource.lock(); // to ensure the resource is still needed + if (!resource) { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + return; + } + + auto url = _url.toString().toStdString(); + + PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0); + texture.reset(resource.dynamicCast()->getTextureLoader()(image, url)); + texture->setSource(url); + } + + // Hash the source image to use as a filename for on-disk caching + std::string hash; + { + QCryptographicHash hasher(QCryptographicHash::Md5); + hasher.addData((const char*)image.bits(), image.byteCount() * sizeof(char)); + hash = hasher.result().toHex().toStdString(); + } + + { + auto resource = _resource.lock(); // to ensure the resource is still needed + if (!resource) { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + return; + } + + PROFILE_RANGE_EX(resource_cache_ktx, __FUNCTION__, 0xffffff00, 0); + auto ktx = gpu::Texture::serialize(*texture); + const char* data = reinterpret_cast(ktx->_storage->data()); + size_t length = ktx->_storage->size(); + KTXFilePointer file; + auto& ktxCache = DependencyManager::get()->_ktxCache; + if (!ktx || !(file = ktxCache.writeFile({ _url, hash, data, length }))) { + qCWarning(modelnetworking) << _url << "file cache failed"; + } else { + resource.dynamicCast()->_file = file; + // FIXME: do I need to set the file as a backing file here? + } + } + + auto resource = _resource.toStrongRef(); // to ensure the resource is still needed + if (resource) { + QMetaObject::invokeMethod(resource.data(), "setImage", + Q_ARG(gpu::TexturePointer, texture), + Q_ARG(int, imageWidth), Q_ARG(int, imageHeight)); + } else { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + } +} diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index 749b5a2ebb..f05ab6b220 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -23,6 +23,8 @@ #include #include +#include "KTXCache.h" + const int ABSOLUTE_MAX_TEXTURE_NUM_PIXELS = 8192 * 8192; namespace gpu { @@ -63,6 +65,7 @@ public: typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName); using TextureLoaderFunc = std::function; + NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file); NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels); NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content); @@ -80,17 +83,20 @@ signals: void networkTextureCreated(const QWeakPointer& self); protected: - virtual bool isCacheable() const override { return _loaded; } virtual void downloadFinished(const QByteArray& data) override; Q_INVOKABLE void loadContent(const QByteArray& content); + Q_INVOKABLE void loadFile(); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); private: + friend class ImageReader; + Type _type; TextureLoaderFunc _textureLoader { [](const QImage&, const std::string&){ return nullptr; } }; + KTXFilePointer _file; int _originalWidth { 0 }; int _originalHeight { 0 }; int _width { 0 }; @@ -143,9 +149,15 @@ protected: const void* extra) override; private: + friend class ImageReader; + friend class DilatableNetworkTexture; + TextureCache(); virtual ~TextureCache(); - friend class DilatableNetworkTexture; + + static const std::string KTX_DIRNAME; + static const std::string KTX_EXT; + KTXCache _ktxCache; gpu::TexturePointer _permutationNormalTexture; gpu::TexturePointer _whiteTexture; diff --git a/libraries/model/src/model/TextureMap.cpp b/libraries/model/src/model/TextureMap.cpp index ce602b7b9b..e8d6d4cd6f 100755 --- a/libraries/model/src/model/TextureMap.cpp +++ b/libraries/model/src/model/TextureMap.cpp @@ -85,87 +85,6 @@ QImage processSourceImage(const QImage& srcImage, bool cubemap) { return srcImage; } -gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bool write = true, bool read = true) { - if (!srcTexture) { - return nullptr; - } - - static QString ktxCacheFolder; - static std::once_flag once; - std::call_once(once, [&] { - // Prepare cache directory - static const QString HIFI_KTX_FOLDER("hifi_ktx"); - QString docsLocation = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); - ktxCacheFolder = docsLocation + "/" + HIFI_KTX_FOLDER + "/"; - QFileInfo info(ktxCacheFolder); - if (!info.exists()) { - QDir(docsLocation).mkpath(HIFI_KTX_FOLDER); - } - }); - - - std::string cleanedName = QCryptographicHash::hash(QUrl::toPercentEncoding(name.c_str()), QCryptographicHash::Sha1).toHex().toStdString(); - std::string cacheFilename(ktxCacheFolder.toStdString()); - cacheFilename += "/"; - cacheFilename += cleanedName; - cacheFilename += ".ktx"; - - gpu::Texture* returnedTexture = srcTexture; - { - if (write && !QFileInfo(cacheFilename.c_str()).exists()) { - auto ktxMemory = gpu::Texture::serialize(*srcTexture); - if (ktxMemory) { - const auto& ktxStorage = ktxMemory->getStorage(); - auto header = ktxMemory->getHeader(); - QFile outFile(cacheFilename.c_str()); - if (!outFile.open(QFile::Truncate | QFile::ReadWrite)) { - throw std::runtime_error("Unable to open file"); - } - //auto ktxSize = sizeof(ktx::Header); // ktxStorage->size() - auto ktxSize = ktxStorage->size(); - outFile.resize(ktxSize); - auto dest = outFile.map(0, ktxSize); - memcpy(dest, ktxStorage->data(), ktxSize); - outFile.unmap(dest); - outFile.close(); - } - } - - if (read && QFileInfo(cacheFilename.c_str()).exists()) { -#define DEBUG_KTX_LOADING 0 -#if DEBUG_KTX_LOADING - { - FILE* file = fopen(cacheFilename.c_str(), "rb"); - if (file != nullptr) { - // obtain file size: - fseek (file , 0 , SEEK_END); - auto size = ftell(file); - rewind(file); - - auto storage = std::make_shared(size); - fread(storage->data(), 1, storage->size(), file); - fclose (file); - - //then create a new texture out of the ktx - auto theNewTexure = Texture::unserialize(srcTexture->getUsage(), srcTexture->getUsageType(), - ktx::KTX::create(std::static_pointer_cast(storage)), srcTexture->getSampler()); - - if (theNewTexure) { - returnedTexture = theNewTexure; - delete srcTexture; - } - } - } -#else - ktx::StoragePointer storage = std::make_shared(cacheFilename.c_str()); - auto ktxFile = ktx::KTX::create(storage); - returnedTexture->setKtxBacking(ktxFile); -#endif - } - } - return returnedTexture; -} - void TextureMap::setTextureSource(TextureSourcePointer& textureSource) { _textureSource = textureSource; } @@ -355,7 +274,6 @@ gpu::Texture* TextureUsage::process2DTextureColorFromImage(const QImage& srcImag ::generateMips(theTexture, image, false); } theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture); } return theTexture; @@ -405,7 +323,6 @@ gpu::Texture* TextureUsage::createNormalTextureFromNormalImage(const QImage& src generateMips(theTexture, image, true); theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture, true, true); } return theTexture; @@ -496,7 +413,6 @@ gpu::Texture* TextureUsage::createNormalTextureFromBumpImage(const QImage& srcIm generateMips(theTexture, image, true); theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture, true, false); } return theTexture; @@ -533,7 +449,6 @@ gpu::Texture* TextureUsage::createRoughnessTextureFromImage(const QImage& srcIma generateMips(theTexture, image, true); theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture); } return theTexture; @@ -574,7 +489,6 @@ gpu::Texture* TextureUsage::createRoughnessTextureFromGlossImage(const QImage& s generateMips(theTexture, image, true); theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture); } return theTexture; @@ -612,7 +526,6 @@ gpu::Texture* TextureUsage::createMetallicTextureFromImage(const QImage& srcImag generateMips(theTexture, image, true); theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture); } return theTexture; @@ -946,7 +859,6 @@ gpu::Texture* TextureUsage::processCubeTextureColorFromImage(const QImage& srcIm } theTexture->setSource(srcImageName); - theTexture = cacheTexture(theTexture->source(), theTexture); } } From b03f51dd26c740bbd086994a2422ad6ed2a66548 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 28 Feb 2017 16:08:54 -0500 Subject: [PATCH 04/11] fix double fclose --- libraries/networking/src/FileCache.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index a717546de4..20b78c281a 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -62,7 +62,6 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length FILE* saveFile = fopen(filepath.c_str(), "wb"); if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { file.reset(createFile(key, filepath, length, extra), &fileDeleter); - fclose(saveFile); file->_cache = this; _files[key] = file; _numTotalFiles += 1; From de23a11dee7a01525cbee3a0e8ec41d967e428be Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 28 Feb 2017 17:25:56 -0500 Subject: [PATCH 05/11] wip --- libraries/networking/src/FileCache.cpp | 69 ++++++++++++++++---------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index 20b78c281a..e4c715d020 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -54,13 +54,14 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length // if file already exists, return it FilePointer file = getFile(key); if (file) { - qCWarning(file_cache) << "Attempted to overwrite" << filepath.c_str(); + qCWarning(file_cache, "Attempted to overwrite %", key.c_str()); return file; } // write the new file FILE* saveFile = fopen(filepath.c_str(), "wb"); if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { + qCInfo(file_cache, "Wrote %s", key.c_str()); file.reset(createFile(key, filepath, length, extra), &fileDeleter); file->_cache = this; _files[key] = file; @@ -69,7 +70,7 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length emit dirty(); } else { - qCWarning(file_cache, "Failed to write %s (%s)", filepath.c_str(), strerror(errno)); + qCWarning(file_cache, "Failed to write %s (%s)", key.c_str(), strerror(errno)); errno = 0; } @@ -88,6 +89,7 @@ FilePointer FileCache::getFile(const Key& key) { if (file) { // if it exists, it is active - remove it from the cache removeUnusedFile(file); + qCInfo(file_cache, "Found %s", key.c_str()); emit dirty(); } else { // if not, remove the weak_ptr @@ -115,28 +117,30 @@ std::string FileCache::createDir(const std::string& dirname) { std::string entry; manifest >> entry; persistedEntries.insert(entry); - - // ZZMP: rm - for (const auto& entry : persistedEntries) - qDebug() << "ZZMP" << entry.c_str(); - qDebug() << "ZZMP" << "---"; + qCInfo(file_cache, "Manifest contents: %s", entry.c_str()); } + } else { + qCWarning(file_cache, "Missing manifest"); } + foreach(QString filename, dir.entryList()) { if (persistedEntries.find(filename.toStdString()) == persistedEntries.cend()) { dir.remove(filename); + qCInfo(file_cache) << "Cleaned" << filename; } } + qCDebug(file_cache, "Initiated %s", dirpath.data()); } else { dir.mkpath(dirpath); + qCDebug(file_cache, "Created %s", dirpath.data()); } return dirpath.toStdString(); } std::string FileCache::getFilepath(const Key& key) { - return _dir + key + '.' + _ext; + return _dir + '/' + key + '.' + _ext; } void FileCache::addUnusedFile(const FilePointer file) { @@ -197,34 +201,46 @@ void FileCache::reserve(size_t length) { } void FileCache::clear() { - std::string manifestPath= _dir + MANIFEST_NAME; - FILE* manifest = fopen(manifestPath.c_str(), "wb"); + try { + std::string manifestPath= _dir + '/' + MANIFEST_NAME; + std::ofstream manifest(manifestPath); - Lock lock(_unusedFilesMutex); - for (const auto& val : _unusedFiles) { - const FilePointer& file = val.second; - file->_cache = nullptr; + bool firstEntry = true; - if (_unusedFilesSize > _offlineFilesMaxSize) { - _unusedFilesSize -= file->getLength(); - } else { - std::string key = file->getKey() + '.' + _ext + '\n'; - if (manifest != nullptr && !fwrite(key.c_str(), key.length(), 1, manifest)) { - manifest = nullptr; // to prevent future writes + { + Lock lock(_unusedFilesMutex); + for (const auto& val : _unusedFiles) { + const FilePointer& file = val.second; + file->_cache = nullptr; + + if (_unusedFilesSize > _offlineFilesMaxSize) { + _unusedFilesSize -= file->getLength(); + } else { + if (!firstEntry) { + manifest << '\n'; + } + firstEntry = false; + manifest << file->getKey(); + + file->_shouldPersist = true; + qCInfo(file_cache, "Persisting %s", file->getKey().c_str()); + } } - file->_shouldPersist = true; } - } - if (manifest == nullptr || fclose(manifest) != 0) { + { + Lock lock(_filesMutex); + for (const auto& val : _files) { + const FilePointer& file = val.second + } + } catch (std::exception& e) { + qCWarning(file_cache, "Failed to write manifest (%s)", e.what()); for (const auto& val : _unusedFiles) { val.second->_shouldPersist = false; } - - qCWarning(file_cache, "Failed to write %s (%s)", manifestPath.c_str(), strerror(errno)); - errno = 0; } + Lock lock(_unusedFilesMutex); _unusedFiles.clear(); } @@ -240,6 +256,7 @@ void File::deleter() { File::~File() { QFile file(getFilepath().c_str()); if (file.exists() && !_shouldPersist) { + qCInfo(file_cache, "Unlinked %s", getFilepath().c_str()); file.remove(); } } From a6a0fd3851a46b3e2f9b6c48d6c75a29928ce7ed Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 1 Mar 2017 11:32:11 -0500 Subject: [PATCH 06/11] fix FileCache persistence --- .../src/model-networking/KTXCache.cpp | 1 + libraries/networking/src/FileCache.cpp | 92 ++++++++++--------- libraries/networking/src/FileCache.h | 3 +- 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp index 1ab32698b6..74926d12a1 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.cpp +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -39,6 +39,7 @@ File* KTXCache::createFile(const Key& key, const std::string& filepath, size_t l const QUrl& url = reinterpret_cast(extra)->url; Lock lock(_urlMutex); _urlMap[url] = key; + qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url; return new KTXFile(key, filepath, length, url); } diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index e4c715d020..61712b383b 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -35,8 +35,9 @@ void FileCache::setOfflineFileCacheSize(size_t offlineFilesMaxSize) { FileCache::FileCache(const std::string& dirname, const std::string& ext, QObject* parent) : QObject(parent), - _dir(createDir(dirname)), - _ext(ext) {} + _ext(ext), + _dirname(dirname), + _dir(createDir(_dirname)) {} FileCache::~FileCache() { clear(); @@ -54,14 +55,13 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length // if file already exists, return it FilePointer file = getFile(key); if (file) { - qCWarning(file_cache, "Attempted to overwrite %", key.c_str()); + qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), key.c_str()); return file; } // write the new file FILE* saveFile = fopen(filepath.c_str(), "wb"); if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { - qCInfo(file_cache, "Wrote %s", key.c_str()); file.reset(createFile(key, filepath, length, extra), &fileDeleter); file->_cache = this; _files[key] = file; @@ -70,7 +70,7 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length emit dirty(); } else { - qCWarning(file_cache, "Failed to write %s (%s)", key.c_str(), strerror(errno)); + qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), key.c_str(), strerror(errno)); errno = 0; } @@ -89,7 +89,7 @@ FilePointer FileCache::getFile(const Key& key) { if (file) { // if it exists, it is active - remove it from the cache removeUnusedFile(file); - qCInfo(file_cache, "Found %s", key.c_str()); + qCInfo(file_cache, "[%s] Found %s", _dirname.c_str(), key.c_str()); emit dirty(); } else { // if not, remove the weak_ptr @@ -101,6 +101,7 @@ FilePointer FileCache::getFile(const Key& key) { } File* FileCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { + qCInfo(file_cache, "Wrote %s", key.c_str()); return new File(key, filepath, length); } @@ -116,24 +117,26 @@ std::string FileCache::createDir(const std::string& dirname) { while (manifest.good()) { std::string entry; manifest >> entry; - persistedEntries.insert(entry); - qCInfo(file_cache, "Manifest contents: %s", entry.c_str()); + if (!entry.empty()) { + qCInfo(file_cache, "[%s] Manifest contains %s", _dirname.c_str(), entry.c_str()); + persistedEntries.insert(entry + '.' + _ext); + } } } else { - qCWarning(file_cache, "Missing manifest"); + qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str()); } - foreach(QString filename, dir.entryList()) { + foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) { if (persistedEntries.find(filename.toStdString()) == persistedEntries.cend()) { dir.remove(filename); - qCInfo(file_cache) << "Cleaned" << filename; + qCInfo(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); } } - qCDebug(file_cache, "Initiated %s", dirpath.data()); + qCDebug(file_cache) << "Initiated" << dirpath; } else { dir.mkpath(dirpath); - qCDebug(file_cache, "Created %s", dirpath.data()); + qCDebug(file_cache) << "Created" << dirpath; } return dirpath.toStdString(); @@ -201,43 +204,46 @@ void FileCache::reserve(size_t length) { } void FileCache::clear() { + auto forAllFiles = [&](std::function functor) { + Lock unusedFilesLock(_unusedFilesMutex); + for (const auto& pair : _unusedFiles) { + functor(pair.second); + } + // clear files so they are not reiterated from _files + _unusedFiles.clear(); + unusedFilesLock.unlock(); + + Lock filesLock(_filesMutex); + for (const auto& pair : _files) { + FilePointer file; + if ((file = pair.second.lock())) { + functor(file); + } + } + }; + try { std::string manifestPath= _dir + '/' + MANIFEST_NAME; std::ofstream manifest(manifestPath); - bool firstEntry = true; + forAllFiles([&](const FilePointer& file) { + file->_cache = nullptr; - { - Lock lock(_unusedFilesMutex); - for (const auto& val : _unusedFiles) { - const FilePointer& file = val.second; - file->_cache = nullptr; - - if (_unusedFilesSize > _offlineFilesMaxSize) { - _unusedFilesSize -= file->getLength(); - } else { - if (!firstEntry) { - manifest << '\n'; - } - firstEntry = false; - manifest << file->getKey(); - - file->_shouldPersist = true; - qCInfo(file_cache, "Persisting %s", file->getKey().c_str()); - } + if (_totalFilesSize > _offlineFilesMaxSize) { + _totalFilesSize -= file->getLength(); + } else { + manifest << file->getKey() << '\n'; + file->_shouldPersist = true; + qCInfo(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str()); } - } - - { - Lock lock(_filesMutex); - for (const auto& val : _files) { - const FilePointer& file = val.second - } + }); } catch (std::exception& e) { - qCWarning(file_cache, "Failed to write manifest (%s)", e.what()); - for (const auto& val : _unusedFiles) { - val.second->_shouldPersist = false; - } + qCWarning(file_cache, "[%s] Failed to write manifest (%s)", _dirname.c_str(), e.what()); + + forAllFiles([](const FilePointer& file) { + file->_cache = nullptr; + file->_shouldPersist = false; + }); } Lock lock(_unusedFilesMutex); diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index f068f6e7d5..09e1760ae5 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -98,8 +98,9 @@ private: std::atomic _totalFilesSize { 0 }; std::atomic _unusedFilesSize { 0 }; - std::string _dir; std::string _ext; + std::string _dirname; + std::string _dir; std::unordered_map> _files; Mutex _filesMutex; From e8319f967dd7c376dbf0418aa5c1aefdd3d07664 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 1 Mar 2017 17:28:51 -0500 Subject: [PATCH 07/11] add loading from KTXCache --- .../src/model-networking/KTXCache.cpp | 7 +++- .../src/model-networking/KTXCache.h | 5 +++ .../src/model-networking/TextureCache.cpp | 39 ++++++++++++++----- .../src/model-networking/TextureCache.h | 1 + 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp index 74926d12a1..3e288c6b27 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.cpp +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -11,7 +11,7 @@ #include "KTXCache.h" -#include +#include KTXFilePointer KTXCache::writeFile(Data data) { return std::static_pointer_cast(FileCache::writeFile(data.key, data.data, data.length, (void*)&data)); @@ -48,3 +48,8 @@ void KTXCache::evictedFile(const FilePointer& file) { Lock lock(_urlMutex); _urlMap.erase(url); } + +std::unique_ptr KTXFile::getKTX() const { + ktx::StoragePointer storage = std::make_shared(getFilepath().c_str()); + return ktx::KTX::create(storage); +} diff --git a/libraries/model-networking/src/model-networking/KTXCache.h b/libraries/model-networking/src/model-networking/KTXCache.h index 5b9cb04061..835c28e6db 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.h +++ b/libraries/model-networking/src/model-networking/KTXCache.h @@ -16,6 +16,10 @@ #include +namespace ktx { + class KTX; +} + class KTXFile; using KTXFilePointer = std::shared_ptr; @@ -55,6 +59,7 @@ class KTXFile : public File { public: QUrl getUrl() const { return _url; } + std::unique_ptr getKTX() const; protected: KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) : diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index 4224cf076c..aef0a6d56f 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -451,12 +451,32 @@ void ImageReader::listSupportedImageFormats() { } void FileReader::read() { - PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0); + gpu::TexturePointer texture; + { + auto resource = _resource.lock(); // to ensure the resource is still needed + if (!resource) { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + return; + } + + PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0); + auto ktx = resource.staticCast()->_file->getKTX(); + gpu::Texture::Usage usage; + gpu::TextureUsageType usageType(gpu::TextureUsageType::RESOURCE); + gpu::Sampler sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR); + texture.reset(gpu::Texture::unserialize(usage, usageType, ktx, sampler)); + texture->setKtxBacking(ktx); + } + + auto resource = _resource.lock(); // to ensure the resource is still needed + if (resource) { + QMetaObject::invokeMethod(resource.data(), "setImage", + Q_ARG(gpu::TexturePointer, texture), + Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight())); + } else { + qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; + } - // TODO: - // auto ktx = ktx::KTX::create(); - // auto texture = gpu::Texture::unserialize(getUsage(), getUsageType(), ktx, getSampler()); - // FIXME: do I need to set the file as a backing file here? } void ImageReader::read() { @@ -503,7 +523,7 @@ void ImageReader::read() { auto url = _url.toString().toStdString(); PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0); - texture.reset(resource.dynamicCast()->getTextureLoader()(image, url)); + texture.reset(resource.staticCast()->getTextureLoader()(image, url)); texture->setSource(url); } @@ -531,12 +551,13 @@ void ImageReader::read() { if (!ktx || !(file = ktxCache.writeFile({ _url, hash, data, length }))) { qCWarning(modelnetworking) << _url << "file cache failed"; } else { - resource.dynamicCast()->_file = file; - // FIXME: do I need to set the file as a backing file here? + resource.staticCast()->_file = file; + auto ktx = file->getKTX(); + texture->setKtxBacking(ktx); } } - auto resource = _resource.toStrongRef(); // to ensure the resource is still needed + auto resource = _resource.lock(); // to ensure the resource is still needed if (resource) { QMetaObject::invokeMethod(resource.data(), "setImage", Q_ARG(gpu::TexturePointer, texture), diff --git a/libraries/model-networking/src/model-networking/TextureCache.h b/libraries/model-networking/src/model-networking/TextureCache.h index f05ab6b220..bc3baa6423 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.h +++ b/libraries/model-networking/src/model-networking/TextureCache.h @@ -92,6 +92,7 @@ protected: Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); private: + friend class FileReader; friend class ImageReader; Type _type; From c71e614dd59e9ece4492cffc77621d05d520781a Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 1 Mar 2017 20:18:31 -0500 Subject: [PATCH 08/11] add loading from persisted files --- .../src/model-networking/KTXCache.cpp | 11 ++ .../src/model-networking/KTXCache.h | 5 +- .../src/model-networking/TextureCache.cpp | 3 +- libraries/networking/src/FileCache.cpp | 112 ++++++++++-------- libraries/networking/src/FileCache.h | 16 ++- 5 files changed, 95 insertions(+), 52 deletions(-) diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp index 3e288c6b27..036e520af3 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.cpp +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -43,12 +43,23 @@ File* KTXCache::createFile(const Key& key, const std::string& filepath, size_t l return new KTXFile(key, filepath, length, url); } +File* KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) { + const QUrl url = QString(metadata.c_str()); + _urlMap[url] = key; + qCInfo(file_cache) << "Loaded KTX" << key.c_str() << url; + return new KTXFile(key, filepath, length, url); +} + void KTXCache::evictedFile(const FilePointer& file) { const QUrl url = std::static_pointer_cast(file)->getUrl(); Lock lock(_urlMutex); _urlMap.erase(url); } +std::string KTXFile::getMetadata() const { + return _url.toString().toStdString(); +} + std::unique_ptr KTXFile::getKTX() const { ktx::StoragePointer storage = std::make_shared(getFilepath().c_str()); return ktx::KTX::create(storage); diff --git a/libraries/model-networking/src/model-networking/KTXCache.h b/libraries/model-networking/src/model-networking/KTXCache.h index 835c28e6db..7fe3ed872b 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.h +++ b/libraries/model-networking/src/model-networking/KTXCache.h @@ -27,7 +27,7 @@ class KTXCache : public FileCache { Q_OBJECT public: - KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) {} + KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) { initialize(); } struct Data { Data(const QUrl& url, const Key& key, const char* data, size_t length) : @@ -43,6 +43,7 @@ public: protected: File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; + File* loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final; void evictedFile(const FilePointer& file) override final; private: @@ -65,6 +66,8 @@ protected: KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) : File(key, filepath, length), _url(url) {} + std::string getMetadata() const override final; + private: friend class KTXCache; diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index aef0a6d56f..56325614ff 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -451,6 +451,8 @@ void ImageReader::listSupportedImageFormats() { } void FileReader::read() { + PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } }); + gpu::TexturePointer texture; { auto resource = _resource.lock(); // to ensure the resource is still needed @@ -459,7 +461,6 @@ void FileReader::read() { return; } - PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0); auto ktx = resource.staticCast()->_file->getKTX(); gpu::Texture::Usage usage; gpu::TextureUsageType usageType(gpu::TextureUsageType::RESOURCE); diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index 61712b383b..40a2509b7c 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -12,6 +12,7 @@ #include "FileCache.h" #include +#include #include #include @@ -37,7 +38,7 @@ FileCache::FileCache(const std::string& dirname, const std::string& ext, QObject QObject(parent), _ext(ext), _dirname(dirname), - _dir(createDir(_dirname)) {} + _dirpath(ServerPathUtils::getDataFilePath(dirname.c_str()).toStdString()) {} FileCache::~FileCache() { clear(); @@ -47,7 +48,63 @@ void fileDeleter(File* file) { file->deleter(); } +void FileCache::initialize() { + QDir dir(_dirpath.c_str()); + + if (dir.exists()) { + std::unordered_map> persistedEntries; + if (dir.exists(MANIFEST_NAME.c_str())) { + std::ifstream manifest; + manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString()); + while (manifest.good()) { + std::string key, metadata; + std::getline(manifest, key, '\t'); + std::getline(manifest, metadata, '\n'); + if (!key.empty()) { + qCInfo(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str()); + auto filename = key + '.' + _ext; + persistedEntries[filename] = { key, metadata }; + } + } + } else { + qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str()); + } + + std::unordered_map entries; + + foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) { + const auto& it = persistedEntries.find(filename.toStdString()); + if (it == persistedEntries.cend()) { + // unlink extra files + dir.remove(filename); + qCInfo(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); + } else { + // load existing files + const Key& key = it->second.first; + const std::string& metadata = it->second.second; + const std::string filepath = dir.filePath(filename).toStdString(); + const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg(); + + FilePointer file(loadFile(key, filepath, length, metadata), &fileDeleter); + file->_cache = this; + _files[key] = file; + _numTotalFiles += 1; + _totalFilesSize += length; + } + } + + qCDebug(file_cache, "[%s] Initialized %s", _dirname.c_str(), _dirpath.c_str()); + } else { + dir.mkpath(_dirpath.c_str()); + qCDebug(file_cache, "[%s] Created %s", _dirname.c_str(), _dirpath.c_str()); + } + + _initialized = true; +} + FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length, void* extra) { + assert(_initialized); + std::string filepath = getFilepath(key); Lock lock(_filesMutex); @@ -78,6 +135,8 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length } FilePointer FileCache::getFile(const Key& key) { + assert(_initialized); + FilePointer file; Lock lock(_filesMutex); @@ -100,50 +159,8 @@ FilePointer FileCache::getFile(const Key& key) { return file; } -File* FileCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { - qCInfo(file_cache, "Wrote %s", key.c_str()); - return new File(key, filepath, length); -} - -std::string FileCache::createDir(const std::string& dirname) { - QString dirpath = ServerPathUtils::getDataFilePath(dirname.c_str()); - QDir dir(dirpath); - - if (dir.exists()) { - std::unordered_set persistedEntries; - if (dir.exists(MANIFEST_NAME.c_str())) { - std::ifstream manifest; - manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString()); - while (manifest.good()) { - std::string entry; - manifest >> entry; - if (!entry.empty()) { - qCInfo(file_cache, "[%s] Manifest contains %s", _dirname.c_str(), entry.c_str()); - persistedEntries.insert(entry + '.' + _ext); - } - } - } else { - qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str()); - } - - - foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) { - if (persistedEntries.find(filename.toStdString()) == persistedEntries.cend()) { - dir.remove(filename); - qCInfo(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); - } - } - qCDebug(file_cache) << "Initiated" << dirpath; - } else { - dir.mkpath(dirpath); - qCDebug(file_cache) << "Created" << dirpath; - } - - return dirpath.toStdString(); -} - std::string FileCache::getFilepath(const Key& key) { - return _dir + '/' + key + '.' + _ext; + return _dirpath + '/' + key + '.' + _ext; } void FileCache::addUnusedFile(const FilePointer file) { @@ -223,7 +240,7 @@ void FileCache::clear() { }; try { - std::string manifestPath= _dir + '/' + MANIFEST_NAME; + std::string manifestPath= _dirpath + '/' + MANIFEST_NAME; std::ofstream manifest(manifestPath); forAllFiles([&](const FilePointer& file) { @@ -232,9 +249,10 @@ void FileCache::clear() { if (_totalFilesSize > _offlineFilesMaxSize) { _totalFilesSize -= file->getLength(); } else { - manifest << file->getKey() << '\n'; + manifest << file->getKey() << '\t' << file->getMetadata() << '\n'; file->_shouldPersist = true; - qCInfo(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str()); + qCInfo(file_cache, "[%s] Persisting %s (%s)", + _dirname.c_str(), file->getKey().c_str(), file->getMetadata().c_str()); } }); } catch (std::exception& e) { diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index 09e1760ae5..c5d8ce3fc2 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -73,10 +73,17 @@ signals: void dirty(); protected: + /// must be called after construction to create the cache on the fs and restore persisted files + void initialize(); + FilePointer writeFile(const Key& key, const char* data, size_t length, void* extra); FilePointer getFile(const Key& key); - virtual File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra); + /// create a file (ex.: create a class derived from File and store it in a secondary map with extra->url) + virtual File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0; + /// load a file + virtual File* loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0; + /// take action when a file is evicted from the cache (ex.: evict it from a secondary map) virtual void evictedFile(const FilePointer& file) = 0; private: @@ -85,7 +92,6 @@ private: friend class File; - std::string createDir(const std::string& dirname); std::string getFilepath(const Key& key); void addUnusedFile(const FilePointer file); @@ -100,7 +106,8 @@ private: std::string _ext; std::string _dirname; - std::string _dir; + std::string _dirpath; + bool _initialized { false }; std::unordered_map> _files; Mutex _filesMutex; @@ -133,6 +140,9 @@ protected: // the destructor should handle unlinking of the actual filepath virtual ~File(); + /// get metadata to store with a file between instances (ex.: return the url of a hash) + virtual std::string getMetadata() const = 0; + const std::string _filepath; private: From febeeeca3ac28e4d4e9d9be953016c84da039476 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Wed, 1 Mar 2017 20:58:00 -0500 Subject: [PATCH 09/11] namespace cache::FileCache and use unique_ptr --- .../src/model-networking/KTXCache.cpp | 20 ++++++++++++------- .../src/model-networking/KTXCache.h | 12 ++++++----- libraries/networking/src/FileCache.cpp | 6 ++++-- libraries/networking/src/FileCache.h | 12 +++++++---- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/libraries/model-networking/src/model-networking/KTXCache.cpp b/libraries/model-networking/src/model-networking/KTXCache.cpp index 036e520af3..d0380c7635 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.cpp +++ b/libraries/model-networking/src/model-networking/KTXCache.cpp @@ -13,6 +13,9 @@ #include +using File = cache::File; +using FilePointer = cache::FilePointer; + KTXFilePointer KTXCache::writeFile(Data data) { return std::static_pointer_cast(FileCache::writeFile(data.key, data.data, data.length, (void*)&data)); } @@ -35,19 +38,22 @@ KTXFilePointer KTXCache::getFile(const QUrl& url) { return file; } -File* KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { - const QUrl& url = reinterpret_cast(extra)->url; +std::unique_ptr KTXCache::createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) { Lock lock(_urlMutex); _urlMap[url] = key; - qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url; - return new KTXFile(key, filepath, length, url); + return std::unique_ptr(new KTXFile(key, filepath, length, url)); } -File* KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) { +std::unique_ptr KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { + const QUrl& url = reinterpret_cast(extra)->url; + qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url; + return createKTXFile(key, filepath, length, url); +} + +std::unique_ptr KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) { const QUrl url = QString(metadata.c_str()); - _urlMap[url] = key; qCInfo(file_cache) << "Loaded KTX" << key.c_str() << url; - return new KTXFile(key, filepath, length, url); + return createKTXFile(key, filepath, length, url); } void KTXCache::evictedFile(const FilePointer& file) { diff --git a/libraries/model-networking/src/model-networking/KTXCache.h b/libraries/model-networking/src/model-networking/KTXCache.h index 7fe3ed872b..84dda48ee2 100644 --- a/libraries/model-networking/src/model-networking/KTXCache.h +++ b/libraries/model-networking/src/model-networking/KTXCache.h @@ -23,7 +23,7 @@ namespace ktx { class KTXFile; using KTXFilePointer = std::shared_ptr; -class KTXCache : public FileCache { +class KTXCache : public cache::FileCache { Q_OBJECT public: @@ -42,11 +42,13 @@ public: KTXFilePointer getFile(const QUrl& url); protected: - File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; - File* loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final; - void evictedFile(const FilePointer& file) override final; + std::unique_ptr createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; + std::unique_ptr loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final; + void evictedFile(const cache::FilePointer& file) override final; private: + std::unique_ptr createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url); + using Mutex = std::mutex; using Lock = std::lock_guard; struct QUrlHasher { std::size_t operator()(QUrl const& url) const { return qHash(url); } }; @@ -55,7 +57,7 @@ private: Mutex _urlMutex; }; -class KTXFile : public File { +class KTXFile : public cache::File { Q_OBJECT public: diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index 40a2509b7c..034e24c8cd 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -22,6 +22,8 @@ Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache") +using namespace cache; + static const std::string MANIFEST_NAME = "manifest"; void FileCache::setUnusedFileCacheSize(size_t unusedFilesMaxSize) { @@ -85,7 +87,7 @@ void FileCache::initialize() { const std::string filepath = dir.filePath(filename).toStdString(); const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg(); - FilePointer file(loadFile(key, filepath, length, metadata), &fileDeleter); + FilePointer file(loadFile(key, filepath, length, metadata).release(), &fileDeleter); file->_cache = this; _files[key] = file; _numTotalFiles += 1; @@ -119,7 +121,7 @@ FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length // write the new file FILE* saveFile = fopen(filepath.c_str(), "wb"); if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { - file.reset(createFile(key, filepath, length, extra), &fileDeleter); + file.reset(createFile(key, filepath, length, extra).release(), &fileDeleter); file->_cache = this; _files[key] = file; _numTotalFiles += 1; diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index c5d8ce3fc2..7e751d56be 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -22,6 +22,8 @@ Q_DECLARE_LOGGING_CATEGORY(file_cache) +namespace cache { + class File; using FilePointer = std::shared_ptr; @@ -80,9 +82,9 @@ protected: FilePointer getFile(const Key& key); /// create a file (ex.: create a class derived from File and store it in a secondary map with extra->url) - virtual File* createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0; + virtual std::unique_ptr createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0; /// load a file - virtual File* loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0; + virtual std::unique_ptr loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0; /// take action when a file is evicted from the cache (ex.: evict it from a secondary map) virtual void evictedFile(const FilePointer& file) = 0; @@ -130,6 +132,8 @@ public: Key getKey() const { return _key; } size_t getLength() const { return _length; } + // the destructor should handle unlinking of the actual filepath + virtual ~File(); // overrides should call File::deleter to maintain caching behavior virtual void deleter(); @@ -137,8 +141,6 @@ protected: // when constructed, the file has already been created/written File(const Key& key, const std::string& filepath, size_t length) : _filepath(filepath), _key(key), _length(length) {} - // the destructor should handle unlinking of the actual filepath - virtual ~File(); /// get metadata to store with a file between instances (ex.: return the url of a hash) virtual std::string getMetadata() const = 0; @@ -157,4 +159,6 @@ private: bool _shouldPersist { false }; }; +} + #endif // hifi_FileCache_h From cca1f2fb257fb921bdb168d565e0b0a6873618ce Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 2 Mar 2017 11:28:49 -0500 Subject: [PATCH 10/11] include in FileCache --- libraries/networking/src/FileCache.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index 7e751d56be..b19f2d10cd 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -10,6 +10,7 @@ #ifndef hifi_FileCache_h #define hifi_FileCache_h +#include #include #include #include From 6a7ee4321ba5e17904aaf88c9c650ef773f7857c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Thu, 2 Mar 2017 13:30:51 -0500 Subject: [PATCH 11/11] suppress logs for file_cache --- libraries/networking/src/FileCache.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index 034e24c8cd..c94d0e1b8c 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -20,7 +20,7 @@ #include -Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache") +Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache", QtWarningMsg) using namespace cache; @@ -63,7 +63,7 @@ void FileCache::initialize() { std::getline(manifest, key, '\t'); std::getline(manifest, metadata, '\n'); if (!key.empty()) { - qCInfo(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str()); + qCDebug(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str()); auto filename = key + '.' + _ext; persistedEntries[filename] = { key, metadata }; } @@ -79,7 +79,7 @@ void FileCache::initialize() { if (it == persistedEntries.cend()) { // unlink extra files dir.remove(filename); - qCInfo(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); + qCDebug(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str()); } else { // load existing files const Key& key = it->second.first; @@ -150,7 +150,7 @@ FilePointer FileCache::getFile(const Key& key) { if (file) { // if it exists, it is active - remove it from the cache removeUnusedFile(file); - qCInfo(file_cache, "[%s] Found %s", _dirname.c_str(), key.c_str()); + qCDebug(file_cache, "[%s] Found %s", _dirname.c_str(), key.c_str()); emit dirty(); } else { // if not, remove the weak_ptr @@ -253,7 +253,7 @@ void FileCache::clear() { } else { manifest << file->getKey() << '\t' << file->getMetadata() << '\n'; file->_shouldPersist = true; - qCInfo(file_cache, "[%s] Persisting %s (%s)", + qCDebug(file_cache, "[%s] Persisting %s (%s)", _dirname.c_str(), file->getKey().c_str(), file->getMetadata().c_str()); } });