Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-09 02:56:50 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #9867 from zzmp/ktx/manifest

Browse source 
Remove manifest from FileCache
This commit is contained in:
Brad Davis 2017-03-13 13:21:03 -07:00 committed by GitHub
commit 02189812da
6 changed files with 186 additions and 320 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
libraries/model-networking/src/model-networking/KTXCache.cpp
View file

@ -16,55 +16,27 @@
using File = cache::File; using File = cache::File;
using FilePointer = cache::FilePointer; using FilePointer = cache::FilePointer;
KTXFilePointer KTXCache::writeFile(Data data) { KTXCache::KTXCache(const std::string& dir, const std::string& ext) :
return std::static_pointer_cast<KTXFile>(FileCache::writeFile(data.key, data.data, data.length, (void*)&data)); FileCache(dir, ext) {
initialize();
} }
KTXFilePointer KTXCache::getFile(const QUrl& url) { KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) {
Key key; FilePointer file = FileCache::writeFile(data, std::move(metadata));
{ return std::static_pointer_cast<KTXFile>(file);
Lock lock(_urlMutex);
const auto it = _urlMap.find(url);
if (it != _urlMap.cend()) {
key = it->second;
}
}
KTXFilePointer file;
if (!key.empty()) {
file = std::static_pointer_cast<KTXFile>(FileCache::getFile(key));
}
return file;
} }
std::unique_ptr<File> KTXCache::createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) { KTXFilePointer KTXCache::getFile(const Key& key) {
Lock lock(_urlMutex); return std::static_pointer_cast<KTXFile>(FileCache::getFile(key));
_urlMap[url] = key;
return std::unique_ptr<File>(new KTXFile(key, filepath, length, url));
} }
std::unique_ptr<File> KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) { std::unique_ptr<File> KTXCache::createFile(Metadata&& metadata, const std::string& filepath) {
const QUrl& url = reinterpret_cast<Data*>(extra)->url; qCInfo(file_cache) << "Wrote KTX" << metadata.key.c_str();
qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url; return std::unique_ptr<File>(new KTXFile(std::move(metadata), filepath));
return createKTXFile(key, filepath, length, url);
} }
std::unique_ptr<File> KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) { KTXFile::KTXFile(Metadata&& metadata, const std::string& filepath) :
const QUrl url = QString(metadata.c_str()); cache::File(std::move(metadata), filepath) {}
qCInfo(file_cache) << "Loaded KTX" << key.c_str() << url;
return createKTXFile(key, filepath, length, url);
}
void KTXCache::evictedFile(const FilePointer& file) {
const QUrl url = std::static_pointer_cast<KTXFile>(file)->getUrl();
Lock lock(_urlMutex);
_urlMap.erase(url);
}
std::string KTXFile::getMetadata() const {
return _url.toString().toStdString();
}
std::unique_ptr<ktx::KTX> KTXFile::getKTX() const { std::unique_ptr<ktx::KTX> KTXFile::getKTX() const {
ktx::StoragePointer storage = std::make_shared<storage::FileStorage>(getFilepath().c_str()); ktx::StoragePointer storage = std::make_shared<storage::FileStorage>(getFilepath().c_str());

38
libraries/model-networking/src/model-networking/KTXCache.h
View file

@ -27,53 +27,25 @@ class KTXCache : public cache::FileCache {
Q_OBJECT Q_OBJECT
public: public:
KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) { initialize(); } KTXCache(const std::string& dir, const std::string& ext);
struct Data { KTXFilePointer writeFile(const char* data, Metadata&& metadata);
Data(const QUrl& url, const Key& key, const char* data, size_t length) : KTXFilePointer getFile(const Key& key);
url(url), key(key), data(data), length(length) {}
const QUrl url;
const Key key;
const char* data;
size_t length;
};
KTXFilePointer writeFile(Data data);
KTXFilePointer getFile(const QUrl& url);
protected: protected:
std::unique_ptr<cache::File> createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final; std::unique_ptr<cache::File> createFile(Metadata&& metadata, const std::string& filepath) override final;
std::unique_ptr<cache::File> loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final;
void evictedFile(const cache::FilePointer& file) override final;
private:
std::unique_ptr<cache::File> createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url);
using Mutex = std::mutex;
using Lock = std::lock_guard<Mutex>;
struct QUrlHasher { std::size_t operator()(QUrl const& url) const { return qHash(url); } };
std::unordered_map<QUrl, Key, QUrlHasher> _urlMap;
Mutex _urlMutex;
}; };
class KTXFile : public cache::File { class KTXFile : public cache::File {
Q_OBJECT Q_OBJECT
public: public:
QUrl getUrl() const { return _url; }
std::unique_ptr<ktx::KTX> getKTX() const; std::unique_ptr<ktx::KTX> getKTX() const;
protected: protected:
KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) :
File(key, filepath, length), _url(url) {}
std::string getMetadata() const override final;
private:
friend class KTXCache; friend class KTXCache;
const QUrl _url; KTXFile(Metadata&& metadata, const std::string& filepath);
}; };
#endif // hifi_KTXCache_h #endif // hifi_KTXCache_h

220
libraries/model-networking/src/model-networking/TextureCache.cpp
View file

@ -43,8 +43,8 @@
#include <StatTracker.h> #include <StatTracker.h>
Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image") Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image")
Q_LOGGING_CATEGORY(trace_resource_parse_ktx, "trace.resource.parse.ktx") Q_LOGGING_CATEGORY(trace_resource_parse_image_raw, "trace.resource.parse.image.raw")
Q_LOGGING_CATEGORY(trace_resource_cache_ktx, "trace.resource.cache.ktx") Q_LOGGING_CATEGORY(trace_resource_parse_image_ktx, "trace.resource.parse.image.ktx")
const std::string TextureCache::KTX_DIRNAME { "ktx_cache" }; const std::string TextureCache::KTX_DIRNAME { "ktx_cache" };
const std::string TextureCache::KTX_EXT { "ktx" }; const std::string TextureCache::KTX_EXT { "ktx" };
@ -296,34 +296,14 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, Type type
QSharedPointer<Resource> TextureCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback, QSharedPointer<Resource> TextureCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
const void* extra) { const void* extra) {
KTXFilePointer file = _ktxCache.getFile(url);
const TextureExtra* textureExtra = static_cast<const TextureExtra*>(extra); const TextureExtra* textureExtra = static_cast<const TextureExtra*>(extra);
auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE; auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE;
auto content = textureExtra ? textureExtra->content : QByteArray();
NetworkTexture* texture; auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
if (file) { NetworkTexture* texture = new NetworkTexture(url, type, content, maxNumPixels);
texture = new NetworkTexture(url, type, file);
} else {
auto content = textureExtra ? textureExtra->content : QByteArray();
auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
texture = new NetworkTexture(url, type, content, maxNumPixels);
}
return QSharedPointer<Resource>(texture, &Resource::deleter); return QSharedPointer<Resource>(texture, &Resource::deleter);
} }
NetworkTexture::NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file) :
Resource(url),
_type(type),
_file(file) {
_textureSource = std::make_shared<gpu::TextureSource>();
if (file) {
_startedLoading = true;
QMetaObject::invokeMethod(this, "loadFile", Qt::QueuedConnection);
}
}
NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) : NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) :
Resource(url), Resource(url),
_type(type), _type(type),
@ -342,12 +322,6 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con
} }
} }
NetworkTexture::NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content) :
NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_NUM_PIXELS)
{
_textureLoader = textureLoader;
}
NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const { NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const {
if (_type == CUSTOM_TEXTURE) { if (_type == CUSTOM_TEXTURE) {
return _textureLoader; return _textureLoader;
@ -387,30 +361,8 @@ gpu::TexturePointer NetworkTexture::getFallbackTexture() const {
class Reader : public QRunnable { class Reader : public QRunnable {
public: public:
Reader(const QWeakPointer<Resource>& resource, const QUrl& url) : _resource(resource), _url(url) { Reader(const QWeakPointer<Resource>& resource, const QUrl& url);
DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing"); void run() override final;
}
void run() override final {
DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
CounterStat counter("Processing");
// Run this with low priority, then restore thread priority
auto originalPriority = QThread::currentThread()->priority();
if (originalPriority == QThread::InheritPriority) {
originalPriority = QThread::NormalPriority;
}
QThread::currentThread()->setPriority(QThread::LowPriority);
Finally restorePriority([originalPriority]{
QThread::currentThread()->setPriority(originalPriority);
});
if (!_resource.lock()) { // to ensure the resource is still needed
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
return;
}
read();
}
virtual void read() = 0; virtual void read() = 0;
protected: protected:
@ -418,42 +370,83 @@ protected:
QUrl _url; QUrl _url;
}; };
class FileReader : public Reader {
public:
FileReader(const QWeakPointer<Resource>& resource, const QUrl& url) : Reader(resource, url) {}
void read() override final;
};
class ImageReader : public Reader { class ImageReader : public Reader {
public: public:
ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url, ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url,
const QByteArray& data, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS); const QByteArray& data, const std::string& hash, int maxNumPixels);
void read() override final; void read() override final;
private: private:
static void listSupportedImageFormats(); static void listSupportedImageFormats();
QByteArray _content; QByteArray _content;
std::string _hash;
int _maxNumPixels; int _maxNumPixels;
}; };
void NetworkTexture::downloadFinished(const QByteArray& data) { class KTXReader : public Reader {
// send the reader off to the thread pool public:
QThreadPool::globalInstance()->start(new ImageReader(_self, _url, data)); KTXReader(const QWeakPointer<Resource>& resource, const QUrl& url, const KTXFilePointer& ktxFile);
} void read() override final;
void NetworkTexture::loadFile() { private:
QThreadPool::globalInstance()->start(new FileReader(_self, _url)); KTXFilePointer _file;
};
void NetworkTexture::downloadFinished(const QByteArray& data) {
loadContent(data);
} }
void NetworkTexture::loadContent(const QByteArray& content) { void NetworkTexture::loadContent(const QByteArray& content) {
QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels)); // Hash the source image to for KTX caching
std::string hash;
{
QCryptographicHash hasher(QCryptographicHash::Md5);
hasher.addData(content);
hash = hasher.result().toHex().toStdString();
}
std::unique_ptr<Reader> reader;
KTXFilePointer ktxFile;
if (!_cache.isNull() && (ktxFile = static_cast<TextureCache*>(_cache.data())->_ktxCache.getFile(hash))) {
reader.reset(new KTXReader(_self, _url, ktxFile));
} else {
reader.reset(new ImageReader(_self, _url, content, hash, _maxNumPixels));
}
QThreadPool::globalInstance()->start(reader.release());
}
Reader::Reader(const QWeakPointer<Resource>& resource, const QUrl& url) :
_resource(resource), _url(url) {
DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
}
void Reader::run() {
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
CounterStat counter("Processing");
auto originalPriority = QThread::currentThread()->priority();
if (originalPriority == QThread::InheritPriority) {
originalPriority = QThread::NormalPriority;
}
QThread::currentThread()->setPriority(QThread::LowPriority);
Finally restorePriority([originalPriority]{ QThread::currentThread()->setPriority(originalPriority); });
if (!_resource.data()) {
qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref";
return;
}
read();
} }
ImageReader::ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url, ImageReader::ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url,
const QByteArray& data, int maxNumPixels) : const QByteArray& data, const std::string& hash, int maxNumPixels) :
Reader(resource, url), _content(data), _maxNumPixels(maxNumPixels) { Reader(resource, url), _content(data), _hash(hash), _maxNumPixels(maxNumPixels) {
listSupportedImageFormats(); listSupportedImageFormats();
#if DEBUG_DUMP_TEXTURE_LOADS #if DEBUG_DUMP_TEXTURE_LOADS
static auto start = usecTimestampNow() / USECS_PER_MSEC; static auto start = usecTimestampNow() / USECS_PER_MSEC;
auto now = usecTimestampNow() / USECS_PER_MSEC - start; auto now = usecTimestampNow() / USECS_PER_MSEC - start;
@ -482,36 +475,7 @@ void ImageReader::listSupportedImageFormats() {
}); });
} }
void FileReader::read() {
PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
gpu::TexturePointer texture;
{
auto resource = _resource.lock(); // to ensure the resource is still needed
if (!resource) {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
return;
}
auto ktx = resource.staticCast<NetworkTexture>()->_file->getKTX();
texture.reset(gpu::Texture::unserialize(ktx));
texture->setKtxBacking(ktx);
}
auto resource = _resource.lock(); // to ensure the resource is still needed
if (resource) {
QMetaObject::invokeMethod(resource.data(), "setImage",
Q_ARG(gpu::TexturePointer, texture),
Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight()));
} else {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
}
}
void ImageReader::read() { void ImageReader::read() {
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
// Help the QImage loader by extracting the image file format from the url filename ext. // Help the QImage loader by extracting the image file format from the url filename ext.
// Some tga are not created properly without it. // Some tga are not created properly without it.
auto filename = _url.fileName().toStdString(); auto filename = _url.fileName().toStdString();
@ -541,7 +505,6 @@ void ImageReader::read() {
QSize(imageWidth, imageHeight) << ")"; QSize(imageWidth, imageHeight) << ")";
} }
// Load the image into a gpu::Texture
gpu::TexturePointer texture = nullptr; gpu::TexturePointer texture = nullptr;
{ {
auto resource = _resource.lock(); // to ensure the resource is still needed auto resource = _resource.lock(); // to ensure the resource is still needed
@ -552,37 +515,22 @@ void ImageReader::read() {
auto url = _url.toString().toStdString(); auto url = _url.toString().toStdString();
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0); PROFILE_RANGE_EX(resource_parse_image_raw, __FUNCTION__, 0xffff0000, 0);
// Load the image into a gpu::Texture
auto networkTexture = resource.staticCast<NetworkTexture>(); auto networkTexture = resource.staticCast<NetworkTexture>();
texture.reset(networkTexture->getTextureLoader()(image, url)); texture.reset(networkTexture->getTextureLoader()(image, url));
texture->setSource(url); texture->setSource(url);
if (texture) { if (texture) {
texture->setFallbackTexture(networkTexture->getFallbackTexture()); texture->setFallbackTexture(networkTexture->getFallbackTexture());
} }
}
// Hash the source image to use as a filename for on-disk caching // Save the image into a KTXFile
std::string hash;
{
QCryptographicHash hasher(QCryptographicHash::Md5);
hasher.addData((const char*)image.bits(), image.byteCount() * sizeof(char));
hash = hasher.result().toHex().toStdString();
}
{
auto resource = _resource.lock(); // to ensure the resource is still needed
if (!resource) {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
return;
}
PROFILE_RANGE_EX(resource_cache_ktx, __FUNCTION__, 0xffffff00, 0);
auto ktx = gpu::Texture::serialize(*texture); auto ktx = gpu::Texture::serialize(*texture);
const char* data = reinterpret_cast<const char*>(ktx->_storage->data()); const char* data = reinterpret_cast<const char*>(ktx->_storage->data());
size_t length = ktx->_storage->size(); size_t length = ktx->_storage->size();
KTXFilePointer file; KTXFilePointer file;
auto& ktxCache = DependencyManager::get<TextureCache>()->_ktxCache; auto& ktxCache = DependencyManager::get<TextureCache>()->_ktxCache;
if (!ktx || !(file = ktxCache.writeFile({ _url, hash, data, length }))) { if (!ktx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(_hash, length)))) {
qCWarning(modelnetworking) << _url << "file cache failed"; qCWarning(modelnetworking) << _url << "file cache failed";
} else { } else {
resource.staticCast<NetworkTexture>()->_file = file; resource.staticCast<NetworkTexture>()->_file = file;
@ -600,3 +548,35 @@ void ImageReader::read() {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope"; qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
} }
} }
KTXReader::KTXReader(const QWeakPointer<Resource>& resource, const QUrl& url,
const KTXFilePointer& ktxFile) :
Reader(resource, url), _file(ktxFile) {}
void KTXReader::read() {
PROFILE_RANGE_EX(resource_parse_image_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
gpu::TexturePointer texture;
{
auto resource = _resource.lock(); // to ensure the resource is still needed
if (!resource) {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
return;
}
resource.staticCast<NetworkTexture>()->_file = _file;
auto ktx = _file->getKTX();
texture.reset(gpu::Texture::unserialize(ktx));
texture->setKtxBacking(ktx);
}
auto resource = _resource.lock(); // to ensure the resource is still needed
if (resource) {
QMetaObject::invokeMethod(resource.data(), "setImage",
Q_ARG(gpu::TexturePointer, texture),
Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight()));
} else {
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
}
}

6
libraries/model-networking/src/model-networking/TextureCache.h
View file

@ -65,9 +65,7 @@ public:
typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName); typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName);
using TextureLoaderFunc = std::function<TextureLoader>; using TextureLoaderFunc = std::function<TextureLoader>;
NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file);
NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels); NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels);
NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content);
QString getType() const override { return "NetworkTexture"; } QString getType() const override { return "NetworkTexture"; }
@ -89,11 +87,10 @@ protected:
virtual void downloadFinished(const QByteArray& data) override; virtual void downloadFinished(const QByteArray& data) override;
Q_INVOKABLE void loadContent(const QByteArray& content); Q_INVOKABLE void loadContent(const QByteArray& content);
Q_INVOKABLE void loadFile();
Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight); Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight);
private: private:
friend class FileReader; friend class KTXReader;
friend class ImageReader; friend class ImageReader;
Type _type; Type _type;
@ -149,6 +146,7 @@ protected:
private: private:
friend class ImageReader; friend class ImageReader;
friend class NetworkTexture;
friend class DilatableNetworkTexture; friend class DilatableNetworkTexture;
TextureCache(); TextureCache();

145
libraries/networking/src/FileCache.cpp
View file

@ -60,45 +60,17 @@ void FileCache::initialize() {
QDir dir(_dirpath.c_str()); QDir dir(_dirpath.c_str());
if (dir.exists()) { if (dir.exists()) {
std::unordered_map<std::string, std::pair<Key, std::string>> persistedEntries; auto nameFilters = QStringList(("*." + _ext).c_str());
if (dir.exists(MANIFEST_NAME.c_str())) { auto filters = QDir::Filters(QDir::NoDotAndDotDot | QDir::Files);
std::ifstream manifest; auto sort = QDir::SortFlags(QDir::Time);
manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString()); auto files = dir.entryList(nameFilters, filters, sort);
while (manifest.good()) {
std::string key, metadata;
std::getline(manifest, key, '\t');
std::getline(manifest, metadata, '\n');
if (!key.empty()) {
qCDebug(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str());
auto filename = key + '.' + _ext;
persistedEntries[filename] = { key, metadata };
}
}
} else {
qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str());
}
std::unordered_map<Key, std::string> entries; // load persisted files
foreach(QString filename, files) {
foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) { const Key key = filename.section('.', 0, 1).toStdString();
const auto& it = persistedEntries.find(filename.toStdString()); const std::string filepath = dir.filePath(filename).toStdString();
if (it == persistedEntries.cend()) { const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg();
// unlink extra files addFile(Metadata(key, length), filepath);
dir.remove(filename);
qCDebug(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str());
} else {
// load existing files
const Key& key = it->second.first;
const std::string& metadata = it->second.second;
const std::string filepath = dir.filePath(filename).toStdString();
const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg();
FilePointer file(loadFile(key, filepath, length, metadata).release(), &fileDeleter);
file->_cache = this;
_files[key] = file;
_numTotalFiles += 1;
_totalFilesSize += length;
}
} }
qCDebug(file_cache, "[%s] Initialized %s", _dirname.c_str(), _dirpath.c_str()); qCDebug(file_cache, "[%s] Initialized %s", _dirname.c_str(), _dirpath.c_str());
@ -110,32 +82,40 @@ void FileCache::initialize() {
_initialized = true; _initialized = true;
} }
FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length, void* extra) { FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath) {
FilePointer file(createFile(std::move(metadata), filepath).release(), &fileDeleter);
if (file) {
_numTotalFiles += 1;
_totalFilesSize += file->getLength();
file->_cache = this;
emit dirty();
Lock lock(_filesMutex);
_files[file->getKey()] = file;
}
return file;
}
FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata) {
assert(_initialized); assert(_initialized);
std::string filepath = getFilepath(key); std::string filepath = getFilepath(metadata.key);
Lock lock(_filesMutex); Lock lock(_filesMutex);
// if file already exists, return it // if file already exists, return it
FilePointer file = getFile(key); FilePointer file = getFile(metadata.key);
if (file) { if (file) {
qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), key.c_str()); qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), metadata.key.c_str());
return file; return file;
} }
// write the new file // write the new file
FILE* saveFile = fopen(filepath.c_str(), "wb"); FILE* saveFile = fopen(filepath.c_str(), "wb");
if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) { if (saveFile != nullptr && fwrite(data, metadata.length, 1, saveFile) && fclose(saveFile) == 0) {
file.reset(createFile(key, filepath, length, extra).release(), &fileDeleter); file = addFile(std::move(metadata), filepath);
file->_cache = this;
_files[key] = file;
_numTotalFiles += 1;
_totalFilesSize += length;
emit dirty();
} else { } else {
qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), key.c_str(), strerror(errno)); qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), metadata.key.c_str(), strerror(errno));
errno = 0; errno = 0;
} }
@ -149,7 +129,7 @@ FilePointer FileCache::getFile(const Key& key) {
Lock lock(_filesMutex); Lock lock(_filesMutex);
// check if file already exists // check if file exists
const auto it = _files.find(key); const auto it = _files.find(key);
if (it != _files.cend()) { if (it != _files.cend()) {
file = it->second.lock(); file = it->second.lock();
@ -221,58 +201,22 @@ void FileCache::reserve(size_t length) {
_numUnusedFiles -= 1; _numUnusedFiles -= 1;
_totalFilesSize -= length; _totalFilesSize -= length;
_unusedFilesSize -= length; _unusedFilesSize -= length;
unusedLock.unlock();
evictedFile(file);
unusedLock.lock();
} }
} }
void FileCache::clear() { void FileCache::clear() {
auto forAllFiles = [&](std::function<void(const FilePointer& file)> functor) { Lock unusedFilesLock(_unusedFilesMutex);
Lock unusedFilesLock(_unusedFilesMutex); for (const auto& pair : _unusedFiles) {
for (const auto& pair : _unusedFiles) { auto& file = pair.second;
functor(pair.second); file->_cache = nullptr;
if (_totalFilesSize > _offlineFilesMaxSize) {
_totalFilesSize -= file->getLength();
} else {
file->_shouldPersist = true;
qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str());
} }
// clear files so they are not reiterated from _files
_unusedFiles.clear();
unusedFilesLock.unlock();
Lock filesLock(_filesMutex);
for (const auto& pair : _files) {
FilePointer file;
if ((file = pair.second.lock())) {
functor(file);
}
}
};
try {
std::string manifestPath= _dirpath + '/' + MANIFEST_NAME;
std::ofstream manifest(manifestPath);
forAllFiles([&](const FilePointer& file) {
file->_cache = nullptr;
if (_totalFilesSize > _offlineFilesMaxSize) {
_totalFilesSize -= file->getLength();
} else {
manifest << file->getKey() << '\t' << file->getMetadata() << '\n';
file->_shouldPersist = true;
qCDebug(file_cache, "[%s] Persisting %s (%s)",
_dirname.c_str(), file->getKey().c_str(), file->getMetadata().c_str());
}
});
} catch (std::exception& e) {
qCWarning(file_cache, "[%s] Failed to write manifest (%s)", _dirname.c_str(), e.what());
forAllFiles([](const FilePointer& file) {
file->_cache = nullptr;
file->_shouldPersist = false;
});
} }
Lock lock(_unusedFilesMutex);
_unusedFiles.clear(); _unusedFiles.clear();
} }
@ -285,6 +229,11 @@ void File::deleter() {
} }
} }
File::File(Metadata&& metadata, const std::string& filepath) :
_key(std::move(metadata.key)),
_length(metadata.length),
_filepath(filepath) {}
File::~File() { File::~File() {
QFile file(getFilepath().c_str()); QFile file(getFilepath().c_str());
if (file.exists() && !_shouldPersist) { if (file.exists() && !_shouldPersist) {

43
libraries/networking/src/FileCache.h
View file

@ -52,21 +52,24 @@ public:
// initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg") // initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg")
FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr); FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr);
// precondition: there should be no references to Files when FileCache is destroyed
virtual ~FileCache(); virtual ~FileCache();
// derived classes are left to implement hashing of the files on their own
using Key = std::string; using Key = std::string;
struct Metadata {
Metadata(const Key& key, size_t length) :
key(key), length(length) {}
Key key;
size_t length;
};
// derived classes should implement a setter/getter, for example, for a FileCache backing a network cache: // derived classes should implement a setter/getter, for example, for a FileCache backing a network cache:
// //
// DerivedFilePointer writeFile(const DerivedData& data) { // DerivedFilePointer writeFile(const char* data, DerivedMetadata&& metadata) {
// return writeFile(data->key, data->data, data->length, &data); // return writeFile(data, std::forward(metadata));
// } // }
// //
// DerivedFilePointer getFile(const QUrl& url) { // DerivedFilePointer getFile(const QUrl& url) {
// // assuming storage/removal of url->hash in createFile/evictedFile overrides // auto key = lookup_hash_for(url); // assuming hashing url in create/evictedFile overrides
// auto key = lookup_hash_for(url);
// return getFile(key); // return getFile(key);
// } // }
@ -77,15 +80,11 @@ protected:
/// must be called after construction to create the cache on the fs and restore persisted files /// must be called after construction to create the cache on the fs and restore persisted files
void initialize(); void initialize();
FilePointer writeFile(const Key& key, const char* data, size_t length, void* extra); FilePointer writeFile(const char* data, Metadata&& metadata);
FilePointer getFile(const Key& key); FilePointer getFile(const Key& key);
/// create a file (ex.: create a class derived from File and store it in a secondary map with extra->url) /// create a file
virtual std::unique_ptr<File> createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0; virtual std::unique_ptr<File> createFile(Metadata&& metadata, const std::string& filepath) = 0;
/// load a file
virtual std::unique_ptr<File> loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0;
/// take action when a file is evicted from the cache (ex.: evict it from a secondary map)
virtual void evictedFile(const FilePointer& file) = 0;
private: private:
using Mutex = std::recursive_mutex; using Mutex = std::recursive_mutex;
@ -95,6 +94,7 @@ private:
std::string getFilepath(const Key& key); std::string getFilepath(const Key& key);
FilePointer addFile(Metadata&& metadata, const std::string& filepath);
void addUnusedFile(const FilePointer file); void addUnusedFile(const FilePointer file);
void removeUnusedFile(const FilePointer file); void removeUnusedFile(const FilePointer file);
void reserve(size_t length); void reserve(size_t length);
@ -126,31 +126,26 @@ class File : public QObject {
public: public:
using Key = FileCache::Key; using Key = FileCache::Key;
using Metadata = FileCache::Metadata;
std::string getFilepath() const { return _filepath; }
Key getKey() const { return _key; } Key getKey() const { return _key; }
size_t getLength() const { return _length; } size_t getLength() const { return _length; }
std::string getFilepath() const { return _filepath; }
// the destructor should handle unlinking of the actual filepath
virtual ~File(); virtual ~File();
// overrides should call File::deleter to maintain caching behavior /// overrides should call File::deleter to maintain caching behavior
virtual void deleter(); virtual void deleter();
protected: protected:
// when constructed, the file has already been created/written /// when constructed, the file has already been created/written
File(const Key& key, const std::string& filepath, size_t length) : File(Metadata&& metadata, const std::string& filepath);
_filepath(filepath), _key(key), _length(length) {}
/// get metadata to store with a file between instances (ex.: return the url of a hash)
virtual std::string getMetadata() const = 0;
const std::string _filepath;
private: private:
friend class FileCache; friend class FileCache;
const Key _key; const Key _key;
const size_t _length; const size_t _length;
const std::string _filepath;
FileCache* _cache; FileCache* _cache;
int _LRUKey { 0 }; int _LRUKey { 0 };