mirror of
https://github.com/overte-org/overte.git
synced 2025-04-20 04:44:11 +02:00
Merge pull request #9867 from zzmp/ktx/manifest
Remove manifest from FileCache
This commit is contained in:
Brad Davis
GitHub
commit
02189812da
6 changed files with 186 additions and 320 deletions
|
|
@ -16,55 +16,27 @@
|
|||
using File = cache::File;
|
||||
using FilePointer = cache::FilePointer;
|
||||
|
||||
KTXFilePointer KTXCache::writeFile(Data data) {
|
||||
return std::static_pointer_cast<KTXFile>(FileCache::writeFile(data.key, data.data, data.length, (void*)&data));
|
||||
KTXCache::KTXCache(const std::string& dir, const std::string& ext) :
|
||||
FileCache(dir, ext) {
|
||||
initialize();
|
||||
}
|
||||
|
||||
KTXFilePointer KTXCache::getFile(const QUrl& url) {
|
||||
Key key;
|
||||
{
|
||||
Lock lock(_urlMutex);
|
||||
const auto it = _urlMap.find(url);
|
||||
if (it != _urlMap.cend()) {
|
||||
key = it->second;
|
||||
}
|
||||
}
|
||||
|
||||
KTXFilePointer file;
|
||||
if (!key.empty()) {
|
||||
file = std::static_pointer_cast<KTXFile>(FileCache::getFile(key));
|
||||
}
|
||||
|
||||
return file;
|
||||
KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) {
|
||||
FilePointer file = FileCache::writeFile(data, std::move(metadata));
|
||||
return std::static_pointer_cast<KTXFile>(file);
|
||||
}
|
||||
|
||||
std::unique_ptr<File> KTXCache::createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) {
|
||||
Lock lock(_urlMutex);
|
||||
_urlMap[url] = key;
|
||||
return std::unique_ptr<File>(new KTXFile(key, filepath, length, url));
|
||||
KTXFilePointer KTXCache::getFile(const Key& key) {
|
||||
return std::static_pointer_cast<KTXFile>(FileCache::getFile(key));
|
||||
}
|
||||
|
||||
std::unique_ptr<File> KTXCache::createFile(const Key& key, const std::string& filepath, size_t length, void* extra) {
|
||||
const QUrl& url = reinterpret_cast<Data*>(extra)->url;
|
||||
qCInfo(file_cache) << "Wrote KTX" << key.c_str() << url;
|
||||
return createKTXFile(key, filepath, length, url);
|
||||
std::unique_ptr<File> KTXCache::createFile(Metadata&& metadata, const std::string& filepath) {
|
||||
qCInfo(file_cache) << "Wrote KTX" << metadata.key.c_str();
|
||||
return std::unique_ptr<File>(new KTXFile(std::move(metadata), filepath));
|
||||
}
|
||||
|
||||
std::unique_ptr<File> KTXCache::loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) {
|
||||
const QUrl url = QString(metadata.c_str());
|
||||
qCInfo(file_cache) << "Loaded KTX" << key.c_str() << url;
|
||||
return createKTXFile(key, filepath, length, url);
|
||||
}
|
||||
|
||||
void KTXCache::evictedFile(const FilePointer& file) {
|
||||
const QUrl url = std::static_pointer_cast<KTXFile>(file)->getUrl();
|
||||
Lock lock(_urlMutex);
|
||||
_urlMap.erase(url);
|
||||
}
|
||||
|
||||
std::string KTXFile::getMetadata() const {
|
||||
return _url.toString().toStdString();
|
||||
}
|
||||
KTXFile::KTXFile(Metadata&& metadata, const std::string& filepath) :
|
||||
cache::File(std::move(metadata), filepath) {}
|
||||
|
||||
std::unique_ptr<ktx::KTX> KTXFile::getKTX() const {
|
||||
ktx::StoragePointer storage = std::make_shared<storage::FileStorage>(getFilepath().c_str());
|
||||
|
|
|
|||
|
|
@ -27,53 +27,25 @@ class KTXCache : public cache::FileCache {
|
|||
Q_OBJECT
|
||||
|
||||
public:
|
||||
KTXCache(const std::string& dir, const std::string& ext) : FileCache(dir, ext) { initialize(); }
|
||||
KTXCache(const std::string& dir, const std::string& ext);
|
||||
|
||||
struct Data {
|
||||
Data(const QUrl& url, const Key& key, const char* data, size_t length) :
|
||||
url(url), key(key), data(data), length(length) {}
|
||||
const QUrl url;
|
||||
const Key key;
|
||||
const char* data;
|
||||
size_t length;
|
||||
};
|
||||
|
||||
KTXFilePointer writeFile(Data data);
|
||||
KTXFilePointer getFile(const QUrl& url);
|
||||
KTXFilePointer writeFile(const char* data, Metadata&& metadata);
|
||||
KTXFilePointer getFile(const Key& key);
|
||||
|
||||
protected:
|
||||
std::unique_ptr<cache::File> createFile(const Key& key, const std::string& filepath, size_t length, void* extra) override final;
|
||||
std::unique_ptr<cache::File> loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) override final;
|
||||
void evictedFile(const cache::FilePointer& file) override final;
|
||||
|
||||
private:
|
||||
std::unique_ptr<cache::File> createKTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url);
|
||||
|
||||
using Mutex = std::mutex;
|
||||
using Lock = std::lock_guard<Mutex>;
|
||||
struct QUrlHasher { std::size_t operator()(QUrl const& url) const { return qHash(url); } };
|
||||
|
||||
std::unordered_map<QUrl, Key, QUrlHasher> _urlMap;
|
||||
Mutex _urlMutex;
|
||||
std::unique_ptr<cache::File> createFile(Metadata&& metadata, const std::string& filepath) override final;
|
||||
};
|
||||
|
||||
class KTXFile : public cache::File {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
QUrl getUrl() const { return _url; }
|
||||
std::unique_ptr<ktx::KTX> getKTX() const;
|
||||
|
||||
protected:
|
||||
KTXFile(const Key& key, const std::string& filepath, size_t length, const QUrl& url) :
|
||||
File(key, filepath, length), _url(url) {}
|
||||
|
||||
std::string getMetadata() const override final;
|
||||
|
||||
private:
|
||||
friend class KTXCache;
|
||||
|
||||
const QUrl _url;
|
||||
KTXFile(Metadata&& metadata, const std::string& filepath);
|
||||
};
|
||||
|
||||
#endif // hifi_KTXCache_h
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@
|
|||
#include <StatTracker.h>
|
||||
|
||||
Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image")
|
||||
Q_LOGGING_CATEGORY(trace_resource_parse_ktx, "trace.resource.parse.ktx")
|
||||
Q_LOGGING_CATEGORY(trace_resource_cache_ktx, "trace.resource.cache.ktx")
|
||||
Q_LOGGING_CATEGORY(trace_resource_parse_image_raw, "trace.resource.parse.image.raw")
|
||||
Q_LOGGING_CATEGORY(trace_resource_parse_image_ktx, "trace.resource.parse.image.ktx")
|
||||
|
||||
const std::string TextureCache::KTX_DIRNAME { "ktx_cache" };
|
||||
const std::string TextureCache::KTX_EXT { "ktx" };
|
||||
|
|
@ -296,34 +296,14 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, Type type
|
|||
|
||||
QSharedPointer<Resource> TextureCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
|
||||
const void* extra) {
|
||||
KTXFilePointer file = _ktxCache.getFile(url);
|
||||
const TextureExtra* textureExtra = static_cast<const TextureExtra*>(extra);
|
||||
auto type = textureExtra ? textureExtra->type : Type::DEFAULT_TEXTURE;
|
||||
|
||||
NetworkTexture* texture;
|
||||
if (file) {
|
||||
texture = new NetworkTexture(url, type, file);
|
||||
} else {
|
||||
auto content = textureExtra ? textureExtra->content : QByteArray();
|
||||
auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
|
||||
texture = new NetworkTexture(url, type, content, maxNumPixels);
|
||||
}
|
||||
|
||||
auto content = textureExtra ? textureExtra->content : QByteArray();
|
||||
auto maxNumPixels = textureExtra ? textureExtra->maxNumPixels : ABSOLUTE_MAX_TEXTURE_NUM_PIXELS;
|
||||
NetworkTexture* texture = new NetworkTexture(url, type, content, maxNumPixels);
|
||||
return QSharedPointer<Resource>(texture, &Resource::deleter);
|
||||
}
|
||||
|
||||
NetworkTexture::NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file) :
|
||||
Resource(url),
|
||||
_type(type),
|
||||
_file(file) {
|
||||
_textureSource = std::make_shared<gpu::TextureSource>();
|
||||
|
||||
if (file) {
|
||||
_startedLoading = true;
|
||||
QMetaObject::invokeMethod(this, "loadFile", Qt::QueuedConnection);
|
||||
}
|
||||
}
|
||||
|
||||
NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels) :
|
||||
Resource(url),
|
||||
_type(type),
|
||||
|
|
@ -342,12 +322,6 @@ NetworkTexture::NetworkTexture(const QUrl& url, Type type, const QByteArray& con
|
|||
}
|
||||
}
|
||||
|
||||
NetworkTexture::NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content) :
|
||||
NetworkTexture(url, CUSTOM_TEXTURE, content, ABSOLUTE_MAX_TEXTURE_NUM_PIXELS)
|
||||
{
|
||||
_textureLoader = textureLoader;
|
||||
}
|
||||
|
||||
NetworkTexture::TextureLoaderFunc NetworkTexture::getTextureLoader() const {
|
||||
if (_type == CUSTOM_TEXTURE) {
|
||||
return _textureLoader;
|
||||
|
|
@ -387,30 +361,8 @@ gpu::TexturePointer NetworkTexture::getFallbackTexture() const {
|
|||
|
||||
class Reader : public QRunnable {
|
||||
public:
|
||||
Reader(const QWeakPointer<Resource>& resource, const QUrl& url) : _resource(resource), _url(url) {
|
||||
DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
|
||||
}
|
||||
void run() override final {
|
||||
DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
|
||||
CounterStat counter("Processing");
|
||||
|
||||
// Run this with low priority, then restore thread priority
|
||||
auto originalPriority = QThread::currentThread()->priority();
|
||||
if (originalPriority == QThread::InheritPriority) {
|
||||
originalPriority = QThread::NormalPriority;
|
||||
}
|
||||
QThread::currentThread()->setPriority(QThread::LowPriority);
|
||||
Finally restorePriority([originalPriority]{
|
||||
QThread::currentThread()->setPriority(originalPriority);
|
||||
});
|
||||
|
||||
if (!_resource.lock()) { // to ensure the resource is still needed
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
return;
|
||||
}
|
||||
|
||||
read();
|
||||
}
|
||||
Reader(const QWeakPointer<Resource>& resource, const QUrl& url);
|
||||
void run() override final;
|
||||
virtual void read() = 0;
|
||||
|
||||
protected:
|
||||
|
|
@ -418,42 +370,83 @@ protected:
|
|||
QUrl _url;
|
||||
};
|
||||
|
||||
class FileReader : public Reader {
|
||||
public:
|
||||
FileReader(const QWeakPointer<Resource>& resource, const QUrl& url) : Reader(resource, url) {}
|
||||
void read() override final;
|
||||
};
|
||||
|
||||
class ImageReader : public Reader {
|
||||
public:
|
||||
ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url,
|
||||
const QByteArray& data, int maxNumPixels = ABSOLUTE_MAX_TEXTURE_NUM_PIXELS);
|
||||
const QByteArray& data, const std::string& hash, int maxNumPixels);
|
||||
void read() override final;
|
||||
|
||||
private:
|
||||
static void listSupportedImageFormats();
|
||||
|
||||
QByteArray _content;
|
||||
std::string _hash;
|
||||
int _maxNumPixels;
|
||||
};
|
||||
|
||||
void NetworkTexture::downloadFinished(const QByteArray& data) {
|
||||
// send the reader off to the thread pool
|
||||
QThreadPool::globalInstance()->start(new ImageReader(_self, _url, data));
|
||||
}
|
||||
class KTXReader : public Reader {
|
||||
public:
|
||||
KTXReader(const QWeakPointer<Resource>& resource, const QUrl& url, const KTXFilePointer& ktxFile);
|
||||
void read() override final;
|
||||
|
||||
void NetworkTexture::loadFile() {
|
||||
QThreadPool::globalInstance()->start(new FileReader(_self, _url));
|
||||
private:
|
||||
KTXFilePointer _file;
|
||||
};
|
||||
|
||||
void NetworkTexture::downloadFinished(const QByteArray& data) {
|
||||
loadContent(data);
|
||||
}
|
||||
|
||||
void NetworkTexture::loadContent(const QByteArray& content) {
|
||||
QThreadPool::globalInstance()->start(new ImageReader(_self, _url, content, _maxNumPixels));
|
||||
// Hash the source image to for KTX caching
|
||||
std::string hash;
|
||||
{
|
||||
QCryptographicHash hasher(QCryptographicHash::Md5);
|
||||
hasher.addData(content);
|
||||
hash = hasher.result().toHex().toStdString();
|
||||
}
|
||||
|
||||
std::unique_ptr<Reader> reader;
|
||||
KTXFilePointer ktxFile;
|
||||
if (!_cache.isNull() && (ktxFile = static_cast<TextureCache*>(_cache.data())->_ktxCache.getFile(hash))) {
|
||||
reader.reset(new KTXReader(_self, _url, ktxFile));
|
||||
} else {
|
||||
reader.reset(new ImageReader(_self, _url, content, hash, _maxNumPixels));
|
||||
}
|
||||
|
||||
QThreadPool::globalInstance()->start(reader.release());
|
||||
}
|
||||
|
||||
Reader::Reader(const QWeakPointer<Resource>& resource, const QUrl& url) :
|
||||
_resource(resource), _url(url) {
|
||||
DependencyManager::get<StatTracker>()->incrementStat("PendingProcessing");
|
||||
}
|
||||
|
||||
void Reader::run() {
|
||||
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
|
||||
DependencyManager::get<StatTracker>()->decrementStat("PendingProcessing");
|
||||
CounterStat counter("Processing");
|
||||
|
||||
auto originalPriority = QThread::currentThread()->priority();
|
||||
if (originalPriority == QThread::InheritPriority) {
|
||||
originalPriority = QThread::NormalPriority;
|
||||
}
|
||||
QThread::currentThread()->setPriority(QThread::LowPriority);
|
||||
Finally restorePriority([originalPriority]{ QThread::currentThread()->setPriority(originalPriority); });
|
||||
|
||||
if (!_resource.data()) {
|
||||
qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref";
|
||||
return;
|
||||
}
|
||||
|
||||
read();
|
||||
}
|
||||
|
||||
ImageReader::ImageReader(const QWeakPointer<Resource>& resource, const QUrl& url,
|
||||
const QByteArray& data, int maxNumPixels) :
|
||||
Reader(resource, url), _content(data), _maxNumPixels(maxNumPixels) {
|
||||
const QByteArray& data, const std::string& hash, int maxNumPixels) :
|
||||
Reader(resource, url), _content(data), _hash(hash), _maxNumPixels(maxNumPixels) {
|
||||
listSupportedImageFormats();
|
||||
|
||||
#if DEBUG_DUMP_TEXTURE_LOADS
|
||||
static auto start = usecTimestampNow() / USECS_PER_MSEC;
|
||||
auto now = usecTimestampNow() / USECS_PER_MSEC - start;
|
||||
|
|
@ -482,36 +475,7 @@ void ImageReader::listSupportedImageFormats() {
|
|||
});
|
||||
}
|
||||
|
||||
void FileReader::read() {
|
||||
PROFILE_RANGE_EX(resource_parse_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
|
||||
|
||||
gpu::TexturePointer texture;
|
||||
{
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
if (!resource) {
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
return;
|
||||
}
|
||||
|
||||
auto ktx = resource.staticCast<NetworkTexture>()->_file->getKTX();
|
||||
texture.reset(gpu::Texture::unserialize(ktx));
|
||||
texture->setKtxBacking(ktx);
|
||||
}
|
||||
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
if (resource) {
|
||||
QMetaObject::invokeMethod(resource.data(), "setImage",
|
||||
Q_ARG(gpu::TexturePointer, texture),
|
||||
Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight()));
|
||||
} else {
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void ImageReader::read() {
|
||||
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
|
||||
|
||||
// Help the QImage loader by extracting the image file format from the url filename ext.
|
||||
// Some tga are not created properly without it.
|
||||
auto filename = _url.fileName().toStdString();
|
||||
|
|
@ -541,7 +505,6 @@ void ImageReader::read() {
|
|||
QSize(imageWidth, imageHeight) << ")";
|
||||
}
|
||||
|
||||
// Load the image into a gpu::Texture
|
||||
gpu::TexturePointer texture = nullptr;
|
||||
{
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
|
|
@ -552,37 +515,22 @@ void ImageReader::read() {
|
|||
|
||||
auto url = _url.toString().toStdString();
|
||||
|
||||
PROFILE_RANGE_EX(resource_parse_image, __FUNCTION__, 0xffff0000, 0);
|
||||
PROFILE_RANGE_EX(resource_parse_image_raw, __FUNCTION__, 0xffff0000, 0);
|
||||
// Load the image into a gpu::Texture
|
||||
auto networkTexture = resource.staticCast<NetworkTexture>();
|
||||
texture.reset(networkTexture->getTextureLoader()(image, url));
|
||||
texture->setSource(url);
|
||||
if (texture) {
|
||||
texture->setFallbackTexture(networkTexture->getFallbackTexture());
|
||||
}
|
||||
}
|
||||
|
||||
// Hash the source image to use as a filename for on-disk caching
|
||||
std::string hash;
|
||||
{
|
||||
QCryptographicHash hasher(QCryptographicHash::Md5);
|
||||
hasher.addData((const char*)image.bits(), image.byteCount() * sizeof(char));
|
||||
hash = hasher.result().toHex().toStdString();
|
||||
}
|
||||
|
||||
{
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
if (!resource) {
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
return;
|
||||
}
|
||||
|
||||
PROFILE_RANGE_EX(resource_cache_ktx, __FUNCTION__, 0xffffff00, 0);
|
||||
// Save the image into a KTXFile
|
||||
auto ktx = gpu::Texture::serialize(*texture);
|
||||
const char* data = reinterpret_cast<const char*>(ktx->_storage->data());
|
||||
size_t length = ktx->_storage->size();
|
||||
KTXFilePointer file;
|
||||
auto& ktxCache = DependencyManager::get<TextureCache>()->_ktxCache;
|
||||
if (!ktx || !(file = ktxCache.writeFile({ _url, hash, data, length }))) {
|
||||
if (!ktx || !(file = ktxCache.writeFile(data, KTXCache::Metadata(_hash, length)))) {
|
||||
qCWarning(modelnetworking) << _url << "file cache failed";
|
||||
} else {
|
||||
resource.staticCast<NetworkTexture>()->_file = file;
|
||||
|
|
@ -600,3 +548,35 @@ void ImageReader::read() {
|
|||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
}
|
||||
}
|
||||
|
||||
KTXReader::KTXReader(const QWeakPointer<Resource>& resource, const QUrl& url,
|
||||
const KTXFilePointer& ktxFile) :
|
||||
Reader(resource, url), _file(ktxFile) {}
|
||||
|
||||
void KTXReader::read() {
|
||||
PROFILE_RANGE_EX(resource_parse_image_ktx, __FUNCTION__, 0xffff0000, 0, { { "url", _url.toString() } });
|
||||
|
||||
gpu::TexturePointer texture;
|
||||
{
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
if (!resource) {
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
return;
|
||||
}
|
||||
|
||||
resource.staticCast<NetworkTexture>()->_file = _file;
|
||||
auto ktx = _file->getKTX();
|
||||
texture.reset(gpu::Texture::unserialize(ktx));
|
||||
texture->setKtxBacking(ktx);
|
||||
}
|
||||
|
||||
auto resource = _resource.lock(); // to ensure the resource is still needed
|
||||
if (resource) {
|
||||
QMetaObject::invokeMethod(resource.data(), "setImage",
|
||||
Q_ARG(gpu::TexturePointer, texture),
|
||||
Q_ARG(int, texture->getWidth()), Q_ARG(int, texture->getHeight()));
|
||||
} else {
|
||||
qCDebug(modelnetworking) << _url << "loading stopped; resource out of scope";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,9 +65,7 @@ public:
|
|||
typedef gpu::Texture* TextureLoader(const QImage& image, const std::string& srcImageName);
|
||||
using TextureLoaderFunc = std::function<TextureLoader>;
|
||||
|
||||
NetworkTexture(const QUrl& url, Type type, const KTXFilePointer& file);
|
||||
NetworkTexture(const QUrl& url, Type type, const QByteArray& content, int maxNumPixels);
|
||||
NetworkTexture(const QUrl& url, const TextureLoaderFunc& textureLoader, const QByteArray& content);
|
||||
|
||||
QString getType() const override { return "NetworkTexture"; }
|
||||
|
||||
|
|
@ -89,11 +87,10 @@ protected:
|
|||
virtual void downloadFinished(const QByteArray& data) override;
|
||||
|
||||
Q_INVOKABLE void loadContent(const QByteArray& content);
|
||||
Q_INVOKABLE void loadFile();
|
||||
Q_INVOKABLE void setImage(gpu::TexturePointer texture, int originalWidth, int originalHeight);
|
||||
|
||||
private:
|
||||
friend class FileReader;
|
||||
friend class KTXReader;
|
||||
friend class ImageReader;
|
||||
|
||||
Type _type;
|
||||
|
|
@ -149,6 +146,7 @@ protected:
|
|||
|
||||
private:
|
||||
friend class ImageReader;
|
||||
friend class NetworkTexture;
|
||||
friend class DilatableNetworkTexture;
|
||||
|
||||
TextureCache();
|
||||
|
|
|
|||
|
|
@ -60,45 +60,17 @@ void FileCache::initialize() {
|
|||
QDir dir(_dirpath.c_str());
|
||||
|
||||
if (dir.exists()) {
|
||||
std::unordered_map<std::string, std::pair<Key, std::string>> persistedEntries;
|
||||
if (dir.exists(MANIFEST_NAME.c_str())) {
|
||||
std::ifstream manifest;
|
||||
manifest.open(dir.absoluteFilePath(MANIFEST_NAME.c_str()).toStdString());
|
||||
while (manifest.good()) {
|
||||
std::string key, metadata;
|
||||
std::getline(manifest, key, '\t');
|
||||
std::getline(manifest, metadata, '\n');
|
||||
if (!key.empty()) {
|
||||
qCDebug(file_cache, "[%s] Manifest contains %s (%s)", _dirname.c_str(), key.c_str(), metadata.c_str());
|
||||
auto filename = key + '.' + _ext;
|
||||
persistedEntries[filename] = { key, metadata };
|
||||
}
|
||||
}
|
||||
} else {
|
||||
qCWarning(file_cache, "[%s] Missing manifest", _dirname.c_str());
|
||||
}
|
||||
auto nameFilters = QStringList(("*." + _ext).c_str());
|
||||
auto filters = QDir::Filters(QDir::NoDotAndDotDot | QDir::Files);
|
||||
auto sort = QDir::SortFlags(QDir::Time);
|
||||
auto files = dir.entryList(nameFilters, filters, sort);
|
||||
|
||||
std::unordered_map<Key, std::string> entries;
|
||||
|
||||
foreach(QString filename, dir.entryList(QDir::Filters(QDir::NoDotAndDotDot | QDir::Files))) {
|
||||
const auto& it = persistedEntries.find(filename.toStdString());
|
||||
if (it == persistedEntries.cend()) {
|
||||
// unlink extra files
|
||||
dir.remove(filename);
|
||||
qCDebug(file_cache, "[%s] Cleaned %s", _dirname.c_str(), filename.toStdString().c_str());
|
||||
} else {
|
||||
// load existing files
|
||||
const Key& key = it->second.first;
|
||||
const std::string& metadata = it->second.second;
|
||||
const std::string filepath = dir.filePath(filename).toStdString();
|
||||
const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg();
|
||||
|
||||
FilePointer file(loadFile(key, filepath, length, metadata).release(), &fileDeleter);
|
||||
file->_cache = this;
|
||||
_files[key] = file;
|
||||
_numTotalFiles += 1;
|
||||
_totalFilesSize += length;
|
||||
}
|
||||
// load persisted files
|
||||
foreach(QString filename, files) {
|
||||
const Key key = filename.section('.', 0, 1).toStdString();
|
||||
const std::string filepath = dir.filePath(filename).toStdString();
|
||||
const size_t length = std::ifstream(filepath, std::ios::binary | std::ios::ate).tellg();
|
||||
addFile(Metadata(key, length), filepath);
|
||||
}
|
||||
|
||||
qCDebug(file_cache, "[%s] Initialized %s", _dirname.c_str(), _dirpath.c_str());
|
||||
|
|
@ -110,32 +82,40 @@ void FileCache::initialize() {
|
|||
_initialized = true;
|
||||
}
|
||||
|
||||
FilePointer FileCache::writeFile(const Key& key, const char* data, size_t length, void* extra) {
|
||||
FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath) {
|
||||
FilePointer file(createFile(std::move(metadata), filepath).release(), &fileDeleter);
|
||||
if (file) {
|
||||
_numTotalFiles += 1;
|
||||
_totalFilesSize += file->getLength();
|
||||
file->_cache = this;
|
||||
emit dirty();
|
||||
|
||||
Lock lock(_filesMutex);
|
||||
_files[file->getKey()] = file;
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata) {
|
||||
assert(_initialized);
|
||||
|
||||
std::string filepath = getFilepath(key);
|
||||
std::string filepath = getFilepath(metadata.key);
|
||||
|
||||
Lock lock(_filesMutex);
|
||||
|
||||
// if file already exists, return it
|
||||
FilePointer file = getFile(key);
|
||||
FilePointer file = getFile(metadata.key);
|
||||
if (file) {
|
||||
qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), key.c_str());
|
||||
qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), metadata.key.c_str());
|
||||
return file;
|
||||
}
|
||||
|
||||
// write the new file
|
||||
FILE* saveFile = fopen(filepath.c_str(), "wb");
|
||||
if (saveFile != nullptr && fwrite(data, length, 1, saveFile) && fclose(saveFile) == 0) {
|
||||
file.reset(createFile(key, filepath, length, extra).release(), &fileDeleter);
|
||||
file->_cache = this;
|
||||
_files[key] = file;
|
||||
_numTotalFiles += 1;
|
||||
_totalFilesSize += length;
|
||||
|
||||
emit dirty();
|
||||
if (saveFile != nullptr && fwrite(data, metadata.length, 1, saveFile) && fclose(saveFile) == 0) {
|
||||
file = addFile(std::move(metadata), filepath);
|
||||
} else {
|
||||
qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), key.c_str(), strerror(errno));
|
||||
qCWarning(file_cache, "[%s] Failed to write %s (%s)", _dirname.c_str(), metadata.key.c_str(), strerror(errno));
|
||||
errno = 0;
|
||||
}
|
||||
|
||||
|
|
@ -149,7 +129,7 @@ FilePointer FileCache::getFile(const Key& key) {
|
|||
|
||||
Lock lock(_filesMutex);
|
||||
|
||||
// check if file already exists
|
||||
// check if file exists
|
||||
const auto it = _files.find(key);
|
||||
if (it != _files.cend()) {
|
||||
file = it->second.lock();
|
||||
|
|
@ -221,58 +201,22 @@ void FileCache::reserve(size_t length) {
|
|||
_numUnusedFiles -= 1;
|
||||
_totalFilesSize -= length;
|
||||
_unusedFilesSize -= length;
|
||||
|
||||
unusedLock.unlock();
|
||||
evictedFile(file);
|
||||
unusedLock.lock();
|
||||
}
|
||||
}
|
||||
|
||||
void FileCache::clear() {
|
||||
auto forAllFiles = [&](std::function<void(const FilePointer& file)> functor) {
|
||||
Lock unusedFilesLock(_unusedFilesMutex);
|
||||
for (const auto& pair : _unusedFiles) {
|
||||
functor(pair.second);
|
||||
Lock unusedFilesLock(_unusedFilesMutex);
|
||||
for (const auto& pair : _unusedFiles) {
|
||||
auto& file = pair.second;
|
||||
file->_cache = nullptr;
|
||||
|
||||
if (_totalFilesSize > _offlineFilesMaxSize) {
|
||||
_totalFilesSize -= file->getLength();
|
||||
} else {
|
||||
file->_shouldPersist = true;
|
||||
qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str());
|
||||
}
|
||||
// clear files so they are not reiterated from _files
|
||||
_unusedFiles.clear();
|
||||
unusedFilesLock.unlock();
|
||||
|
||||
Lock filesLock(_filesMutex);
|
||||
for (const auto& pair : _files) {
|
||||
FilePointer file;
|
||||
if ((file = pair.second.lock())) {
|
||||
functor(file);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
std::string manifestPath= _dirpath + '/' + MANIFEST_NAME;
|
||||
std::ofstream manifest(manifestPath);
|
||||
|
||||
forAllFiles([&](const FilePointer& file) {
|
||||
file->_cache = nullptr;
|
||||
|
||||
if (_totalFilesSize > _offlineFilesMaxSize) {
|
||||
_totalFilesSize -= file->getLength();
|
||||
} else {
|
||||
manifest << file->getKey() << '\t' << file->getMetadata() << '\n';
|
||||
file->_shouldPersist = true;
|
||||
qCDebug(file_cache, "[%s] Persisting %s (%s)",
|
||||
_dirname.c_str(), file->getKey().c_str(), file->getMetadata().c_str());
|
||||
}
|
||||
});
|
||||
} catch (std::exception& e) {
|
||||
qCWarning(file_cache, "[%s] Failed to write manifest (%s)", _dirname.c_str(), e.what());
|
||||
|
||||
forAllFiles([](const FilePointer& file) {
|
||||
file->_cache = nullptr;
|
||||
file->_shouldPersist = false;
|
||||
});
|
||||
}
|
||||
|
||||
Lock lock(_unusedFilesMutex);
|
||||
_unusedFiles.clear();
|
||||
}
|
||||
|
||||
|
|
@ -285,6 +229,11 @@ void File::deleter() {
|
|||
}
|
||||
}
|
||||
|
||||
File::File(Metadata&& metadata, const std::string& filepath) :
|
||||
_key(std::move(metadata.key)),
|
||||
_length(metadata.length),
|
||||
_filepath(filepath) {}
|
||||
|
||||
File::~File() {
|
||||
QFile file(getFilepath().c_str());
|
||||
if (file.exists() && !_shouldPersist) {
|
||||
|
|
|
|||
|
|
@ -52,21 +52,24 @@ public:
|
|||
|
||||
// initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg")
|
||||
FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr);
|
||||
// precondition: there should be no references to Files when FileCache is destroyed
|
||||
virtual ~FileCache();
|
||||
|
||||
// derived classes are left to implement hashing of the files on their own
|
||||
using Key = std::string;
|
||||
struct Metadata {
|
||||
Metadata(const Key& key, size_t length) :
|
||||
key(key), length(length) {}
|
||||
Key key;
|
||||
size_t length;
|
||||
};
|
||||
|
||||
// derived classes should implement a setter/getter, for example, for a FileCache backing a network cache:
|
||||
//
|
||||
// DerivedFilePointer writeFile(const DerivedData& data) {
|
||||
// return writeFile(data->key, data->data, data->length, &data);
|
||||
// DerivedFilePointer writeFile(const char* data, DerivedMetadata&& metadata) {
|
||||
// return writeFile(data, std::forward(metadata));
|
||||
// }
|
||||
//
|
||||
// DerivedFilePointer getFile(const QUrl& url) {
|
||||
// // assuming storage/removal of url->hash in createFile/evictedFile overrides
|
||||
// auto key = lookup_hash_for(url);
|
||||
// auto key = lookup_hash_for(url); // assuming hashing url in create/evictedFile overrides
|
||||
// return getFile(key);
|
||||
// }
|
||||
|
||||
|
|
@ -77,15 +80,11 @@ protected:
|
|||
/// must be called after construction to create the cache on the fs and restore persisted files
|
||||
void initialize();
|
||||
|
||||
FilePointer writeFile(const Key& key, const char* data, size_t length, void* extra);
|
||||
FilePointer writeFile(const char* data, Metadata&& metadata);
|
||||
FilePointer getFile(const Key& key);
|
||||
|
||||
/// create a file (ex.: create a class derived from File and store it in a secondary map with extra->url)
|
||||
virtual std::unique_ptr<File> createFile(const Key& key, const std::string& filepath, size_t length, void* extra) = 0;
|
||||
/// load a file
|
||||
virtual std::unique_ptr<File> loadFile(const Key& key, const std::string& filepath, size_t length, const std::string& metadata) = 0;
|
||||
/// take action when a file is evicted from the cache (ex.: evict it from a secondary map)
|
||||
virtual void evictedFile(const FilePointer& file) = 0;
|
||||
/// create a file
|
||||
virtual std::unique_ptr<File> createFile(Metadata&& metadata, const std::string& filepath) = 0;
|
||||
|
||||
private:
|
||||
using Mutex = std::recursive_mutex;
|
||||
|
|
@ -95,6 +94,7 @@ private:
|
|||
|
||||
std::string getFilepath(const Key& key);
|
||||
|
||||
FilePointer addFile(Metadata&& metadata, const std::string& filepath);
|
||||
void addUnusedFile(const FilePointer file);
|
||||
void removeUnusedFile(const FilePointer file);
|
||||
void reserve(size_t length);
|
||||
|
|
@ -126,31 +126,26 @@ class File : public QObject {
|
|||
|
||||
public:
|
||||
using Key = FileCache::Key;
|
||||
using Metadata = FileCache::Metadata;
|
||||
|
||||
std::string getFilepath() const { return _filepath; }
|
||||
Key getKey() const { return _key; }
|
||||
size_t getLength() const { return _length; }
|
||||
std::string getFilepath() const { return _filepath; }
|
||||
|
||||
// the destructor should handle unlinking of the actual filepath
|
||||
virtual ~File();
|
||||
// overrides should call File::deleter to maintain caching behavior
|
||||
/// overrides should call File::deleter to maintain caching behavior
|
||||
virtual void deleter();
|
||||
|
||||
protected:
|
||||
// when constructed, the file has already been created/written
|
||||
File(const Key& key, const std::string& filepath, size_t length) :
|
||||
_filepath(filepath), _key(key), _length(length) {}
|
||||
|
||||
/// get metadata to store with a file between instances (ex.: return the url of a hash)
|
||||
virtual std::string getMetadata() const = 0;
|
||||
|
||||
const std::string _filepath;
|
||||
/// when constructed, the file has already been created/written
|
||||
File(Metadata&& metadata, const std::string& filepath);
|
||||
|
||||
private:
|
||||
friend class FileCache;
|
||||
|
||||
const Key _key;
|
||||
const size_t _length;
|
||||
const std::string _filepath;
|
||||
|
||||
FileCache* _cache;
|
||||
int _LRUKey { 0 };
|
||||
|
|
|
|||
Loading…
Reference in a new issue