From 765bd3890eee93e7af0a5b516f96c22922443065 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Tue, 30 May 2017 10:00:30 -0700 Subject: [PATCH 1/2] Improving test code for KTX library --- libraries/ktx/src/khronos/KHR.h | 2 + tests/ktx/CMakeLists.txt | 20 +-- tests/ktx/src/KtxTests.cpp | 195 ++++++++++++++++++++++++++++ tests/ktx/src/KtxTests.h | 21 +++ tests/ktx/src/main.cpp | 223 -------------------------------- 5 files changed, 225 insertions(+), 236 deletions(-) create mode 100644 tests/ktx/src/KtxTests.cpp create mode 100644 tests/ktx/src/KtxTests.h delete mode 100644 tests/ktx/src/main.cpp diff --git a/libraries/ktx/src/khronos/KHR.h b/libraries/ktx/src/khronos/KHR.h index a98f2cc0d4..98cc1a4736 100644 --- a/libraries/ktx/src/khronos/KHR.h +++ b/libraries/ktx/src/khronos/KHR.h @@ -211,6 +211,8 @@ namespace khronos { template inline uint32_t evalAlignedCompressedBlockCount(uint32_t value) { + enum { val = ALIGNMENT && !(ALIGNMENT & (ALIGNMENT - 1)) }; + static_assert(val, "template parameter ALIGNMENT must be a power of 2"); // FIXME add static assert that ALIGNMENT is a power of 2 static uint32_t ALIGNMENT_REMAINDER = ALIGNMENT - 1; return (value + ALIGNMENT_REMAINDER) / ALIGNMENT; diff --git a/tests/ktx/CMakeLists.txt b/tests/ktx/CMakeLists.txt index 7133c30898..599435a90b 100644 --- a/tests/ktx/CMakeLists.txt +++ b/tests/ktx/CMakeLists.txt @@ -1,15 +1,9 @@ +# Declare dependencies +macro (SETUP_TESTCASE_DEPENDENCIES) + # link in the shared libraries + link_hifi_libraries(shared ktx gpu image) -set(TARGET_NAME ktx-test) - -if (WIN32) - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4049 /ignore:4217") -endif() + package_libraries_for_deployment() +endmacro () -# This is not a testcase -- just set it up as a regular hifi project -setup_hifi_project(Quick Gui OpenGL) -set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Tests/manual-tests/") - -# link in the shared libraries -link_hifi_libraries(shared octree ktx gl gpu gpu-gl render model model-networking networking render-utils fbx entities entities-renderer animation audio avatars script-engine physics image) - -package_libraries_for_deployment() +setup_hifi_testcase() diff --git a/tests/ktx/src/KtxTests.cpp b/tests/ktx/src/KtxTests.cpp new file mode 100644 index 0000000000..94e5d7e8e7 --- /dev/null +++ b/tests/ktx/src/KtxTests.cpp @@ -0,0 +1,195 @@ +// +// Created by Bradley Austin Davis on 2016/07/01 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "KtxTests.h" + +#include + +#include + +#include +#include +#include + + +QTEST_GUILESS_MAIN(KtxTests) + +QString getRootPath() { + static std::once_flag once; + static QString result; + std::call_once(once, [&] { + QFileInfo file(__FILE__); + QDir parent = file.absolutePath(); + result = QDir::cleanPath(parent.currentPath() + "/../../.."); + }); + return result; +} + +void KtxTests::initTestCase() { +} + +void KtxTests::cleanupTestCase() { +} + +void KtxTests::testKhronosCompressionFunctions() { + using namespace khronos::gl::texture; + QCOMPARE(evalAlignedCompressedBlockCount<4>(0), (uint32_t)0x0); + QCOMPARE(evalAlignedCompressedBlockCount<4>(1), (uint32_t)0x1); + QCOMPARE(evalAlignedCompressedBlockCount<4>(4), (uint32_t)0x1); + QCOMPARE(evalAlignedCompressedBlockCount<4>(5), (uint32_t)0x2); + QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x00), (uint32_t)0x00); + QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x01), (uint32_t)0x01); + QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x04), (uint32_t)0x01); + QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x05), (uint32_t)0x02); + QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x1000), (uint32_t)0x400); + + QVERIFY_EXCEPTION_THROWN(evalCompressedBlockCount(InternalFormat::RGBA8, 0x00), std::runtime_error); +} + +void KtxTests::testKtxEvalFunctions() { + QCOMPARE(sizeof(ktx::Header), (size_t)64); + QCOMPARE(ktx::evalPadding(0x0), (uint8_t)0); + QCOMPARE(ktx::evalPadding(0x1), (uint8_t)3); + QCOMPARE(ktx::evalPadding(0x2), (uint8_t)2); + QCOMPARE(ktx::evalPadding(0x3), (uint8_t)1); + QCOMPARE(ktx::evalPadding(0x4), (uint8_t)0); + QCOMPARE(ktx::evalPadding(0x400), (uint8_t)0); + QCOMPARE(ktx::evalPadding(0x401), (uint8_t)3); + QCOMPARE(ktx::evalPaddedSize(0x0), 0x0); + QCOMPARE(ktx::evalPaddedSize(0x1), 0x4); + QCOMPARE(ktx::evalPaddedSize(0x2), 0x4); + QCOMPARE(ktx::evalPaddedSize(0x3), 0x4); + QCOMPARE(ktx::evalPaddedSize(0x4), 0x4); + QCOMPARE(ktx::evalPaddedSize(0x400), 0x400); + QCOMPARE(ktx::evalPaddedSize(0x401), 0x404); + QCOMPARE(ktx::evalAlignedCount((uint32_t)0x0), (uint32_t)0x0); + QCOMPARE(ktx::evalAlignedCount((uint32_t)0x1), (uint32_t)0x1); + QCOMPARE(ktx::evalAlignedCount((uint32_t)0x4), (uint32_t)0x1); + QCOMPARE(ktx::evalAlignedCount((uint32_t)0x5), (uint32_t)0x2); +} + +void KtxTests::testKtxSerialization() { + const QString TEST_IMAGE = getRootPath() + "/scripts/developer/tests/cube_texture.png"; + QImage image(TEST_IMAGE); + gpu::TexturePointer testTexture = image::TextureUsage::process2DTextureColorFromImage(image, TEST_IMAGE.toStdString(), true); + auto ktxMemory = gpu::Texture::serialize(*testTexture); + QVERIFY(ktxMemory.get()); + + // Serialize the image to a file + QTemporaryFile TEST_IMAGE_KTX; + { + const auto& ktxStorage = ktxMemory->getStorage(); + QVERIFY(ktx::KTX::validate(ktxStorage)); + QVERIFY(ktxMemory->isValid()); + + auto& outFile = TEST_IMAGE_KTX; + if (!outFile.open()) { + QFAIL("Unable to open file"); + } + auto ktxSize = ktxStorage->size(); + outFile.resize(ktxSize); + auto dest = outFile.map(0, ktxSize); + memcpy(dest, ktxStorage->data(), ktxSize); + outFile.unmap(dest); + outFile.close(); + } + + + { + auto ktxStorage = std::make_shared(TEST_IMAGE_KTX.fileName()); + QVERIFY(ktx::KTX::validate(ktxStorage)); + auto ktxFile = ktx::KTX::create(ktxStorage); + QVERIFY(ktxFile.get()); + QVERIFY(ktxFile->isValid()); + { + const auto& memStorage = ktxMemory->getStorage(); + const auto& fileStorage = ktxFile->getStorage(); + QVERIFY(memStorage->size() == fileStorage->size()); + QVERIFY(memStorage->data() != fileStorage->data()); + QVERIFY(0 == memcmp(memStorage->data(), fileStorage->data(), memStorage->size())); + QVERIFY(ktxFile->_images.size() == ktxMemory->_images.size()); + auto imageCount = ktxFile->_images.size(); + auto startMemory = ktxMemory->_storage->data(); + auto startFile = ktxFile->_storage->data(); + for (size_t i = 0; i < imageCount; ++i) { + auto memImages = ktxMemory->_images[i]; + auto fileImages = ktxFile->_images[i]; + QVERIFY(memImages._padding == fileImages._padding); + QVERIFY(memImages._numFaces == fileImages._numFaces); + QVERIFY(memImages._imageSize == fileImages._imageSize); + QVERIFY(memImages._faceSize == fileImages._faceSize); + QVERIFY(memImages._faceBytes.size() == memImages._numFaces); + QVERIFY(fileImages._faceBytes.size() == fileImages._numFaces); + auto faceCount = fileImages._numFaces; + for (uint32_t face = 0; face < faceCount; ++face) { + auto memFace = memImages._faceBytes[face]; + auto memOffset = memFace - startMemory; + auto fileFace = fileImages._faceBytes[face]; + auto fileOffset = fileFace - startFile; + QVERIFY(memOffset % 4 == 0); + QVERIFY(memOffset == fileOffset); + } + } + } + } + testTexture->setKtxBacking(TEST_IMAGE_KTX.fileName().toStdString()); +} + +#if 0 + +static const QString TEST_FOLDER { "H:/ktx_cacheold" }; +//static const QString TEST_FOLDER { "C:/Users/bdavis/Git/KTX/testimages" }; + +//static const QString EXTENSIONS { "4bbdf8f786470e4ab3e672d44b8e8df2.ktx" }; +static const QString EXTENSIONS { "*.ktx" }; + +int mainTemp(int, char**) { + qInstallMessageHandler(messageHandler); + auto fileInfoList = QDir { TEST_FOLDER }.entryInfoList(QStringList { EXTENSIONS }); + for (auto fileInfo : fileInfoList) { + qDebug() << fileInfo.filePath(); + std::shared_ptr storage { new storage::FileStorage { fileInfo.filePath() } }; + + if (!ktx::KTX::validate(storage)) { + qDebug() << "KTX invalid"; + } + + auto ktxFile = ktx::KTX::create(storage); + ktx::KTXDescriptor ktxDescriptor = ktxFile->toDescriptor(); + + qDebug() << "Contains " << ktxDescriptor.keyValues.size() << " key value pairs"; + for (const auto& kv : ktxDescriptor.keyValues) { + qDebug() << "\t" << kv._key.c_str(); + } + + auto offsetToMinMipKV = ktxDescriptor.getValueOffsetForKey(ktx::HIFI_MIN_POPULATED_MIP_KEY); + if (offsetToMinMipKV) { + auto data = storage->data() + ktx::KTX_HEADER_SIZE + offsetToMinMipKV; + auto minMipLevelAvailable = *data; + qDebug() << "\tMin mip available " << minMipLevelAvailable; + assert(minMipLevelAvailable < ktxDescriptor.header.numberOfMipmapLevels); + } + auto storageSize = storage->size(); + for (const auto& faceImageDesc : ktxDescriptor.images) { + //assert(0 == (faceImageDesc._faceSize % 4)); + for (const auto& faceOffset : faceImageDesc._faceOffsets) { + assert(0 == (faceOffset % 4)); + auto faceEndOffset = faceOffset + faceImageDesc._faceSize; + assert(faceEndOffset <= storageSize); + } + } + + for (const auto& faceImage : ktxFile->_images) { + for (const ktx::Byte* faceBytes : faceImage._faceBytes) { + assert(0 == (reinterpret_cast(faceBytes) % 4)); + } + } + } + return 0; +} +#endif diff --git a/tests/ktx/src/KtxTests.h b/tests/ktx/src/KtxTests.h new file mode 100644 index 0000000000..5627dc313d --- /dev/null +++ b/tests/ktx/src/KtxTests.h @@ -0,0 +1,21 @@ +// +// Created by Bradley Austin Davis on 2016/07/01 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +class KtxTests : public QObject { + Q_OBJECT +private slots: + void initTestCase(); + void cleanupTestCase(); + void testKtxEvalFunctions(); + void testKhronosCompressionFunctions(); + void testKtxSerialization(); +}; + + diff --git a/tests/ktx/src/main.cpp b/tests/ktx/src/main.cpp deleted file mode 100644 index 3b62b89948..0000000000 --- a/tests/ktx/src/main.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// -// Created by Bradley Austin Davis on 2016/07/01 -// Copyright 2014 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -QSharedPointer logger; - -gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bool write = true, bool read = true); - - -void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) { - QString logMessage = LogHandler::getInstance().printMessage((LogMsgType)type, context, message); - - if (!logMessage.isEmpty()) { -#ifdef Q_OS_WIN - OutputDebugStringA(logMessage.toLocal8Bit().constData()); - OutputDebugStringA("\n"); -#endif - if (logger) { - logger->addMessage(qPrintable(logMessage + "\n")); - } - } -} - -const char * LOG_FILTER_RULES = R"V0G0N( -hifi.gpu=true -)V0G0N"; - -QString getRootPath() { - static std::once_flag once; - static QString result; - std::call_once(once, [&] { - QFileInfo file(__FILE__); - QDir parent = file.absolutePath(); - result = QDir::cleanPath(parent.currentPath() + "/../../.."); - }); - return result; -} - -const QString TEST_IMAGE = getRootPath() + "/scripts/developer/tests/cube_texture.png"; -const QString TEST_IMAGE_KTX = getRootPath() + "/scripts/developer/tests/cube_texture.ktx"; - -int main(int argc, char** argv) { - QApplication app(argc, argv); - QCoreApplication::setApplicationName("KTX"); - QCoreApplication::setOrganizationName("High Fidelity"); - QCoreApplication::setOrganizationDomain("highfidelity.com"); - logger.reset(new FileLogger()); - - Q_ASSERT(ktx::evalPadding(0) == 0); - Q_ASSERT(ktx::evalPadding(1) == 3); - Q_ASSERT(ktx::evalPadding(2) == 2); - Q_ASSERT(ktx::evalPadding(3) == 1); - Q_ASSERT(ktx::evalPadding(4) == 0); - Q_ASSERT(ktx::evalPadding(1024) == 0); - Q_ASSERT(ktx::evalPadding(1025) == 3); - Q_ASSERT(ktx::evalPaddedSize(0) == 0); - Q_ASSERT(ktx::evalPaddedSize(1) == 4); - Q_ASSERT(ktx::evalPaddedSize(2) == 4); - Q_ASSERT(ktx::evalPaddedSize(3) == 4); - Q_ASSERT(ktx::evalPaddedSize(4) == 4); - Q_ASSERT(ktx::evalPaddedSize(1024) == 1024); - Q_ASSERT(ktx::evalPaddedSize(1025) == 1028); - Q_ASSERT(sizeof(ktx::Header) == 12 + (sizeof(uint32_t) * 13)); - - DependencyManager::set(); - qInstallMessageHandler(messageHandler); - QLoggingCategory::setFilterRules(LOG_FILTER_RULES); - - QImage image(TEST_IMAGE); - gpu::TexturePointer testTexture = image::TextureUsage::process2DTextureColorFromImage(image, TEST_IMAGE.toStdString(), true); - - auto ktxMemory = gpu::Texture::serialize(*testTexture); - { - const auto& ktxStorage = ktxMemory->getStorage(); - Q_ASSERT_X(ktx::KTX::validate(ktxStorage), __FUNCTION__, "KTX storage validation failed"); - Q_ASSERT_X(ktxMemory->isValid(), __FUNCTION__, "KTX self-validation failed"); - QSaveFile outFile(TEST_IMAGE_KTX); - if (!outFile.open(QFile::WriteOnly)) { - throw std::runtime_error("Unable to open file"); - } - auto ktxSize = ktxStorage->size(); - outFile.resize(ktxSize); - auto dest = outFile.map(0, ktxSize); - memcpy(dest, ktxStorage->data(), ktxSize); - outFile.unmap(dest); - outFile.commit(); - } - - { - auto ktxFile = ktx::KTX::create(std::shared_ptr(new storage::FileStorage(TEST_IMAGE_KTX))); - { - const auto& memStorage = ktxMemory->getStorage(); - const auto& fileStorage = ktxFile->getStorage(); - Q_ASSERT(memStorage->size() == fileStorage->size()); - Q_ASSERT(memStorage->data() != fileStorage->data()); - Q_ASSERT(0 == memcmp(memStorage->data(), fileStorage->data(), memStorage->size())); - Q_ASSERT(ktxFile->_images.size() == ktxMemory->_images.size()); - auto imageCount = ktxFile->_images.size(); - auto startMemory = ktxMemory->_storage->data(); - auto startFile = ktxFile->_storage->data(); - for (size_t i = 0; i < imageCount; ++i) { - auto memImages = ktxMemory->_images[i]; - auto fileImages = ktxFile->_images[i]; - Q_ASSERT(memImages._padding == fileImages._padding); - Q_ASSERT(memImages._numFaces == fileImages._numFaces); - Q_ASSERT(memImages._imageSize == fileImages._imageSize); - Q_ASSERT(memImages._faceSize == fileImages._faceSize); - Q_ASSERT(memImages._faceBytes.size() == memImages._numFaces); - Q_ASSERT(fileImages._faceBytes.size() == fileImages._numFaces); - auto faceCount = fileImages._numFaces; - for (uint32_t face = 0; face < faceCount; ++face) { - auto memFace = memImages._faceBytes[face]; - auto memOffset = memFace - startMemory; - auto fileFace = fileImages._faceBytes[face]; - auto fileOffset = fileFace - startFile; - Q_ASSERT(memOffset % 4 == 0); - Q_ASSERT(memOffset == fileOffset); - } - } - } - } - testTexture->setKtxBacking(TEST_IMAGE_KTX.toStdString()); - return 0; -} - -#if 0 -static const QString TEST_FOLDER { "H:/ktx_cacheold" }; -//static const QString TEST_FOLDER { "C:/Users/bdavis/Git/KTX/testimages" }; - -//static const QString EXTENSIONS { "4bbdf8f786470e4ab3e672d44b8e8df2.ktx" }; -static const QString EXTENSIONS { "*.ktx" }; - -int mainTemp(int, char**) { - qInstallMessageHandler(messageHandler); - auto fileInfoList = QDir { TEST_FOLDER }.entryInfoList(QStringList { EXTENSIONS }); - for (auto fileInfo : fileInfoList) { - qDebug() << fileInfo.filePath(); - std::shared_ptr storage { new storage::FileStorage { fileInfo.filePath() } }; - - if (!ktx::KTX::validate(storage)) { - qDebug() << "KTX invalid"; - } - - auto ktxFile = ktx::KTX::create(storage); - ktx::KTXDescriptor ktxDescriptor = ktxFile->toDescriptor(); - - qDebug() << "Contains " << ktxDescriptor.keyValues.size() << " key value pairs"; - for (const auto& kv : ktxDescriptor.keyValues) { - qDebug() << "\t" << kv._key.c_str(); - } - - auto offsetToMinMipKV = ktxDescriptor.getValueOffsetForKey(ktx::HIFI_MIN_POPULATED_MIP_KEY); - if (offsetToMinMipKV) { - auto data = storage->data() + ktx::KTX_HEADER_SIZE + offsetToMinMipKV; - auto minMipLevelAvailable = *data; - qDebug() << "\tMin mip available " << minMipLevelAvailable; - assert(minMipLevelAvailable < ktxDescriptor.header.numberOfMipmapLevels); - } - auto storageSize = storage->size(); - for (const auto& faceImageDesc : ktxDescriptor.images) { - //assert(0 == (faceImageDesc._faceSize % 4)); - for (const auto& faceOffset : faceImageDesc._faceOffsets) { - assert(0 == (faceOffset % 4)); - auto faceEndOffset = faceOffset + faceImageDesc._faceSize; - assert(faceEndOffset <= storageSize); - } - } - - for (const auto& faceImage : ktxFile->_images) { - for (const ktx::Byte* faceBytes : faceImage._faceBytes) { - assert(0 == (reinterpret_cast(faceBytes) % 4)); - } - } - } - return 0; -} -#endif - -#include "main.moc" - From a77491ccb1128679fbe92f47d8caa9eb75b08b80 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Thu, 25 May 2017 10:33:34 -0700 Subject: [PATCH 2/2] FileCache refactoring and tests --- libraries/networking/src/FileCache.cpp | 173 +++++++++++++++------- libraries/networking/src/FileCache.h | 52 ++++--- libraries/shared/src/NumericalConstants.h | 3 + tests/networking/src/FileCacheTests.cpp | 170 +++++++++++++++++++++ tests/networking/src/FileCacheTests.h | 30 ++++ 5 files changed, 353 insertions(+), 75 deletions(-) create mode 100644 tests/networking/src/FileCacheTests.cpp create mode 100644 tests/networking/src/FileCacheTests.h diff --git a/libraries/networking/src/FileCache.cpp b/libraries/networking/src/FileCache.cpp index 8f3509d8f3..6cb8cd8f7c 100644 --- a/libraries/networking/src/FileCache.cpp +++ b/libraries/networking/src/FileCache.cpp @@ -11,52 +11,81 @@ #include "FileCache.h" -#include -#include -#include -#include -#include -#include +#include +#include +#include + +#include +#include +#include +#include #include +#include +#ifdef Q_OS_WIN +#include +#else +#include +#endif + +#ifdef NDEBUG Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache", QtWarningMsg) - +#else +Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache") +#endif using namespace cache; -static const std::string MANIFEST_NAME = "manifest"; +static const char DIR_SEP = '/'; +static const char EXT_SEP = '.'; -static const size_t BYTES_PER_MEGABYTES = 1024 * 1024; -static const size_t BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES; -const size_t FileCache::DEFAULT_UNUSED_MAX_SIZE = 5 * BYTES_PER_GIGABYTES; // 5GB -const size_t FileCache::MAX_UNUSED_MAX_SIZE = 100 * BYTES_PER_GIGABYTES; // 100GB -const size_t FileCache::DEFAULT_OFFLINE_MAX_SIZE = 2 * BYTES_PER_GIGABYTES; // 2GB +const size_t FileCache::DEFAULT_MAX_SIZE { GB_TO_BYTES(5) }; +const size_t FileCache::MAX_MAX_SIZE { GB_TO_BYTES(100) }; +const size_t FileCache::DEFAULT_MIN_FREE_STORAGE_SPACE { GB_TO_BYTES(1) }; -void FileCache::setUnusedFileCacheSize(size_t unusedFilesMaxSize) { - _unusedFilesMaxSize = std::min(unusedFilesMaxSize, MAX_UNUSED_MAX_SIZE); - reserve(0); + +std::string getCacheName(const std::string& dirname_str) { + QString dirname { dirname_str.c_str() }; + QDir dir(dirname); + if (!dir.isAbsolute()) { + return dirname_str; + } + return dir.dirName().toStdString(); +} + +std::string getCachePath(const std::string& dirname_str) { + QString dirname { dirname_str.c_str() }; + QDir dir(dirname); + if (dir.isAbsolute()) { + return dirname_str; + } + return PathUtils::getAppLocalDataFilePath(dirname).toStdString(); +} + +void FileCache::setMinFreeSize(size_t size) { + _minFreeSpaceSize = size; + clean(); emit dirty(); } -void FileCache::setOfflineFileCacheSize(size_t offlineFilesMaxSize) { - _offlineFilesMaxSize = std::min(offlineFilesMaxSize, MAX_UNUSED_MAX_SIZE); +void FileCache::setMaxSize(size_t maxSize) { + _maxSize = std::min(maxSize, MAX_MAX_SIZE); + clean(); + emit dirty(); } FileCache::FileCache(const std::string& dirname, const std::string& ext, QObject* parent) : QObject(parent), _ext(ext), - _dirname(dirname), - _dirpath(PathUtils::getAppLocalDataFilePath(dirname.c_str()).toStdString()) {} + _dirname(getCacheName(dirname)), + _dirpath(getCachePath(dirname)) { +} FileCache::~FileCache() { clear(); } -void fileDeleter(File* file) { - file->deleter(); -} - void FileCache::initialize() { QDir dir(_dirpath.c_str()); @@ -84,7 +113,7 @@ void FileCache::initialize() { } FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath) { - FilePointer file(createFile(std::move(metadata), filepath).release(), &fileDeleter); + FilePointer file(createFile(std::move(metadata), filepath).release(), std::mem_fn(&File::deleter)); if (file) { _numTotalFiles += 1; _totalFilesSize += file->getLength(); @@ -141,6 +170,7 @@ FilePointer FileCache::getFile(const Key& key) { if (it != _files.cend()) { file = it->second.lock(); if (file) { + file->touch(); // if it exists, it is active - remove it from the cache removeUnusedFile(file); qCDebug(file_cache, "[%s] Found %s", _dirname.c_str(), key.c_str()); @@ -155,44 +185,74 @@ FilePointer FileCache::getFile(const Key& key) { } std::string FileCache::getFilepath(const Key& key) { - return _dirpath + '/' + key + '.' + _ext; + return _dirpath + DIR_SEP + key + EXT_SEP + _ext; } -void FileCache::addUnusedFile(const FilePointer file) { +void FileCache::addUnusedFile(const FilePointer& file) { { Lock lock(_filesMutex); _files[file->getKey()] = file; } - reserve(file->getLength()); - file->_LRUKey = ++_lastLRUKey; - { Lock lock(_unusedFilesMutex); - _unusedFiles.insert({ file->_LRUKey, file }); + _unusedFiles.insert(file); _numUnusedFiles += 1; _unusedFilesSize += file->getLength(); } + clean(); emit dirty(); } -void FileCache::removeUnusedFile(const FilePointer file) { +void FileCache::removeUnusedFile(const FilePointer& file) { Lock lock(_unusedFilesMutex); - const auto it = _unusedFiles.find(file->_LRUKey); - if (it != _unusedFiles.cend()) { - _unusedFiles.erase(it); + if (_unusedFiles.erase(file)) { _numUnusedFiles -= 1; _unusedFilesSize -= file->getLength(); } } -void FileCache::reserve(size_t length) { +size_t FileCache::getOverbudgetAmount() const { + size_t result = 0; + + size_t currentFreeSpace = QStorageInfo(_dirpath.c_str()).bytesFree(); + if (_minFreeSpaceSize > currentFreeSpace) { + result = _minFreeSpaceSize - currentFreeSpace; + } + + if (_totalFilesSize > _maxSize) { + result = std::max(_totalFilesSize - _maxSize, result); + } + + return result; +} + +namespace cache { + struct FilePointerComparator { + bool operator()(const FilePointer& a, const FilePointer& b) { + return a->_modified > b->_modified; + } + }; +} + +void FileCache::clean() { + size_t overbudgetAmount = getOverbudgetAmount(); + + // Avoid sorting the unused files by LRU if we're not over budget / under free space + if (0 == overbudgetAmount) { + return; + } + Lock unusedLock(_unusedFilesMutex); - while (!_unusedFiles.empty() && - _unusedFilesSize + length > _unusedFilesMaxSize) { - auto it = _unusedFiles.begin(); - auto file = it->second; + using Queue = std::priority_queue, FilePointerComparator>; + Queue queue; + for (const auto& file : _unusedFiles) { + queue.push(file); + } + while (!queue.empty() && overbudgetAmount > 0) { + auto file = queue.top(); + queue.pop(); auto length = file->getLength(); unusedLock.unlock(); @@ -203,34 +263,32 @@ void FileCache::reserve(size_t length) { } unusedLock.lock(); - _unusedFiles.erase(it); + _unusedFiles.erase(file); _numTotalFiles -= 1; _numUnusedFiles -= 1; _totalFilesSize -= length; _unusedFilesSize -= length; + overbudgetAmount -= std::min(length, overbudgetAmount); } } void FileCache::clear() { - Lock unusedFilesLock(_unusedFilesMutex); - for (const auto& pair : _unusedFiles) { - auto& file = pair.second; - file->_cache = nullptr; + // Eliminate any overbudget files + clean(); - if (_totalFilesSize > _offlineFilesMaxSize) { - _totalFilesSize -= file->getLength(); - } else { - file->_shouldPersist = true; - qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str()); - } + // Mark everything remaining as persisted + Lock unusedFilesLock(_unusedFilesMutex); + for (auto& file : _unusedFiles) { + file->_shouldPersist = true; + file->_cache = nullptr; + qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str()); } _unusedFiles.clear(); } void File::deleter() { if (_cache) { - FilePointer self(this, &fileDeleter); - _cache->addUnusedFile(self); + _cache->addUnusedFile(FilePointer(this, std::mem_fn(&File::deleter))); } else { deleteLater(); } @@ -239,7 +297,9 @@ void File::deleter() { File::File(Metadata&& metadata, const std::string& filepath) : _key(std::move(metadata.key)), _length(metadata.length), - _filepath(filepath) {} + _filepath(filepath), + _modified(QFileInfo(_filepath.c_str()).lastRead().toMSecsSinceEpoch()) { +} File::~File() { QFile file(getFilepath().c_str()); @@ -248,3 +308,8 @@ File::~File() { file.remove(); } } + +void File::touch() { + utime(_filepath.c_str(), nullptr); + _modified = std::max(QFileInfo(_filepath.c_str()).lastRead().toMSecsSinceEpoch(), _modified); +} \ No newline at end of file diff --git a/libraries/networking/src/FileCache.h b/libraries/networking/src/FileCache.h index 908ddcd285..040e1ab592 100644 --- a/libraries/networking/src/FileCache.h +++ b/libraries/networking/src/FileCache.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -35,24 +36,28 @@ class FileCache : public QObject { Q_PROPERTY(size_t sizeTotal READ getSizeTotalFiles NOTIFY dirty) Q_PROPERTY(size_t sizeCached READ getSizeCachedFiles NOTIFY dirty) - static const size_t DEFAULT_UNUSED_MAX_SIZE; - static const size_t MAX_UNUSED_MAX_SIZE; - static const size_t DEFAULT_OFFLINE_MAX_SIZE; + static const size_t DEFAULT_MAX_SIZE; + static const size_t MAX_MAX_SIZE; + static const size_t DEFAULT_MIN_FREE_STORAGE_SPACE; public: + // You can initialize the FileCache with a directory name (ex.: "temp_jpgs") that will be relative to the application local data, OR with a full path + // The file cache will ignore any file that doesn't match the extension provided + FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr); + virtual ~FileCache(); + size_t getNumTotalFiles() const { return _numTotalFiles; } size_t getNumCachedFiles() const { return _numUnusedFiles; } size_t getSizeTotalFiles() const { return _totalFilesSize; } size_t getSizeCachedFiles() const { return _unusedFilesSize; } - void setUnusedFileCacheSize(size_t unusedFilesMaxSize); - size_t getUnusedFileCacheSize() const { return _unusedFilesSize; } + // Set the maximum amount of disk space to use on disk + void setMaxSize(size_t maxCacheSize); - void setOfflineFileCacheSize(size_t offlineFilesMaxSize); - - // initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg") - FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr); - virtual ~FileCache(); + // Set the minumum amount of free disk space to retain. This supercedes the max size, + // so if the cache is consuming all but 500 MB of the drive, unused entries will be ejected + // to free up more space, regardless of the cache max size + void setMinFreeSize(size_t size); using Key = std::string; struct Metadata { @@ -76,10 +81,11 @@ public: signals: void dirty(); -protected: +public: /// must be called after construction to create the cache on the fs and restore persisted files void initialize(); + // Add file to the cache and return the cache entry. FilePointer writeFile(const char* data, Metadata&& metadata, bool overwrite = false); FilePointer getFile(const Key& key); @@ -95,11 +101,17 @@ private: std::string getFilepath(const Key& key); FilePointer addFile(Metadata&& metadata, const std::string& filepath); - void addUnusedFile(const FilePointer file); - void removeUnusedFile(const FilePointer file); - void reserve(size_t length); + void addUnusedFile(const FilePointer& file); + void removeUnusedFile(const FilePointer& file); + void clean(); void clear(); + size_t getOverbudgetAmount() const; + + // FIXME it might be desirable to have the min free space variable be static so it can be + // shared among multiple instances of FileCache + std::atomic _minFreeSpaceSize { DEFAULT_MIN_FREE_STORAGE_SPACE }; + std::atomic _maxSize { DEFAULT_MAX_SIZE }; std::atomic _numTotalFiles { 0 }; std::atomic _numUnusedFiles { 0 }; std::atomic _totalFilesSize { 0 }; @@ -113,12 +125,8 @@ private: std::unordered_map> _files; Mutex _filesMutex; - std::map _unusedFiles; + std::unordered_set _unusedFiles; Mutex _unusedFilesMutex; - size_t _unusedFilesMaxSize { DEFAULT_UNUSED_MAX_SIZE }; - int _lastLRUKey { 0 }; - - size_t _offlineFilesMaxSize { DEFAULT_OFFLINE_MAX_SIZE }; }; class File : public QObject { @@ -142,13 +150,15 @@ protected: private: friend class FileCache; + friend struct FilePointerComparator; const Key _key; const size_t _length; const std::string _filepath; - FileCache* _cache; - int _LRUKey { 0 }; + void touch(); + FileCache* _cache { nullptr }; + int64_t _modified { 0 }; bool _shouldPersist { false }; }; diff --git a/libraries/shared/src/NumericalConstants.h b/libraries/shared/src/NumericalConstants.h index d37e1e31c5..e2d2409d56 100644 --- a/libraries/shared/src/NumericalConstants.h +++ b/libraries/shared/src/NumericalConstants.h @@ -50,10 +50,13 @@ const int KILO_PER_MEGA = 1000; #define KB_TO_BYTES_SHIFT 10 #define MB_TO_BYTES_SHIFT 20 +#define GB_TO_BYTES_SHIFT 30 +#define GB_TO_BYTES(X) ((size_t)(X) << GB_TO_BYTES_SHIFT) #define MB_TO_BYTES(X) ((size_t)(X) << MB_TO_BYTES_SHIFT) #define KB_TO_BYTES(X) ((size_t)(X) << KB_TO_BYTES_SHIFT) +#define BYTES_TO_GB(X) (X >> GB_TO_BYTES_SHIFT) #define BYTES_TO_MB(X) (X >> MB_TO_BYTES_SHIFT) #define BYTES_TO_KB(X) (X >> KB_TO_BYTES_SHIFT) diff --git a/tests/networking/src/FileCacheTests.cpp b/tests/networking/src/FileCacheTests.cpp new file mode 100644 index 0000000000..0813d05a54 --- /dev/null +++ b/tests/networking/src/FileCacheTests.cpp @@ -0,0 +1,170 @@ +// +// ResoruceTests.cpp +// +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#include "FileCacheTests.h" + +#include + +QTEST_GUILESS_MAIN(FileCacheTests) + +using namespace cache; + +// Limit the file size to 10 MB +static const size_t MAX_UNUSED_SIZE { 1024 * 1024 * 10 }; +static const QByteArray TEST_DATA { 1024 * 1024, '0' }; + +static std::string getFileKey(int i) { + return QString(QByteArray { 1, (char)i }.toHex()).toStdString(); +} + +class TestFile : public File { + using Parent = File; +public: + TestFile(Metadata&& metadata, const std::string& filepath) + : Parent(std::move(metadata), filepath) { + } +}; + +class TestFileCache : public FileCache { + using Parent = FileCache; + +public: + TestFileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr) : Parent(dirname, ext, nullptr) { + initialize(); + } + + std::unique_ptr createFile(Metadata&& metadata, const std::string& filepath) override { + qCInfo(file_cache) << "Wrote KTX" << metadata.key.c_str(); + return std::unique_ptr(new TestFile(std::move(metadata), filepath)); + } +}; + +using CachePointer = std::shared_ptr; + +// The FileCache relies on deleteLater to clear unused files, but QTest classes don't run a conventional event loop +// so we need to call this function to force any pending deletes to occur in the File destructor +static void forceDeletes() { + while (QCoreApplication::hasPendingEvents()) { + QCoreApplication::sendPostedEvents(); + QCoreApplication::sendPostedEvents(0, QEvent::DeferredDelete); + QCoreApplication::processEvents(); + } +} + +size_t FileCacheTests::getCacheDirectorySize() const { + size_t result = 0; + QDir dir(_testDir.path()); + for (const auto& file : dir.entryList({ "*.tmp" })) { + result += QFileInfo(dir.absoluteFilePath(file)).size(); + } + return result; +} + +CachePointer makeFileCache(QString& location) { + auto result = std::make_shared(location.toStdString(), "tmp"); + result->setMaxSize(MAX_UNUSED_SIZE); + return result; +} + +void FileCacheTests::initTestCase() { +} + +void FileCacheTests::testUnusedFiles() { + auto cache = makeFileCache(_testDir.path()); + std::list inUseFiles; + + { + for (int i = 0; i < 100; ++i) { + std::string key = getFileKey(i); + auto file = cache->writeFile(TEST_DATA.data(), TestFileCache::Metadata(key, TEST_DATA.size())); + inUseFiles.push_back(file); + forceDeletes(); + QThread::msleep(10); + } + QCOMPARE(cache->getNumCachedFiles(), (size_t)0); + QCOMPARE(cache->getNumTotalFiles(), (size_t)100); + // Release the in-use files + inUseFiles.clear(); + // Cache state is updated, but the directory state is unchanged, + // because the file deletes are triggered by an event loop + QCOMPARE(cache->getNumCachedFiles(), (size_t)10); + QCOMPARE(cache->getNumTotalFiles(), (size_t)10); + QVERIFY(getCacheDirectorySize() > MAX_UNUSED_SIZE); + forceDeletes(); + QCOMPARE(cache->getNumCachedFiles(), (size_t)10); + QCOMPARE(cache->getNumTotalFiles(), (size_t)10); + QVERIFY(getCacheDirectorySize() <= MAX_UNUSED_SIZE); + } + + // Reset the cache + cache = makeFileCache(_testDir.path()); + { + // Test files 0 to 89 are missing, because the LRU algorithm deleted them when we released the files + for (int i = 0; i < 90; ++i) { + std::string key = getFileKey(i); + auto file = cache->getFile(key); + QVERIFY(!file.get()); + } + + QThread::msleep(1000); + // Test files 90 to 99 are present + for (int i = 90; i < 100; ++i) { + std::string key = getFileKey(i); + auto file = cache->getFile(key); + QVERIFY(file.get()); + inUseFiles.push_back(file); + // Each access touches the file, so we need to sleep here to ensure that the files are + // spaced out in numeric order, otherwise later tests can't reliably determine the order + // for cache ejection + QThread::msleep(1000); + } + QCOMPARE(cache->getNumCachedFiles(), (size_t)0); + QCOMPARE(cache->getNumTotalFiles(), (size_t)10); + inUseFiles.clear(); + QCOMPARE(cache->getNumCachedFiles(), (size_t)10); + QCOMPARE(cache->getNumTotalFiles(), (size_t)10); + } +} + +size_t FileCacheTests::getFreeSpace() const { + return QStorageInfo(_testDir.path()).bytesFree(); +} + +// FIXME if something else is changing the amount of free space on the target drive concurrently with this test +// running, then it may fail +void FileCacheTests::testFreeSpacePreservation() { + QCOMPARE(getCacheDirectorySize(), MAX_UNUSED_SIZE); + // Set the target free space to slightly above whatever the current free space is... + size_t targetFreeSpace = getFreeSpace() + MAX_UNUSED_SIZE / 2; + + // Reset the cache + auto cache = makeFileCache(_testDir.path()); + // Setting the min free space causes it to eject the oldest files that cause the cache to exceed the minimum space + cache->setMinFreeSize(targetFreeSpace); + QVERIFY(getFreeSpace() < targetFreeSpace); + forceDeletes(); + QCOMPARE(cache->getNumCachedFiles(), (size_t)5); + QCOMPARE(cache->getNumTotalFiles(), (size_t)5); + QVERIFY(getFreeSpace() >= targetFreeSpace); + for (int i = 0; i < 95; ++i) { + std::string key = getFileKey(i); + auto file = cache->getFile(key); + QVERIFY(!file.get()); + } + for (int i = 95; i < 100; ++i) { + std::string key = getFileKey(i); + auto file = cache->getFile(key); + QVERIFY(file.get()); + } +} + +void FileCacheTests::cleanupTestCase() { +} + diff --git a/tests/networking/src/FileCacheTests.h b/tests/networking/src/FileCacheTests.h new file mode 100644 index 0000000000..838c15afb8 --- /dev/null +++ b/tests/networking/src/FileCacheTests.h @@ -0,0 +1,30 @@ +// +// ResourceTests.h +// +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ResourceTests_h +#define hifi_ResourceTests_h + +#include +#include + +class FileCacheTests : public QObject { + Q_OBJECT +private slots: + void initTestCase(); + void testUnusedFiles(); + void testFreeSpacePreservation(); + void cleanupTestCase(); + +private: + size_t getFreeSpace() const; + size_t getCacheDirectorySize() const; + QTemporaryDir _testDir; +}; + +#endif // hifi_ResourceTests_h