Merge pull request #10571 from jherico/ktx_tests

FileCache refactoring and test, plus KTX unit tests

libraries/ktx/src/khronos/KHR.h

@@ -211,6 +211,8 @@ namespace khronos {
 
             template <uint32_t ALIGNMENT> 
             inline uint32_t evalAlignedCompressedBlockCount(uint32_t value) {
+                enum { val = ALIGNMENT && !(ALIGNMENT & (ALIGNMENT - 1)) };
+                static_assert(val, "template parameter ALIGNMENT must be a power of 2");
                 // FIXME add static assert that ALIGNMENT is a power of 2
                 static uint32_t ALIGNMENT_REMAINDER = ALIGNMENT - 1;
                 return (value + ALIGNMENT_REMAINDER) / ALIGNMENT;

libraries/networking/src/FileCache.cpp

@@ -11,52 +11,81 @@
 
 #include "FileCache.h"
 
-#include <cstdio>
-#include <cassert>
-#include <fstream>
-#include <unordered_set>
 
-#include <QDir>
+#include <unordered_set>
-#include <QSaveFile>
+#include <queue>
+#include <cassert>
+
+#include <QtCore/QDateTime>
+#include <QtCore/QDir>
+#include <QtCore/QSaveFile>
+#include <QtCore/QStorageInfo>
 
 #include <PathUtils.h>
+#include <NumericalConstants.h>
 
+#ifdef Q_OS_WIN
+#include <sys/utime.h>
+#else
+#include <utime.h>
+#endif
+
+#ifdef NDEBUG
 Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache", QtWarningMsg)
-
+#else
+Q_LOGGING_CATEGORY(file_cache, "hifi.file_cache")
+#endif
 using namespace cache;
 
-static const std::string MANIFEST_NAME = "manifest";
+static const char DIR_SEP = '/';
+static const char EXT_SEP = '.';
 
-static const size_t BYTES_PER_MEGABYTES = 1024 * 1024;
+const size_t FileCache::DEFAULT_MAX_SIZE { GB_TO_BYTES(5) };
-static const size_t BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES;
+const size_t FileCache::MAX_MAX_SIZE { GB_TO_BYTES(100) };
-const size_t FileCache::DEFAULT_UNUSED_MAX_SIZE = 5 * BYTES_PER_GIGABYTES; // 5GB
+const size_t FileCache::DEFAULT_MIN_FREE_STORAGE_SPACE { GB_TO_BYTES(1) };
-const size_t FileCache::MAX_UNUSED_MAX_SIZE = 100 * BYTES_PER_GIGABYTES; // 100GB
-const size_t FileCache::DEFAULT_OFFLINE_MAX_SIZE = 2 * BYTES_PER_GIGABYTES; // 2GB
 
-void FileCache::setUnusedFileCacheSize(size_t unusedFilesMaxSize) {
+
-    _unusedFilesMaxSize = std::min(unusedFilesMaxSize, MAX_UNUSED_MAX_SIZE);
+std::string getCacheName(const std::string& dirname_str) {
-    reserve(0);
+    QString dirname { dirname_str.c_str() };
+    QDir dir(dirname);
+    if (!dir.isAbsolute()) {
+        return dirname_str;
+    }
+    return dir.dirName().toStdString();
+}
+
+std::string getCachePath(const std::string& dirname_str) {
+    QString dirname { dirname_str.c_str() };
+    QDir dir(dirname);
+    if (dir.isAbsolute()) {
+        return dirname_str;
+    }
+    return PathUtils::getAppLocalDataFilePath(dirname).toStdString();
+}
+
+void FileCache::setMinFreeSize(size_t size) {
+    _minFreeSpaceSize = size;
+    clean();
     emit dirty();
 }
 
-void FileCache::setOfflineFileCacheSize(size_t offlineFilesMaxSize) {
+void FileCache::setMaxSize(size_t maxSize) {
-    _offlineFilesMaxSize = std::min(offlineFilesMaxSize, MAX_UNUSED_MAX_SIZE);
+    _maxSize = std::min(maxSize, MAX_MAX_SIZE);
+    clean();
+    emit dirty();
 }
 
 FileCache::FileCache(const std::string& dirname, const std::string& ext, QObject* parent) :
     QObject(parent),
     _ext(ext),
-    _dirname(dirname),
+    _dirname(getCacheName(dirname)),
-    _dirpath(PathUtils::getAppLocalDataFilePath(dirname.c_str()).toStdString()) {}
+    _dirpath(getCachePath(dirname)) {
+}
 
 FileCache::~FileCache() {
     clear();
 }
 
-void fileDeleter(File* file) {
-    file->deleter();
-}
-
 void FileCache::initialize() {
     QDir dir(_dirpath.c_str());
 
@@ -84,7 +113,7 @@ void FileCache::initialize() {
 }
 
 FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath) {
-    FilePointer file(createFile(std::move(metadata), filepath).release(), &fileDeleter);
+    FilePointer file(createFile(std::move(metadata), filepath).release(), std::mem_fn(&File::deleter));
     if (file) {
         _numTotalFiles += 1;
         _totalFilesSize += file->getLength();
@@ -141,6 +170,7 @@ FilePointer FileCache::getFile(const Key& key) {
     if (it != _files.cend()) {
         file = it->second.lock();
         if (file) {
+            file->touch();
             // if it exists, it is active - remove it from the cache
             removeUnusedFile(file);
             qCDebug(file_cache, "[%s] Found %s", _dirname.c_str(), key.c_str());
@@ -155,44 +185,74 @@ FilePointer FileCache::getFile(const Key& key) {
 }
 
 std::string FileCache::getFilepath(const Key& key) {
-    return _dirpath + '/' + key + '.' + _ext;
+    return _dirpath + DIR_SEP + key + EXT_SEP + _ext;
 }
 
-void FileCache::addUnusedFile(const FilePointer file) {
+void FileCache::addUnusedFile(const FilePointer& file) {
     {
         Lock lock(_filesMutex);
         _files[file->getKey()] = file;
     }
 
-    reserve(file->getLength());
-    file->_LRUKey = ++_lastLRUKey;
-
     {
         Lock lock(_unusedFilesMutex);
-        _unusedFiles.insert({ file->_LRUKey, file });
+        _unusedFiles.insert(file);
         _numUnusedFiles += 1;
         _unusedFilesSize += file->getLength();
     }
+    clean();
 
     emit dirty();
 }
 
-void FileCache::removeUnusedFile(const FilePointer file) {
+void FileCache::removeUnusedFile(const FilePointer& file) {
     Lock lock(_unusedFilesMutex);
-    const auto it = _unusedFiles.find(file->_LRUKey);
+    if (_unusedFiles.erase(file)) {
-    if (it != _unusedFiles.cend()) {
-        _unusedFiles.erase(it);
         _numUnusedFiles -= 1;
         _unusedFilesSize -= file->getLength();
     }
 }
 
-void FileCache::reserve(size_t length) {
+size_t FileCache::getOverbudgetAmount() const {
+    size_t result = 0;
+
+    size_t currentFreeSpace = QStorageInfo(_dirpath.c_str()).bytesFree();
+    if (_minFreeSpaceSize > currentFreeSpace) {
+        result = _minFreeSpaceSize - currentFreeSpace;
+    }
+
+    if (_totalFilesSize > _maxSize) {
+        result = std::max(_totalFilesSize - _maxSize, result);
+    }
+
+    return result;
+}
+
+namespace cache {
+    struct FilePointerComparator {
+        bool operator()(const FilePointer& a, const FilePointer& b) {
+            return a->_modified > b->_modified;
+        }
+    };
+}
+
+void FileCache::clean() {
+    size_t overbudgetAmount = getOverbudgetAmount();
+
+    // Avoid sorting the unused files by LRU if we're not over budget / under free space
+    if (0 == overbudgetAmount) {
+        return;
+    }
+
     Lock unusedLock(_unusedFilesMutex);
-    while (!_unusedFiles.empty() &&
+    using Queue = std::priority_queue<FilePointer, std::vector<FilePointer>, FilePointerComparator>;
-            _unusedFilesSize + length > _unusedFilesMaxSize) {
+    Queue queue;
-        auto it = _unusedFiles.begin();
+    for (const auto& file : _unusedFiles) {
-        auto file = it->second;
+        queue.push(file);
+    }
+    while (!queue.empty() && overbudgetAmount > 0) {
+        auto file = queue.top();
+        queue.pop();
         auto length = file->getLength();
 
         unusedLock.unlock();
@@ -203,34 +263,32 @@ void FileCache::reserve(size_t length) {
         }
         unusedLock.lock();
 
-        _unusedFiles.erase(it);
+        _unusedFiles.erase(file);
         _numTotalFiles -= 1;
         _numUnusedFiles -= 1;
         _totalFilesSize -= length;
         _unusedFilesSize -= length;
+        overbudgetAmount -= std::min(length, overbudgetAmount);
     }
 }
 
 void FileCache::clear() {
-    Lock unusedFilesLock(_unusedFilesMutex);
+    // Eliminate any overbudget files
-    for (const auto& pair : _unusedFiles) {
+    clean();
-        auto& file = pair.second;
-        file->_cache = nullptr;
 
-        if (_totalFilesSize > _offlineFilesMaxSize) {
+    // Mark everything remaining as persisted
-            _totalFilesSize -= file->getLength();
+    Lock unusedFilesLock(_unusedFilesMutex);
-        } else {
+    for (auto& file : _unusedFiles) {
-            file->_shouldPersist = true;
+        file->_shouldPersist = true;
-            qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str());
+        file->_cache = nullptr;
-        }
+        qCDebug(file_cache, "[%s] Persisting %s", _dirname.c_str(), file->getKey().c_str());
     }
     _unusedFiles.clear();
 }
 
 void File::deleter() {
     if (_cache) {
-        FilePointer self(this, &fileDeleter);
+        _cache->addUnusedFile(FilePointer(this, std::mem_fn(&File::deleter)));
-        _cache->addUnusedFile(self);
     } else {
         deleteLater();
     }
@@ -239,7 +297,9 @@ void File::deleter() {
 File::File(Metadata&& metadata, const std::string& filepath) :
     _key(std::move(metadata.key)),
     _length(metadata.length),
-    _filepath(filepath) {}
+    _filepath(filepath),
+    _modified(QFileInfo(_filepath.c_str()).lastRead().toMSecsSinceEpoch()) {
+}
 
 File::~File() {
     QFile file(getFilepath().c_str());
@@ -248,3 +308,8 @@ File::~File() {
         file.remove();
     }
 }
+
+void File::touch() {
+    utime(_filepath.c_str(), nullptr);
+    _modified = std::max<int64_t>(QFileInfo(_filepath.c_str()).lastRead().toMSecsSinceEpoch(), _modified);
+}

libraries/networking/src/FileCache.h

@@ -14,6 +14,7 @@
 #include <memory>
 #include <cstddef>
 #include <map>
+#include <unordered_set>
 #include <mutex>
 #include <string>
 #include <unordered_map>
@@ -35,24 +36,28 @@ class FileCache : public QObject {
     Q_PROPERTY(size_t sizeTotal READ getSizeTotalFiles NOTIFY dirty)
     Q_PROPERTY(size_t sizeCached READ getSizeCachedFiles NOTIFY dirty)
 
-    static const size_t DEFAULT_UNUSED_MAX_SIZE;
+    static const size_t DEFAULT_MAX_SIZE;
-    static const size_t MAX_UNUSED_MAX_SIZE;
+    static const size_t MAX_MAX_SIZE;
-    static const size_t DEFAULT_OFFLINE_MAX_SIZE;
+    static const size_t DEFAULT_MIN_FREE_STORAGE_SPACE;
 
 public:
+    // You can initialize the FileCache with a directory name (ex.: "temp_jpgs") that will be relative to the application local data, OR with a full path
+    // The file cache will ignore any file that doesn't match the extension provided
+    FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr);
+    virtual ~FileCache();
+
     size_t getNumTotalFiles() const { return _numTotalFiles; }
     size_t getNumCachedFiles() const { return _numUnusedFiles; }
     size_t getSizeTotalFiles() const { return _totalFilesSize; }
     size_t getSizeCachedFiles() const { return _unusedFilesSize; }
 
-    void setUnusedFileCacheSize(size_t unusedFilesMaxSize);
+    // Set the maximum amount of disk space to use on disk
-    size_t getUnusedFileCacheSize() const { return _unusedFilesSize; }
+    void setMaxSize(size_t maxCacheSize);
 
-    void setOfflineFileCacheSize(size_t offlineFilesMaxSize);
+    // Set the minumum amount of free disk space to retain.  This supercedes the max size,
-
+    // so if the cache is consuming all but 500 MB of the drive, unused entries will be ejected 
-    // initialize FileCache with a directory name (not a path, ex.: "temp_jpgs") and an ext (ex.: "jpg")
+    // to free up more space, regardless of the cache max size
-    FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr);
+    void setMinFreeSize(size_t size);
-    virtual ~FileCache();
 
     using Key = std::string;
     struct Metadata {
@@ -76,10 +81,11 @@ public:
 signals:
     void dirty();
 
-protected:
+public:
     /// must be called after construction to create the cache on the fs and restore persisted files
     void initialize();
 
+    // Add file to the cache and return the cache entry.  
     FilePointer writeFile(const char* data, Metadata&& metadata, bool overwrite = false);
     FilePointer getFile(const Key& key);
 
@@ -95,11 +101,17 @@ private:
     std::string getFilepath(const Key& key);
 
     FilePointer addFile(Metadata&& metadata, const std::string& filepath);
-    void addUnusedFile(const FilePointer file);
+    void addUnusedFile(const FilePointer& file);
-    void removeUnusedFile(const FilePointer file);
+    void removeUnusedFile(const FilePointer& file);
-    void reserve(size_t length);
+    void clean();
     void clear();
 
+    size_t getOverbudgetAmount() const;
+
+    // FIXME it might be desirable to have the min free space variable be static so it can be
+    // shared among multiple instances of FileCache
+    std::atomic<size_t> _minFreeSpaceSize { DEFAULT_MIN_FREE_STORAGE_SPACE };
+    std::atomic<size_t> _maxSize { DEFAULT_MAX_SIZE };
     std::atomic<size_t> _numTotalFiles { 0 };
     std::atomic<size_t> _numUnusedFiles { 0 };
     std::atomic<size_t> _totalFilesSize { 0 };
@@ -113,12 +125,8 @@ private:
     std::unordered_map<Key, std::weak_ptr<File>> _files;
     Mutex _filesMutex;
 
-    std::map<int, FilePointer> _unusedFiles;
+    std::unordered_set<FilePointer> _unusedFiles;
     Mutex _unusedFilesMutex;
-    size_t _unusedFilesMaxSize { DEFAULT_UNUSED_MAX_SIZE };
-    int _lastLRUKey { 0 };
-
-    size_t _offlineFilesMaxSize { DEFAULT_OFFLINE_MAX_SIZE };
 };
 
 class File : public QObject {
@@ -142,13 +150,15 @@ protected:
 
 private:
     friend class FileCache;
+    friend struct FilePointerComparator;
 
     const Key _key;
     const size_t _length;
     const std::string _filepath;
 
-    FileCache* _cache;
+    void touch();
-    int _LRUKey { 0 };
+    FileCache* _cache { nullptr };
+    int64_t _modified { 0 };
 
     bool _shouldPersist { false };
 };

libraries/shared/src/NumericalConstants.h

@@ -50,10 +50,13 @@ const int KILO_PER_MEGA = 1000;
 
 #define KB_TO_BYTES_SHIFT 10
 #define MB_TO_BYTES_SHIFT 20
+#define GB_TO_BYTES_SHIFT 30
 
+#define GB_TO_BYTES(X) ((size_t)(X) << GB_TO_BYTES_SHIFT)
 #define MB_TO_BYTES(X) ((size_t)(X) << MB_TO_BYTES_SHIFT)
 #define KB_TO_BYTES(X) ((size_t)(X) << KB_TO_BYTES_SHIFT)
 
+#define BYTES_TO_GB(X) (X >> GB_TO_BYTES_SHIFT)
 #define BYTES_TO_MB(X) (X >> MB_TO_BYTES_SHIFT)
 #define BYTES_TO_KB(X) (X >> KB_TO_BYTES_SHIFT)
 

tests/ktx/CMakeLists.txt

@@ -1,15 +1,9 @@
+# Declare dependencies
+macro (SETUP_TESTCASE_DEPENDENCIES)
+  # link in the shared libraries
+  link_hifi_libraries(shared ktx gpu image)
 
-set(TARGET_NAME ktx-test)
+  package_libraries_for_deployment()
- 
+endmacro ()
-if (WIN32)
-  SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4049 /ignore:4217")
-endif()
 
-# This is not a testcase -- just set it up as a regular hifi project
+setup_hifi_testcase()
-setup_hifi_project(Quick Gui OpenGL)
-set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Tests/manual-tests/")
-
-# link in the shared libraries
-link_hifi_libraries(shared octree ktx gl gpu gpu-gl render model model-networking networking render-utils fbx entities entities-renderer animation audio avatars script-engine physics image)
-
-package_libraries_for_deployment()

tests/ktx/src/KtxTests.cpp

@@ -0,0 +1,195 @@
+//
+//  Created by Bradley Austin Davis on 2016/07/01
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "KtxTests.h"
+
+#include <mutex>
+
+#include <QtTest/QtTest>
+
+#include <ktx/KTX.h>
+#include <gpu/Texture.h>
+#include <image/Image.h>
+
+
+QTEST_GUILESS_MAIN(KtxTests)
+
+QString getRootPath() {
+    static std::once_flag once;
+    static QString result;
+    std::call_once(once, [&] {
+        QFileInfo file(__FILE__);
+        QDir parent = file.absolutePath();
+        result = QDir::cleanPath(parent.currentPath() + "/../../..");
+    });
+    return result;
+}
+
+void KtxTests::initTestCase() {
+}
+
+void KtxTests::cleanupTestCase() {
+}
+
+void KtxTests::testKhronosCompressionFunctions() {
+    using namespace khronos::gl::texture;
+    QCOMPARE(evalAlignedCompressedBlockCount<4>(0), (uint32_t)0x0);
+    QCOMPARE(evalAlignedCompressedBlockCount<4>(1), (uint32_t)0x1);
+    QCOMPARE(evalAlignedCompressedBlockCount<4>(4), (uint32_t)0x1);
+    QCOMPARE(evalAlignedCompressedBlockCount<4>(5), (uint32_t)0x2);
+    QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x00), (uint32_t)0x00);
+    QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x01), (uint32_t)0x01);
+    QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x04), (uint32_t)0x01);
+    QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x05), (uint32_t)0x02);
+    QCOMPARE(evalCompressedBlockCount(InternalFormat::COMPRESSED_SRGB_S3TC_DXT1_EXT, 0x1000), (uint32_t)0x400);
+
+    QVERIFY_EXCEPTION_THROWN(evalCompressedBlockCount(InternalFormat::RGBA8, 0x00), std::runtime_error);
+}
+
+void KtxTests::testKtxEvalFunctions() {
+    QCOMPARE(sizeof(ktx::Header), (size_t)64);
+    QCOMPARE(ktx::evalPadding(0x0), (uint8_t)0);
+    QCOMPARE(ktx::evalPadding(0x1), (uint8_t)3);
+    QCOMPARE(ktx::evalPadding(0x2), (uint8_t)2);
+    QCOMPARE(ktx::evalPadding(0x3), (uint8_t)1);
+    QCOMPARE(ktx::evalPadding(0x4), (uint8_t)0);
+    QCOMPARE(ktx::evalPadding(0x400), (uint8_t)0);
+    QCOMPARE(ktx::evalPadding(0x401), (uint8_t)3);
+    QCOMPARE(ktx::evalPaddedSize(0x0), 0x0);
+    QCOMPARE(ktx::evalPaddedSize(0x1), 0x4);
+    QCOMPARE(ktx::evalPaddedSize(0x2), 0x4);
+    QCOMPARE(ktx::evalPaddedSize(0x3), 0x4);
+    QCOMPARE(ktx::evalPaddedSize(0x4), 0x4);
+    QCOMPARE(ktx::evalPaddedSize(0x400), 0x400);
+    QCOMPARE(ktx::evalPaddedSize(0x401), 0x404);
+    QCOMPARE(ktx::evalAlignedCount((uint32_t)0x0), (uint32_t)0x0);
+    QCOMPARE(ktx::evalAlignedCount((uint32_t)0x1), (uint32_t)0x1);
+    QCOMPARE(ktx::evalAlignedCount((uint32_t)0x4), (uint32_t)0x1);
+    QCOMPARE(ktx::evalAlignedCount((uint32_t)0x5), (uint32_t)0x2);
+}
+
+void KtxTests::testKtxSerialization() {
+    const QString TEST_IMAGE = getRootPath() + "/scripts/developer/tests/cube_texture.png";
+    QImage image(TEST_IMAGE);
+    gpu::TexturePointer testTexture = image::TextureUsage::process2DTextureColorFromImage(image, TEST_IMAGE.toStdString(), true);
+    auto ktxMemory = gpu::Texture::serialize(*testTexture);
+    QVERIFY(ktxMemory.get());
+
+    // Serialize the image to a file
+    QTemporaryFile TEST_IMAGE_KTX;
+    {
+        const auto& ktxStorage = ktxMemory->getStorage();
+        QVERIFY(ktx::KTX::validate(ktxStorage));
+        QVERIFY(ktxMemory->isValid());
+
+        auto& outFile = TEST_IMAGE_KTX;
+        if (!outFile.open()) {
+            QFAIL("Unable to open file");
+        }
+        auto ktxSize = ktxStorage->size();
+        outFile.resize(ktxSize);
+        auto dest = outFile.map(0, ktxSize);
+        memcpy(dest, ktxStorage->data(), ktxSize);
+        outFile.unmap(dest);
+        outFile.close();
+    }
+
+
+    {
+        auto ktxStorage = std::make_shared<storage::FileStorage>(TEST_IMAGE_KTX.fileName());
+        QVERIFY(ktx::KTX::validate(ktxStorage));
+        auto ktxFile = ktx::KTX::create(ktxStorage);
+        QVERIFY(ktxFile.get());
+        QVERIFY(ktxFile->isValid());
+        {
+            const auto& memStorage = ktxMemory->getStorage();
+            const auto& fileStorage = ktxFile->getStorage();
+            QVERIFY(memStorage->size() == fileStorage->size());
+            QVERIFY(memStorage->data() != fileStorage->data());
+            QVERIFY(0 == memcmp(memStorage->data(), fileStorage->data(), memStorage->size()));
+            QVERIFY(ktxFile->_images.size() == ktxMemory->_images.size());
+            auto imageCount = ktxFile->_images.size();
+            auto startMemory = ktxMemory->_storage->data();
+            auto startFile = ktxFile->_storage->data();
+            for (size_t i = 0; i < imageCount; ++i) {
+                auto memImages = ktxMemory->_images[i];
+                auto fileImages = ktxFile->_images[i];
+                QVERIFY(memImages._padding == fileImages._padding);
+                QVERIFY(memImages._numFaces == fileImages._numFaces);
+                QVERIFY(memImages._imageSize == fileImages._imageSize);
+                QVERIFY(memImages._faceSize == fileImages._faceSize);
+                QVERIFY(memImages._faceBytes.size() == memImages._numFaces);
+                QVERIFY(fileImages._faceBytes.size() == fileImages._numFaces);
+                auto faceCount = fileImages._numFaces;
+                for (uint32_t face = 0; face < faceCount; ++face) {
+                    auto memFace = memImages._faceBytes[face];
+                    auto memOffset = memFace - startMemory;
+                    auto fileFace = fileImages._faceBytes[face];
+                    auto fileOffset = fileFace - startFile;
+                    QVERIFY(memOffset % 4 == 0);
+                    QVERIFY(memOffset == fileOffset);
+                }
+            }
+        }
+    }
+    testTexture->setKtxBacking(TEST_IMAGE_KTX.fileName().toStdString());
+}
+
+#if 0
+
+static const QString TEST_FOLDER { "H:/ktx_cacheold" };
+//static const QString TEST_FOLDER { "C:/Users/bdavis/Git/KTX/testimages" };
+
+//static const QString EXTENSIONS { "4bbdf8f786470e4ab3e672d44b8e8df2.ktx" };
+static const QString EXTENSIONS { "*.ktx" };
+
+int mainTemp(int, char**) {
+    qInstallMessageHandler(messageHandler);
+    auto fileInfoList = QDir { TEST_FOLDER }.entryInfoList(QStringList  { EXTENSIONS });
+    for (auto fileInfo : fileInfoList) {
+        qDebug() << fileInfo.filePath();
+        std::shared_ptr<storage::Storage> storage { new storage::FileStorage { fileInfo.filePath() } };
+
+        if (!ktx::KTX::validate(storage)) {
+            qDebug() << "KTX invalid";
+        }
+
+        auto ktxFile = ktx::KTX::create(storage);
+        ktx::KTXDescriptor ktxDescriptor = ktxFile->toDescriptor();
+
+        qDebug() << "Contains " << ktxDescriptor.keyValues.size() << " key value pairs";
+        for (const auto& kv : ktxDescriptor.keyValues) {
+            qDebug() << "\t" << kv._key.c_str();
+        }
+
+        auto offsetToMinMipKV = ktxDescriptor.getValueOffsetForKey(ktx::HIFI_MIN_POPULATED_MIP_KEY);
+        if (offsetToMinMipKV) {
+            auto data = storage->data() + ktx::KTX_HEADER_SIZE + offsetToMinMipKV;
+            auto minMipLevelAvailable = *data;
+            qDebug() << "\tMin mip available " << minMipLevelAvailable;
+            assert(minMipLevelAvailable < ktxDescriptor.header.numberOfMipmapLevels);
+        }
+        auto storageSize = storage->size();
+        for (const auto& faceImageDesc : ktxDescriptor.images) {
+            //assert(0 == (faceImageDesc._faceSize % 4));
+            for (const auto& faceOffset : faceImageDesc._faceOffsets) {
+                assert(0 == (faceOffset % 4));
+                auto faceEndOffset = faceOffset + faceImageDesc._faceSize;
+                assert(faceEndOffset <= storageSize);
+            }
+        }
+
+        for (const auto& faceImage : ktxFile->_images) {
+            for (const ktx::Byte* faceBytes : faceImage._faceBytes) {
+                assert(0 == (reinterpret_cast<size_t>(faceBytes) % 4));
+            }
+        }
+    }
+    return 0;
+}
+#endif

tests/ktx/src/KtxTests.h

@@ -0,0 +1,21 @@
+//
+//  Created by Bradley Austin Davis on 2016/07/01
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <QtCore/QObject>
+
+class KtxTests : public QObject {
+    Q_OBJECT
+private slots:
+    void initTestCase();
+    void cleanupTestCase();
+    void testKtxEvalFunctions();
+    void testKhronosCompressionFunctions();
+    void testKtxSerialization();
+};
+
+

tests/ktx/src/main.cpp

@@ -1,223 +0,0 @@
-//
-//  Created by Bradley Austin Davis on 2016/07/01
-//  Copyright 2014 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#include <iostream>
-#include <string>
-#include <vector>
-#include <sstream>
-
-#include <QProcessEnvironment>
-
-#include <QtCore/QDir>
-#include <QtCore/QElapsedTimer>
-#include <QtCore/QLoggingCategory>
-#include <QtCore/QRegularExpression>
-#include <QtCore/QSettings>
-#include <QtCore/QTimer>
-#include <QtCore/QThread>
-#include <QtCore/QThreadPool>
-#include <QtCore/QSaveFile>
-
-#include <QtGui/QGuiApplication>
-#include <QtGui/QResizeEvent>
-#include <QtGui/QWindow>
-
-#include <QtWidgets/QFileDialog>
-#include <QtWidgets/QInputDialog>
-#include <QtWidgets/QMessageBox>
-#include <QtWidgets/QApplication>
-
-#include <shared/RateCounter.h>
-#include <shared/NetworkUtils.h>
-#include <shared/FileLogger.h>
-#include <shared/FileUtils.h>
-#include <StatTracker.h>
-#include <LogHandler.h>
-
-#include <gpu/Texture.h>
-#include <gl/Config.h>
-#include <model/TextureMap.h>
-#include <ktx/KTX.h>
-#include <image/Image.h>
-
-QSharedPointer<FileLogger> logger;
-
-gpu::Texture* cacheTexture(const std::string& name, gpu::Texture* srcTexture, bool write = true, bool read = true);
-
-
-void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) {
-    QString logMessage = LogHandler::getInstance().printMessage((LogMsgType)type, context, message);
-
-    if (!logMessage.isEmpty()) {
-#ifdef Q_OS_WIN
-        OutputDebugStringA(logMessage.toLocal8Bit().constData());
-        OutputDebugStringA("\n");
-#endif
-        if (logger) {
-            logger->addMessage(qPrintable(logMessage + "\n"));
-        }
-    }
-}
-
-const char * LOG_FILTER_RULES = R"V0G0N(
-hifi.gpu=true
-)V0G0N";
-
-QString getRootPath() {
-    static std::once_flag once;
-    static QString result;
-    std::call_once(once, [&] {
-        QFileInfo file(__FILE__);
-        QDir parent = file.absolutePath();
-        result = QDir::cleanPath(parent.currentPath() + "/../../..");
-    });
-    return result;
-}
-
-const QString TEST_IMAGE = getRootPath() + "/scripts/developer/tests/cube_texture.png";
-const QString TEST_IMAGE_KTX = getRootPath() + "/scripts/developer/tests/cube_texture.ktx";
-
-int main(int argc, char** argv) {
-    QApplication app(argc, argv);
-    QCoreApplication::setApplicationName("KTX");
-    QCoreApplication::setOrganizationName("High Fidelity");
-    QCoreApplication::setOrganizationDomain("highfidelity.com");
-    logger.reset(new FileLogger());
-
-    Q_ASSERT(ktx::evalPadding(0) == 0);
-    Q_ASSERT(ktx::evalPadding(1) == 3);
-    Q_ASSERT(ktx::evalPadding(2) == 2);
-    Q_ASSERT(ktx::evalPadding(3) == 1);
-    Q_ASSERT(ktx::evalPadding(4) == 0);
-    Q_ASSERT(ktx::evalPadding(1024) == 0);
-    Q_ASSERT(ktx::evalPadding(1025) == 3);
-    Q_ASSERT(ktx::evalPaddedSize(0) == 0);
-    Q_ASSERT(ktx::evalPaddedSize(1) == 4);
-    Q_ASSERT(ktx::evalPaddedSize(2) == 4);
-    Q_ASSERT(ktx::evalPaddedSize(3) == 4);
-    Q_ASSERT(ktx::evalPaddedSize(4) == 4);
-    Q_ASSERT(ktx::evalPaddedSize(1024) == 1024);
-    Q_ASSERT(ktx::evalPaddedSize(1025) == 1028);
-    Q_ASSERT(sizeof(ktx::Header) == 12 + (sizeof(uint32_t) * 13));
-
-    DependencyManager::set<tracing::Tracer>();
-    qInstallMessageHandler(messageHandler);
-    QLoggingCategory::setFilterRules(LOG_FILTER_RULES);
-
-    QImage image(TEST_IMAGE);
-    gpu::TexturePointer testTexture = image::TextureUsage::process2DTextureColorFromImage(image, TEST_IMAGE.toStdString(), true);
-
-    auto ktxMemory = gpu::Texture::serialize(*testTexture);
-    {
-        const auto& ktxStorage = ktxMemory->getStorage();
-        Q_ASSERT_X(ktx::KTX::validate(ktxStorage), __FUNCTION__, "KTX storage validation failed");
-        Q_ASSERT_X(ktxMemory->isValid(), __FUNCTION__, "KTX self-validation failed");
-        QSaveFile outFile(TEST_IMAGE_KTX);
-        if (!outFile.open(QFile::WriteOnly)) {
-            throw std::runtime_error("Unable to open file");
-        }
-        auto ktxSize = ktxStorage->size();
-        outFile.resize(ktxSize);
-        auto dest = outFile.map(0, ktxSize);
-        memcpy(dest, ktxStorage->data(), ktxSize);
-        outFile.unmap(dest);
-        outFile.commit();
-    }
-
-    {
-        auto ktxFile = ktx::KTX::create(std::shared_ptr<storage::Storage>(new storage::FileStorage(TEST_IMAGE_KTX)));
-        {
-            const auto& memStorage = ktxMemory->getStorage();
-            const auto& fileStorage = ktxFile->getStorage();
-            Q_ASSERT(memStorage->size() == fileStorage->size());
-            Q_ASSERT(memStorage->data() != fileStorage->data());
-            Q_ASSERT(0 == memcmp(memStorage->data(), fileStorage->data(), memStorage->size()));
-            Q_ASSERT(ktxFile->_images.size() == ktxMemory->_images.size());
-            auto imageCount = ktxFile->_images.size();
-            auto startMemory = ktxMemory->_storage->data();
-            auto startFile = ktxFile->_storage->data();
-            for (size_t i = 0; i < imageCount; ++i) {
-                auto memImages = ktxMemory->_images[i];
-                auto fileImages = ktxFile->_images[i];
-                Q_ASSERT(memImages._padding == fileImages._padding);
-                Q_ASSERT(memImages._numFaces == fileImages._numFaces);
-                Q_ASSERT(memImages._imageSize == fileImages._imageSize);
-                Q_ASSERT(memImages._faceSize == fileImages._faceSize);
-                Q_ASSERT(memImages._faceBytes.size() == memImages._numFaces);
-                Q_ASSERT(fileImages._faceBytes.size() == fileImages._numFaces);
-                auto faceCount = fileImages._numFaces;
-                for (uint32_t face = 0; face < faceCount; ++face) {
-                    auto memFace = memImages._faceBytes[face];
-                    auto memOffset = memFace - startMemory;
-                    auto fileFace = fileImages._faceBytes[face];
-                    auto fileOffset = fileFace - startFile;
-                    Q_ASSERT(memOffset % 4 == 0);
-                    Q_ASSERT(memOffset == fileOffset);
-                }
-            }
-        }
-    }
-    testTexture->setKtxBacking(TEST_IMAGE_KTX.toStdString());
-    return 0;
-}
-
-#if 0
-static const QString TEST_FOLDER { "H:/ktx_cacheold" };
-//static const QString TEST_FOLDER { "C:/Users/bdavis/Git/KTX/testimages" };
-
-//static const QString EXTENSIONS { "4bbdf8f786470e4ab3e672d44b8e8df2.ktx" };
-static const QString EXTENSIONS { "*.ktx" };
-
-int mainTemp(int, char**) {
-    qInstallMessageHandler(messageHandler);
-    auto fileInfoList = QDir { TEST_FOLDER }.entryInfoList(QStringList  { EXTENSIONS });
-    for (auto fileInfo : fileInfoList) {
-        qDebug() << fileInfo.filePath();
-        std::shared_ptr<storage::Storage> storage { new storage::FileStorage { fileInfo.filePath() } };
-
-        if (!ktx::KTX::validate(storage)) {
-            qDebug() << "KTX invalid";
-        }
-
-        auto ktxFile = ktx::KTX::create(storage);
-        ktx::KTXDescriptor ktxDescriptor = ktxFile->toDescriptor();
-
-        qDebug() << "Contains " << ktxDescriptor.keyValues.size() << " key value pairs";
-        for (const auto& kv : ktxDescriptor.keyValues) {
-            qDebug() << "\t" << kv._key.c_str();
-        }
-
-        auto offsetToMinMipKV = ktxDescriptor.getValueOffsetForKey(ktx::HIFI_MIN_POPULATED_MIP_KEY);
-        if (offsetToMinMipKV) {
-            auto data = storage->data() + ktx::KTX_HEADER_SIZE + offsetToMinMipKV;
-            auto minMipLevelAvailable = *data;
-            qDebug() << "\tMin mip available " << minMipLevelAvailable;
-            assert(minMipLevelAvailable < ktxDescriptor.header.numberOfMipmapLevels);
-        }
-        auto storageSize = storage->size();
-        for (const auto& faceImageDesc : ktxDescriptor.images) {
-            //assert(0 == (faceImageDesc._faceSize % 4));
-            for (const auto& faceOffset : faceImageDesc._faceOffsets) {
-                assert(0 == (faceOffset % 4));
-                auto faceEndOffset = faceOffset + faceImageDesc._faceSize;
-                assert(faceEndOffset <= storageSize);
-            }
-        }
-
-        for (const auto& faceImage : ktxFile->_images) {
-            for (const ktx::Byte* faceBytes : faceImage._faceBytes) {
-                assert(0 == (reinterpret_cast<size_t>(faceBytes) % 4));
-            }
-        }
-    }
-    return 0;
-}
-#endif
-
-#include "main.moc"
-

tests/networking/src/FileCacheTests.cpp

@@ -0,0 +1,170 @@
+//
+//  ResoruceTests.cpp
+//
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+
+#include "FileCacheTests.h"
+
+#include <FileCache.h>
+
+QTEST_GUILESS_MAIN(FileCacheTests)
+
+using namespace cache;
+
+// Limit the file size to 10 MB
+static const size_t MAX_UNUSED_SIZE { 1024 * 1024 * 10 };
+static const QByteArray TEST_DATA { 1024 * 1024, '0' };
+
+static std::string getFileKey(int i) {
+    return QString(QByteArray { 1, (char)i }.toHex()).toStdString();
+}
+
+class TestFile : public File {
+    using Parent = File;
+public:
+    TestFile(Metadata&& metadata, const std::string& filepath)
+        : Parent(std::move(metadata), filepath) {
+    }
+};
+
+class TestFileCache : public FileCache {
+    using Parent = FileCache;
+
+public:
+    TestFileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr) : Parent(dirname, ext, nullptr) {
+        initialize();
+    }
+
+    std::unique_ptr<File> createFile(Metadata&& metadata, const std::string& filepath) override {
+        qCInfo(file_cache) << "Wrote KTX" << metadata.key.c_str();
+        return std::unique_ptr<File>(new TestFile(std::move(metadata), filepath));
+    }
+};
+
+using CachePointer = std::shared_ptr<TestFileCache>;
+
+// The FileCache relies on deleteLater to clear unused files, but QTest classes don't run a conventional event loop
+// so we need to call this function to force any pending deletes to occur in the File destructor
+static void forceDeletes() {
+    while (QCoreApplication::hasPendingEvents()) {
+        QCoreApplication::sendPostedEvents();
+        QCoreApplication::sendPostedEvents(0, QEvent::DeferredDelete);
+        QCoreApplication::processEvents();
+    }
+}
+
+size_t FileCacheTests::getCacheDirectorySize() const {
+    size_t result = 0;
+    QDir dir(_testDir.path());
+    for (const auto& file : dir.entryList({ "*.tmp" })) {
+        result += QFileInfo(dir.absoluteFilePath(file)).size();
+    }
+    return result;
+}
+
+CachePointer makeFileCache(QString& location) {
+    auto result = std::make_shared<TestFileCache>(location.toStdString(), "tmp");
+    result->setMaxSize(MAX_UNUSED_SIZE);
+    return result;
+}
+
+void FileCacheTests::initTestCase() {
+}
+
+void FileCacheTests::testUnusedFiles() {
+    auto cache = makeFileCache(_testDir.path());
+    std::list<FilePointer> inUseFiles;
+
+    {
+        for (int i = 0; i < 100; ++i) {
+            std::string key = getFileKey(i);
+            auto file = cache->writeFile(TEST_DATA.data(), TestFileCache::Metadata(key, TEST_DATA.size()));
+            inUseFiles.push_back(file);
+            forceDeletes();
+            QThread::msleep(10);
+        }
+        QCOMPARE(cache->getNumCachedFiles(), (size_t)0);
+        QCOMPARE(cache->getNumTotalFiles(), (size_t)100);
+        // Release the in-use files
+        inUseFiles.clear();
+        // Cache state is updated, but the directory state is unchanged, 
+        // because the file deletes are triggered by an event loop
+        QCOMPARE(cache->getNumCachedFiles(), (size_t)10);
+        QCOMPARE(cache->getNumTotalFiles(), (size_t)10);
+        QVERIFY(getCacheDirectorySize() >  MAX_UNUSED_SIZE);
+        forceDeletes();
+        QCOMPARE(cache->getNumCachedFiles(), (size_t)10);
+        QCOMPARE(cache->getNumTotalFiles(), (size_t)10);
+        QVERIFY(getCacheDirectorySize() <= MAX_UNUSED_SIZE);
+    }
+
+    // Reset the cache
+    cache = makeFileCache(_testDir.path());
+    {
+        // Test files 0 to 89 are missing, because the LRU algorithm deleted them when we released the files
+        for (int i = 0; i < 90; ++i) {
+            std::string key = getFileKey(i);
+            auto file = cache->getFile(key);
+            QVERIFY(!file.get());
+        }
+
+        QThread::msleep(1000);
+        // Test files 90 to 99 are present
+        for (int i = 90; i < 100; ++i) {
+            std::string key = getFileKey(i);
+            auto file = cache->getFile(key);
+            QVERIFY(file.get());
+            inUseFiles.push_back(file);
+            // Each access touches the file, so we need to sleep here to ensure that the files are 
+            // spaced out in numeric order, otherwise later tests can't reliably determine the order 
+            // for cache ejection
+            QThread::msleep(1000);
+        }
+        QCOMPARE(cache->getNumCachedFiles(), (size_t)0);
+        QCOMPARE(cache->getNumTotalFiles(), (size_t)10);
+        inUseFiles.clear();
+        QCOMPARE(cache->getNumCachedFiles(), (size_t)10);
+        QCOMPARE(cache->getNumTotalFiles(), (size_t)10);
+    }
+}
+
+size_t FileCacheTests::getFreeSpace() const {
+    return QStorageInfo(_testDir.path()).bytesFree();
+}
+
+// FIXME if something else is changing the amount of free space on the target drive concurrently with this test 
+// running, then it may fail
+void FileCacheTests::testFreeSpacePreservation() {
+    QCOMPARE(getCacheDirectorySize(), MAX_UNUSED_SIZE);
+    // Set the target free space to slightly above whatever the current free space is...
+    size_t targetFreeSpace = getFreeSpace() + MAX_UNUSED_SIZE / 2;
+
+    // Reset the cache
+    auto cache = makeFileCache(_testDir.path());
+    // Setting the min free space causes it to eject the oldest files that cause the cache to exceed the minimum space
+    cache->setMinFreeSize(targetFreeSpace);
+    QVERIFY(getFreeSpace() < targetFreeSpace);
+    forceDeletes();
+    QCOMPARE(cache->getNumCachedFiles(), (size_t)5);
+    QCOMPARE(cache->getNumTotalFiles(), (size_t)5);
+    QVERIFY(getFreeSpace() >= targetFreeSpace);
+    for (int i = 0; i < 95; ++i) {
+        std::string key = getFileKey(i);
+        auto file = cache->getFile(key);
+        QVERIFY(!file.get());
+    }
+    for (int i = 95; i < 100; ++i) {
+        std::string key = getFileKey(i);
+        auto file = cache->getFile(key);
+        QVERIFY(file.get());
+    }
+}
+
+void FileCacheTests::cleanupTestCase() {
+}
+

tests/networking/src/FileCacheTests.h

@@ -0,0 +1,30 @@
+//
+//  ResourceTests.h
+//
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_ResourceTests_h
+#define hifi_ResourceTests_h
+
+#include <QtTest/QtTest>
+#include <QtCore/QTemporaryDir>
+
+class FileCacheTests : public QObject {
+    Q_OBJECT
+private slots:
+    void initTestCase();
+    void testUnusedFiles();
+    void testFreeSpacePreservation();
+    void cleanupTestCase();
+
+private:
+    size_t getFreeSpace() const;
+    size_t getCacheDirectorySize() const;
+    QTemporaryDir _testDir;
+};
+
+#endif // hifi_ResourceTests_h