Merge pull request #10599 from jherico/ktx_cache_wipe

Add versioning support to the KTX cache

interface/src/Menu.cpp

@@ -407,6 +407,12 @@ Menu::Menu() {
 #endif
 
 
+    {
+        auto action = addActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::RenderClearKtxCache);
+        connect(action, &QAction::triggered, []{
+            Setting::Handle<int>(KTXCache::SETTING_VERSION_NAME, KTXCache::INVALID_VERSION).set(KTXCache::INVALID_VERSION);
+        });
+    }
 
     // Developer > Render > LOD Tools
     addActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::LodTools, 0,

interface/src/Menu.h

@@ -145,6 +145,7 @@ namespace MenuOption {
     const QString Quit =  "Quit";
     const QString ReloadAllScripts = "Reload All Scripts";
     const QString ReloadContent = "Reload Content (Clears all caches)";
+    const QString RenderClearKtxCache = "Clear KTX Cache (requires restart)";
     const QString RenderMaxTextureMemory = "Maximum Texture Memory";
     const QString RenderMaxTextureAutomatic = "Automatic Texture Memory";
     const QString RenderMaxTexture4MB = "4 MB";

libraries/model-networking/src/model-networking/KTXCache.cpp

@@ -11,14 +11,28 @@
 
 #include "KTXCache.h"
 
+#include <SettingHandle.h>
 #include <ktx/KTX.h>
 
 using File = cache::File;
 using FilePointer = cache::FilePointer;
 
+// Whenever a change is made to the serialized format for the KTX cache that isn't backward compatible,
+// this value should be incremented.  This will force the KTX cache to be wiped
+const int KTXCache::CURRENT_VERSION = 0x01;
+const int KTXCache::INVALID_VERSION = 0x00;
+const char* KTXCache::SETTING_VERSION_NAME = "hifi.ktx.cache_version";
+
 KTXCache::KTXCache(const std::string& dir, const std::string& ext) :
     FileCache(dir, ext) {
     initialize();
+
+    Setting::Handle<int> cacheVersionHandle(SETTING_VERSION_NAME, INVALID_VERSION);
+    auto cacheVersion = cacheVersionHandle.get();
+    if (cacheVersion != CURRENT_VERSION) {
+        wipe();
+        cacheVersionHandle.set(CURRENT_VERSION);
+    }
 }
 
 KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) {

libraries/model-networking/src/model-networking/KTXCache.h

@@ -27,6 +27,12 @@ class KTXCache : public cache::FileCache {
     Q_OBJECT
 
 public:
+    // Whenever a change is made to the serialized format for the KTX cache that isn't backward compatible,
+    // this value should be incremented.  This will force the KTX cache to be wiped
+    static const int CURRENT_VERSION;
+    static const int INVALID_VERSION;
+    static const char* SETTING_VERSION_NAME;
+
     KTXCache(const std::string& dir, const std::string& ext);
 
     KTXFilePointer writeFile(const char* data, Metadata&& metadata);

libraries/networking/src/FileCache.cpp

@@ -236,6 +236,28 @@ namespace cache {
     };
 }
 
+void FileCache::eject(const FilePointer& file) {
+    file->_cache = nullptr;
+    const auto& length = file->getLength();
+    const auto& key = file->getKey();
+
+    {
+        Lock lock(_filesMutex);
+        if (0 != _files.erase(key)) {
+            _numTotalFiles -= 1;
+            _totalFilesSize -= length;
+        }
+    }
+
+    {
+        Lock unusedLock(_unusedFilesMutex);
+        if (0 != _unusedFiles.erase(file)) {
+            _numUnusedFiles -= 1;
+            _unusedFilesSize -= length;
+        }
+    }
+}
+
 void FileCache::clean() {
     size_t overbudgetAmount = getOverbudgetAmount();
 
@@ -250,28 +272,23 @@ void FileCache::clean() {
     for (const auto& file : _unusedFiles) {
         queue.push(file);
     }
+
     while (!queue.empty() && overbudgetAmount > 0) {
         auto file = queue.top();
         queue.pop();
+        eject(file);
         auto length = file->getLength();
-
-        unusedLock.unlock();
-        {
-            file->_cache = nullptr;
-            Lock lock(_filesMutex);
-            _files.erase(file->getKey());
-        }
-        unusedLock.lock();
-
-        _unusedFiles.erase(file);
-        _numTotalFiles -= 1;
-        _numUnusedFiles -= 1;
-        _totalFilesSize -= length;
-        _unusedFilesSize -= length;
         overbudgetAmount -= std::min(length, overbudgetAmount);
     }
 }
 
+void FileCache::wipe() {
+    Lock unusedFilesLock(_unusedFilesMutex);
+    while (!_unusedFiles.empty()) {
+        eject(*_unusedFiles.begin());
+    }
+}
+
 void FileCache::clear() {
     // Eliminate any overbudget files
     clean();

libraries/networking/src/FileCache.h

@@ -46,6 +46,9 @@ public:
     FileCache(const std::string& dirname, const std::string& ext, QObject* parent = nullptr);
     virtual ~FileCache();
 
+    // Remove all unlocked items from the cache
+    void wipe();
+
     size_t getNumTotalFiles() const { return _numTotalFiles; }
     size_t getNumCachedFiles() const { return _numUnusedFiles; }
     size_t getSizeTotalFiles() const { return _totalFilesSize; }
@@ -95,6 +98,9 @@ public:
 private:
     using Mutex = std::recursive_mutex;
     using Lock = std::unique_lock<Mutex>;
+    using Map = std::unordered_map<Key, std::weak_ptr<File>>;
+    using Set = std::unordered_set<FilePointer>;
+    using KeySet = std::unordered_set<Key>;
 
     friend class File;
 
@@ -105,6 +111,8 @@ private:
     void removeUnusedFile(const FilePointer& file);
     void clean();
     void clear();
+    // Remove a file from the cache
+    void eject(const FilePointer& file);
 
     size_t getOverbudgetAmount() const;
 
@@ -122,10 +130,10 @@ private:
     std::string _dirpath;
     bool _initialized { false };
 
-    std::unordered_map<Key, std::weak_ptr<File>> _files;
+    Map _files;
     Mutex _filesMutex;
 
-    std::unordered_set<FilePointer> _unusedFiles;
+    Set _unusedFiles;
     Mutex _unusedFilesMutex;
 };
 
@@ -136,8 +144,8 @@ public:
     using Key = FileCache::Key;
     using Metadata = FileCache::Metadata;
 
-    Key getKey() const { return _key; }
+    const Key& getKey() const { return _key; }
-    size_t getLength() const { return _length; }
+    const size_t& getLength() const { return _length; }
     std::string getFilepath() const { return _filepath; }
 
     virtual ~File();

tests/networking/src/FileCacheTests.cpp

@@ -113,18 +113,21 @@ void FileCacheTests::testUnusedFiles() {
             QVERIFY(!file.get());
         }
 
-        QThread::msleep(1000);
         // Test files 90 to 99 are present
         for (int i = 90; i < 100; ++i) {
             std::string key = getFileKey(i);
             auto file = cache->getFile(key);
             QVERIFY(file.get());
             inUseFiles.push_back(file);
-            // Each access touches the file, so we need to sleep here to ensure that the files are 
+
-            // spaced out in numeric order, otherwise later tests can't reliably determine the order 
+            if (i == 94) {
-            // for cache ejection
+                // Each access touches the file, so we need to sleep here to ensure that the the last 5 files 
+                // have later times for cache ejection priority, otherwise the test runs too fast to reliably
+                // differentiate 
                 QThread::msleep(1000);
             }
+        }
+        
         QCOMPARE(cache->getNumCachedFiles(), (size_t)0);
         QCOMPARE(cache->getNumTotalFiles(), (size_t)10);
         inUseFiles.clear();
@@ -165,6 +168,20 @@ void FileCacheTests::testFreeSpacePreservation() {
     }
 }
 
+void FileCacheTests::testWipe() {
+    // Reset the cache
+    auto cache = makeFileCache(_testDir.path());
+    QCOMPARE(cache->getNumCachedFiles(), (size_t)5);
+    QCOMPARE(cache->getNumTotalFiles(), (size_t)5);
+    cache->wipe();
+    QCOMPARE(cache->getNumCachedFiles(), (size_t)0);
+    QCOMPARE(cache->getNumTotalFiles(), (size_t)0);
+    QVERIFY(getCacheDirectorySize() > 0);
+    forceDeletes();
+    QCOMPARE(getCacheDirectorySize(), (size_t)0);
+}
+
+
 void FileCacheTests::cleanupTestCase() {
 }
 

tests/networking/src/FileCacheTests.h

@@ -20,6 +20,7 @@ private slots:
     void testUnusedFiles();
     void testFreeSpacePreservation();
     void cleanupTestCase();
+    void testWipe();
 
 private:
     size_t getFreeSpace() const;