Merge pull request #5806 from Atlante45/as_caching

Asset Server caching

interface/src/Application.cpp

@@ -182,9 +182,6 @@ public:
 
 using namespace std;
 
-//  Starfield information
-const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES;  // 10GB
-
 static QTimer* locationUpdateTimer = NULL;
 static QTimer* balanceUpdateTimer = NULL;
 static QTimer* identityPacketTimer = NULL;
@@ -370,6 +367,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
         _lastFaceTrackerUpdate(0),
         _applicationOverlay()
 {
+    thread()->setObjectName("Main Thread");
+    
     setInstance(this);
 
     _entityClipboard->createRootElement();
@@ -460,13 +459,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
 
     audioThread->start();
 
-    QThread* assetThread = new QThread;
+    // Setup AssetClient
-
-    assetThread->setObjectName("Asset Thread");
     auto assetClient = DependencyManager::get<AssetClient>();
-
+    QThread* assetThread = new QThread;
+    assetThread->setObjectName("Asset Thread");
     assetClient->moveToThread(assetThread);
-
+    connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init);
     assetThread->start();
 
     const DomainHandler& domainHandler = nodeList->getDomainHandler();
@@ -855,8 +853,7 @@ void Application::cleanupBeforeQuit() {
 }
 
 void Application::emptyLocalCache() {
-    QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
+    if (auto cache = NetworkAccessManager::getInstance().cache()) {
-    if (cache) {
         qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
         cache->clear();
     }

interface/src/ui/DiskCacheEditor.cpp

@@ -140,8 +140,7 @@ void DiskCacheEditor::clear() {
                                               "You are about to erase all the content of the disk cache,"
                                               "are you sure you want to do that?");
     if (buttonClicked == QMessageBox::Yes) {
-        QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
+        if (auto cache = NetworkAccessManager::getInstance().cache()) {
-        if (cache) {
             qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
             cache->clear();
         }

libraries/networking/src/AssetClient.cpp

@@ -11,16 +11,21 @@
 
 #include "AssetClient.h"
 
-#include <QBuffer>
-#include <QThread>
-
 #include <cstdint>
 
+#include <QtCore/QBuffer>
+#include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
+#include <QtNetwork/QNetworkDiskCache>
+
 #include "AssetRequest.h"
 #include "AssetUpload.h"
+#include "AssetUtils.h"
+#include "NetworkAccessManager.h"
+#include "NetworkLogging.h"
 #include "NodeList.h"
 #include "PacketReceiver.h"
-#include "AssetUtils.h"
+#include "ResourceCache.h"
 
 MessageID AssetClient::_currentID = 0;
 
@@ -40,9 +45,29 @@ AssetClient::AssetClient() {
     connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &AssetClient::handleNodeKilled);
 }
 
+void AssetClient::init() {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection);
+    }
+    
+    // Setup disk cache if not already
+    QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+    if (!networkAccessManager.cache()) {
+        QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
+        cachePath = !cachePath.isEmpty() ? cachePath : "interfaceCache";
+        
+        QNetworkDiskCache* cache = new QNetworkDiskCache();
+        cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE);
+        cache->setCacheDirectory(cachePath);
+        networkAccessManager.setCache(cache);
+        qCDebug(asset_client) << "AssetClient disk cache setup at" << cachePath
+                                << "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)";
+    }
+}
+
 AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) {
     if (hash.length() != SHA256_HASH_HEX_LENGTH) {
-        qDebug() << "Invalid hash size";
+        qCWarning(asset_client) << "Invalid hash size";
         return nullptr;
     }
 
@@ -50,7 +75,7 @@ AssetRequest* AssetClient::createRequest(const QString& hash, const QString& ext
     SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
 
     if (!assetServer) {
-        qDebug().nospace() << "Could not request " << hash << "." << extension
+        qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension
             << " since you are not currently connected to an asset-server.";
         return nullptr;
     }
@@ -68,7 +93,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
     SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
     
     if (!assetServer) {
-        qDebug() << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
+        qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
         return nullptr;
     }
     
@@ -82,7 +107,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
 bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,
                            ReceivedAssetCallback callback) {
     if (hash.length() != SHA256_HASH_HEX_LENGTH) {
-        qDebug() << "Invalid hash size";
+        qCWarning(asset_client) << "Invalid hash size";
         return false;
     }
 
@@ -97,7 +122,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
             + sizeof(start) + sizeof(end);
         auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true);
         
-        qDebug() << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
+        qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
         
         packet->writePrimitive(messageID);
 
@@ -184,7 +209,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
     packet.open(QIODevice::ReadOnly);
 
     auto assetHash = packet.read(SHA256_HASH_LENGTH);
-    qDebug() << "Got reply for asset: " << assetHash.toHex();
+    qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex();
 
     MessageID messageID;
     packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
@@ -198,7 +223,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
         packet.read(reinterpret_cast<char*>(&length), sizeof(DataOffset));
         data = packet.read(length);
     } else {
-        qDebug() << "Failure getting asset: " << error;
+        qCWarning(asset_client) << "Failure getting asset: " << error;
     }
 
     // Check if we have any pending requests for this node
@@ -254,15 +279,15 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, Shared
     AssetServerError error;
     packet->readPrimitive(&error);
 
-    QString hashString { "" };
+    QString hashString;
 
     if (error) {
-        qDebug() << "Error uploading file to asset server";
+        qCWarning(asset_client) << "Error uploading file to asset server";
     } else {
         auto hash = packet->read(SHA256_HASH_LENGTH);
         hashString = hash.toHex();
         
-        qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
+        qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
     }
 
     // Check if we have any pending requests for this node

libraries/networking/src/AssetClient.h

@@ -40,6 +40,8 @@ class AssetClient : public QObject, public Dependency {
     Q_OBJECT
 public:
     AssetClient();
+    
+    Q_INVOKABLE void init();
 
     Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension);
     Q_INVOKABLE AssetUpload* createUpload(const QString& filename);

libraries/networking/src/AssetRequest.cpp

@@ -13,18 +13,20 @@
 
 #include <algorithm>
 
-#include <QThread>
+#include <QtCore/QThread>
+#include <QtNetwork/QAbstractNetworkCache>
 
 #include "AssetClient.h"
+#include "NetworkAccessManager.h"
 #include "NetworkLogging.h"
 #include "NodeList.h"
+#include "ResourceCache.h"
 
 AssetRequest::AssetRequest(const QString& hash, const QString& extension) :
     QObject(),
     _hash(hash),
     _extension(extension)
 {
-    
 }
 
 void AssetRequest::start() {
@@ -34,7 +36,19 @@ void AssetRequest::start() {
     }
 
     if (_state != NotStarted) {
-        qCWarning(networking) << "AssetRequest already started.";
+        qCWarning(asset_client) << "AssetRequest already started.";
+        return;
+    }
+    
+    // Try to load from cache
+    if (loadFromCache()) {
+        _info.hash = _hash;
+        _info.size = _data.size();
+        _error = NoError;
+        
+        _state = Finished;
+        emit finished(this);
+        qCDebug(asset_client) << getUrl().toDisplayString() << "loaded from disk cache.";
         return;
     }
     
@@ -58,8 +72,7 @@ void AssetRequest::start() {
         }
 
         if (_error != NoError) {
-            qCDebug(networking) << "Got error retrieving asset info for" << _hash;
+            qCWarning(asset_client) << "Got error retrieving asset info for" << _hash;
-
             _state = Finished;
             emit finished(this);
             
@@ -69,7 +82,7 @@ void AssetRequest::start() {
         _state = WaitingForData;
         _data.resize(info.size);
         
-        qCDebug(networking) << "Got size of " << _hash << " : " << info.size << " bytes";
+        qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes";
         
         int start = 0, end = _info.size;
         
@@ -98,6 +111,10 @@ void AssetRequest::start() {
                     memcpy(_data.data() + start, data.constData(), data.size());
                     _totalReceived += data.size();
                     emit progress(_totalReceived, _info.size);
+                    
+                    if (saveToCache(data)) {
+                        qCDebug(asset_client) << getUrl().toDisplayString() << "saved to disk cache";
+                    }
                 } else {
                     // hash doesn't match - we have an error
                     _error = HashVerificationFailed;
@@ -106,7 +123,7 @@ void AssetRequest::start() {
             }
             
             if (_error != NoError) {
-                qCDebug(networking) << "Got error retrieving asset" << _hash << "- error code" << _error;
+                qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error;
             }
             
             _state = Finished;
@@ -114,3 +131,51 @@ void AssetRequest::start() {
         });
     });
 }
+
+QUrl AssetRequest::getUrl() const {
+    if (!_extension.isEmpty()) {
+        return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, _hash, _extension));
+    } else {
+        return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, _hash));
+    }
+}
+
+bool AssetRequest::loadFromCache() {
+    if (auto cache = NetworkAccessManager::getInstance().cache()) {
+        auto url = getUrl();
+        if (auto ioDevice = cache->data(url)) {
+            _data = ioDevice->readAll();
+            return true;
+        } else {
+            qCDebug(asset_client) << url.toDisplayString() << "not in disk cache";
+        }
+    } else {
+        qCWarning(asset_client) << "No disk cache to load assets from.";
+    }
+    return false;
+}
+
+bool AssetRequest::saveToCache(const QByteArray& file) const {
+    if (auto cache = NetworkAccessManager::getInstance().cache()) {
+        auto url = getUrl();
+        
+        if (!cache->metaData(url).isValid()) {
+            QNetworkCacheMetaData metaData;
+            metaData.setUrl(url);
+            metaData.setSaveToDisk(true);
+            metaData.setLastModified(QDateTime::currentDateTime());
+            metaData.setExpirationDate(QDateTime()); // Never expires
+            
+            if (auto ioDevice = cache->prepare(metaData)) {
+                ioDevice->write(file);
+                cache->insert(ioDevice);
+                return true;
+            }
+            qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
+        }
+    } else {
+        qCWarning(asset_client) << "No disk cache to save assets to.";
+    }
+    return false;
+}
+

libraries/networking/src/AssetRequest.h

@@ -46,12 +46,16 @@ public:
     const QByteArray& getData() const { return _data; }
     const State& getState() const { return _state; }
     const Error& getError() const { return _error; }
+    QUrl getUrl() const;
 
 signals:
     void finished(AssetRequest* thisRequest);
     void progress(qint64 totalReceived, qint64 total);
 
 private:
+    bool loadFromCache();
+    bool saveToCache(const QByteArray& file) const;
+    
     State _state = NotStarted;
     Error _error = NoError;
     AssetInfo _info;

libraries/networking/src/AssetResourceRequest.cpp

@@ -32,7 +32,6 @@ void AssetResourceRequest::doSend() {
         _state = Finished;
 
         emit finished();
-
         return;
     }
 
@@ -43,12 +42,11 @@ void AssetResourceRequest::doSend() {
         _state = Finished;
 
         emit finished();
-
         return;
     }
 
     connect(_assetRequest, &AssetRequest::progress, this, &AssetResourceRequest::progress);
-    QObject::connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) mutable {
+    connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) {
         Q_ASSERT(_state == InProgress);
         Q_ASSERT(req == _assetRequest);
         Q_ASSERT(req->getState() == AssetRequest::Finished);

libraries/networking/src/AssetUpload.cpp

@@ -15,6 +15,7 @@
 #include <QtCore/QThread>
 
 #include "AssetClient.h"
+#include "NetworkLogging.h"
 
 AssetUpload::AssetUpload(QObject* object, const QString& filename) :
     _filename(filename)
@@ -41,7 +42,7 @@ void AssetUpload::start() {
         // ask the AssetClient to upload the asset and emit the proper signals from the passed callback
         auto assetClient = DependencyManager::get<AssetClient>();
         
-        qDebug() << "Attempting to upload" << _filename << "to asset-server.";
+        qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
         
         assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
             if (!responseReceived) {

libraries/networking/src/HTTPResourceRequest.cpp

@@ -29,8 +29,7 @@ HTTPResourceRequest::~HTTPResourceRequest() {
 }
 
 void HTTPResourceRequest::doSend() {
-    QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+    QNetworkRequest networkRequest(_url);
-    QNetworkRequest networkRequest = QNetworkRequest(_url);
     networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
 
     if (_cacheEnabled) {
@@ -39,7 +38,7 @@ void HTTPResourceRequest::doSend() {
         networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
     }
 
-    _reply = networkAccessManager.get(networkRequest);
+    _reply = NetworkAccessManager::getInstance().get(networkRequest);
     
     connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished);
     connect(_reply, &QNetworkReply::downloadProgress, this, &HTTPResourceRequest::onDownloadProgress);

libraries/networking/src/NetworkAccessManager.cpp

@@ -18,6 +18,7 @@ QThreadStorage<QNetworkAccessManager*> networkAccessManagers;
 QNetworkAccessManager& NetworkAccessManager::getInstance() {
     if (!networkAccessManagers.hasLocalData()) {
         networkAccessManagers.setLocalData(new QNetworkAccessManager());
+        
     }
     
     return *networkAccessManagers.localData();

libraries/networking/src/NetworkAccessManager.h

@@ -12,7 +12,7 @@
 #ifndef hifi_NetworkAccessManager_h
 #define hifi_NetworkAccessManager_h
 
-#include <QtNetwork/qnetworkaccessmanager.h>
+#include <QtNetwork/QNetworkAccessManager>
 
 /// Wrapper around QNetworkAccessManager to restrict at one instance by thread
 class NetworkAccessManager : public QObject {

libraries/networking/src/NetworkLogging.cpp

@@ -12,3 +12,4 @@
 #include "NetworkLogging.h"
 
 Q_LOGGING_CATEGORY(networking, "hifi.networking")
+Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client")

libraries/networking/src/NetworkLogging.h

@@ -15,5 +15,6 @@
 #include <QLoggingCategory>
 
 Q_DECLARE_LOGGING_CATEGORY(networking)
+Q_DECLARE_LOGGING_CATEGORY(asset_client)
 
 #endif // hifi_NetworkLogging_h

libraries/networking/src/ResourceCache.cpp

@@ -12,8 +12,6 @@
 #include <cfloat>
 #include <cmath>
 
-#include <QDebug>
-#include <QNetworkDiskCache>
 #include <QThread>
 #include <QTimer>
 

libraries/networking/src/ResourceCache.h

@@ -35,6 +35,7 @@ class Resource;
 
 static const qint64 BYTES_PER_MEGABYTES = 1024 * 1024;
 static const qint64 BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES;
+static const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES;  // 10GB
 
 // Windows can have troubles allocating that much memory in ram sometimes
 // so default cache size at 100 MB on windows (1GB otherwise)