From 9678e704440544d3c15e426eac23c363e98053bc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 15 Sep 2015 15:27:38 +0200 Subject: [PATCH] Asset caching --- interface/src/Application.cpp | 15 ++--- interface/src/ui/DiskCacheEditor.cpp | 3 +- libraries/networking/src/AssetClient.cpp | 41 ++++++++--- libraries/networking/src/AssetClient.h | 2 + libraries/networking/src/AssetRequest.cpp | 75 +++++++++++++++++++-- libraries/networking/src/AssetRequest.h | 4 ++ libraries/networking/src/AssetUpload.cpp | 3 +- libraries/networking/src/NetworkLogging.cpp | 1 + libraries/networking/src/NetworkLogging.h | 1 + libraries/networking/src/ResourceCache.cpp | 2 - libraries/networking/src/ResourceCache.h | 1 + 11 files changed, 118 insertions(+), 30 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 261cd5f2e0..3cb75b55ec 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -182,9 +182,6 @@ public: using namespace std; -// Starfield information -const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB - static QTimer* locationUpdateTimer = NULL; static QTimer* balanceUpdateTimer = NULL; static QTimer* identityPacketTimer = NULL; @@ -462,13 +459,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : audioThread->start(); - QThread* assetThread = new QThread; - - assetThread->setObjectName("Asset Thread"); + // Setup AssetClient auto assetClient = DependencyManager::get(); - + QThread* assetThread = new QThread; + assetThread->setObjectName("Asset Thread"); assetClient->moveToThread(assetThread); - + connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); assetThread->start(); const DomainHandler& domainHandler = nodeList->getDomainHandler(); @@ -857,8 +853,7 @@ void Application::cleanupBeforeQuit() { } void Application::emptyLocalCache() { - QNetworkDiskCache* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); - if (cache) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { qDebug() << "DiskCacheEditor::clear(): Clearing disk cache."; cache->clear(); } diff --git a/interface/src/ui/DiskCacheEditor.cpp b/interface/src/ui/DiskCacheEditor.cpp index 2f2b924e13..a33f28e240 100644 --- a/interface/src/ui/DiskCacheEditor.cpp +++ b/interface/src/ui/DiskCacheEditor.cpp @@ -140,8 +140,7 @@ void DiskCacheEditor::clear() { "You are about to erase all the content of the disk cache," "are you sure you want to do that?"); if (buttonClicked == QMessageBox::Yes) { - QNetworkDiskCache* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); - if (cache) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { qDebug() << "DiskCacheEditor::clear(): Clearing disk cache."; cache->clear(); } diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 12f974796a..87e9b4c060 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -14,13 +14,18 @@ #include #include +#include #include +#include #include "AssetRequest.h" #include "AssetUpload.h" #include "AssetUtils.h" +#include "NetworkAccessManager.h" +#include "NetworkLogging.h" #include "NodeList.h" #include "PacketReceiver.h" +#include "ResourceCache.h" MessageID AssetClient::_currentID = 0; @@ -40,9 +45,25 @@ AssetClient::AssetClient() { connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &AssetClient::handleNodeKilled); } +void AssetClient::init() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection); + } + + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + if (!networkAccessManager.cache()) { + QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation); + QNetworkDiskCache* cache = new QNetworkDiskCache(); + cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE); + cache->setCacheDirectory(!cachePath.isEmpty() ? cachePath : "interfaceCache"); + networkAccessManager.setCache(cache); + qCDebug(asset_client) << "AssetClient disk cache setup."; + } +} + AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { - qDebug() << "Invalid hash size"; + qCWarning(asset_client) << "Invalid hash size"; return nullptr; } @@ -50,7 +71,7 @@ AssetRequest* AssetClient::createRequest(const QString& hash, const QString& ext SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer) { - qDebug().nospace() << "Could not request " << hash << "." << extension + qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension << " since you are not currently connected to an asset-server."; return nullptr; } @@ -68,7 +89,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) { SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer) { - qDebug() << "Could not upload" << filename << "since you are not currently connected to an asset-server."; + qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server."; return nullptr; } @@ -82,7 +103,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) { bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { - qDebug() << "Invalid hash size"; + qCWarning(asset_client) << "Invalid hash size"; return false; } @@ -97,7 +118,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf + sizeof(start) + sizeof(end); auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true); - qDebug() << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server."; + qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server."; packet->writePrimitive(messageID); @@ -184,7 +205,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S packet.open(QIODevice::ReadOnly); auto assetHash = packet.read(SHA256_HASH_LENGTH); - qDebug() << "Got reply for asset: " << assetHash.toHex(); + qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex(); MessageID messageID; packet.read(reinterpret_cast(&messageID), sizeof(messageID)); @@ -198,7 +219,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S packet.read(reinterpret_cast(&length), sizeof(DataOffset)); data = packet.read(length); } else { - qDebug() << "Failure getting asset: " << error; + qCWarning(asset_client) << "Failure getting asset: " << error; } // Check if we have any pending requests for this node @@ -254,15 +275,15 @@ void AssetClient::handleAssetUploadReply(QSharedPointer packet, Shared AssetServerError error; packet->readPrimitive(&error); - QString hashString { "" }; + QString hashString; if (error) { - qDebug() << "Error uploading file to asset server"; + qCWarning(asset_client) << "Error uploading file to asset server"; } else { auto hash = packet->read(SHA256_HASH_LENGTH); hashString = hash.toHex(); - qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; + qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; } // Check if we have any pending requests for this node diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 6557954467..9b82e63b58 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -40,6 +40,8 @@ class AssetClient : public QObject, public Dependency { Q_OBJECT public: AssetClient(); + + Q_INVOKABLE void init(); Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 7490e2eb27..934d64c263 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -14,17 +14,19 @@ #include #include +#include #include "AssetClient.h" +#include "NetworkAccessManager.h" #include "NetworkLogging.h" #include "NodeList.h" +#include "ResourceCache.h" AssetRequest::AssetRequest(const QString& hash, const QString& extension) : QObject(), _hash(hash), _extension(extension) { - } void AssetRequest::start() { @@ -34,7 +36,18 @@ void AssetRequest::start() { } if (_state != NotStarted) { - qCWarning(networking) << "AssetRequest already started."; + qCWarning(asset_client) << "AssetRequest already started."; + return; + } + + if (loadFromCache()) { + _info.hash = _hash; + _info.size = _data.size(); + _error = NoError; + + _state = Finished; + emit finished(this); + qCDebug(asset_client) << getUrl().toDisplayString() << "loaded from disk cache."; return; } @@ -58,7 +71,7 @@ void AssetRequest::start() { } if (_error != NoError) { - qCDebug(networking) << "Got error retrieving asset info for" << _hash; + qCWarning(asset_client) << "Got error retrieving asset info for" << _hash; _state = Finished; emit finished(this); @@ -68,7 +81,7 @@ void AssetRequest::start() { _state = WaitingForData; _data.resize(info.size); - qCDebug(networking) << "Got size of " << _hash << " : " << info.size << " bytes"; + qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes"; int start = 0, end = _info.size; @@ -97,6 +110,10 @@ void AssetRequest::start() { memcpy(_data.data() + start, data.constData(), data.size()); _totalReceived += data.size(); emit progress(_totalReceived, _info.size); + + if (saveToCache(data)) { + qCDebug(asset_client) << getUrl().toDisplayString() << "saved to disk cache"; + } } else { // hash doesn't match - we have an error _error = HashVerificationFailed; @@ -105,7 +122,7 @@ void AssetRequest::start() { } if (_error != NoError) { - qCDebug(networking) << "Got error retrieving asset" << _hash << "- error code" << _error; + qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error; } _state = Finished; @@ -113,3 +130,51 @@ void AssetRequest::start() { }); }); } + +QUrl AssetRequest::getUrl() const { + if (!_extension.isEmpty()) { + return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, _hash, _extension)); + } else { + return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, _hash)); + } +} + +bool AssetRequest::loadFromCache() { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + auto url = getUrl(); + if (auto ioDevice = cache->data(url)) { + _data = ioDevice->readAll(); + return true; + } else { + qCDebug(asset_client) << url.toDisplayString() << "not in disk cache"; + } + } else { + qCWarning(asset_client) << "No disk cache to load assets from."; + } + return false; +} + +bool AssetRequest::saveToCache(const QByteArray& file) const { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + auto url = getUrl(); + + if (!cache->metaData(url).isValid()) { + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(QDateTime::currentDateTime()); + metaData.setExpirationDate(QDateTime()); // Never expires + + if (auto ioDevice = cache->prepare(metaData)) { + ioDevice->write(file); + cache->insert(ioDevice); + return true; + } + qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache."; + } + } else { + qCWarning(asset_client) << "No disk cache to save assets to."; + } + return false; +} + diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 9bd224b35e..75e2353425 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -46,12 +46,16 @@ public: const QByteArray& getData() const { return _data; } const State& getState() const { return _state; } const Error& getError() const { return _error; } + QUrl getUrl() const; signals: void finished(AssetRequest* thisRequest); void progress(qint64 totalReceived, qint64 total); private: + bool loadFromCache(); + bool saveToCache(const QByteArray& file) const; + State _state = NotStarted; Error _error = NoError; AssetInfo _info; diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index 9c9d172959..92632a43e5 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -15,6 +15,7 @@ #include #include "AssetClient.h" +#include "NetworkLogging.h" AssetUpload::AssetUpload(QObject* object, const QString& filename) : _filename(filename) @@ -41,7 +42,7 @@ void AssetUpload::start() { // ask the AssetClient to upload the asset and emit the proper signals from the passed callback auto assetClient = DependencyManager::get(); - qDebug() << "Attempting to upload" << _filename << "to asset-server."; + qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server."; assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){ if (!responseReceived) { diff --git a/libraries/networking/src/NetworkLogging.cpp b/libraries/networking/src/NetworkLogging.cpp index 45ff716a97..28b209960e 100644 --- a/libraries/networking/src/NetworkLogging.cpp +++ b/libraries/networking/src/NetworkLogging.cpp @@ -12,3 +12,4 @@ #include "NetworkLogging.h" Q_LOGGING_CATEGORY(networking, "hifi.networking") +Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client") diff --git a/libraries/networking/src/NetworkLogging.h b/libraries/networking/src/NetworkLogging.h index 47a6a34264..838bbb57d2 100644 --- a/libraries/networking/src/NetworkLogging.h +++ b/libraries/networking/src/NetworkLogging.h @@ -15,5 +15,6 @@ #include Q_DECLARE_LOGGING_CATEGORY(networking) +Q_DECLARE_LOGGING_CATEGORY(asset_client) #endif // hifi_NetworkLogging_h diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 18135cf6df..5c47a831e8 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -12,8 +12,6 @@ #include #include -#include -#include #include #include diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 3bae7f5b9d..97e46f088a 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -35,6 +35,7 @@ class Resource; static const qint64 BYTES_PER_MEGABYTES = 1024 * 1024; static const qint64 BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES; +static const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB // Windows can have troubles allocating that much memory in ram sometimes // so default cache size at 100 MB on windows (1GB otherwise)