diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b6e0a871b3..3cb75b55ec 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -182,9 +182,6 @@ public: using namespace std; -// Starfield information -const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB - static QTimer* locationUpdateTimer = NULL; static QTimer* balanceUpdateTimer = NULL; static QTimer* identityPacketTimer = NULL; @@ -370,6 +367,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : _lastFaceTrackerUpdate(0), _applicationOverlay() { + thread()->setObjectName("Main Thread"); + setInstance(this); _entityClipboard->createRootElement(); @@ -460,13 +459,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : audioThread->start(); - QThread* assetThread = new QThread; - - assetThread->setObjectName("Asset Thread"); + // Setup AssetClient auto assetClient = DependencyManager::get(); - + QThread* assetThread = new QThread; + assetThread->setObjectName("Asset Thread"); assetClient->moveToThread(assetThread); - + connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); assetThread->start(); const DomainHandler& domainHandler = nodeList->getDomainHandler(); @@ -855,8 +853,7 @@ void Application::cleanupBeforeQuit() { } void Application::emptyLocalCache() { - QNetworkDiskCache* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); - if (cache) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { qDebug() << "DiskCacheEditor::clear(): Clearing disk cache."; cache->clear(); } diff --git a/interface/src/ui/DiskCacheEditor.cpp b/interface/src/ui/DiskCacheEditor.cpp index 2f2b924e13..a33f28e240 100644 --- a/interface/src/ui/DiskCacheEditor.cpp +++ b/interface/src/ui/DiskCacheEditor.cpp @@ -140,8 +140,7 @@ void DiskCacheEditor::clear() { "You are about to erase all the content of the disk cache," "are you sure you want to do that?"); if (buttonClicked == QMessageBox::Yes) { - QNetworkDiskCache* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); - if (cache) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { qDebug() << "DiskCacheEditor::clear(): Clearing disk cache."; cache->clear(); } diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index c13bc07be1..87af2a5cf8 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -11,16 +11,21 @@ #include "AssetClient.h" -#include -#include - #include +#include +#include +#include +#include + #include "AssetRequest.h" #include "AssetUpload.h" +#include "AssetUtils.h" +#include "NetworkAccessManager.h" +#include "NetworkLogging.h" #include "NodeList.h" #include "PacketReceiver.h" -#include "AssetUtils.h" +#include "ResourceCache.h" MessageID AssetClient::_currentID = 0; @@ -40,9 +45,29 @@ AssetClient::AssetClient() { connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &AssetClient::handleNodeKilled); } +void AssetClient::init() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection); + } + + // Setup disk cache if not already + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + if (!networkAccessManager.cache()) { + QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation); + cachePath = !cachePath.isEmpty() ? cachePath : "interfaceCache"; + + QNetworkDiskCache* cache = new QNetworkDiskCache(); + cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE); + cache->setCacheDirectory(cachePath); + networkAccessManager.setCache(cache); + qCDebug(asset_client) << "AssetClient disk cache setup at" << cachePath + << "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)"; + } +} + AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { - qDebug() << "Invalid hash size"; + qCWarning(asset_client) << "Invalid hash size"; return nullptr; } @@ -50,7 +75,7 @@ AssetRequest* AssetClient::createRequest(const QString& hash, const QString& ext SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer) { - qDebug().nospace() << "Could not request " << hash << "." << extension + qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension << " since you are not currently connected to an asset-server."; return nullptr; } @@ -68,7 +93,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) { SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (!assetServer) { - qDebug() << "Could not upload" << filename << "since you are not currently connected to an asset-server."; + qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server."; return nullptr; } @@ -82,7 +107,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) { bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { - qDebug() << "Invalid hash size"; + qCWarning(asset_client) << "Invalid hash size"; return false; } @@ -97,7 +122,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf + sizeof(start) + sizeof(end); auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true); - qDebug() << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server."; + qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server."; packet->writePrimitive(messageID); @@ -184,7 +209,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S packet.open(QIODevice::ReadOnly); auto assetHash = packet.read(SHA256_HASH_LENGTH); - qDebug() << "Got reply for asset: " << assetHash.toHex(); + qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex(); MessageID messageID; packet.read(reinterpret_cast(&messageID), sizeof(messageID)); @@ -198,7 +223,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S packet.read(reinterpret_cast(&length), sizeof(DataOffset)); data = packet.read(length); } else { - qDebug() << "Failure getting asset: " << error; + qCWarning(asset_client) << "Failure getting asset: " << error; } // Check if we have any pending requests for this node @@ -254,15 +279,15 @@ void AssetClient::handleAssetUploadReply(QSharedPointer packet, Shared AssetServerError error; packet->readPrimitive(&error); - QString hashString { "" }; + QString hashString; if (error) { - qDebug() << "Error uploading file to asset server"; + qCWarning(asset_client) << "Error uploading file to asset server"; } else { auto hash = packet->read(SHA256_HASH_LENGTH); hashString = hash.toHex(); - qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; + qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; } // Check if we have any pending requests for this node diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 6557954467..9b82e63b58 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -40,6 +40,8 @@ class AssetClient : public QObject, public Dependency { Q_OBJECT public: AssetClient(); + + Q_INVOKABLE void init(); Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 45e925389d..b4e3882a5e 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -13,18 +13,20 @@ #include -#include +#include +#include #include "AssetClient.h" +#include "NetworkAccessManager.h" #include "NetworkLogging.h" #include "NodeList.h" +#include "ResourceCache.h" AssetRequest::AssetRequest(const QString& hash, const QString& extension) : QObject(), _hash(hash), _extension(extension) { - } void AssetRequest::start() { @@ -34,7 +36,19 @@ void AssetRequest::start() { } if (_state != NotStarted) { - qCWarning(networking) << "AssetRequest already started."; + qCWarning(asset_client) << "AssetRequest already started."; + return; + } + + // Try to load from cache + if (loadFromCache()) { + _info.hash = _hash; + _info.size = _data.size(); + _error = NoError; + + _state = Finished; + emit finished(this); + qCDebug(asset_client) << getUrl().toDisplayString() << "loaded from disk cache."; return; } @@ -58,8 +72,7 @@ void AssetRequest::start() { } if (_error != NoError) { - qCDebug(networking) << "Got error retrieving asset info for" << _hash; - + qCWarning(asset_client) << "Got error retrieving asset info for" << _hash; _state = Finished; emit finished(this); @@ -69,7 +82,7 @@ void AssetRequest::start() { _state = WaitingForData; _data.resize(info.size); - qCDebug(networking) << "Got size of " << _hash << " : " << info.size << " bytes"; + qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes"; int start = 0, end = _info.size; @@ -98,6 +111,10 @@ void AssetRequest::start() { memcpy(_data.data() + start, data.constData(), data.size()); _totalReceived += data.size(); emit progress(_totalReceived, _info.size); + + if (saveToCache(data)) { + qCDebug(asset_client) << getUrl().toDisplayString() << "saved to disk cache"; + } } else { // hash doesn't match - we have an error _error = HashVerificationFailed; @@ -106,7 +123,7 @@ void AssetRequest::start() { } if (_error != NoError) { - qCDebug(networking) << "Got error retrieving asset" << _hash << "- error code" << _error; + qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error; } _state = Finished; @@ -114,3 +131,51 @@ void AssetRequest::start() { }); }); } + +QUrl AssetRequest::getUrl() const { + if (!_extension.isEmpty()) { + return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, _hash, _extension)); + } else { + return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, _hash)); + } +} + +bool AssetRequest::loadFromCache() { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + auto url = getUrl(); + if (auto ioDevice = cache->data(url)) { + _data = ioDevice->readAll(); + return true; + } else { + qCDebug(asset_client) << url.toDisplayString() << "not in disk cache"; + } + } else { + qCWarning(asset_client) << "No disk cache to load assets from."; + } + return false; +} + +bool AssetRequest::saveToCache(const QByteArray& file) const { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + auto url = getUrl(); + + if (!cache->metaData(url).isValid()) { + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(QDateTime::currentDateTime()); + metaData.setExpirationDate(QDateTime()); // Never expires + + if (auto ioDevice = cache->prepare(metaData)) { + ioDevice->write(file); + cache->insert(ioDevice); + return true; + } + qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache."; + } + } else { + qCWarning(asset_client) << "No disk cache to save assets to."; + } + return false; +} + diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 9bd224b35e..75e2353425 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -46,12 +46,16 @@ public: const QByteArray& getData() const { return _data; } const State& getState() const { return _state; } const Error& getError() const { return _error; } + QUrl getUrl() const; signals: void finished(AssetRequest* thisRequest); void progress(qint64 totalReceived, qint64 total); private: + bool loadFromCache(); + bool saveToCache(const QByteArray& file) const; + State _state = NotStarted; Error _error = NoError; AssetInfo _info; diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 6b419fc293..ecbe80cddb 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -32,7 +32,6 @@ void AssetResourceRequest::doSend() { _state = Finished; emit finished(); - return; } @@ -43,12 +42,11 @@ void AssetResourceRequest::doSend() { _state = Finished; emit finished(); - return; } connect(_assetRequest, &AssetRequest::progress, this, &AssetResourceRequest::progress); - QObject::connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) mutable { + connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) { Q_ASSERT(_state == InProgress); Q_ASSERT(req == _assetRequest); Q_ASSERT(req->getState() == AssetRequest::Finished); diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index 9c9d172959..92632a43e5 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -15,6 +15,7 @@ #include #include "AssetClient.h" +#include "NetworkLogging.h" AssetUpload::AssetUpload(QObject* object, const QString& filename) : _filename(filename) @@ -41,7 +42,7 @@ void AssetUpload::start() { // ask the AssetClient to upload the asset and emit the proper signals from the passed callback auto assetClient = DependencyManager::get(); - qDebug() << "Attempting to upload" << _filename << "to asset-server."; + qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server."; assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){ if (!responseReceived) { diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index fd650df348..34c9c1dad2 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -29,8 +29,7 @@ HTTPResourceRequest::~HTTPResourceRequest() { } void HTTPResourceRequest::doSend() { - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest networkRequest = QNetworkRequest(_url); + QNetworkRequest networkRequest(_url); networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); if (_cacheEnabled) { @@ -39,7 +38,7 @@ void HTTPResourceRequest::doSend() { networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); } - _reply = networkAccessManager.get(networkRequest); + _reply = NetworkAccessManager::getInstance().get(networkRequest); connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished); connect(_reply, &QNetworkReply::downloadProgress, this, &HTTPResourceRequest::onDownloadProgress); diff --git a/libraries/networking/src/NetworkAccessManager.cpp b/libraries/networking/src/NetworkAccessManager.cpp index 841b7491c7..97171d5ad7 100644 --- a/libraries/networking/src/NetworkAccessManager.cpp +++ b/libraries/networking/src/NetworkAccessManager.cpp @@ -18,6 +18,7 @@ QThreadStorage networkAccessManagers; QNetworkAccessManager& NetworkAccessManager::getInstance() { if (!networkAccessManagers.hasLocalData()) { networkAccessManagers.setLocalData(new QNetworkAccessManager()); + } return *networkAccessManagers.localData(); diff --git a/libraries/networking/src/NetworkAccessManager.h b/libraries/networking/src/NetworkAccessManager.h index 5a64f4ae0a..c4b435adb6 100644 --- a/libraries/networking/src/NetworkAccessManager.h +++ b/libraries/networking/src/NetworkAccessManager.h @@ -12,7 +12,7 @@ #ifndef hifi_NetworkAccessManager_h #define hifi_NetworkAccessManager_h -#include +#include /// Wrapper around QNetworkAccessManager to restrict at one instance by thread class NetworkAccessManager : public QObject { diff --git a/libraries/networking/src/NetworkLogging.cpp b/libraries/networking/src/NetworkLogging.cpp index 45ff716a97..28b209960e 100644 --- a/libraries/networking/src/NetworkLogging.cpp +++ b/libraries/networking/src/NetworkLogging.cpp @@ -12,3 +12,4 @@ #include "NetworkLogging.h" Q_LOGGING_CATEGORY(networking, "hifi.networking") +Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client") diff --git a/libraries/networking/src/NetworkLogging.h b/libraries/networking/src/NetworkLogging.h index 47a6a34264..838bbb57d2 100644 --- a/libraries/networking/src/NetworkLogging.h +++ b/libraries/networking/src/NetworkLogging.h @@ -15,5 +15,6 @@ #include Q_DECLARE_LOGGING_CATEGORY(networking) +Q_DECLARE_LOGGING_CATEGORY(asset_client) #endif // hifi_NetworkLogging_h diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 18135cf6df..5c47a831e8 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -12,8 +12,6 @@ #include #include -#include -#include #include #include diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 3bae7f5b9d..97e46f088a 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -35,6 +35,7 @@ class Resource; static const qint64 BYTES_PER_MEGABYTES = 1024 * 1024; static const qint64 BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES; +static const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB // Windows can have troubles allocating that much memory in ram sometimes // so default cache size at 100 MB on windows (1GB otherwise)