From 395cc663ddb9c082f22d1cb6ff7bab45f43b189c Mon Sep 17 00:00:00 2001 From: humbletim Date: Fri, 19 Jan 2018 15:10:36 -0500 Subject: [PATCH] reuse same disk cache as AssetClient --- libraries/networking/src/AssetClient.cpp | 99 +++++++++++++++++++ libraries/networking/src/AssetClient.h | 3 + .../src/BaseAssetScriptingInterface.cpp | 99 ++++--------------- .../src/BaseAssetScriptingInterface.h | 20 ++-- .../src/AssetScriptingInterface.cpp | 22 ----- 5 files changed, 128 insertions(+), 115 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 6980621219..c126fc2e5a 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -100,6 +100,105 @@ void AssetClient::cacheInfoRequest(MiniPromise::Promise deferred) { } } +void AssetClient::queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); + return; + } + if (auto cache = NetworkAccessManager::getInstance().cache()) { + QNetworkCacheMetaData metaData = cache->metaData(url); + QVariantMap attributes, rawHeaders; + + QHashIterator i(metaData.attributes()); + while (i.hasNext()) { + i.next(); + attributes[QString::number(i.key())] = i.value(); + } + for (const auto& i : metaData.rawHeaders()) { + rawHeaders[i.first] = i.second; + } + deferred->resolve({ + { "isValid", metaData.isValid() }, + { "url", metaData.url() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + { "saveToDisk", metaData.saveToDisk() }, + { "attributes", attributes }, + { "rawHeaders", rawHeaders }, + }); + } else { + deferred->reject("cache currently unavailable"); + } +} + +void AssetClient::loadFromCache(MiniPromise::Promise deferred, const QUrl& url) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadFromCache", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); + return; + } + if (auto cache = NetworkAccessManager::getInstance().cache()) { + MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); + queryCacheMeta(metaRequest, url); + metaRequest->then([&](QString error, QVariantMap metadata) { + if (!error.isEmpty()) { + deferred->reject(error, metadata); + return; + } + QVariantMap result = { + { "metadata", metadata }, + { "data", QByteArray() }, + }; + // caller is responsible for the deletion of the ioDevice, hence the unique_ptr + if (auto ioDevice = std::unique_ptr(cache->data(url))) { + QByteArray data = ioDevice->readAll(); + result["data"] = data; + } else { + error = "cache data unavailable"; + } + deferred->handle(error, result); + }); + } else { + deferred->reject("cache currently unavailable"); + } +} + +namespace { + // parse RFC 1123 HTTP date format + QDateTime parseHttpDate(const QString& dateString) { + QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); + dt.setTimeSpec(Qt::UTC); + return dt; + } +} + +void AssetClient::saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& headers) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + QDateTime lastModified = headers.contains("last-modified") ? + parseHttpDate(headers["last-modified"].toString()) : + QDateTime::currentDateTimeUtc(); + QDateTime expirationDate = headers.contains("expires") ? + parseHttpDate(headers["expires"].toString()) : + QDateTime(); // never expires + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(lastModified); + metaData.setExpirationDate(expirationDate); + if (auto ioDevice = cache->prepare(metaData)) { + ioDevice->write(data); + cache->insert(ioDevice); + qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; + deferred->resolve({{ "success", true }}); + } else { + auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); + qCWarning(asset_client) << error; + deferred->reject(error); + } + } else { + deferred->reject("cache currently unavailable"); + } +} + void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "cacheInfoRequest", Qt::QueuedConnection, diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index c10ecf78a3..81149bf3d6 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -68,6 +68,9 @@ public slots: void cacheInfoRequest(QObject* reciever, QString slot); void cacheInfoRequest(MiniPromise::Promise deferred); + void queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url); + void loadFromCache(MiniPromise::Promise deferred, const QUrl& url); + void saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); void clearCache(); private slots: diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index 8172f12c79..f6f7fd87e3 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -31,31 +31,23 @@ QSharedPointer BaseAssetScriptingInterface::assetClient() { return DependencyManager::get(); } -BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent), _cache(this) { -} - +BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent) {} bool BaseAssetScriptingInterface::initializeCache() { - // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively - // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings - // become available we configure our instance to match. auto assets = assetClient(); if (!assets) { return false; // not yet possible to initialize the cache } - if (_cache.cacheDirectory().size()) { + if (!_cacheDirectory.isEmpty()) { return true; // cache is ready } // attempt to initialize the cache - QMetaObject::invokeMethod(assetClient().data(), "init"); + QMetaObject::invokeMethod(assets.data(), "init"); Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); deferred->then([&](QVariantMap result) { - auto cacheDirectory = result.value("cacheDirectory").toString(); - auto maximumCacheSize = result.value("maximumCacheSize").toLongLong(); - _cache.setCacheDirectory(cacheDirectory); - _cache.setMaximumCacheSize(maximumCacheSize); + _cacheDirectory = result.value("cacheDirectory").toString(); }); deferred->fail([&](QString error) { qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error; @@ -64,79 +56,28 @@ bool BaseAssetScriptingInterface::initializeCache() { return false; // cache is not ready yet } -QVariantMap BaseAssetScriptingInterface::getCacheStatus() { - return { - { "cacheDirectory", _cache.cacheDirectory() }, - { "cacheSize", _cache.cacheSize() }, - { "maximumCacheSize", _cache.maximumCacheSize() }, - }; +Promise BaseAssetScriptingInterface::getCacheStatus() { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->cacheInfoRequest(deferred); + return deferred; } -QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { - QNetworkCacheMetaData metaData = _cache.metaData(url); - QVariantMap attributes, rawHeaders; - - QHashIterator i(metaData.attributes()); - while (i.hasNext()) { - i.next(); - attributes[QString::number(i.key())] = i.value(); - } - for (const auto& i : metaData.rawHeaders()) { - rawHeaders[i.first] = i.second; - } - return { - { "isValid", metaData.isValid() }, - { "url", metaData.url() }, - { "expirationDate", metaData.expirationDate() }, - { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, - { "saveToDisk", metaData.saveToDisk() }, - { "attributes", attributes }, - { "rawHeaders", rawHeaders }, - }; +Promise BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->queryCacheMeta(deferred, url); + return deferred; } -QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { - QVariantMap result = { - { "metadata", queryCacheMeta(url) }, - { "data", QByteArray() }, - }; - // caller is responsible for the deletion of the ioDevice, hence the unique_ptr - if (auto ioDevice = std::unique_ptr(_cache.data(url))) { - QByteArray data = ioDevice->readAll(); - result["data"] = data; - } - return result; +Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->loadFromCache(deferred, url); + return deferred; } -namespace { - // parse RFC 1123 HTTP date format - QDateTime parseHttpDate(const QString& dateString) { - QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); - dt.setTimeSpec(Qt::UTC); - return dt; - } -} - -bool BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - QDateTime lastModified = headers.contains("last-modified") ? - parseHttpDate(headers["last-modified"].toString()) : - QDateTime::currentDateTimeUtc(); - QDateTime expirationDate = headers.contains("expires") ? - parseHttpDate(headers["expires"].toString()) : - QDateTime(); // never expires - QNetworkCacheMetaData metaData; - metaData.setUrl(url); - metaData.setSaveToDisk(true); - metaData.setLastModified(lastModified); - metaData.setExpirationDate(expirationDate); - if (auto ioDevice = _cache.prepare(metaData)) { - ioDevice->write(data); - _cache.insert(ioDevice); - qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; - return true; - } - qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache."; - return false; +Promise BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->saveToCache(deferred, url, data, headers); + return deferred; } Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) { diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h index 3aca3d394e..35c829fd37 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.h +++ b/libraries/networking/src/BaseAssetScriptingInterface.h @@ -33,15 +33,7 @@ public: BaseAssetScriptingInterface(QObject* parent = nullptr); public slots: - /**jsdoc - * Get the current status of the disk cache (if available) - * @function Assets.uploadData - * @static - * @return {String} result.cacheDirectory (path to the current disk cache) - * @return {Number} result.cacheSize (used cache size in bytes) - * @return {Number} result.maximumCacheSize (maxmimum cache size in bytes) - */ - QVariantMap getCacheStatus(); + Promise getCacheStatus(); /**jsdoc * Initialize the disk cache (returns true if already initialized) @@ -56,14 +48,14 @@ public slots: QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); } bool isValidHash(QString input) { return AssetUtils::isValidHash(input); } QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); } + QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); } - virtual QVariantMap queryCacheMeta(const QUrl& url); - virtual QVariantMap loadFromCache(const QUrl& url); - virtual bool saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + virtual Promise queryCacheMeta(const QUrl& url); + virtual Promise loadFromCache(const QUrl& url); + virtual Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); protected: - //void onCacheInfoResponse(QString cacheDirectory, qint64 cacheSize, qint64 maximumCacheSize); - QNetworkDiskCache _cache; + QString _cacheDirectory; const QString NoError{}; //virtual bool jsAssert(bool condition, const QString& error) = 0; Promise loadAsset(QString asset, bool decompress, QString responseType); diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index d80491b2a6..0870460a41 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -75,28 +75,6 @@ void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValu setMappingRequest->start(); } -void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) { - auto request = DependencyManager::get()->createGetMappingRequest(path); - QObject::connect(request, &GetMappingRequest::finished, this, [=](GetMappingRequest* request) mutable { - auto result = request->getError(); - if (callback.isFunction()) { - if (result == GetMappingRequest::NotFound) { - QScriptValueList args { "", true }; - callback.call(_engine->currentContext()->thisObject(), args); - } else if (result == GetMappingRequest::NoError) { - QScriptValueList args { request->getHash(), true }; - callback.call(_engine->currentContext()->thisObject(), args); - } else { - qCDebug(scriptengine) << "error -- " << request->getError() << " -- " << request->getErrorString(); - QScriptValueList args { "", false }; - callback.call(_engine->currentContext()->thisObject(), args); - } - request->deleteLater(); - } - }); - request->start(); -} - void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callback) { // FIXME: historically this API method failed silently when given a non-atp prefixed // urlString (or if the AssetRequest failed).