From 7e1be2b17c9a1640aa1dc5fb314ecdcf589ab24a Mon Sep 17 00:00:00 2001 From: humbletim Date: Thu, 21 Dec 2017 11:35:14 -0500 Subject: [PATCH 01/21] shared promises library --- libraries/shared/src/shared/MiniPromises.cpp | 4 + libraries/shared/src/shared/MiniPromises.h | 249 +++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 libraries/shared/src/shared/MiniPromises.cpp create mode 100644 libraries/shared/src/shared/MiniPromises.h diff --git a/libraries/shared/src/shared/MiniPromises.cpp b/libraries/shared/src/shared/MiniPromises.cpp new file mode 100644 index 0000000000..fdfb509608 --- /dev/null +++ b/libraries/shared/src/shared/MiniPromises.cpp @@ -0,0 +1,4 @@ +#include "MiniPromises.h" +namespace { + int promiseMetaTypeId = qRegisterMetaType("MiniPromise::Promise"); +} diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h new file mode 100644 index 0000000000..17f2394a2f --- /dev/null +++ b/libraries/shared/src/shared/MiniPromises.h @@ -0,0 +1,249 @@ +#pragma once + +// Minimalist threadsafe Promise-like helper for instrumenting asynchronous results +// +// This class pivots around composable continuation-style callback handlers: +// auto successCallback = [=](QVariantMap result) { .... } +// auto errorCallback = [=](QString error) { .... } +// auto combinedCallback = [=](QString error, QVariantMap result) { .... } +// +// * Callback Handlers are automatically invoked on the right thread (the Promise's thread). +// * Callbacks can be assigned anytime during a Promise's life and "do the right thing". +// - ie: for code clarity you can define success cases first (or maintain time order) +// * "Deferred" concept can be used to publish outcomes. +// * "Promise" concept be used to subscribe to outcomes. +// +// See AssetScriptingInterface.cpp for some examples of using to simplify chained async result. + +#include +#include +#include +#include +#include "ReadWriteLockable.h" +#include + +class MiniPromise : public QObject, public std::enable_shared_from_this, public ReadWriteLockable { + Q_OBJECT +public: + using handlerFunction = std::function; + using successFunction = std::function; + using errorFunction = std::function; + using handlers = QVector; + using Promise = std::shared_ptr; + MiniPromise(const QString debugName) { setObjectName(debugName); } + MiniPromise() {} + ~MiniPromise() { + qDebug() << "~MiniPromise" << objectName(); + if (!_rejected && !_resolved) { + qWarning() << "====== WARNING: unhandled MiniPromise" << objectName() << _error << _result; + } + } + + QString _error; + QVariantMap _result; + std::atomic _rejected{false}; + std::atomic _resolved{false}; + handlers _onresolve; + handlers _onreject; + handlers _onfinally; + + Promise self() { return shared_from_this(); } + + // result aggregation helpers -- eg: deferred->defaults(interimResultMap)->ready(...) + + // copy values from the input map, but only for keys that don't already exist + Promise mixin(const QVariantMap& source) { + qDebug() << objectName() << "mixin"; + withWriteLock([&]{ + for (const auto& key : source.keys()) { + if (!_result.contains(key)) { + _result[key] = source[key]; + } + } + }); + return self(); + } + // copy values from the input map, replacing any existing keys + Promise assignResult(const QVariantMap& source) { + qDebug() << objectName() << "assignResult"; + withWriteLock([&]{ + for (const auto& key : source.keys()) { + _result[key] = source[key]; + } + }); + return self(); + } + + // TODO: I think calling as "ready" makes it read better, but is customary Promise "finally" sufficient? + Promise ready(handlerFunction always) { return finally(always); } + Promise finally(handlerFunction always) { + if (!_rejected && !_resolved) { + withWriteLock([&]{ + _onfinally << always; + }); + } else { + qDebug() << "finally (already resolved/rejected)" << objectName(); + executeOnPromiseThread([&]{ + withReadLock([&]{ + always(_error, _result); + }); + }); + } + return self(); + } + Promise fail(errorFunction errorOnly) { + return fail([this, errorOnly](QString error, QVariantMap result) { + errorOnly(error); + }); + } + + Promise fail(handlerFunction failFunc) { + if (!_rejected) { + withWriteLock([&]{ + _onreject << failFunc; + }); + } else { + executeOnPromiseThread([&]{ + qDebug() << "fail (already rejected)" << objectName(); + withReadLock([&]{ + failFunc(_error, _result); + }); + }); + } + return self(); + } + + Promise then(successFunction successOnly) { + return then([this, successOnly](QString error, QVariantMap result) { + successOnly(result); + }); + } + Promise then(handlerFunction successFunc) { + if (!_resolved) { + withWriteLock([&]{ + _onresolve << successFunc; + }); + } else { + executeOnPromiseThread([&]{ + qDebug() << "fail (already resolved)" << objectName(); + withReadLock([&]{ + successFunc(_error, _result); + }); + }); + } + return self(); + } + // register combined success/error handlers + Promise then(successFunction successOnly, errorFunction errorOnly) { + // note: first arg can be null (see ES6 .then(null, errorHandler) conventions) + if (successOnly) { + then(successOnly); + } + if (errorOnly) { + fail(errorOnly); + } + return self(); + } + + // trigger methods + Promise handle(QString error, const QVariantMap& result) { + qDebug() << "handle" << objectName() << error; + if (error.isEmpty()) { + resolve(error, result); + } else { + reject(error, result); + } + return self(); + } + Promise resolve(QVariantMap result) { + return resolve(QString(), result); + } + + Q_INVOKABLE void executeOnPromiseThread(std::function function) { + if (QThread::currentThread() != thread()) { + qDebug() << "-0-0-00-0--0" << objectName() << "executeOnPromiseThread -- wrong thread" << QThread::currentThread(); + QMetaObject::invokeMethod( + this, "executeOnPromiseThread", Qt::BlockingQueuedConnection, + Q_ARG(std::function, function)); + } else { + function(); + } + } + + Promise setState(bool resolved, QString error, const QVariantMap& result) { + qDebug() << "setState" << objectName() << resolved << error; + if (resolved) { + _resolved = true; + } else { + _rejected = true; + } + withWriteLock([&]{ + _error = error; + }); + assignResult(result); + qDebug() << "//setState" << objectName() << resolved << error; + return self(); + } + Promise resolve(QString error, const QVariantMap& result) { + setState(true, error, result); + qDebug() << "handle" << objectName() << error; + { + QString error; + QVariantMap result; + handlers toresolve; + handlers tofinally; + withReadLock([&]{ + error = _error; + result = _result; + toresolve = _onresolve; + tofinally = _onfinally; + }); + executeOnPromiseThread([&]{ + for (const auto& onresolve : toresolve) { + onresolve(error, result); + } + for (const auto& onfinally : tofinally) { + onfinally(error, result); + } + }); + } + return self(); + } + Promise reject(QString error) { + return reject(error, QVariantMap()); + } + Promise reject(QString error, const QVariantMap& result) { + setState(false, error, result); + qDebug() << "handle" << objectName() << error; + { + QString error; + QVariantMap result; + handlers toreject; + handlers tofinally; + withReadLock([&]{ + error = _error; + result = _result; + toreject = _onreject; + tofinally = _onfinally; + }); + executeOnPromiseThread([&]{ + for (const auto& onreject : toreject) { + onreject(error, result); + } + for (const auto& onfinally : tofinally) { + onfinally(error, result); + } + if (toreject.isEmpty() && tofinally.isEmpty()) { + qWarning() << "WARNING: unhandled MiniPromise::reject" << objectName() << error << result; + } + }); + } + return self(); + } +}; + +inline MiniPromise::Promise makePromise(const QString& hint = QString()) { + return std::make_shared(hint); +} + +Q_DECLARE_METATYPE(MiniPromise::Promise) From 8c7a8f0df348c1a94e450c19e3b8a0d30ec77548 Mon Sep 17 00:00:00 2001 From: humbletim Date: Fri, 15 Dec 2017 14:12:04 -0500 Subject: [PATCH 02/21] expanded Assets scripting interface --- libraries/networking/src/AssetClient.cpp | 25 ++ libraries/networking/src/AssetClient.h | 2 + libraries/networking/src/AssetRequest.cpp | 11 + libraries/networking/src/AssetRequest.h | 3 +- .../networking/src/AssetResourceRequest.cpp | 2 +- libraries/networking/src/AssetUtils.cpp | 33 +- libraries/networking/src/AssetUtils.h | 17 +- .../src/BaseAssetScriptingInterface.cpp | 399 ++++++++++++++++++ .../src/BaseAssetScriptingInterface.h | 78 ++++ .../src/AssetScriptingInterface.cpp | 349 +++++++++++++-- .../src/AssetScriptingInterface.h | 37 +- .../tests/unit_tests/assetUnitTests.js | 181 ++++++++ 12 files changed, 1090 insertions(+), 47 deletions(-) create mode 100644 libraries/networking/src/BaseAssetScriptingInterface.cpp create mode 100644 libraries/networking/src/BaseAssetScriptingInterface.h create mode 100644 scripts/developer/tests/unit_tests/assetUnitTests.js diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 6fdfc5a42b..725975c9fb 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "AssetRequest.h" #include "AssetUpload.h" @@ -72,6 +73,30 @@ void AssetClient::init() { networkAccessManager.setCache(cache); qInfo() << "ResourceManager disk cache setup at" << _cacheDir << "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)"; + } else { + auto cache = qobject_cast(networkAccessManager.cache()); + qInfo() << "ResourceManager disk cache already setup at" << cache->cacheDirectory() + << "(size:" << cache->maximumCacheSize() / BYTES_PER_GIGABYTES << "GB)"; + } + +} + +void AssetClient::cacheInfoRequest(MiniPromise::Promise deferred) { + if (QThread::currentThread() != thread()) { + if (!QMetaType::isRegistered(qMetaTypeId())) { + qRegisterMetaType(); + } + QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred)); + return; + } + if (auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache())) { + deferred->resolve({ + { "cacheDirectory", cache->cacheDirectory() }, + { "cacheSize", cache->cacheSize() }, + { "maximumCacheSize", cache->maximumCacheSize() }, + }); + } else { + deferred->reject("Cache not available"); } } diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 8035aa886e..5d854390e2 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -19,6 +19,7 @@ #include #include +#include #include "AssetUtils.h" #include "ByteRange.h" @@ -66,6 +67,7 @@ public slots: void init(); void cacheInfoRequest(QObject* reciever, QString slot); + void cacheInfoRequest(MiniPromise::Promise deferred); void clearCache(); private slots: diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 7fa563d4ad..6a2bcdca9c 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -134,3 +134,14 @@ void AssetRequest::start() { emit progress(totalReceived, total); }); } + + +const QString AssetRequest::getErrorString() const { + QString result; + if (_error != Error::NoError) { + QVariant v; + v.setValue(_error); + result = v.toString(); // courtesy of Q_ENUM + } + return result; +} diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index a7213a90d7..b469e2f012 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -42,7 +42,7 @@ public: NetworkError, UnknownError }; - + Q_ENUM(Error) AssetRequest(const QString& hash, const ByteRange& byteRange = ByteRange()); virtual ~AssetRequest() override; @@ -51,6 +51,7 @@ public: const QByteArray& getData() const { return _data; } const State& getState() const { return _state; } const Error& getError() const { return _error; } + const QString getErrorString() const; QUrl getUrl() const { return ::getATPUrl(_hash); } QString getHash() const { return _hash; } diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 55d0ad4f75..e1a155a561 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -146,7 +146,7 @@ void AssetResourceRequest::requestHash(const AssetHash& hash) { _assetRequest = assetClient->createRequest(hash, _byteRange); connect(_assetRequest, &AssetRequest::progress, this, &AssetResourceRequest::onDownloadProgress); - connect(_assetRequest, &AssetRequest::finished, this, [this](AssetRequest* req) { + connect(_assetRequest, &AssetRequest::finished, this, [this, hash](AssetRequest* req) { Q_ASSERT(_state == InProgress); Q_ASSERT(req == _assetRequest); Q_ASSERT(req->getState() == AssetRequest::Finished); diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 76fda6aed4..fa9885f053 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -15,6 +15,7 @@ #include #include +#include // for baseName #include #include "NetworkAccessManager.h" @@ -22,8 +23,36 @@ #include "ResourceManager.h" -QUrl getATPUrl(const QString& hash) { - return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, hash)); +// Extract the valid AssetHash portion from atp: URLs like "[atp:]HASH[.fbx][?query]" +// (or an invalid AssetHash if not found) +AssetHash extractAssetHash(const QString& input) { + if (isValidHash(input)) { + return input; + } + QString path = getATPUrl(input).path(); + QString baseName = QFileInfo(path).baseName(); + if (isValidHash(baseName)) { + return baseName; + } + return AssetHash(); +} + +// Get the normalized ATP URL for a raw hash, /path or "atp:" input string. +QUrl getATPUrl(const QString& input) { + QUrl url = input; + if (!url.scheme().isEmpty() && url.scheme() != URL_SCHEME_ATP) { + return QUrl(); + } + // this strips extraneous info from the URL (while preserving fragment/querystring) + QString path = url.toEncoded( + QUrl::RemoveAuthority | QUrl::RemoveScheme | + QUrl::StripTrailingSlash | QUrl::NormalizePathSegments + ); + QString baseName = QFileInfo(path).baseName(); + if (isValidPath(path) || isValidHash(baseName)) { + return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP).arg(path)); + } + return QUrl(); } QByteArray hashData(const QByteArray& data) { diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index a7c053c3d6..8c89b4701c 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -71,7 +71,8 @@ struct MappingInfo { using AssetMapping = std::map; -QUrl getATPUrl(const QString& hash); +QUrl getATPUrl(const QString& input); +AssetHash extractAssetHash(const QString& input); QByteArray hashData(const QByteArray& data); @@ -84,4 +85,18 @@ bool isValidHash(const QString& hashString); QString bakingStatusToString(BakingStatus status); +// backwards-compatible namespace aliases +// (allows new code to be explicit -- eg: `AssetUtils::isValidPath(path)` vs. `isValidPath(path)`) +namespace AssetUtils { + static const auto& loadFromCache = ::loadFromCache; + static const auto& saveToCache = ::saveToCache; + static const auto& hashData = ::hashData; + static const auto& getATPUrl = ::getATPUrl; + static const auto& extractAssetHash = ::extractAssetHash; + static const auto& isValidFilePath = ::isValidFilePath; + static const auto& isValidPath = ::isValidPath; + static const auto& isValidHash = ::isValidHash; +}; + + #endif // hifi_AssetUtils_h diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp new file mode 100644 index 0000000000..bfdc21c7d5 --- /dev/null +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -0,0 +1,399 @@ +// +// BaseAssetScriptingInterface.cpp +// libraries/networking/src +// +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include "BaseAssetScriptingInterface.h" + +#include +#include +#include +#include + +#include "AssetRequest.h" +#include "AssetUpload.h" +#include "AssetUtils.h" +#include "MappingRequest.h" +#include "NetworkLogging.h" + +#include + +#include +#include "Gzip.h" + +using Promise = MiniPromise::Promise; + +QSharedPointer BaseAssetScriptingInterface::assetClient() { + return DependencyManager::get(); +} + +BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent), _cache(this) { +} + + +bool BaseAssetScriptingInterface::initializeCache() { + qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- current values" << _cache.cacheDirectory() << _cache.cacheSize() << _cache.maximumCacheSize(); + + // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively + // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings + // become available we configure our instance to match. + auto assets = assetClient(); + if (!assets) { + return false; // not yet possible to initialize the cache + } + if (_cache.cacheDirectory().size()) { + return true; // cache is ready + } + + // attempt to initialize the cache + qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- invoking AssetClient::init" << assetClient().data(); + QMetaObject::invokeMethod(assetClient().data(), "init"); + + qDebug() << "BaseAssetScriptingInterface::getCacheStatus querying cache status" << QThread::currentThread(); + Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); + deferred->then([&](QVariantMap result) { + qDebug() << "//queryCacheStatus" << QThread::currentThread(); + auto cacheDirectory = result.value("cacheDirectory").toString(); + auto cacheSize = result.value("cacheSize").toLongLong(); + auto maximumCacheSize = result.value("maximumCacheSize").toLongLong(); + qDebug() << "///queryCacheStatus" << cacheDirectory << cacheSize << maximumCacheSize; + _cache.setCacheDirectory(cacheDirectory); + _cache.setMaximumCacheSize(maximumCacheSize); + }); + deferred->fail([&](QString error) { + qDebug() << "//queryCacheStatus ERROR" << QThread::currentThread() << error; + }); + assets->cacheInfoRequest(deferred); + return false; // cache is not ready yet +} + +QVariantMap BaseAssetScriptingInterface::getCacheStatus() { + //assetClient()->cacheInfoRequest(this, "onCacheInfoResponse"); + return { + { "cacheDirectory", _cache.cacheDirectory() }, + { "cacheSize", _cache.cacheSize() }, + { "maximumCacheSize", _cache.maximumCacheSize() }, + }; +} + +QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { + QNetworkCacheMetaData metaData = _cache.metaData(url); + QVariantMap attributes, rawHeaders; + + QHashIterator i(metaData.attributes()); + while (i.hasNext()) { + i.next(); + attributes[QString::number(i.key())] = i.value(); + } + for (const auto& i : metaData.rawHeaders()) { + rawHeaders[i.first] = i.second; + } + return { + { "isValid", metaData.isValid() }, + { "url", metaData.url() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + { "saveToDisk", metaData.saveToDisk() }, + { "attributes", attributes }, + { "rawHeaders", rawHeaders }, + }; +} + +QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { + qDebug() << "loadFromCache" << url; + QVariantMap result = { + { "metadata", queryCacheMeta(url) }, + { "data", QByteArray() }, + }; + // caller is responsible for the deletion of the ioDevice, hence the unique_ptr + if (auto ioDevice = std::unique_ptr(_cache.data(url))) { + QByteArray data = ioDevice->readAll(); + qCDebug(asset_client) << url.toDisplayString() << "loaded from disk cache (" << data.size() << " bytes)"; + result["data"] = data; + } + return result; +} + +namespace { + // parse RFC 1123 HTTP date format + QDateTime parseHttpDate(const QString& dateString) { + QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); + dt.setTimeSpec(Qt::UTC); + return dt; + } +} + +bool BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { + QDateTime lastModified = headers.contains("last-modified") ? + parseHttpDate(headers["last-modified"].toString()) : + QDateTime::currentDateTimeUtc(); + QDateTime expirationDate = headers.contains("expires") ? + parseHttpDate(headers["expires"].toString()) : + QDateTime(); // never expires + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(lastModified); + metaData.setExpirationDate(expirationDate); + if (auto ioDevice = _cache.prepare(metaData)) { + ioDevice->write(data); + _cache.insert(ioDevice); + qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; + return true; + } + qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache."; + return false; +} + +Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) { + auto hash = AssetUtils::extractAssetHash(asset); + auto url = AssetUtils::getATPUrl(hash).toString(); + + QVariantMap metaData = { + { "_asset", asset }, + { "_type", "download" }, + { "hash", hash }, + { "url", url }, + { "responseType", responseType }, + }; + + Promise fetched = makePromise("loadAsset::fetched"), + loaded = makePromise("loadAsset::loaded"); + + downloadBytes(hash) + ->mixin(metaData) + ->ready([=](QString error, QVariantMap result) { + Q_ASSERT(thread() == QThread::currentThread()); + fetched->mixin(result); + if (decompress) { + qDebug() << "loadAsset::decompressBytes..."; + decompressBytes(result.value("data").toByteArray()) + ->mixin(result) + ->ready([=](QString error, QVariantMap result) { + fetched->handle(error, result); + }); + } else { + fetched->handle(error, result); + } + }); + + fetched->ready([=](QString error, QVariantMap result) { + qDebug() << "loadAsset::fetched" << error; + if (responseType == "arraybuffer") { + loaded->resolve(NoError, result); + } else { + convertBytes(result.value("data").toByteArray(), responseType) + ->mixin(result) + ->ready([=](QString error, QVariantMap result) { + loaded->resolve(NoError, result); + }); + } + }); + + return loaded; +} + +Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) { + QVariantMap result; + Promise conversion = makePromise(__FUNCTION__); + if (dataByteArray.size() == 0) { + result["response"] = QString(); + } else if (responseType == "text") { + result["response"] = QString::fromUtf8(dataByteArray); + } else if (responseType == "json") { + QJsonParseError status; + auto parsed = QJsonDocument::fromJson(dataByteArray, &status); + if (status.error == QJsonParseError::NoError) { + result["response"] = parsed.isArray() ? + QVariant(parsed.array().toVariantList()) : + QVariant(parsed.object().toVariantMap()); + } else { + QVariantMap errorResult = { + { "error", status.error }, + { "offset", status.offset }, + }; + return conversion->reject("JSON Parse Error: " + status.errorString(), errorResult); + } + } else if (responseType == "arraybuffer") { + result["response"] = dataByteArray; + } + return conversion->resolve(NoError, result); +} + +Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) { + QByteArray inflated; + auto start = usecTimestampNow(); + if (gunzip(dataByteArray, inflated)) { + auto end = usecTimestampNow(); + return makePromise(__FUNCTION__)->resolve(NoError, { + { "_compressedByteLength", dataByteArray.size() }, + { "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, + { "_compressMS", (double)(end - start) / 1000.0 }, + { "decompressed", true }, + { "byteLength", inflated.size() }, + { "contentType", QMimeDatabase().mimeTypeForData(inflated).name() }, + { "data", inflated }, + }); + } else { + return makePromise(__FUNCTION__)->reject("gunzip error", {}); + } +} + +Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) { + QByteArray deflated; + auto start = usecTimestampNow(); + if (gzip(dataByteArray, deflated, level)) { + auto end = usecTimestampNow(); + return makePromise(__FUNCTION__)->resolve(NoError, { + { "_uncompressedByteLength", dataByteArray.size() }, + { "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, + { "_compressMS", (double)(end - start) / 1000.0 }, + { "compressed", true }, + { "byteLength", deflated.size() }, + { "contentType", QMimeDatabase().mimeTypeForData(deflated).name() }, + { "data", deflated }, + }); + } else { + return makePromise(__FUNCTION__)->reject("gzip error", {}); + } +} + +Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { + auto assetClient = DependencyManager::get(); + QPointer assetRequest = assetClient->createRequest(hash); + Promise deferred = makePromise(__FUNCTION__); + + QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { + qDebug() << "...BaseAssetScriptingInterface::downloadBytes" << request->getErrorString(); + // note: we are now on the "Resource Manager" thread + Q_ASSERT(QThread::currentThread() == request->thread()); + Q_ASSERT(request->getState() == AssetRequest::Finished); + QString error; + QVariantMap result; + if (request->getError() == AssetRequest::Error::NoError) { + QByteArray data = request->getData(); + result = { + { "url", request->getUrl() }, + { "hash", request->getHash() }, + { "cached", request->loadedFromCache() }, + { "content-type", QMimeDatabase().mimeTypeForData(data).name() }, + { "data", data }, + }; + } else { + error = request->getError(); + result = { { "error", request->getError() } }; + } + qDebug() << "//BaseAssetScriptingInterface::downloadBytes" << error << result.keys(); + // forward thread-safe copies back to our thread + deferred->handle(error, result); + request->deleteLater(); + }); + assetRequest->start(); + return deferred; +} + +Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { + Promise deferred = makePromise(__FUNCTION__); + QPointer upload = DependencyManager::get()->createUpload(bytes); + + QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) { + Q_ASSERT(QThread::currentThread() == upload->thread()); + // note: we are now on the "Resource Manager" thread + QString error; + QVariantMap result; + if (upload->getError() == AssetUpload::NoError) { + result = { + { "hash", hash }, + { "url", AssetUtils::getATPUrl(hash).toString() }, + { "filename", upload->getFilename() }, + }; + } else { + error = upload->getErrorString(); + result = { { "error", upload->getError() } }; + } + // forward thread-safe copies back to our thread + deferred->handle(error, result); + upload->deleteLater(); + }); + upload->start(); + return deferred; +} + +Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { + auto deferred = makePromise(__FUNCTION__); + auto url = AssetUtils::getATPUrl(asset); + auto path = url.path(); + auto hash = AssetUtils::extractAssetHash(asset); + if (AssetUtils::isValidHash(hash)) { + // already a valid ATP hash -- nothing to do + deferred->resolve(NoError, { + { "hash", hash }, + { "path", path }, + { "url", url }, + }); + } else if (AssetUtils::isValidFilePath(path)) { + auto assetClient = DependencyManager::get(); + QPointer request = assetClient->createGetMappingRequest(path); + + QObject::connect(request, &GetMappingRequest::finished, request, [=]() { + Q_ASSERT(QThread::currentThread() == request->thread()); + // note: we are now on the "Resource Manager" thread + QString error; + QVariantMap result; + if (request->getError() == GetMappingRequest::NoError) { + result = { + { "_hash", hash }, + { "_path", path }, + { "_url", url }, + { "hash", request->getHash() }, + { "wasRedirected", request->wasRedirected() }, + { "path", request->wasRedirected() ? request->getRedirectedPath() : path }, + }; + } else { + error = request->getErrorString(); + result = { { "error", request->getError() } }; + } + // forward thread-safe copies back to our thread + deferred->handle(error, result); + request->deleteLater(); + }); + request->start(); + } else { + deferred->reject("invalid ATP file path: " + asset + "("+path+")", {}); + } + return deferred; +} + +Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) { + auto deferred = makePromise(__FUNCTION__); + auto assetClient = DependencyManager::get(); + QPointer setMappingRequest = assetClient->createSetMappingRequest(path, hash); + + connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) { + Q_ASSERT(QThread::currentThread() == request->thread()); + // we are now on the "Resource Manager" thread + QString error; + QVariantMap result; + if (request->getError() == SetMappingRequest::NoError) { + result = { + { "_hash", hash }, + { "_path", path }, + { "hash", request->getHash() }, + { "path", request->getPath() }, + { "url", AssetUtils::getATPUrl(request->getPath()).toString() }, + }; + } else { + error = request->getErrorString(); + result = { { "error", request->getError() } }; + } + // forward results back to our thread + deferred->handle(error, result); + request->deleteLater(); + }); + setMappingRequest->start(); + return deferred; +} diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h new file mode 100644 index 0000000000..3aca3d394e --- /dev/null +++ b/libraries/networking/src/BaseAssetScriptingInterface.h @@ -0,0 +1,78 @@ +// +// BaseAssetScriptingInterface.h +// libraries/networking/src +// +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +// BaseAssetScriptingInterface contains the engine-agnostic support code that can be used from +// both QML JS and QScriptEngine JS engine implementations + +#ifndef hifi_BaseAssetScriptingInterface_h +#define hifi_BaseAssetScriptingInterface_h + +#include +#include +#include "AssetClient.h" +#include +#include "NetworkAccessManager.h" +#include + +/**jsdoc + * @namespace Assets + */ +class BaseAssetScriptingInterface : public QObject { + Q_OBJECT +public: + using Promise = MiniPromise::Promise; + QSharedPointer assetClient(); + + BaseAssetScriptingInterface(QObject* parent = nullptr); + +public slots: + /**jsdoc + * Get the current status of the disk cache (if available) + * @function Assets.uploadData + * @static + * @return {String} result.cacheDirectory (path to the current disk cache) + * @return {Number} result.cacheSize (used cache size in bytes) + * @return {Number} result.maximumCacheSize (maxmimum cache size in bytes) + */ + QVariantMap getCacheStatus(); + + /**jsdoc + * Initialize the disk cache (returns true if already initialized) + * @function Assets.initializeCache + * @static + */ + bool initializeCache(); + + virtual bool isValidPath(QString input) { return AssetUtils::isValidPath(input); } + virtual bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); } + QUrl getATPUrl(QString input) { return AssetUtils::getATPUrl(input); } + QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); } + bool isValidHash(QString input) { return AssetUtils::isValidHash(input); } + QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); } + + virtual QVariantMap queryCacheMeta(const QUrl& url); + virtual QVariantMap loadFromCache(const QUrl& url); + virtual bool saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + +protected: + //void onCacheInfoResponse(QString cacheDirectory, qint64 cacheSize, qint64 maximumCacheSize); + QNetworkDiskCache _cache; + const QString NoError{}; + //virtual bool jsAssert(bool condition, const QString& error) = 0; + Promise loadAsset(QString asset, bool decompress, QString responseType); + Promise getAssetInfo(QString asset); + Promise downloadBytes(QString hash); + Promise uploadBytes(const QByteArray& bytes); + Promise compressBytes(const QByteArray& bytes, int level = -1); + Promise convertBytes(const QByteArray& dataByteArray, const QString& responseType); + Promise decompressBytes(const QByteArray& bytes); + Promise symlinkAsset(QString hash, QString path); +}; +#endif // hifi_BaseAssetScriptingInterface_h diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 25e8c0dcf3..b5492db304 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -11,43 +11,70 @@ #include "AssetScriptingInterface.h" +#include +#include #include #include #include +#include #include #include #include -AssetScriptingInterface::AssetScriptingInterface(QScriptEngine* engine) : - _engine(engine) -{ +#include + +#include +#include "Gzip.h" +#include "ScriptEngine.h" + +//using Promise = MiniPromise::Promise; + +AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) { + if (auto engine = qobject_cast(parent)) { + registerMetaTypes(engine); + } } +#define JS_ASSERT(cond, error) { if (!this->jsAssert(cond, error)) { return; } } void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { + auto handler = makeScopedHandlerObject(thisObject(), callback); QByteArray dataByteArray = data.toUtf8(); auto upload = DependencyManager::get()->createUpload(dataByteArray); - QObject::connect(upload, &AssetUpload::finished, this, [this, callback](AssetUpload* upload, const QString& hash) mutable { - if (callback.isFunction()) { - QString url = "atp:" + hash; - QScriptValueList args { url, hash }; - callback.call(_engine->currentContext()->thisObject(), args); - } + Promise deferred = makePromise(__FUNCTION__) + ->ready([this, handler](QString error, QVariantMap result) { + auto url = result.value("url").toString(); + auto hash = result.value("hash").toString(); + jsCallback(handler, url, hash); + }); + + connect(upload, &AssetUpload::finished, upload, [this, deferred](AssetUpload* upload, const QString& hash) { + // we are now on the "Resource Manager" thread (and "hash" being a *reference* makes it unsafe to use directly) + Q_ASSERT(QThread::currentThread() == upload->thread()); + deferred->resolve(NoError, { + { "url", "atp:" + hash }, + { "hash", hash }, + }); upload->deleteLater(); }); upload->start(); } void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) { - auto setMappingRequest = DependencyManager::get()->createSetMappingRequest(path, hash); + auto handler = makeScopedHandlerObject(thisObject(), callback); + auto setMappingRequest = assetClient()->createSetMappingRequest(path, hash); + Promise deferred = makePromise(__FUNCTION__) + ->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); - QObject::connect(setMappingRequest, &SetMappingRequest::finished, this, [this, callback](SetMappingRequest* request) mutable { - if (callback.isFunction()) { - QString error = request->getErrorString(); - QScriptValueList args { error }; - callback.call(_engine->currentContext()->thisObject(), args); - } + connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [this, deferred](SetMappingRequest* request) { + Q_ASSERT(QThread::currentThread() == request->thread()); + // we are now on the "Resource Manager" thread + QString error = request->getErrorString(); + // forward a thread-safe values back to our thread + deferred->handle(error, { { "error", request->getError() } }); request->deleteLater(); }); setMappingRequest->start(); @@ -55,48 +82,63 @@ void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValu void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callback) { + // FIXME: historically this API method failed silently when given a non-atp prefixed + // urlString (or if the AssetRequest failed). + // .. is that by design or could we update without breaking things to provide better feedback to scripts? if (!urlString.startsWith(ATP_SCHEME)) { + // ... for now at least log a message so user can check logs + qDebug() << "AssetScriptingInterface::downloadData ERROR: url does not start with " << ATP_SCHEME; return; } - - // Make request to atp - auto path = urlString.right(urlString.length() - ATP_SCHEME.length()); - auto parts = path.split(".", QString::SkipEmptyParts); - auto hash = parts.length() > 0 ? parts[0] : ""; - + QString hash = AssetUtils::extractAssetHash(urlString); + auto handler = makeScopedHandlerObject(thisObject(), callback); auto assetClient = DependencyManager::get(); auto assetRequest = assetClient->createRequest(hash); - _pendingRequests << assetRequest; + Promise deferred = makePromise(__FUNCTION__) + ->ready([=](QString error, QVariantMap result) { + // FIXME: to remain backwards-compatible the signature here is "callback(data, n/a)" + jsCallback(handler, result.value("data").toString(), { { "errorMessage", error } }); + }); - connect(assetRequest, &AssetRequest::finished, this, [this, callback](AssetRequest* request) mutable { + connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { + Q_ASSERT(QThread::currentThread() == request->thread()); + // we are now on the "Resource Manager" thread Q_ASSERT(request->getState() == AssetRequest::Finished); if (request->getError() == AssetRequest::Error::NoError) { - if (callback.isFunction()) { - QString data = QString::fromUtf8(request->getData()); - QScriptValueList args { data }; - callback.call(_engine->currentContext()->thisObject(), args); - } + QString data = QString::fromUtf8(request->getData()); + // forward a thread-safe values back to our thread + deferred->resolve(NoError, { { "data", data } }); + } else { + // FIXME: propagate error to scripts? (requires changing signature or inverting param order above..) + //deferred->resolve(request->getErrorString(), { { "error", requet->getError() } }); + qDebug() << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString(); } request->deleteLater(); - _pendingRequests.remove(request); }); assetRequest->start(); } void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScriptValue callback) { + auto handler = makeScopedHandlerObject(thisObject(), callback); auto setBakingEnabledRequest = DependencyManager::get()->createSetBakingEnabledRequest({ path }, enabled); - QObject::connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, this, [this, callback](SetBakingEnabledRequest* request) mutable { - if (callback.isFunction()) { - QString error = request->getErrorString(); - QScriptValueList args{ error }; - callback.call(_engine->currentContext()->thisObject(), args); - } + Promise deferred = makePromise(__FUNCTION__) + ->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); + + connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, setBakingEnabledRequest, [this, deferred](SetBakingEnabledRequest* request) { + Q_ASSERT(QThread::currentThread() == request->thread()); + // we are now on the "Resource Manager" thread + + QString error = request->getErrorString(); + // forward thread-safe values back to our thread + deferred->handle(error, {}); request->deleteLater(); }); setBakingEnabledRequest->start(); @@ -111,3 +153,238 @@ void AssetScriptingInterface::sendFakedHandshake() { } #endif + +void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { + auto path = AssetUtils::getATPUrl(asset).path(); + auto handler = makeScopedHandlerObject(thisObject(), callback); + JS_ASSERT(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); + JS_ASSERT(callback.isFunction(), "expected second parameter to be a callback function"); + qDebug() << ">>getMapping//getAssetInfo" << path; + getAssetInfo(path)->ready([this, handler](QString error, QVariantMap result) { + qDebug() << "//getMapping//getAssetInfo" << error << result.keys(); + jsCallback(handler, error, result.value("hash").toString()); + }); +} + +/////////////////////////// new APIS //////////////////////////////////// + +bool AssetScriptingInterface::jsAssert(bool condition, const QString& error) { + if (condition) { + return true; + } + if (context()) { + context()->throwError(error); + } else { + qDebug() << "WARNING -- jsAssert failed outside of a valid JS context: " + error; + } + return false; +} + +void AssetScriptingInterface::jsCallback(const QScriptValue& handler, + const QScriptValue& error, const QScriptValue& result) { + Q_ASSERT(thread() == QThread::currentThread()); + auto errorValue = !error.toBool() ? QScriptValue::NullValue : error; + JS_ASSERT(handler.isObject() && handler.property("callback").isFunction(), + QString("jsCallback -- .callback is not a function (%1)") + .arg(handler.property("callback").toVariant().typeName())); +#if 1 || DEGUG_JSCALLBACK + QScriptValue debug = result; + debug.setProperty("toString", handler.engine()->evaluate("1,function() { return JSON.stringify(this, 0, 2); }")); +#endif + ::callScopedHandlerObject(handler, errorValue, result); +} + +void AssetScriptingInterface::jsCallback(const QScriptValue& handler, + const QScriptValue& error, const QVariantMap& result) { + Q_ASSERT(thread() == QThread::currentThread()); + Q_ASSERT(handler.engine()); + auto engine = handler.engine(); + jsCallback(handler, error, engine->toScriptValue(result)); +} + +void AssetScriptingInterface::deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { + jsAssert(false, "TODO: deleteAsset API"); +} + +void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { + JS_ASSERT(options.isObject() || options.isString(), "expected request options Object or URL as first parameter"); + + auto decompress = options.property("decompress").toBool() || options.property("compressed").toBool(); + auto responseType = options.property("responseType").toString().toLower(); + auto url = options.property("url").toString(); + if (options.isString()) { + url = options.toString(); + } + if (responseType.isEmpty() || responseType == "string") { + responseType = "text"; + } + auto asset = AssetUtils::getATPUrl(url).path(); + auto handler = makeScopedHandlerObject(scope, callback); + + JS_ASSERT(handler.property("callback").isFunction(), + QString("Invalid callback function (%1)").arg(handler.property("callback").toVariant().typeName())); + JS_ASSERT(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset), + QString("Invalid ATP url '%1'").arg(url)); + JS_ASSERT(RESPONSE_TYPES.contains(responseType), + QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); + + Promise resolved = makePromise("resolved"), + loaded = makePromise("loaded"); + + loaded->ready([=](QString error, QVariantMap result) { + qDebug() << "//loaded" << error; + jsCallback(handler, error, result); + }); + + resolved->ready([=](QString error, QVariantMap result) { + qDebug() << "//resolved" << result.value("hash"); + QString hash = result.value("hash").toString(); + if (!error.isEmpty() || !AssetUtils::isValidHash(hash)) { + loaded->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); + } else { + loadAsset(hash, decompress, responseType) + ->mixin(result) + ->ready([this, loaded, hash](QString error, QVariantMap result) { + qDebug() << "//getAssetInfo/loadAsset" << error << hash; + loaded->resolve(NoError, result); + }); + } + }); + + if (AssetUtils::isValidHash(asset)) { + resolved->resolve(NoError, { { "hash", asset } }); + } else { + getAssetInfo(asset) + ->ready([this, resolved](QString error, QVariantMap result) { + qDebug() << "//getAssetInfo" << error << result.value("hash") << result.value("path"); + resolved->resolve(error, result); + }); + } +} + +void AssetScriptingInterface::resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { + const QString& URL{ "url" }; + + auto url = (options.isString() ? options : options.property(URL)).toString(); + auto asset = AssetUtils::getATPUrl(url).path(); + auto handler = makeScopedHandlerObject(scope, callback); + + JS_ASSERT(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset), + "expected options to be an asset URL or request options containing .url property"); + JS_ASSERT(handler.property("callback").isFunction(), "invalid callback function"); + getAssetInfo(asset)->ready([=](QString error, QVariantMap result) { + qDebug() << "//resolveAsset/getAssetInfo" << error << result.value("hash"); + jsCallback(handler, error, result); + }); +} + +void AssetScriptingInterface::decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { + auto data = options.property("data"); + QByteArray dataByteArray = qscriptvalue_cast(data); + auto handler = makeScopedHandlerObject(scope, callback); + auto responseType = options.property("responseType").toString().toLower(); + if (responseType.isEmpty()) { + responseType = "text"; + } + decompressBytes(dataByteArray) + ->ready([=](QString error, QVariantMap result) { + if (responseType == "arraybuffer") { + jsCallback(handler, error, result); + } else { + convertBytes(result.value("data").toByteArray(), responseType) + ->mixin(result) + ->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); + } + }); +} + +void AssetScriptingInterface::compressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { + auto data = options.property("data"); + QByteArray dataByteArray = data.isString() ? + data.toString().toUtf8() : + qscriptvalue_cast(data); + auto handler = makeScopedHandlerObject(scope, callback); + auto level = options.property("level").toInt32(); + if (level < -1 || level > 9) { + level = -1; + } + compressBytes(dataByteArray, level) + ->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); +} + +void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { + auto compress = options.property("compress").toBool() || + options.property("compressed").toBool(); + auto handler = makeScopedHandlerObject(scope, callback); + auto data = options.property("data"); + auto rawPath = options.property("path").toString(); + auto path = AssetUtils::getATPUrl(rawPath).path(); + + QByteArray dataByteArray = data.isString() ? + data.toString().toUtf8() : + qscriptvalue_cast(data); + //auto rawByteLength = dataByteArray.size(); + + JS_ASSERT(path.isEmpty() || AssetUtils::isValidFilePath(path), + QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path)); + JS_ASSERT(handler.property("callback").isFunction(), + "invalid callback function"); + JS_ASSERT(dataByteArray.size() > 0, + QString("expected non-zero .data (got %1 / #%2 bytes)") + .arg(data.toVariant().typeName()) + .arg(dataByteArray.size())); + + + // [compressed] => uploaded to server => [mapped to path] + Promise prepared = makePromise("putAsset::prepared"), + uploaded = makePromise("putAsset::uploaded"), + finished = makePromise("putAsset::finished"); + + if (compress) { + qDebug() << "putAsset::compressBytes..."; + compressBytes(dataByteArray, -1) + ->finally([=](QString error, QVariantMap result) { + qDebug() << "//putAsset::compressedBytes" << error << result.keys(); + prepared->handle(error, result); + }); + } else { + prepared->resolve(NoError, {{ "data", dataByteArray }}); + } + + prepared->ready([=](QString error, QVariantMap result) { + qDebug() << "//putAsset::prepared" << error << result.value("data").toByteArray().size() << result.keys(); + uploadBytes(result.value("data").toByteArray()) + ->mixin(result) + ->ready([=](QString error, QVariantMap result) { + qDebug() << "===========//putAsset::prepared/uploadBytes" << error << result.keys(); + uploaded->handle(error, result); + }); + }); + + uploaded->ready([=](QString error, QVariantMap result) { + QString hash = result.value("hash").toString(); + qDebug() << "//putAsset::uploaded" << error << hash << result.keys(); + if (path.isEmpty()) { + finished->handle(error, result); + } else if (!AssetUtils::isValidHash(hash)) { + finished->reject("path mapping requested, but did not receive valid hash", result); + } else { + qDebug() << "symlinkAsset" << hash << path << QThread::currentThread(); + symlinkAsset(hash, path) + ->mixin(result) + ->ready([=](QString error, QVariantMap result) { + finished->handle(error, result); + qDebug() << "//symlinkAsset" << hash << path << result.keys(); + }); + } + }); + + finished->ready([=](QString error, QVariantMap result) { + qDebug() << "//putAsset::finished" << error << result.keys(); + jsCallback(handler, error, result); + }); +} diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 2812be65f9..9022316634 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -15,17 +15,21 @@ #define hifi_AssetScriptingInterface_h #include +#include #include - +#include #include +#include +#include +#include /**jsdoc * @namespace Assets */ -class AssetScriptingInterface : public QObject { +class AssetScriptingInterface : public BaseAssetScriptingInterface, QScriptable { Q_OBJECT public: - AssetScriptingInterface(QScriptEngine* engine); + AssetScriptingInterface(QObject* parent = nullptr); /**jsdoc * Upload content to the connected domain's asset server. @@ -75,16 +79,37 @@ public: * @param {string} error */ Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); - + Q_INVOKABLE void getMapping(QString path, QScriptValue callback); + Q_INVOKABLE void setBakingEnabled(QString path, bool enabled, QScriptValue callback); #if (PR_BUILD || DEV_BUILD) Q_INVOKABLE void sendFakedHandshake(); #endif + // Advanced APIs + // getAsset(options, scope[callback(error, result)]) -- fetches an Asset from the Server + // [options.url] an "atp:" style URL, hash, or relative mapped path to fetch + // [options.responseType] the desired reponse type (text | arraybuffer | json) + // [options.decompress] whether to apply gunzip decompression on the stream + // [scope[callback]] continuation-style (error, { responseType, data, byteLength, ... }) callback + const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" }; + Q_INVOKABLE void getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + // putAsset(options, scope[callback(error, result)]) -- upload a new Aset to the Server + // [options.data] -- (ArrayBuffer|String) + // [options.compress] -- (true|false) + // [options.path=undefined] -- option path mapping to set on the created hash result + // [ + Q_INVOKABLE void putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void compressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + protected: - QSet _pendingRequests; - QScriptEngine* _engine; + void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QVariantMap& result); + void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result); + bool jsAssert(bool condition, const QString& error); }; #endif // hifi_AssetScriptingInterface_h diff --git a/scripts/developer/tests/unit_tests/assetUnitTests.js b/scripts/developer/tests/unit_tests/assetUnitTests.js new file mode 100644 index 0000000000..be8710e50d --- /dev/null +++ b/scripts/developer/tests/unit_tests/assetUnitTests.js @@ -0,0 +1,181 @@ +/* eslint-env jasmine */ + +instrument_testrunner(true); + +describe("Assets", function () { + var context = { + definedHash: null, + definedPath: null, + definedContent: null, + }; + var NULL_HASH = new Array(64+1).join('0'); // 64 hex + var SAMPLE_FOLDER = '/assetUnitTests'; + var SAMPLE_FILE_PATH = SAMPLE_FOLDER + "/test.json"; + var SAMPLE_CONTENTS = 'Test Run on ' + JSON.stringify(Date()); + var IS_ASSET_HASH_REGEX = /^[a-f0-9]{64}$/i; + var IS_ASSET_URL = /^atp:/; + + it('Entities.canWriteAssets', function() { + expect(Entities.canWriteAssets()).toBe(true); + }); + + it('Assets.extractAssetHash(input)', function() { + // new helper method that provides a catch-all way to get at the sha256 hash + // considering the multiple, different ways ATP hash URLs are found across the + // system and in content. + + var POSITIVE_TEST_URLS = [ + 'atp:HASH', + 'atp:HASH.obj', + 'atp:HASH.fbx?cache-buster', + 'atp:/.baked/HASH/asset.fbx', + 'HASH' + ]; + var NEGATIVE_TEST_URLS = [ + 'asdf', + 'http://domain.tld', + '/path/filename.fbx', + 'atp:/path/filename.fbx?#HASH', + '' + ]; + pending(); + }); + + // new APIs + + it('Assets.getAsset(options, {callback(error, result)})', function() { pending(); }); + it('Assets.putAsset(options, {callback(error, result)})', function() { pending(); }); + it('Assets.deleteAsset(options, {callback(error, result)})', function() { pending(); }); + it('Assets.resolveAsset(options, {callback(error, result)})', function() { pending(); }); + + + // existing APIs + it('Assets.getATPUrl(input)', function() { pending(); }); + it('Assets.isValidPath(input)', function() { pending(); }); + it('Assets.isValidFilePath(input)', function() { pending(); }); + it('Assets.isValidHash(input)', function() { pending(); }); + it('Assets.setBakingEnabled(path, enabled, {callback(error)})', function() { pending(); }); + + it("Assets.uploadData(data, {callback(url, hash)})", function (done) { + Assets.uploadData(SAMPLE_CONTENTS, function(url, hash) { + expect(url).toMatch(IS_ASSET_URL); + expect(hash).toMatch(IS_ASSET_HASH_REGEX); + context.definedHash = hash; // used in later tests + context.definedContent = SAMPLE_CONTENTS; + print('context.definedHash = ' + context.definedHash); + print('context.definedContent = ' + context.definedContent); + done(); + }); + }); + + it("Assets.setMapping(path, hash {callback(error)})", function (done) { + expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX); + Assets.setMapping(SAMPLE_FILE_PATH, context.definedHash, function(error) { + if (error) error += ' ('+JSON.stringify([SAMPLE_FILE_PATH, context.definedHash])+')'; + expect(error).toBe(null); + context.definedPath = SAMPLE_FILE_PATH; + print('context.definedPath = ' + context.definedPath); + done(); + }); + }); + + it("Assets.getMapping(path, {callback(error, hash)})", function (done) { + expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf'); + expect(context.definedPath).toMatch(/^\//, 'asfasdf'); + Assets.getMapping(context.definedPath, function(error, hash) { + if (error) error += ' ('+JSON.stringify([context.definedPath])+')'; + expect(error).toBe(null); + expect(hash).toMatch(IS_ASSET_HASH_REGEX); + expect(hash).toEqual(context.definedHash); + done(); + }); + }); + + it('Assets.getMapping(nullHash, {callback(data)})', function(done) { + // FIXME: characterization test -- current behavior is that downloadData silently fails + // when given an asset that doesn't exist + Assets.downloadData(NULL_HASH, function(result) { + throw new Error("this callback 'should' not be triggered"); + }); + setTimeout(function() { done(); }, 2000); + }); + + it('Assets.downloadData(hash, {callback(data)})', function(done) { + expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf'); + Assets.downloadData('atp:' + context.definedHash, function(result) { + expect(result).toEqual(context.definedContent); + done(); + }); + }); + + it('Assets.downloadData(nullHash, {callback(data)})', function(done) { + // FIXME: characterization test -- current behavior is that downloadData silently fails + // when given an asset doesn't exist + Assets.downloadData(NULL_HASH, function(result) { + throw new Error("this callback 'should' not be triggered"); + }); + setTimeout(function() { + done(); + }, 2000); + }); + + describe('exceptions', function() { + describe('Asset.setMapping', function() { + it('-- invalid path', function(done) { + Assets.setMapping('foo', NULL_HASH, function(error/*, result*/) { + expect(error).toEqual('Path is invalid'); + /* expect(result).not.toMatch(IS_ASSET_URL); */ + done(); + }); + }); + it('-- invalid hash', function(done) { + Assets.setMapping(SAMPLE_FILE_PATH, 'bar', function(error/*, result*/) { + expect(error).toEqual('Hash is invalid'); + /* expect(result).not.toMatch(IS_ASSET_URL); */ + done(); + }); + }); + }); + describe('Asset.getMapping', function() { + it('-- invalid path throws immediate', function() { + expect(function() { + Assets.getMapping('foo', function(error, hash) { + throw new Error("should never make it here..."); + }); + throw new Error("should never make it here..."); + }).toThrowError(/invalid.*path/i); + }); + it('-- non-existing path', function(done) { + Assets.getMapping('/foo/bar/'+Date.now(), function(error, hash) { + expect(error).toEqual('Asset not found'); + expect(hash).not.toMatch(IS_ASSET_HASH_REGEX); + done(); + }); + }); + }); + }); +}); + +// ---------------------------------------------------------------------------- +// this stuff allows the unit tests to be loaded indepenently and/or as part of testRunner.js execution +function run() {} +function instrument_testrunner(force) { + if (force || typeof describe === 'undefined') { + // Include testing library + Script.include('../../libraries/jasmine/jasmine.js'); + Script.include('../../libraries/jasmine/hifi-boot.js'); + + run = function() { + if (!/:console/.test(Script.resolvePath(''))) { + // invoke Script.stop (after any async tests complete) + jasmine.getEnv().addReporter({ jasmineDone: Script.stop }); + } else { + jasmine.getEnv().addReporter({ jasmineDone: function() { print("JASMINE DONE"); } }); + } + + // Run the tests + jasmine.getEnv().execute(); + }; + } +} +run(); From e94103633705d80c7095bd298b5e3703a5a5f9d9 Mon Sep 17 00:00:00 2001 From: humbletim Date: Tue, 2 Jan 2018 15:28:01 -0500 Subject: [PATCH 03/21] renames per CR feedback / code cleanup --- libraries/shared/src/shared/MiniPromises.cpp | 4 +- libraries/shared/src/shared/MiniPromises.h | 203 +++++++++---------- 2 files changed, 99 insertions(+), 108 deletions(-) diff --git a/libraries/shared/src/shared/MiniPromises.cpp b/libraries/shared/src/shared/MiniPromises.cpp index fdfb509608..28e2c18857 100644 --- a/libraries/shared/src/shared/MiniPromises.cpp +++ b/libraries/shared/src/shared/MiniPromises.cpp @@ -1,4 +1,2 @@ #include "MiniPromises.h" -namespace { - int promiseMetaTypeId = qRegisterMetaType("MiniPromise::Promise"); -} +int MiniPromise::metaTypeID = qRegisterMetaType("MiniPromise::Promise"); diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index 17f2394a2f..c5d21a22fb 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -1,19 +1,19 @@ #pragma once -// Minimalist threadsafe Promise-like helper for instrumenting asynchronous results +// Minimalist threadsafe Promise-like helper for managing asynchronous results // -// This class pivots around composable continuation-style callback handlers: +// This class pivots around continuation-style callback handlers: // auto successCallback = [=](QVariantMap result) { .... } // auto errorCallback = [=](QString error) { .... } // auto combinedCallback = [=](QString error, QVariantMap result) { .... } // // * Callback Handlers are automatically invoked on the right thread (the Promise's thread). // * Callbacks can be assigned anytime during a Promise's life and "do the right thing". -// - ie: for code clarity you can define success cases first (or maintain time order) +// - ie: for code clarity you can define success cases first or choose to maintain time order // * "Deferred" concept can be used to publish outcomes. // * "Promise" concept be used to subscribe to outcomes. // -// See AssetScriptingInterface.cpp for some examples of using to simplify chained async result. +// See AssetScriptingInterface.cpp for some examples of using to simplify chained async results. #include #include @@ -25,35 +25,37 @@ class MiniPromise : public QObject, public std::enable_shared_from_this, public ReadWriteLockable { Q_OBJECT public: - using handlerFunction = std::function; - using successFunction = std::function; - using errorFunction = std::function; - using handlers = QVector; + using HandlerFunction = std::function; + using SuccessFunction = std::function; + using ErrorFunction = std::function; + using HandlerFunctions = QVector; using Promise = std::shared_ptr; - MiniPromise(const QString debugName) { setObjectName(debugName); } + + static int metaTypeID; + MiniPromise() {} + MiniPromise(const QString debugName) { setObjectName(debugName); } + ~MiniPromise() { - qDebug() << "~MiniPromise" << objectName(); if (!_rejected && !_resolved) { - qWarning() << "====== WARNING: unhandled MiniPromise" << objectName() << _error << _result; + qWarning() << "MiniPromise::~MiniPromise -- destroying unhandled promise:" << objectName() << _error << _result; + } + } + Promise self() { return shared_from_this(); } + + Q_INVOKABLE void executeOnPromiseThread(std::function function) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod( + this, "executeOnPromiseThread", Qt::BlockingQueuedConnection, + Q_ARG(std::function, function)); + } else { + function(); } } - QString _error; - QVariantMap _result; - std::atomic _rejected{false}; - std::atomic _resolved{false}; - handlers _onresolve; - handlers _onreject; - handlers _onfinally; - - Promise self() { return shared_from_this(); } - // result aggregation helpers -- eg: deferred->defaults(interimResultMap)->ready(...) - // copy values from the input map, but only for keys that don't already exist Promise mixin(const QVariantMap& source) { - qDebug() << objectName() << "mixin"; withWriteLock([&]{ for (const auto& key : source.keys()) { if (!_result.contains(key)) { @@ -65,7 +67,6 @@ public: } // copy values from the input map, replacing any existing keys Promise assignResult(const QVariantMap& source) { - qDebug() << objectName() << "assignResult"; withWriteLock([&]{ for (const auto& key : source.keys()) { _result[key] = source[key]; @@ -74,15 +75,15 @@ public: return self(); } - // TODO: I think calling as "ready" makes it read better, but is customary Promise "finally" sufficient? - Promise ready(handlerFunction always) { return finally(always); } - Promise finally(handlerFunction always) { + // callback registration methods + Promise ready(HandlerFunction always) { return finally(always); } + Promise finally(HandlerFunction always) { if (!_rejected && !_resolved) { withWriteLock([&]{ _onfinally << always; }); } else { - qDebug() << "finally (already resolved/rejected)" << objectName(); + qWarning() << "MiniPromise::finally() called after promise was already rejected or resolved:" << objectName(); executeOnPromiseThread([&]{ withReadLock([&]{ always(_error, _result); @@ -91,20 +92,20 @@ public: } return self(); } - Promise fail(errorFunction errorOnly) { + Promise fail(ErrorFunction errorOnly) { return fail([this, errorOnly](QString error, QVariantMap result) { errorOnly(error); }); } - Promise fail(handlerFunction failFunc) { + Promise fail(HandlerFunction failFunc) { if (!_rejected) { withWriteLock([&]{ _onreject << failFunc; }); } else { executeOnPromiseThread([&]{ - qDebug() << "fail (already rejected)" << objectName(); + qWarning() << "MiniPromise::fail() called after promise was already rejected:" << objectName(); withReadLock([&]{ failFunc(_error, _result); }); @@ -113,19 +114,19 @@ public: return self(); } - Promise then(successFunction successOnly) { + Promise then(SuccessFunction successOnly) { return then([this, successOnly](QString error, QVariantMap result) { successOnly(result); }); } - Promise then(handlerFunction successFunc) { + Promise then(HandlerFunction successFunc) { if (!_resolved) { withWriteLock([&]{ _onresolve << successFunc; }); } else { executeOnPromiseThread([&]{ - qDebug() << "fail (already resolved)" << objectName(); + qWarning() << "MiniPromise::then() called after promise was already resolved:" << objectName(); withReadLock([&]{ successFunc(_error, _result); }); @@ -133,9 +134,9 @@ public: } return self(); } - // register combined success/error handlers - Promise then(successFunction successOnly, errorFunction errorOnly) { - // note: first arg can be null (see ES6 .then(null, errorHandler) conventions) + + // NOTE: first arg may be null (see ES6 .then(null, errorHandler) conventions) + Promise then(SuccessFunction successOnly, ErrorFunction errorOnly) { if (successOnly) { then(successOnly); } @@ -146,8 +147,8 @@ public: } // trigger methods + // handle() automatically resolves or rejects the promise (based on whether an error value occurred) Promise handle(QString error, const QVariantMap& result) { - qDebug() << "handle" << objectName() << error; if (error.isEmpty()) { resolve(error, result); } else { @@ -155,23 +156,64 @@ public: } return self(); } + Promise resolve(QVariantMap result) { return resolve(QString(), result); } + Promise resolve(QString error, const QVariantMap& result) { + setState(true, error, result); - Q_INVOKABLE void executeOnPromiseThread(std::function function) { - if (QThread::currentThread() != thread()) { - qDebug() << "-0-0-00-0--0" << objectName() << "executeOnPromiseThread -- wrong thread" << QThread::currentThread(); - QMetaObject::invokeMethod( - this, "executeOnPromiseThread", Qt::BlockingQueuedConnection, - Q_ARG(std::function, function)); - } else { - function(); - } + QString localError; + QVariantMap localResult; + HandlerFunctions resolveHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + localError = _error; + localResult = _result; + resolveHandlers = _onresolve; + finallyHandlers = _onfinally; + }); + executeOnPromiseThread([&]{ + for (const auto& onresolve : resolveHandlers) { + onresolve(localError, localResult); + } + for (const auto& onfinally : finallyHandlers) { + onfinally(localError, localResult); + } + }); + return self(); } + Promise reject(QString error) { + return reject(error, QVariantMap()); + } + Promise reject(QString error, const QVariantMap& result) { + setState(false, error, result); + + QString localError; + QVariantMap localResult; + HandlerFunctions rejectHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + localError = _error; + localResult = _result; + rejectHandlers = _onreject; + finallyHandlers = _onfinally; + }); + executeOnPromiseThread([&]{ + for (const auto& onreject : rejectHandlers) { + onreject(localError, localResult); + } + for (const auto& onfinally : finallyHandlers) { + onfinally(localError, localResult); + } + }); + return self(); + } + +private: + Promise setState(bool resolved, QString error, const QVariantMap& result) { - qDebug() << "setState" << objectName() << resolved << error; if (resolved) { _resolved = true; } else { @@ -181,65 +223,16 @@ public: _error = error; }); assignResult(result); - qDebug() << "//setState" << objectName() << resolved << error; - return self(); - } - Promise resolve(QString error, const QVariantMap& result) { - setState(true, error, result); - qDebug() << "handle" << objectName() << error; - { - QString error; - QVariantMap result; - handlers toresolve; - handlers tofinally; - withReadLock([&]{ - error = _error; - result = _result; - toresolve = _onresolve; - tofinally = _onfinally; - }); - executeOnPromiseThread([&]{ - for (const auto& onresolve : toresolve) { - onresolve(error, result); - } - for (const auto& onfinally : tofinally) { - onfinally(error, result); - } - }); - } - return self(); - } - Promise reject(QString error) { - return reject(error, QVariantMap()); - } - Promise reject(QString error, const QVariantMap& result) { - setState(false, error, result); - qDebug() << "handle" << objectName() << error; - { - QString error; - QVariantMap result; - handlers toreject; - handlers tofinally; - withReadLock([&]{ - error = _error; - result = _result; - toreject = _onreject; - tofinally = _onfinally; - }); - executeOnPromiseThread([&]{ - for (const auto& onreject : toreject) { - onreject(error, result); - } - for (const auto& onfinally : tofinally) { - onfinally(error, result); - } - if (toreject.isEmpty() && tofinally.isEmpty()) { - qWarning() << "WARNING: unhandled MiniPromise::reject" << objectName() << error << result; - } - }); - } return self(); } + + QString _error; + QVariantMap _result; + std::atomic _rejected{false}; + std::atomic _resolved{false}; + HandlerFunctions _onresolve; + HandlerFunctions _onreject; + HandlerFunctions _onfinally; }; inline MiniPromise::Promise makePromise(const QString& hint = QString()) { From 8d2e04167832be70e90f54926d909d30e7784389 Mon Sep 17 00:00:00 2001 From: humbletim Date: Wed, 3 Jan 2018 03:39:24 -0500 Subject: [PATCH 04/21] remove debug prints; CR feedback --- .../src/BaseAssetScriptingInterface.cpp | 16 +- .../src/AssetScriptingInterface.cpp | 139 +++++++++--------- 2 files changed, 70 insertions(+), 85 deletions(-) diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index bfdc21c7d5..8172f12c79 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -36,8 +36,6 @@ BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObj bool BaseAssetScriptingInterface::initializeCache() { - qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- current values" << _cache.cacheDirectory() << _cache.cacheSize() << _cache.maximumCacheSize(); - // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings // become available we configure our instance to match. @@ -50,29 +48,23 @@ bool BaseAssetScriptingInterface::initializeCache() { } // attempt to initialize the cache - qDebug() << "BaseAssetScriptingInterface::getCacheStatus -- invoking AssetClient::init" << assetClient().data(); QMetaObject::invokeMethod(assetClient().data(), "init"); - qDebug() << "BaseAssetScriptingInterface::getCacheStatus querying cache status" << QThread::currentThread(); Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); deferred->then([&](QVariantMap result) { - qDebug() << "//queryCacheStatus" << QThread::currentThread(); auto cacheDirectory = result.value("cacheDirectory").toString(); - auto cacheSize = result.value("cacheSize").toLongLong(); auto maximumCacheSize = result.value("maximumCacheSize").toLongLong(); - qDebug() << "///queryCacheStatus" << cacheDirectory << cacheSize << maximumCacheSize; _cache.setCacheDirectory(cacheDirectory); _cache.setMaximumCacheSize(maximumCacheSize); }); deferred->fail([&](QString error) { - qDebug() << "//queryCacheStatus ERROR" << QThread::currentThread() << error; + qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error; }); assets->cacheInfoRequest(deferred); return false; // cache is not ready yet } QVariantMap BaseAssetScriptingInterface::getCacheStatus() { - //assetClient()->cacheInfoRequest(this, "onCacheInfoResponse"); return { { "cacheDirectory", _cache.cacheDirectory() }, { "cacheSize", _cache.cacheSize() }, @@ -104,7 +96,6 @@ QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { } QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { - qDebug() << "loadFromCache" << url; QVariantMap result = { { "metadata", queryCacheMeta(url) }, { "data", QByteArray() }, @@ -112,7 +103,6 @@ QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { // caller is responsible for the deletion of the ioDevice, hence the unique_ptr if (auto ioDevice = std::unique_ptr(_cache.data(url))) { QByteArray data = ioDevice->readAll(); - qCDebug(asset_client) << url.toDisplayString() << "loaded from disk cache (" << data.size() << " bytes)"; result["data"] = data; } return result; @@ -170,7 +160,6 @@ Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, Q Q_ASSERT(thread() == QThread::currentThread()); fetched->mixin(result); if (decompress) { - qDebug() << "loadAsset::decompressBytes..."; decompressBytes(result.value("data").toByteArray()) ->mixin(result) ->ready([=](QString error, QVariantMap result) { @@ -182,7 +171,6 @@ Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, Q }); fetched->ready([=](QString error, QVariantMap result) { - qDebug() << "loadAsset::fetched" << error; if (responseType == "arraybuffer") { loaded->resolve(NoError, result); } else { @@ -268,7 +256,6 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { Promise deferred = makePromise(__FUNCTION__); QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { - qDebug() << "...BaseAssetScriptingInterface::downloadBytes" << request->getErrorString(); // note: we are now on the "Resource Manager" thread Q_ASSERT(QThread::currentThread() == request->thread()); Q_ASSERT(request->getState() == AssetRequest::Finished); @@ -287,7 +274,6 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { error = request->getError(); result = { { "error", request->getError() } }; } - qDebug() << "//BaseAssetScriptingInterface::downloadBytes" << error << result.keys(); // forward thread-safe copies back to our thread deferred->handle(error, result); request->deleteLater(); diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index b5492db304..a7d0bad3c2 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -42,12 +42,12 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { QByteArray dataByteArray = data.toUtf8(); auto upload = DependencyManager::get()->createUpload(dataByteArray); - Promise deferred = makePromise(__FUNCTION__) - ->ready([this, handler](QString error, QVariantMap result) { - auto url = result.value("url").toString(); - auto hash = result.value("hash").toString(); - jsCallback(handler, url, hash); - }); + Promise deferred = makePromise(__FUNCTION__); + deferred->ready([this, handler](QString error, QVariantMap result) { + auto url = result.value("url").toString(); + auto hash = result.value("hash").toString(); + jsCallback(handler, url, hash); + }); connect(upload, &AssetUpload::finished, upload, [this, deferred](AssetUpload* upload, const QString& hash) { // we are now on the "Resource Manager" thread (and "hash" being a *reference* makes it unsafe to use directly) @@ -64,10 +64,10 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) { auto handler = makeScopedHandlerObject(thisObject(), callback); auto setMappingRequest = assetClient()->createSetMappingRequest(path, hash); - Promise deferred = makePromise(__FUNCTION__) - ->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + Promise deferred = makePromise(__FUNCTION__); + deferred->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [this, deferred](SetMappingRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -96,11 +96,11 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb auto assetClient = DependencyManager::get(); auto assetRequest = assetClient->createRequest(hash); - Promise deferred = makePromise(__FUNCTION__) - ->ready([=](QString error, QVariantMap result) { - // FIXME: to remain backwards-compatible the signature here is "callback(data, n/a)" - jsCallback(handler, result.value("data").toString(), { { "errorMessage", error } }); - }); + Promise deferred = makePromise(__FUNCTION__); + deferred->ready([=](QString error, QVariantMap result) { + // FIXME: to remain backwards-compatible the signature here is "callback(data, n/a)" + jsCallback(handler, result.value("data").toString(), { { "errorMessage", error } }); + }); connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -127,10 +127,10 @@ void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScri auto handler = makeScopedHandlerObject(thisObject(), callback); auto setBakingEnabledRequest = DependencyManager::get()->createSetBakingEnabledRequest({ path }, enabled); - Promise deferred = makePromise(__FUNCTION__) - ->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + Promise deferred = makePromise(__FUNCTION__); + deferred->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, setBakingEnabledRequest, [this, deferred](SetBakingEnabledRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -160,14 +160,13 @@ void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { JS_ASSERT(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); JS_ASSERT(callback.isFunction(), "expected second parameter to be a callback function"); qDebug() << ">>getMapping//getAssetInfo" << path; - getAssetInfo(path)->ready([this, handler](QString error, QVariantMap result) { + Promise promise = getAssetInfo(path); + promise->ready([this, handler](QString error, QVariantMap result) { qDebug() << "//getMapping//getAssetInfo" << error << result.keys(); jsCallback(handler, error, result.value("hash").toString()); }); } -/////////////////////////// new APIS //////////////////////////////////// - bool AssetScriptingInterface::jsAssert(bool condition, const QString& error) { if (condition) { return true; @@ -187,10 +186,6 @@ void AssetScriptingInterface::jsCallback(const QScriptValue& handler, JS_ASSERT(handler.isObject() && handler.property("callback").isFunction(), QString("jsCallback -- .callback is not a function (%1)") .arg(handler.property("callback").toVariant().typeName())); -#if 1 || DEGUG_JSCALLBACK - QScriptValue debug = result; - debug.setProperty("toString", handler.engine()->evaluate("1,function() { return JSON.stringify(this, 0, 2); }")); -#endif ::callScopedHandlerObject(handler, errorValue, result); } @@ -228,8 +223,8 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, JS_ASSERT(RESPONSE_TYPES.contains(responseType), QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); - Promise resolved = makePromise("resolved"), - loaded = makePromise("loaded"); + Promise resolved = makePromise("resolved"); + Promise loaded = makePromise("loaded"); loaded->ready([=](QString error, QVariantMap result) { qDebug() << "//loaded" << error; @@ -242,9 +237,9 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, if (!error.isEmpty() || !AssetUtils::isValidHash(hash)) { loaded->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); } else { - loadAsset(hash, decompress, responseType) - ->mixin(result) - ->ready([this, loaded, hash](QString error, QVariantMap result) { + Promise promise = loadAsset(hash, decompress, responseType); + promise->mixin(result); + promise->ready([this, loaded, hash](QString error, QVariantMap result) { qDebug() << "//getAssetInfo/loadAsset" << error << hash; loaded->resolve(NoError, result); }); @@ -254,11 +249,11 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, if (AssetUtils::isValidHash(asset)) { resolved->resolve(NoError, { { "hash", asset } }); } else { - getAssetInfo(asset) - ->ready([this, resolved](QString error, QVariantMap result) { - qDebug() << "//getAssetInfo" << error << result.value("hash") << result.value("path"); + Promise promise = getAssetInfo(asset); + promise->ready([this, resolved](QString error, QVariantMap result) { + qDebug() << "//getAssetInfo" << error << result.value("hash") << result.value("path"); resolved->resolve(error, result); - }); + }); } } @@ -286,34 +281,40 @@ void AssetScriptingInterface::decompressData(QScriptValue options, QScriptValue if (responseType.isEmpty()) { responseType = "text"; } - decompressBytes(dataByteArray) - ->ready([=](QString error, QVariantMap result) { - if (responseType == "arraybuffer") { + Promise promise = decompressBytes(dataByteArray); + promise->ready([=](QString error, QVariantMap result) { + if (responseType == "arraybuffer") { + jsCallback(handler, error, result); + } else { + Promise promise = convertBytes(result.value("data").toByteArray(), responseType); + promise->mixin(result); + promise->ready([=](QString error, QVariantMap result) { jsCallback(handler, error, result); - } else { - convertBytes(result.value("data").toByteArray(), responseType) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); - } + }); + } }); } +namespace { + const int32_t DEFAULT_GZIP_COMPRESSION_LEVEL = -1; + const int32_t MAX_GZIP_COMPRESSION_LEVEL = 9; +} + void AssetScriptingInterface::compressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { + auto data = options.property("data"); QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); auto handler = makeScopedHandlerObject(scope, callback); auto level = options.property("level").toInt32(); - if (level < -1 || level > 9) { - level = -1; + if (level < DEFAULT_GZIP_COMPRESSION_LEVEL || level > MAX_GZIP_COMPRESSION_LEVEL) { + level = DEFAULT_GZIP_COMPRESSION_LEVEL; } - compressBytes(dataByteArray, level) - ->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + Promise promise = compressBytes(dataByteArray, level); + promise->ready([=](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); } void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { @@ -327,7 +328,6 @@ void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); - //auto rawByteLength = dataByteArray.size(); JS_ASSERT(path.isEmpty() || AssetUtils::isValidFilePath(path), QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path)); @@ -338,18 +338,17 @@ void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, .arg(data.toVariant().typeName()) .arg(dataByteArray.size())); - // [compressed] => uploaded to server => [mapped to path] - Promise prepared = makePromise("putAsset::prepared"), - uploaded = makePromise("putAsset::uploaded"), - finished = makePromise("putAsset::finished"); + Promise prepared = makePromise("putAsset::prepared"); + Promise uploaded = makePromise("putAsset::uploaded"); + Promise finished = makePromise("putAsset::finished"); if (compress) { qDebug() << "putAsset::compressBytes..."; - compressBytes(dataByteArray, -1) - ->finally([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::compressedBytes" << error << result.keys(); - prepared->handle(error, result); + Promise promise = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL); + promise->finally([=](QString error, QVariantMap result) { + qDebug() << "//putAsset::compressedBytes" << error << result.keys(); + prepared->handle(error, result); }); } else { prepared->resolve(NoError, {{ "data", dataByteArray }}); @@ -357,12 +356,12 @@ void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, prepared->ready([=](QString error, QVariantMap result) { qDebug() << "//putAsset::prepared" << error << result.value("data").toByteArray().size() << result.keys(); - uploadBytes(result.value("data").toByteArray()) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - qDebug() << "===========//putAsset::prepared/uploadBytes" << error << result.keys(); - uploaded->handle(error, result); - }); + Promise promise = uploadBytes(result.value("data").toByteArray()); + promise->mixin(result); + promise->ready([=](QString error, QVariantMap result) { + qDebug() << "===========//putAsset::prepared/uploadBytes" << error << result.keys(); + uploaded->handle(error, result); + }); }); uploaded->ready([=](QString error, QVariantMap result) { @@ -374,10 +373,10 @@ void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, finished->reject("path mapping requested, but did not receive valid hash", result); } else { qDebug() << "symlinkAsset" << hash << path << QThread::currentThread(); - symlinkAsset(hash, path) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - finished->handle(error, result); + Promise promise = symlinkAsset(hash, path); + promise->mixin(result); + promise->ready([=](QString error, QVariantMap result) { + finished->handle(error, result); qDebug() << "//symlinkAsset" << hash << path << result.keys(); }); } From a863aec4a031fe7bd3f60501448baa1546542226 Mon Sep 17 00:00:00 2001 From: humbletim Date: Wed, 3 Jan 2018 04:01:43 -0500 Subject: [PATCH 05/21] put existing AssetUtils functions in namespace per CR feedback --- assignment-client/src/assets/AssetServer.h | 1 + assignment-client/src/assets/BakeAssetTask.h | 2 ++ assignment-client/src/assets/UploadAssetTask.cpp | 5 +++-- libraries/networking/src/AssetClient.cpp | 14 ++++++++------ libraries/networking/src/AssetClient.h | 2 ++ libraries/networking/src/AssetRequest.cpp | 8 ++++---- libraries/networking/src/AssetRequest.h | 2 +- libraries/networking/src/AssetUpload.cpp | 4 ++-- libraries/networking/src/AssetUtils.cpp | 4 ++++ libraries/networking/src/AssetUtils.h | 16 +++------------- 10 files changed, 30 insertions(+), 28 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index e6393e6a98..7db37860b9 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -21,6 +21,7 @@ #include "AssetUtils.h" #include "ReceivedMessage.h" +using namespace AssetUtils; namespace std { template <> diff --git a/assignment-client/src/assets/BakeAssetTask.h b/assignment-client/src/assets/BakeAssetTask.h index c73a8bff65..8fe93a85e3 100644 --- a/assignment-client/src/assets/BakeAssetTask.h +++ b/assignment-client/src/assets/BakeAssetTask.h @@ -22,6 +22,8 @@ #include +using namespace AssetUtils; + class BakeAssetTask : public QObject, public QRunnable { Q_OBJECT public: diff --git a/assignment-client/src/assets/UploadAssetTask.cpp b/assignment-client/src/assets/UploadAssetTask.cpp index 5e6d59d032..238572c451 100644 --- a/assignment-client/src/assets/UploadAssetTask.cpp +++ b/assignment-client/src/assets/UploadAssetTask.cpp @@ -20,6 +20,7 @@ #include "ClientServerUtils.h" +using namespace AssetUtils; UploadAssetTask::UploadAssetTask(QSharedPointer receivedMessage, SharedNodePointer senderNode, const QDir& resourcesDir, uint64_t filesizeLimit) : @@ -54,7 +55,7 @@ void UploadAssetTask::run() { } else { QByteArray fileData = buffer.read(fileSize); - auto hash = hashData(fileData); + auto hash = AssetUtils::hashData(fileData); auto hexHash = hash.toHex(); qDebug() << "Hash for uploaded file from" << uuidStringWithoutCurlyBraces(_senderNode->getUUID()) @@ -66,7 +67,7 @@ void UploadAssetTask::run() { if (file.exists()) { // check if the local file has the correct contents, otherwise we overwrite - if (file.open(QIODevice::ReadOnly) && hashData(file.readAll()) == hash) { + if (file.open(QIODevice::ReadOnly) && AssetUtils::hashData(file.readAll()) == hash) { qDebug() << "Not overwriting existing verified file: " << hexHash; existingCorrectFile = true; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 725975c9fb..be58c0384c 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -34,6 +34,8 @@ MessageID AssetClient::_currentID = 0; +using AssetUtils::AssetMappingOperationType; + AssetClient::AssetClient() { _cacheDir = qApp->property(hifi::properties::APP_LOCAL_DATA_PATH).toString(); setCustomDeleter([](Dependency* dependency){ @@ -251,7 +253,7 @@ MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffse ReceivedAssetCallback callback, ProgressCallback progressCallback) { Q_ASSERT(QThread::currentThread() == thread()); - if (hash.length() != SHA256_HASH_HEX_LENGTH) { + if (hash.length() != AssetUtils::SHA256_HASH_HEX_LENGTH) { qCWarning(asset_client) << "Invalid hash size"; return false; } @@ -263,7 +265,7 @@ MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffse auto messageID = ++_currentID; - auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(start) + sizeof(end); + auto payloadSize = sizeof(messageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(start) + sizeof(end); auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true); qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server."; @@ -295,7 +297,7 @@ MessageID AssetClient::getAssetInfo(const QString& hash, GetInfoCallback callbac if (assetServer) { auto messageID = ++_currentID; - auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH; + auto payloadSize = sizeof(messageID) + AssetUtils::SHA256_HASH_LENGTH; auto packet = NLPacket::create(PacketType::AssetGetInfo, payloadSize, true); packet->writePrimitive(messageID); @@ -317,7 +319,7 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer messag MessageID messageID; message->readPrimitive(&messageID); - auto assetHash = message->read(SHA256_HASH_LENGTH); + auto assetHash = message->read(AssetUtils::SHA256_HASH_LENGTH); AssetServerError error; message->readPrimitive(&error); @@ -351,7 +353,7 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer messag void AssetClient::handleAssetGetReply(QSharedPointer message, SharedNodePointer senderNode) { Q_ASSERT(QThread::currentThread() == thread()); - auto assetHash = message->readHead(SHA256_HASH_LENGTH); + auto assetHash = message->readHead(AssetUtils::SHA256_HASH_LENGTH); qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex(); MessageID messageID; @@ -751,7 +753,7 @@ void AssetClient::handleAssetUploadReply(QSharedPointer message if (error) { qCWarning(asset_client) << "Error uploading file to asset server"; } else { - auto hash = message->read(SHA256_HASH_LENGTH); + auto hash = message->read(AssetUtils::SHA256_HASH_LENGTH); hashString = hash.toHex(); qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 5d854390e2..6f55866740 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -42,6 +42,8 @@ struct AssetInfo { int64_t size; }; +using namespace AssetUtils; + using MappingOperationCallback = std::function message)>; using ReceivedAssetCallback = std::function; using GetInfoCallback = std::function; diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 6a2bcdca9c..a036a28cfd 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -52,7 +52,7 @@ void AssetRequest::start() { } // in case we haven't parsed a valid hash, return an error now - if (!isValidHash(_hash)) { + if (!AssetUtils::isValidHash(_hash)) { _error = InvalidHash; _state = Finished; @@ -61,7 +61,7 @@ void AssetRequest::start() { } // Try to load from cache - _data = loadFromCache(getUrl()); + _data = AssetUtils::loadFromCache(getUrl()); if (!_data.isNull()) { _error = NoError; @@ -104,7 +104,7 @@ void AssetRequest::start() { break; } } else { - if (!_byteRange.isSet() && hashData(data).toHex() != _hash) { + if (!_byteRange.isSet() && AssetUtils::hashData(data).toHex() != _hash) { // the hash of the received data does not match what we expect, so we return an error _error = HashVerificationFailed; } @@ -115,7 +115,7 @@ void AssetRequest::start() { emit progress(_totalReceived, data.size()); if (!_byteRange.isSet()) { - saveToCache(getUrl(), data); + AssetUtils::saveToCache(getUrl(), data); } } } diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index b469e2f012..bf901c4e8f 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -52,7 +52,7 @@ public: const State& getState() const { return _state; } const Error& getError() const { return _error; } const QString getErrorString() const; - QUrl getUrl() const { return ::getATPUrl(_hash); } + QUrl getUrl() const { return AssetUtils::getATPUrl(_hash); } QString getHash() const { return _hash; } bool loadedFromCache() const { return _loadedFromCache; } diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index 077a0388cf..fabdd1febc 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -104,8 +104,8 @@ void AssetUpload::start() { } } - if (_error == NoError && hash == hashData(_data).toHex()) { - saveToCache(getATPUrl(hash), _data); + if (_error == NoError && hash == AssetUtils::hashData(_data).toHex()) { + AssetUtils::saveToCache(AssetUtils::getATPUrl(hash), _data); } emit finished(this, hash); diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index fa9885f053..45150e80c1 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -23,6 +23,8 @@ #include "ResourceManager.h" +namespace AssetUtils { + // Extract the valid AssetHash portion from atp: URLs like "[atp:]HASH[.fbx][?query]" // (or an invalid AssetHash if not found) AssetHash extractAssetHash(const QString& input) { @@ -131,3 +133,5 @@ QString bakingStatusToString(BakingStatus status) { return "--"; } } + +} // namespace AssetUtils diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 8c89b4701c..42e8dfad62 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -19,6 +19,8 @@ #include #include +namespace AssetUtils { + using DataOffset = int64_t; using AssetPath = QString; @@ -85,18 +87,6 @@ bool isValidHash(const QString& hashString); QString bakingStatusToString(BakingStatus status); -// backwards-compatible namespace aliases -// (allows new code to be explicit -- eg: `AssetUtils::isValidPath(path)` vs. `isValidPath(path)`) -namespace AssetUtils { - static const auto& loadFromCache = ::loadFromCache; - static const auto& saveToCache = ::saveToCache; - static const auto& hashData = ::hashData; - static const auto& getATPUrl = ::getATPUrl; - static const auto& extractAssetHash = ::extractAssetHash; - static const auto& isValidFilePath = ::isValidFilePath; - static const auto& isValidPath = ::isValidPath; - static const auto& isValidHash = ::isValidHash; -}; - +} // namespace AssetUtils #endif // hifi_AssetUtils_h From fc41bcca5af66e995de46551e72298cb0bdf1449 Mon Sep 17 00:00:00 2001 From: humbletim Date: Fri, 5 Jan 2018 18:41:51 -0500 Subject: [PATCH 06/21] removing unnecessary qWarning() debug output --- libraries/shared/src/shared/MiniPromises.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index c5d21a22fb..0222a91d44 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -83,7 +83,6 @@ public: _onfinally << always; }); } else { - qWarning() << "MiniPromise::finally() called after promise was already rejected or resolved:" << objectName(); executeOnPromiseThread([&]{ withReadLock([&]{ always(_error, _result); @@ -105,7 +104,6 @@ public: }); } else { executeOnPromiseThread([&]{ - qWarning() << "MiniPromise::fail() called after promise was already rejected:" << objectName(); withReadLock([&]{ failFunc(_error, _result); }); @@ -126,7 +124,6 @@ public: }); } else { executeOnPromiseThread([&]{ - qWarning() << "MiniPromise::then() called after promise was already resolved:" << objectName(); withReadLock([&]{ successFunc(_error, _result); }); From 744da485512c691ea110145a8bb3a809402cf57e Mon Sep 17 00:00:00 2001 From: humbletim Date: Wed, 17 Jan 2018 14:00:50 -0500 Subject: [PATCH 07/21] add apache 2.0 headers --- libraries/shared/src/shared/MiniPromises.cpp | 8 ++++++++ libraries/shared/src/shared/MiniPromises.h | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/libraries/shared/src/shared/MiniPromises.cpp b/libraries/shared/src/shared/MiniPromises.cpp index 28e2c18857..faada3627a 100644 --- a/libraries/shared/src/shared/MiniPromises.cpp +++ b/libraries/shared/src/shared/MiniPromises.cpp @@ -1,2 +1,10 @@ +// +// Created by Timothy Dedischew on 2017/12/21 +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #include "MiniPromises.h" int MiniPromise::metaTypeID = qRegisterMetaType("MiniPromise::Promise"); diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index 0222a91d44..3385118666 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -1,3 +1,11 @@ +// +// Created by Timothy Dedischew on 2017/12/21 +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #pragma once // Minimalist threadsafe Promise-like helper for managing asynchronous results From 0930cc4f0e3bfbcc4b001a08df1eab78521f4f7c Mon Sep 17 00:00:00 2001 From: humbletim Date: Wed, 17 Jan 2018 17:08:57 -0500 Subject: [PATCH 08/21] partial getMapping PR integration --- .../script-engine/src/AssetScriptingInterface.cpp | 5 ++--- .../script-engine/src/AssetScriptingInterface.h | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index a7d0bad3c2..54e74564ef 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -27,8 +27,7 @@ #include #include "Gzip.h" #include "ScriptEngine.h" - -//using Promise = MiniPromise::Promise; +#include "ScriptEngineLogging.h" AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) { if (auto engine = qobject_cast(parent)) { @@ -88,7 +87,7 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb if (!urlString.startsWith(ATP_SCHEME)) { // ... for now at least log a message so user can check logs - qDebug() << "AssetScriptingInterface::downloadData ERROR: url does not start with " << ATP_SCHEME; + qCDebug(scriptengine) << "AssetScriptingInterface::downloadData url must be of form atp:"; return; } QString hash = AssetUtils::extractAssetHash(urlString); diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 9022316634..f1a270b6c7 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -79,6 +79,21 @@ public: * @param {string} error */ Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); + + /**jsdoc + * Look up a path to hash mapping within the connected domain's asset server + * @function Assets.getMapping + * @static + * @param path {string} + * @param callback {Assets~getMappingCallback} + */ + + /**jsdoc + * Called when getMapping is complete. + * @callback Assets~getMappingCallback + * @param error {string} error description if the path could not be resolved; otherwise a null value. + * @param assetID {string} hash value if found, else an empty string + */ Q_INVOKABLE void getMapping(QString path, QScriptValue callback); Q_INVOKABLE void setBakingEnabled(QString path, bool enabled, QScriptValue callback); From d2a3a71f0036efc8af22051d61cc612d51596f3b Mon Sep 17 00:00:00 2001 From: humbletim Date: Thu, 18 Jan 2018 21:48:41 -0500 Subject: [PATCH 09/21] trying JS_VERIFY on for size (instead of JS_ASSERT) --- .../src/AssetScriptingInterface.cpp | 39 +++++++++---------- .../src/AssetScriptingInterface.h | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 47f338e419..d80491b2a6 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -28,12 +28,9 @@ #include "ScriptEngine.h" #include "ScriptEngineLogging.h" -AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) { - if (auto engine = qobject_cast(parent)) { - registerMetaTypes(engine); - } -} -#define JS_ASSERT(cond, error) { if (!this->jsAssert(cond, error)) { return; } } +AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) {} + +#define JS_VERIFY(cond, error) { if (!this->jsVerify(cond, error)) { return; } } void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { auto handler = makeScopedHandlerObject(thisObject(), callback); @@ -176,8 +173,8 @@ void AssetScriptingInterface::sendFakedHandshake() { void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { auto path = AssetUtils::getATPUrl(asset).path(); auto handler = makeScopedHandlerObject(thisObject(), callback); - JS_ASSERT(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); - JS_ASSERT(callback.isFunction(), "expected second parameter to be a callback function"); + JS_VERIFY(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); + JS_VERIFY(callback.isFunction(), "expected second parameter to be a callback function"); qDebug() << ">>getMapping//getAssetInfo" << path; Promise promise = getAssetInfo(path); promise->ready([this, handler](QString error, QVariantMap result) { @@ -186,14 +183,14 @@ void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { }); } -bool AssetScriptingInterface::jsAssert(bool condition, const QString& error) { +bool AssetScriptingInterface::jsVerify(bool condition, const QString& error) { if (condition) { return true; } if (context()) { context()->throwError(error); } else { - qDebug() << "WARNING -- jsAssert failed outside of a valid JS context: " + error; + qDebug() << "WARNING -- jsVerify failed outside of a valid JS context: " + error; } return false; } @@ -202,7 +199,7 @@ void AssetScriptingInterface::jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result) { Q_ASSERT(thread() == QThread::currentThread()); auto errorValue = !error.toBool() ? QScriptValue::NullValue : error; - JS_ASSERT(handler.isObject() && handler.property("callback").isFunction(), + JS_VERIFY(handler.isObject() && handler.property("callback").isFunction(), QString("jsCallback -- .callback is not a function (%1)") .arg(handler.property("callback").toVariant().typeName())); ::callScopedHandlerObject(handler, errorValue, result); @@ -217,11 +214,11 @@ void AssetScriptingInterface::jsCallback(const QScriptValue& handler, } void AssetScriptingInterface::deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { - jsAssert(false, "TODO: deleteAsset API"); + jsVerify(false, "TODO: deleteAsset API"); } void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { - JS_ASSERT(options.isObject() || options.isString(), "expected request options Object or URL as first parameter"); + JS_VERIFY(options.isObject() || options.isString(), "expected request options Object or URL as first parameter"); auto decompress = options.property("decompress").toBool() || options.property("compressed").toBool(); auto responseType = options.property("responseType").toString().toLower(); @@ -235,11 +232,11 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, auto asset = AssetUtils::getATPUrl(url).path(); auto handler = makeScopedHandlerObject(scope, callback); - JS_ASSERT(handler.property("callback").isFunction(), + JS_VERIFY(handler.property("callback").isFunction(), QString("Invalid callback function (%1)").arg(handler.property("callback").toVariant().typeName())); - JS_ASSERT(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset), + JS_VERIFY(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset), QString("Invalid ATP url '%1'").arg(url)); - JS_ASSERT(RESPONSE_TYPES.contains(responseType), + JS_VERIFY(RESPONSE_TYPES.contains(responseType), QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); Promise resolved = makePromise("resolved"); @@ -283,9 +280,9 @@ void AssetScriptingInterface::resolveAsset(QScriptValue options, QScriptValue sc auto asset = AssetUtils::getATPUrl(url).path(); auto handler = makeScopedHandlerObject(scope, callback); - JS_ASSERT(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset), + JS_VERIFY(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset), "expected options to be an asset URL or request options containing .url property"); - JS_ASSERT(handler.property("callback").isFunction(), "invalid callback function"); + JS_VERIFY(handler.property("callback").isFunction(), "invalid callback function"); getAssetInfo(asset)->ready([=](QString error, QVariantMap result) { qDebug() << "//resolveAsset/getAssetInfo" << error << result.value("hash"); jsCallback(handler, error, result); @@ -348,11 +345,11 @@ void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, data.toString().toUtf8() : qscriptvalue_cast(data); - JS_ASSERT(path.isEmpty() || AssetUtils::isValidFilePath(path), + JS_VERIFY(path.isEmpty() || AssetUtils::isValidFilePath(path), QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path)); - JS_ASSERT(handler.property("callback").isFunction(), + JS_VERIFY(handler.property("callback").isFunction(), "invalid callback function"); - JS_ASSERT(dataByteArray.size() > 0, + JS_VERIFY(dataByteArray.size() > 0, QString("expected non-zero .data (got %1 / #%2 bytes)") .arg(data.toVariant().typeName()) .arg(dataByteArray.size())); diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index f1a270b6c7..7001c64634 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -124,7 +124,7 @@ public: protected: void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QVariantMap& result); void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result); - bool jsAssert(bool condition, const QString& error); + bool jsVerify(bool condition, const QString& error); }; #endif // hifi_AssetScriptingInterface_h From dfb1619dc6ab956f2973371c670fd60a1fb480e4 Mon Sep 17 00:00:00 2001 From: humbletim Date: Thu, 18 Jan 2018 21:49:53 -0500 Subject: [PATCH 10/21] CR feedback; using namespace AssetUtils => inline AssetUtils:: --- assignment-client/src/assets/AssetServer.cpp | 163 +++++++++--------- assignment-client/src/assets/AssetServer.h | 32 ++-- .../src/assets/BakeAssetTask.cpp | 2 +- assignment-client/src/assets/BakeAssetTask.h | 8 +- .../src/assets/SendAssetTask.cpp | 12 +- .../src/assets/UploadAssetTask.cpp | 10 +- .../AssetMappingsScriptingInterface.cpp | 4 +- libraries/networking/src/AssetClient.cpp | 84 +++++---- libraries/networking/src/AssetClient.h | 38 ++-- libraries/networking/src/AssetRequest.cpp | 8 +- .../networking/src/AssetResourceRequest.cpp | 4 +- .../networking/src/AssetResourceRequest.h | 4 +- libraries/networking/src/AssetUpload.cpp | 10 +- libraries/networking/src/MappingRequest.cpp | 66 +++---- libraries/networking/src/MappingRequest.h | 40 ++--- 15 files changed, 238 insertions(+), 247 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 1b533f10f3..0a868737b0 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -39,7 +39,6 @@ #include "SendAssetTask.h" #include "UploadAssetTask.h" - static const uint8_t MIN_CORES_FOR_MULTICORE = 4; static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2; static const uint8_t CPU_AFFINITY_COUNT_LOW = 1; @@ -56,7 +55,7 @@ static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx"; static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx"; static const QString BAKED_SCRIPT_SIMPLE_NAME = "asset.js"; -void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) { +void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) { qDebug() << "Starting bake for: " << assetPath << assetHash; auto it = _pendingBakes.find(assetHash); if (it == _pendingBakes.end()) { @@ -74,23 +73,23 @@ void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPa } } -QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) { +QString AssetServer::getPathToAssetHash(const AssetUtils::AssetHash& assetHash) { return _filesDirectory.absoluteFilePath(assetHash); } -std::pair AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) { +std::pair AssetServer::getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) { auto it = _pendingBakes.find(hash); if (it != _pendingBakes.end()) { - return { (*it)->isBaking() ? Baking : Pending, "" }; + return { (*it)->isBaking() ? AssetUtils::Baking : AssetUtils::Pending, "" }; } - if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { - return { Baked, "" }; + if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { + return { AssetUtils::Baked, "" }; } auto dotIndex = path.lastIndexOf("."); if (dotIndex == -1) { - return { Irrelevant, "" }; + return { AssetUtils::Irrelevant, "" }; } auto extension = path.mid(dotIndex + 1); @@ -104,16 +103,16 @@ std::pair AssetServer::getAssetStatus(const AssetPath& pa } else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) { bakedFilename = BAKED_SCRIPT_SIMPLE_NAME; } else { - return { Irrelevant, "" }; + return { AssetUtils::Irrelevant, "" }; } - auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename; + auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename; auto jt = _fileMappings.find(bakedPath); if (jt != _fileMappings.end()) { if (jt->second == hash) { - return { NotBaked, "" }; + return { AssetUtils::NotBaked, "" }; } else { - return { Baked, "" }; + return { AssetUtils::Baked, "" }; } } else { bool loaded; @@ -121,11 +120,11 @@ std::pair AssetServer::getAssetStatus(const AssetPath& pa std::tie(loaded, meta) = readMetaFile(hash); if (loaded && meta.failedLastBake) { - return { Error, meta.lastBakeErrors }; + return { AssetUtils::Error, meta.lastBakeErrors }; } } - return { Pending, "" }; + return { AssetUtils::Pending, "" }; } void AssetServer::bakeAssets() { @@ -137,14 +136,14 @@ void AssetServer::bakeAssets() { } } -void AssetServer::maybeBake(const AssetPath& path, const AssetHash& hash) { +void AssetServer::maybeBake(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) { if (needsToBeBaked(path, hash)) { qDebug() << "Queuing bake of: " << path; bakeAsset(hash, path, getPathToAssetHash(hash)); } } -void AssetServer::createEmptyMetaFile(const AssetHash& hash) { +void AssetServer::createEmptyMetaFile(const AssetUtils::AssetHash& hash) { QString metaFilePath = "atp:/" + hash + "/meta.json"; QFile metaFile { metaFilePath }; @@ -157,14 +156,14 @@ void AssetServer::createEmptyMetaFile(const AssetHash& hash) { } } -bool AssetServer::hasMetaFile(const AssetHash& hash) { - QString metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json"; +bool AssetServer::hasMetaFile(const AssetUtils::AssetHash& hash) { + QString metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json"; return _fileMappings.find(metaFilePath) != _fileMappings.end(); } -bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) { - if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { +bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& assetHash) { + if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { return false; } @@ -196,7 +195,7 @@ bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHa return false; } - auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename; + auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename; return _fileMappings.find(bakedPath) == _fileMappings.end(); } @@ -235,7 +234,7 @@ AssetServer::AssetServer(ReceivedMessage& message) : ThreadedAssignment(message), _transferTaskPool(this), _bakingTaskPool(this), - _filesizeLimit(MAX_UPLOAD_SIZE) + _filesizeLimit(AssetUtils::MAX_UPLOAD_SIZE) { // store the current state of image compression so we can reset it when this assignment is complete _wasColorTextureCompressionEnabled = image::isColorTexturesCompressionEnabled(); @@ -390,7 +389,7 @@ void AssetServer::completeSetup() { // Check the asset directory to output some information about what we have auto files = _filesDirectory.entryList(QDir::Files); - QRegExp hashFileRegex { ASSET_HASH_REGEX_STRING }; + QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING }; auto hashedFiles = files.filter(hashFileRegex); qCInfo(asset_server) << "There are" << hashedFiles.size() << "asset files in the asset directory."; @@ -410,9 +409,9 @@ void AssetServer::completeSetup() { // get file size limit for an asset static const QString ASSETS_FILESIZE_LIMIT_OPTION = "assets_filesize_limit"; auto assetsFilesizeLimitJSONValue = assetServerObject[ASSETS_FILESIZE_LIMIT_OPTION]; - auto assetsFilesizeLimit = (uint64_t)assetsFilesizeLimitJSONValue.toInt(MAX_UPLOAD_SIZE); + auto assetsFilesizeLimit = (uint64_t)assetsFilesizeLimitJSONValue.toInt(AssetUtils::MAX_UPLOAD_SIZE); - if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < MAX_UPLOAD_SIZE) { + if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < AssetUtils::MAX_UPLOAD_SIZE) { _filesizeLimit = assetsFilesizeLimit * BITS_PER_MEGABITS; } @@ -421,7 +420,7 @@ void AssetServer::completeSetup() { } void AssetServer::cleanupUnmappedFiles() { - QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}" }; + QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(AssetUtils::SHA256_HASH_HEX_LENGTH) + "}" }; auto files = _filesDirectory.entryInfoList(QDir::Files); @@ -454,6 +453,8 @@ void AssetServer::cleanupUnmappedFiles() { } void AssetServer::handleAssetMappingOperation(QSharedPointer message, SharedNodePointer senderNode) { + using AssetMappingOperationType = AssetUtils::AssetMappingOperationType; + MessageID messageID; message->readPrimitive(&messageID); @@ -519,7 +520,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode if (!bakedRootFile.isEmpty()) { // we ran into an asset for which we could have a baked version, let's check if it's ready - bakedAssetPath = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile; + bakedAssetPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile; auto bakedIt = _fileMappings.find(bakedAssetPath); if (bakedIt != _fileMappings.end()) { @@ -537,7 +538,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode } } - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); if (wasRedirected) { qDebug() << "Writing re-directed hash for" << originalAssetHash << "to" << redirectedAssetHash; @@ -563,12 +564,12 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode } } } else { - replyPacket.writePrimitive(AssetServerError::AssetNotFound); + replyPacket.writePrimitive(AssetUtils::AssetServerError::AssetNotFound); } } void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) { - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); uint32_t count = (uint32_t)_fileMappings.size(); @@ -580,11 +581,11 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN replyPacket.writeString(mapping); replyPacket.write(QByteArray::fromHex(hash.toUtf8())); - BakingStatus status; + AssetUtils::BakingStatus status; QString lastBakeErrors; std::tie(status, lastBakeErrors) = getAssetStatus(mapping, hash); replyPacket.writePrimitive(status); - if (status == Error) { + if (status == AssetUtils::Error) { replyPacket.writeString(lastBakeErrors); } } @@ -594,22 +595,22 @@ void AssetServer::handleSetMappingOperation(ReceivedMessage& message, SharedNode if (senderNode->getCanWriteToAssetServer()) { QString assetPath = message.readString(); - auto assetHash = message.read(SHA256_HASH_LENGTH).toHex(); + auto assetHash = message.read(AssetUtils::SHA256_HASH_LENGTH).toHex(); // don't process a set mapping operation that is inside the hidden baked folder - if (assetPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { - qCDebug(asset_server) << "Refusing to process a set mapping operation inside" << HIDDEN_BAKED_CONTENT_FOLDER; - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + if (assetPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { + qCDebug(asset_server) << "Refusing to process a set mapping operation inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER; + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } else { if (setMapping(assetPath, assetHash)) { - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed); } } } else { - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } } @@ -623,21 +624,21 @@ void AssetServer::handleDeleteMappingsOperation(ReceivedMessage& message, Shared for (int i = 0; i < numberOfDeletedMappings; ++i) { auto mapping = message.readString(); - if (!mapping.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + if (!mapping.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { mappingsToDelete << mapping; } else { qCDebug(asset_server) << "Refusing to delete mapping" << mapping - << "that is inside" << HIDDEN_BAKED_CONTENT_FOLDER; + << "that is inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER; } } if (deleteMappings(mappingsToDelete)) { - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed); } } else { - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } } @@ -646,20 +647,20 @@ void AssetServer::handleRenameMappingOperation(ReceivedMessage& message, SharedN QString oldPath = message.readString(); QString newPath = message.readString(); - if (oldPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER) || newPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + if (oldPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER) || newPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { qCDebug(asset_server) << "Cannot rename" << oldPath << "to" << newPath - << "since one of the paths is inside" << HIDDEN_BAKED_CONTENT_FOLDER; - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + << "since one of the paths is inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER; + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } else { if (renameMapping(oldPath, newPath)) { - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed); } } } else { - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } } @@ -678,12 +679,12 @@ void AssetServer::handleSetBakingEnabledOperation(ReceivedMessage& message, Shar } if (setBakingEnabled(mappings, enabled)) { - replyPacket.writePrimitive(AssetServerError::NoError); + replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError); } else { - replyPacket.writePrimitive(AssetServerError::MappingOperationFailed); + replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed); } } else { - replyPacket.writePrimitive(AssetServerError::PermissionDenied); + replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied); } } @@ -691,15 +692,15 @@ void AssetServer::handleAssetGetInfo(QSharedPointer message, Sh QByteArray assetHash; MessageID messageID; - if (message->getSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID))) { + if (message->getSize() < qint64(AssetUtils::SHA256_HASH_LENGTH + sizeof(messageID))) { qCDebug(asset_server) << "ERROR bad file request"; return; } message->readPrimitive(&messageID); - assetHash = message->readWithoutCopy(SHA256_HASH_LENGTH); + assetHash = message->readWithoutCopy(AssetUtils::SHA256_HASH_LENGTH); - auto size = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(AssetServerError) + sizeof(qint64)); + auto size = qint64(sizeof(MessageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(AssetUtils::AssetServerError) + sizeof(qint64)); auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply, size, true); QByteArray hexHash = assetHash.toHex(); @@ -712,11 +713,11 @@ void AssetServer::handleAssetGetInfo(QSharedPointer message, Sh if (fileInfo.exists() && fileInfo.isReadable()) { qCDebug(asset_server) << "Opening file: " << fileInfo.filePath(); - replyPacket->writePrimitive(AssetServerError::NoError); + replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError); replyPacket->writePrimitive(fileInfo.size()); } else { qCDebug(asset_server) << "Asset not found: " << QString(hexHash); - replyPacket->writePrimitive(AssetServerError::AssetNotFound); + replyPacket->writePrimitive(AssetUtils::AssetServerError::AssetNotFound); } auto nodeList = DependencyManager::get(); @@ -725,7 +726,7 @@ void AssetServer::handleAssetGetInfo(QSharedPointer message, Sh void AssetServer::handleAssetGet(QSharedPointer message, SharedNodePointer senderNode) { - auto minSize = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(DataOffset) + sizeof(DataOffset)); + auto minSize = qint64(sizeof(MessageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(AssetUtils::DataOffset) + sizeof(AssetUtils::DataOffset)); if (message->getSize() < minSize) { qCDebug(asset_server) << "ERROR bad file request"; @@ -749,14 +750,14 @@ void AssetServer::handleAssetUpload(QSharedPointer message, Sha // for now this also means it isn't allowed to add assets // so return a packet with error that indicates that - auto permissionErrorPacket = NLPacket::create(PacketType::AssetUploadReply, sizeof(MessageID) + sizeof(AssetServerError), true); + auto permissionErrorPacket = NLPacket::create(PacketType::AssetUploadReply, sizeof(MessageID) + sizeof(AssetUtils::AssetServerError), true); MessageID messageID; message->readPrimitive(&messageID); // write the message ID and a permission denied error permissionErrorPacket->writePrimitive(messageID); - permissionErrorPacket->writePrimitive(AssetServerError::PermissionDenied); + permissionErrorPacket->writePrimitive(AssetUtils::AssetServerError::PermissionDenied); // send off the packet auto nodeList = DependencyManager::get(); @@ -863,12 +864,12 @@ bool AssetServer::loadMappingsFromFile() { continue; } - if (!isValidFilePath(key)) { + if (!AssetUtils::isValidFilePath(key)) { qCWarning(asset_server) << "Will not keep mapping for" << key << "since it is not a valid path."; continue; } - if (!isValidHash(value.toString())) { + if (!AssetUtils::isValidHash(value.toString())) { qCWarning(asset_server) << "Will not keep mapping for" << key << "since it does not have a valid hash."; continue; } @@ -918,15 +919,15 @@ bool AssetServer::writeMappingsToFile() { return false; } -bool AssetServer::setMapping(AssetPath path, AssetHash hash) { +bool AssetServer::setMapping(AssetUtils::AssetPath path, AssetUtils::AssetHash hash) { path = path.trimmed(); - if (!isValidFilePath(path)) { + if (!AssetUtils::isValidFilePath(path)) { qCWarning(asset_server) << "Cannot set a mapping for invalid path:" << path << "=>" << hash; return false; } - if (!isValidHash(hash)) { + if (!AssetUtils::isValidHash(hash)) { qCWarning(asset_server) << "Cannot set a mapping for invalid hash" << path << "=>" << hash; return false; } @@ -958,23 +959,23 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) { } } -bool pathIsFolder(const AssetPath& path) { +bool pathIsFolder(const AssetUtils::AssetPath& path) { return path.endsWith('/'); } -void AssetServer::removeBakedPathsForDeletedAsset(AssetHash hash) { +void AssetServer::removeBakedPathsForDeletedAsset(AssetUtils::AssetHash hash) { // we deleted the file with this hash // check if we had baked content for that file that should also now be removed // by calling deleteMappings for the hidden baked content folder for this hash - AssetPathList hiddenBakedFolder { HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" }; + AssetUtils::AssetPathList hiddenBakedFolder { AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" }; qCDebug(asset_server) << "Deleting baked content below" << hiddenBakedFolder << "since" << hash << "was deleted"; deleteMappings(hiddenBakedFolder); } -bool AssetServer::deleteMappings(const AssetPathList& paths) { +bool AssetServer::deleteMappings(const AssetUtils::AssetPathList& paths) { // take a copy of the current mappings in case persistence of these deletes fails auto oldMappings = _fileMappings; @@ -1060,11 +1061,11 @@ bool AssetServer::deleteMappings(const AssetPathList& paths) { } } -bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) { +bool AssetServer::renameMapping(AssetUtils::AssetPath oldPath, AssetUtils::AssetPath newPath) { oldPath = oldPath.trimmed(); newPath = newPath.trimmed(); - if (!isValidFilePath(oldPath) || !isValidFilePath(newPath)) { + if (!AssetUtils::isValidFilePath(oldPath) || !AssetUtils::isValidFilePath(newPath)) { qCWarning(asset_server) << "Cannot perform rename with invalid paths - both should have leading forward and no ending slashes:" << oldPath << "=>" << newPath; @@ -1164,8 +1165,8 @@ static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx"; static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx"; static const QString BAKED_ASSET_SIMPLE_JS_NAME = "asset.js"; -QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) { - return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; +QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativeFilePath) { + return AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath; } void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) { @@ -1197,7 +1198,7 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina qDebug() << "File path: " << filePath; - AssetHash bakedFileHash; + AssetUtils::AssetHash bakedFileHash; if (file.open(QIODevice::ReadOnly)) { QCryptographicHash hasher(QCryptographicHash::Sha256); @@ -1290,8 +1291,8 @@ static const QString BAKE_VERSION_KEY = "bake_version"; static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake"; static const QString LAST_BAKE_ERRORS_KEY = "last_bake_errors"; -std::pair AssetServer::readMetaFile(AssetHash hash) { - auto metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json"; +std::pair AssetServer::readMetaFile(AssetUtils::AssetHash hash) { + auto metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json"; auto it = _fileMappings.find(metaFilePath); if (it == _fileMappings.end()) { @@ -1335,7 +1336,7 @@ std::pair AssetServer::readMetaFile(AssetHash hash) { return { false, {} }; } -bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta) { +bool AssetServer::writeMetaFile(AssetUtils::AssetHash originalAssetHash, const AssetMeta& meta) { // construct the JSON that will be in the meta file QJsonObject metaFileObject; @@ -1349,7 +1350,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me auto metaFileJSON = metaFileDoc.toJson(); // get a hash for the contents of the meta-file - AssetHash metaFileHash = QCryptographicHash::hash(metaFileJSON, QCryptographicHash::Sha256).toHex(); + AssetUtils::AssetHash metaFileHash = QCryptographicHash::hash(metaFileJSON, QCryptographicHash::Sha256).toHex(); // create the meta file in our files folder, named by the hash of its contents QFile metaFile(_filesDirectory.absoluteFilePath(metaFileHash)); @@ -1359,7 +1360,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me metaFile.close(); // add a mapping to the meta file so it doesn't get deleted because it is unmapped - auto metaFileMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + "meta.json"; + auto metaFileMapping = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + "meta.json"; return setMapping(metaFileMapping, metaFileHash); } else { @@ -1367,7 +1368,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me } } -bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) { +bool AssetServer::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled) { for (const auto& path : paths) { auto it = _fileMappings.find(path); if (it != _fileMappings.end()) { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 7db37860b9..5e7a3c1700 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -21,8 +21,6 @@ #include "AssetUtils.h" #include "ReceivedMessage.h" -using namespace AssetUtils; - namespace std { template <> struct hash { @@ -76,29 +74,29 @@ private: bool writeMappingsToFile(); /// Set the mapping for path to hash - bool setMapping(AssetPath path, AssetHash hash); + bool setMapping(AssetUtils::AssetPath path, AssetUtils::AssetHash hash); /// Delete mapping `path`. Returns `true` if deletion of mappings succeeds, else `false`. - bool deleteMappings(const AssetPathList& paths); + bool deleteMappings(const AssetUtils::AssetPathList& paths); /// Rename mapping from `oldPath` to `newPath`. Returns true if successful - bool renameMapping(AssetPath oldPath, AssetPath newPath); + bool renameMapping(AssetUtils::AssetPath oldPath, AssetUtils::AssetPath newPath); - bool setBakingEnabled(const AssetPathList& paths, bool enabled); + bool setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled); /// Delete any unmapped files from the local asset directory void cleanupUnmappedFiles(); - QString getPathToAssetHash(const AssetHash& assetHash); + QString getPathToAssetHash(const AssetUtils::AssetHash& assetHash); - std::pair getAssetStatus(const AssetPath& path, const AssetHash& hash); + std::pair getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash); void bakeAssets(); - void maybeBake(const AssetPath& path, const AssetHash& hash); - void createEmptyMetaFile(const AssetHash& hash); - bool hasMetaFile(const AssetHash& hash); - bool needsToBeBaked(const AssetPath& path, const AssetHash& assetHash); - void bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); + void maybeBake(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash); + void createEmptyMetaFile(const AssetUtils::AssetHash& hash); + bool hasMetaFile(const AssetUtils::AssetHash& hash); + bool needsToBeBaked(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& assetHash); + void bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath); /// Move baked content for asset to baked directory and update baked status void handleCompletedBake(QString originalAssetHash, QString assetPath, QString bakedTempOutputDir, @@ -107,11 +105,11 @@ private: void handleAbortedBake(QString originalAssetHash, QString assetPath); /// Create meta file to describe baked content for original asset - std::pair readMetaFile(AssetHash hash); - bool writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta = AssetMeta()); + std::pair readMetaFile(AssetUtils::AssetHash hash); + bool writeMetaFile(AssetUtils::AssetHash originalAssetHash, const AssetMeta& meta = AssetMeta()); /// Remove baked paths when the original asset is deleteds - void removeBakedPathsForDeletedAsset(AssetHash originalAssetHash); + void removeBakedPathsForDeletedAsset(AssetUtils::AssetHash originalAssetHash); Mappings _fileMappings; @@ -121,7 +119,7 @@ private: /// Task pool for handling uploads and downloads of assets QThreadPool _transferTaskPool; - QHash> _pendingBakes; + QHash> _pendingBakes; QThreadPool _bakingTaskPool; bool _wasColorTextureCompressionEnabled { false }; diff --git a/assignment-client/src/assets/BakeAssetTask.cpp b/assignment-client/src/assets/BakeAssetTask.cpp index 49322ca4cb..5b9add7467 100644 --- a/assignment-client/src/assets/BakeAssetTask.cpp +++ b/assignment-client/src/assets/BakeAssetTask.cpp @@ -24,7 +24,7 @@ static const int OVEN_STATUS_CODE_ABORT { 2 }; std::once_flag registerMetaTypesFlag; -BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) : +BakeAssetTask::BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) : _assetHash(assetHash), _assetPath(assetPath), _filePath(filePath) diff --git a/assignment-client/src/assets/BakeAssetTask.h b/assignment-client/src/assets/BakeAssetTask.h index 8fe93a85e3..7b003fa1bd 100644 --- a/assignment-client/src/assets/BakeAssetTask.h +++ b/assignment-client/src/assets/BakeAssetTask.h @@ -22,12 +22,10 @@ #include -using namespace AssetUtils; - class BakeAssetTask : public QObject, public QRunnable { Q_OBJECT public: - BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath); + BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath); bool isBaking() { return _isBaking.load(); } @@ -43,8 +41,8 @@ signals: private: std::atomic _isBaking { false }; - AssetHash _assetHash; - AssetPath _assetPath; + AssetUtils::AssetHash _assetHash; + AssetUtils::AssetPath _assetPath; QString _filePath; std::unique_ptr _ovenProcess { nullptr }; std::atomic _wasAborted { false }; diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index eab88e0d46..6da092357f 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -40,7 +40,7 @@ void SendAssetTask::run() { ByteRange byteRange; _message->readPrimitive(&messageID); - QByteArray assetHash = _message->read(SHA256_HASH_LENGTH); + QByteArray assetHash = _message->read(AssetUtils::SHA256_HASH_LENGTH); // `start` and `end` indicate the range of data to retrieve for the asset identified by `assetHash`. // `start` is inclusive, `end` is exclusive. Requesting `start` = 1, `end` = 10 will retrieve 9 bytes of data, @@ -61,7 +61,7 @@ void SendAssetTask::run() { replyPacketList->writePrimitive(messageID); if (!byteRange.isValid()) { - replyPacketList->writePrimitive(AssetServerError::InvalidByteRange); + replyPacketList->writePrimitive(AssetUtils::AssetServerError::InvalidByteRange); } else { QString filePath = _resourcesDir.filePath(QString(hexHash)); @@ -75,7 +75,7 @@ void SendAssetTask::run() { // check if we're being asked to read data that we just don't have // because of the file size if (file.size() < byteRange.fromInclusive || file.size() < byteRange.toExclusive) { - replyPacketList->writePrimitive(AssetServerError::InvalidByteRange); + replyPacketList->writePrimitive(AssetUtils::AssetServerError::InvalidByteRange); qCDebug(networking) << "Bad byte range: " << hexHash << " " << byteRange.fromInclusive << ":" << byteRange.toExclusive; } else { @@ -86,7 +86,7 @@ void SendAssetTask::run() { // this range is positive, meaning we just need to seek into the file and then read from there file.seek(byteRange.fromInclusive); - replyPacketList->writePrimitive(AssetServerError::NoError); + replyPacketList->writePrimitive(AssetUtils::AssetServerError::NoError); replyPacketList->writePrimitive(size); replyPacketList->write(file.read(size)); } else { @@ -95,7 +95,7 @@ void SendAssetTask::run() { // seek to the part of the file where the negative range begins file.seek(file.size() + byteRange.fromInclusive); - replyPacketList->writePrimitive(AssetServerError::NoError); + replyPacketList->writePrimitive(AssetUtils::AssetServerError::NoError); replyPacketList->writePrimitive(size); // first write everything from the negative range to the end of the file @@ -107,7 +107,7 @@ void SendAssetTask::run() { file.close(); } else { qCDebug(networking) << "Asset not found: " << filePath << "(" << hexHash << ")"; - replyPacketList->writePrimitive(AssetServerError::AssetNotFound); + replyPacketList->writePrimitive(AssetUtils::AssetServerError::AssetNotFound); } } diff --git a/assignment-client/src/assets/UploadAssetTask.cpp b/assignment-client/src/assets/UploadAssetTask.cpp index 238572c451..68bf4db5fd 100644 --- a/assignment-client/src/assets/UploadAssetTask.cpp +++ b/assignment-client/src/assets/UploadAssetTask.cpp @@ -20,8 +20,6 @@ #include "ClientServerUtils.h" -using namespace AssetUtils; - UploadAssetTask::UploadAssetTask(QSharedPointer receivedMessage, SharedNodePointer senderNode, const QDir& resourcesDir, uint64_t filesizeLimit) : _receivedMessage(receivedMessage), @@ -51,7 +49,7 @@ void UploadAssetTask::run() { replyPacket->writePrimitive(messageID); if (fileSize > _filesizeLimit) { - replyPacket->writePrimitive(AssetServerError::AssetTooLarge); + replyPacket->writePrimitive(AssetUtils::AssetServerError::AssetTooLarge); } else { QByteArray fileData = buffer.read(fileSize); @@ -72,7 +70,7 @@ void UploadAssetTask::run() { existingCorrectFile = true; - replyPacket->writePrimitive(AssetServerError::NoError); + replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError); replyPacket->write(hash); } else { qDebug() << "Overwriting an existing file whose contents did not match the expected hash: " << hexHash; @@ -85,7 +83,7 @@ void UploadAssetTask::run() { qDebug() << "Wrote file" << hexHash << "to disk. Upload complete"; file.close(); - replyPacket->writePrimitive(AssetServerError::NoError); + replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError); replyPacket->write(hash); } else { qWarning() << "Failed to upload or write to file" << hexHash << " - upload failed."; @@ -97,7 +95,7 @@ void UploadAssetTask::run() { qWarning() << "Removal of failed upload file" << hexHash << "failed."; } - replyPacket->writePrimitive(AssetServerError::FileOperationFailed); + replyPacket->writePrimitive(AssetUtils::AssetServerError::FileOperationFailed); } } diff --git a/interface/src/scripting/AssetMappingsScriptingInterface.cpp b/interface/src/scripting/AssetMappingsScriptingInterface.cpp index 72b6f401e9..211545bf05 100644 --- a/interface/src/scripting/AssetMappingsScriptingInterface.cpp +++ b/interface/src/scripting/AssetMappingsScriptingInterface.cpp @@ -263,7 +263,7 @@ void AssetMappingModel::refresh() { for (auto& mapping : mappings) { auto& path = mapping.first; - if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) { + if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) { // Hide baked mappings continue; } @@ -306,7 +306,7 @@ void AssetMappingModel::refresh() { auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status); lastItem->setData(statusString, Qt::UserRole + 5); lastItem->setData(mapping.second.bakingErrors, Qt::UserRole + 6); - if (mapping.second.status == Pending) { + if (mapping.second.status == AssetUtils::Pending) { ++numPendingBakes; } } diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index be58c0384c..6980621219 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -34,8 +34,6 @@ MessageID AssetClient::_currentID = 0; -using AssetUtils::AssetMappingOperationType; - AssetClient::AssetClient() { _cacheDir = qApp->property(hifi::properties::APP_LOCAL_DATA_PATH).toString(); setCustomDeleter([](Dependency* dependency){ @@ -140,7 +138,7 @@ void AssetClient::handleAssetMappingOperationReply(QSharedPointerreadPrimitive(&messageID); - AssetServerError error; + AssetUtils::AssetServerError error; message->readPrimitive(&error); // Check if we have any pending requests for this node @@ -176,7 +174,7 @@ bool haveAssetServer() { return true; } -GetMappingRequest* AssetClient::createGetMappingRequest(const AssetPath& path) { +GetMappingRequest* AssetClient::createGetMappingRequest(const AssetUtils::AssetPath& path) { auto request = new GetMappingRequest(path); request->moveToThread(thread()); @@ -192,7 +190,7 @@ GetAllMappingsRequest* AssetClient::createGetAllMappingsRequest() { return request; } -DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetPathList& paths) { +DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetUtils::AssetPathList& paths) { auto request = new DeleteMappingsRequest(paths); request->moveToThread(thread()); @@ -200,7 +198,7 @@ DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetPathL return request; } -SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, const AssetHash& hash) { +SetMappingRequest* AssetClient::createSetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) { auto request = new SetMappingRequest(path, hash); request->moveToThread(thread()); @@ -208,7 +206,7 @@ SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, c return request; } -RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath) { +RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath) { auto request = new RenameMappingRequest(oldPath, newPath); request->moveToThread(thread()); @@ -216,7 +214,7 @@ RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& o return request; } -SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetPathList& path, bool enabled) { +SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled) { auto bakingEnabledRequest = new SetBakingEnabledRequest(path, enabled); bakingEnabledRequest->moveToThread(thread()); @@ -224,7 +222,7 @@ SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetP return bakingEnabledRequest; } -AssetRequest* AssetClient::createRequest(const AssetHash& hash, const ByteRange& byteRange) { +AssetRequest* AssetClient::createRequest(const AssetUtils::AssetHash& hash, const ByteRange& byteRange) { auto request = new AssetRequest(hash, byteRange); // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case) @@ -249,7 +247,7 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data) { return upload; } -MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffset end, +MessageID AssetClient::getAsset(const QString& hash, AssetUtils::DataOffset start, AssetUtils::DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback) { Q_ASSERT(QThread::currentThread() == thread()); @@ -284,7 +282,7 @@ MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffse } } - callback(false, AssetServerError::NoError, QByteArray()); + callback(false, AssetUtils::AssetServerError::NoError, QByteArray()); return INVALID_MESSAGE_ID; } @@ -310,7 +308,7 @@ MessageID AssetClient::getAssetInfo(const QString& hash, GetInfoCallback callbac } } - callback(false, AssetServerError::NoError, { "", 0 }); + callback(false, AssetUtils::AssetServerError::NoError, { "", 0 }); return INVALID_MESSAGE_ID; } @@ -321,12 +319,12 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer messag message->readPrimitive(&messageID); auto assetHash = message->read(AssetUtils::SHA256_HASH_LENGTH); - AssetServerError error; + AssetUtils::AssetServerError error; message->readPrimitive(&error); AssetInfo info { assetHash.toHex(), 0 }; - if (error == AssetServerError::NoError) { + if (error == AssetUtils::AssetServerError::NoError) { message->readPrimitive(&info.size); } @@ -359,10 +357,10 @@ void AssetClient::handleAssetGetReply(QSharedPointer message, S MessageID messageID; message->readHeadPrimitive(&messageID); - AssetServerError error; + AssetUtils::AssetServerError error; message->readHeadPrimitive(&error); - DataOffset length = 0; + AssetUtils::DataOffset length = 0; if (!error) { message->readHeadPrimitive(&length); } else { @@ -415,7 +413,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer message, S } void AssetClient::handleProgressCallback(const QWeakPointer& node, MessageID messageID, - qint64 size, DataOffset length) { + qint64 size, AssetUtils::DataOffset length) { auto senderNode = node.toStrongRef(); if (!senderNode) { @@ -441,7 +439,7 @@ void AssetClient::handleProgressCallback(const QWeakPointer& node, Message callbacks.progressCallback(size, length); } -void AssetClient::handleCompleteCallback(const QWeakPointer& node, MessageID messageID, DataOffset length) { +void AssetClient::handleCompleteCallback(const QWeakPointer& node, MessageID messageID, AssetUtils::DataOffset length) { auto senderNode = node.toStrongRef(); if (!senderNode) { @@ -475,9 +473,9 @@ void AssetClient::handleCompleteCallback(const QWeakPointer& node, Message } if (message->failed() || length != message->getBytesLeftToRead()) { - callbacks.completeCallback(false, AssetServerError::NoError, QByteArray()); + callbacks.completeCallback(false, AssetUtils::AssetServerError::NoError, QByteArray()); } else { - callbacks.completeCallback(true, AssetServerError::NoError, message->readAll()); + callbacks.completeCallback(true, AssetUtils::AssetServerError::NoError, message->readAll()); } // We should never get to this point without the associated senderNode and messageID @@ -488,7 +486,7 @@ void AssetClient::handleCompleteCallback(const QWeakPointer& node, Message } -MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCallback callback) { +MessageID AssetClient::getAssetMapping(const AssetUtils::AssetPath& path, MappingOperationCallback callback) { Q_ASSERT(QThread::currentThread() == thread()); auto nodeList = DependencyManager::get(); @@ -500,7 +498,7 @@ MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCa auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::Get); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Get); packetList->writeString(path); @@ -511,7 +509,7 @@ MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCa } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } @@ -527,7 +525,7 @@ MessageID AssetClient::getAllAssetMappings(MappingOperationCallback callback) { auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::GetAll); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::GetAll); if (nodeList->sendPacketList(std::move(packetList), *assetServer) != -1) { _pendingMappingRequests[assetServer][messageID] = callback; @@ -536,11 +534,11 @@ MessageID AssetClient::getAllAssetMappings(MappingOperationCallback callback) { } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } -MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOperationCallback callback) { +MessageID AssetClient::deleteAssetMappings(const AssetUtils::AssetPathList& paths, MappingOperationCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -550,7 +548,7 @@ MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOp auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::Delete); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Delete); packetList->writePrimitive(int(paths.size())); @@ -565,11 +563,11 @@ MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOp } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } -MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback) { +MessageID AssetClient::setAssetMapping(const QString& path, const AssetUtils::AssetHash& hash, MappingOperationCallback callback) { Q_ASSERT(QThread::currentThread() == thread()); auto nodeList = DependencyManager::get(); @@ -581,7 +579,7 @@ MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& has auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::Set); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Set); packetList->writeString(path); packetList->write(QByteArray::fromHex(hash.toUtf8())); @@ -593,11 +591,11 @@ MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& has } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } -MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetPath& newPath, MappingOperationCallback callback) { +MessageID AssetClient::renameAssetMapping(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath, MappingOperationCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -607,7 +605,7 @@ MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetP auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::Rename); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Rename); packetList->writeString(oldPath); packetList->writeString(newPath); @@ -620,11 +618,11 @@ MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetP } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } -MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback) { +MessageID AssetClient::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled, MappingOperationCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -634,7 +632,7 @@ MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled auto messageID = ++_currentID; packetList->writePrimitive(messageID); - packetList->writePrimitive(AssetMappingOperationType::SetBakingEnabled); + packetList->writePrimitive(AssetUtils::AssetMappingOperationType::SetBakingEnabled); packetList->writePrimitive(enabled); @@ -651,7 +649,7 @@ MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled } } - callback(false, AssetServerError::NoError, QSharedPointer()); + callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); return INVALID_MESSAGE_ID; } @@ -735,7 +733,7 @@ MessageID AssetClient::uploadAsset(const QByteArray& data, UploadResultCallback } } - callback(false, AssetServerError::NoError, QString()); + callback(false, AssetUtils::AssetServerError::NoError, QString()); return INVALID_MESSAGE_ID; } @@ -745,7 +743,7 @@ void AssetClient::handleAssetUploadReply(QSharedPointer message MessageID messageID; message->readPrimitive(&messageID); - AssetServerError error; + AssetUtils::AssetServerError error; message->readPrimitive(&error); QString hashString; @@ -792,7 +790,7 @@ void AssetClient::handleNodeKilled(SharedNodePointer node) { auto messageMapIt = _pendingUploads.find(node); if (messageMapIt != _pendingUploads.end()) { for (const auto& value : messageMapIt->second) { - value.second(false, AssetServerError::NoError, ""); + value.second(false, AssetUtils::AssetServerError::NoError, ""); } messageMapIt->second.clear(); } @@ -824,7 +822,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) { disconnect(message.data(), nullptr, this, nullptr); } - value.second.completeCallback(false, AssetServerError::NoError, QByteArray()); + value.second.completeCallback(false, AssetUtils::AssetServerError::NoError, QByteArray()); } messageMapIt->second.clear(); } @@ -835,7 +833,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) { if (messageMapIt != _pendingInfoRequests.end()) { AssetInfo info { "", 0 }; for (const auto& value : messageMapIt->second) { - value.second(false, AssetServerError::NoError, info); + value.second(false, AssetUtils::AssetServerError::NoError, info); } messageMapIt->second.clear(); } @@ -845,7 +843,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) { auto messageMapIt = _pendingMappingRequests.find(node); if (messageMapIt != _pendingMappingRequests.end()) { for (const auto& value : messageMapIt->second) { - value.second(false, AssetServerError::NoError, QSharedPointer()); + value.second(false, AssetUtils::AssetServerError::NoError, QSharedPointer()); } messageMapIt->second.clear(); } diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 6f55866740..c10ecf78a3 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -42,12 +42,10 @@ struct AssetInfo { int64_t size; }; -using namespace AssetUtils; - -using MappingOperationCallback = std::function message)>; -using ReceivedAssetCallback = std::function; -using GetInfoCallback = std::function; -using UploadResultCallback = std::function; +using MappingOperationCallback = std::function message)>; +using ReceivedAssetCallback = std::function; +using GetInfoCallback = std::function; +using UploadResultCallback = std::function; using ProgressCallback = std::function; class AssetClient : public QObject, public Dependency { @@ -55,13 +53,13 @@ class AssetClient : public QObject, public Dependency { public: AssetClient(); - Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetPath& path); + Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetUtils::AssetPath& path); Q_INVOKABLE GetAllMappingsRequest* createGetAllMappingsRequest(); - Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetPathList& paths); - Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash); - Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath); - Q_INVOKABLE SetBakingEnabledRequest* createSetBakingEnabledRequest(const AssetPathList& path, bool enabled); - Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const ByteRange& byteRange = ByteRange()); + Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetUtils::AssetPathList& paths); + Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash); + Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath); + Q_INVOKABLE SetBakingEnabledRequest* createSetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled); + Q_INVOKABLE AssetRequest* createRequest(const AssetUtils::AssetHash& hash, const ByteRange& byteRange = ByteRange()); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data); @@ -82,15 +80,15 @@ private slots: void handleNodeClientConnectionReset(SharedNodePointer node); private: - MessageID getAssetMapping(const AssetHash& hash, MappingOperationCallback callback); + MessageID getAssetMapping(const AssetUtils::AssetHash& hash, MappingOperationCallback callback); MessageID getAllAssetMappings(MappingOperationCallback callback); - MessageID setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback); - MessageID deleteAssetMappings(const AssetPathList& paths, MappingOperationCallback callback); - MessageID renameAssetMapping(const AssetPath& oldPath, const AssetPath& newPath, MappingOperationCallback callback); - MessageID setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback); + MessageID setAssetMapping(const QString& path, const AssetUtils::AssetHash& hash, MappingOperationCallback callback); + MessageID deleteAssetMappings(const AssetUtils::AssetPathList& paths, MappingOperationCallback callback); + MessageID renameAssetMapping(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath, MappingOperationCallback callback); + MessageID setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled, MappingOperationCallback callback); MessageID getAssetInfo(const QString& hash, GetInfoCallback callback); - MessageID getAsset(const QString& hash, DataOffset start, DataOffset end, + MessageID getAsset(const QString& hash, AssetUtils::DataOffset start, AssetUtils::DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback); MessageID uploadAsset(const QByteArray& data, UploadResultCallback callback); @@ -99,8 +97,8 @@ private: bool cancelGetAssetRequest(MessageID id); bool cancelUploadAssetRequest(MessageID id); - void handleProgressCallback(const QWeakPointer& node, MessageID messageID, qint64 size, DataOffset length); - void handleCompleteCallback(const QWeakPointer& node, MessageID messageID, DataOffset length); + void handleProgressCallback(const QWeakPointer& node, MessageID messageID, qint64 size, AssetUtils::DataOffset length); + void handleCompleteCallback(const QWeakPointer& node, MessageID messageID, AssetUtils::DataOffset length); void forceFailureOfPendingRequests(SharedNodePointer node); diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index a036a28cfd..c42e9aff14 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -80,7 +80,7 @@ void AssetRequest::start() { auto hash = _hash; _assetRequestID = assetClient->getAsset(_hash, _byteRange.fromInclusive, _byteRange.toExclusive, - [this, that, hash](bool responseReceived, AssetServerError serverError, const QByteArray& data) { + [this, that, hash](bool responseReceived, AssetUtils::AssetServerError serverError, const QByteArray& data) { if (!that) { qCWarning(asset_client) << "Got reply for dead asset request " << hash << "- error code" << _error; @@ -91,12 +91,12 @@ void AssetRequest::start() { if (!responseReceived) { _error = NetworkError; - } else if (serverError != AssetServerError::NoError) { + } else if (serverError != AssetUtils::AssetServerError::NoError) { switch (serverError) { - case AssetServerError::AssetNotFound: + case AssetUtils::AssetServerError::AssetNotFound: _error = NotFound; break; - case AssetServerError::InvalidByteRange: + case AssetUtils::AssetServerError::InvalidByteRange: _error = InvalidByteRange; break; default: diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index e1a155a561..9523a9aa26 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -68,7 +68,7 @@ void AssetResourceRequest::doSend() { } } -void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { +void AssetResourceRequest::requestMappingForPath(const AssetUtils::AssetPath& path) { auto statTracker = DependencyManager::get(); statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_STARTED); @@ -140,7 +140,7 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) { _assetMappingRequest->start(); } -void AssetResourceRequest::requestHash(const AssetHash& hash) { +void AssetResourceRequest::requestHash(const AssetUtils::AssetHash& hash) { // Make request to atp auto assetClient = DependencyManager::get(); _assetRequest = assetClient->createRequest(hash, _byteRange); diff --git a/libraries/networking/src/AssetResourceRequest.h b/libraries/networking/src/AssetResourceRequest.h index ac36c83985..2fe79040ca 100644 --- a/libraries/networking/src/AssetResourceRequest.h +++ b/libraries/networking/src/AssetResourceRequest.h @@ -34,8 +34,8 @@ private slots: private: static bool urlIsAssetHash(const QUrl& url); - void requestMappingForPath(const AssetPath& path); - void requestHash(const AssetHash& hash); + void requestMappingForPath(const AssetUtils::AssetPath& path); + void requestHash(const AssetUtils::AssetHash& hash); GetMappingRequest* _assetMappingRequest { nullptr }; AssetRequest* _assetRequest { nullptr }; diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index fabdd1febc..f1c84e474a 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -81,21 +81,21 @@ void AssetUpload::start() { qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server."; } - assetClient->uploadAsset(_data, [this](bool responseReceived, AssetServerError error, const QString& hash){ + assetClient->uploadAsset(_data, [this](bool responseReceived, AssetUtils::AssetServerError error, const QString& hash){ if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::AssetTooLarge: + case AssetUtils::AssetServerError::AssetTooLarge: _error = TooLarge; break; - case AssetServerError::PermissionDenied: + case AssetUtils::AssetServerError::PermissionDenied: _error = PermissionDenied; break; - case AssetServerError::FileOperationFailed: + case AssetUtils::AssetServerError::FileOperationFailed: _error = ServerFileError; break; default: diff --git a/libraries/networking/src/MappingRequest.cpp b/libraries/networking/src/MappingRequest.cpp index a79105e3ab..07639d4994 100644 --- a/libraries/networking/src/MappingRequest.cpp +++ b/libraries/networking/src/MappingRequest.cpp @@ -51,13 +51,13 @@ QString MappingRequest::getErrorString() const { } } -GetMappingRequest::GetMappingRequest(const AssetPath& path) : _path(path.trimmed()) { +GetMappingRequest::GetMappingRequest(const AssetUtils::AssetPath& path) : _path(path.trimmed()) { }; void GetMappingRequest::doStart() { // short circuit the request if the path is invalid - if (!isValidFilePath(_path)) { + if (!AssetUtils::isValidFilePath(_path)) { _error = MappingRequest::InvalidPath; emit finished(this); return; @@ -66,17 +66,17 @@ void GetMappingRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->getAssetMapping(_path, - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::AssetNotFound: + case AssetUtils::AssetServerError::AssetNotFound: _error = NotFound; break; default: @@ -86,7 +86,7 @@ void GetMappingRequest::doStart() { } if (!_error) { - _hash = message->read(SHA256_HASH_LENGTH).toHex(); + _hash = message->read(AssetUtils::SHA256_HASH_LENGTH).toHex(); // check the boolean to see if this request got re-directed quint8 wasRedirected; @@ -112,7 +112,7 @@ GetAllMappingsRequest::GetAllMappingsRequest() { void GetAllMappingsRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->getAllAssetMappings( - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; @@ -120,7 +120,7 @@ void GetAllMappingsRequest::doStart() { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; default: @@ -135,11 +135,11 @@ void GetAllMappingsRequest::doStart() { message->readPrimitive(&numberOfMappings); for (uint32_t i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); - auto hash = message->read(SHA256_HASH_LENGTH).toHex(); - BakingStatus status; + auto hash = message->read(AssetUtils::SHA256_HASH_LENGTH).toHex(); + AssetUtils::BakingStatus status; QString lastBakeErrors; message->readPrimitive(&status); - if (status == BakingStatus::Error) { + if (status == AssetUtils::BakingStatus::Error) { lastBakeErrors = message->readString(); } _mappings[path] = { hash, status, lastBakeErrors }; @@ -149,7 +149,7 @@ void GetAllMappingsRequest::doStart() { }); }; -SetMappingRequest::SetMappingRequest(const AssetPath& path, const AssetHash& hash) : +SetMappingRequest::SetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) : _path(path.trimmed()), _hash(hash) { @@ -159,8 +159,8 @@ SetMappingRequest::SetMappingRequest(const AssetPath& path, const AssetHash& has void SetMappingRequest::doStart() { // short circuit the request if the hash or path are invalid - auto validPath = isValidFilePath(_path); - auto validHash = isValidHash(_hash); + auto validPath = AssetUtils::isValidFilePath(_path); + auto validHash = AssetUtils::isValidHash(_hash); if (!validPath || !validHash) { _error = !validPath ? MappingRequest::InvalidPath : MappingRequest::InvalidHash; emit finished(this); @@ -170,17 +170,17 @@ void SetMappingRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->setAssetMapping(_path, _hash, - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::PermissionDenied: + case AssetUtils::AssetServerError::PermissionDenied: _error = PermissionDenied; break; default: @@ -193,7 +193,7 @@ void SetMappingRequest::doStart() { }); }; -DeleteMappingsRequest::DeleteMappingsRequest(const AssetPathList& paths) : _paths(paths) { +DeleteMappingsRequest::DeleteMappingsRequest(const AssetUtils::AssetPathList& paths) : _paths(paths) { for (auto& path : _paths) { path = path.trimmed(); } @@ -203,7 +203,7 @@ void DeleteMappingsRequest::doStart() { // short circuit the request if any of the paths are invalid for (auto& path : _paths) { - if (!isValidPath(path)) { + if (!AssetUtils::isValidPath(path)) { _error = MappingRequest::InvalidPath; emit finished(this); return; @@ -213,17 +213,17 @@ void DeleteMappingsRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->deleteAssetMappings(_paths, - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::PermissionDenied: + case AssetUtils::AssetServerError::PermissionDenied: _error = PermissionDenied; break; default: @@ -236,7 +236,7 @@ void DeleteMappingsRequest::doStart() { }); }; -RenameMappingRequest::RenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath) : +RenameMappingRequest::RenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath) : _oldPath(oldPath.trimmed()), _newPath(newPath.trimmed()) { @@ -246,7 +246,7 @@ RenameMappingRequest::RenameMappingRequest(const AssetPath& oldPath, const Asset void RenameMappingRequest::doStart() { // short circuit the request if either of the paths are invalid - if (!isValidFilePath(_oldPath) || !isValidFilePath(_newPath)) { + if (!AssetUtils::isValidFilePath(_oldPath) || !AssetUtils::isValidFilePath(_newPath)) { _error = InvalidPath; emit finished(this); return; @@ -255,17 +255,17 @@ void RenameMappingRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->renameAssetMapping(_oldPath, _newPath, - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::PermissionDenied: + case AssetUtils::AssetServerError::PermissionDenied: _error = PermissionDenied; break; default: @@ -278,7 +278,7 @@ void RenameMappingRequest::doStart() { }); } -SetBakingEnabledRequest::SetBakingEnabledRequest(const AssetPathList& paths, bool enabled) : _paths(paths), _enabled(enabled) { +SetBakingEnabledRequest::SetBakingEnabledRequest(const AssetUtils::AssetPathList& paths, bool enabled) : _paths(paths), _enabled(enabled) { for (auto& path : _paths) { path = path.trimmed(); } @@ -288,7 +288,7 @@ void SetBakingEnabledRequest::doStart() { // short circuit the request if any of the paths are invalid for (auto& path : _paths) { - if (!isValidPath(path)) { + if (!AssetUtils::isValidPath(path)) { _error = MappingRequest::InvalidPath; emit finished(this); return; @@ -298,17 +298,17 @@ void SetBakingEnabledRequest::doStart() { auto assetClient = DependencyManager::get(); _mappingRequestID = assetClient->setBakingEnabled(_paths, _enabled, - [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { + [this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer message) { _mappingRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else { switch (error) { - case AssetServerError::NoError: + case AssetUtils::AssetServerError::NoError: _error = NoError; break; - case AssetServerError::PermissionDenied: + case AssetUtils::AssetServerError::PermissionDenied: _error = PermissionDenied; break; default: @@ -319,4 +319,4 @@ void SetBakingEnabledRequest::doStart() { emit finished(this); }); -}; \ No newline at end of file +}; diff --git a/libraries/networking/src/MappingRequest.h b/libraries/networking/src/MappingRequest.h index fc43375469..5286849dd1 100644 --- a/libraries/networking/src/MappingRequest.h +++ b/libraries/networking/src/MappingRequest.h @@ -50,10 +50,10 @@ private: class GetMappingRequest : public MappingRequest { Q_OBJECT public: - GetMappingRequest(const AssetPath& path); + GetMappingRequest(const AssetUtils::AssetPath& path); - AssetHash getHash() const { return _hash; } - AssetPath getRedirectedPath() const { return _redirectedPath; } + AssetUtils::AssetHash getHash() const { return _hash; } + AssetUtils::AssetPath getRedirectedPath() const { return _redirectedPath; } bool wasRedirected() const { return _wasRedirected; } signals: @@ -62,21 +62,21 @@ signals: private: virtual void doStart() override; - AssetPath _path; - AssetHash _hash; + AssetUtils::AssetPath _path; + AssetUtils::AssetHash _hash; - AssetPath _redirectedPath; + AssetUtils::AssetPath _redirectedPath; bool _wasRedirected { false }; }; class SetMappingRequest : public MappingRequest { Q_OBJECT public: - SetMappingRequest(const AssetPath& path, const AssetHash& hash); + SetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash); - AssetPath getPath() const { return _path; } - AssetHash getHash() const { return _hash; } + AssetUtils::AssetPath getPath() const { return _path; } + AssetUtils::AssetHash getHash() const { return _hash; } signals: void finished(SetMappingRequest* thisRequest); @@ -84,14 +84,14 @@ signals: private: virtual void doStart() override; - AssetPath _path; - AssetHash _hash; + AssetUtils::AssetPath _path; + AssetUtils::AssetHash _hash; }; class DeleteMappingsRequest : public MappingRequest { Q_OBJECT public: - DeleteMappingsRequest(const AssetPathList& path); + DeleteMappingsRequest(const AssetUtils::AssetPathList& path); signals: void finished(DeleteMappingsRequest* thisRequest); @@ -99,13 +99,13 @@ signals: private: virtual void doStart() override; - AssetPathList _paths; + AssetUtils::AssetPathList _paths; }; class RenameMappingRequest : public MappingRequest { Q_OBJECT public: - RenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath); + RenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath); signals: void finished(RenameMappingRequest* thisRequest); @@ -113,8 +113,8 @@ signals: private: virtual void doStart() override; - AssetPath _oldPath; - AssetPath _newPath; + AssetUtils::AssetPath _oldPath; + AssetUtils::AssetPath _newPath; }; class GetAllMappingsRequest : public MappingRequest { @@ -122,7 +122,7 @@ class GetAllMappingsRequest : public MappingRequest { public: GetAllMappingsRequest(); - AssetMapping getMappings() const { return _mappings; } + AssetUtils::AssetMapping getMappings() const { return _mappings; } signals: void finished(GetAllMappingsRequest* thisRequest); @@ -130,13 +130,13 @@ signals: private: virtual void doStart() override; - AssetMapping _mappings; + AssetUtils::AssetMapping _mappings; }; class SetBakingEnabledRequest : public MappingRequest { Q_OBJECT public: - SetBakingEnabledRequest(const AssetPathList& path, bool enabled); + SetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled); signals: void finished(SetBakingEnabledRequest* thisRequest); @@ -144,7 +144,7 @@ signals: private: virtual void doStart() override; - AssetPathList _paths; + AssetUtils::AssetPathList _paths; bool _enabled; }; From 31cdd5cca248f2b744a83d7d076406a190f46e20 Mon Sep 17 00:00:00 2001 From: humbletim Date: Thu, 18 Jan 2018 22:20:28 -0500 Subject: [PATCH 11/21] QueuedConnection per CR feedback --- libraries/shared/src/shared/MiniPromises.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index 3385118666..2f17760aa8 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -54,7 +54,7 @@ public: Q_INVOKABLE void executeOnPromiseThread(std::function function) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod( - this, "executeOnPromiseThread", Qt::BlockingQueuedConnection, + this, "executeOnPromiseThread", Qt::QueuedConnection, Q_ARG(std::function, function)); } else { function(); From 395cc663ddb9c082f22d1cb6ff7bab45f43b189c Mon Sep 17 00:00:00 2001 From: humbletim Date: Fri, 19 Jan 2018 15:10:36 -0500 Subject: [PATCH 12/21] reuse same disk cache as AssetClient --- libraries/networking/src/AssetClient.cpp | 99 +++++++++++++++++++ libraries/networking/src/AssetClient.h | 3 + .../src/BaseAssetScriptingInterface.cpp | 99 ++++--------------- .../src/BaseAssetScriptingInterface.h | 20 ++-- .../src/AssetScriptingInterface.cpp | 22 ----- 5 files changed, 128 insertions(+), 115 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 6980621219..c126fc2e5a 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -100,6 +100,105 @@ void AssetClient::cacheInfoRequest(MiniPromise::Promise deferred) { } } +void AssetClient::queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); + return; + } + if (auto cache = NetworkAccessManager::getInstance().cache()) { + QNetworkCacheMetaData metaData = cache->metaData(url); + QVariantMap attributes, rawHeaders; + + QHashIterator i(metaData.attributes()); + while (i.hasNext()) { + i.next(); + attributes[QString::number(i.key())] = i.value(); + } + for (const auto& i : metaData.rawHeaders()) { + rawHeaders[i.first] = i.second; + } + deferred->resolve({ + { "isValid", metaData.isValid() }, + { "url", metaData.url() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + { "saveToDisk", metaData.saveToDisk() }, + { "attributes", attributes }, + { "rawHeaders", rawHeaders }, + }); + } else { + deferred->reject("cache currently unavailable"); + } +} + +void AssetClient::loadFromCache(MiniPromise::Promise deferred, const QUrl& url) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadFromCache", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); + return; + } + if (auto cache = NetworkAccessManager::getInstance().cache()) { + MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); + queryCacheMeta(metaRequest, url); + metaRequest->then([&](QString error, QVariantMap metadata) { + if (!error.isEmpty()) { + deferred->reject(error, metadata); + return; + } + QVariantMap result = { + { "metadata", metadata }, + { "data", QByteArray() }, + }; + // caller is responsible for the deletion of the ioDevice, hence the unique_ptr + if (auto ioDevice = std::unique_ptr(cache->data(url))) { + QByteArray data = ioDevice->readAll(); + result["data"] = data; + } else { + error = "cache data unavailable"; + } + deferred->handle(error, result); + }); + } else { + deferred->reject("cache currently unavailable"); + } +} + +namespace { + // parse RFC 1123 HTTP date format + QDateTime parseHttpDate(const QString& dateString) { + QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); + dt.setTimeSpec(Qt::UTC); + return dt; + } +} + +void AssetClient::saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& headers) { + if (auto cache = NetworkAccessManager::getInstance().cache()) { + QDateTime lastModified = headers.contains("last-modified") ? + parseHttpDate(headers["last-modified"].toString()) : + QDateTime::currentDateTimeUtc(); + QDateTime expirationDate = headers.contains("expires") ? + parseHttpDate(headers["expires"].toString()) : + QDateTime(); // never expires + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(lastModified); + metaData.setExpirationDate(expirationDate); + if (auto ioDevice = cache->prepare(metaData)) { + ioDevice->write(data); + cache->insert(ioDevice); + qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; + deferred->resolve({{ "success", true }}); + } else { + auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); + qCWarning(asset_client) << error; + deferred->reject(error); + } + } else { + deferred->reject("cache currently unavailable"); + } +} + void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "cacheInfoRequest", Qt::QueuedConnection, diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index c10ecf78a3..81149bf3d6 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -68,6 +68,9 @@ public slots: void cacheInfoRequest(QObject* reciever, QString slot); void cacheInfoRequest(MiniPromise::Promise deferred); + void queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url); + void loadFromCache(MiniPromise::Promise deferred, const QUrl& url); + void saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); void clearCache(); private slots: diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index 8172f12c79..f6f7fd87e3 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -31,31 +31,23 @@ QSharedPointer BaseAssetScriptingInterface::assetClient() { return DependencyManager::get(); } -BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent), _cache(this) { -} - +BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent) {} bool BaseAssetScriptingInterface::initializeCache() { - // NOTE: *instances* of QNetworkDiskCache are not thread-safe -- however, different threads can effectively - // use the same underlying cache if configured with identical settings. Once AssetClient's disk cache settings - // become available we configure our instance to match. auto assets = assetClient(); if (!assets) { return false; // not yet possible to initialize the cache } - if (_cache.cacheDirectory().size()) { + if (!_cacheDirectory.isEmpty()) { return true; // cache is ready } // attempt to initialize the cache - QMetaObject::invokeMethod(assetClient().data(), "init"); + QMetaObject::invokeMethod(assets.data(), "init"); Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); deferred->then([&](QVariantMap result) { - auto cacheDirectory = result.value("cacheDirectory").toString(); - auto maximumCacheSize = result.value("maximumCacheSize").toLongLong(); - _cache.setCacheDirectory(cacheDirectory); - _cache.setMaximumCacheSize(maximumCacheSize); + _cacheDirectory = result.value("cacheDirectory").toString(); }); deferred->fail([&](QString error) { qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error; @@ -64,79 +56,28 @@ bool BaseAssetScriptingInterface::initializeCache() { return false; // cache is not ready yet } -QVariantMap BaseAssetScriptingInterface::getCacheStatus() { - return { - { "cacheDirectory", _cache.cacheDirectory() }, - { "cacheSize", _cache.cacheSize() }, - { "maximumCacheSize", _cache.maximumCacheSize() }, - }; +Promise BaseAssetScriptingInterface::getCacheStatus() { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->cacheInfoRequest(deferred); + return deferred; } -QVariantMap BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { - QNetworkCacheMetaData metaData = _cache.metaData(url); - QVariantMap attributes, rawHeaders; - - QHashIterator i(metaData.attributes()); - while (i.hasNext()) { - i.next(); - attributes[QString::number(i.key())] = i.value(); - } - for (const auto& i : metaData.rawHeaders()) { - rawHeaders[i.first] = i.second; - } - return { - { "isValid", metaData.isValid() }, - { "url", metaData.url() }, - { "expirationDate", metaData.expirationDate() }, - { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, - { "saveToDisk", metaData.saveToDisk() }, - { "attributes", attributes }, - { "rawHeaders", rawHeaders }, - }; +Promise BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->queryCacheMeta(deferred, url); + return deferred; } -QVariantMap BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { - QVariantMap result = { - { "metadata", queryCacheMeta(url) }, - { "data", QByteArray() }, - }; - // caller is responsible for the deletion of the ioDevice, hence the unique_ptr - if (auto ioDevice = std::unique_ptr(_cache.data(url))) { - QByteArray data = ioDevice->readAll(); - result["data"] = data; - } - return result; +Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->loadFromCache(deferred, url); + return deferred; } -namespace { - // parse RFC 1123 HTTP date format - QDateTime parseHttpDate(const QString& dateString) { - QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); - dt.setTimeSpec(Qt::UTC); - return dt; - } -} - -bool BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - QDateTime lastModified = headers.contains("last-modified") ? - parseHttpDate(headers["last-modified"].toString()) : - QDateTime::currentDateTimeUtc(); - QDateTime expirationDate = headers.contains("expires") ? - parseHttpDate(headers["expires"].toString()) : - QDateTime(); // never expires - QNetworkCacheMetaData metaData; - metaData.setUrl(url); - metaData.setSaveToDisk(true); - metaData.setLastModified(lastModified); - metaData.setExpirationDate(expirationDate); - if (auto ioDevice = _cache.prepare(metaData)) { - ioDevice->write(data); - _cache.insert(ioDevice); - qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; - return true; - } - qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache."; - return false; +Promise BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { + Promise deferred = makePromise(__FUNCTION__); + DependencyManager::get()->saveToCache(deferred, url, data, headers); + return deferred; } Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) { diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h index 3aca3d394e..35c829fd37 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.h +++ b/libraries/networking/src/BaseAssetScriptingInterface.h @@ -33,15 +33,7 @@ public: BaseAssetScriptingInterface(QObject* parent = nullptr); public slots: - /**jsdoc - * Get the current status of the disk cache (if available) - * @function Assets.uploadData - * @static - * @return {String} result.cacheDirectory (path to the current disk cache) - * @return {Number} result.cacheSize (used cache size in bytes) - * @return {Number} result.maximumCacheSize (maxmimum cache size in bytes) - */ - QVariantMap getCacheStatus(); + Promise getCacheStatus(); /**jsdoc * Initialize the disk cache (returns true if already initialized) @@ -56,14 +48,14 @@ public slots: QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); } bool isValidHash(QString input) { return AssetUtils::isValidHash(input); } QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); } + QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); } - virtual QVariantMap queryCacheMeta(const QUrl& url); - virtual QVariantMap loadFromCache(const QUrl& url); - virtual bool saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + virtual Promise queryCacheMeta(const QUrl& url); + virtual Promise loadFromCache(const QUrl& url); + virtual Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); protected: - //void onCacheInfoResponse(QString cacheDirectory, qint64 cacheSize, qint64 maximumCacheSize); - QNetworkDiskCache _cache; + QString _cacheDirectory; const QString NoError{}; //virtual bool jsAssert(bool condition, const QString& error) = 0; Promise loadAsset(QString asset, bool decompress, QString responseType); diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index d80491b2a6..0870460a41 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -75,28 +75,6 @@ void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValu setMappingRequest->start(); } -void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) { - auto request = DependencyManager::get()->createGetMappingRequest(path); - QObject::connect(request, &GetMappingRequest::finished, this, [=](GetMappingRequest* request) mutable { - auto result = request->getError(); - if (callback.isFunction()) { - if (result == GetMappingRequest::NotFound) { - QScriptValueList args { "", true }; - callback.call(_engine->currentContext()->thisObject(), args); - } else if (result == GetMappingRequest::NoError) { - QScriptValueList args { request->getHash(), true }; - callback.call(_engine->currentContext()->thisObject(), args); - } else { - qCDebug(scriptengine) << "error -- " << request->getError() << " -- " << request->getErrorString(); - QScriptValueList args { "", false }; - callback.call(_engine->currentContext()->thisObject(), args); - } - request->deleteLater(); - } - }); - request->start(); -} - void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callback) { // FIXME: historically this API method failed silently when given a non-atp prefixed // urlString (or if the AssetRequest failed). From 3a735c1fc7d12809289fc225cc336bc0c4902060 Mon Sep 17 00:00:00 2001 From: humbletim Date: Tue, 23 Jan 2018 03:12:26 -0500 Subject: [PATCH 13/21] CR feedback and code cleanup --- libraries/networking/src/AssetClient.cpp | 234 ++++++++------ libraries/networking/src/AssetClient.h | 8 +- libraries/networking/src/AssetUtils.cpp | 2 +- .../src/BaseAssetScriptingInterface.cpp | 187 ++++++----- .../src/BaseAssetScriptingInterface.h | 26 +- .../src/ArrayBufferViewClass.cpp | 35 +- .../script-engine/src/ArrayBufferViewClass.h | 4 + .../src/AssetScriptingInterface.cpp | 306 ++++++++++-------- .../src/AssetScriptingInterface.h | 91 +++++- libraries/script-engine/src/ScriptEngine.cpp | 4 +- libraries/script-engine/src/ScriptEngine.h | 2 +- libraries/shared/src/BaseScriptEngine.cpp | 6 + libraries/shared/src/shared/MiniPromises.cpp | 17 + libraries/shared/src/shared/MiniPromises.h | 107 +++--- 14 files changed, 658 insertions(+), 371 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index c126fc2e5a..a0c86a25e8 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -81,122 +81,170 @@ void AssetClient::init() { } -void AssetClient::cacheInfoRequest(MiniPromise::Promise deferred) { - if (QThread::currentThread() != thread()) { - if (!QMetaType::isRegistered(qMetaTypeId())) { - qRegisterMetaType(); - } - QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred)); - return; - } - if (auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache())) { - deferred->resolve({ - { "cacheDirectory", cache->cacheDirectory() }, - { "cacheSize", cache->cacheSize() }, - { "maximumCacheSize", cache->maximumCacheSize() }, - }); - } else { - deferred->reject("Cache not available"); - } +namespace { + const QString& CACHE_ERROR_MESSAGE{ "AssetClient::Error: %1 %2" }; } -void AssetClient::queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url) { +MiniPromise::Promise AssetClient::cacheInfoRequestAsync(MiniPromise::Promise deferred) { + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread + } if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); - return; - } - if (auto cache = NetworkAccessManager::getInstance().cache()) { - QNetworkCacheMetaData metaData = cache->metaData(url); - QVariantMap attributes, rawHeaders; - - QHashIterator i(metaData.attributes()); - while (i.hasNext()) { - i.next(); - attributes[QString::number(i.key())] = i.value(); - } - for (const auto& i : metaData.rawHeaders()) { - rawHeaders[i.first] = i.second; - } - deferred->resolve({ - { "isValid", metaData.isValid() }, - { "url", metaData.url() }, - { "expirationDate", metaData.expirationDate() }, - { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, - { "saveToDisk", metaData.saveToDisk() }, - { "attributes", attributes }, - { "rawHeaders", rawHeaders }, - }); + QMetaObject::invokeMethod(this, "cacheInfoRequestAsync", Q_ARG(MiniPromise::Promise, deferred)); } else { - deferred->reject("cache currently unavailable"); + auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); + if (cache) { + deferred->resolve({ + { "cacheDirectory", cache->cacheDirectory() }, + { "cacheSize", cache->cacheSize() }, + { "maximumCacheSize", cache->maximumCacheSize() }, + }); + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable")); + } } + return deferred; } -void AssetClient::loadFromCache(MiniPromise::Promise deferred, const QUrl& url) { +MiniPromise::Promise AssetClient::queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred) { if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "loadFromCache", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); - return; - } - if (auto cache = NetworkAccessManager::getInstance().cache()) { - MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); - queryCacheMeta(metaRequest, url); - metaRequest->then([&](QString error, QVariantMap metadata) { - if (!error.isEmpty()) { - deferred->reject(error, metadata); - return; - } - QVariantMap result = { - { "metadata", metadata }, - { "data", QByteArray() }, - }; - // caller is responsible for the deletion of the ioDevice, hence the unique_ptr - if (auto ioDevice = std::unique_ptr(cache->data(url))) { - QByteArray data = ioDevice->readAll(); - result["data"] = data; + QMetaObject::invokeMethod(this, "queryCacheMetaAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + QNetworkCacheMetaData metaData = cache->metaData(url); + QVariantMap attributes, rawHeaders; + if (!metaData.isValid()) { + deferred->reject("invalid cache entry", { + { "_url", url }, + { "isValid", metaData.isValid() }, + { "metaDataURL", metaData.url() }, + }); } else { - error = "cache data unavailable"; + QHashIterator i(metaData.attributes()); + while (i.hasNext()) { + i.next(); + attributes[QString::number(i.key())] = i.value(); + } + for (const auto& i : metaData.rawHeaders()) { + rawHeaders[i.first] = i.second; + } + deferred->resolve({ + { "_url", url }, + { "isValid", metaData.isValid() }, + { "url", metaData.url() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + { "saveToDisk", metaData.saveToDisk() }, + { "attributes", attributes }, + { "rawHeaders", rawHeaders }, + }); } - deferred->handle(error, result); - }); - } else { - deferred->reject("cache currently unavailable"); + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable")); + } } + return deferred; +} + +MiniPromise::Promise AssetClient::loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred) { + auto errorMessage = CACHE_ERROR_MESSAGE.arg(__FUNCTION__); + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread + } + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadFromCacheAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); + queryCacheMetaAsync(url, metaRequest); + metaRequest->finally([&](QString error, QVariantMap metadata) { + QVariantMap result = { + { "url", url }, + { "metadata", metadata }, + { "data", QByteArray() }, + }; + if (!error.isEmpty()) { + deferred->reject(error, result); + return; + } + // caller is responsible for the deletion of the ioDevice, hence the unique_ptr + auto ioDevice = std::unique_ptr(cache->data(url)); + if (ioDevice) { + result["data"] = ioDevice->readAll(); + } else { + error = errorMessage.arg("error reading data"); + } + deferred->handle(error, result); + }); + } else { + deferred->reject(errorMessage.arg("cache unavailable")); + } + } + return deferred; } namespace { // parse RFC 1123 HTTP date format QDateTime parseHttpDate(const QString& dateString) { QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); + if (!dt.isValid()) { + dt = QDateTime::fromString(dateString, Qt::ISODateWithMs); + } + if (!dt.isValid()) { + qDebug() << __FUNCTION__ << "unrecognized date format:" << dateString; + } dt.setTimeSpec(Qt::UTC); return dt; } + QDateTime getHttpDateValue(const QVariantMap& headers, const QString& keyName, const QDateTime& defaultValue) { + return headers.contains(keyName) ? parseHttpDate(headers[keyName].toString()) : defaultValue; + } } -void AssetClient::saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - if (auto cache = NetworkAccessManager::getInstance().cache()) { - QDateTime lastModified = headers.contains("last-modified") ? - parseHttpDate(headers["last-modified"].toString()) : - QDateTime::currentDateTimeUtc(); - QDateTime expirationDate = headers.contains("expires") ? - parseHttpDate(headers["expires"].toString()) : - QDateTime(); // never expires - QNetworkCacheMetaData metaData; - metaData.setUrl(url); - metaData.setSaveToDisk(true); - metaData.setLastModified(lastModified); - metaData.setExpirationDate(expirationDate); - if (auto ioDevice = cache->prepare(metaData)) { - ioDevice->write(data); - cache->insert(ioDevice); - qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; - deferred->resolve({{ "success", true }}); - } else { - auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); - qCWarning(asset_client) << error; - deferred->reject(error); - } - } else { - deferred->reject("cache currently unavailable"); +MiniPromise::Promise AssetClient::saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& headers, MiniPromise::Promise deferred) { + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread } + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod( + this, "saveToCacheAsync", Qt::QueuedConnection, + Q_ARG(const QUrl&, url), + Q_ARG(const QByteArray&, data), + Q_ARG(const QVariantMap&, headers), + Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(getHttpDateValue(headers, "last-modified", QDateTime::currentDateTimeUtc())); + metaData.setExpirationDate(getHttpDateValue(headers, "expires", QDateTime())); // nil defaultValue == never expires + auto ioDevice = cache->prepare(metaData); + if (ioDevice) { + ioDevice->write(data); + cache->insert(ioDevice); + qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; + deferred->resolve({ + { "url", url }, + { "success", true }, + { "metaDataURL", metaData.url() }, + { "byteLength", data.size() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + }); + } else { + auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); + qCWarning(asset_client) << error; + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg(error)); + } + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("unavailable")); + } + } + return deferred; } void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) { diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 81149bf3d6..3ec96c3dd4 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -67,10 +67,10 @@ public slots: void init(); void cacheInfoRequest(QObject* reciever, QString slot); - void cacheInfoRequest(MiniPromise::Promise deferred); - void queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url); - void loadFromCache(MiniPromise::Promise deferred, const QUrl& url); - void saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + MiniPromise::Promise cacheInfoRequestAsync(MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap(), MiniPromise::Promise deferred = nullptr); void clearCache(); private slots: diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 45150e80c1..117274eab8 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -50,7 +50,7 @@ QUrl getATPUrl(const QString& input) { QUrl::RemoveAuthority | QUrl::RemoveScheme | QUrl::StripTrailingSlash | QUrl::NormalizePathSegments ); - QString baseName = QFileInfo(path).baseName(); + QString baseName = QFileInfo(url.path()).baseName(); if (isValidPath(path) || isValidHash(baseName)) { return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP).arg(path)); } diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index f6f7fd87e3..d62e992822 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -28,14 +28,18 @@ using Promise = MiniPromise::Promise; QSharedPointer BaseAssetScriptingInterface::assetClient() { - return DependencyManager::get(); + auto client = DependencyManager::get(); + Q_ASSERT(client); + if (!client) { + qDebug() << "BaseAssetScriptingInterface::assetClient unavailable"; + } + return client; } BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent) {} bool BaseAssetScriptingInterface::initializeCache() { - auto assets = assetClient(); - if (!assets) { + if (!assetClient()) { return false; // not yet possible to initialize the cache } if (!_cacheDirectory.isEmpty()) { @@ -43,41 +47,64 @@ bool BaseAssetScriptingInterface::initializeCache() { } // attempt to initialize the cache - QMetaObject::invokeMethod(assets.data(), "init"); + QMetaObject::invokeMethod(assetClient().data(), "init"); Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); - deferred->then([&](QVariantMap result) { + deferred->then([this](QVariantMap result) { _cacheDirectory = result.value("cacheDirectory").toString(); }); - deferred->fail([&](QString error) { + deferred->fail([](QString error) { qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error; }); - assets->cacheInfoRequest(deferred); + assetClient()->cacheInfoRequestAsync(deferred); return false; // cache is not ready yet } Promise BaseAssetScriptingInterface::getCacheStatus() { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->cacheInfoRequest(deferred); - return deferred; + return assetClient()->cacheInfoRequestAsync(makePromise(__FUNCTION__)); } Promise BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->queryCacheMeta(deferred, url); - return deferred; + return assetClient()->queryCacheMetaAsync(url, makePromise(__FUNCTION__)); } -Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->loadFromCache(deferred, url); - return deferred; +Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url, bool decompress, const QString& responseType) { + QVariantMap metaData = { + { "_type", "cache" }, + { "url", url }, + { "responseType", responseType }, + }; + + Promise completed = makePromise("loadFromCache::completed"); + Promise fetched = makePromise("loadFromCache::fetched"); + + Promise downloaded = assetClient()->loadFromCacheAsync(url, makePromise("loadFromCache-retrieval")); + downloaded->mixin(metaData); + downloaded->fail(fetched); + + if (decompress) { + downloaded->then([=](QVariantMap result) { + fetched->mixin(result); + Promise decompressed = decompressBytes(result.value("data").toByteArray()); + decompressed->mixin(result); + decompressed->ready(fetched); + }); + } else { + downloaded->then(fetched); + } + + fetched->fail(completed); + fetched->then([=](QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); + }); + + return completed; } Promise BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->saveToCache(deferred, url, data, headers); - return deferred; + return assetClient()->saveToCacheAsync(url, data, headers, makePromise(__FUNCTION__)); } Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) { @@ -92,73 +119,81 @@ Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, Q { "responseType", responseType }, }; - Promise fetched = makePromise("loadAsset::fetched"), - loaded = makePromise("loadAsset::loaded"); + Promise completed = makePromise("loadAsset::completed"); + Promise fetched = makePromise("loadAsset::fetched"); - downloadBytes(hash) - ->mixin(metaData) - ->ready([=](QString error, QVariantMap result) { - Q_ASSERT(thread() == QThread::currentThread()); - fetched->mixin(result); - if (decompress) { - decompressBytes(result.value("data").toByteArray()) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - fetched->handle(error, result); - }); - } else { - fetched->handle(error, result); - } + Promise downloaded = downloadBytes(hash); + downloaded->mixin(metaData); + downloaded->fail(fetched); + + if (decompress) { + downloaded->then([=](QVariantMap result) { + Q_ASSERT(thread() == QThread::currentThread()); + fetched->mixin(result); + Promise decompressed = decompressBytes(result.value("data").toByteArray()); + decompressed->mixin(result); + decompressed->ready(fetched); + }); + } else { + downloaded->then(fetched); + } + + fetched->fail(completed); + fetched->then([=](QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); }); - fetched->ready([=](QString error, QVariantMap result) { - if (responseType == "arraybuffer") { - loaded->resolve(NoError, result); - } else { - convertBytes(result.value("data").toByteArray(), responseType) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - loaded->resolve(NoError, result); - }); - } - }); - - return loaded; + return completed; } Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) { - QVariantMap result; + QVariantMap result = { + { "_contentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, + { "_byteLength", dataByteArray.size() }, + { "_responseType", responseType }, + }; + QString error; Promise conversion = makePromise(__FUNCTION__); - if (dataByteArray.size() == 0) { - result["response"] = QString(); + if (!RESPONSE_TYPES.contains(responseType)) { + error = QString("convertBytes: invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | ")); + } else if (responseType == "arraybuffer") { + // interpret as bytes + result["response"] = dataByteArray; } else if (responseType == "text") { + // interpret as utf-8 text result["response"] = QString::fromUtf8(dataByteArray); } else if (responseType == "json") { + // interpret as JSON QJsonParseError status; auto parsed = QJsonDocument::fromJson(dataByteArray, &status); if (status.error == QJsonParseError::NoError) { - result["response"] = parsed.isArray() ? - QVariant(parsed.array().toVariantList()) : - QVariant(parsed.object().toVariantMap()); + result["response"] = parsed.isArray() ? QVariant(parsed.array().toVariantList()) : QVariant(parsed.object().toVariantMap()); } else { - QVariantMap errorResult = { + result = { { "error", status.error }, { "offset", status.offset }, }; - return conversion->reject("JSON Parse Error: " + status.errorString(), errorResult); + error = "JSON Parse Error: " + status.errorString(); } - } else if (responseType == "arraybuffer") { - result["response"] = dataByteArray; } - return conversion->resolve(NoError, result); + if (result.value("response").canConvert()) { + auto data = result.value("response").toByteArray(); + result["contentType"] = QMimeDatabase().mimeTypeForData(data).name(); + result["byteLength"] = data.size(); + result["responseType"] = responseType; + } + return conversion->handle(error, result); } Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) { QByteArray inflated; + Promise decompressed = makePromise(__FUNCTION__); auto start = usecTimestampNow(); if (gunzip(dataByteArray, inflated)) { auto end = usecTimestampNow(); - return makePromise(__FUNCTION__)->resolve(NoError, { + decompressed->resolve({ { "_compressedByteLength", dataByteArray.size() }, { "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, { "_compressMS", (double)(end - start) / 1000.0 }, @@ -168,16 +203,18 @@ Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteA { "data", inflated }, }); } else { - return makePromise(__FUNCTION__)->reject("gunzip error", {}); + decompressed->reject("gunzip error"); } + return decompressed; } Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) { QByteArray deflated; auto start = usecTimestampNow(); + Promise compressed = makePromise(__FUNCTION__); if (gzip(dataByteArray, deflated, level)) { auto end = usecTimestampNow(); - return makePromise(__FUNCTION__)->resolve(NoError, { + compressed->resolve({ { "_uncompressedByteLength", dataByteArray.size() }, { "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, { "_compressMS", (double)(end - start) / 1000.0 }, @@ -187,13 +224,13 @@ Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArr { "data", deflated }, }); } else { - return makePromise(__FUNCTION__)->reject("gzip error", {}); + compressed->reject("gzip error", {}); } + return compressed; } Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { - auto assetClient = DependencyManager::get(); - QPointer assetRequest = assetClient->createRequest(hash); + QPointer assetRequest = assetClient()->createRequest(hash); Promise deferred = makePromise(__FUNCTION__); QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { @@ -208,7 +245,7 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { { "url", request->getUrl() }, { "hash", request->getHash() }, { "cached", request->loadedFromCache() }, - { "content-type", QMimeDatabase().mimeTypeForData(data).name() }, + { "contentType", QMimeDatabase().mimeTypeForData(data).name() }, { "data", data }, }; } else { @@ -225,8 +262,9 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { Promise deferred = makePromise(__FUNCTION__); - QPointer upload = DependencyManager::get()->createUpload(bytes); + QPointer upload = assetClient()->createUpload(bytes); + const auto byteLength = bytes.size(); QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) { Q_ASSERT(QThread::currentThread() == upload->thread()); // note: we are now on the "Resource Manager" thread @@ -237,6 +275,7 @@ Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { { "hash", hash }, { "url", AssetUtils::getATPUrl(hash).toString() }, { "filename", upload->getFilename() }, + { "byteLength", byteLength }, }; } else { error = upload->getErrorString(); @@ -251,20 +290,19 @@ Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { } Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { - auto deferred = makePromise(__FUNCTION__); + Promise deferred = makePromise(__FUNCTION__); auto url = AssetUtils::getATPUrl(asset); auto path = url.path(); auto hash = AssetUtils::extractAssetHash(asset); if (AssetUtils::isValidHash(hash)) { // already a valid ATP hash -- nothing to do - deferred->resolve(NoError, { + deferred->resolve({ { "hash", hash }, { "path", path }, { "url", url }, }); } else if (AssetUtils::isValidFilePath(path)) { - auto assetClient = DependencyManager::get(); - QPointer request = assetClient->createGetMappingRequest(path); + QPointer request = assetClient()->createGetMappingRequest(path); QObject::connect(request, &GetMappingRequest::finished, request, [=]() { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -276,7 +314,9 @@ Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { { "_hash", hash }, { "_path", path }, { "_url", url }, + { "url", url }, { "hash", request->getHash() }, + { "hashURL", AssetUtils::getATPUrl(request->getHash()).toString() }, { "wasRedirected", request->wasRedirected() }, { "path", request->wasRedirected() ? request->getRedirectedPath() : path }, }; @@ -297,8 +337,7 @@ Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) { auto deferred = makePromise(__FUNCTION__); - auto assetClient = DependencyManager::get(); - QPointer setMappingRequest = assetClient->createSetMappingRequest(path, hash); + QPointer setMappingRequest = assetClient()->createSetMappingRequest(path, hash); connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h index 35c829fd37..5ac391a4d7 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.h +++ b/libraries/networking/src/BaseAssetScriptingInterface.h @@ -27,37 +27,29 @@ class BaseAssetScriptingInterface : public QObject { Q_OBJECT public: + const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" }; using Promise = MiniPromise::Promise; QSharedPointer assetClient(); BaseAssetScriptingInterface(QObject* parent = nullptr); public slots: - Promise getCacheStatus(); - - /**jsdoc - * Initialize the disk cache (returns true if already initialized) - * @function Assets.initializeCache - * @static - */ - bool initializeCache(); - - virtual bool isValidPath(QString input) { return AssetUtils::isValidPath(input); } - virtual bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); } + bool isValidPath(QString input) { return AssetUtils::isValidPath(input); } + bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); } QUrl getATPUrl(QString input) { return AssetUtils::getATPUrl(input); } QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); } bool isValidHash(QString input) { return AssetUtils::isValidHash(input); } QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); } QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); } - virtual Promise queryCacheMeta(const QUrl& url); - virtual Promise loadFromCache(const QUrl& url); - virtual Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); - protected: QString _cacheDirectory; - const QString NoError{}; - //virtual bool jsAssert(bool condition, const QString& error) = 0; + bool initializeCache(); + Promise getCacheStatus(); + Promise queryCacheMeta(const QUrl& url); + Promise loadFromCache(const QUrl& url, bool decompress = false, const QString& responseType = "arraybuffer"); + Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + Promise loadAsset(QString asset, bool decompress, QString responseType); Promise getAssetInfo(QString asset); Promise downloadBytes(QString hash); diff --git a/libraries/script-engine/src/ArrayBufferViewClass.cpp b/libraries/script-engine/src/ArrayBufferViewClass.cpp index cf776ed834..84cb32d665 100644 --- a/libraries/script-engine/src/ArrayBufferViewClass.cpp +++ b/libraries/script-engine/src/ArrayBufferViewClass.cpp @@ -11,7 +11,8 @@ #include "ArrayBufferViewClass.h" -Q_DECLARE_METATYPE(QByteArray*) +int qScriptClassPointerMetaTypeId = qRegisterMetaType(); +int qByteArrayMetaTypeId = qRegisterMetaType(); ArrayBufferViewClass::ArrayBufferViewClass(ScriptEngine* scriptEngine) : QObject(scriptEngine), @@ -21,6 +22,7 @@ _scriptEngine(scriptEngine) { _bufferName = engine()->toStringHandle(BUFFER_PROPERTY_NAME.toLatin1()); _byteOffsetName = engine()->toStringHandle(BYTE_OFFSET_PROPERTY_NAME.toLatin1()); _byteLengthName = engine()->toStringHandle(BYTE_LENGTH_PROPERTY_NAME.toLatin1()); + registerMetaTypes(scriptEngine); } QScriptClass::QueryFlags ArrayBufferViewClass::queryProperty(const QScriptValue& object, @@ -50,3 +52,34 @@ QScriptValue::PropertyFlags ArrayBufferViewClass::propertyFlags(const QScriptVal const QScriptString& name, uint id) { return QScriptValue::Undeletable; } + +namespace { + void byteArrayFromScriptValue(const QScriptValue& object, QByteArray& byteArray) { + if (object.isValid()) { + if (object.isObject()) { + if (object.isArray()) { + auto Uint8Array = object.engine()->globalObject().property("Uint8Array"); + auto typedArray = Uint8Array.construct(QScriptValueList{object}); + byteArray = qvariant_cast(typedArray.property("buffer").toVariant()); + } else { + byteArray = qvariant_cast(object.data().toVariant()); + } + } else { + byteArray = object.toString().toUtf8(); + } + } + } + + QScriptValue byteArrayToScriptValue(QScriptEngine *engine, const QByteArray& byteArray) { + QScriptValue data = engine->newVariant(QVariant::fromValue(byteArray)); + QScriptValue constructor = engine->globalObject().property("ArrayBuffer"); + Q_ASSERT(constructor.isValid()); + auto array = qscriptvalue_cast(constructor.data()); + Q_ASSERT(array); + return engine->newObject(array, data); + } +} + +void ArrayBufferViewClass::registerMetaTypes(QScriptEngine* scriptEngine) { + qScriptRegisterMetaType(scriptEngine, byteArrayToScriptValue, byteArrayFromScriptValue); +} diff --git a/libraries/script-engine/src/ArrayBufferViewClass.h b/libraries/script-engine/src/ArrayBufferViewClass.h index 67af4a3fc3..038cc75ffd 100644 --- a/libraries/script-engine/src/ArrayBufferViewClass.h +++ b/libraries/script-engine/src/ArrayBufferViewClass.h @@ -29,6 +29,7 @@ static const QString BYTE_LENGTH_PROPERTY_NAME = "byteLength"; class ArrayBufferViewClass : public QObject, public QScriptClass { Q_OBJECT public: + static void registerMetaTypes(QScriptEngine* scriptEngine); ArrayBufferViewClass(ScriptEngine* scriptEngine); ScriptEngine* getScriptEngine() { return _scriptEngine; } @@ -49,4 +50,7 @@ protected: ScriptEngine* _scriptEngine; }; +Q_DECLARE_METATYPE(QScriptClass*) +Q_DECLARE_METATYPE(QByteArray) + #endif // hifi_ArrayBufferViewClass_h diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 0870460a41..7c9bf2c4eb 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -18,27 +18,33 @@ #include #include #include +#include #include #include #include - -#include -#include "Gzip.h" #include "ScriptEngine.h" #include "ScriptEngineLogging.h" -AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) {} +#include +#include + +using Promise = MiniPromise::Promise; + +AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) { + qCDebug(scriptengine) << "AssetScriptingInterface::AssetScriptingInterface" << parent; + MiniPromise::registerMetaTypes(parent); +} #define JS_VERIFY(cond, error) { if (!this->jsVerify(cond, error)) { return; } } void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); QByteArray dataByteArray = data.toUtf8(); auto upload = DependencyManager::get()->createUpload(dataByteArray); Promise deferred = makePromise(__FUNCTION__); - deferred->ready([this, handler](QString error, QVariantMap result) { + deferred->ready([=](QString error, QVariantMap result) { auto url = result.value("url").toString(); auto hash = result.value("hash").toString(); jsCallback(handler, url, hash); @@ -47,7 +53,7 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { connect(upload, &AssetUpload::finished, upload, [this, deferred](AssetUpload* upload, const QString& hash) { // we are now on the "Resource Manager" thread (and "hash" being a *reference* makes it unsafe to use directly) Q_ASSERT(QThread::currentThread() == upload->thread()); - deferred->resolve(NoError, { + deferred->resolve({ { "url", "atp:" + hash }, { "hash", hash }, }); @@ -57,7 +63,7 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { } void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); auto setMappingRequest = assetClient()->createSetMappingRequest(path, hash); Promise deferred = makePromise(__FUNCTION__); deferred->ready([=](QString error, QVariantMap result) { @@ -86,7 +92,7 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb return; } QString hash = AssetUtils::extractAssetHash(urlString); - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); auto assetClient = DependencyManager::get(); auto assetRequest = assetClient->createRequest(hash); @@ -104,11 +110,11 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb if (request->getError() == AssetRequest::Error::NoError) { QString data = QString::fromUtf8(request->getData()); // forward a thread-safe values back to our thread - deferred->resolve(NoError, { { "data", data } }); + deferred->resolve({ { "data", data } }); } else { // FIXME: propagate error to scripts? (requires changing signature or inverting param order above..) //deferred->resolve(request->getErrorString(), { { "error", requet->getError() } }); - qDebug() << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString(); + qCDebug(scriptengine) << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString(); } request->deleteLater(); @@ -118,13 +124,9 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb } void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); auto setBakingEnabledRequest = DependencyManager::get()->createSetBakingEnabledRequest({ path }, enabled); - Promise deferred = makePromise(__FUNCTION__); - deferred->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + Promise deferred = jsPromiseReady(makePromise(__FUNCTION__), thisObject(), callback); connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, setBakingEnabledRequest, [this, deferred](SetBakingEnabledRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -150,13 +152,11 @@ void AssetScriptingInterface::sendFakedHandshake() { void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { auto path = AssetUtils::getATPUrl(asset).path(); - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); JS_VERIFY(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); JS_VERIFY(callback.isFunction(), "expected second parameter to be a callback function"); - qDebug() << ">>getMapping//getAssetInfo" << path; Promise promise = getAssetInfo(path); - promise->ready([this, handler](QString error, QVariantMap result) { - qDebug() << "//getMapping//getAssetInfo" << error << result.keys(); + promise->ready([=](QString error, QVariantMap result) { jsCallback(handler, error, result.value("hash").toString()); }); } @@ -168,11 +168,31 @@ bool AssetScriptingInterface::jsVerify(bool condition, const QString& error) { if (context()) { context()->throwError(error); } else { - qDebug() << "WARNING -- jsVerify failed outside of a valid JS context: " + error; + qCDebug(scriptengine) << "WARNING -- jsVerify failed outside of a valid JS context: " + error; } return false; } +QScriptValue AssetScriptingInterface::jsBindCallback(QScriptValue scope, QScriptValue callback) { + QScriptValue handler = ::makeScopedHandlerObject(scope, callback); + QScriptValue value = handler.property("callback"); + if (!jsVerify(handler.isObject() && value.isFunction(), + QString("jsBindCallback -- .callback is not a function (%1)").arg(value.toVariant().typeName()))) { + return QScriptValue(); + } + return handler; +} + +Promise AssetScriptingInterface::jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback) { + auto handler = jsBindCallback(scope, callback); + if (!jsVerify(handler.isValid(), "jsPromiseReady -- invalid callback handler")) { + return nullptr; + } + return promise->ready([this, handler](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); +} + void AssetScriptingInterface::jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result) { Q_ASSERT(thread() == QThread::currentThread()); @@ -208,46 +228,36 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, responseType = "text"; } auto asset = AssetUtils::getATPUrl(url).path(); - auto handler = makeScopedHandlerObject(scope, callback); - - JS_VERIFY(handler.property("callback").isFunction(), - QString("Invalid callback function (%1)").arg(handler.property("callback").toVariant().typeName())); JS_VERIFY(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset), QString("Invalid ATP url '%1'").arg(url)); JS_VERIFY(RESPONSE_TYPES.contains(responseType), QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); - Promise resolved = makePromise("resolved"); - Promise loaded = makePromise("loaded"); + Promise fetched = jsPromiseReady(makePromise("fetched"), scope, callback); + Promise mapped = makePromise("mapped"); - loaded->ready([=](QString error, QVariantMap result) { - qDebug() << "//loaded" << error; - jsCallback(handler, error, result); - }); - - resolved->ready([=](QString error, QVariantMap result) { - qDebug() << "//resolved" << result.value("hash"); + mapped->ready([=](QString error, QVariantMap result) { QString hash = result.value("hash").toString(); + QString url = result.value("url").toString(); if (!error.isEmpty() || !AssetUtils::isValidHash(hash)) { - loaded->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); + fetched->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); } else { Promise promise = loadAsset(hash, decompress, responseType); promise->mixin(result); - promise->ready([this, loaded, hash](QString error, QVariantMap result) { - qDebug() << "//getAssetInfo/loadAsset" << error << hash; - loaded->resolve(NoError, result); + promise->ready([=](QString error, QVariantMap loadResult) { + loadResult["url"] = url; // maintain mapped .url in results (vs. atp:hash returned by loadAsset) + fetched->handle(error, loadResult); }); } }); if (AssetUtils::isValidHash(asset)) { - resolved->resolve(NoError, { { "hash", asset } }); - } else { - Promise promise = getAssetInfo(asset); - promise->ready([this, resolved](QString error, QVariantMap result) { - qDebug() << "//getAssetInfo" << error << result.value("hash") << result.value("path"); - resolved->resolve(error, result); + mapped->resolve({ + { "hash", asset }, + { "url", url }, }); + } else { + getAssetInfo(asset)->ready(mapped); } } @@ -256,128 +266,166 @@ void AssetScriptingInterface::resolveAsset(QScriptValue options, QScriptValue sc auto url = (options.isString() ? options : options.property(URL)).toString(); auto asset = AssetUtils::getATPUrl(url).path(); - auto handler = makeScopedHandlerObject(scope, callback); JS_VERIFY(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset), "expected options to be an asset URL or request options containing .url property"); - JS_VERIFY(handler.property("callback").isFunction(), "invalid callback function"); - getAssetInfo(asset)->ready([=](QString error, QVariantMap result) { - qDebug() << "//resolveAsset/getAssetInfo" << error << result.value("hash"); - jsCallback(handler, error, result); - }); + + jsPromiseReady(getAssetInfo(asset), scope, callback); } void AssetScriptingInterface::decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { auto data = options.property("data"); QByteArray dataByteArray = qscriptvalue_cast(data); - auto handler = makeScopedHandlerObject(scope, callback); auto responseType = options.property("responseType").toString().toLower(); if (responseType.isEmpty()) { responseType = "text"; } - Promise promise = decompressBytes(dataByteArray); - promise->ready([=](QString error, QVariantMap result) { - if (responseType == "arraybuffer") { - jsCallback(handler, error, result); - } else { - Promise promise = convertBytes(result.value("data").toByteArray(), responseType); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); - } - }); + Promise completed = jsPromiseReady(makePromise(__FUNCTION__), scope, callback); + Promise decompressed = decompressBytes(dataByteArray); + if (responseType == "arraybuffer") { + decompressed->ready(completed); + } else { + decompressed->ready([=](QString error, QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); + }); + } } namespace { const int32_t DEFAULT_GZIP_COMPRESSION_LEVEL = -1; const int32_t MAX_GZIP_COMPRESSION_LEVEL = 9; } - void AssetScriptingInterface::compressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { - - auto data = options.property("data"); - QByteArray dataByteArray = data.isString() ? - data.toString().toUtf8() : - qscriptvalue_cast(data); - auto handler = makeScopedHandlerObject(scope, callback); - auto level = options.property("level").toInt32(); - if (level < DEFAULT_GZIP_COMPRESSION_LEVEL || level > MAX_GZIP_COMPRESSION_LEVEL) { - level = DEFAULT_GZIP_COMPRESSION_LEVEL; - } - Promise promise = compressBytes(dataByteArray, level); - promise->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + auto data = options.property("data").isValid() ? options.property("data") : options; + QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); + int level = options.property("level").isNumber() ? options.property("level").toInt32() : DEFAULT_GZIP_COMPRESSION_LEVEL; + JS_VERIFY(level >= DEFAULT_GZIP_COMPRESSION_LEVEL || level <= MAX_GZIP_COMPRESSION_LEVEL, QString("invalid .level %1").arg(level)); + jsPromiseReady(compressBytes(dataByteArray, level), scope, callback); } void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { - auto compress = options.property("compress").toBool() || - options.property("compressed").toBool(); - auto handler = makeScopedHandlerObject(scope, callback); - auto data = options.property("data"); + auto compress = options.property("compress").toBool() || options.property("compressed").toBool(); + auto data = options.isObject() ? options.property("data") : options; auto rawPath = options.property("path").toString(); auto path = AssetUtils::getATPUrl(rawPath).path(); - QByteArray dataByteArray = data.isString() ? - data.toString().toUtf8() : - qscriptvalue_cast(data); + QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); JS_VERIFY(path.isEmpty() || AssetUtils::isValidFilePath(path), QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path)); - JS_VERIFY(handler.property("callback").isFunction(), - "invalid callback function"); JS_VERIFY(dataByteArray.size() > 0, - QString("expected non-zero .data (got %1 / #%2 bytes)") - .arg(data.toVariant().typeName()) - .arg(dataByteArray.size())); + QString("expected non-zero .data (got %1 / #%2 bytes)").arg(data.toVariant().typeName()).arg(dataByteArray.size())); // [compressed] => uploaded to server => [mapped to path] Promise prepared = makePromise("putAsset::prepared"); Promise uploaded = makePromise("putAsset::uploaded"); - Promise finished = makePromise("putAsset::finished"); + Promise completed = makePromise("putAsset::completed"); + jsPromiseReady(completed, scope, callback); if (compress) { - qDebug() << "putAsset::compressBytes..."; - Promise promise = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL); - promise->finally([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::compressedBytes" << error << result.keys(); - prepared->handle(error, result); - }); + Promise compress = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL); + compress->ready(prepared); } else { - prepared->resolve(NoError, {{ "data", dataByteArray }}); + prepared->resolve({{ "data", dataByteArray }}); } - prepared->ready([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::prepared" << error << result.value("data").toByteArray().size() << result.keys(); - Promise promise = uploadBytes(result.value("data").toByteArray()); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - qDebug() << "===========//putAsset::prepared/uploadBytes" << error << result.keys(); - uploaded->handle(error, result); + prepared->fail(completed); + prepared->then([=](QVariantMap result) { + Promise upload = uploadBytes(result.value("data").toByteArray()); + upload->mixin(result); + upload->ready(uploaded); + }); + + uploaded->fail(completed); + if (path.isEmpty()) { + uploaded->then(completed); + } else { + uploaded->then([=](QVariantMap result) { + QString hash = result.value("hash").toString(); + if (!AssetUtils::isValidHash(hash)) { + completed->reject("path mapping requested, but did not receive valid hash", result); + } else { + Promise link = symlinkAsset(hash, path); + link->mixin(result); + link->ready(completed); + } }); - }); - - uploaded->ready([=](QString error, QVariantMap result) { - QString hash = result.value("hash").toString(); - qDebug() << "//putAsset::uploaded" << error << hash << result.keys(); - if (path.isEmpty()) { - finished->handle(error, result); - } else if (!AssetUtils::isValidHash(hash)) { - finished->reject("path mapping requested, but did not receive valid hash", result); - } else { - qDebug() << "symlinkAsset" << hash << path << QThread::currentThread(); - Promise promise = symlinkAsset(hash, path); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - finished->handle(error, result); - qDebug() << "//symlinkAsset" << hash << path << result.keys(); - }); - } - }); - - finished->ready([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::finished" << error << result.keys(); - jsCallback(handler, error, result); - }); + } +} + +void AssetScriptingInterface::queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback) { + QString url = options.isString() ? options.toString() : options.property("url").toString(); + JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url)); + jsPromiseReady(Parent::queryCacheMeta(url), scope, callback); +} + +void AssetScriptingInterface::loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback) { + QString url, responseType; + bool decompress = false; + if (options.isString()) { + url = options.toString(); + responseType = "text"; + } else { + url = options.property("url").toString(); + responseType = options.property("responseType").isValid() ? options.property("responseType").toString() : "text"; + decompress = options.property("decompress").toBool() || options.property("compressed").toBool(); + } + JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url)); + JS_VERIFY(RESPONSE_TYPES.contains(responseType), + QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); + + jsPromiseReady(Parent::loadFromCache(url, decompress, responseType), scope, callback); +} + +bool AssetScriptingInterface::canWriteCacheValue(const QUrl& url) { + auto scriptEngine = qobject_cast(engine()); + if (!scriptEngine) { + qCDebug(scriptengine) << __FUNCTION__ << "invalid script engine" << url; + return false; + } + // allow cache writes only from Client, EntityServer and Agent scripts + bool isAllowedContext = ( + scriptEngine->isClientScript() || + scriptEngine->isAgentScript() || + scriptEngine->isEntityServerScript() + ); + if (!isAllowedContext) { + qCDebug(scriptengine) << __FUNCTION__ << "invalid context" << scriptEngine->getContext() << url; + return false; + } + return true; +} + +void AssetScriptingInterface::saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback) { + JS_VERIFY(options.isObject(), QString("expected options object as first parameter not: %1").arg(options.toVariant().typeName())); + + QString url = options.property("url").toString(); + QByteArray data = qscriptvalue_cast(options.property("data")); + QVariantMap headers = qscriptvalue_cast(options.property("headers")); + + saveToCache(url, data, headers, scope, callback); +} + +void AssetScriptingInterface::saveToCache(const QUrl& rawURL, const QByteArray& data, const QVariantMap& metadata, QScriptValue scope, QScriptValue callback) { + QUrl url = rawURL; + if (url.path().isEmpty() && !data.isEmpty()) { + // generate a valid ATP URL from the data -- appending any existing fragment or querystring values + auto atpURL = AssetUtils::getATPUrl(hashDataHex(data)); + atpURL.setQuery(url.query()); + atpURL.setFragment(url.fragment()); + qCDebug(scriptengine) << "autogenerated ATP URL" << url << "=>" << atpURL; + url = atpURL; + } + auto hash = AssetUtils::extractAssetHash(url.toDisplayString()); + + JS_VERIFY(url.isValid(), QString("Invalid URL '%1'").arg(url.toString())); + JS_VERIFY(canWriteCacheValue(url), "Invalid cache write URL: " + url.toString()); + JS_VERIFY(url.scheme() == "atp" || url.scheme() == "cache", "only 'atp' and 'cache' URL schemes supported"); + JS_VERIFY(hash.isEmpty() || hash == hashDataHex(data), QString("invalid checksum hash for atp:HASH style URL (%1 != %2)").arg(hash, hashDataHex(data))); + + qCDebug(scriptengine) << "saveToCache" << url.toDisplayString() << data << hash << metadata; + + jsPromiseReady(Parent::saveToCache(url, data, metadata), scope, callback); } diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 7001c64634..fdce173dfe 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -21,6 +21,7 @@ #include #include #include +#include #include /**jsdoc @@ -29,6 +30,7 @@ class AssetScriptingInterface : public BaseAssetScriptingInterface, QScriptable { Q_OBJECT public: + using Parent = BaseAssetScriptingInterface; AssetScriptingInterface(QObject* parent = nullptr); /**jsdoc @@ -102,26 +104,91 @@ public: Q_INVOKABLE void sendFakedHandshake(); #endif - // Advanced APIs - // getAsset(options, scope[callback(error, result)]) -- fetches an Asset from the Server - // [options.url] an "atp:" style URL, hash, or relative mapped path to fetch - // [options.responseType] the desired reponse type (text | arraybuffer | json) - // [options.decompress] whether to apply gunzip decompression on the stream - // [scope[callback]] continuation-style (error, { responseType, data, byteLength, ... }) callback - const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" }; + /**jsdoc + * Request Asset data from the ATP Server + * @function Assets.getAsset + * @param {URL|Assets.GetOptions} options An atp: style URL, hash, or relative mapped path; or an {@link Assets.GetOptions} object with request parameters + * @param {Assets~getAssetCallback} scope[callback] A scope callback function to receive (error, results) values + */ + /**jsdoc + * A set of properties that can be passed to {@link Assets.getAsset}. + * @typedef {Object} Assets.GetOptions + * @property {URL} [url] an "atp:" style URL, hash, or relative mapped path to fetch + * @property {string} [responseType=text] the desired reponse type (text | arraybuffer | json) + * @property {boolean} [decompress=false] whether to attempt gunzip decompression on the fetched data + * See: {@link Assets.putAsset} and its .compress=true option + */ + /**jsdoc + * Called when Assets.getAsset is complete. + * @callback Assets~getAssetCallback + * @param {string} error - contains error message or null value if no error occured fetching the asset + * @param {Asset~getAssetResult} result - result object containing, on success containing asset metadata and contents + */ + /**jsdoc + * Result value returned by {@link Assets.getAsset}. + * @typedef {Object} Assets~getAssetResult + * @property {url} [url] the resolved "atp:" style URL for the fetched asset + * @property {string} [hash] the resolved hash for the fetched asset + * @property {string|ArrayBuffer|Object} [response] response data (possibly converted per .responseType value) + * @property {string} [responseType] response type (text | arraybuffer | json) + * @property {string} [contentType] detected asset mime-type (autodetected) + * @property {number} [byteLength] response data size in bytes + * @property {number} [decompressed] flag indicating whether data was decompressed + */ Q_INVOKABLE void getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); - // putAsset(options, scope[callback(error, result)]) -- upload a new Aset to the Server - // [options.data] -- (ArrayBuffer|String) - // [options.compress] -- (true|false) - // [options.path=undefined] -- option path mapping to set on the created hash result - // [ + + /**jsdoc + * Upload Asset data to the ATP Server + * @function Assets.putAsset + * @param {Assets.PutOptions} options A PutOptions object with upload parameters + * @param {Assets~putAssetCallback} scope[callback] A scoped callback function invoked with (error, results) + */ + /**jsdoc + * A set of properties that can be passed to {@link Assets.putAsset}. + * @typedef {Object} Assets.PutOptions + * @property {ArrayBuffer|string} [data] byte buffer or string value representing the new asset's content + * @property {string} [path=null] ATP path mapping to automatically create (upon successful upload to hash) + * @property {boolean} [compress=false] whether to gzip compress data before uploading + */ + /**jsdoc + * Called when Assets.putAsset is complete. + * @callback Assets~puttAssetCallback + * @param {string} error - contains error message (or null value if no error occured while uploading/mapping the new asset) + * @param {Asset~putAssetResult} result - result object containing error or result status of asset upload + */ + /**jsdoc + * Result value returned by {@link Assets.putAsset}. + * @typedef {Object} Assets~putAssetResult + * @property {url} [url] the resolved "atp:" style URL for the uploaded asset (based on .path if specified, otherwise on the resulting ATP hash) + * @property {string} [path] the uploaded asset's resulting ATP path (or undefined if no path mapping was assigned) + * @property {string} [hash] the uploaded asset's resulting ATP hash + * @property {boolean} [compressed] flag indicating whether the data was compressed before upload + * @property {number} [byteLength] flag indicating final byte size of the data uploaded to the ATP server + */ Q_INVOKABLE void putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void compressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE bool initializeCache() { return Parent::initializeCache(); } + + Q_INVOKABLE bool canWriteCacheValue(const QUrl& url); + + Q_INVOKABLE void getCacheStatus(QScriptValue scope, QScriptValue callback = QScriptValue()) { + jsPromiseReady(Parent::getCacheStatus(), scope, callback); + } + + Q_INVOKABLE void queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata, + QScriptValue scope, QScriptValue callback = QScriptValue()); protected: + QScriptValue jsBindCallback(QScriptValue scope, QScriptValue callback = QScriptValue()); + Promise jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback = QScriptValue()); + void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QVariantMap& result); void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result); bool jsVerify(bool condition, const QString& error); diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index c7034adf35..65ef2025d9 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -56,6 +56,7 @@ #include #include "ArrayBufferViewClass.h" +#include "AssetScriptingInterface.h" #include "BatchLoader.h" #include "BaseScriptEngine.h" #include "DataViewClass.h" @@ -175,6 +176,7 @@ ScriptEngine::ScriptEngine(Context context, const QString& scriptContents, const _timerFunctionMap(), _fileNameString(fileNameString), _arrayBufferClass(new ArrayBufferClass(this)), + _assetScriptingInterface(new AssetScriptingInterface(this)), // don't delete `ScriptEngines` until all `ScriptEngine`s are gone _scriptEngines(DependencyManager::get()) { @@ -704,7 +706,7 @@ void ScriptEngine::init() { // constants globalObject().setProperty("TREE_SCALE", newVariant(QVariant(TREE_SCALE))); - registerGlobalObject("Assets", &_assetScriptingInterface); + registerGlobalObject("Assets", _assetScriptingInterface); registerGlobalObject("Resources", DependencyManager::get().data()); registerGlobalObject("DebugDraw", &DebugDraw::getInstance()); diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index 17c0e0713a..7f69eee990 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -321,7 +321,7 @@ protected: ArrayBufferClass* _arrayBufferClass; - AssetScriptingInterface _assetScriptingInterface{ this }; + AssetScriptingInterface* _assetScriptingInterface; std::function _emitScriptUpdates{ []() { return true; } }; diff --git a/libraries/shared/src/BaseScriptEngine.cpp b/libraries/shared/src/BaseScriptEngine.cpp index c92d629b75..22ae01d72f 100644 --- a/libraries/shared/src/BaseScriptEngine.cpp +++ b/libraries/shared/src/BaseScriptEngine.cpp @@ -325,6 +325,12 @@ QScriptValue makeScopedHandlerObject(QScriptValue scopeOrCallback, QScriptValue } else if (methodOrName.isFunction()) { scope = scopeOrCallback; callback = methodOrName; + } else if (!methodOrName.isValid()) { + // instantiate from an existing scoped handler object + if (scopeOrCallback.property("callback").isFunction()) { + scope = scopeOrCallback.property("scope"); + callback = scopeOrCallback.property("callback"); + } } } auto handler = engine->newObject(); diff --git a/libraries/shared/src/shared/MiniPromises.cpp b/libraries/shared/src/shared/MiniPromises.cpp index faada3627a..bb78852c29 100644 --- a/libraries/shared/src/shared/MiniPromises.cpp +++ b/libraries/shared/src/shared/MiniPromises.cpp @@ -7,4 +7,21 @@ // #include "MiniPromises.h" +#include +#include + int MiniPromise::metaTypeID = qRegisterMetaType("MiniPromise::Promise"); + +namespace { + void promiseFromScriptValue(const QScriptValue& object, MiniPromise::Promise& promise) { + Q_ASSERT(false); + } + QScriptValue promiseToScriptValue(QScriptEngine *engine, const MiniPromise::Promise& promise) { + return engine->newQObject(promise.get()); + } +} +void MiniPromise::registerMetaTypes(QObject* engine) { + auto scriptEngine = qobject_cast(engine); + qDebug() << "----------------------- MiniPromise::registerMetaTypes ------------" << scriptEngine; + qScriptRegisterMetaType(scriptEngine, promiseToScriptValue, promiseFromScriptValue); +} diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index 2f17760aa8..5983f135b7 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -32,6 +32,9 @@ class MiniPromise : public QObject, public std::enable_shared_from_this, public ReadWriteLockable { Q_OBJECT + Q_PROPERTY(QString state READ getStateString) + Q_PROPERTY(QString error READ getError) + Q_PROPERTY(QVariantMap result READ getResult) public: using HandlerFunction = std::function; using SuccessFunction = std::function; @@ -39,23 +42,25 @@ public: using HandlerFunctions = QVector; using Promise = std::shared_ptr; + static void registerMetaTypes(QObject* engine); static int metaTypeID; MiniPromise() {} MiniPromise(const QString debugName) { setObjectName(debugName); } ~MiniPromise() { - if (!_rejected && !_resolved) { - qWarning() << "MiniPromise::~MiniPromise -- destroying unhandled promise:" << objectName() << _error << _result; + if (getStateString() == "pending") { + qWarning() << "MiniPromise::~MiniPromise -- destroying pending promise:" << objectName() << _error << _result << "handlers:" << getPendingHandlerCount(); } } Promise self() { return shared_from_this(); } - Q_INVOKABLE void executeOnPromiseThread(std::function function) { + Q_INVOKABLE void executeOnPromiseThread(std::function function, MiniPromise::Promise root = nullptr) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod( this, "executeOnPromiseThread", Qt::QueuedConnection, - Q_ARG(std::function, function)); + Q_ARG(std::function, function), + Q_ARG(MiniPromise::Promise, self())); } else { function(); } @@ -92,9 +97,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - always(_error, _result); - }); + always(getError(), getResult()); }); } return self(); @@ -112,9 +115,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - failFunc(_error, _result); - }); + failFunc(getError(), getResult()); }); } return self(); @@ -132,9 +133,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - successFunc(_error, _result); - }); + successFunc(getError(), getResult()); }); } return self(); @@ -151,6 +150,26 @@ public: return self(); } + + // helper functions for forwarding results on to a next Promise + Promise ready(Promise next) { return finally(next); } + Promise finally(Promise next) { + return finally([next](QString error, QVariantMap result) { + next->handle(error, result); + }); + } + Promise fail(Promise next) { + return fail([next](QString error, QVariantMap result) { + next->reject(error, result); + }); + } + Promise then(Promise next) { + return then([next](QString error, QVariantMap result) { + next->resolve(error, result); + }); + } + + // trigger methods // handle() automatically resolves or rejects the promise (based on whether an error value occurred) Promise handle(QString error, const QVariantMap& result) { @@ -168,17 +187,15 @@ public: Promise resolve(QString error, const QVariantMap& result) { setState(true, error, result); - QString localError; - QVariantMap localResult; - HandlerFunctions resolveHandlers; - HandlerFunctions finallyHandlers; - withReadLock([&]{ - localError = _error; - localResult = _result; - resolveHandlers = _onresolve; - finallyHandlers = _onfinally; - }); executeOnPromiseThread([&]{ + const QString localError{ getError() }; + const QVariantMap localResult{ getResult() }; + HandlerFunctions resolveHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + resolveHandlers = _onresolve; + finallyHandlers = _onfinally; + }); for (const auto& onresolve : resolveHandlers) { onresolve(localError, localResult); } @@ -195,17 +212,15 @@ public: Promise reject(QString error, const QVariantMap& result) { setState(false, error, result); - QString localError; - QVariantMap localResult; - HandlerFunctions rejectHandlers; - HandlerFunctions finallyHandlers; - withReadLock([&]{ - localError = _error; - localResult = _result; - rejectHandlers = _onreject; - finallyHandlers = _onfinally; - }); executeOnPromiseThread([&]{ + const QString localError{ getError() }; + const QVariantMap localResult{ getResult() }; + HandlerFunctions rejectHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + rejectHandlers = _onreject; + finallyHandlers = _onfinally; + }); for (const auto& onreject : rejectHandlers) { onreject(localError, localResult); } @@ -224,13 +239,25 @@ private: } else { _rejected = true; } - withWriteLock([&]{ - _error = error; - }); + setError(error); assignResult(result); return self(); } + void setError(const QString error) { withWriteLock([&]{ _error = error; }); } + QString getError() const { return resultWithReadLock([this]() -> QString { return _error; }); } + QVariantMap getResult() const { return resultWithReadLock([this]() -> QVariantMap { return _result; }); } + int getPendingHandlerCount() const { + return resultWithReadLock([this]() -> int { + return _onresolve.size() + _onreject.size() + _onfinally.size(); + }); + } + QString getStateString() const { + return _rejected ? "rejected" : + _resolved ? "resolved" : + getPendingHandlerCount() ? "pending" : + "unknown"; + } QString _error; QVariantMap _result; std::atomic _rejected{false}; @@ -240,8 +267,12 @@ private: HandlerFunctions _onfinally; }; +Q_DECLARE_METATYPE(MiniPromise::Promise) + inline MiniPromise::Promise makePromise(const QString& hint = QString()) { + if (!QMetaType::isRegistered(qMetaTypeId())) { + int type = qRegisterMetaType(); + qDebug() << "makePromise -- registered MetaType:" << type; + } return std::make_shared(hint); } - -Q_DECLARE_METATYPE(MiniPromise::Promise) From 51d8b9aea688f936591dce79f40d9a209422bb7f Mon Sep 17 00:00:00 2001 From: humbletim Date: Tue, 23 Jan 2018 03:12:50 -0500 Subject: [PATCH 14/21] update and expand on js unit tests --- .../tests/unit_tests/assetUnitTests.js | 383 ++++++++++++++++-- 1 file changed, 352 insertions(+), 31 deletions(-) diff --git a/scripts/developer/tests/unit_tests/assetUnitTests.js b/scripts/developer/tests/unit_tests/assetUnitTests.js index be8710e50d..5d43eaf1de 100644 --- a/scripts/developer/tests/unit_tests/assetUnitTests.js +++ b/scripts/developer/tests/unit_tests/assetUnitTests.js @@ -1,34 +1,46 @@ /* eslint-env jasmine */ +if (!Entities.canWriteAssets()) { + Window.alert('!Entities.canWriteAssets -- please goto a domain with asset rights and reload this test script'); + throw new Error('!Entities.canWriteAssets'); +} + instrument_testrunner(true); describe("Assets", function () { var context = { + memo: {}, + cache: {}, definedHash: null, definedPath: null, definedContent: null, }; + var ATP_TIMEOUT_MS = 1000; // ms + var NULL_HASH = new Array(64+1).join('0'); // 64 hex var SAMPLE_FOLDER = '/assetUnitTests'; var SAMPLE_FILE_PATH = SAMPLE_FOLDER + "/test.json"; var SAMPLE_CONTENTS = 'Test Run on ' + JSON.stringify(Date()); + var SAMPLE_JSON = JSON.stringify({ content: SAMPLE_CONTENTS }); + var SAMPLE_FLOATS = [ Math.PI, 1.1, 2.2, 3.3 ]; + var SAMPLE_BUFFER = new Float32Array(SAMPLE_FLOATS).buffer; + var IS_ASSET_HASH_REGEX = /^[a-f0-9]{64}$/i; var IS_ASSET_URL = /^atp:/; - it('Entities.canWriteAssets', function() { expect(Entities.canWriteAssets()).toBe(true); }); - it('Assets.extractAssetHash(input)', function() { + describe('extractAssetHash(input)', function() { // new helper method that provides a catch-all way to get at the sha256 hash // considering the multiple, different ways ATP hash URLs are found across the // system and in content. - + var POSITIVE_TEST_URLS = [ 'atp:HASH', 'atp:HASH.obj', + 'atp:HASH.fbx#cache-buster', 'atp:HASH.fbx?cache-buster', - 'atp:/.baked/HASH/asset.fbx', 'HASH' ]; var NEGATIVE_TEST_URLS = [ @@ -36,25 +48,278 @@ describe("Assets", function () { 'http://domain.tld', '/path/filename.fbx', 'atp:/path/filename.fbx?#HASH', + 'atp:/.baked/HASH/asset.fbx', '' ]; - pending(); + it("POSITIVE_TEST_URLS", function() { + POSITIVE_TEST_URLS.forEach(function(url) { + url = url.replace('HASH', NULL_HASH); + expect([url, Assets.extractAssetHash(url)].join('|')).toEqual([url, NULL_HASH].join('|')); + }); + }); + it("NEGATIVE_TEST_URLS", function() { + NEGATIVE_TEST_URLS.forEach(function(url) { + expect(Assets.extractAssetHash(url.replace('HASH', NULL_HASH))).toEqual(''); + }); + }); }); // new APIs + describe('.putAsset', function() { + it("data", function(done) { + Assets.putAsset(SAMPLE_CONTENTS, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.hash).toMatch(IS_ASSET_HASH_REGEX); + context.memo.stringURL = result.url; + done(); + }); + }); + it('nopath.text', function(done) { + Assets.putAsset({ + data: SAMPLE_CONTENTS, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.hash).toMatch(IS_ASSET_HASH_REGEX); + context.memo.textHash = result.hash; + context.memo.textURL = result.url; + done(); + }); + }); + it('path.text', function(done) { + var samplePath = SAMPLE_FOLDER + '/content.json'; + Assets.putAsset({ + path: samplePath, + data: SAMPLE_JSON, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.path).toEqual(samplePath); + expect(result.hash).toMatch(IS_ASSET_HASH_REGEX); + context.memo.jsonPath = result.path; + done(); + }); + }); + it('path.buffer', function(done) { + var samplePath = SAMPLE_FOLDER + '/content.buffer'; + Assets.putAsset({ + path: samplePath, + data: SAMPLE_BUFFER, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.path).toEqual(samplePath); + expect(result.hash).toMatch(IS_ASSET_HASH_REGEX); + expect(result.byteLength).toEqual(SAMPLE_BUFFER.byteLength); + context.memo.bufferURL = result.url; + context.memo.bufferHash = result.hash; + done(); + }); + }); + it('path.json.gz', function(done) { + var samplePath = SAMPLE_FOLDER + '/content.json.gz'; + Assets.putAsset({ + path: samplePath, + data: SAMPLE_JSON, + compress: true, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.path).toEqual(samplePath); + expect(result.hash).toMatch(IS_ASSET_HASH_REGEX); + context.memo.jsonCompressedPath = result.path; + done(); + }); + }); + }); + // it('.deleteAsset(options, {callback(error, result)})', function() { pending(); }); + it('.resolveAsset', function(done) { + expect(context.memo.bufferURL).toBeDefined(); + Assets.resolveAsset(context.memo.bufferURL, function(error, result) { + expect(error).toBe(null); + expect(result.url).toEqual(context.memo.bufferURL); + expect(result.hash).toEqual(context.memo.bufferHash); + done(); + }); + }); + describe('.getAsset', function() { + it('path/', function(done) { + expect(context.memo.jsonPath).toBeDefined(); + Assets.getAsset(context.memo.jsonPath, function(error, result) { + expect(error).toBe(null); + expect(result.response).toEqual(SAMPLE_JSON); + done(); + }); + }); + it('text', function(done) { + expect(context.memo.textURL).toBeDefined(); + Assets.getAsset({ + url: context.memo.textURL, + responseType: 'text', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response).toEqual(SAMPLE_CONTENTS); + expect(result.url).toEqual(context.memo.textURL); + done(); + }); + }); + it('arraybuffer', function(done) { + expect(context.memo.bufferURL).toBeDefined(); + Assets.getAsset({ + url: context.memo.bufferURL, + responseType: 'arraybuffer', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response.byteLength).toEqual(SAMPLE_BUFFER.byteLength); + var expected = [].slice.call(new Float32Array(SAMPLE_BUFFER)).join(','); + var actual = [].slice.call(new Float32Array(result.response)).join(','); + expect(actual).toEqual(expected); + done(); + }); + }); + it('json', function(done) { + expect(context.memo.jsonPath).toBeDefined(); + Assets.getAsset({ + url: context.memo.jsonPath, + responseType: 'json', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response.content).toEqual(SAMPLE_CONTENTS); + done(); + }); + }); + it('json.gz', function(done) { + expect(context.memo.jsonCompressedPath).toBeDefined(); + Assets.getAsset({ + url: context.memo.jsonCompressedPath, + responseType: 'json', + decompress: true, + }, function(error, result) { + expect(error).toBe(null); + expect(result.decompressed).toBe(true); + expect(result.response.content).toEqual(SAMPLE_CONTENTS); + done(); + }); + }); + }); - it('Assets.getAsset(options, {callback(error, result)})', function() { pending(); }); - it('Assets.putAsset(options, {callback(error, result)})', function() { pending(); }); - it('Assets.deleteAsset(options, {callback(error, result)})', function() { pending(); }); - it('Assets.resolveAsset(options, {callback(error, result)})', function() { pending(); }); + // cache APIs + it('.getCacheStatus', function(done) { + Assets.getCacheStatus(function(error, result) { + expect(error).toBe(null); + expect(result.cacheSize).toBeGreaterThan(0); + expect(result.maximumCacheSize).toBeGreaterThan(0); + done(); + }); + }); + describe('.saveToCache', function() { + it(".data", function(done) { + Assets.saveToCache({ data: SAMPLE_CONTENTS }, function(error, result) { + expect(error).toBe(null); + expect(result.success).toBe(true); + context.cache.textURL = result.url; + done(); + }); + }); + it('.url', function(done) { + var sampleURL = 'atp:' + SAMPLE_FOLDER + '/cache.json'; + Assets.saveToCache({ + url: sampleURL, + data: SAMPLE_JSON, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.url).toEqual(sampleURL); + expect(result.success).toBe(true); + context.cache.jsonURL = result.url; + done(); + }); + }); + it('.buffer', function(done) { + var sampleURL = 'atp:' + SAMPLE_FOLDER + '/cache.buffer'; + Assets.saveToCache({ + url: sampleURL, + data: SAMPLE_BUFFER, + }, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.url).toEqual(sampleURL); + expect(result.success).toBe(true); + expect(result.byteLength).toEqual(SAMPLE_BUFFER.byteLength); + context.cache.bufferURL = result.url; + done(); + }); + }); + }); + it('.queryCacheMeta', function(done) { + expect(context.cache.bufferURL).toBeDefined(); + Assets.queryCacheMeta(context.cache.bufferURL, function(error, result) { + expect(error).toBe(null); + expect(result.url).toMatch(IS_ASSET_URL); + expect(result.url).toEqual(context.cache.bufferURL); + done(); + }); + }); + describe('.loadFromCache', function() { + it('urlString', function(done) { + expect(context.cache.jsonURL).toBeDefined(); + Assets.loadFromCache(context.cache.jsonURL, function(error, result) { + expect(error).toBe(null); + expect(result.response).toEqual(SAMPLE_JSON); + done(); + }); + }); + it('.responseType=text', function(done) { + expect(context.cache.textURL).toBeDefined(); + Assets.loadFromCache({ + url: context.cache.textURL, + responseType: 'text', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response).toEqual(SAMPLE_CONTENTS); + done(); + }); + }); + it('.responseType=arraybuffer', function(done) { + expect(context.cache.bufferURL).toBeDefined(); + Assets.loadFromCache({ + url: context.cache.bufferURL, + responseType: 'arraybuffer', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response.byteLength).toEqual(SAMPLE_BUFFER.byteLength); + var expected = [].slice.call(new Float32Array(SAMPLE_BUFFER)).join(','); + var actual = [].slice.call(new Float32Array(result.response)).join(','); + expect(actual).toEqual(expected); + done(); + }); + }); + it('.responseType=json', function(done) { + expect(context.cache.jsonURL).toBeDefined(); + Assets.loadFromCache({ + url: context.cache.jsonURL, + responseType: 'json', + }, function(error, result) { + expect(error).toBe(null); + expect(result.response.content).toEqual(SAMPLE_CONTENTS); + done(); + }); + }); + }); - - // existing APIs - it('Assets.getATPUrl(input)', function() { pending(); }); - it('Assets.isValidPath(input)', function() { pending(); }); - it('Assets.isValidFilePath(input)', function() { pending(); }); - it('Assets.isValidHash(input)', function() { pending(); }); - it('Assets.setBakingEnabled(path, enabled, {callback(error)})', function() { pending(); }); + // existing newly-mapped APIs + it('.getATPUrl', function() { + expect(Assets.getATPUrl(SAMPLE_FILE_PATH)).toEqual('atp:' + SAMPLE_FILE_PATH); + expect(Assets.getATPUrl(NULL_HASH)).toEqual('atp:' + NULL_HASH); + expect(Assets.getATPUrl("/file.ext?a=b#c=d")).toEqual('atp:/file.ext?a=b#c=d'); + expect(Assets.getATPUrl("atp:xxx")).toEqual(''); + expect(Assets.getATPUrl("xxx")).toEqual(''); + }); + xit('.isValidPath', function(done) { pending(); }); + xit('.isValidFilePath', function() { pending(); }); + xit('.isValidHash', function() { pending(); }); + xit('.setBakingEnabled', function() { pending(); }); it("Assets.uploadData(data, {callback(url, hash)})", function (done) { Assets.uploadData(SAMPLE_CONTENTS, function(url, hash) { @@ -68,7 +333,7 @@ describe("Assets", function () { }); }); - it("Assets.setMapping(path, hash {callback(error)})", function (done) { + it("Assets.setMapping", function (done) { expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX); Assets.setMapping(SAMPLE_FILE_PATH, context.definedHash, function(error) { if (error) error += ' ('+JSON.stringify([SAMPLE_FILE_PATH, context.definedHash])+')'; @@ -79,7 +344,7 @@ describe("Assets", function () { }); }); - it("Assets.getMapping(path, {callback(error, hash)})", function (done) { + it("Assets.getMapping", function (done) { expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf'); expect(context.definedPath).toMatch(/^\//, 'asfasdf'); Assets.getMapping(context.definedPath, function(error, hash) { @@ -91,36 +356,47 @@ describe("Assets", function () { }); }); - it('Assets.getMapping(nullHash, {callback(data)})', function(done) { + it('.getMapping(nullHash)', function(done) { // FIXME: characterization test -- current behavior is that downloadData silently fails // when given an asset that doesn't exist Assets.downloadData(NULL_HASH, function(result) { throw new Error("this callback 'should' not be triggered"); }); - setTimeout(function() { done(); }, 2000); + setTimeout(function() { done(); }, ATP_TIMEOUT_MS); }); - it('Assets.downloadData(hash, {callback(data)})', function(done) { + it('.downloadData(hash)', function(done) { expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf'); Assets.downloadData('atp:' + context.definedHash, function(result) { expect(result).toEqual(context.definedContent); done(); }); }); - - it('Assets.downloadData(nullHash, {callback(data)})', function(done) { + + it('.downloadData(nullHash)', function(done) { // FIXME: characterization test -- current behavior is that downloadData silently fails // when given an asset doesn't exist Assets.downloadData(NULL_HASH, function(result) { throw new Error("this callback 'should' not be triggered"); }); - setTimeout(function() { - done(); - }, 2000); + setTimeout(function() { done(); }, ATP_TIMEOUT_MS); }); - + describe('exceptions', function() { - describe('Asset.setMapping', function() { + describe('.getAsset', function() { + it('-- invalid data.gz', function(done) { + expect(context.memo.jsonPath).toBeDefined(); + Assets.getAsset({ + url: context.memo.jsonPath, + responseType: 'json', + decompress: true, + }, function(error, result) { + expect(error).toMatch(/gunzip error/); + done(); + }); + }); + }); + describe('.setMapping', function() { it('-- invalid path', function(done) { Assets.setMapping('foo', NULL_HASH, function(error/*, result*/) { expect(error).toEqual('Path is invalid'); @@ -136,7 +412,7 @@ describe("Assets", function () { }); }); }); - describe('Asset.getMapping', function() { + describe('.getMapping', function() { it('-- invalid path throws immediate', function() { expect(function() { Assets.getMapping('foo', function(error, hash) { @@ -161,12 +437,57 @@ describe("Assets", function () { function run() {} function instrument_testrunner(force) { if (force || typeof describe === 'undefined') { + var oldPrint = print; + window = new OverlayWebWindow({ + title: 'assetUnitTests.js', + width: 640, + height: 480, + source: 'about:blank', + }); + Script.scriptEnding.connect(window, 'close'); + window.closed.connect(Script, 'stop'); + // wait for window (ready) and test runner (ran) both to initialize + var ready = false; + var ran = false; + window.webEventReceived.connect(function() { ready = true; maybeRun(); }); + run = function() { ran = true; maybeRun(); }; + + window.setURL([ + 'data:text/html;text,', + '', + '
',
+            '',
+            '',
+        ].join('\n'));
+        print = function() {
+            var str = [].slice.call(arguments).join(' ');
+            if (ready) {
+                window.emitScriptEvent(str + '\n');
+            } else {
+                oldPrint('!ready', str);
+            }
+        };
+
         // Include testing library
         Script.include('../../libraries/jasmine/jasmine.js');
         Script.include('../../libraries/jasmine/hifi-boot.js');
 
-        run = function() {
-            if (!/:console/.test(Script.resolvePath(''))) {
+        function maybeRun() {
+            if (!ran || !ready) {
+                return oldPrint('doRun -- waiting for both script and web window to become available');
+            }
+            if (!force) {
                 // invoke Script.stop (after any async tests complete)
                 jasmine.getEnv().addReporter({ jasmineDone: Script.stop });
             } else {

From 97889b30d5cc420065fa7459c7bcd4cbc869955e Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Tue, 23 Jan 2018 03:21:19 -0500
Subject: [PATCH 15/21] remove unnecessary lambda capture

---
 libraries/networking/src/AssetResourceRequest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp
index 9523a9aa26..6d5bbb3ac5 100644
--- a/libraries/networking/src/AssetResourceRequest.cpp
+++ b/libraries/networking/src/AssetResourceRequest.cpp
@@ -146,7 +146,7 @@ void AssetResourceRequest::requestHash(const AssetUtils::AssetHash& hash) {
     _assetRequest = assetClient->createRequest(hash, _byteRange);
 
     connect(_assetRequest, &AssetRequest::progress, this, &AssetResourceRequest::onDownloadProgress);
-    connect(_assetRequest, &AssetRequest::finished, this, [this, hash](AssetRequest* req) {
+    connect(_assetRequest, &AssetRequest::finished, this, [this](AssetRequest* req) {
         Q_ASSERT(_state == InProgress);
         Q_ASSERT(req == _assetRequest);
         Q_ASSERT(req->getState() == AssetRequest::Finished);

From 16ca0eebed0962651fb1c8da3f9fced7c7b5661b Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Tue, 23 Jan 2018 03:36:00 -0500
Subject: [PATCH 16/21] fix atp-client AssetUtils:: namespace

---
 tools/atp-client/src/ATPClientApp.cpp | 2 +-
 tools/atp-client/src/ATPClientApp.h   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/atp-client/src/ATPClientApp.cpp b/tools/atp-client/src/ATPClientApp.cpp
index 9fd1bf8d4f..307c913a96 100644
--- a/tools/atp-client/src/ATPClientApp.cpp
+++ b/tools/atp-client/src/ATPClientApp.cpp
@@ -361,7 +361,7 @@ void ATPClientApp::lookupAsset() {
     request->start();
 }
 
-void ATPClientApp::download(AssetHash hash) {
+void ATPClientApp::download(AssetUtils::AssetHash hash) {
     auto assetClient = DependencyManager::get();
     auto assetRequest = new AssetRequest(hash);
 
diff --git a/tools/atp-client/src/ATPClientApp.h b/tools/atp-client/src/ATPClientApp.h
index a3904d6e50..42760ceca9 100644
--- a/tools/atp-client/src/ATPClientApp.h
+++ b/tools/atp-client/src/ATPClientApp.h
@@ -45,7 +45,7 @@ private:
     void setMapping(QString hash);
     void lookupAsset();
     void listAssets();
-    void download(AssetHash hash);
+    void download(AssetUtils::AssetHash hash);
     void finish(int exitCode);
     bool _verbose;
 

From 76cbc4ee55ec7349b19647b8eaba654b62444f44 Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Tue, 23 Jan 2018 14:47:49 -0500
Subject: [PATCH 17/21] separate getAsset fail/then handlers

---
 libraries/script-engine/src/AssetScriptingInterface.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp
index 7c9bf2c4eb..78d0b323ee 100644
--- a/libraries/script-engine/src/AssetScriptingInterface.cpp
+++ b/libraries/script-engine/src/AssetScriptingInterface.cpp
@@ -236,11 +236,12 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope,
     Promise fetched = jsPromiseReady(makePromise("fetched"), scope, callback);
     Promise mapped = makePromise("mapped");
 
-    mapped->ready([=](QString error, QVariantMap result) {
+    mapped->fail(fetched);
+    mapped->then([=](QVariantMap result) {
         QString hash = result.value("hash").toString();
         QString url = result.value("url").toString();
-        if (!error.isEmpty() || !AssetUtils::isValidHash(hash)) {
-            fetched->reject(error.isEmpty() ? "internal hash error: " + hash : error, result);
+        if (!AssetUtils::isValidHash(hash)) {
+            fetched->reject("internal hash error: " + hash, result);
         } else {
             Promise promise = loadAsset(hash, decompress, responseType);
             promise->mixin(result);

From c2ee13931ec7d01dd1417fbf1e08d06ca2b526e5 Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Wed, 24 Jan 2018 13:38:10 -0500
Subject: [PATCH 18/21] changes per CR feedback; additional jsdoc

---
 libraries/networking/src/AssetClient.cpp      |  2 +-
 .../src/BaseAssetScriptingInterface.h         |  3 ---
 .../src/AssetScriptingInterface.cpp           | 20 ++++++++++++++++---
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp
index a0c86a25e8..4d987a3200 100644
--- a/libraries/networking/src/AssetClient.cpp
+++ b/libraries/networking/src/AssetClient.cpp
@@ -92,7 +92,7 @@ MiniPromise::Promise AssetClient::cacheInfoRequestAsync(MiniPromise::Promise def
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "cacheInfoRequestAsync", Q_ARG(MiniPromise::Promise, deferred));
     } else {
-        auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache());
+        auto cache = qobject_cast(NetworkAccessManager::getInstance().cache());
         if (cache) {
             deferred->resolve({
                 { "cacheDirectory", cache->cacheDirectory() },
diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h
index 5ac391a4d7..a51f4a5105 100644
--- a/libraries/networking/src/BaseAssetScriptingInterface.h
+++ b/libraries/networking/src/BaseAssetScriptingInterface.h
@@ -21,9 +21,6 @@
 #include "NetworkAccessManager.h"
 #include 
 
-/**jsdoc
- * @namespace Assets
- */
 class BaseAssetScriptingInterface : public QObject {
     Q_OBJECT
 public:
diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp
index 78d0b323ee..1c811573fb 100644
--- a/libraries/script-engine/src/AssetScriptingInterface.cpp
+++ b/libraries/script-engine/src/AssetScriptingInterface.cpp
@@ -215,6 +215,21 @@ void AssetScriptingInterface::deleteAsset(QScriptValue options, QScriptValue sco
     jsVerify(false, "TODO: deleteAsset API");
 }
 
+/**jsdoc
+ * @typedef {string} Assets.GetOptions.ResponseType
+ * 
Available responseType values for use with @{link Assets.getAsset} and @{link Assets.loadFromCache} configuration option. 

+ * 
+ *   
+ *     
+ *   
+ *   
+ *     
+ *     
+ *     
+ *   
+ * 
responseTypetypeof response value
"text"contents returned as utf-8 decoded String value
"arraybuffer"contents as a binary ArrayBuffer object
"json"contents as a parsed JSON object

+ */
+
 void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) {
     JS_VERIFY(options.isObject() || options.isString(), "expected request options Object or URL as first parameter");
 
@@ -224,7 +239,7 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope,
     if (options.isString()) {
         url = options.toString();
     }
-    if (responseType.isEmpty() || responseType == "string") {
+    if (responseType.isEmpty()) {
         responseType = "text";
     }
     auto asset = AssetUtils::getATPUrl(url).path();
@@ -389,8 +404,7 @@ bool AssetScriptingInterface::canWriteCacheValue(const QUrl& url) {
     // allow cache writes only from Client, EntityServer and Agent scripts
     bool isAllowedContext = (
         scriptEngine->isClientScript() ||
-        scriptEngine->isAgentScript() ||
-        scriptEngine->isEntityServerScript()
+        scriptEngine->isAgentScript()
     );
     if (!isAllowedContext) {
         qCDebug(scriptengine) << __FUNCTION__ << "invalid context" << scriptEngine->getContext() << url;

From 63a226f9994a025093ef9043603d3a84dc410b17 Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Wed, 24 Jan 2018 13:49:58 -0500
Subject: [PATCH 19/21] make fromScriptValue logic easier to follow for CR;
 eliminate extra QByteArray conversion

---
 .../script-engine/src/ArrayBufferClass.cpp    | 33 ++++++++++++++---
 .../script-engine/src/ArrayBufferClass.h      |  2 --
 .../src/ArrayBufferViewClass.cpp              | 35 +------------------
 .../script-engine/src/ArrayBufferViewClass.h  |  4 ---
 4 files changed, 29 insertions(+), 45 deletions(-)

diff --git a/libraries/script-engine/src/ArrayBufferClass.cpp b/libraries/script-engine/src/ArrayBufferClass.cpp
index b84188f707..4a06dee391 100644
--- a/libraries/script-engine/src/ArrayBufferClass.cpp
+++ b/libraries/script-engine/src/ArrayBufferClass.cpp
@@ -18,14 +18,19 @@
 
 #include "ArrayBufferClass.h"
 
+
 static const QString CLASS_NAME = "ArrayBuffer";
 
+// FIXME: Q_DECLARE_METATYPE is global and really belongs in a shared header file, not per .cpp like this
+// (see DataViewClass.cpp, etc. which would also have to be updated to resolve)
+Q_DECLARE_METATYPE(QScriptClass*)
 Q_DECLARE_METATYPE(QByteArray*)
+int qScriptClassPointerMetaTypeId = qRegisterMetaType();
+int qByteArrayPointerMetaTypeId = qRegisterMetaType();
 
 ArrayBufferClass::ArrayBufferClass(ScriptEngine* scriptEngine) :
 QObject(scriptEngine),
-QScriptClass(scriptEngine),
-_scriptEngine(scriptEngine) {
+QScriptClass(scriptEngine) {
     qScriptRegisterMetaType(engine(), toScriptValue, fromScriptValue);
     QScriptValue global = engine()->globalObject();
     
@@ -144,12 +149,30 @@ QScriptValue ArrayBufferClass::toScriptValue(QScriptEngine* engine, const QByteA
     QScriptValue ctor = engine->globalObject().property(CLASS_NAME);
     ArrayBufferClass* cls = qscriptvalue_cast(ctor.data());
     if (!cls) {
-        return engine->newVariant(QVariant::fromValue(ba));
+        if (engine->currentContext()) {
+            engine->currentContext()->throwError("arrayBufferClass::toScriptValue -- could not get " + CLASS_NAME + " class constructor");
+        }
+        return QScriptValue::NullValue;
     }
     return cls->newInstance(ba);
 }
 
-void ArrayBufferClass::fromScriptValue(const QScriptValue& obj, QByteArray& ba) {
-    ba = qvariant_cast(obj.data().toVariant());
+void ArrayBufferClass::fromScriptValue(const QScriptValue& object, QByteArray& byteArray) {
+    if (object.isString()) {
+        // UTF-8 encoded String
+        byteArray = object.toString().toUtf8();
+    } else if (object.isArray()) {
+        // Array of uint8s eg: [ 128, 3, 25, 234 ]
+        auto Uint8Array = object.engine()->globalObject().property("Uint8Array");
+        auto typedArray = Uint8Array.construct(QScriptValueList{object});
+        if (QByteArray* buffer = qscriptvalue_cast(typedArray.property("buffer"))) {
+            byteArray = *buffer;
+        }
+    } else if (object.isObject()) {
+        // ArrayBuffer instance (or any JS class that supports coercion into QByteArray*)
+        if (QByteArray* buffer = qscriptvalue_cast(object.data())) {
+            byteArray = *buffer;
+        }
+    }
 }
 
diff --git a/libraries/script-engine/src/ArrayBufferClass.h b/libraries/script-engine/src/ArrayBufferClass.h
index 69c2cc0799..a2cf08cc06 100644
--- a/libraries/script-engine/src/ArrayBufferClass.h
+++ b/libraries/script-engine/src/ArrayBufferClass.h
@@ -40,7 +40,6 @@ public:
     QString name() const override;
     QScriptValue prototype() const override;
 
-    ScriptEngine* getEngine() { return _scriptEngine; }
 
 private:
     static QScriptValue construct(QScriptContext* context, QScriptEngine* engine);
@@ -55,7 +54,6 @@ private:
     QScriptString _name;
     QScriptString _byteLength;
 
-    ScriptEngine* _scriptEngine;
 };
 
 #endif // hifi_ArrayBufferClass_h
diff --git a/libraries/script-engine/src/ArrayBufferViewClass.cpp b/libraries/script-engine/src/ArrayBufferViewClass.cpp
index 84cb32d665..cf776ed834 100644
--- a/libraries/script-engine/src/ArrayBufferViewClass.cpp
+++ b/libraries/script-engine/src/ArrayBufferViewClass.cpp
@@ -11,8 +11,7 @@
 
 #include "ArrayBufferViewClass.h"
 
-int qScriptClassPointerMetaTypeId = qRegisterMetaType();
-int qByteArrayMetaTypeId = qRegisterMetaType();
+Q_DECLARE_METATYPE(QByteArray*)
 
 ArrayBufferViewClass::ArrayBufferViewClass(ScriptEngine* scriptEngine) :
 QObject(scriptEngine),
@@ -22,7 +21,6 @@ _scriptEngine(scriptEngine) {
     _bufferName = engine()->toStringHandle(BUFFER_PROPERTY_NAME.toLatin1());
     _byteOffsetName = engine()->toStringHandle(BYTE_OFFSET_PROPERTY_NAME.toLatin1());
     _byteLengthName = engine()->toStringHandle(BYTE_LENGTH_PROPERTY_NAME.toLatin1());
-    registerMetaTypes(scriptEngine);
 }
 
 QScriptClass::QueryFlags ArrayBufferViewClass::queryProperty(const QScriptValue& object,
@@ -52,34 +50,3 @@ QScriptValue::PropertyFlags ArrayBufferViewClass::propertyFlags(const QScriptVal
                                                                 const QScriptString& name, uint id) {
     return QScriptValue::Undeletable;
 }
-
-namespace {
-    void byteArrayFromScriptValue(const QScriptValue& object, QByteArray& byteArray) {
-        if (object.isValid()) {
-            if (object.isObject()) {
-                if (object.isArray()) {
-                    auto Uint8Array = object.engine()->globalObject().property("Uint8Array");
-                    auto typedArray = Uint8Array.construct(QScriptValueList{object});
-                    byteArray = qvariant_cast(typedArray.property("buffer").toVariant());
-                } else {
-                    byteArray = qvariant_cast(object.data().toVariant());
-                }
-            } else {
-                byteArray = object.toString().toUtf8();
-            }
-        }
-    }
-
-    QScriptValue byteArrayToScriptValue(QScriptEngine *engine, const QByteArray& byteArray) {
-        QScriptValue data = engine->newVariant(QVariant::fromValue(byteArray));
-        QScriptValue constructor = engine->globalObject().property("ArrayBuffer");
-        Q_ASSERT(constructor.isValid());
-        auto array = qscriptvalue_cast(constructor.data());
-        Q_ASSERT(array);
-        return engine->newObject(array, data);
-    }
-}
-
-void ArrayBufferViewClass::registerMetaTypes(QScriptEngine* scriptEngine) {
-    qScriptRegisterMetaType(scriptEngine, byteArrayToScriptValue, byteArrayFromScriptValue);
-}
diff --git a/libraries/script-engine/src/ArrayBufferViewClass.h b/libraries/script-engine/src/ArrayBufferViewClass.h
index 038cc75ffd..67af4a3fc3 100644
--- a/libraries/script-engine/src/ArrayBufferViewClass.h
+++ b/libraries/script-engine/src/ArrayBufferViewClass.h
@@ -29,7 +29,6 @@ static const QString BYTE_LENGTH_PROPERTY_NAME = "byteLength";
 class ArrayBufferViewClass : public QObject, public QScriptClass {
     Q_OBJECT
 public:
-    static void registerMetaTypes(QScriptEngine* scriptEngine);
     ArrayBufferViewClass(ScriptEngine* scriptEngine);
 
     ScriptEngine* getScriptEngine() { return _scriptEngine; }
@@ -50,7 +49,4 @@ protected:
     ScriptEngine* _scriptEngine;
 };
 
-Q_DECLARE_METATYPE(QScriptClass*)
-Q_DECLARE_METATYPE(QByteArray)
-
 #endif // hifi_ArrayBufferViewClass_h

From 3ebf322fcf994ff866e3e8d368393daab7156518 Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Wed, 24 Jan 2018 14:13:34 -0500
Subject: [PATCH 20/21] switch to foreach per CR feeddback

---
 libraries/networking/src/AssetClient.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp
index 4d987a3200..1db93e8cb9 100644
--- a/libraries/networking/src/AssetClient.cpp
+++ b/libraries/networking/src/AssetClient.cpp
@@ -121,10 +121,9 @@ MiniPromise::Promise AssetClient::queryCacheMetaAsync(const QUrl& url, MiniPromi
                     { "metaDataURL", metaData.url() },
                 });
             } else {
-                QHashIterator i(metaData.attributes());
-                while (i.hasNext()) {
-                    i.next();
-                    attributes[QString::number(i.key())] = i.value();
+                auto metaAttributes = metaData.attributes();
+                foreach(QNetworkRequest::Attribute k, metaAttributes.keys()) {
+                    attributes[QString::number(k)] = metaAttributes[k];
                 }
                 for (const auto& i : metaData.rawHeaders()) {
                     rawHeaders[i.first] = i.second;

From e8f0df43638ca5cee51f6558c781d4031e8f799b Mon Sep 17 00:00:00 2001
From: humbletim 
Date: Wed, 24 Jan 2018 14:28:22 -0500
Subject: [PATCH 21/21] switch to bool per CR feedback

---
 libraries/networking/src/BaseAssetScriptingInterface.cpp | 4 ++--
 libraries/networking/src/BaseAssetScriptingInterface.h   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp
index d62e992822..3149bbc768 100644
--- a/libraries/networking/src/BaseAssetScriptingInterface.cpp
+++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp
@@ -42,7 +42,7 @@ bool BaseAssetScriptingInterface::initializeCache() {
     if (!assetClient()) {
         return false; // not yet possible to initialize the cache
     }
-    if (!_cacheDirectory.isEmpty()) {
+    if (_cacheReady) {
         return true; // cache is ready
     }
 
@@ -51,7 +51,7 @@ bool BaseAssetScriptingInterface::initializeCache() {
 
     Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus");
     deferred->then([this](QVariantMap result) {
-        _cacheDirectory = result.value("cacheDirectory").toString();
+        _cacheReady = !result.value("cacheDirectory").toString().isEmpty();
     });
     deferred->fail([](QString error) {
         qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error;
diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h
index a51f4a5105..336f3f81db 100644
--- a/libraries/networking/src/BaseAssetScriptingInterface.h
+++ b/libraries/networking/src/BaseAssetScriptingInterface.h
@@ -40,7 +40,7 @@ public slots:
     QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); }
 
 protected:
-    QString _cacheDirectory;
+    bool _cacheReady{ false };
     bool initializeCache();
     Promise getCacheStatus();
     Promise queryCacheMeta(const QUrl& url);