From 3a735c1fc7d12809289fc225cc336bc0c4902060 Mon Sep 17 00:00:00 2001 From: humbletim Date: Tue, 23 Jan 2018 03:12:26 -0500 Subject: [PATCH] CR feedback and code cleanup --- libraries/networking/src/AssetClient.cpp | 234 ++++++++------ libraries/networking/src/AssetClient.h | 8 +- libraries/networking/src/AssetUtils.cpp | 2 +- .../src/BaseAssetScriptingInterface.cpp | 187 ++++++----- .../src/BaseAssetScriptingInterface.h | 26 +- .../src/ArrayBufferViewClass.cpp | 35 +- .../script-engine/src/ArrayBufferViewClass.h | 4 + .../src/AssetScriptingInterface.cpp | 306 ++++++++++-------- .../src/AssetScriptingInterface.h | 91 +++++- libraries/script-engine/src/ScriptEngine.cpp | 4 +- libraries/script-engine/src/ScriptEngine.h | 2 +- libraries/shared/src/BaseScriptEngine.cpp | 6 + libraries/shared/src/shared/MiniPromises.cpp | 17 + libraries/shared/src/shared/MiniPromises.h | 107 +++--- 14 files changed, 658 insertions(+), 371 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index c126fc2e5a..a0c86a25e8 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -81,122 +81,170 @@ void AssetClient::init() { } -void AssetClient::cacheInfoRequest(MiniPromise::Promise deferred) { - if (QThread::currentThread() != thread()) { - if (!QMetaType::isRegistered(qMetaTypeId())) { - qRegisterMetaType(); - } - QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred)); - return; - } - if (auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache())) { - deferred->resolve({ - { "cacheDirectory", cache->cacheDirectory() }, - { "cacheSize", cache->cacheSize() }, - { "maximumCacheSize", cache->maximumCacheSize() }, - }); - } else { - deferred->reject("Cache not available"); - } +namespace { + const QString& CACHE_ERROR_MESSAGE{ "AssetClient::Error: %1 %2" }; } -void AssetClient::queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url) { +MiniPromise::Promise AssetClient::cacheInfoRequestAsync(MiniPromise::Promise deferred) { + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread + } if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "cacheInfoRequest", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); - return; - } - if (auto cache = NetworkAccessManager::getInstance().cache()) { - QNetworkCacheMetaData metaData = cache->metaData(url); - QVariantMap attributes, rawHeaders; - - QHashIterator i(metaData.attributes()); - while (i.hasNext()) { - i.next(); - attributes[QString::number(i.key())] = i.value(); - } - for (const auto& i : metaData.rawHeaders()) { - rawHeaders[i.first] = i.second; - } - deferred->resolve({ - { "isValid", metaData.isValid() }, - { "url", metaData.url() }, - { "expirationDate", metaData.expirationDate() }, - { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, - { "saveToDisk", metaData.saveToDisk() }, - { "attributes", attributes }, - { "rawHeaders", rawHeaders }, - }); + QMetaObject::invokeMethod(this, "cacheInfoRequestAsync", Q_ARG(MiniPromise::Promise, deferred)); } else { - deferred->reject("cache currently unavailable"); + auto* cache = qobject_cast(NetworkAccessManager::getInstance().cache()); + if (cache) { + deferred->resolve({ + { "cacheDirectory", cache->cacheDirectory() }, + { "cacheSize", cache->cacheSize() }, + { "maximumCacheSize", cache->maximumCacheSize() }, + }); + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable")); + } } + return deferred; } -void AssetClient::loadFromCache(MiniPromise::Promise deferred, const QUrl& url) { +MiniPromise::Promise AssetClient::queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred) { if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "loadFromCache", Q_ARG(MiniPromise::Promise, deferred), Q_ARG(const QUrl&, url)); - return; - } - if (auto cache = NetworkAccessManager::getInstance().cache()) { - MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); - queryCacheMeta(metaRequest, url); - metaRequest->then([&](QString error, QVariantMap metadata) { - if (!error.isEmpty()) { - deferred->reject(error, metadata); - return; - } - QVariantMap result = { - { "metadata", metadata }, - { "data", QByteArray() }, - }; - // caller is responsible for the deletion of the ioDevice, hence the unique_ptr - if (auto ioDevice = std::unique_ptr(cache->data(url))) { - QByteArray data = ioDevice->readAll(); - result["data"] = data; + QMetaObject::invokeMethod(this, "queryCacheMetaAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + QNetworkCacheMetaData metaData = cache->metaData(url); + QVariantMap attributes, rawHeaders; + if (!metaData.isValid()) { + deferred->reject("invalid cache entry", { + { "_url", url }, + { "isValid", metaData.isValid() }, + { "metaDataURL", metaData.url() }, + }); } else { - error = "cache data unavailable"; + QHashIterator i(metaData.attributes()); + while (i.hasNext()) { + i.next(); + attributes[QString::number(i.key())] = i.value(); + } + for (const auto& i : metaData.rawHeaders()) { + rawHeaders[i.first] = i.second; + } + deferred->resolve({ + { "_url", url }, + { "isValid", metaData.isValid() }, + { "url", metaData.url() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + { "saveToDisk", metaData.saveToDisk() }, + { "attributes", attributes }, + { "rawHeaders", rawHeaders }, + }); } - deferred->handle(error, result); - }); - } else { - deferred->reject("cache currently unavailable"); + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable")); + } } + return deferred; +} + +MiniPromise::Promise AssetClient::loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred) { + auto errorMessage = CACHE_ERROR_MESSAGE.arg(__FUNCTION__); + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread + } + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadFromCacheAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + MiniPromise::Promise metaRequest = makePromise(__FUNCTION__); + queryCacheMetaAsync(url, metaRequest); + metaRequest->finally([&](QString error, QVariantMap metadata) { + QVariantMap result = { + { "url", url }, + { "metadata", metadata }, + { "data", QByteArray() }, + }; + if (!error.isEmpty()) { + deferred->reject(error, result); + return; + } + // caller is responsible for the deletion of the ioDevice, hence the unique_ptr + auto ioDevice = std::unique_ptr(cache->data(url)); + if (ioDevice) { + result["data"] = ioDevice->readAll(); + } else { + error = errorMessage.arg("error reading data"); + } + deferred->handle(error, result); + }); + } else { + deferred->reject(errorMessage.arg("cache unavailable")); + } + } + return deferred; } namespace { // parse RFC 1123 HTTP date format QDateTime parseHttpDate(const QString& dateString) { QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss"); + if (!dt.isValid()) { + dt = QDateTime::fromString(dateString, Qt::ISODateWithMs); + } + if (!dt.isValid()) { + qDebug() << __FUNCTION__ << "unrecognized date format:" << dateString; + } dt.setTimeSpec(Qt::UTC); return dt; } + QDateTime getHttpDateValue(const QVariantMap& headers, const QString& keyName, const QDateTime& defaultValue) { + return headers.contains(keyName) ? parseHttpDate(headers[keyName].toString()) : defaultValue; + } } -void AssetClient::saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - if (auto cache = NetworkAccessManager::getInstance().cache()) { - QDateTime lastModified = headers.contains("last-modified") ? - parseHttpDate(headers["last-modified"].toString()) : - QDateTime::currentDateTimeUtc(); - QDateTime expirationDate = headers.contains("expires") ? - parseHttpDate(headers["expires"].toString()) : - QDateTime(); // never expires - QNetworkCacheMetaData metaData; - metaData.setUrl(url); - metaData.setSaveToDisk(true); - metaData.setLastModified(lastModified); - metaData.setExpirationDate(expirationDate); - if (auto ioDevice = cache->prepare(metaData)) { - ioDevice->write(data); - cache->insert(ioDevice); - qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; - deferred->resolve({{ "success", true }}); - } else { - auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); - qCWarning(asset_client) << error; - deferred->reject(error); - } - } else { - deferred->reject("cache currently unavailable"); +MiniPromise::Promise AssetClient::saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& headers, MiniPromise::Promise deferred) { + if (!deferred) { + deferred = makePromise(__FUNCTION__); // create on caller's thread } + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod( + this, "saveToCacheAsync", Qt::QueuedConnection, + Q_ARG(const QUrl&, url), + Q_ARG(const QByteArray&, data), + Q_ARG(const QVariantMap&, headers), + Q_ARG(MiniPromise::Promise, deferred)); + } else { + auto cache = NetworkAccessManager::getInstance().cache(); + if (cache) { + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData.setSaveToDisk(true); + metaData.setLastModified(getHttpDateValue(headers, "last-modified", QDateTime::currentDateTimeUtc())); + metaData.setExpirationDate(getHttpDateValue(headers, "expires", QDateTime())); // nil defaultValue == never expires + auto ioDevice = cache->prepare(metaData); + if (ioDevice) { + ioDevice->write(data); + cache->insert(ioDevice); + qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)"; + deferred->resolve({ + { "url", url }, + { "success", true }, + { "metaDataURL", metaData.url() }, + { "byteLength", data.size() }, + { "expirationDate", metaData.expirationDate() }, + { "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() }, + }); + } else { + auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString()); + qCWarning(asset_client) << error; + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg(error)); + } + } else { + deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("unavailable")); + } + } + return deferred; } void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) { diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 81149bf3d6..3ec96c3dd4 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -67,10 +67,10 @@ public slots: void init(); void cacheInfoRequest(QObject* reciever, QString slot); - void cacheInfoRequest(MiniPromise::Promise deferred); - void queryCacheMeta(MiniPromise::Promise deferred, const QUrl& url); - void loadFromCache(MiniPromise::Promise deferred, const QUrl& url); - void saveToCache(MiniPromise::Promise deferred, const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + MiniPromise::Promise cacheInfoRequestAsync(MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr); + MiniPromise::Promise saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap(), MiniPromise::Promise deferred = nullptr); void clearCache(); private slots: diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 45150e80c1..117274eab8 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -50,7 +50,7 @@ QUrl getATPUrl(const QString& input) { QUrl::RemoveAuthority | QUrl::RemoveScheme | QUrl::StripTrailingSlash | QUrl::NormalizePathSegments ); - QString baseName = QFileInfo(path).baseName(); + QString baseName = QFileInfo(url.path()).baseName(); if (isValidPath(path) || isValidHash(baseName)) { return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP).arg(path)); } diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index f6f7fd87e3..d62e992822 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -28,14 +28,18 @@ using Promise = MiniPromise::Promise; QSharedPointer BaseAssetScriptingInterface::assetClient() { - return DependencyManager::get(); + auto client = DependencyManager::get(); + Q_ASSERT(client); + if (!client) { + qDebug() << "BaseAssetScriptingInterface::assetClient unavailable"; + } + return client; } BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent) {} bool BaseAssetScriptingInterface::initializeCache() { - auto assets = assetClient(); - if (!assets) { + if (!assetClient()) { return false; // not yet possible to initialize the cache } if (!_cacheDirectory.isEmpty()) { @@ -43,41 +47,64 @@ bool BaseAssetScriptingInterface::initializeCache() { } // attempt to initialize the cache - QMetaObject::invokeMethod(assets.data(), "init"); + QMetaObject::invokeMethod(assetClient().data(), "init"); Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus"); - deferred->then([&](QVariantMap result) { + deferred->then([this](QVariantMap result) { _cacheDirectory = result.value("cacheDirectory").toString(); }); - deferred->fail([&](QString error) { + deferred->fail([](QString error) { qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error; }); - assets->cacheInfoRequest(deferred); + assetClient()->cacheInfoRequestAsync(deferred); return false; // cache is not ready yet } Promise BaseAssetScriptingInterface::getCacheStatus() { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->cacheInfoRequest(deferred); - return deferred; + return assetClient()->cacheInfoRequestAsync(makePromise(__FUNCTION__)); } Promise BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->queryCacheMeta(deferred, url); - return deferred; + return assetClient()->queryCacheMetaAsync(url, makePromise(__FUNCTION__)); } -Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->loadFromCache(deferred, url); - return deferred; +Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url, bool decompress, const QString& responseType) { + QVariantMap metaData = { + { "_type", "cache" }, + { "url", url }, + { "responseType", responseType }, + }; + + Promise completed = makePromise("loadFromCache::completed"); + Promise fetched = makePromise("loadFromCache::fetched"); + + Promise downloaded = assetClient()->loadFromCacheAsync(url, makePromise("loadFromCache-retrieval")); + downloaded->mixin(metaData); + downloaded->fail(fetched); + + if (decompress) { + downloaded->then([=](QVariantMap result) { + fetched->mixin(result); + Promise decompressed = decompressBytes(result.value("data").toByteArray()); + decompressed->mixin(result); + decompressed->ready(fetched); + }); + } else { + downloaded->then(fetched); + } + + fetched->fail(completed); + fetched->then([=](QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); + }); + + return completed; } Promise BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) { - Promise deferred = makePromise(__FUNCTION__); - DependencyManager::get()->saveToCache(deferred, url, data, headers); - return deferred; + return assetClient()->saveToCacheAsync(url, data, headers, makePromise(__FUNCTION__)); } Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) { @@ -92,73 +119,81 @@ Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, Q { "responseType", responseType }, }; - Promise fetched = makePromise("loadAsset::fetched"), - loaded = makePromise("loadAsset::loaded"); + Promise completed = makePromise("loadAsset::completed"); + Promise fetched = makePromise("loadAsset::fetched"); - downloadBytes(hash) - ->mixin(metaData) - ->ready([=](QString error, QVariantMap result) { - Q_ASSERT(thread() == QThread::currentThread()); - fetched->mixin(result); - if (decompress) { - decompressBytes(result.value("data").toByteArray()) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - fetched->handle(error, result); - }); - } else { - fetched->handle(error, result); - } + Promise downloaded = downloadBytes(hash); + downloaded->mixin(metaData); + downloaded->fail(fetched); + + if (decompress) { + downloaded->then([=](QVariantMap result) { + Q_ASSERT(thread() == QThread::currentThread()); + fetched->mixin(result); + Promise decompressed = decompressBytes(result.value("data").toByteArray()); + decompressed->mixin(result); + decompressed->ready(fetched); + }); + } else { + downloaded->then(fetched); + } + + fetched->fail(completed); + fetched->then([=](QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); }); - fetched->ready([=](QString error, QVariantMap result) { - if (responseType == "arraybuffer") { - loaded->resolve(NoError, result); - } else { - convertBytes(result.value("data").toByteArray(), responseType) - ->mixin(result) - ->ready([=](QString error, QVariantMap result) { - loaded->resolve(NoError, result); - }); - } - }); - - return loaded; + return completed; } Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) { - QVariantMap result; + QVariantMap result = { + { "_contentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, + { "_byteLength", dataByteArray.size() }, + { "_responseType", responseType }, + }; + QString error; Promise conversion = makePromise(__FUNCTION__); - if (dataByteArray.size() == 0) { - result["response"] = QString(); + if (!RESPONSE_TYPES.contains(responseType)) { + error = QString("convertBytes: invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | ")); + } else if (responseType == "arraybuffer") { + // interpret as bytes + result["response"] = dataByteArray; } else if (responseType == "text") { + // interpret as utf-8 text result["response"] = QString::fromUtf8(dataByteArray); } else if (responseType == "json") { + // interpret as JSON QJsonParseError status; auto parsed = QJsonDocument::fromJson(dataByteArray, &status); if (status.error == QJsonParseError::NoError) { - result["response"] = parsed.isArray() ? - QVariant(parsed.array().toVariantList()) : - QVariant(parsed.object().toVariantMap()); + result["response"] = parsed.isArray() ? QVariant(parsed.array().toVariantList()) : QVariant(parsed.object().toVariantMap()); } else { - QVariantMap errorResult = { + result = { { "error", status.error }, { "offset", status.offset }, }; - return conversion->reject("JSON Parse Error: " + status.errorString(), errorResult); + error = "JSON Parse Error: " + status.errorString(); } - } else if (responseType == "arraybuffer") { - result["response"] = dataByteArray; } - return conversion->resolve(NoError, result); + if (result.value("response").canConvert()) { + auto data = result.value("response").toByteArray(); + result["contentType"] = QMimeDatabase().mimeTypeForData(data).name(); + result["byteLength"] = data.size(); + result["responseType"] = responseType; + } + return conversion->handle(error, result); } Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) { QByteArray inflated; + Promise decompressed = makePromise(__FUNCTION__); auto start = usecTimestampNow(); if (gunzip(dataByteArray, inflated)) { auto end = usecTimestampNow(); - return makePromise(__FUNCTION__)->resolve(NoError, { + decompressed->resolve({ { "_compressedByteLength", dataByteArray.size() }, { "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, { "_compressMS", (double)(end - start) / 1000.0 }, @@ -168,16 +203,18 @@ Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteA { "data", inflated }, }); } else { - return makePromise(__FUNCTION__)->reject("gunzip error", {}); + decompressed->reject("gunzip error"); } + return decompressed; } Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) { QByteArray deflated; auto start = usecTimestampNow(); + Promise compressed = makePromise(__FUNCTION__); if (gzip(dataByteArray, deflated, level)) { auto end = usecTimestampNow(); - return makePromise(__FUNCTION__)->resolve(NoError, { + compressed->resolve({ { "_uncompressedByteLength", dataByteArray.size() }, { "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() }, { "_compressMS", (double)(end - start) / 1000.0 }, @@ -187,13 +224,13 @@ Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArr { "data", deflated }, }); } else { - return makePromise(__FUNCTION__)->reject("gzip error", {}); + compressed->reject("gzip error", {}); } + return compressed; } Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { - auto assetClient = DependencyManager::get(); - QPointer assetRequest = assetClient->createRequest(hash); + QPointer assetRequest = assetClient()->createRequest(hash); Promise deferred = makePromise(__FUNCTION__); QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) { @@ -208,7 +245,7 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { { "url", request->getUrl() }, { "hash", request->getHash() }, { "cached", request->loadedFromCache() }, - { "content-type", QMimeDatabase().mimeTypeForData(data).name() }, + { "contentType", QMimeDatabase().mimeTypeForData(data).name() }, { "data", data }, }; } else { @@ -225,8 +262,9 @@ Promise BaseAssetScriptingInterface::downloadBytes(QString hash) { Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { Promise deferred = makePromise(__FUNCTION__); - QPointer upload = DependencyManager::get()->createUpload(bytes); + QPointer upload = assetClient()->createUpload(bytes); + const auto byteLength = bytes.size(); QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) { Q_ASSERT(QThread::currentThread() == upload->thread()); // note: we are now on the "Resource Manager" thread @@ -237,6 +275,7 @@ Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { { "hash", hash }, { "url", AssetUtils::getATPUrl(hash).toString() }, { "filename", upload->getFilename() }, + { "byteLength", byteLength }, }; } else { error = upload->getErrorString(); @@ -251,20 +290,19 @@ Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) { } Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { - auto deferred = makePromise(__FUNCTION__); + Promise deferred = makePromise(__FUNCTION__); auto url = AssetUtils::getATPUrl(asset); auto path = url.path(); auto hash = AssetUtils::extractAssetHash(asset); if (AssetUtils::isValidHash(hash)) { // already a valid ATP hash -- nothing to do - deferred->resolve(NoError, { + deferred->resolve({ { "hash", hash }, { "path", path }, { "url", url }, }); } else if (AssetUtils::isValidFilePath(path)) { - auto assetClient = DependencyManager::get(); - QPointer request = assetClient->createGetMappingRequest(path); + QPointer request = assetClient()->createGetMappingRequest(path); QObject::connect(request, &GetMappingRequest::finished, request, [=]() { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -276,7 +314,9 @@ Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { { "_hash", hash }, { "_path", path }, { "_url", url }, + { "url", url }, { "hash", request->getHash() }, + { "hashURL", AssetUtils::getATPUrl(request->getHash()).toString() }, { "wasRedirected", request->wasRedirected() }, { "path", request->wasRedirected() ? request->getRedirectedPath() : path }, }; @@ -297,8 +337,7 @@ Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) { Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) { auto deferred = makePromise(__FUNCTION__); - auto assetClient = DependencyManager::get(); - QPointer setMappingRequest = assetClient->createSetMappingRequest(path, hash); + QPointer setMappingRequest = assetClient()->createSetMappingRequest(path, hash); connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); diff --git a/libraries/networking/src/BaseAssetScriptingInterface.h b/libraries/networking/src/BaseAssetScriptingInterface.h index 35c829fd37..5ac391a4d7 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.h +++ b/libraries/networking/src/BaseAssetScriptingInterface.h @@ -27,37 +27,29 @@ class BaseAssetScriptingInterface : public QObject { Q_OBJECT public: + const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" }; using Promise = MiniPromise::Promise; QSharedPointer assetClient(); BaseAssetScriptingInterface(QObject* parent = nullptr); public slots: - Promise getCacheStatus(); - - /**jsdoc - * Initialize the disk cache (returns true if already initialized) - * @function Assets.initializeCache - * @static - */ - bool initializeCache(); - - virtual bool isValidPath(QString input) { return AssetUtils::isValidPath(input); } - virtual bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); } + bool isValidPath(QString input) { return AssetUtils::isValidPath(input); } + bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); } QUrl getATPUrl(QString input) { return AssetUtils::getATPUrl(input); } QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); } bool isValidHash(QString input) { return AssetUtils::isValidHash(input); } QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); } QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); } - virtual Promise queryCacheMeta(const QUrl& url); - virtual Promise loadFromCache(const QUrl& url); - virtual Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); - protected: QString _cacheDirectory; - const QString NoError{}; - //virtual bool jsAssert(bool condition, const QString& error) = 0; + bool initializeCache(); + Promise getCacheStatus(); + Promise queryCacheMeta(const QUrl& url); + Promise loadFromCache(const QUrl& url, bool decompress = false, const QString& responseType = "arraybuffer"); + Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap()); + Promise loadAsset(QString asset, bool decompress, QString responseType); Promise getAssetInfo(QString asset); Promise downloadBytes(QString hash); diff --git a/libraries/script-engine/src/ArrayBufferViewClass.cpp b/libraries/script-engine/src/ArrayBufferViewClass.cpp index cf776ed834..84cb32d665 100644 --- a/libraries/script-engine/src/ArrayBufferViewClass.cpp +++ b/libraries/script-engine/src/ArrayBufferViewClass.cpp @@ -11,7 +11,8 @@ #include "ArrayBufferViewClass.h" -Q_DECLARE_METATYPE(QByteArray*) +int qScriptClassPointerMetaTypeId = qRegisterMetaType(); +int qByteArrayMetaTypeId = qRegisterMetaType(); ArrayBufferViewClass::ArrayBufferViewClass(ScriptEngine* scriptEngine) : QObject(scriptEngine), @@ -21,6 +22,7 @@ _scriptEngine(scriptEngine) { _bufferName = engine()->toStringHandle(BUFFER_PROPERTY_NAME.toLatin1()); _byteOffsetName = engine()->toStringHandle(BYTE_OFFSET_PROPERTY_NAME.toLatin1()); _byteLengthName = engine()->toStringHandle(BYTE_LENGTH_PROPERTY_NAME.toLatin1()); + registerMetaTypes(scriptEngine); } QScriptClass::QueryFlags ArrayBufferViewClass::queryProperty(const QScriptValue& object, @@ -50,3 +52,34 @@ QScriptValue::PropertyFlags ArrayBufferViewClass::propertyFlags(const QScriptVal const QScriptString& name, uint id) { return QScriptValue::Undeletable; } + +namespace { + void byteArrayFromScriptValue(const QScriptValue& object, QByteArray& byteArray) { + if (object.isValid()) { + if (object.isObject()) { + if (object.isArray()) { + auto Uint8Array = object.engine()->globalObject().property("Uint8Array"); + auto typedArray = Uint8Array.construct(QScriptValueList{object}); + byteArray = qvariant_cast(typedArray.property("buffer").toVariant()); + } else { + byteArray = qvariant_cast(object.data().toVariant()); + } + } else { + byteArray = object.toString().toUtf8(); + } + } + } + + QScriptValue byteArrayToScriptValue(QScriptEngine *engine, const QByteArray& byteArray) { + QScriptValue data = engine->newVariant(QVariant::fromValue(byteArray)); + QScriptValue constructor = engine->globalObject().property("ArrayBuffer"); + Q_ASSERT(constructor.isValid()); + auto array = qscriptvalue_cast(constructor.data()); + Q_ASSERT(array); + return engine->newObject(array, data); + } +} + +void ArrayBufferViewClass::registerMetaTypes(QScriptEngine* scriptEngine) { + qScriptRegisterMetaType(scriptEngine, byteArrayToScriptValue, byteArrayFromScriptValue); +} diff --git a/libraries/script-engine/src/ArrayBufferViewClass.h b/libraries/script-engine/src/ArrayBufferViewClass.h index 67af4a3fc3..038cc75ffd 100644 --- a/libraries/script-engine/src/ArrayBufferViewClass.h +++ b/libraries/script-engine/src/ArrayBufferViewClass.h @@ -29,6 +29,7 @@ static const QString BYTE_LENGTH_PROPERTY_NAME = "byteLength"; class ArrayBufferViewClass : public QObject, public QScriptClass { Q_OBJECT public: + static void registerMetaTypes(QScriptEngine* scriptEngine); ArrayBufferViewClass(ScriptEngine* scriptEngine); ScriptEngine* getScriptEngine() { return _scriptEngine; } @@ -49,4 +50,7 @@ protected: ScriptEngine* _scriptEngine; }; +Q_DECLARE_METATYPE(QScriptClass*) +Q_DECLARE_METATYPE(QByteArray) + #endif // hifi_ArrayBufferViewClass_h diff --git a/libraries/script-engine/src/AssetScriptingInterface.cpp b/libraries/script-engine/src/AssetScriptingInterface.cpp index 0870460a41..7c9bf2c4eb 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.cpp +++ b/libraries/script-engine/src/AssetScriptingInterface.cpp @@ -18,27 +18,33 @@ #include #include #include +#include #include #include #include - -#include -#include "Gzip.h" #include "ScriptEngine.h" #include "ScriptEngineLogging.h" -AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) {} +#include +#include + +using Promise = MiniPromise::Promise; + +AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) { + qCDebug(scriptengine) << "AssetScriptingInterface::AssetScriptingInterface" << parent; + MiniPromise::registerMetaTypes(parent); +} #define JS_VERIFY(cond, error) { if (!this->jsVerify(cond, error)) { return; } } void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); QByteArray dataByteArray = data.toUtf8(); auto upload = DependencyManager::get()->createUpload(dataByteArray); Promise deferred = makePromise(__FUNCTION__); - deferred->ready([this, handler](QString error, QVariantMap result) { + deferred->ready([=](QString error, QVariantMap result) { auto url = result.value("url").toString(); auto hash = result.value("hash").toString(); jsCallback(handler, url, hash); @@ -47,7 +53,7 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { connect(upload, &AssetUpload::finished, upload, [this, deferred](AssetUpload* upload, const QString& hash) { // we are now on the "Resource Manager" thread (and "hash" being a *reference* makes it unsafe to use directly) Q_ASSERT(QThread::currentThread() == upload->thread()); - deferred->resolve(NoError, { + deferred->resolve({ { "url", "atp:" + hash }, { "hash", hash }, }); @@ -57,7 +63,7 @@ void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) { } void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); auto setMappingRequest = assetClient()->createSetMappingRequest(path, hash); Promise deferred = makePromise(__FUNCTION__); deferred->ready([=](QString error, QVariantMap result) { @@ -86,7 +92,7 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb return; } QString hash = AssetUtils::extractAssetHash(urlString); - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); auto assetClient = DependencyManager::get(); auto assetRequest = assetClient->createRequest(hash); @@ -104,11 +110,11 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb if (request->getError() == AssetRequest::Error::NoError) { QString data = QString::fromUtf8(request->getData()); // forward a thread-safe values back to our thread - deferred->resolve(NoError, { { "data", data } }); + deferred->resolve({ { "data", data } }); } else { // FIXME: propagate error to scripts? (requires changing signature or inverting param order above..) //deferred->resolve(request->getErrorString(), { { "error", requet->getError() } }); - qDebug() << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString(); + qCDebug(scriptengine) << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString(); } request->deleteLater(); @@ -118,13 +124,9 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb } void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScriptValue callback) { - auto handler = makeScopedHandlerObject(thisObject(), callback); auto setBakingEnabledRequest = DependencyManager::get()->createSetBakingEnabledRequest({ path }, enabled); - Promise deferred = makePromise(__FUNCTION__); - deferred->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + Promise deferred = jsPromiseReady(makePromise(__FUNCTION__), thisObject(), callback); connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, setBakingEnabledRequest, [this, deferred](SetBakingEnabledRequest* request) { Q_ASSERT(QThread::currentThread() == request->thread()); @@ -150,13 +152,11 @@ void AssetScriptingInterface::sendFakedHandshake() { void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) { auto path = AssetUtils::getATPUrl(asset).path(); - auto handler = makeScopedHandlerObject(thisObject(), callback); + auto handler = jsBindCallback(thisObject(), callback); JS_VERIFY(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")"); JS_VERIFY(callback.isFunction(), "expected second parameter to be a callback function"); - qDebug() << ">>getMapping//getAssetInfo" << path; Promise promise = getAssetInfo(path); - promise->ready([this, handler](QString error, QVariantMap result) { - qDebug() << "//getMapping//getAssetInfo" << error << result.keys(); + promise->ready([=](QString error, QVariantMap result) { jsCallback(handler, error, result.value("hash").toString()); }); } @@ -168,11 +168,31 @@ bool AssetScriptingInterface::jsVerify(bool condition, const QString& error) { if (context()) { context()->throwError(error); } else { - qDebug() << "WARNING -- jsVerify failed outside of a valid JS context: " + error; + qCDebug(scriptengine) << "WARNING -- jsVerify failed outside of a valid JS context: " + error; } return false; } +QScriptValue AssetScriptingInterface::jsBindCallback(QScriptValue scope, QScriptValue callback) { + QScriptValue handler = ::makeScopedHandlerObject(scope, callback); + QScriptValue value = handler.property("callback"); + if (!jsVerify(handler.isObject() && value.isFunction(), + QString("jsBindCallback -- .callback is not a function (%1)").arg(value.toVariant().typeName()))) { + return QScriptValue(); + } + return handler; +} + +Promise AssetScriptingInterface::jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback) { + auto handler = jsBindCallback(scope, callback); + if (!jsVerify(handler.isValid(), "jsPromiseReady -- invalid callback handler")) { + return nullptr; + } + return promise->ready([this, handler](QString error, QVariantMap result) { + jsCallback(handler, error, result); + }); +} + void AssetScriptingInterface::jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result) { Q_ASSERT(thread() == QThread::currentThread()); @@ -208,46 +228,36 @@ void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, responseType = "text"; } auto asset = AssetUtils::getATPUrl(url).path(); - auto handler = makeScopedHandlerObject(scope, callback); - - JS_VERIFY(handler.property("callback").isFunction(), - QString("Invalid callback function (%1)").arg(handler.property("callback").toVariant().typeName())); JS_VERIFY(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset), QString("Invalid ATP url '%1'").arg(url)); JS_VERIFY(RESPONSE_TYPES.contains(responseType), QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); - Promise resolved = makePromise("resolved"); - Promise loaded = makePromise("loaded"); + Promise fetched = jsPromiseReady(makePromise("fetched"), scope, callback); + Promise mapped = makePromise("mapped"); - loaded->ready([=](QString error, QVariantMap result) { - qDebug() << "//loaded" << error; - jsCallback(handler, error, result); - }); - - resolved->ready([=](QString error, QVariantMap result) { - qDebug() << "//resolved" << result.value("hash"); + mapped->ready([=](QString error, QVariantMap result) { QString hash = result.value("hash").toString(); + QString url = result.value("url").toString(); if (!error.isEmpty() || !AssetUtils::isValidHash(hash)) { - loaded->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); + fetched->reject(error.isEmpty() ? "internal hash error: " + hash : error, result); } else { Promise promise = loadAsset(hash, decompress, responseType); promise->mixin(result); - promise->ready([this, loaded, hash](QString error, QVariantMap result) { - qDebug() << "//getAssetInfo/loadAsset" << error << hash; - loaded->resolve(NoError, result); + promise->ready([=](QString error, QVariantMap loadResult) { + loadResult["url"] = url; // maintain mapped .url in results (vs. atp:hash returned by loadAsset) + fetched->handle(error, loadResult); }); } }); if (AssetUtils::isValidHash(asset)) { - resolved->resolve(NoError, { { "hash", asset } }); - } else { - Promise promise = getAssetInfo(asset); - promise->ready([this, resolved](QString error, QVariantMap result) { - qDebug() << "//getAssetInfo" << error << result.value("hash") << result.value("path"); - resolved->resolve(error, result); + mapped->resolve({ + { "hash", asset }, + { "url", url }, }); + } else { + getAssetInfo(asset)->ready(mapped); } } @@ -256,128 +266,166 @@ void AssetScriptingInterface::resolveAsset(QScriptValue options, QScriptValue sc auto url = (options.isString() ? options : options.property(URL)).toString(); auto asset = AssetUtils::getATPUrl(url).path(); - auto handler = makeScopedHandlerObject(scope, callback); JS_VERIFY(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset), "expected options to be an asset URL or request options containing .url property"); - JS_VERIFY(handler.property("callback").isFunction(), "invalid callback function"); - getAssetInfo(asset)->ready([=](QString error, QVariantMap result) { - qDebug() << "//resolveAsset/getAssetInfo" << error << result.value("hash"); - jsCallback(handler, error, result); - }); + + jsPromiseReady(getAssetInfo(asset), scope, callback); } void AssetScriptingInterface::decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { auto data = options.property("data"); QByteArray dataByteArray = qscriptvalue_cast(data); - auto handler = makeScopedHandlerObject(scope, callback); auto responseType = options.property("responseType").toString().toLower(); if (responseType.isEmpty()) { responseType = "text"; } - Promise promise = decompressBytes(dataByteArray); - promise->ready([=](QString error, QVariantMap result) { - if (responseType == "arraybuffer") { - jsCallback(handler, error, result); - } else { - Promise promise = convertBytes(result.value("data").toByteArray(), responseType); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); - } - }); + Promise completed = jsPromiseReady(makePromise(__FUNCTION__), scope, callback); + Promise decompressed = decompressBytes(dataByteArray); + if (responseType == "arraybuffer") { + decompressed->ready(completed); + } else { + decompressed->ready([=](QString error, QVariantMap result) { + Promise converted = convertBytes(result.value("data").toByteArray(), responseType); + converted->mixin(result); + converted->ready(completed); + }); + } } namespace { const int32_t DEFAULT_GZIP_COMPRESSION_LEVEL = -1; const int32_t MAX_GZIP_COMPRESSION_LEVEL = 9; } - void AssetScriptingInterface::compressData(QScriptValue options, QScriptValue scope, QScriptValue callback) { - - auto data = options.property("data"); - QByteArray dataByteArray = data.isString() ? - data.toString().toUtf8() : - qscriptvalue_cast(data); - auto handler = makeScopedHandlerObject(scope, callback); - auto level = options.property("level").toInt32(); - if (level < DEFAULT_GZIP_COMPRESSION_LEVEL || level > MAX_GZIP_COMPRESSION_LEVEL) { - level = DEFAULT_GZIP_COMPRESSION_LEVEL; - } - Promise promise = compressBytes(dataByteArray, level); - promise->ready([=](QString error, QVariantMap result) { - jsCallback(handler, error, result); - }); + auto data = options.property("data").isValid() ? options.property("data") : options; + QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); + int level = options.property("level").isNumber() ? options.property("level").toInt32() : DEFAULT_GZIP_COMPRESSION_LEVEL; + JS_VERIFY(level >= DEFAULT_GZIP_COMPRESSION_LEVEL || level <= MAX_GZIP_COMPRESSION_LEVEL, QString("invalid .level %1").arg(level)); + jsPromiseReady(compressBytes(dataByteArray, level), scope, callback); } void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) { - auto compress = options.property("compress").toBool() || - options.property("compressed").toBool(); - auto handler = makeScopedHandlerObject(scope, callback); - auto data = options.property("data"); + auto compress = options.property("compress").toBool() || options.property("compressed").toBool(); + auto data = options.isObject() ? options.property("data") : options; auto rawPath = options.property("path").toString(); auto path = AssetUtils::getATPUrl(rawPath).path(); - QByteArray dataByteArray = data.isString() ? - data.toString().toUtf8() : - qscriptvalue_cast(data); + QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast(data); JS_VERIFY(path.isEmpty() || AssetUtils::isValidFilePath(path), QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path)); - JS_VERIFY(handler.property("callback").isFunction(), - "invalid callback function"); JS_VERIFY(dataByteArray.size() > 0, - QString("expected non-zero .data (got %1 / #%2 bytes)") - .arg(data.toVariant().typeName()) - .arg(dataByteArray.size())); + QString("expected non-zero .data (got %1 / #%2 bytes)").arg(data.toVariant().typeName()).arg(dataByteArray.size())); // [compressed] => uploaded to server => [mapped to path] Promise prepared = makePromise("putAsset::prepared"); Promise uploaded = makePromise("putAsset::uploaded"); - Promise finished = makePromise("putAsset::finished"); + Promise completed = makePromise("putAsset::completed"); + jsPromiseReady(completed, scope, callback); if (compress) { - qDebug() << "putAsset::compressBytes..."; - Promise promise = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL); - promise->finally([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::compressedBytes" << error << result.keys(); - prepared->handle(error, result); - }); + Promise compress = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL); + compress->ready(prepared); } else { - prepared->resolve(NoError, {{ "data", dataByteArray }}); + prepared->resolve({{ "data", dataByteArray }}); } - prepared->ready([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::prepared" << error << result.value("data").toByteArray().size() << result.keys(); - Promise promise = uploadBytes(result.value("data").toByteArray()); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - qDebug() << "===========//putAsset::prepared/uploadBytes" << error << result.keys(); - uploaded->handle(error, result); + prepared->fail(completed); + prepared->then([=](QVariantMap result) { + Promise upload = uploadBytes(result.value("data").toByteArray()); + upload->mixin(result); + upload->ready(uploaded); + }); + + uploaded->fail(completed); + if (path.isEmpty()) { + uploaded->then(completed); + } else { + uploaded->then([=](QVariantMap result) { + QString hash = result.value("hash").toString(); + if (!AssetUtils::isValidHash(hash)) { + completed->reject("path mapping requested, but did not receive valid hash", result); + } else { + Promise link = symlinkAsset(hash, path); + link->mixin(result); + link->ready(completed); + } }); - }); - - uploaded->ready([=](QString error, QVariantMap result) { - QString hash = result.value("hash").toString(); - qDebug() << "//putAsset::uploaded" << error << hash << result.keys(); - if (path.isEmpty()) { - finished->handle(error, result); - } else if (!AssetUtils::isValidHash(hash)) { - finished->reject("path mapping requested, but did not receive valid hash", result); - } else { - qDebug() << "symlinkAsset" << hash << path << QThread::currentThread(); - Promise promise = symlinkAsset(hash, path); - promise->mixin(result); - promise->ready([=](QString error, QVariantMap result) { - finished->handle(error, result); - qDebug() << "//symlinkAsset" << hash << path << result.keys(); - }); - } - }); - - finished->ready([=](QString error, QVariantMap result) { - qDebug() << "//putAsset::finished" << error << result.keys(); - jsCallback(handler, error, result); - }); + } +} + +void AssetScriptingInterface::queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback) { + QString url = options.isString() ? options.toString() : options.property("url").toString(); + JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url)); + jsPromiseReady(Parent::queryCacheMeta(url), scope, callback); +} + +void AssetScriptingInterface::loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback) { + QString url, responseType; + bool decompress = false; + if (options.isString()) { + url = options.toString(); + responseType = "text"; + } else { + url = options.property("url").toString(); + responseType = options.property("responseType").isValid() ? options.property("responseType").toString() : "text"; + decompress = options.property("decompress").toBool() || options.property("compressed").toBool(); + } + JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url)); + JS_VERIFY(RESPONSE_TYPES.contains(responseType), + QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "))); + + jsPromiseReady(Parent::loadFromCache(url, decompress, responseType), scope, callback); +} + +bool AssetScriptingInterface::canWriteCacheValue(const QUrl& url) { + auto scriptEngine = qobject_cast(engine()); + if (!scriptEngine) { + qCDebug(scriptengine) << __FUNCTION__ << "invalid script engine" << url; + return false; + } + // allow cache writes only from Client, EntityServer and Agent scripts + bool isAllowedContext = ( + scriptEngine->isClientScript() || + scriptEngine->isAgentScript() || + scriptEngine->isEntityServerScript() + ); + if (!isAllowedContext) { + qCDebug(scriptengine) << __FUNCTION__ << "invalid context" << scriptEngine->getContext() << url; + return false; + } + return true; +} + +void AssetScriptingInterface::saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback) { + JS_VERIFY(options.isObject(), QString("expected options object as first parameter not: %1").arg(options.toVariant().typeName())); + + QString url = options.property("url").toString(); + QByteArray data = qscriptvalue_cast(options.property("data")); + QVariantMap headers = qscriptvalue_cast(options.property("headers")); + + saveToCache(url, data, headers, scope, callback); +} + +void AssetScriptingInterface::saveToCache(const QUrl& rawURL, const QByteArray& data, const QVariantMap& metadata, QScriptValue scope, QScriptValue callback) { + QUrl url = rawURL; + if (url.path().isEmpty() && !data.isEmpty()) { + // generate a valid ATP URL from the data -- appending any existing fragment or querystring values + auto atpURL = AssetUtils::getATPUrl(hashDataHex(data)); + atpURL.setQuery(url.query()); + atpURL.setFragment(url.fragment()); + qCDebug(scriptengine) << "autogenerated ATP URL" << url << "=>" << atpURL; + url = atpURL; + } + auto hash = AssetUtils::extractAssetHash(url.toDisplayString()); + + JS_VERIFY(url.isValid(), QString("Invalid URL '%1'").arg(url.toString())); + JS_VERIFY(canWriteCacheValue(url), "Invalid cache write URL: " + url.toString()); + JS_VERIFY(url.scheme() == "atp" || url.scheme() == "cache", "only 'atp' and 'cache' URL schemes supported"); + JS_VERIFY(hash.isEmpty() || hash == hashDataHex(data), QString("invalid checksum hash for atp:HASH style URL (%1 != %2)").arg(hash, hashDataHex(data))); + + qCDebug(scriptengine) << "saveToCache" << url.toDisplayString() << data << hash << metadata; + + jsPromiseReady(Parent::saveToCache(url, data, metadata), scope, callback); } diff --git a/libraries/script-engine/src/AssetScriptingInterface.h b/libraries/script-engine/src/AssetScriptingInterface.h index 7001c64634..fdce173dfe 100644 --- a/libraries/script-engine/src/AssetScriptingInterface.h +++ b/libraries/script-engine/src/AssetScriptingInterface.h @@ -21,6 +21,7 @@ #include #include #include +#include #include /**jsdoc @@ -29,6 +30,7 @@ class AssetScriptingInterface : public BaseAssetScriptingInterface, QScriptable { Q_OBJECT public: + using Parent = BaseAssetScriptingInterface; AssetScriptingInterface(QObject* parent = nullptr); /**jsdoc @@ -102,26 +104,91 @@ public: Q_INVOKABLE void sendFakedHandshake(); #endif - // Advanced APIs - // getAsset(options, scope[callback(error, result)]) -- fetches an Asset from the Server - // [options.url] an "atp:" style URL, hash, or relative mapped path to fetch - // [options.responseType] the desired reponse type (text | arraybuffer | json) - // [options.decompress] whether to apply gunzip decompression on the stream - // [scope[callback]] continuation-style (error, { responseType, data, byteLength, ... }) callback - const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" }; + /**jsdoc + * Request Asset data from the ATP Server + * @function Assets.getAsset + * @param {URL|Assets.GetOptions} options An atp: style URL, hash, or relative mapped path; or an {@link Assets.GetOptions} object with request parameters + * @param {Assets~getAssetCallback} scope[callback] A scope callback function to receive (error, results) values + */ + /**jsdoc + * A set of properties that can be passed to {@link Assets.getAsset}. + * @typedef {Object} Assets.GetOptions + * @property {URL} [url] an "atp:" style URL, hash, or relative mapped path to fetch + * @property {string} [responseType=text] the desired reponse type (text | arraybuffer | json) + * @property {boolean} [decompress=false] whether to attempt gunzip decompression on the fetched data + * See: {@link Assets.putAsset} and its .compress=true option + */ + /**jsdoc + * Called when Assets.getAsset is complete. + * @callback Assets~getAssetCallback + * @param {string} error - contains error message or null value if no error occured fetching the asset + * @param {Asset~getAssetResult} result - result object containing, on success containing asset metadata and contents + */ + /**jsdoc + * Result value returned by {@link Assets.getAsset}. + * @typedef {Object} Assets~getAssetResult + * @property {url} [url] the resolved "atp:" style URL for the fetched asset + * @property {string} [hash] the resolved hash for the fetched asset + * @property {string|ArrayBuffer|Object} [response] response data (possibly converted per .responseType value) + * @property {string} [responseType] response type (text | arraybuffer | json) + * @property {string} [contentType] detected asset mime-type (autodetected) + * @property {number} [byteLength] response data size in bytes + * @property {number} [decompressed] flag indicating whether data was decompressed + */ Q_INVOKABLE void getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); - // putAsset(options, scope[callback(error, result)]) -- upload a new Aset to the Server - // [options.data] -- (ArrayBuffer|String) - // [options.compress] -- (true|false) - // [options.path=undefined] -- option path mapping to set on the created hash result - // [ + + /**jsdoc + * Upload Asset data to the ATP Server + * @function Assets.putAsset + * @param {Assets.PutOptions} options A PutOptions object with upload parameters + * @param {Assets~putAssetCallback} scope[callback] A scoped callback function invoked with (error, results) + */ + /**jsdoc + * A set of properties that can be passed to {@link Assets.putAsset}. + * @typedef {Object} Assets.PutOptions + * @property {ArrayBuffer|string} [data] byte buffer or string value representing the new asset's content + * @property {string} [path=null] ATP path mapping to automatically create (upon successful upload to hash) + * @property {boolean} [compress=false] whether to gzip compress data before uploading + */ + /**jsdoc + * Called when Assets.putAsset is complete. + * @callback Assets~puttAssetCallback + * @param {string} error - contains error message (or null value if no error occured while uploading/mapping the new asset) + * @param {Asset~putAssetResult} result - result object containing error or result status of asset upload + */ + /**jsdoc + * Result value returned by {@link Assets.putAsset}. + * @typedef {Object} Assets~putAssetResult + * @property {url} [url] the resolved "atp:" style URL for the uploaded asset (based on .path if specified, otherwise on the resulting ATP hash) + * @property {string} [path] the uploaded asset's resulting ATP path (or undefined if no path mapping was assigned) + * @property {string} [hash] the uploaded asset's resulting ATP hash + * @property {boolean} [compressed] flag indicating whether the data was compressed before upload + * @property {number} [byteLength] flag indicating final byte size of the data uploaded to the ATP server + */ Q_INVOKABLE void putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); Q_INVOKABLE void compressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE bool initializeCache() { return Parent::initializeCache(); } + + Q_INVOKABLE bool canWriteCacheValue(const QUrl& url); + + Q_INVOKABLE void getCacheStatus(QScriptValue scope, QScriptValue callback = QScriptValue()) { + jsPromiseReady(Parent::getCacheStatus(), scope, callback); + } + + Q_INVOKABLE void queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue()); + Q_INVOKABLE void saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata, + QScriptValue scope, QScriptValue callback = QScriptValue()); protected: + QScriptValue jsBindCallback(QScriptValue scope, QScriptValue callback = QScriptValue()); + Promise jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback = QScriptValue()); + void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QVariantMap& result); void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result); bool jsVerify(bool condition, const QString& error); diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index c7034adf35..65ef2025d9 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -56,6 +56,7 @@ #include #include "ArrayBufferViewClass.h" +#include "AssetScriptingInterface.h" #include "BatchLoader.h" #include "BaseScriptEngine.h" #include "DataViewClass.h" @@ -175,6 +176,7 @@ ScriptEngine::ScriptEngine(Context context, const QString& scriptContents, const _timerFunctionMap(), _fileNameString(fileNameString), _arrayBufferClass(new ArrayBufferClass(this)), + _assetScriptingInterface(new AssetScriptingInterface(this)), // don't delete `ScriptEngines` until all `ScriptEngine`s are gone _scriptEngines(DependencyManager::get()) { @@ -704,7 +706,7 @@ void ScriptEngine::init() { // constants globalObject().setProperty("TREE_SCALE", newVariant(QVariant(TREE_SCALE))); - registerGlobalObject("Assets", &_assetScriptingInterface); + registerGlobalObject("Assets", _assetScriptingInterface); registerGlobalObject("Resources", DependencyManager::get().data()); registerGlobalObject("DebugDraw", &DebugDraw::getInstance()); diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index 17c0e0713a..7f69eee990 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -321,7 +321,7 @@ protected: ArrayBufferClass* _arrayBufferClass; - AssetScriptingInterface _assetScriptingInterface{ this }; + AssetScriptingInterface* _assetScriptingInterface; std::function _emitScriptUpdates{ []() { return true; } }; diff --git a/libraries/shared/src/BaseScriptEngine.cpp b/libraries/shared/src/BaseScriptEngine.cpp index c92d629b75..22ae01d72f 100644 --- a/libraries/shared/src/BaseScriptEngine.cpp +++ b/libraries/shared/src/BaseScriptEngine.cpp @@ -325,6 +325,12 @@ QScriptValue makeScopedHandlerObject(QScriptValue scopeOrCallback, QScriptValue } else if (methodOrName.isFunction()) { scope = scopeOrCallback; callback = methodOrName; + } else if (!methodOrName.isValid()) { + // instantiate from an existing scoped handler object + if (scopeOrCallback.property("callback").isFunction()) { + scope = scopeOrCallback.property("scope"); + callback = scopeOrCallback.property("callback"); + } } } auto handler = engine->newObject(); diff --git a/libraries/shared/src/shared/MiniPromises.cpp b/libraries/shared/src/shared/MiniPromises.cpp index faada3627a..bb78852c29 100644 --- a/libraries/shared/src/shared/MiniPromises.cpp +++ b/libraries/shared/src/shared/MiniPromises.cpp @@ -7,4 +7,21 @@ // #include "MiniPromises.h" +#include +#include + int MiniPromise::metaTypeID = qRegisterMetaType("MiniPromise::Promise"); + +namespace { + void promiseFromScriptValue(const QScriptValue& object, MiniPromise::Promise& promise) { + Q_ASSERT(false); + } + QScriptValue promiseToScriptValue(QScriptEngine *engine, const MiniPromise::Promise& promise) { + return engine->newQObject(promise.get()); + } +} +void MiniPromise::registerMetaTypes(QObject* engine) { + auto scriptEngine = qobject_cast(engine); + qDebug() << "----------------------- MiniPromise::registerMetaTypes ------------" << scriptEngine; + qScriptRegisterMetaType(scriptEngine, promiseToScriptValue, promiseFromScriptValue); +} diff --git a/libraries/shared/src/shared/MiniPromises.h b/libraries/shared/src/shared/MiniPromises.h index 2f17760aa8..5983f135b7 100644 --- a/libraries/shared/src/shared/MiniPromises.h +++ b/libraries/shared/src/shared/MiniPromises.h @@ -32,6 +32,9 @@ class MiniPromise : public QObject, public std::enable_shared_from_this, public ReadWriteLockable { Q_OBJECT + Q_PROPERTY(QString state READ getStateString) + Q_PROPERTY(QString error READ getError) + Q_PROPERTY(QVariantMap result READ getResult) public: using HandlerFunction = std::function; using SuccessFunction = std::function; @@ -39,23 +42,25 @@ public: using HandlerFunctions = QVector; using Promise = std::shared_ptr; + static void registerMetaTypes(QObject* engine); static int metaTypeID; MiniPromise() {} MiniPromise(const QString debugName) { setObjectName(debugName); } ~MiniPromise() { - if (!_rejected && !_resolved) { - qWarning() << "MiniPromise::~MiniPromise -- destroying unhandled promise:" << objectName() << _error << _result; + if (getStateString() == "pending") { + qWarning() << "MiniPromise::~MiniPromise -- destroying pending promise:" << objectName() << _error << _result << "handlers:" << getPendingHandlerCount(); } } Promise self() { return shared_from_this(); } - Q_INVOKABLE void executeOnPromiseThread(std::function function) { + Q_INVOKABLE void executeOnPromiseThread(std::function function, MiniPromise::Promise root = nullptr) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod( this, "executeOnPromiseThread", Qt::QueuedConnection, - Q_ARG(std::function, function)); + Q_ARG(std::function, function), + Q_ARG(MiniPromise::Promise, self())); } else { function(); } @@ -92,9 +97,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - always(_error, _result); - }); + always(getError(), getResult()); }); } return self(); @@ -112,9 +115,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - failFunc(_error, _result); - }); + failFunc(getError(), getResult()); }); } return self(); @@ -132,9 +133,7 @@ public: }); } else { executeOnPromiseThread([&]{ - withReadLock([&]{ - successFunc(_error, _result); - }); + successFunc(getError(), getResult()); }); } return self(); @@ -151,6 +150,26 @@ public: return self(); } + + // helper functions for forwarding results on to a next Promise + Promise ready(Promise next) { return finally(next); } + Promise finally(Promise next) { + return finally([next](QString error, QVariantMap result) { + next->handle(error, result); + }); + } + Promise fail(Promise next) { + return fail([next](QString error, QVariantMap result) { + next->reject(error, result); + }); + } + Promise then(Promise next) { + return then([next](QString error, QVariantMap result) { + next->resolve(error, result); + }); + } + + // trigger methods // handle() automatically resolves or rejects the promise (based on whether an error value occurred) Promise handle(QString error, const QVariantMap& result) { @@ -168,17 +187,15 @@ public: Promise resolve(QString error, const QVariantMap& result) { setState(true, error, result); - QString localError; - QVariantMap localResult; - HandlerFunctions resolveHandlers; - HandlerFunctions finallyHandlers; - withReadLock([&]{ - localError = _error; - localResult = _result; - resolveHandlers = _onresolve; - finallyHandlers = _onfinally; - }); executeOnPromiseThread([&]{ + const QString localError{ getError() }; + const QVariantMap localResult{ getResult() }; + HandlerFunctions resolveHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + resolveHandlers = _onresolve; + finallyHandlers = _onfinally; + }); for (const auto& onresolve : resolveHandlers) { onresolve(localError, localResult); } @@ -195,17 +212,15 @@ public: Promise reject(QString error, const QVariantMap& result) { setState(false, error, result); - QString localError; - QVariantMap localResult; - HandlerFunctions rejectHandlers; - HandlerFunctions finallyHandlers; - withReadLock([&]{ - localError = _error; - localResult = _result; - rejectHandlers = _onreject; - finallyHandlers = _onfinally; - }); executeOnPromiseThread([&]{ + const QString localError{ getError() }; + const QVariantMap localResult{ getResult() }; + HandlerFunctions rejectHandlers; + HandlerFunctions finallyHandlers; + withReadLock([&]{ + rejectHandlers = _onreject; + finallyHandlers = _onfinally; + }); for (const auto& onreject : rejectHandlers) { onreject(localError, localResult); } @@ -224,13 +239,25 @@ private: } else { _rejected = true; } - withWriteLock([&]{ - _error = error; - }); + setError(error); assignResult(result); return self(); } + void setError(const QString error) { withWriteLock([&]{ _error = error; }); } + QString getError() const { return resultWithReadLock([this]() -> QString { return _error; }); } + QVariantMap getResult() const { return resultWithReadLock([this]() -> QVariantMap { return _result; }); } + int getPendingHandlerCount() const { + return resultWithReadLock([this]() -> int { + return _onresolve.size() + _onreject.size() + _onfinally.size(); + }); + } + QString getStateString() const { + return _rejected ? "rejected" : + _resolved ? "resolved" : + getPendingHandlerCount() ? "pending" : + "unknown"; + } QString _error; QVariantMap _result; std::atomic _rejected{false}; @@ -240,8 +267,12 @@ private: HandlerFunctions _onfinally; }; +Q_DECLARE_METATYPE(MiniPromise::Promise) + inline MiniPromise::Promise makePromise(const QString& hint = QString()) { + if (!QMetaType::isRegistered(qMetaTypeId())) { + int type = qRegisterMetaType(); + qDebug() << "makePromise -- registered MetaType:" << type; + } return std::make_shared(hint); } - -Q_DECLARE_METATYPE(MiniPromise::Promise)