Mirrors/overte-thingvellir
3
0
Fork 0
mirror of https://thingvellir.net/git/overte synced 2025-03-27 23:52:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #12036 from humbletim/Leopoly_Phase1_003_Asset-Request

Browse source 
Leopoly_Phase1_003_Asset-Request
This commit is contained in:
MiladNazeri 2018-01-29 11:40:41 -08:00 committed by GitHub
commit 9baf02b7a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 2218 additions and 334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
assignment-client/src/assets/AssetServer.cpp
View file

@ -39,7 +39,6 @@
#include "SendAssetTask.h"
#include "UploadAssetTask.h"
static const uint8_t MIN_CORES_FOR_MULTICORE = 4;
static const uint8_t CPU_AFFINITY_COUNT_HIGH = 2;
static const uint8_t CPU_AFFINITY_COUNT_LOW = 1;
@ -56,7 +55,7 @@ static const QString BAKED_MODEL_SIMPLE_NAME = "asset.fbx";
static const QString BAKED_TEXTURE_SIMPLE_NAME = "texture.ktx";
static const QString BAKED_SCRIPT_SIMPLE_NAME = "asset.js";
void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) {
void AssetServer::bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) {
qDebug() << "Starting bake for: " << assetPath << assetHash;
auto it = _pendingBakes.find(assetHash);
if (it == _pendingBakes.end()) {
@ -74,23 +73,23 @@ void AssetServer::bakeAsset(const AssetHash& assetHash, const AssetPath& assetPa
}
}
QString AssetServer::getPathToAssetHash(const AssetHash& assetHash) {
QString AssetServer::getPathToAssetHash(const AssetUtils::AssetHash& assetHash) {
return _filesDirectory.absoluteFilePath(assetHash);
}
std::pair<BakingStatus, QString> AssetServer::getAssetStatus(const AssetPath& path, const AssetHash& hash) {
std::pair<AssetUtils::BakingStatus, QString> AssetServer::getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) {
auto it = _pendingBakes.find(hash);
if (it != _pendingBakes.end()) {
return { (*it)->isBaking() ? Baking : Pending, "" };
return { (*it)->isBaking() ? AssetUtils::Baking : AssetUtils::Pending, "" };
}
if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
return { Baked, "" };
if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
return { AssetUtils::Baked, "" };
}
auto dotIndex = path.lastIndexOf(".");
if (dotIndex == -1) {
return { Irrelevant, "" };
return { AssetUtils::Irrelevant, "" };
}
auto extension = path.mid(dotIndex + 1);
@ -104,16 +103,16 @@ std::pair<BakingStatus, QString> AssetServer::getAssetStatus(const AssetPath& pa
} else if (BAKEABLE_SCRIPT_EXTENSIONS.contains(extension)) {
bakedFilename = BAKED_SCRIPT_SIMPLE_NAME;
} else {
return { Irrelevant, "" };
return { AssetUtils::Irrelevant, "" };
}
auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename;
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + bakedFilename;
auto jt = _fileMappings.find(bakedPath);
if (jt != _fileMappings.end()) {
if (jt->second == hash) {
return { NotBaked, "" };
return { AssetUtils::NotBaked, "" };
} else {
return { Baked, "" };
return { AssetUtils::Baked, "" };
}
} else {
bool loaded;
@ -121,11 +120,11 @@ std::pair<BakingStatus, QString> AssetServer::getAssetStatus(const AssetPath& pa
std::tie(loaded, meta) = readMetaFile(hash);
if (loaded && meta.failedLastBake) {
return { Error, meta.lastBakeErrors };
return { AssetUtils::Error, meta.lastBakeErrors };
}
}
return { Pending, "" };
return { AssetUtils::Pending, "" };
}
void AssetServer::bakeAssets() {
@ -137,14 +136,14 @@ void AssetServer::bakeAssets() {
}
}
void AssetServer::maybeBake(const AssetPath& path, const AssetHash& hash) {
void AssetServer::maybeBake(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) {
if (needsToBeBaked(path, hash)) {
qDebug() << "Queuing bake of: " << path;
bakeAsset(hash, path, getPathToAssetHash(hash));
}
}
void AssetServer::createEmptyMetaFile(const AssetHash& hash) {
void AssetServer::createEmptyMetaFile(const AssetUtils::AssetHash& hash) {
QString metaFilePath = "atp:/" + hash + "/meta.json";
QFile metaFile { metaFilePath };
@ -157,14 +156,14 @@ void AssetServer::createEmptyMetaFile(const AssetHash& hash) {
}
}
bool AssetServer::hasMetaFile(const AssetHash& hash) {
QString metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json";
bool AssetServer::hasMetaFile(const AssetUtils::AssetHash& hash) {
QString metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/meta.json";
return _fileMappings.find(metaFilePath) != _fileMappings.end();
}
bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHash) {
if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
bool AssetServer::needsToBeBaked(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& assetHash) {
if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
return false;
}
@ -196,7 +195,7 @@ bool AssetServer::needsToBeBaked(const AssetPath& path, const AssetHash& assetHa
return false;
}
auto bakedPath = HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
auto bakedPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + assetHash + "/" + bakedFilename;
return _fileMappings.find(bakedPath) == _fileMappings.end();
}
@ -235,7 +234,7 @@ AssetServer::AssetServer(ReceivedMessage& message) :
ThreadedAssignment(message),
_transferTaskPool(this),
_bakingTaskPool(this),
_filesizeLimit(MAX_UPLOAD_SIZE)
_filesizeLimit(AssetUtils::MAX_UPLOAD_SIZE)
{
// store the current state of image compression so we can reset it when this assignment is complete
_wasColorTextureCompressionEnabled = image::isColorTexturesCompressionEnabled();
@ -390,7 +389,7 @@ void AssetServer::completeSetup() {
// Check the asset directory to output some information about what we have
auto files = _filesDirectory.entryList(QDir::Files);
QRegExp hashFileRegex { ASSET_HASH_REGEX_STRING };
QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING };
auto hashedFiles = files.filter(hashFileRegex);
qCInfo(asset_server) << "There are" << hashedFiles.size() << "asset files in the asset directory.";
@ -410,9 +409,9 @@ void AssetServer::completeSetup() {
// get file size limit for an asset
static const QString ASSETS_FILESIZE_LIMIT_OPTION = "assets_filesize_limit";
auto assetsFilesizeLimitJSONValue = assetServerObject[ASSETS_FILESIZE_LIMIT_OPTION];
auto assetsFilesizeLimit = (uint64_t)assetsFilesizeLimitJSONValue.toInt(MAX_UPLOAD_SIZE);
auto assetsFilesizeLimit = (uint64_t)assetsFilesizeLimitJSONValue.toInt(AssetUtils::MAX_UPLOAD_SIZE);
if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < MAX_UPLOAD_SIZE) {
if (assetsFilesizeLimit != 0 && assetsFilesizeLimit < AssetUtils::MAX_UPLOAD_SIZE) {
_filesizeLimit = assetsFilesizeLimit * BITS_PER_MEGABITS;
}
@ -421,7 +420,7 @@ void AssetServer::completeSetup() {
}
void AssetServer::cleanupUnmappedFiles() {
QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}" };
QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(AssetUtils::SHA256_HASH_HEX_LENGTH) + "}" };
auto files = _filesDirectory.entryInfoList(QDir::Files);
@ -454,6 +453,8 @@ void AssetServer::cleanupUnmappedFiles() {
}
void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
using AssetMappingOperationType = AssetUtils::AssetMappingOperationType;
MessageID messageID;
message->readPrimitive(&messageID);
@ -519,7 +520,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode
if (!bakedRootFile.isEmpty()) {
// we ran into an asset for which we could have a baked version, let's check if it's ready
bakedAssetPath = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile;
bakedAssetPath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + bakedRootFile;
auto bakedIt = _fileMappings.find(bakedAssetPath);
if (bakedIt != _fileMappings.end()) {
@ -537,7 +538,7 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode
}
}
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
if (wasRedirected) {
qDebug() << "Writing re-directed hash for" << originalAssetHash << "to" << redirectedAssetHash;
@ -563,12 +564,12 @@ void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNode
}
}
} else {
replyPacket.writePrimitive(AssetServerError::AssetNotFound);
replyPacket.writePrimitive(AssetUtils::AssetServerError::AssetNotFound);
}
}
void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
uint32_t count = (uint32_t)_fileMappings.size();
@ -580,11 +581,11 @@ void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedN
replyPacket.writeString(mapping);
replyPacket.write(QByteArray::fromHex(hash.toUtf8()));
BakingStatus status;
AssetUtils::BakingStatus status;
QString lastBakeErrors;
std::tie(status, lastBakeErrors) = getAssetStatus(mapping, hash);
replyPacket.writePrimitive(status);
if (status == Error) {
if (status == AssetUtils::Error) {
replyPacket.writeString(lastBakeErrors);
}
}
@ -594,22 +595,22 @@ void AssetServer::handleSetMappingOperation(ReceivedMessage& message, SharedNode
if (senderNode->getCanWriteToAssetServer()) {
QString assetPath = message.readString();
auto assetHash = message.read(SHA256_HASH_LENGTH).toHex();
auto assetHash = message.read(AssetUtils::SHA256_HASH_LENGTH).toHex();
// don't process a set mapping operation that is inside the hidden baked folder
if (assetPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
qCDebug(asset_server) << "Refusing to process a set mapping operation inside" << HIDDEN_BAKED_CONTENT_FOLDER;
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
if (assetPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
qCDebug(asset_server) << "Refusing to process a set mapping operation inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER;
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
} else {
if (setMapping(assetPath, assetHash)) {
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed);
}
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
}
}
@ -623,21 +624,21 @@ void AssetServer::handleDeleteMappingsOperation(ReceivedMessage& message, Shared
for (int i = 0; i < numberOfDeletedMappings; ++i) {
auto mapping = message.readString();
if (!mapping.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
if (!mapping.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
mappingsToDelete << mapping;
} else {
qCDebug(asset_server) << "Refusing to delete mapping" << mapping
<< "that is inside" << HIDDEN_BAKED_CONTENT_FOLDER;
<< "that is inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER;
}
}
if (deleteMappings(mappingsToDelete)) {
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed);
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
}
}
@ -646,20 +647,20 @@ void AssetServer::handleRenameMappingOperation(ReceivedMessage& message, SharedN
QString oldPath = message.readString();
QString newPath = message.readString();
if (oldPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER) || newPath.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
if (oldPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER) || newPath.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
qCDebug(asset_server) << "Cannot rename" << oldPath << "to" << newPath
<< "since one of the paths is inside" << HIDDEN_BAKED_CONTENT_FOLDER;
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
<< "since one of the paths is inside" << AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER;
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
} else {
if (renameMapping(oldPath, newPath)) {
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed);
}
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
}
}
@ -678,12 +679,12 @@ void AssetServer::handleSetBakingEnabledOperation(ReceivedMessage& message, Shar
}
if (setBakingEnabled(mappings, enabled)) {
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.writePrimitive(AssetUtils::AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
replyPacket.writePrimitive(AssetUtils::AssetServerError::MappingOperationFailed);
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
replyPacket.writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
}
}
@ -691,15 +692,15 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, Sh
QByteArray assetHash;
MessageID messageID;
if (message->getSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID))) {
if (message->getSize() < qint64(AssetUtils::SHA256_HASH_LENGTH + sizeof(messageID))) {
qCDebug(asset_server) << "ERROR bad file request";
return;
}
message->readPrimitive(&messageID);
assetHash = message->readWithoutCopy(SHA256_HASH_LENGTH);
assetHash = message->readWithoutCopy(AssetUtils::SHA256_HASH_LENGTH);
auto size = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(AssetServerError) + sizeof(qint64));
auto size = qint64(sizeof(MessageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(AssetUtils::AssetServerError) + sizeof(qint64));
auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply, size, true);
QByteArray hexHash = assetHash.toHex();
@ -712,11 +713,11 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, Sh
if (fileInfo.exists() && fileInfo.isReadable()) {
qCDebug(asset_server) << "Opening file: " << fileInfo.filePath();
replyPacket->writePrimitive(AssetServerError::NoError);
replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError);
replyPacket->writePrimitive(fileInfo.size());
} else {
qCDebug(asset_server) << "Asset not found: " << QString(hexHash);
replyPacket->writePrimitive(AssetServerError::AssetNotFound);
replyPacket->writePrimitive(AssetUtils::AssetServerError::AssetNotFound);
}
auto nodeList = DependencyManager::get<NodeList>();
@ -725,7 +726,7 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, Sh
void AssetServer::handleAssetGet(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
auto minSize = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(DataOffset) + sizeof(DataOffset));
auto minSize = qint64(sizeof(MessageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(AssetUtils::DataOffset) + sizeof(AssetUtils::DataOffset));
if (message->getSize() < minSize) {
qCDebug(asset_server) << "ERROR bad file request";
@ -749,14 +750,14 @@ void AssetServer::handleAssetUpload(QSharedPointer<ReceivedMessage> message, Sha
// for now this also means it isn't allowed to add assets
// so return a packet with error that indicates that
auto permissionErrorPacket = NLPacket::create(PacketType::AssetUploadReply, sizeof(MessageID) + sizeof(AssetServerError), true);
auto permissionErrorPacket = NLPacket::create(PacketType::AssetUploadReply, sizeof(MessageID) + sizeof(AssetUtils::AssetServerError), true);
MessageID messageID;
message->readPrimitive(&messageID);
// write the message ID and a permission denied error
permissionErrorPacket->writePrimitive(messageID);
permissionErrorPacket->writePrimitive(AssetServerError::PermissionDenied);
permissionErrorPacket->writePrimitive(AssetUtils::AssetServerError::PermissionDenied);
// send off the packet
auto nodeList = DependencyManager::get<NodeList>();
@ -863,12 +864,12 @@ bool AssetServer::loadMappingsFromFile() {
continue;
}
if (!isValidFilePath(key)) {
if (!AssetUtils::isValidFilePath(key)) {
qCWarning(asset_server) << "Will not keep mapping for" << key << "since it is not a valid path.";
continue;
}
if (!isValidHash(value.toString())) {
if (!AssetUtils::isValidHash(value.toString())) {
qCWarning(asset_server) << "Will not keep mapping for" << key << "since it does not have a valid hash.";
continue;
}
@ -918,15 +919,15 @@ bool AssetServer::writeMappingsToFile() {
return false;
}
bool AssetServer::setMapping(AssetPath path, AssetHash hash) {
bool AssetServer::setMapping(AssetUtils::AssetPath path, AssetUtils::AssetHash hash) {
path = path.trimmed();
if (!isValidFilePath(path)) {
if (!AssetUtils::isValidFilePath(path)) {
qCWarning(asset_server) << "Cannot set a mapping for invalid path:" << path << "=>" << hash;
return false;
}
if (!isValidHash(hash)) {
if (!AssetUtils::isValidHash(hash)) {
qCWarning(asset_server) << "Cannot set a mapping for invalid hash" << path << "=>" << hash;
return false;
}
@ -958,23 +959,23 @@ bool AssetServer::setMapping(AssetPath path, AssetHash hash) {
}
}
bool pathIsFolder(const AssetPath& path) {
bool pathIsFolder(const AssetUtils::AssetPath& path) {
return path.endsWith('/');
}
void AssetServer::removeBakedPathsForDeletedAsset(AssetHash hash) {
void AssetServer::removeBakedPathsForDeletedAsset(AssetUtils::AssetHash hash) {
// we deleted the file with this hash
// check if we had baked content for that file that should also now be removed
// by calling deleteMappings for the hidden baked content folder for this hash
AssetPathList hiddenBakedFolder { HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" };
AssetUtils::AssetPathList hiddenBakedFolder { AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" };
qCDebug(asset_server) << "Deleting baked content below" << hiddenBakedFolder << "since" << hash << "was deleted";
deleteMappings(hiddenBakedFolder);
}
bool AssetServer::deleteMappings(const AssetPathList& paths) {
bool AssetServer::deleteMappings(const AssetUtils::AssetPathList& paths) {
// take a copy of the current mappings in case persistence of these deletes fails
auto oldMappings = _fileMappings;
@ -1060,11 +1061,11 @@ bool AssetServer::deleteMappings(const AssetPathList& paths) {
}
}
bool AssetServer::renameMapping(AssetPath oldPath, AssetPath newPath) {
bool AssetServer::renameMapping(AssetUtils::AssetPath oldPath, AssetUtils::AssetPath newPath) {
oldPath = oldPath.trimmed();
newPath = newPath.trimmed();
if (!isValidFilePath(oldPath) || !isValidFilePath(newPath)) {
if (!AssetUtils::isValidFilePath(oldPath) || !AssetUtils::isValidFilePath(newPath)) {
qCWarning(asset_server) << "Cannot perform rename with invalid paths - both should have leading forward and no ending slashes:"
<< oldPath << "=>" << newPath;
@ -1164,8 +1165,8 @@ static const QString BAKED_ASSET_SIMPLE_FBX_NAME = "asset.fbx";
static const QString BAKED_ASSET_SIMPLE_TEXTURE_NAME = "texture.ktx";
static const QString BAKED_ASSET_SIMPLE_JS_NAME = "asset.js";
QString getBakeMapping(const AssetHash& hash, const QString& relativeFilePath) {
return HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath;
QString getBakeMapping(const AssetUtils::AssetHash& hash, const QString& relativeFilePath) {
return AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + relativeFilePath;
}
void AssetServer::handleFailedBake(QString originalAssetHash, QString assetPath, QString errors) {
@ -1197,7 +1198,7 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
qDebug() << "File path: " << filePath;
AssetHash bakedFileHash;
AssetUtils::AssetHash bakedFileHash;
if (file.open(QIODevice::ReadOnly)) {
QCryptographicHash hasher(QCryptographicHash::Sha256);
@ -1290,8 +1291,8 @@ static const QString BAKE_VERSION_KEY = "bake_version";
static const QString FAILED_LAST_BAKE_KEY = "failed_last_bake";
static const QString LAST_BAKE_ERRORS_KEY = "last_bake_errors";
std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetHash hash) {
auto metaFilePath = HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json";
std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetUtils::AssetHash hash) {
auto metaFilePath = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + hash + "/" + "meta.json";
auto it = _fileMappings.find(metaFilePath);
if (it == _fileMappings.end()) {
@ -1335,7 +1336,7 @@ std::pair<bool, AssetMeta> AssetServer::readMetaFile(AssetHash hash) {
return { false, {} };
}
bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta) {
bool AssetServer::writeMetaFile(AssetUtils::AssetHash originalAssetHash, const AssetMeta& meta) {
// construct the JSON that will be in the meta file
QJsonObject metaFileObject;
@ -1349,7 +1350,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me
auto metaFileJSON = metaFileDoc.toJson();
// get a hash for the contents of the meta-file
AssetHash metaFileHash = QCryptographicHash::hash(metaFileJSON, QCryptographicHash::Sha256).toHex();
AssetUtils::AssetHash metaFileHash = QCryptographicHash::hash(metaFileJSON, QCryptographicHash::Sha256).toHex();
// create the meta file in our files folder, named by the hash of its contents
QFile metaFile(_filesDirectory.absoluteFilePath(metaFileHash));
@ -1359,7 +1360,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me
metaFile.close();
// add a mapping to the meta file so it doesn't get deleted because it is unmapped
auto metaFileMapping = HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + "meta.json";
auto metaFileMapping = AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER + originalAssetHash + "/" + "meta.json";
return setMapping(metaFileMapping, metaFileHash);
} else {
@ -1367,7 +1368,7 @@ bool AssetServer::writeMetaFile(AssetHash originalAssetHash, const AssetMeta& me
}
}
bool AssetServer::setBakingEnabled(const AssetPathList& paths, bool enabled) {
bool AssetServer::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled) {
for (const auto& path : paths) {
auto it = _fileMappings.find(path);
if (it != _fileMappings.end()) {

31
assignment-client/src/assets/AssetServer.h
View file

@ -21,7 +21,6 @@
#include "AssetUtils.h"
#include "ReceivedMessage.h"
namespace std {
template <>
struct hash<QString> {
@ -75,29 +74,29 @@ private:
bool writeMappingsToFile();
/// Set the mapping for path to hash
bool setMapping(AssetPath path, AssetHash hash);
bool setMapping(AssetUtils::AssetPath path, AssetUtils::AssetHash hash);
/// Delete mapping `path`. Returns `true` if deletion of mappings succeeds, else `false`.
bool deleteMappings(const AssetPathList& paths);
bool deleteMappings(const AssetUtils::AssetPathList& paths);
/// Rename mapping from `oldPath` to `newPath`. Returns true if successful
bool renameMapping(AssetPath oldPath, AssetPath newPath);
bool renameMapping(AssetUtils::AssetPath oldPath, AssetUtils::AssetPath newPath);
bool setBakingEnabled(const AssetPathList& paths, bool enabled);
bool setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled);
/// Delete any unmapped files from the local asset directory
void cleanupUnmappedFiles();
QString getPathToAssetHash(const AssetHash& assetHash);
QString getPathToAssetHash(const AssetUtils::AssetHash& assetHash);
std::pair<BakingStatus, QString> getAssetStatus(const AssetPath& path, const AssetHash& hash);
std::pair<AssetUtils::BakingStatus, QString> getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
void bakeAssets();
void maybeBake(const AssetPath& path, const AssetHash& hash);
void createEmptyMetaFile(const AssetHash& hash);
bool hasMetaFile(const AssetHash& hash);
bool needsToBeBaked(const AssetPath& path, const AssetHash& assetHash);
void bakeAsset(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath);
void maybeBake(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
void createEmptyMetaFile(const AssetUtils::AssetHash& hash);
bool hasMetaFile(const AssetUtils::AssetHash& hash);
bool needsToBeBaked(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& assetHash);
void bakeAsset(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
/// Move baked content for asset to baked directory and update baked status
void handleCompletedBake(QString originalAssetHash, QString assetPath, QString bakedTempOutputDir,
@ -106,11 +105,11 @@ private:
void handleAbortedBake(QString originalAssetHash, QString assetPath);
/// Create meta file to describe baked content for original asset
std::pair<bool, AssetMeta> readMetaFile(AssetHash hash);
bool writeMetaFile(AssetHash originalAssetHash, const AssetMeta& meta = AssetMeta());
std::pair<bool, AssetMeta> readMetaFile(AssetUtils::AssetHash hash);
bool writeMetaFile(AssetUtils::AssetHash originalAssetHash, const AssetMeta& meta = AssetMeta());
/// Remove baked paths when the original asset is deleteds
void removeBakedPathsForDeletedAsset(AssetHash originalAssetHash);
void removeBakedPathsForDeletedAsset(AssetUtils::AssetHash originalAssetHash);
Mappings _fileMappings;
@ -120,7 +119,7 @@ private:
/// Task pool for handling uploads and downloads of assets
QThreadPool _transferTaskPool;
QHash<AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
QHash<AssetUtils::AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
QThreadPool _bakingTaskPool;
bool _wasColorTextureCompressionEnabled { false };

2
assignment-client/src/assets/BakeAssetTask.cpp
View file

@ -24,7 +24,7 @@ static const int OVEN_STATUS_CODE_ABORT { 2 };
std::once_flag registerMetaTypesFlag;
BakeAssetTask::BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath) :
BakeAssetTask::BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath) :
_assetHash(assetHash),
_assetPath(assetPath),
_filePath(filePath)

6
assignment-client/src/assets/BakeAssetTask.h
View file

@ -25,7 +25,7 @@
class BakeAssetTask : public QObject, public QRunnable {
Q_OBJECT
public:
BakeAssetTask(const AssetHash& assetHash, const AssetPath& assetPath, const QString& filePath);
BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
bool isBaking() { return _isBaking.load(); }
@ -41,8 +41,8 @@ signals:
private:
std::atomic<bool> _isBaking { false };
AssetHash _assetHash;
AssetPath _assetPath;
AssetUtils::AssetHash _assetHash;
AssetUtils::AssetPath _assetPath;
QString _filePath;
std::unique_ptr<QProcess> _ovenProcess { nullptr };
std::atomic<bool> _wasAborted { false };

12
assignment-client/src/assets/SendAssetTask.cpp
View file

@ -40,7 +40,7 @@ void SendAssetTask::run() {
ByteRange byteRange;
_message->readPrimitive(&messageID);
QByteArray assetHash = _message->read(SHA256_HASH_LENGTH);
QByteArray assetHash = _message->read(AssetUtils::SHA256_HASH_LENGTH);
// `start` and `end` indicate the range of data to retrieve for the asset identified by `assetHash`.
// `start` is inclusive, `end` is exclusive. Requesting `start` = 1, `end` = 10 will retrieve 9 bytes of data,
@ -61,7 +61,7 @@ void SendAssetTask::run() {
replyPacketList->writePrimitive(messageID);
if (!byteRange.isValid()) {
replyPacketList->writePrimitive(AssetServerError::InvalidByteRange);
replyPacketList->writePrimitive(AssetUtils::AssetServerError::InvalidByteRange);
} else {
QString filePath = _resourcesDir.filePath(QString(hexHash));
@ -75,7 +75,7 @@ void SendAssetTask::run() {
// check if we're being asked to read data that we just don't have
// because of the file size
if (file.size() < byteRange.fromInclusive || file.size() < byteRange.toExclusive) {
replyPacketList->writePrimitive(AssetServerError::InvalidByteRange);
replyPacketList->writePrimitive(AssetUtils::AssetServerError::InvalidByteRange);
qCDebug(networking) << "Bad byte range: " << hexHash << " "
<< byteRange.fromInclusive << ":" << byteRange.toExclusive;
} else {
@ -86,7 +86,7 @@ void SendAssetTask::run() {
// this range is positive, meaning we just need to seek into the file and then read from there
file.seek(byteRange.fromInclusive);
replyPacketList->writePrimitive(AssetServerError::NoError);
replyPacketList->writePrimitive(AssetUtils::AssetServerError::NoError);
replyPacketList->writePrimitive(size);
replyPacketList->write(file.read(size));
} else {
@ -95,7 +95,7 @@ void SendAssetTask::run() {
// seek to the part of the file where the negative range begins
file.seek(file.size() + byteRange.fromInclusive);
replyPacketList->writePrimitive(AssetServerError::NoError);
replyPacketList->writePrimitive(AssetUtils::AssetServerError::NoError);
replyPacketList->writePrimitive(size);
// first write everything from the negative range to the end of the file
@ -107,7 +107,7 @@ void SendAssetTask::run() {
file.close();
} else {
qCDebug(networking) << "Asset not found: " << filePath << "(" << hexHash << ")";
replyPacketList->writePrimitive(AssetServerError::AssetNotFound);
replyPacketList->writePrimitive(AssetUtils::AssetServerError::AssetNotFound);
}
}

13
assignment-client/src/assets/UploadAssetTask.cpp
View file

@ -20,7 +20,6 @@
#include "ClientServerUtils.h"
UploadAssetTask::UploadAssetTask(QSharedPointer<ReceivedMessage> receivedMessage, SharedNodePointer senderNode,
const QDir& resourcesDir, uint64_t filesizeLimit) :
_receivedMessage(receivedMessage),
@ -50,11 +49,11 @@ void UploadAssetTask::run() {
replyPacket->writePrimitive(messageID);
if (fileSize > _filesizeLimit) {
replyPacket->writePrimitive(AssetServerError::AssetTooLarge);
replyPacket->writePrimitive(AssetUtils::AssetServerError::AssetTooLarge);
} else {
QByteArray fileData = buffer.read(fileSize);
auto hash = hashData(fileData);
auto hash = AssetUtils::hashData(fileData);
auto hexHash = hash.toHex();
qDebug() << "Hash for uploaded file from" << uuidStringWithoutCurlyBraces(_senderNode->getUUID())
@ -66,12 +65,12 @@ void UploadAssetTask::run() {
if (file.exists()) {
// check if the local file has the correct contents, otherwise we overwrite
if (file.open(QIODevice::ReadOnly) && hashData(file.readAll()) == hash) {
if (file.open(QIODevice::ReadOnly) && AssetUtils::hashData(file.readAll()) == hash) {
qDebug() << "Not overwriting existing verified file: " << hexHash;
existingCorrectFile = true;
replyPacket->writePrimitive(AssetServerError::NoError);
replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError);
replyPacket->write(hash);
} else {
qDebug() << "Overwriting an existing file whose contents did not match the expected hash: " << hexHash;
@ -84,7 +83,7 @@ void UploadAssetTask::run() {
qDebug() << "Wrote file" << hexHash << "to disk. Upload complete";
file.close();
replyPacket->writePrimitive(AssetServerError::NoError);
replyPacket->writePrimitive(AssetUtils::AssetServerError::NoError);
replyPacket->write(hash);
} else {
qWarning() << "Failed to upload or write to file" << hexHash << " - upload failed.";
@ -96,7 +95,7 @@ void UploadAssetTask::run() {
qWarning() << "Removal of failed upload file" << hexHash << "failed.";
}
replyPacket->writePrimitive(AssetServerError::FileOperationFailed);
replyPacket->writePrimitive(AssetUtils::AssetServerError::FileOperationFailed);
}
}

4
interface/src/scripting/AssetMappingsScriptingInterface.cpp
View file

@ -260,7 +260,7 @@ void AssetMappingModel::refresh() {
for (auto& mapping : mappings) {
auto& path = mapping.first;
if (path.startsWith(HIDDEN_BAKED_CONTENT_FOLDER)) {
if (path.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
// Hide baked mappings
continue;
}
@ -303,7 +303,7 @@ void AssetMappingModel::refresh() {
auto statusString = isFolder ? "--" : bakingStatusToString(mapping.second.status);
lastItem->setData(statusString, Qt::UserRole + 5);
lastItem->setData(mapping.second.bakingErrors, Qt::UserRole + 6);
if (mapping.second.status == Pending) {
if (mapping.second.status == AssetUtils::Pending) {
++numPendingBakes;
}
}

265
libraries/networking/src/AssetClient.cpp
View file

@ -20,6 +20,7 @@
#include <QtNetwork/QNetworkDiskCache>
#include <shared/GlobalAppProperties.h>
#include <shared/MiniPromises.h>
#include "AssetRequest.h"
#include "AssetUpload.h"
@ -72,7 +73,177 @@ void AssetClient::init() {
networkAccessManager.setCache(cache);
qInfo() << "ResourceManager disk cache setup at" << _cacheDir
<< "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)";
} else {
auto cache = qobject_cast<QNetworkDiskCache*>(networkAccessManager.cache());
qInfo() << "ResourceManager disk cache already setup at" << cache->cacheDirectory()
<< "(size:" << cache->maximumCacheSize() / BYTES_PER_GIGABYTES << "GB)";
}
}
namespace {
const QString& CACHE_ERROR_MESSAGE{ "AssetClient::Error: %1 %2" };
}
MiniPromise::Promise AssetClient::cacheInfoRequestAsync(MiniPromise::Promise deferred) {
if (!deferred) {
deferred = makePromise(__FUNCTION__); // create on caller's thread
}
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "cacheInfoRequestAsync", Q_ARG(MiniPromise::Promise, deferred));
} else {
auto cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
if (cache) {
deferred->resolve({
{ "cacheDirectory", cache->cacheDirectory() },
{ "cacheSize", cache->cacheSize() },
{ "maximumCacheSize", cache->maximumCacheSize() },
});
} else {
deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable"));
}
}
return deferred;
}
MiniPromise::Promise AssetClient::queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "queryCacheMetaAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred));
} else {
auto cache = NetworkAccessManager::getInstance().cache();
if (cache) {
QNetworkCacheMetaData metaData = cache->metaData(url);
QVariantMap attributes, rawHeaders;
if (!metaData.isValid()) {
deferred->reject("invalid cache entry", {
{ "_url", url },
{ "isValid", metaData.isValid() },
{ "metaDataURL", metaData.url() },
});
} else {
auto metaAttributes = metaData.attributes();
foreach(QNetworkRequest::Attribute k, metaAttributes.keys()) {
attributes[QString::number(k)] = metaAttributes[k];
}
for (const auto& i : metaData.rawHeaders()) {
rawHeaders[i.first] = i.second;
}
deferred->resolve({
{ "_url", url },
{ "isValid", metaData.isValid() },
{ "url", metaData.url() },
{ "expirationDate", metaData.expirationDate() },
{ "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() },
{ "saveToDisk", metaData.saveToDisk() },
{ "attributes", attributes },
{ "rawHeaders", rawHeaders },
});
}
} else {
deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("cache unavailable"));
}
}
return deferred;
}
MiniPromise::Promise AssetClient::loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred) {
auto errorMessage = CACHE_ERROR_MESSAGE.arg(__FUNCTION__);
if (!deferred) {
deferred = makePromise(__FUNCTION__); // create on caller's thread
}
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "loadFromCacheAsync", Q_ARG(const QUrl&, url), Q_ARG(MiniPromise::Promise, deferred));
} else {
auto cache = NetworkAccessManager::getInstance().cache();
if (cache) {
MiniPromise::Promise metaRequest = makePromise(__FUNCTION__);
queryCacheMetaAsync(url, metaRequest);
metaRequest->finally([&](QString error, QVariantMap metadata) {
QVariantMap result = {
{ "url", url },
{ "metadata", metadata },
{ "data", QByteArray() },
};
if (!error.isEmpty()) {
deferred->reject(error, result);
return;
}
// caller is responsible for the deletion of the ioDevice, hence the unique_ptr
auto ioDevice = std::unique_ptr<QIODevice>(cache->data(url));
if (ioDevice) {
result["data"] = ioDevice->readAll();
} else {
error = errorMessage.arg("error reading data");
}
deferred->handle(error, result);
});
} else {
deferred->reject(errorMessage.arg("cache unavailable"));
}
}
return deferred;
}
namespace {
// parse RFC 1123 HTTP date format
QDateTime parseHttpDate(const QString& dateString) {
QDateTime dt = QDateTime::fromString(dateString.left(25), "ddd, dd MMM yyyy HH:mm:ss");
if (!dt.isValid()) {
dt = QDateTime::fromString(dateString, Qt::ISODateWithMs);
}
if (!dt.isValid()) {
qDebug() << __FUNCTION__ << "unrecognized date format:" << dateString;
}
dt.setTimeSpec(Qt::UTC);
return dt;
}
QDateTime getHttpDateValue(const QVariantMap& headers, const QString& keyName, const QDateTime& defaultValue) {
return headers.contains(keyName) ? parseHttpDate(headers[keyName].toString()) : defaultValue;
}
}
MiniPromise::Promise AssetClient::saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& headers, MiniPromise::Promise deferred) {
if (!deferred) {
deferred = makePromise(__FUNCTION__); // create on caller's thread
}
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(
this, "saveToCacheAsync", Qt::QueuedConnection,
Q_ARG(const QUrl&, url),
Q_ARG(const QByteArray&, data),
Q_ARG(const QVariantMap&, headers),
Q_ARG(MiniPromise::Promise, deferred));
} else {
auto cache = NetworkAccessManager::getInstance().cache();
if (cache) {
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData.setSaveToDisk(true);
metaData.setLastModified(getHttpDateValue(headers, "last-modified", QDateTime::currentDateTimeUtc()));
metaData.setExpirationDate(getHttpDateValue(headers, "expires", QDateTime())); // nil defaultValue == never expires
auto ioDevice = cache->prepare(metaData);
if (ioDevice) {
ioDevice->write(data);
cache->insert(ioDevice);
qCDebug(asset_client) << url.toDisplayString() << "saved to disk cache ("<< data.size()<<" bytes)";
deferred->resolve({
{ "url", url },
{ "success", true },
{ "metaDataURL", metaData.url() },
{ "byteLength", data.size() },
{ "expirationDate", metaData.expirationDate() },
{ "lastModified", metaData.lastModified().toString().isEmpty() ? QDateTime() : metaData.lastModified() },
});
} else {
auto error = QString("Could not save %1 to disk cache").arg(url.toDisplayString());
qCWarning(asset_client) << error;
deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg(error));
}
} else {
deferred->reject(CACHE_ERROR_MESSAGE.arg(__FUNCTION__).arg("unavailable"));
}
}
return deferred;
}
void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) {
@ -113,7 +284,7 @@ void AssetClient::handleAssetMappingOperationReply(QSharedPointer<ReceivedMessag
MessageID messageID;
message->readPrimitive(&messageID);
AssetServerError error;
AssetUtils::AssetServerError error;
message->readPrimitive(&error);
// Check if we have any pending requests for this node
@ -149,7 +320,7 @@ bool haveAssetServer() {
return true;
}
GetMappingRequest* AssetClient::createGetMappingRequest(const AssetPath& path) {
GetMappingRequest* AssetClient::createGetMappingRequest(const AssetUtils::AssetPath& path) {
auto request = new GetMappingRequest(path);
request->moveToThread(thread());
@ -165,7 +336,7 @@ GetAllMappingsRequest* AssetClient::createGetAllMappingsRequest() {
return request;
}
DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetPathList& paths) {
DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetUtils::AssetPathList& paths) {
auto request = new DeleteMappingsRequest(paths);
request->moveToThread(thread());
@ -173,7 +344,7 @@ DeleteMappingsRequest* AssetClient::createDeleteMappingsRequest(const AssetPathL
return request;
}
SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, const AssetHash& hash) {
SetMappingRequest* AssetClient::createSetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) {
auto request = new SetMappingRequest(path, hash);
request->moveToThread(thread());
@ -181,7 +352,7 @@ SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, c
return request;
}
RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath) {
RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath) {
auto request = new RenameMappingRequest(oldPath, newPath);
request->moveToThread(thread());
@ -189,7 +360,7 @@ RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& o
return request;
}
SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetPathList& path, bool enabled) {
SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled) {
auto bakingEnabledRequest = new SetBakingEnabledRequest(path, enabled);
bakingEnabledRequest->moveToThread(thread());
@ -197,7 +368,7 @@ SetBakingEnabledRequest* AssetClient::createSetBakingEnabledRequest(const AssetP
return bakingEnabledRequest;
}
AssetRequest* AssetClient::createRequest(const AssetHash& hash, const ByteRange& byteRange) {
AssetRequest* AssetClient::createRequest(const AssetUtils::AssetHash& hash, const ByteRange& byteRange) {
auto request = new AssetRequest(hash, byteRange);
// Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
@ -222,11 +393,11 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data) {
return upload;
}
MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffset end,
MessageID AssetClient::getAsset(const QString& hash, AssetUtils::DataOffset start, AssetUtils::DataOffset end,
ReceivedAssetCallback callback, ProgressCallback progressCallback) {
Q_ASSERT(QThread::currentThread() == thread());
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
if (hash.length() != AssetUtils::SHA256_HASH_HEX_LENGTH) {
qCWarning(asset_client) << "Invalid hash size";
return false;
}
@ -238,7 +409,7 @@ MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffse
auto messageID = ++_currentID;
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(start) + sizeof(end);
auto payloadSize = sizeof(messageID) + AssetUtils::SHA256_HASH_LENGTH + sizeof(start) + sizeof(end);
auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true);
qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
@ -257,7 +428,7 @@ MessageID AssetClient::getAsset(const QString& hash, DataOffset start, DataOffse
}
}
callback(false, AssetServerError::NoError, QByteArray());
callback(false, AssetUtils::AssetServerError::NoError, QByteArray());
return INVALID_MESSAGE_ID;
}
@ -270,7 +441,7 @@ MessageID AssetClient::getAssetInfo(const QString& hash, GetInfoCallback callbac
if (assetServer) {
auto messageID = ++_currentID;
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH;
auto payloadSize = sizeof(messageID) + AssetUtils::SHA256_HASH_LENGTH;
auto packet = NLPacket::create(PacketType::AssetGetInfo, payloadSize, true);
packet->writePrimitive(messageID);
@ -283,7 +454,7 @@ MessageID AssetClient::getAssetInfo(const QString& hash, GetInfoCallback callbac
}
}
callback(false, AssetServerError::NoError, { "", 0 });
callback(false, AssetUtils::AssetServerError::NoError, { "", 0 });
return INVALID_MESSAGE_ID;
}
@ -292,14 +463,14 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer<ReceivedMessage> messag
MessageID messageID;
message->readPrimitive(&messageID);
auto assetHash = message->read(SHA256_HASH_LENGTH);
auto assetHash = message->read(AssetUtils::SHA256_HASH_LENGTH);
AssetServerError error;
AssetUtils::AssetServerError error;
message->readPrimitive(&error);
AssetInfo info { assetHash.toHex(), 0 };
if (error == AssetServerError::NoError) {
if (error == AssetUtils::AssetServerError::NoError) {
message->readPrimitive(&info.size);
}
@ -326,16 +497,16 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer<ReceivedMessage> messag
void AssetClient::handleAssetGetReply(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
Q_ASSERT(QThread::currentThread() == thread());
auto assetHash = message->readHead(SHA256_HASH_LENGTH);
auto assetHash = message->readHead(AssetUtils::SHA256_HASH_LENGTH);
qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex();
MessageID messageID;
message->readHeadPrimitive(&messageID);
AssetServerError error;
AssetUtils::AssetServerError error;
message->readHeadPrimitive(&error);
DataOffset length = 0;
AssetUtils::DataOffset length = 0;
if (!error) {
message->readHeadPrimitive(&length);
} else {
@ -388,7 +559,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<ReceivedMessage> message, S
}
void AssetClient::handleProgressCallback(const QWeakPointer<Node>& node, MessageID messageID,
qint64 size, DataOffset length) {
qint64 size, AssetUtils::DataOffset length) {
auto senderNode = node.toStrongRef();
if (!senderNode) {
@ -414,7 +585,7 @@ void AssetClient::handleProgressCallback(const QWeakPointer<Node>& node, Message
callbacks.progressCallback(size, length);
}
void AssetClient::handleCompleteCallback(const QWeakPointer<Node>& node, MessageID messageID, DataOffset length) {
void AssetClient::handleCompleteCallback(const QWeakPointer<Node>& node, MessageID messageID, AssetUtils::DataOffset length) {
auto senderNode = node.toStrongRef();
if (!senderNode) {
@ -448,9 +619,9 @@ void AssetClient::handleCompleteCallback(const QWeakPointer<Node>& node, Message
}
if (message->failed() || length != message->getBytesLeftToRead()) {
callbacks.completeCallback(false, AssetServerError::NoError, QByteArray());
callbacks.completeCallback(false, AssetUtils::AssetServerError::NoError, QByteArray());
} else {
callbacks.completeCallback(true, AssetServerError::NoError, message->readAll());
callbacks.completeCallback(true, AssetUtils::AssetServerError::NoError, message->readAll());
}
// We should never get to this point without the associated senderNode and messageID
@ -461,7 +632,7 @@ void AssetClient::handleCompleteCallback(const QWeakPointer<Node>& node, Message
}
MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCallback callback) {
MessageID AssetClient::getAssetMapping(const AssetUtils::AssetPath& path, MappingOperationCallback callback) {
Q_ASSERT(QThread::currentThread() == thread());
auto nodeList = DependencyManager::get<NodeList>();
@ -473,7 +644,7 @@ MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCa
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::Get);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Get);
packetList->writeString(path);
@ -484,7 +655,7 @@ MessageID AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCa
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
@ -500,7 +671,7 @@ MessageID AssetClient::getAllAssetMappings(MappingOperationCallback callback) {
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::GetAll);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::GetAll);
if (nodeList->sendPacketList(std::move(packetList), *assetServer) != -1) {
_pendingMappingRequests[assetServer][messageID] = callback;
@ -509,11 +680,11 @@ MessageID AssetClient::getAllAssetMappings(MappingOperationCallback callback) {
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOperationCallback callback) {
MessageID AssetClient::deleteAssetMappings(const AssetUtils::AssetPathList& paths, MappingOperationCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
@ -523,7 +694,7 @@ MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOp
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::Delete);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Delete);
packetList->writePrimitive(int(paths.size()));
@ -538,11 +709,11 @@ MessageID AssetClient::deleteAssetMappings(const AssetPathList& paths, MappingOp
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback) {
MessageID AssetClient::setAssetMapping(const QString& path, const AssetUtils::AssetHash& hash, MappingOperationCallback callback) {
Q_ASSERT(QThread::currentThread() == thread());
auto nodeList = DependencyManager::get<NodeList>();
@ -554,7 +725,7 @@ MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& has
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::Set);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Set);
packetList->writeString(path);
packetList->write(QByteArray::fromHex(hash.toUtf8()));
@ -566,11 +737,11 @@ MessageID AssetClient::setAssetMapping(const QString& path, const AssetHash& has
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetPath& newPath, MappingOperationCallback callback) {
MessageID AssetClient::renameAssetMapping(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath, MappingOperationCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
@ -580,7 +751,7 @@ MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetP
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::Rename);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::Rename);
packetList->writeString(oldPath);
packetList->writeString(newPath);
@ -593,11 +764,11 @@ MessageID AssetClient::renameAssetMapping(const AssetPath& oldPath, const AssetP
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback) {
MessageID AssetClient::setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled, MappingOperationCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
@ -607,7 +778,7 @@ MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(AssetMappingOperationType::SetBakingEnabled);
packetList->writePrimitive(AssetUtils::AssetMappingOperationType::SetBakingEnabled);
packetList->writePrimitive(enabled);
@ -624,7 +795,7 @@ MessageID AssetClient::setBakingEnabled(const AssetPathList& paths, bool enabled
}
}
callback(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
callback(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
return INVALID_MESSAGE_ID;
}
@ -708,7 +879,7 @@ MessageID AssetClient::uploadAsset(const QByteArray& data, UploadResultCallback
}
}
callback(false, AssetServerError::NoError, QString());
callback(false, AssetUtils::AssetServerError::NoError, QString());
return INVALID_MESSAGE_ID;
}
@ -718,7 +889,7 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<ReceivedMessage> message
MessageID messageID;
message->readPrimitive(&messageID);
AssetServerError error;
AssetUtils::AssetServerError error;
message->readPrimitive(&error);
QString hashString;
@ -726,7 +897,7 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<ReceivedMessage> message
if (error) {
qCWarning(asset_client) << "Error uploading file to asset server";
} else {
auto hash = message->read(SHA256_HASH_LENGTH);
auto hash = message->read(AssetUtils::SHA256_HASH_LENGTH);
hashString = hash.toHex();
qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
@ -765,7 +936,7 @@ void AssetClient::handleNodeKilled(SharedNodePointer node) {
auto messageMapIt = _pendingUploads.find(node);
if (messageMapIt != _pendingUploads.end()) {
for (const auto& value : messageMapIt->second) {
value.second(false, AssetServerError::NoError, "");
value.second(false, AssetUtils::AssetServerError::NoError, "");
}
messageMapIt->second.clear();
}
@ -797,7 +968,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) {
disconnect(message.data(), nullptr, this, nullptr);
}
value.second.completeCallback(false, AssetServerError::NoError, QByteArray());
value.second.completeCallback(false, AssetUtils::AssetServerError::NoError, QByteArray());
}
messageMapIt->second.clear();
}
@ -808,7 +979,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) {
if (messageMapIt != _pendingInfoRequests.end()) {
AssetInfo info { "", 0 };
for (const auto& value : messageMapIt->second) {
value.second(false, AssetServerError::NoError, info);
value.second(false, AssetUtils::AssetServerError::NoError, info);
}
messageMapIt->second.clear();
}
@ -818,7 +989,7 @@ void AssetClient::forceFailureOfPendingRequests(SharedNodePointer node) {
auto messageMapIt = _pendingMappingRequests.find(node);
if (messageMapIt != _pendingMappingRequests.end()) {
for (const auto& value : messageMapIt->second) {
value.second(false, AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
value.second(false, AssetUtils::AssetServerError::NoError, QSharedPointer<ReceivedMessage>());
}
messageMapIt->second.clear();
}

41
libraries/networking/src/AssetClient.h
View file

@ -19,6 +19,7 @@
#include <map>
#include <DependencyManager.h>
#include <shared/MiniPromises.h>
#include "AssetUtils.h"
#include "ByteRange.h"
@ -41,10 +42,10 @@ struct AssetInfo {
int64_t size;
};
using MappingOperationCallback = std::function<void(bool responseReceived, AssetServerError serverError, QSharedPointer<ReceivedMessage> message)>;
using ReceivedAssetCallback = std::function<void(bool responseReceived, AssetServerError serverError, const QByteArray& data)>;
using GetInfoCallback = std::function<void(bool responseReceived, AssetServerError serverError, AssetInfo info)>;
using UploadResultCallback = std::function<void(bool responseReceived, AssetServerError serverError, const QString& hash)>;
using MappingOperationCallback = std::function<void(bool responseReceived, AssetUtils::AssetServerError serverError, QSharedPointer<ReceivedMessage> message)>;
using ReceivedAssetCallback = std::function<void(bool responseReceived, AssetUtils::AssetServerError serverError, const QByteArray& data)>;
using GetInfoCallback = std::function<void(bool responseReceived, AssetUtils::AssetServerError serverError, AssetInfo info)>;
using UploadResultCallback = std::function<void(bool responseReceived, AssetUtils::AssetServerError serverError, const QString& hash)>;
using ProgressCallback = std::function<void(qint64 totalReceived, qint64 total)>;
class AssetClient : public QObject, public Dependency {
@ -52,13 +53,13 @@ class AssetClient : public QObject, public Dependency {
public:
AssetClient();
Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetPath& path);
Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetUtils::AssetPath& path);
Q_INVOKABLE GetAllMappingsRequest* createGetAllMappingsRequest();
Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetPathList& paths);
Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash);
Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath);
Q_INVOKABLE SetBakingEnabledRequest* createSetBakingEnabledRequest(const AssetPathList& path, bool enabled);
Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const ByteRange& byteRange = ByteRange());
Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetUtils::AssetPathList& paths);
Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath);
Q_INVOKABLE SetBakingEnabledRequest* createSetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled);
Q_INVOKABLE AssetRequest* createRequest(const AssetUtils::AssetHash& hash, const ByteRange& byteRange = ByteRange());
Q_INVOKABLE AssetUpload* createUpload(const QString& filename);
Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data);
@ -66,6 +67,10 @@ public slots:
void init();
void cacheInfoRequest(QObject* reciever, QString slot);
MiniPromise::Promise cacheInfoRequestAsync(MiniPromise::Promise deferred = nullptr);
MiniPromise::Promise queryCacheMetaAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr);
MiniPromise::Promise loadFromCacheAsync(const QUrl& url, MiniPromise::Promise deferred = nullptr);
MiniPromise::Promise saveToCacheAsync(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap(), MiniPromise::Promise deferred = nullptr);
void clearCache();
private slots:
@ -78,15 +83,15 @@ private slots:
void handleNodeClientConnectionReset(SharedNodePointer node);
private:
MessageID getAssetMapping(const AssetHash& hash, MappingOperationCallback callback);
MessageID getAssetMapping(const AssetUtils::AssetHash& hash, MappingOperationCallback callback);
MessageID getAllAssetMappings(MappingOperationCallback callback);
MessageID setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback);
MessageID deleteAssetMappings(const AssetPathList& paths, MappingOperationCallback callback);
MessageID renameAssetMapping(const AssetPath& oldPath, const AssetPath& newPath, MappingOperationCallback callback);
MessageID setBakingEnabled(const AssetPathList& paths, bool enabled, MappingOperationCallback callback);
MessageID setAssetMapping(const QString& path, const AssetUtils::AssetHash& hash, MappingOperationCallback callback);
MessageID deleteAssetMappings(const AssetUtils::AssetPathList& paths, MappingOperationCallback callback);
MessageID renameAssetMapping(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath, MappingOperationCallback callback);
MessageID setBakingEnabled(const AssetUtils::AssetPathList& paths, bool enabled, MappingOperationCallback callback);
MessageID getAssetInfo(const QString& hash, GetInfoCallback callback);
MessageID getAsset(const QString& hash, DataOffset start, DataOffset end,
MessageID getAsset(const QString& hash, AssetUtils::DataOffset start, AssetUtils::DataOffset end,
ReceivedAssetCallback callback, ProgressCallback progressCallback);
MessageID uploadAsset(const QByteArray& data, UploadResultCallback callback);
@ -95,8 +100,8 @@ private:
bool cancelGetAssetRequest(MessageID id);
bool cancelUploadAssetRequest(MessageID id);
void handleProgressCallback(const QWeakPointer<Node>& node, MessageID messageID, qint64 size, DataOffset length);
void handleCompleteCallback(const QWeakPointer<Node>& node, MessageID messageID, DataOffset length);
void handleProgressCallback(const QWeakPointer<Node>& node, MessageID messageID, qint64 size, AssetUtils::DataOffset length);
void handleCompleteCallback(const QWeakPointer<Node>& node, MessageID messageID, AssetUtils::DataOffset length);
void forceFailureOfPendingRequests(SharedNodePointer node);

27
libraries/networking/src/AssetRequest.cpp
View file

@ -52,7 +52,7 @@ void AssetRequest::start() {
}
// in case we haven't parsed a valid hash, return an error now
if (!isValidHash(_hash)) {
if (!AssetUtils::isValidHash(_hash)) {
_error = InvalidHash;
_state = Finished;
@ -61,7 +61,7 @@ void AssetRequest::start() {
}
// Try to load from cache
_data = loadFromCache(getUrl());
_data = AssetUtils::loadFromCache(getUrl());
if (!_data.isNull()) {
_error = NoError;
@ -80,7 +80,7 @@ void AssetRequest::start() {
auto hash = _hash;
_assetRequestID = assetClient->getAsset(_hash, _byteRange.fromInclusive, _byteRange.toExclusive,
[this, that, hash](bool responseReceived, AssetServerError serverError, const QByteArray& data) {
[this, that, hash](bool responseReceived, AssetUtils::AssetServerError serverError, const QByteArray& data) {
if (!that) {
qCWarning(asset_client) << "Got reply for dead asset request " << hash << "- error code" << _error;
@ -91,12 +91,12 @@ void AssetRequest::start() {
if (!responseReceived) {
_error = NetworkError;
} else if (serverError != AssetServerError::NoError) {
} else if (serverError != AssetUtils::AssetServerError::NoError) {
switch (serverError) {
case AssetServerError::AssetNotFound:
case AssetUtils::AssetServerError::AssetNotFound:
_error = NotFound;
break;
case AssetServerError::InvalidByteRange:
case AssetUtils::AssetServerError::InvalidByteRange:
_error = InvalidByteRange;
break;
default:
@ -104,7 +104,7 @@ void AssetRequest::start() {
break;
}
} else {
if (!_byteRange.isSet() && hashData(data).toHex() != _hash) {
if (!_byteRange.isSet() && AssetUtils::hashData(data).toHex() != _hash) {
// the hash of the received data does not match what we expect, so we return an error
_error = HashVerificationFailed;
}
@ -115,7 +115,7 @@ void AssetRequest::start() {
emit progress(_totalReceived, data.size());
if (!_byteRange.isSet()) {
saveToCache(getUrl(), data);
AssetUtils::saveToCache(getUrl(), data);
}
}
}
@ -134,3 +134,14 @@ void AssetRequest::start() {
emit progress(totalReceived, total);
});
}
const QString AssetRequest::getErrorString() const {
QString result;
if (_error != Error::NoError) {
QVariant v;
v.setValue(_error);
result = v.toString(); // courtesy of Q_ENUM
}
return result;
}

5
libraries/networking/src/AssetRequest.h
View file

@ -42,7 +42,7 @@ public:
NetworkError,
UnknownError
};
Q_ENUM(Error)
AssetRequest(const QString& hash, const ByteRange& byteRange = ByteRange());
virtual ~AssetRequest() override;
@ -51,7 +51,8 @@ public:
const QByteArray& getData() const { return _data; }
const State& getState() const { return _state; }
const Error& getError() const { return _error; }
QUrl getUrl() const { return ::getATPUrl(_hash); }
const QString getErrorString() const;
QUrl getUrl() const { return AssetUtils::getATPUrl(_hash); }
QString getHash() const { return _hash; }
bool loadedFromCache() const { return _loadedFromCache; }

4
libraries/networking/src/AssetResourceRequest.cpp
View file

@ -68,7 +68,7 @@ void AssetResourceRequest::doSend() {
}
}
void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
void AssetResourceRequest::requestMappingForPath(const AssetUtils::AssetPath& path) {
auto statTracker = DependencyManager::get<StatTracker>();
statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_STARTED);
@ -140,7 +140,7 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
_assetMappingRequest->start();
}
void AssetResourceRequest::requestHash(const AssetHash& hash) {
void AssetResourceRequest::requestHash(const AssetUtils::AssetHash& hash) {
// Make request to atp
auto assetClient = DependencyManager::get<AssetClient>();
_assetRequest = assetClient->createRequest(hash, _byteRange);

4
libraries/networking/src/AssetResourceRequest.h
View file

@ -34,8 +34,8 @@ private slots:
private:
static bool urlIsAssetHash(const QUrl& url);
void requestMappingForPath(const AssetPath& path);
void requestHash(const AssetHash& hash);
void requestMappingForPath(const AssetUtils::AssetPath& path);
void requestHash(const AssetUtils::AssetHash& hash);
GetMappingRequest* _assetMappingRequest { nullptr };
AssetRequest* _assetRequest { nullptr };

14
libraries/networking/src/AssetUpload.cpp
View file

@ -81,21 +81,21 @@ void AssetUpload::start() {
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
}
assetClient->uploadAsset(_data, [this](bool responseReceived, AssetServerError error, const QString& hash){
assetClient->uploadAsset(_data, [this](bool responseReceived, AssetUtils::AssetServerError error, const QString& hash){
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::AssetTooLarge:
case AssetUtils::AssetServerError::AssetTooLarge:
_error = TooLarge;
break;
case AssetServerError::PermissionDenied:
case AssetUtils::AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
case AssetServerError::FileOperationFailed:
case AssetUtils::AssetServerError::FileOperationFailed:
_error = ServerFileError;
break;
default:
@ -104,8 +104,8 @@ void AssetUpload::start() {
}
}
if (_error == NoError && hash == hashData(_data).toHex()) {
saveToCache(getATPUrl(hash), _data);
if (_error == NoError && hash == AssetUtils::hashData(_data).toHex()) {
AssetUtils::saveToCache(AssetUtils::getATPUrl(hash), _data);
}
emit finished(this, hash);

37
libraries/networking/src/AssetUtils.cpp
View file

@ -15,6 +15,7 @@
#include <QtCore/QCryptographicHash>
#include <QtCore/QDateTime>
#include <QtCore/QFileInfo> // for baseName
#include <QtNetwork/QAbstractNetworkCache>
#include "NetworkAccessManager.h"
@ -22,8 +23,38 @@
#include "ResourceManager.h"
QUrl getATPUrl(const QString& hash) {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, hash));
namespace AssetUtils {
// Extract the valid AssetHash portion from atp: URLs like "[atp:]HASH[.fbx][?query]"
// (or an invalid AssetHash if not found)
AssetHash extractAssetHash(const QString& input) {
if (isValidHash(input)) {
return input;
}
QString path = getATPUrl(input).path();
QString baseName = QFileInfo(path).baseName();
if (isValidHash(baseName)) {
return baseName;
}
return AssetHash();
}
// Get the normalized ATP URL for a raw hash, /path or "atp:" input string.
QUrl getATPUrl(const QString& input) {
QUrl url = input;
if (!url.scheme().isEmpty() && url.scheme() != URL_SCHEME_ATP) {
return QUrl();
}
// this strips extraneous info from the URL (while preserving fragment/querystring)
QString path = url.toEncoded(
QUrl::RemoveAuthority | QUrl::RemoveScheme |
QUrl::StripTrailingSlash | QUrl::NormalizePathSegments
);
QString baseName = QFileInfo(url.path()).baseName();
if (isValidPath(path) || isValidHash(baseName)) {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP).arg(path));
}
return QUrl();
}
QByteArray hashData(const QByteArray& data) {
@ -102,3 +133,5 @@ QString bakingStatusToString(BakingStatus status) {
return "--";
}
}
} // namespace AssetUtils

7
libraries/networking/src/AssetUtils.h
View file

@ -19,6 +19,8 @@
#include <QtCore/QByteArray>
#include <QtCore/QUrl>
namespace AssetUtils {
using DataOffset = int64_t;
using AssetPath = QString;
@ -71,7 +73,8 @@ struct MappingInfo {
using AssetMapping = std::map<AssetPath, MappingInfo>;
QUrl getATPUrl(const QString& hash);
QUrl getATPUrl(const QString& input);
AssetHash extractAssetHash(const QString& input);
QByteArray hashData(const QByteArray& data);
@ -84,4 +87,6 @@ bool isValidHash(const QString& hashString);
QString bakingStatusToString(BakingStatus status);
} // namespace AssetUtils
#endif // hifi_AssetUtils_h

365
libraries/networking/src/BaseAssetScriptingInterface.cpp Normal file
View file

@ -0,0 +1,365 @@
//
// BaseAssetScriptingInterface.cpp
// libraries/networking/src
//
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
#include "BaseAssetScriptingInterface.h"
#include <QJsonDocument>
#include <QJsonArray>
#include <QMimeDatabase>
#include <QThread>
#include "AssetRequest.h"
#include "AssetUpload.h"
#include "AssetUtils.h"
#include "MappingRequest.h"
#include "NetworkLogging.h"
#include <RegisteredMetaTypes.h>
#include <shared/QtHelpers.h>
#include "Gzip.h"
using Promise = MiniPromise::Promise;
QSharedPointer<AssetClient> BaseAssetScriptingInterface::assetClient() {
auto client = DependencyManager::get<AssetClient>();
Q_ASSERT(client);
if (!client) {
qDebug() << "BaseAssetScriptingInterface::assetClient unavailable";
}
return client;
}
BaseAssetScriptingInterface::BaseAssetScriptingInterface(QObject* parent) : QObject(parent) {}
bool BaseAssetScriptingInterface::initializeCache() {
if (!assetClient()) {
return false; // not yet possible to initialize the cache
}
if (_cacheReady) {
return true; // cache is ready
}
// attempt to initialize the cache
QMetaObject::invokeMethod(assetClient().data(), "init");
Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus");
deferred->then([this](QVariantMap result) {
_cacheReady = !result.value("cacheDirectory").toString().isEmpty();
});
deferred->fail([](QString error) {
qDebug() << "BaseAssetScriptingInterface::queryCacheStatus ERROR" << QThread::currentThread() << error;
});
assetClient()->cacheInfoRequestAsync(deferred);
return false; // cache is not ready yet
}
Promise BaseAssetScriptingInterface::getCacheStatus() {
return assetClient()->cacheInfoRequestAsync(makePromise(__FUNCTION__));
}
Promise BaseAssetScriptingInterface::queryCacheMeta(const QUrl& url) {
return assetClient()->queryCacheMetaAsync(url, makePromise(__FUNCTION__));
}
Promise BaseAssetScriptingInterface::loadFromCache(const QUrl& url, bool decompress, const QString& responseType) {
QVariantMap metaData = {
{ "_type", "cache" },
{ "url", url },
{ "responseType", responseType },
};
Promise completed = makePromise("loadFromCache::completed");
Promise fetched = makePromise("loadFromCache::fetched");
Promise downloaded = assetClient()->loadFromCacheAsync(url, makePromise("loadFromCache-retrieval"));
downloaded->mixin(metaData);
downloaded->fail(fetched);
if (decompress) {
downloaded->then([=](QVariantMap result) {
fetched->mixin(result);
Promise decompressed = decompressBytes(result.value("data").toByteArray());
decompressed->mixin(result);
decompressed->ready(fetched);
});
} else {
downloaded->then(fetched);
}
fetched->fail(completed);
fetched->then([=](QVariantMap result) {
Promise converted = convertBytes(result.value("data").toByteArray(), responseType);
converted->mixin(result);
converted->ready(completed);
});
return completed;
}
Promise BaseAssetScriptingInterface::saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& headers) {
return assetClient()->saveToCacheAsync(url, data, headers, makePromise(__FUNCTION__));
}
Promise BaseAssetScriptingInterface::loadAsset(QString asset, bool decompress, QString responseType) {
auto hash = AssetUtils::extractAssetHash(asset);
auto url = AssetUtils::getATPUrl(hash).toString();
QVariantMap metaData = {
{ "_asset", asset },
{ "_type", "download" },
{ "hash", hash },
{ "url", url },
{ "responseType", responseType },
};
Promise completed = makePromise("loadAsset::completed");
Promise fetched = makePromise("loadAsset::fetched");
Promise downloaded = downloadBytes(hash);
downloaded->mixin(metaData);
downloaded->fail(fetched);
if (decompress) {
downloaded->then([=](QVariantMap result) {
Q_ASSERT(thread() == QThread::currentThread());
fetched->mixin(result);
Promise decompressed = decompressBytes(result.value("data").toByteArray());
decompressed->mixin(result);
decompressed->ready(fetched);
});
} else {
downloaded->then(fetched);
}
fetched->fail(completed);
fetched->then([=](QVariantMap result) {
Promise converted = convertBytes(result.value("data").toByteArray(), responseType);
converted->mixin(result);
converted->ready(completed);
});
return completed;
}
Promise BaseAssetScriptingInterface::convertBytes(const QByteArray& dataByteArray, const QString& responseType) {
QVariantMap result = {
{ "_contentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
{ "_byteLength", dataByteArray.size() },
{ "_responseType", responseType },
};
QString error;
Promise conversion = makePromise(__FUNCTION__);
if (!RESPONSE_TYPES.contains(responseType)) {
error = QString("convertBytes: invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | "));
} else if (responseType == "arraybuffer") {
// interpret as bytes
result["response"] = dataByteArray;
} else if (responseType == "text") {
// interpret as utf-8 text
result["response"] = QString::fromUtf8(dataByteArray);
} else if (responseType == "json") {
// interpret as JSON
QJsonParseError status;
auto parsed = QJsonDocument::fromJson(dataByteArray, &status);
if (status.error == QJsonParseError::NoError) {
result["response"] = parsed.isArray() ? QVariant(parsed.array().toVariantList()) : QVariant(parsed.object().toVariantMap());
} else {
result = {
{ "error", status.error },
{ "offset", status.offset },
};
error = "JSON Parse Error: " + status.errorString();
}
}
if (result.value("response").canConvert<QByteArray>()) {
auto data = result.value("response").toByteArray();
result["contentType"] = QMimeDatabase().mimeTypeForData(data).name();
result["byteLength"] = data.size();
result["responseType"] = responseType;
}
return conversion->handle(error, result);
}
Promise BaseAssetScriptingInterface::decompressBytes(const QByteArray& dataByteArray) {
QByteArray inflated;
Promise decompressed = makePromise(__FUNCTION__);
auto start = usecTimestampNow();
if (gunzip(dataByteArray, inflated)) {
auto end = usecTimestampNow();
decompressed->resolve({
{ "_compressedByteLength", dataByteArray.size() },
{ "_compressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
{ "_compressMS", (double)(end - start) / 1000.0 },
{ "decompressed", true },
{ "byteLength", inflated.size() },
{ "contentType", QMimeDatabase().mimeTypeForData(inflated).name() },
{ "data", inflated },
});
} else {
decompressed->reject("gunzip error");
}
return decompressed;
}
Promise BaseAssetScriptingInterface::compressBytes(const QByteArray& dataByteArray, int level) {
QByteArray deflated;
auto start = usecTimestampNow();
Promise compressed = makePromise(__FUNCTION__);
if (gzip(dataByteArray, deflated, level)) {
auto end = usecTimestampNow();
compressed->resolve({
{ "_uncompressedByteLength", dataByteArray.size() },
{ "_uncompressedContentType", QMimeDatabase().mimeTypeForData(dataByteArray).name() },
{ "_compressMS", (double)(end - start) / 1000.0 },
{ "compressed", true },
{ "byteLength", deflated.size() },
{ "contentType", QMimeDatabase().mimeTypeForData(deflated).name() },
{ "data", deflated },
});
} else {
compressed->reject("gzip error", {});
}
return compressed;
}
Promise BaseAssetScriptingInterface::downloadBytes(QString hash) {
QPointer<AssetRequest> assetRequest = assetClient()->createRequest(hash);
Promise deferred = makePromise(__FUNCTION__);
QObject::connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) {
// note: we are now on the "Resource Manager" thread
Q_ASSERT(QThread::currentThread() == request->thread());
Q_ASSERT(request->getState() == AssetRequest::Finished);
QString error;
QVariantMap result;
if (request->getError() == AssetRequest::Error::NoError) {
QByteArray data = request->getData();
result = {
{ "url", request->getUrl() },
{ "hash", request->getHash() },
{ "cached", request->loadedFromCache() },
{ "contentType", QMimeDatabase().mimeTypeForData(data).name() },
{ "data", data },
};
} else {
error = request->getError();
result = { { "error", request->getError() } };
}
// forward thread-safe copies back to our thread
deferred->handle(error, result);
request->deleteLater();
});
assetRequest->start();
return deferred;
}
Promise BaseAssetScriptingInterface::uploadBytes(const QByteArray& bytes) {
Promise deferred = makePromise(__FUNCTION__);
QPointer<AssetUpload> upload = assetClient()->createUpload(bytes);
const auto byteLength = bytes.size();
QObject::connect(upload, &AssetUpload::finished, upload, [=](AssetUpload* upload, const QString& hash) {
Q_ASSERT(QThread::currentThread() == upload->thread());
// note: we are now on the "Resource Manager" thread
QString error;
QVariantMap result;
if (upload->getError() == AssetUpload::NoError) {
result = {
{ "hash", hash },
{ "url", AssetUtils::getATPUrl(hash).toString() },
{ "filename", upload->getFilename() },
{ "byteLength", byteLength },
};
} else {
error = upload->getErrorString();
result = { { "error", upload->getError() } };
}
// forward thread-safe copies back to our thread
deferred->handle(error, result);
upload->deleteLater();
});
upload->start();
return deferred;
}
Promise BaseAssetScriptingInterface::getAssetInfo(QString asset) {
Promise deferred = makePromise(__FUNCTION__);
auto url = AssetUtils::getATPUrl(asset);
auto path = url.path();
auto hash = AssetUtils::extractAssetHash(asset);
if (AssetUtils::isValidHash(hash)) {
// already a valid ATP hash -- nothing to do
deferred->resolve({
{ "hash", hash },
{ "path", path },
{ "url", url },
});
} else if (AssetUtils::isValidFilePath(path)) {
QPointer<GetMappingRequest> request = assetClient()->createGetMappingRequest(path);
QObject::connect(request, &GetMappingRequest::finished, request, [=]() {
Q_ASSERT(QThread::currentThread() == request->thread());
// note: we are now on the "Resource Manager" thread
QString error;
QVariantMap result;
if (request->getError() == GetMappingRequest::NoError) {
result = {
{ "_hash", hash },
{ "_path", path },
{ "_url", url },
{ "url", url },
{ "hash", request->getHash() },
{ "hashURL", AssetUtils::getATPUrl(request->getHash()).toString() },
{ "wasRedirected", request->wasRedirected() },
{ "path", request->wasRedirected() ? request->getRedirectedPath() : path },
};
} else {
error = request->getErrorString();
result = { { "error", request->getError() } };
}
// forward thread-safe copies back to our thread
deferred->handle(error, result);
request->deleteLater();
});
request->start();
} else {
deferred->reject("invalid ATP file path: " + asset + "("+path+")", {});
}
return deferred;
}
Promise BaseAssetScriptingInterface::symlinkAsset(QString hash, QString path) {
auto deferred = makePromise(__FUNCTION__);
QPointer<SetMappingRequest> setMappingRequest = assetClient()->createSetMappingRequest(path, hash);
connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [=](SetMappingRequest* request) {
Q_ASSERT(QThread::currentThread() == request->thread());
// we are now on the "Resource Manager" thread
QString error;
QVariantMap result;
if (request->getError() == SetMappingRequest::NoError) {
result = {
{ "_hash", hash },
{ "_path", path },
{ "hash", request->getHash() },
{ "path", request->getPath() },
{ "url", AssetUtils::getATPUrl(request->getPath()).toString() },
};
} else {
error = request->getErrorString();
result = { { "error", request->getError() } };
}
// forward results back to our thread
deferred->handle(error, result);
request->deleteLater();
});
setMappingRequest->start();
return deferred;
}

59
libraries/networking/src/BaseAssetScriptingInterface.h Normal file
View file

@ -0,0 +1,59 @@
//
// BaseAssetScriptingInterface.h
// libraries/networking/src
//
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
// BaseAssetScriptingInterface contains the engine-agnostic support code that can be used from
// both QML JS and QScriptEngine JS engine implementations
#ifndef hifi_BaseAssetScriptingInterface_h
#define hifi_BaseAssetScriptingInterface_h
#include <QtCore/QObject>
#include <QtCore/QThread>
#include "AssetClient.h"
#include <shared/MiniPromises.h>
#include "NetworkAccessManager.h"
#include <QtNetwork/QNetworkDiskCache>
class BaseAssetScriptingInterface : public QObject {
Q_OBJECT
public:
const QStringList RESPONSE_TYPES{ "text", "arraybuffer", "json" };
using Promise = MiniPromise::Promise;
QSharedPointer<AssetClient> assetClient();
BaseAssetScriptingInterface(QObject* parent = nullptr);
public slots:
bool isValidPath(QString input) { return AssetUtils::isValidPath(input); }
bool isValidFilePath(QString input) { return AssetUtils::isValidFilePath(input); }
QUrl getATPUrl(QString input) { return AssetUtils::getATPUrl(input); }
QString extractAssetHash(QString input) { return AssetUtils::extractAssetHash(input); }
bool isValidHash(QString input) { return AssetUtils::isValidHash(input); }
QByteArray hashData(const QByteArray& data) { return AssetUtils::hashData(data); }
QString hashDataHex(const QByteArray& data) { return hashData(data).toHex(); }
protected:
bool _cacheReady{ false };
bool initializeCache();
Promise getCacheStatus();
Promise queryCacheMeta(const QUrl& url);
Promise loadFromCache(const QUrl& url, bool decompress = false, const QString& responseType = "arraybuffer");
Promise saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata = QVariantMap());
Promise loadAsset(QString asset, bool decompress, QString responseType);
Promise getAssetInfo(QString asset);
Promise downloadBytes(QString hash);
Promise uploadBytes(const QByteArray& bytes);
Promise compressBytes(const QByteArray& bytes, int level = -1);
Promise convertBytes(const QByteArray& dataByteArray, const QString& responseType);
Promise decompressBytes(const QByteArray& bytes);
Promise symlinkAsset(QString hash, QString path);
};
#endif // hifi_BaseAssetScriptingInterface_h

66
libraries/networking/src/MappingRequest.cpp
View file

@ -51,13 +51,13 @@ QString MappingRequest::getErrorString() const {
}
}
GetMappingRequest::GetMappingRequest(const AssetPath& path) : _path(path.trimmed()) {
GetMappingRequest::GetMappingRequest(const AssetUtils::AssetPath& path) : _path(path.trimmed()) {
};
void GetMappingRequest::doStart() {
// short circuit the request if the path is invalid
if (!isValidFilePath(_path)) {
if (!AssetUtils::isValidFilePath(_path)) {
_error = MappingRequest::InvalidPath;
emit finished(this);
return;
@ -66,17 +66,17 @@ void GetMappingRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->getAssetMapping(_path,
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::AssetNotFound:
case AssetUtils::AssetServerError::AssetNotFound:
_error = NotFound;
break;
default:
@ -86,7 +86,7 @@ void GetMappingRequest::doStart() {
}
if (!_error) {
_hash = message->read(SHA256_HASH_LENGTH).toHex();
_hash = message->read(AssetUtils::SHA256_HASH_LENGTH).toHex();
// check the boolean to see if this request got re-directed
quint8 wasRedirected;
@ -112,7 +112,7 @@ GetAllMappingsRequest::GetAllMappingsRequest() {
void GetAllMappingsRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->getAllAssetMappings(
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
@ -120,7 +120,7 @@ void GetAllMappingsRequest::doStart() {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
default:
@ -135,11 +135,11 @@ void GetAllMappingsRequest::doStart() {
message->readPrimitive(&numberOfMappings);
for (uint32_t i = 0; i < numberOfMappings; ++i) {
auto path = message->readString();
auto hash = message->read(SHA256_HASH_LENGTH).toHex();
BakingStatus status;
auto hash = message->read(AssetUtils::SHA256_HASH_LENGTH).toHex();
AssetUtils::BakingStatus status;
QString lastBakeErrors;
message->readPrimitive(&status);
if (status == BakingStatus::Error) {
if (status == AssetUtils::BakingStatus::Error) {
lastBakeErrors = message->readString();
}
_mappings[path] = { hash, status, lastBakeErrors };
@ -149,7 +149,7 @@ void GetAllMappingsRequest::doStart() {
});
};
SetMappingRequest::SetMappingRequest(const AssetPath& path, const AssetHash& hash) :
SetMappingRequest::SetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash) :
_path(path.trimmed()),
_hash(hash)
{
@ -159,8 +159,8 @@ SetMappingRequest::SetMappingRequest(const AssetPath& path, const AssetHash& has
void SetMappingRequest::doStart() {
// short circuit the request if the hash or path are invalid
auto validPath = isValidFilePath(_path);
auto validHash = isValidHash(_hash);
auto validPath = AssetUtils::isValidFilePath(_path);
auto validHash = AssetUtils::isValidHash(_hash);
if (!validPath || !validHash) {
_error = !validPath ? MappingRequest::InvalidPath : MappingRequest::InvalidHash;
emit finished(this);
@ -170,17 +170,17 @@ void SetMappingRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->setAssetMapping(_path, _hash,
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::PermissionDenied:
case AssetUtils::AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
@ -193,7 +193,7 @@ void SetMappingRequest::doStart() {
});
};
DeleteMappingsRequest::DeleteMappingsRequest(const AssetPathList& paths) : _paths(paths) {
DeleteMappingsRequest::DeleteMappingsRequest(const AssetUtils::AssetPathList& paths) : _paths(paths) {
for (auto& path : _paths) {
path = path.trimmed();
}
@ -203,7 +203,7 @@ void DeleteMappingsRequest::doStart() {
// short circuit the request if any of the paths are invalid
for (auto& path : _paths) {
if (!isValidPath(path)) {
if (!AssetUtils::isValidPath(path)) {
_error = MappingRequest::InvalidPath;
emit finished(this);
return;
@ -213,17 +213,17 @@ void DeleteMappingsRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->deleteAssetMappings(_paths,
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::PermissionDenied:
case AssetUtils::AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
@ -236,7 +236,7 @@ void DeleteMappingsRequest::doStart() {
});
};
RenameMappingRequest::RenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath) :
RenameMappingRequest::RenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath) :
_oldPath(oldPath.trimmed()),
_newPath(newPath.trimmed())
{
@ -246,7 +246,7 @@ RenameMappingRequest::RenameMappingRequest(const AssetPath& oldPath, const Asset
void RenameMappingRequest::doStart() {
// short circuit the request if either of the paths are invalid
if (!isValidFilePath(_oldPath) || !isValidFilePath(_newPath)) {
if (!AssetUtils::isValidFilePath(_oldPath) || !AssetUtils::isValidFilePath(_newPath)) {
_error = InvalidPath;
emit finished(this);
return;
@ -255,17 +255,17 @@ void RenameMappingRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->renameAssetMapping(_oldPath, _newPath,
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::PermissionDenied:
case AssetUtils::AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
@ -278,7 +278,7 @@ void RenameMappingRequest::doStart() {
});
}
SetBakingEnabledRequest::SetBakingEnabledRequest(const AssetPathList& paths, bool enabled) : _paths(paths), _enabled(enabled) {
SetBakingEnabledRequest::SetBakingEnabledRequest(const AssetUtils::AssetPathList& paths, bool enabled) : _paths(paths), _enabled(enabled) {
for (auto& path : _paths) {
path = path.trimmed();
}
@ -288,7 +288,7 @@ void SetBakingEnabledRequest::doStart() {
// short circuit the request if any of the paths are invalid
for (auto& path : _paths) {
if (!isValidPath(path)) {
if (!AssetUtils::isValidPath(path)) {
_error = MappingRequest::InvalidPath;
emit finished(this);
return;
@ -298,17 +298,17 @@ void SetBakingEnabledRequest::doStart() {
auto assetClient = DependencyManager::get<AssetClient>();
_mappingRequestID = assetClient->setBakingEnabled(_paths, _enabled,
[this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer<ReceivedMessage> message) {
[this, assetClient](bool responseReceived, AssetUtils::AssetServerError error, QSharedPointer<ReceivedMessage> message) {
_mappingRequestID = INVALID_MESSAGE_ID;
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
case AssetUtils::AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::PermissionDenied:
case AssetUtils::AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
@ -319,4 +319,4 @@ void SetBakingEnabledRequest::doStart() {
emit finished(this);
});
};
};

40
libraries/networking/src/MappingRequest.h
View file

@ -50,10 +50,10 @@ private:
class GetMappingRequest : public MappingRequest {
Q_OBJECT
public:
GetMappingRequest(const AssetPath& path);
GetMappingRequest(const AssetUtils::AssetPath& path);
AssetHash getHash() const { return _hash; }
AssetPath getRedirectedPath() const { return _redirectedPath; }
AssetUtils::AssetHash getHash() const { return _hash; }
AssetUtils::AssetPath getRedirectedPath() const { return _redirectedPath; }
bool wasRedirected() const { return _wasRedirected; }
signals:
@ -62,21 +62,21 @@ signals:
private:
virtual void doStart() override;
AssetPath _path;
AssetHash _hash;
AssetUtils::AssetPath _path;
AssetUtils::AssetHash _hash;
AssetPath _redirectedPath;
AssetUtils::AssetPath _redirectedPath;
bool _wasRedirected { false };
};
class SetMappingRequest : public MappingRequest {
Q_OBJECT
public:
SetMappingRequest(const AssetPath& path, const AssetHash& hash);
SetMappingRequest(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
AssetPath getPath() const { return _path; }
AssetHash getHash() const { return _hash; }
AssetUtils::AssetPath getPath() const { return _path; }
AssetUtils::AssetHash getHash() const { return _hash; }
signals:
void finished(SetMappingRequest* thisRequest);
@ -84,14 +84,14 @@ signals:
private:
virtual void doStart() override;
AssetPath _path;
AssetHash _hash;
AssetUtils::AssetPath _path;
AssetUtils::AssetHash _hash;
};
class DeleteMappingsRequest : public MappingRequest {
Q_OBJECT
public:
DeleteMappingsRequest(const AssetPathList& path);
DeleteMappingsRequest(const AssetUtils::AssetPathList& path);
signals:
void finished(DeleteMappingsRequest* thisRequest);
@ -99,13 +99,13 @@ signals:
private:
virtual void doStart() override;
AssetPathList _paths;
AssetUtils::AssetPathList _paths;
};
class RenameMappingRequest : public MappingRequest {
Q_OBJECT
public:
RenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath);
RenameMappingRequest(const AssetUtils::AssetPath& oldPath, const AssetUtils::AssetPath& newPath);
signals:
void finished(RenameMappingRequest* thisRequest);
@ -113,8 +113,8 @@ signals:
private:
virtual void doStart() override;
AssetPath _oldPath;
AssetPath _newPath;
AssetUtils::AssetPath _oldPath;
AssetUtils::AssetPath _newPath;
};
class GetAllMappingsRequest : public MappingRequest {
@ -122,7 +122,7 @@ class GetAllMappingsRequest : public MappingRequest {
public:
GetAllMappingsRequest();
AssetMapping getMappings() const { return _mappings; }
AssetUtils::AssetMapping getMappings() const { return _mappings; }
signals:
void finished(GetAllMappingsRequest* thisRequest);
@ -130,13 +130,13 @@ signals:
private:
virtual void doStart() override;
AssetMapping _mappings;
AssetUtils::AssetMapping _mappings;
};
class SetBakingEnabledRequest : public MappingRequest {
Q_OBJECT
public:
SetBakingEnabledRequest(const AssetPathList& path, bool enabled);
SetBakingEnabledRequest(const AssetUtils::AssetPathList& path, bool enabled);
signals:
void finished(SetBakingEnabledRequest* thisRequest);
@ -144,7 +144,7 @@ signals:
private:
virtual void doStart() override;
AssetPathList _paths;
AssetUtils::AssetPathList _paths;
bool _enabled;
};

33
libraries/script-engine/src/ArrayBufferClass.cpp
View file

@ -18,14 +18,19 @@
#include "ArrayBufferClass.h"
static const QString CLASS_NAME = "ArrayBuffer";
// FIXME: Q_DECLARE_METATYPE is global and really belongs in a shared header file, not per .cpp like this
// (see DataViewClass.cpp, etc. which would also have to be updated to resolve)
Q_DECLARE_METATYPE(QScriptClass*)
Q_DECLARE_METATYPE(QByteArray*)
int qScriptClassPointerMetaTypeId = qRegisterMetaType<QScriptClass*>();
int qByteArrayPointerMetaTypeId = qRegisterMetaType<QByteArray*>();
ArrayBufferClass::ArrayBufferClass(ScriptEngine* scriptEngine) :
QObject(scriptEngine),
QScriptClass(scriptEngine),
_scriptEngine(scriptEngine) {
QScriptClass(scriptEngine) {
qScriptRegisterMetaType<QByteArray>(engine(), toScriptValue, fromScriptValue);
QScriptValue global = engine()->globalObject();
@ -144,12 +149,30 @@ QScriptValue ArrayBufferClass::toScriptValue(QScriptEngine* engine, const QByteA
QScriptValue ctor = engine->globalObject().property(CLASS_NAME);
ArrayBufferClass* cls = qscriptvalue_cast<ArrayBufferClass*>(ctor.data());
if (!cls) {
return engine->newVariant(QVariant::fromValue(ba));
if (engine->currentContext()) {
engine->currentContext()->throwError("arrayBufferClass::toScriptValue -- could not get " + CLASS_NAME + " class constructor");
}
return QScriptValue::NullValue;
}
return cls->newInstance(ba);
}
void ArrayBufferClass::fromScriptValue(const QScriptValue& obj, QByteArray& ba) {
ba = qvariant_cast<QByteArray>(obj.data().toVariant());
void ArrayBufferClass::fromScriptValue(const QScriptValue& object, QByteArray& byteArray) {
if (object.isString()) {
// UTF-8 encoded String
byteArray = object.toString().toUtf8();
} else if (object.isArray()) {
// Array of uint8s eg: [ 128, 3, 25, 234 ]
auto Uint8Array = object.engine()->globalObject().property("Uint8Array");
auto typedArray = Uint8Array.construct(QScriptValueList{object});
if (QByteArray* buffer = qscriptvalue_cast<QByteArray*>(typedArray.property("buffer"))) {
byteArray = *buffer;
}
} else if (object.isObject()) {
// ArrayBuffer instance (or any JS class that supports coercion into QByteArray*)
if (QByteArray* buffer = qscriptvalue_cast<QByteArray*>(object.data())) {
byteArray = *buffer;
}
}
}

2
libraries/script-engine/src/ArrayBufferClass.h
View file

@ -41,7 +41,6 @@ public:
QString name() const override;
QScriptValue prototype() const override;
ScriptEngine* getEngine() { return _scriptEngine; }
private:
static QScriptValue construct(QScriptContext* context, QScriptEngine* engine);
@ -56,7 +55,6 @@ private:
QScriptString _name;
QScriptString _byteLength;
ScriptEngine* _scriptEngine;
};
#endif // hifi_ArrayBufferClass_h

425
libraries/script-engine/src/AssetScriptingInterface.cpp
View file

@ -11,102 +11,113 @@
#include "AssetScriptingInterface.h"
#include <QMimeDatabase>
#include <QThread>
#include <QtScript/QScriptEngine>
#include <AssetRequest.h>
#include <AssetUpload.h>
#include <AssetUtils.h>
#include <BaseScriptEngine.h>
#include <MappingRequest.h>
#include <NodeList.h>
#include <RegisteredMetaTypes.h>
#include "ScriptEngine.h"
#include "ScriptEngineLogging.h"
#include <shared/QtHelpers.h>
#include <Gzip.h>
AssetScriptingInterface::AssetScriptingInterface(QScriptEngine* engine) :
_engine(engine)
{
using Promise = MiniPromise::Promise;
AssetScriptingInterface::AssetScriptingInterface(QObject* parent) : BaseAssetScriptingInterface(parent) {
qCDebug(scriptengine) << "AssetScriptingInterface::AssetScriptingInterface" << parent;
MiniPromise::registerMetaTypes(parent);
}
#define JS_VERIFY(cond, error) { if (!this->jsVerify(cond, error)) { return; } }
void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) {
auto handler = jsBindCallback(thisObject(), callback);
QByteArray dataByteArray = data.toUtf8();
auto upload = DependencyManager::get<AssetClient>()->createUpload(dataByteArray);
QObject::connect(upload, &AssetUpload::finished, this, [this, callback](AssetUpload* upload, const QString& hash) mutable {
if (callback.isFunction()) {
QString url = "atp:" + hash;
QScriptValueList args { url, hash };
callback.call(_engine->currentContext()->thisObject(), args);
}
Promise deferred = makePromise(__FUNCTION__);
deferred->ready([=](QString error, QVariantMap result) {
auto url = result.value("url").toString();
auto hash = result.value("hash").toString();
jsCallback(handler, url, hash);
});
connect(upload, &AssetUpload::finished, upload, [this, deferred](AssetUpload* upload, const QString& hash) {
// we are now on the "Resource Manager" thread (and "hash" being a *reference* makes it unsafe to use directly)
Q_ASSERT(QThread::currentThread() == upload->thread());
deferred->resolve({
{ "url", "atp:" + hash },
{ "hash", hash },
});
upload->deleteLater();
});
upload->start();
}
void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) {
auto setMappingRequest = DependencyManager::get<AssetClient>()->createSetMappingRequest(path, hash);
auto handler = jsBindCallback(thisObject(), callback);
auto setMappingRequest = assetClient()->createSetMappingRequest(path, hash);
Promise deferred = makePromise(__FUNCTION__);
deferred->ready([=](QString error, QVariantMap result) {
jsCallback(handler, error, result);
});
QObject::connect(setMappingRequest, &SetMappingRequest::finished, this, [this, callback](SetMappingRequest* request) mutable {
if (callback.isFunction()) {
QString error = request->getErrorString();
QScriptValueList args { error };
callback.call(_engine->currentContext()->thisObject(), args);
}
connect(setMappingRequest, &SetMappingRequest::finished, setMappingRequest, [this, deferred](SetMappingRequest* request) {
Q_ASSERT(QThread::currentThread() == request->thread());
// we are now on the "Resource Manager" thread
QString error = request->getErrorString();
// forward a thread-safe values back to our thread
deferred->handle(error, { { "error", request->getError() } });
request->deleteLater();
});
setMappingRequest->start();
}
void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) {
auto request = DependencyManager::get<AssetClient>()->createGetMappingRequest(path);
QObject::connect(request, &GetMappingRequest::finished, this, [=](GetMappingRequest* request) mutable {
auto result = request->getError();
if (callback.isFunction()) {
if (result == GetMappingRequest::NotFound) {
QScriptValueList args { "", true };
callback.call(_engine->currentContext()->thisObject(), args);
} else if (result == GetMappingRequest::NoError) {
QScriptValueList args { request->getHash(), true };
callback.call(_engine->currentContext()->thisObject(), args);
} else {
qCDebug(scriptengine) << "error -- " << request->getError() << " -- " << request->getErrorString();
QScriptValueList args { "", false };
callback.call(_engine->currentContext()->thisObject(), args);
}
request->deleteLater();
}
});
request->start();
}
void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callback) {
// FIXME: historically this API method failed silently when given a non-atp prefixed
// urlString (or if the AssetRequest failed).
// .. is that by design or could we update without breaking things to provide better feedback to scripts?
if (!urlString.startsWith(ATP_SCHEME)) {
// ... for now at least log a message so user can check logs
qCDebug(scriptengine) << "AssetScriptingInterface::downloadData url must be of form atp:<hash-value>";
return;
}
// Make request to atp
auto path = urlString.right(urlString.length() - ATP_SCHEME.length());
auto parts = path.split(".", QString::SkipEmptyParts);
auto hash = parts.length() > 0 ? parts[0] : "";
QString hash = AssetUtils::extractAssetHash(urlString);
auto handler = jsBindCallback(thisObject(), callback);
auto assetClient = DependencyManager::get<AssetClient>();
auto assetRequest = assetClient->createRequest(hash);
_pendingRequests << assetRequest;
Promise deferred = makePromise(__FUNCTION__);
deferred->ready([=](QString error, QVariantMap result) {
// FIXME: to remain backwards-compatible the signature here is "callback(data, n/a)"
jsCallback(handler, result.value("data").toString(), { { "errorMessage", error } });
});
connect(assetRequest, &AssetRequest::finished, this, [this, callback](AssetRequest* request) mutable {
connect(assetRequest, &AssetRequest::finished, assetRequest, [this, deferred](AssetRequest* request) {
Q_ASSERT(QThread::currentThread() == request->thread());
// we are now on the "Resource Manager" thread
Q_ASSERT(request->getState() == AssetRequest::Finished);
if (request->getError() == AssetRequest::Error::NoError) {
if (callback.isFunction()) {
QString data = QString::fromUtf8(request->getData());
QScriptValueList args { data };
callback.call(_engine->currentContext()->thisObject(), args);
}
QString data = QString::fromUtf8(request->getData());
// forward a thread-safe values back to our thread
deferred->resolve({ { "data", data } });
} else {
// FIXME: propagate error to scripts? (requires changing signature or inverting param order above..)
//deferred->resolve(request->getErrorString(), { { "error", requet->getError() } });
qCDebug(scriptengine) << "AssetScriptingInterface::downloadData ERROR: " << request->getErrorString();
}
request->deleteLater();
_pendingRequests.remove(request);
});
assetRequest->start();
@ -115,12 +126,15 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb
void AssetScriptingInterface::setBakingEnabled(QString path, bool enabled, QScriptValue callback) {
auto setBakingEnabledRequest = DependencyManager::get<AssetClient>()->createSetBakingEnabledRequest({ path }, enabled);
QObject::connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, this, [this, callback](SetBakingEnabledRequest* request) mutable {
if (callback.isFunction()) {
QString error = request->getErrorString();
QScriptValueList args{ error };
callback.call(_engine->currentContext()->thisObject(), args);
}
Promise deferred = jsPromiseReady(makePromise(__FUNCTION__), thisObject(), callback);
connect(setBakingEnabledRequest, &SetBakingEnabledRequest::finished, setBakingEnabledRequest, [this, deferred](SetBakingEnabledRequest* request) {
Q_ASSERT(QThread::currentThread() == request->thread());
// we are now on the "Resource Manager" thread
QString error = request->getErrorString();
// forward thread-safe values back to our thread
deferred->handle(error, {});
request->deleteLater();
});
setBakingEnabledRequest->start();
@ -135,3 +149,298 @@ void AssetScriptingInterface::sendFakedHandshake() {
}
#endif
void AssetScriptingInterface::getMapping(QString asset, QScriptValue callback) {
auto path = AssetUtils::getATPUrl(asset).path();
auto handler = jsBindCallback(thisObject(), callback);
JS_VERIFY(AssetUtils::isValidFilePath(path), "invalid ATP file path: " + asset + "(path:"+path+")");
JS_VERIFY(callback.isFunction(), "expected second parameter to be a callback function");
Promise promise = getAssetInfo(path);
promise->ready([=](QString error, QVariantMap result) {
jsCallback(handler, error, result.value("hash").toString());
});
}
bool AssetScriptingInterface::jsVerify(bool condition, const QString& error) {
if (condition) {
return true;
}
if (context()) {
context()->throwError(error);
} else {
qCDebug(scriptengine) << "WARNING -- jsVerify failed outside of a valid JS context: " + error;
}
return false;
}
QScriptValue AssetScriptingInterface::jsBindCallback(QScriptValue scope, QScriptValue callback) {
QScriptValue handler = ::makeScopedHandlerObject(scope, callback);
QScriptValue value = handler.property("callback");
if (!jsVerify(handler.isObject() && value.isFunction(),
QString("jsBindCallback -- .callback is not a function (%1)").arg(value.toVariant().typeName()))) {
return QScriptValue();
}
return handler;
}
Promise AssetScriptingInterface::jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback) {
auto handler = jsBindCallback(scope, callback);
if (!jsVerify(handler.isValid(), "jsPromiseReady -- invalid callback handler")) {
return nullptr;
}
return promise->ready([this, handler](QString error, QVariantMap result) {
jsCallback(handler, error, result);
});
}
void AssetScriptingInterface::jsCallback(const QScriptValue& handler,
const QScriptValue& error, const QScriptValue& result) {
Q_ASSERT(thread() == QThread::currentThread());
auto errorValue = !error.toBool() ? QScriptValue::NullValue : error;
JS_VERIFY(handler.isObject() && handler.property("callback").isFunction(),
QString("jsCallback -- .callback is not a function (%1)")
.arg(handler.property("callback").toVariant().typeName()));
::callScopedHandlerObject(handler, errorValue, result);
}
void AssetScriptingInterface::jsCallback(const QScriptValue& handler,
const QScriptValue& error, const QVariantMap& result) {
Q_ASSERT(thread() == QThread::currentThread());
Q_ASSERT(handler.engine());
auto engine = handler.engine();
jsCallback(handler, error, engine->toScriptValue(result));
}
void AssetScriptingInterface::deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) {
jsVerify(false, "TODO: deleteAsset API");
}
/**jsdoc
* @typedef {string} Assets.GetOptions.ResponseType
* <p>Available <code>responseType</code> values for use with @{link Assets.getAsset} and @{link Assets.loadFromCache} configuration option. </p>
* <table>
* <thead>
* <tr><th>responseType</th><th>typeof response value</th></tr>
* </thead>
* <tbody>
* <tr><td><code>"text"</code></td><td>contents returned as utf-8 decoded <code>String</code> value</td></tr>
* <tr><td><code>"arraybuffer"</code></td><td>contents as a binary <code>ArrayBuffer</code> object</td></tr>
* <tr><td><code>"json"</code></td><td>contents as a parsed <code>JSON</code> object</td></tr>
* </tbody>
* </table>
*/
void AssetScriptingInterface::getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) {
JS_VERIFY(options.isObject() || options.isString(), "expected request options Object or URL as first parameter");
auto decompress = options.property("decompress").toBool() || options.property("compressed").toBool();
auto responseType = options.property("responseType").toString().toLower();
auto url = options.property("url").toString();
if (options.isString()) {
url = options.toString();
}
if (responseType.isEmpty()) {
responseType = "text";
}
auto asset = AssetUtils::getATPUrl(url).path();
JS_VERIFY(AssetUtils::isValidHash(asset) || AssetUtils::isValidFilePath(asset),
QString("Invalid ATP url '%1'").arg(url));
JS_VERIFY(RESPONSE_TYPES.contains(responseType),
QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | ")));
Promise fetched = jsPromiseReady(makePromise("fetched"), scope, callback);
Promise mapped = makePromise("mapped");
mapped->fail(fetched);
mapped->then([=](QVariantMap result) {
QString hash = result.value("hash").toString();
QString url = result.value("url").toString();
if (!AssetUtils::isValidHash(hash)) {
fetched->reject("internal hash error: " + hash, result);
} else {
Promise promise = loadAsset(hash, decompress, responseType);
promise->mixin(result);
promise->ready([=](QString error, QVariantMap loadResult) {
loadResult["url"] = url; // maintain mapped .url in results (vs. atp:hash returned by loadAsset)
fetched->handle(error, loadResult);
});
}
});
if (AssetUtils::isValidHash(asset)) {
mapped->resolve({
{ "hash", asset },
{ "url", url },
});
} else {
getAssetInfo(asset)->ready(mapped);
}
}
void AssetScriptingInterface::resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) {
const QString& URL{ "url" };
auto url = (options.isString() ? options : options.property(URL)).toString();
auto asset = AssetUtils::getATPUrl(url).path();
JS_VERIFY(AssetUtils::isValidFilePath(asset) || AssetUtils::isValidHash(asset),
"expected options to be an asset URL or request options containing .url property");
jsPromiseReady(getAssetInfo(asset), scope, callback);
}
void AssetScriptingInterface::decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback) {
auto data = options.property("data");
QByteArray dataByteArray = qscriptvalue_cast<QByteArray>(data);
auto responseType = options.property("responseType").toString().toLower();
if (responseType.isEmpty()) {
responseType = "text";
}
Promise completed = jsPromiseReady(makePromise(__FUNCTION__), scope, callback);
Promise decompressed = decompressBytes(dataByteArray);
if (responseType == "arraybuffer") {
decompressed->ready(completed);
} else {
decompressed->ready([=](QString error, QVariantMap result) {
Promise converted = convertBytes(result.value("data").toByteArray(), responseType);
converted->mixin(result);
converted->ready(completed);
});
}
}
namespace {
const int32_t DEFAULT_GZIP_COMPRESSION_LEVEL = -1;
const int32_t MAX_GZIP_COMPRESSION_LEVEL = 9;
}
void AssetScriptingInterface::compressData(QScriptValue options, QScriptValue scope, QScriptValue callback) {
auto data = options.property("data").isValid() ? options.property("data") : options;
QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast<QByteArray>(data);
int level = options.property("level").isNumber() ? options.property("level").toInt32() : DEFAULT_GZIP_COMPRESSION_LEVEL;
JS_VERIFY(level >= DEFAULT_GZIP_COMPRESSION_LEVEL || level <= MAX_GZIP_COMPRESSION_LEVEL, QString("invalid .level %1").arg(level));
jsPromiseReady(compressBytes(dataByteArray, level), scope, callback);
}
void AssetScriptingInterface::putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback) {
auto compress = options.property("compress").toBool() || options.property("compressed").toBool();
auto data = options.isObject() ? options.property("data") : options;
auto rawPath = options.property("path").toString();
auto path = AssetUtils::getATPUrl(rawPath).path();
QByteArray dataByteArray = data.isString() ? data.toString().toUtf8() : qscriptvalue_cast<QByteArray>(data);
JS_VERIFY(path.isEmpty() || AssetUtils::isValidFilePath(path),
QString("expected valid ATP file path '%1' ('%2')").arg(rawPath).arg(path));
JS_VERIFY(dataByteArray.size() > 0,
QString("expected non-zero .data (got %1 / #%2 bytes)").arg(data.toVariant().typeName()).arg(dataByteArray.size()));
// [compressed] => uploaded to server => [mapped to path]
Promise prepared = makePromise("putAsset::prepared");
Promise uploaded = makePromise("putAsset::uploaded");
Promise completed = makePromise("putAsset::completed");
jsPromiseReady(completed, scope, callback);
if (compress) {
Promise compress = compressBytes(dataByteArray, DEFAULT_GZIP_COMPRESSION_LEVEL);
compress->ready(prepared);
} else {
prepared->resolve({{ "data", dataByteArray }});
}
prepared->fail(completed);
prepared->then([=](QVariantMap result) {
Promise upload = uploadBytes(result.value("data").toByteArray());
upload->mixin(result);
upload->ready(uploaded);
});
uploaded->fail(completed);
if (path.isEmpty()) {
uploaded->then(completed);
} else {
uploaded->then([=](QVariantMap result) {
QString hash = result.value("hash").toString();
if (!AssetUtils::isValidHash(hash)) {
completed->reject("path mapping requested, but did not receive valid hash", result);
} else {
Promise link = symlinkAsset(hash, path);
link->mixin(result);
link->ready(completed);
}
});
}
}
void AssetScriptingInterface::queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback) {
QString url = options.isString() ? options.toString() : options.property("url").toString();
JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url));
jsPromiseReady(Parent::queryCacheMeta(url), scope, callback);
}
void AssetScriptingInterface::loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback) {
QString url, responseType;
bool decompress = false;
if (options.isString()) {
url = options.toString();
responseType = "text";
} else {
url = options.property("url").toString();
responseType = options.property("responseType").isValid() ? options.property("responseType").toString() : "text";
decompress = options.property("decompress").toBool() || options.property("compressed").toBool();
}
JS_VERIFY(QUrl(url).isValid(), QString("Invalid URL '%1'").arg(url));
JS_VERIFY(RESPONSE_TYPES.contains(responseType),
QString("Invalid responseType: '%1' (expected: %2)").arg(responseType).arg(RESPONSE_TYPES.join(" | ")));
jsPromiseReady(Parent::loadFromCache(url, decompress, responseType), scope, callback);
}
bool AssetScriptingInterface::canWriteCacheValue(const QUrl& url) {
auto scriptEngine = qobject_cast<ScriptEngine*>(engine());
if (!scriptEngine) {
qCDebug(scriptengine) << __FUNCTION__ << "invalid script engine" << url;
return false;
}
// allow cache writes only from Client, EntityServer and Agent scripts
bool isAllowedContext = (
scriptEngine->isClientScript() ||
scriptEngine->isAgentScript()
);
if (!isAllowedContext) {
qCDebug(scriptengine) << __FUNCTION__ << "invalid context" << scriptEngine->getContext() << url;
return false;
}
return true;
}
void AssetScriptingInterface::saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback) {
JS_VERIFY(options.isObject(), QString("expected options object as first parameter not: %1").arg(options.toVariant().typeName()));
QString url = options.property("url").toString();
QByteArray data = qscriptvalue_cast<QByteArray>(options.property("data"));
QVariantMap headers = qscriptvalue_cast<QVariantMap>(options.property("headers"));
saveToCache(url, data, headers, scope, callback);
}
void AssetScriptingInterface::saveToCache(const QUrl& rawURL, const QByteArray& data, const QVariantMap& metadata, QScriptValue scope, QScriptValue callback) {
QUrl url = rawURL;
if (url.path().isEmpty() && !data.isEmpty()) {
// generate a valid ATP URL from the data -- appending any existing fragment or querystring values
auto atpURL = AssetUtils::getATPUrl(hashDataHex(data));
atpURL.setQuery(url.query());
atpURL.setFragment(url.fragment());
qCDebug(scriptengine) << "autogenerated ATP URL" << url << "=>" << atpURL;
url = atpURL;
}
auto hash = AssetUtils::extractAssetHash(url.toDisplayString());
JS_VERIFY(url.isValid(), QString("Invalid URL '%1'").arg(url.toString()));
JS_VERIFY(canWriteCacheValue(url), "Invalid cache write URL: " + url.toString());
JS_VERIFY(url.scheme() == "atp" || url.scheme() == "cache", "only 'atp' and 'cache' URL schemes supported");
JS_VERIFY(hash.isEmpty() || hash == hashDataHex(data), QString("invalid checksum hash for atp:HASH style URL (%1 != %2)").arg(hash, hashDataHex(data)));
qCDebug(scriptengine) << "saveToCache" << url.toDisplayString() << data << hash << metadata;
jsPromiseReady(Parent::saveToCache(url, data, metadata), scope, callback);
}

104
libraries/script-engine/src/AssetScriptingInterface.h
View file

@ -15,17 +15,23 @@
#define hifi_AssetScriptingInterface_h
#include <QtCore/QObject>
#include <QtCore/QThread>
#include <QtScript/QScriptValue>
#include <QtScript/QScriptable>
#include <AssetClient.h>
#include <NetworkAccessManager.h>
#include <BaseAssetScriptingInterface.h>
#include <BaseScriptEngine.h>
#include <QtNetwork/QNetworkDiskCache>
/**jsdoc
* @namespace Assets
*/
class AssetScriptingInterface : public QObject {
class AssetScriptingInterface : public BaseAssetScriptingInterface, QScriptable {
Q_OBJECT
public:
AssetScriptingInterface(QScriptEngine* engine);
using Parent = BaseAssetScriptingInterface;
AssetScriptingInterface(QObject* parent = nullptr);
/**jsdoc
* Upload content to the connected domain's asset server.
@ -87,21 +93,105 @@ public:
/**jsdoc
* Called when getMapping is complete.
* @callback Assets~getMappingCallback
* @param error {string} error description if the path could not be resolved; otherwise a null value.
* @param assetID {string} hash value if found, else an empty string
* @param success {boolean} false for errors other than "not found", else true
*/
Q_INVOKABLE void getMapping(QString path, QScriptValue callback);
Q_INVOKABLE void setBakingEnabled(QString path, bool enabled, QScriptValue callback);
#if (PR_BUILD || DEV_BUILD)
Q_INVOKABLE void sendFakedHandshake();
#endif
/**jsdoc
* Request Asset data from the ATP Server
* @function Assets.getAsset
* @param {URL|Assets.GetOptions} options An atp: style URL, hash, or relative mapped path; or an {@link Assets.GetOptions} object with request parameters
* @param {Assets~getAssetCallback} scope[callback] A scope callback function to receive (error, results) values
*/
/**jsdoc
* A set of properties that can be passed to {@link Assets.getAsset}.
* @typedef {Object} Assets.GetOptions
* @property {URL} [url] an "atp:" style URL, hash, or relative mapped path to fetch
* @property {string} [responseType=text] the desired reponse type (text | arraybuffer | json)
* @property {boolean} [decompress=false] whether to attempt gunzip decompression on the fetched data
* See: {@link Assets.putAsset} and its .compress=true option
*/
/**jsdoc
* Called when Assets.getAsset is complete.
* @callback Assets~getAssetCallback
* @param {string} error - contains error message or null value if no error occured fetching the asset
* @param {Asset~getAssetResult} result - result object containing, on success containing asset metadata and contents
*/
/**jsdoc
* Result value returned by {@link Assets.getAsset}.
* @typedef {Object} Assets~getAssetResult
* @property {url} [url] the resolved "atp:" style URL for the fetched asset
* @property {string} [hash] the resolved hash for the fetched asset
* @property {string|ArrayBuffer|Object} [response] response data (possibly converted per .responseType value)
* @property {string} [responseType] response type (text | arraybuffer | json)
* @property {string} [contentType] detected asset mime-type (autodetected)
* @property {number} [byteLength] response data size in bytes
* @property {number} [decompressed] flag indicating whether data was decompressed
*/
Q_INVOKABLE void getAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
/**jsdoc
* Upload Asset data to the ATP Server
* @function Assets.putAsset
* @param {Assets.PutOptions} options A PutOptions object with upload parameters
* @param {Assets~putAssetCallback} scope[callback] A scoped callback function invoked with (error, results)
*/
/**jsdoc
* A set of properties that can be passed to {@link Assets.putAsset}.
* @typedef {Object} Assets.PutOptions
* @property {ArrayBuffer|string} [data] byte buffer or string value representing the new asset's content
* @property {string} [path=null] ATP path mapping to automatically create (upon successful upload to hash)
* @property {boolean} [compress=false] whether to gzip compress data before uploading
*/
/**jsdoc
* Called when Assets.putAsset is complete.
* @callback Assets~puttAssetCallback
* @param {string} error - contains error message (or null value if no error occured while uploading/mapping the new asset)
* @param {Asset~putAssetResult} result - result object containing error or result status of asset upload
*/
/**jsdoc
* Result value returned by {@link Assets.putAsset}.
* @typedef {Object} Assets~putAssetResult
* @property {url} [url] the resolved "atp:" style URL for the uploaded asset (based on .path if specified, otherwise on the resulting ATP hash)
* @property {string} [path] the uploaded asset's resulting ATP path (or undefined if no path mapping was assigned)
* @property {string} [hash] the uploaded asset's resulting ATP hash
* @property {boolean} [compressed] flag indicating whether the data was compressed before upload
* @property {number} [byteLength] flag indicating final byte size of the data uploaded to the ATP server
*/
Q_INVOKABLE void putAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void deleteAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void resolveAsset(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void decompressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void compressData(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE bool initializeCache() { return Parent::initializeCache(); }
Q_INVOKABLE bool canWriteCacheValue(const QUrl& url);
Q_INVOKABLE void getCacheStatus(QScriptValue scope, QScriptValue callback = QScriptValue()) {
jsPromiseReady(Parent::getCacheStatus(), scope, callback);
}
Q_INVOKABLE void queryCacheMeta(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void loadFromCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void saveToCache(QScriptValue options, QScriptValue scope, QScriptValue callback = QScriptValue());
Q_INVOKABLE void saveToCache(const QUrl& url, const QByteArray& data, const QVariantMap& metadata,
QScriptValue scope, QScriptValue callback = QScriptValue());
protected:
QSet<AssetRequest*> _pendingRequests;
QScriptEngine* _engine;
QScriptValue jsBindCallback(QScriptValue scope, QScriptValue callback = QScriptValue());
Promise jsPromiseReady(Promise promise, QScriptValue scope, QScriptValue callback = QScriptValue());
void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QVariantMap& result);
void jsCallback(const QScriptValue& handler, const QScriptValue& error, const QScriptValue& result);
bool jsVerify(bool condition, const QString& error);
};
#endif // hifi_AssetScriptingInterface_h

4
libraries/script-engine/src/ScriptEngine.cpp
View file

@ -56,6 +56,7 @@
#include <AnimationObject.h>
#include "ArrayBufferViewClass.h"
#include "AssetScriptingInterface.h"
#include "BatchLoader.h"
#include "BaseScriptEngine.h"
#include "DataViewClass.h"
@ -175,6 +176,7 @@ ScriptEngine::ScriptEngine(Context context, const QString& scriptContents, const
_timerFunctionMap(),
_fileNameString(fileNameString),
_arrayBufferClass(new ArrayBufferClass(this)),
_assetScriptingInterface(new AssetScriptingInterface(this)),
// don't delete `ScriptEngines` until all `ScriptEngine`s are gone
_scriptEngines(DependencyManager::get<ScriptEngines>())
{
@ -704,7 +706,7 @@ void ScriptEngine::init() {
// constants
globalObject().setProperty("TREE_SCALE", newVariant(QVariant(TREE_SCALE)));
registerGlobalObject("Assets", &_assetScriptingInterface);
registerGlobalObject("Assets", _assetScriptingInterface);
registerGlobalObject("Resources", DependencyManager::get<ResourceScriptingInterface>().data());
registerGlobalObject("DebugDraw", &DebugDraw::getInstance());

2
libraries/script-engine/src/ScriptEngine.h
View file

@ -321,7 +321,7 @@ protected:
ArrayBufferClass* _arrayBufferClass;
AssetScriptingInterface _assetScriptingInterface{ this };
AssetScriptingInterface* _assetScriptingInterface;
std::function<bool()> _emitScriptUpdates{ []() { return true; } };

6
libraries/shared/src/BaseScriptEngine.cpp
View file

@ -325,6 +325,12 @@ QScriptValue makeScopedHandlerObject(QScriptValue scopeOrCallback, QScriptValue
} else if (methodOrName.isFunction()) {
scope = scopeOrCallback;
callback = methodOrName;
} else if (!methodOrName.isValid()) {
// instantiate from an existing scoped handler object
if (scopeOrCallback.property("callback").isFunction()) {
scope = scopeOrCallback.property("scope");
callback = scopeOrCallback.property("callback");
}
}
}
auto handler = engine->newObject();

27
libraries/shared/src/shared/MiniPromises.cpp Normal file
View file

@ -0,0 +1,27 @@
//
// Created by Timothy Dedischew on 2017/12/21
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "MiniPromises.h"
#include <QtScript/QScriptEngine>
#include <QtScript/QScriptValue>
int MiniPromise::metaTypeID = qRegisterMetaType<MiniPromise::Promise>("MiniPromise::Promise");
namespace {
void promiseFromScriptValue(const QScriptValue& object, MiniPromise::Promise& promise) {
Q_ASSERT(false);
}
QScriptValue promiseToScriptValue(QScriptEngine *engine, const MiniPromise::Promise& promise) {
return engine->newQObject(promise.get());
}
}
void MiniPromise::registerMetaTypes(QObject* engine) {
auto scriptEngine = qobject_cast<QScriptEngine*>(engine);
qDebug() << "----------------------- MiniPromise::registerMetaTypes ------------" << scriptEngine;
qScriptRegisterMetaType(scriptEngine, promiseToScriptValue, promiseFromScriptValue);
}

278
libraries/shared/src/shared/MiniPromises.h Normal file
View file

@ -0,0 +1,278 @@
//
// Created by Timothy Dedischew on 2017/12/21
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#pragma once
// Minimalist threadsafe Promise-like helper for managing asynchronous results
//
// This class pivots around continuation-style callback handlers:
// auto successCallback = [=](QVariantMap result) { .... }
// auto errorCallback = [=](QString error) { .... }
// auto combinedCallback = [=](QString error, QVariantMap result) { .... }
//
// * Callback Handlers are automatically invoked on the right thread (the Promise's thread).
// * Callbacks can be assigned anytime during a Promise's life and "do the right thing".
// - ie: for code clarity you can define success cases first or choose to maintain time order
// * "Deferred" concept can be used to publish outcomes.
// * "Promise" concept be used to subscribe to outcomes.
//
// See AssetScriptingInterface.cpp for some examples of using to simplify chained async results.
#include <QtCore/QObject>
#include <QtCore/QThread>
#include <QtCore/QVariantMap>
#include <QDebug>
#include "ReadWriteLockable.h"
#include <memory>
class MiniPromise : public QObject, public std::enable_shared_from_this<MiniPromise>, public ReadWriteLockable {
Q_OBJECT
Q_PROPERTY(QString state READ getStateString)
Q_PROPERTY(QString error READ getError)
Q_PROPERTY(QVariantMap result READ getResult)
public:
using HandlerFunction = std::function<void(QString error, QVariantMap result)>;
using SuccessFunction = std::function<void(QVariantMap result)>;
using ErrorFunction = std::function<void(QString error)>;
using HandlerFunctions = QVector<HandlerFunction>;
using Promise = std::shared_ptr<MiniPromise>;
static void registerMetaTypes(QObject* engine);
static int metaTypeID;
MiniPromise() {}
MiniPromise(const QString debugName) { setObjectName(debugName); }
~MiniPromise() {
if (getStateString() == "pending") {
qWarning() << "MiniPromise::~MiniPromise -- destroying pending promise:" << objectName() << _error << _result << "handlers:" << getPendingHandlerCount();
}
}
Promise self() { return shared_from_this(); }
Q_INVOKABLE void executeOnPromiseThread(std::function<void()> function, MiniPromise::Promise root = nullptr) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(
this, "executeOnPromiseThread", Qt::QueuedConnection,
Q_ARG(std::function<void()>, function),
Q_ARG(MiniPromise::Promise, self()));
} else {
function();
}
}
// result aggregation helpers -- eg: deferred->defaults(interimResultMap)->ready(...)
// copy values from the input map, but only for keys that don't already exist
Promise mixin(const QVariantMap& source) {
withWriteLock([&]{
for (const auto& key : source.keys()) {
if (!_result.contains(key)) {
_result[key] = source[key];
}
}
});
return self();
}
// copy values from the input map, replacing any existing keys
Promise assignResult(const QVariantMap& source) {
withWriteLock([&]{
for (const auto& key : source.keys()) {
_result[key] = source[key];
}
});
return self();
}
// callback registration methods
Promise ready(HandlerFunction always) { return finally(always); }
Promise finally(HandlerFunction always) {
if (!_rejected && !_resolved) {
withWriteLock([&]{
_onfinally << always;
});
} else {
executeOnPromiseThread([&]{
always(getError(), getResult());
});
}
return self();
}
Promise fail(ErrorFunction errorOnly) {
return fail([this, errorOnly](QString error, QVariantMap result) {
errorOnly(error);
});
}
Promise fail(HandlerFunction failFunc) {
if (!_rejected) {
withWriteLock([&]{
_onreject << failFunc;
});
} else {
executeOnPromiseThread([&]{
failFunc(getError(), getResult());
});
}
return self();
}
Promise then(SuccessFunction successOnly) {
return then([this, successOnly](QString error, QVariantMap result) {
successOnly(result);
});
}
Promise then(HandlerFunction successFunc) {
if (!_resolved) {
withWriteLock([&]{
_onresolve << successFunc;
});
} else {
executeOnPromiseThread([&]{
successFunc(getError(), getResult());
});
}
return self();
}
// NOTE: first arg may be null (see ES6 .then(null, errorHandler) conventions)
Promise then(SuccessFunction successOnly, ErrorFunction errorOnly) {
if (successOnly) {
then(successOnly);
}
if (errorOnly) {
fail(errorOnly);
}
return self();
}
// helper functions for forwarding results on to a next Promise
Promise ready(Promise next) { return finally(next); }
Promise finally(Promise next) {
return finally([next](QString error, QVariantMap result) {
next->handle(error, result);
});
}
Promise fail(Promise next) {
return fail([next](QString error, QVariantMap result) {
next->reject(error, result);
});
}
Promise then(Promise next) {
return then([next](QString error, QVariantMap result) {
next->resolve(error, result);
});
}
// trigger methods
// handle() automatically resolves or rejects the promise (based on whether an error value occurred)
Promise handle(QString error, const QVariantMap& result) {
if (error.isEmpty()) {
resolve(error, result);
} else {
reject(error, result);
}
return self();
}
Promise resolve(QVariantMap result) {
return resolve(QString(), result);
}
Promise resolve(QString error, const QVariantMap& result) {
setState(true, error, result);
executeOnPromiseThread([&]{
const QString localError{ getError() };
const QVariantMap localResult{ getResult() };
HandlerFunctions resolveHandlers;
HandlerFunctions finallyHandlers;
withReadLock([&]{
resolveHandlers = _onresolve;
finallyHandlers = _onfinally;
});
for (const auto& onresolve : resolveHandlers) {
onresolve(localError, localResult);
}
for (const auto& onfinally : finallyHandlers) {
onfinally(localError, localResult);
}
});
return self();
}
Promise reject(QString error) {
return reject(error, QVariantMap());
}
Promise reject(QString error, const QVariantMap& result) {
setState(false, error, result);
executeOnPromiseThread([&]{
const QString localError{ getError() };
const QVariantMap localResult{ getResult() };
HandlerFunctions rejectHandlers;
HandlerFunctions finallyHandlers;
withReadLock([&]{
rejectHandlers = _onreject;
finallyHandlers = _onfinally;
});
for (const auto& onreject : rejectHandlers) {
onreject(localError, localResult);
}
for (const auto& onfinally : finallyHandlers) {
onfinally(localError, localResult);
}
});
return self();
}
private:
Promise setState(bool resolved, QString error, const QVariantMap& result) {
if (resolved) {
_resolved = true;
} else {
_rejected = true;
}
setError(error);
assignResult(result);
return self();
}
void setError(const QString error) { withWriteLock([&]{ _error = error; }); }
QString getError() const { return resultWithReadLock<QString>([this]() -> QString { return _error; }); }
QVariantMap getResult() const { return resultWithReadLock<QVariantMap>([this]() -> QVariantMap { return _result; }); }
int getPendingHandlerCount() const {
return resultWithReadLock<int>([this]() -> int {
return _onresolve.size() + _onreject.size() + _onfinally.size();
});
}
QString getStateString() const {
return _rejected ? "rejected" :
_resolved ? "resolved" :
getPendingHandlerCount() ? "pending" :
"unknown";
}
QString _error;
QVariantMap _result;
std::atomic<bool> _rejected{false};
std::atomic<bool> _resolved{false};
HandlerFunctions _onresolve;
HandlerFunctions _onreject;
HandlerFunctions _onfinally;
};
Q_DECLARE_METATYPE(MiniPromise::Promise)
inline MiniPromise::Promise makePromise(const QString& hint = QString()) {
if (!QMetaType::isRegistered(qMetaTypeId<MiniPromise::Promise>())) {
int type = qRegisterMetaType<MiniPromise::Promise>();
qDebug() << "makePromise -- registered MetaType<MiniPromise::Promise>:" << type;
}
return std::make_shared<MiniPromise>(hint);
}

502
scripts/developer/tests/unit_tests/assetUnitTests.js Normal file
View file

@ -0,0 +1,502 @@
/* eslint-env jasmine */
if (!Entities.canWriteAssets()) {
Window.alert('!Entities.canWriteAssets -- please goto a domain with asset rights and reload this test script');
throw new Error('!Entities.canWriteAssets');
}
instrument_testrunner(true);
describe("Assets", function () {
var context = {
memo: {},
cache: {},
definedHash: null,
definedPath: null,
definedContent: null,
};
var ATP_TIMEOUT_MS = 1000; // ms
var NULL_HASH = new Array(64+1).join('0'); // 64 hex
var SAMPLE_FOLDER = '/assetUnitTests';
var SAMPLE_FILE_PATH = SAMPLE_FOLDER + "/test.json";
var SAMPLE_CONTENTS = 'Test Run on ' + JSON.stringify(Date());
var SAMPLE_JSON = JSON.stringify({ content: SAMPLE_CONTENTS });
var SAMPLE_FLOATS = [ Math.PI, 1.1, 2.2, 3.3 ];
var SAMPLE_BUFFER = new Float32Array(SAMPLE_FLOATS).buffer;
var IS_ASSET_HASH_REGEX = /^[a-f0-9]{64}$/i;
var IS_ASSET_URL = /^atp:/;
it('Entities.canWriteAssets', function() {
expect(Entities.canWriteAssets()).toBe(true);
});
describe('extractAssetHash(input)', function() {
// new helper method that provides a catch-all way to get at the sha256 hash
// considering the multiple, different ways ATP hash URLs are found across the
// system and in content.
var POSITIVE_TEST_URLS = [
'atp:HASH',
'atp:HASH.obj',
'atp:HASH.fbx#cache-buster',
'atp:HASH.fbx?cache-buster',
'HASH'
];
var NEGATIVE_TEST_URLS = [
'asdf',
'http://domain.tld',
'/path/filename.fbx',
'atp:/path/filename.fbx?#HASH',
'atp:/.baked/HASH/asset.fbx',
''
];
it("POSITIVE_TEST_URLS", function() {
POSITIVE_TEST_URLS.forEach(function(url) {
url = url.replace('HASH', NULL_HASH);
expect([url, Assets.extractAssetHash(url)].join('|')).toEqual([url, NULL_HASH].join('|'));
});
});
it("NEGATIVE_TEST_URLS", function() {
NEGATIVE_TEST_URLS.forEach(function(url) {
expect(Assets.extractAssetHash(url.replace('HASH', NULL_HASH))).toEqual('');
});
});
});
// new APIs
describe('.putAsset', function() {
it("data", function(done) {
Assets.putAsset(SAMPLE_CONTENTS, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.hash).toMatch(IS_ASSET_HASH_REGEX);
context.memo.stringURL = result.url;
done();
});
});
it('nopath.text', function(done) {
Assets.putAsset({
data: SAMPLE_CONTENTS,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.hash).toMatch(IS_ASSET_HASH_REGEX);
context.memo.textHash = result.hash;
context.memo.textURL = result.url;
done();
});
});
it('path.text', function(done) {
var samplePath = SAMPLE_FOLDER + '/content.json';
Assets.putAsset({
path: samplePath,
data: SAMPLE_JSON,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.path).toEqual(samplePath);
expect(result.hash).toMatch(IS_ASSET_HASH_REGEX);
context.memo.jsonPath = result.path;
done();
});
});
it('path.buffer', function(done) {
var samplePath = SAMPLE_FOLDER + '/content.buffer';
Assets.putAsset({
path: samplePath,
data: SAMPLE_BUFFER,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.path).toEqual(samplePath);
expect(result.hash).toMatch(IS_ASSET_HASH_REGEX);
expect(result.byteLength).toEqual(SAMPLE_BUFFER.byteLength);
context.memo.bufferURL = result.url;
context.memo.bufferHash = result.hash;
done();
});
});
it('path.json.gz', function(done) {
var samplePath = SAMPLE_FOLDER + '/content.json.gz';
Assets.putAsset({
path: samplePath,
data: SAMPLE_JSON,
compress: true,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.path).toEqual(samplePath);
expect(result.hash).toMatch(IS_ASSET_HASH_REGEX);
context.memo.jsonCompressedPath = result.path;
done();
});
});
});
// it('.deleteAsset(options, {callback(error, result)})', function() { pending(); });
it('.resolveAsset', function(done) {
expect(context.memo.bufferURL).toBeDefined();
Assets.resolveAsset(context.memo.bufferURL, function(error, result) {
expect(error).toBe(null);
expect(result.url).toEqual(context.memo.bufferURL);
expect(result.hash).toEqual(context.memo.bufferHash);
done();
});
});
describe('.getAsset', function() {
it('path/', function(done) {
expect(context.memo.jsonPath).toBeDefined();
Assets.getAsset(context.memo.jsonPath, function(error, result) {
expect(error).toBe(null);
expect(result.response).toEqual(SAMPLE_JSON);
done();
});
});
it('text', function(done) {
expect(context.memo.textURL).toBeDefined();
Assets.getAsset({
url: context.memo.textURL,
responseType: 'text',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response).toEqual(SAMPLE_CONTENTS);
expect(result.url).toEqual(context.memo.textURL);
done();
});
});
it('arraybuffer', function(done) {
expect(context.memo.bufferURL).toBeDefined();
Assets.getAsset({
url: context.memo.bufferURL,
responseType: 'arraybuffer',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response.byteLength).toEqual(SAMPLE_BUFFER.byteLength);
var expected = [].slice.call(new Float32Array(SAMPLE_BUFFER)).join(',');
var actual = [].slice.call(new Float32Array(result.response)).join(',');
expect(actual).toEqual(expected);
done();
});
});
it('json', function(done) {
expect(context.memo.jsonPath).toBeDefined();
Assets.getAsset({
url: context.memo.jsonPath,
responseType: 'json',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response.content).toEqual(SAMPLE_CONTENTS);
done();
});
});
it('json.gz', function(done) {
expect(context.memo.jsonCompressedPath).toBeDefined();
Assets.getAsset({
url: context.memo.jsonCompressedPath,
responseType: 'json',
decompress: true,
}, function(error, result) {
expect(error).toBe(null);
expect(result.decompressed).toBe(true);
expect(result.response.content).toEqual(SAMPLE_CONTENTS);
done();
});
});
});
// cache APIs
it('.getCacheStatus', function(done) {
Assets.getCacheStatus(function(error, result) {
expect(error).toBe(null);
expect(result.cacheSize).toBeGreaterThan(0);
expect(result.maximumCacheSize).toBeGreaterThan(0);
done();
});
});
describe('.saveToCache', function() {
it(".data", function(done) {
Assets.saveToCache({ data: SAMPLE_CONTENTS }, function(error, result) {
expect(error).toBe(null);
expect(result.success).toBe(true);
context.cache.textURL = result.url;
done();
});
});
it('.url', function(done) {
var sampleURL = 'atp:' + SAMPLE_FOLDER + '/cache.json';
Assets.saveToCache({
url: sampleURL,
data: SAMPLE_JSON,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.url).toEqual(sampleURL);
expect(result.success).toBe(true);
context.cache.jsonURL = result.url;
done();
});
});
it('.buffer', function(done) {
var sampleURL = 'atp:' + SAMPLE_FOLDER + '/cache.buffer';
Assets.saveToCache({
url: sampleURL,
data: SAMPLE_BUFFER,
}, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.url).toEqual(sampleURL);
expect(result.success).toBe(true);
expect(result.byteLength).toEqual(SAMPLE_BUFFER.byteLength);
context.cache.bufferURL = result.url;
done();
});
});
});
it('.queryCacheMeta', function(done) {
expect(context.cache.bufferURL).toBeDefined();
Assets.queryCacheMeta(context.cache.bufferURL, function(error, result) {
expect(error).toBe(null);
expect(result.url).toMatch(IS_ASSET_URL);
expect(result.url).toEqual(context.cache.bufferURL);
done();
});
});
describe('.loadFromCache', function() {
it('urlString', function(done) {
expect(context.cache.jsonURL).toBeDefined();
Assets.loadFromCache(context.cache.jsonURL, function(error, result) {
expect(error).toBe(null);
expect(result.response).toEqual(SAMPLE_JSON);
done();
});
});
it('.responseType=text', function(done) {
expect(context.cache.textURL).toBeDefined();
Assets.loadFromCache({
url: context.cache.textURL,
responseType: 'text',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response).toEqual(SAMPLE_CONTENTS);
done();
});
});
it('.responseType=arraybuffer', function(done) {
expect(context.cache.bufferURL).toBeDefined();
Assets.loadFromCache({
url: context.cache.bufferURL,
responseType: 'arraybuffer',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response.byteLength).toEqual(SAMPLE_BUFFER.byteLength);
var expected = [].slice.call(new Float32Array(SAMPLE_BUFFER)).join(',');
var actual = [].slice.call(new Float32Array(result.response)).join(',');
expect(actual).toEqual(expected);
done();
});
});
it('.responseType=json', function(done) {
expect(context.cache.jsonURL).toBeDefined();
Assets.loadFromCache({
url: context.cache.jsonURL,
responseType: 'json',
}, function(error, result) {
expect(error).toBe(null);
expect(result.response.content).toEqual(SAMPLE_CONTENTS);
done();
});
});
});
// existing newly-mapped APIs
it('.getATPUrl', function() {
expect(Assets.getATPUrl(SAMPLE_FILE_PATH)).toEqual('atp:' + SAMPLE_FILE_PATH);
expect(Assets.getATPUrl(NULL_HASH)).toEqual('atp:' + NULL_HASH);
expect(Assets.getATPUrl("/file.ext?a=b#c=d")).toEqual('atp:/file.ext?a=b#c=d');
expect(Assets.getATPUrl("atp:xxx")).toEqual('');
expect(Assets.getATPUrl("xxx")).toEqual('');
});
xit('.isValidPath', function(done) { pending(); });
xit('.isValidFilePath', function() { pending(); });
xit('.isValidHash', function() { pending(); });
xit('.setBakingEnabled', function() { pending(); });
it("Assets.uploadData(data, {callback(url, hash)})", function (done) {
Assets.uploadData(SAMPLE_CONTENTS, function(url, hash) {
expect(url).toMatch(IS_ASSET_URL);
expect(hash).toMatch(IS_ASSET_HASH_REGEX);
context.definedHash = hash; // used in later tests
context.definedContent = SAMPLE_CONTENTS;
print('context.definedHash = ' + context.definedHash);
print('context.definedContent = ' + context.definedContent);
done();
});
});
it("Assets.setMapping", function (done) {
expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX);
Assets.setMapping(SAMPLE_FILE_PATH, context.definedHash, function(error) {
if (error) error += ' ('+JSON.stringify([SAMPLE_FILE_PATH, context.definedHash])+')';
expect(error).toBe(null);
context.definedPath = SAMPLE_FILE_PATH;
print('context.definedPath = ' + context.definedPath);
done();
});
});
it("Assets.getMapping", function (done) {
expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf');
expect(context.definedPath).toMatch(/^\//, 'asfasdf');
Assets.getMapping(context.definedPath, function(error, hash) {
if (error) error += ' ('+JSON.stringify([context.definedPath])+')';
expect(error).toBe(null);
expect(hash).toMatch(IS_ASSET_HASH_REGEX);
expect(hash).toEqual(context.definedHash);
done();
});
});
it('.getMapping(nullHash)', function(done) {
// FIXME: characterization test -- current behavior is that downloadData silently fails
// when given an asset that doesn't exist
Assets.downloadData(NULL_HASH, function(result) {
throw new Error("this callback 'should' not be triggered");
});
setTimeout(function() { done(); }, ATP_TIMEOUT_MS);
});
it('.downloadData(hash)', function(done) {
expect(context.definedHash).toMatch(IS_ASSET_HASH_REGEX, 'asfasdf');
Assets.downloadData('atp:' + context.definedHash, function(result) {
expect(result).toEqual(context.definedContent);
done();
});
});
it('.downloadData(nullHash)', function(done) {
// FIXME: characterization test -- current behavior is that downloadData silently fails
// when given an asset doesn't exist
Assets.downloadData(NULL_HASH, function(result) {
throw new Error("this callback 'should' not be triggered");
});
setTimeout(function() { done(); }, ATP_TIMEOUT_MS);
});
describe('exceptions', function() {
describe('.getAsset', function() {
it('-- invalid data.gz', function(done) {
expect(context.memo.jsonPath).toBeDefined();
Assets.getAsset({
url: context.memo.jsonPath,
responseType: 'json',
decompress: true,
}, function(error, result) {
expect(error).toMatch(/gunzip error/);
done();
});
});
});
describe('.setMapping', function() {
it('-- invalid path', function(done) {
Assets.setMapping('foo', NULL_HASH, function(error/*, result*/) {
expect(error).toEqual('Path is invalid');
/* expect(result).not.toMatch(IS_ASSET_URL); */
done();
});
});
it('-- invalid hash', function(done) {
Assets.setMapping(SAMPLE_FILE_PATH, 'bar', function(error/*, result*/) {
expect(error).toEqual('Hash is invalid');
/* expect(result).not.toMatch(IS_ASSET_URL); */
done();
});
});
});
describe('.getMapping', function() {
it('-- invalid path throws immediate', function() {
expect(function() {
Assets.getMapping('foo', function(error, hash) {
throw new Error("should never make it here...");
});
throw new Error("should never make it here...");
}).toThrowError(/invalid.*path/i);
});
it('-- non-existing path', function(done) {
Assets.getMapping('/foo/bar/'+Date.now(), function(error, hash) {
expect(error).toEqual('Asset not found');
expect(hash).not.toMatch(IS_ASSET_HASH_REGEX);
done();
});
});
});
});
});
// ----------------------------------------------------------------------------
// this stuff allows the unit tests to be loaded indepenently and/or as part of testRunner.js execution
function run() {}
function instrument_testrunner(force) {
if (force || typeof describe === 'undefined') {
var oldPrint = print;
window = new OverlayWebWindow({
title: 'assetUnitTests.js',
width: 640,
height: 480,
source: 'about:blank',
});
Script.scriptEnding.connect(window, 'close');
window.closed.connect(Script, 'stop');
// wait for window (ready) and test runner (ran) both to initialize
var ready = false;
var ran = false;
window.webEventReceived.connect(function() { ready = true; maybeRun(); });
run = function() { ran = true; maybeRun(); };
window.setURL([
'data:text/html;text,',
'<body style="height:100%;width:100%;background:#eee;whitespace:pre;font-size:8px;">',
'<pre id=output></pre><div style="height:2px;"></div>',
'<body>',
'<script>('+function(){
window.addEventListener("DOMContentLoaded",function(){
setTimeout(function() {
EventBridge.scriptEventReceived.connect(function(msg){
output.innerHTML += msg;
window.scrollTo(0, 1e10);
document.body.scrollTop = 1e10;
});
EventBridge.emitWebEvent('ready');
}, 1000);
});
}+')();</script>',
].join('\n'));
print = function() {
var str = [].slice.call(arguments).join(' ');
if (ready) {
window.emitScriptEvent(str + '\n');
} else {
oldPrint('!ready', str);
}
};
// Include testing library
Script.include('../../libraries/jasmine/jasmine.js');
Script.include('../../libraries/jasmine/hifi-boot.js');
function maybeRun() {
if (!ran || !ready) {
return oldPrint('doRun -- waiting for both script and web window to become available');
}
if (!force) {
// invoke Script.stop (after any async tests complete)
jasmine.getEnv().addReporter({ jasmineDone: Script.stop });
} else {
jasmine.getEnv().addReporter({ jasmineDone: function() { print("JASMINE DONE"); } });
}
// Run the tests
jasmine.getEnv().execute();
};
}
}
run();

2
tools/atp-client/src/ATPClientApp.cpp
View file

@ -361,7 +361,7 @@ void ATPClientApp::lookupAsset() {
request->start();
}
void ATPClientApp::download(AssetHash hash) {
void ATPClientApp::download(AssetUtils::AssetHash hash) {
auto assetClient = DependencyManager::get<AssetClient>();
auto assetRequest = new AssetRequest(hash);

2
tools/atp-client/src/ATPClientApp.h
View file

@ -45,7 +45,7 @@ private:
void setMapping(QString hash);
void lookupAsset();
void listAssets();
void download(AssetHash hash);
void download(AssetUtils::AssetHash hash);
void finish(int exitCode);
bool _verbose;