Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-13 06:47:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #78 from birarda/atp-mappings

Browse source 
client side handling of mappings, persistence of mappings in AssetServer
This commit is contained in:
Ryan Huffman 2016-03-08 13:12:38 -08:00
commit 4d49283f25
19 changed files with 400 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

331
assignment-client/src/assets/AssetServer.cpp
View file

@ -12,13 +12,14 @@
#include "AssetServer.h"
#include <QCoreApplication>
#include <QCryptographicHash>
#include <QDateTime>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QString>
#include <QtCore/QCoreApplication>
#include <QtCore/QCryptographicHash>
#include <QtCore/QDateTime>
#include <QtCore/QDir>
#include <QtCore/QFile>
#include <QtCore/QFileInfo>
#include <QtCore/QJsonDocument>
#include <QtCore/QString>
#include "NetworkLogging.h"
#include "NodeType.h"
@ -57,6 +58,8 @@ void AssetServer::run() {
ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer);
}
static const QString ASSET_FILES_SUBDIR = "files";
void AssetServer::completeSetup() {
auto nodeList = DependencyManager::get<NodeList>();
@ -97,71 +100,81 @@ void AssetServer::completeSetup() {
qDebug() << "Creating resources directory";
_resourcesDirectory.mkpath(".");
_filesDirectory = _resourcesDirectory;
bool noExistingAssets = !_resourcesDirectory.exists() || _resourcesDirectory.entryList(QDir::Files).size() == 0;
if (noExistingAssets) {
qDebug() << "Asset resources directory empty, searching for existing asset resources to migrate";
QString oldDataDirectory = QCoreApplication::applicationDirPath();
const QString OLD_RESOURCES_PATH = "assets";
auto oldResourcesDirectory = QDir(oldDataDirectory).filePath("resources/" + OLD_RESOURCES_PATH);
if (QDir(oldResourcesDirectory).exists()) {
qDebug() << "Existing assets found in " << oldResourcesDirectory << ", copying to " << _resourcesDirectory;
QDir resourcesParentDirectory = _resourcesDirectory.filePath("..");
if (!resourcesParentDirectory.exists()) {
qDebug() << "Creating data directory " << resourcesParentDirectory.absolutePath();
resourcesParentDirectory.mkpath(".");
}
auto files = QDir(oldResourcesDirectory).entryList(QDir::Files);
for (auto& file : files) {
auto from = oldResourcesDirectory + QDir::separator() + file;
auto to = _resourcesDirectory.absoluteFilePath(file);
qDebug() << "\tCopying from " << from << " to " << to;
QFile::copy(from, to);
}
}
if (!_resourcesDirectory.mkpath(ASSET_FILES_SUBDIR) || !_filesDirectory.cd(ASSET_FILES_SUBDIR)) {
qCritical() << "Unable to create file directory for asset-server files. Stopping assignment.";
setFinished(true);
return;
}
qDebug() << "Serving files from: " << _resourcesDirectory.path();
// Scan for new files
qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) {
qDebug() << "Found file: " << filename;
if (!fileInfo.isReadable()) {
qDebug() << "\tCan't open file for reading: " << filename;
continue;
}
// load whatever mappings we currently have from the local file
loadMappingsFromFile();
qInfo() << "Serving files from: " << _filesDirectory.path();
// Read file
QFile file { fileInfo.absoluteFilePath() };
file.open(QFile::ReadOnly);
QByteArray data = file.readAll();
// Check the asset directory to output some information about what we have
auto files = _filesDirectory.entryList(QDir::Files);
auto hash = hashData(data);
auto hexHash = hash.toHex();
QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}$" };
auto hashedFiles = files.filter(hashFileRegex);
qDebug() << "\tMoving " << filename << " to " << hexHash;
file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix());
}
}
qInfo() << "There are" << hashedFiles.size() << "asset files in the asset directory.";
performMappingMigration();
nodeList->addNodeTypeToInterestSet(NodeType::Agent);
}
void AssetServer::performMappingMigration() {
QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\.[\\w]+)+$" };
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
for (const auto& fileInfo : files) {
if (hashFileRegex.exactMatch(fileInfo.fileName())) {
// we have a pre-mapping file that we should migrate to the new mapping system
qDebug() << "Migrating pre-mapping file" << fileInfo.fileName();
// rename the file to the same name with no extension
QFile oldFile { fileInfo.absoluteFilePath() };
auto oldAbsolutePath = fileInfo.absoluteFilePath();
auto oldFilename = fileInfo.fileName();
auto hash = oldFilename.left(SHA256_HASH_HEX_LENGTH);
auto fullExtension = oldFilename.mid(oldFilename.indexOf('.'));
qDebug() << "\tMoving" << oldAbsolutePath << "to" << oldAbsolutePath.replace(fullExtension, "");
bool renamed = oldFile.copy(_filesDirectory.filePath(hash));
if (!renamed) {
qWarning() << "\tCould not migrate pre-mapping file" << fileInfo.fileName();
} else {
qDebug() << "\tRenamed pre-mapping file" << fileInfo.fileName();
// add a new mapping with the old extension and a truncated version of the hash
static const int TRUNCATED_HASH_NUM_CHAR = 16;
auto fakeFileName = hash.left(TRUNCATED_HASH_NUM_CHAR) + fullExtension;
qDebug() << "\tAdding a migration mapping from" << fakeFileName << "to" << hash;
auto it = _fileMappings.find(fakeFileName);
if (it == _fileMappings.end()) {
_fileMappings[fakeFileName] = hash;
if (writeMappingsToFile()) {
// mapping added and persisted, we can remove the migrated file
oldFile.remove();
qDebug() << "\tMigration completed for" << oldFilename;
}
} else {
qDebug() << "\tCould not add migration mapping for" << hash << "since a mapping for" << fakeFileName
<< "already exists.";
}
}
}
}
}
void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
MessageID messageID;
@ -175,46 +188,19 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> me
switch (operationType) {
case AssetMappingOperationType::Get: {
QString assetPath = message->readString();
auto it = _fileMapping.find(assetPath);
if (it != _fileMapping.end()) {
auto assetHash = it->second;
qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash;
replyPacket->writePrimitive(AssetServerError::NoError);
//replyPacket->write(assetHash.toLatin1().toHex());
replyPacket->writeString(assetHash.toLatin1());
}
else {
qDebug() << "Mapping not found for: " << assetPath;
replyPacket->writePrimitive(AssetServerError::AssetNotFound);
}
handleGetMappingOperation(*message, senderNode, *replyPacket);
break;
}
case AssetMappingOperationType::GetAll: {
replyPacket->writePrimitive(AssetServerError::NoError);
auto count = _fileMapping.size();
replyPacket->writePrimitive(count);
for (auto& kv : _fileMapping) {
replyPacket->writeString(kv.first);
replyPacket->writeString(kv.second);
}
handleGetAllMappingOperation(*message, senderNode, *replyPacket);
break;
}
case AssetMappingOperationType::Set: {
QString assetPath = message->readString();
//auto assetHash = message->read(SHA256_HASH_LENGTH);
auto assetHash = message->readString();
_fileMapping[assetPath] = assetHash;
replyPacket->writePrimitive(AssetServerError::NoError);
handleSetMappingOperation(*message, senderNode, *replyPacket);
break;
}
case AssetMappingOperationType::Delete: {
QString assetPath = message->readString();
bool removed = _fileMapping.erase(assetPath) > 0;
replyPacket->writePrimitive(AssetServerError::NoError);
handleDeleteMappingOperation(*message, senderNode, *replyPacket);
break;
}
}
@ -223,20 +209,75 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> me
nodeList->sendPacketList(std::move(replyPacket), *senderNode);
}
void AssetServer::handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
QString assetPath = message.readString();
auto it = _fileMappings.find(assetPath);
if (it != _fileMappings.end()) {
auto assetHash = it->toString();
qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash;
replyPacket.writePrimitive(AssetServerError::NoError);
replyPacket.write(QByteArray::fromHex(assetHash.toUtf8()));
}
else {
qDebug() << "Mapping not found for: " << assetPath;
replyPacket.writePrimitive(AssetServerError::AssetNotFound);
}
}
void AssetServer::handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
replyPacket.writePrimitive(AssetServerError::NoError);
auto count = _fileMappings.size();
replyPacket.writePrimitive(count);
for (auto it = _fileMappings.cbegin(); it != _fileMappings.cend(); ++ it) {
replyPacket.writeString(it.key());
replyPacket.write(QByteArray::fromHex(it.value().toString().toUtf8()));
}
}
void AssetServer::handleSetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
if (senderNode->getCanRez()) {
QString assetPath = message.readString();
auto assetHash = message.read(SHA256_HASH_LENGTH).toHex();
if (setMapping(assetPath, assetHash)) {
replyPacket.writePrimitive(AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
}
}
void AssetServer::handleDeleteMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket) {
if (senderNode->getCanRez()) {
QString assetPath = message.readString();
if (deleteMapping(assetPath)) {
replyPacket.writePrimitive(AssetServerError::NoError);
} else {
replyPacket.writePrimitive(AssetServerError::MappingOperationFailed);
}
} else {
replyPacket.writePrimitive(AssetServerError::PermissionDenied);
}
}
void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
QByteArray assetHash;
MessageID messageID;
uint8_t extensionLength;
if (message->getSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID) + sizeof(extensionLength))) {
if (message->getSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID))) {
qDebug() << "ERROR bad file request";
return;
}
message->readPrimitive(&messageID);
assetHash = message->readWithoutCopy(SHA256_HASH_LENGTH);
message->readPrimitive(&extensionLength);
QByteArray extension = message->read(extensionLength);
auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply);
@ -245,7 +286,7 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, Sh
replyPacket->writePrimitive(messageID);
replyPacket->write(assetHash);
QString fileName = QString(hexHash) + "." + extension;
QString fileName = QString(hexHash);
QFileInfo fileInfo { _resourcesDirectory.filePath(fileName) };
if (fileInfo.exists() && fileInfo.isReadable()) {
@ -263,7 +304,7 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<ReceivedMessage> message, Sh
void AssetServer::handleAssetGet(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
auto minSize = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(uint8_t) + sizeof(DataOffset) + sizeof(DataOffset));
auto minSize = qint64(sizeof(MessageID) + SHA256_HASH_LENGTH + sizeof(DataOffset) + sizeof(DataOffset));
if (message->getSize() < minSize) {
qDebug() << "ERROR bad file request";
@ -271,7 +312,7 @@ void AssetServer::handleAssetGet(QSharedPointer<ReceivedMessage> message, Shared
}
// Queue task
auto task = new SendAssetTask(message, senderNode, _resourcesDirectory);
auto task = new SendAssetTask(message, senderNode, _filesDirectory);
_taskPool.start(task);
}
@ -280,7 +321,7 @@ void AssetServer::handleAssetUpload(QSharedPointer<ReceivedMessage> message, Sha
if (senderNode->getCanRez()) {
qDebug() << "Starting an UploadAssetTask for upload from" << uuidStringWithoutCurlyBraces(senderNode->getUUID());
auto task = new UploadAssetTask(message, senderNode, _resourcesDirectory);
auto task = new UploadAssetTask(message, senderNode, _filesDirectory);
_taskPool.start(task);
} else {
// this is a node the domain told us is not allowed to rez entities
@ -371,8 +412,92 @@ void AssetServer::sendStatsPacket() {
ThreadedAssignment::addPacketStatsAndSendStatsPacket(serverStats);
}
void AssetServer::loadMappingFromFile() {
static const QString MAP_FILE_NAME = "map.json";
void AssetServer::loadMappingsFromFile() {
auto mapFilePath = _resourcesDirectory.absoluteFilePath(MAP_FILE_NAME);
QFile mapFile { mapFilePath };
if (mapFile.exists()) {
if (mapFile.open(QIODevice::ReadOnly)) {
QJsonParseError error;
auto jsonDocument = QJsonDocument::fromJson(mapFile.readAll(), &error);
if (error.error == QJsonParseError::NoError) {
_fileMappings = jsonDocument.object().toVariantHash();
qInfo() << "Loaded" << _fileMappings.count() << "mappings from map file at" << mapFilePath;
return;
}
}
qCritical() << "Failed to read mapping file at" << mapFilePath << "- assignment with not continue.";
setFinished(true);
} else {
qInfo() << "No existing mappings loaded from file since no file was found at" << mapFilePath;
}
}
void AssetServer::writeMappingToFile() {
bool AssetServer::writeMappingsToFile() {
auto mapFilePath = _resourcesDirectory.absoluteFilePath(MAP_FILE_NAME);
QFile mapFile { mapFilePath };
if (mapFile.open(QIODevice::WriteOnly)) {
auto jsonObject = QJsonObject::fromVariantHash(_fileMappings);
QJsonDocument jsonDocument { jsonObject };
if (mapFile.write(jsonDocument.toJson()) != -1) {
qDebug() << "Wrote JSON mappings to file at" << mapFilePath;
return true;
} else {
qWarning() << "Failed to write JSON mappings to file at" << mapFilePath;
}
} else {
qWarning() << "Failed to open map file at" << mapFilePath;
}
return false;
}
bool AssetServer::setMapping(AssetPath path, AssetHash hash) {
// remember what the old mapping was in case persistence fails
auto oldMapping = _fileMappings.value(path).toString();
// update the in memory QHash
_fileMappings[path] = hash;
// attempt to write to file
if (writeMappingsToFile()) {
// persistence succeeded, we are good to go
return true;
} else {
// failed to persist this mapping to file - put back the old one in our in-memory representation
if (oldMapping.isEmpty()) {
_fileMappings.remove(path);
} else {
_fileMappings[path] = oldMapping;
}
return false;
}
}
bool AssetServer::deleteMapping(AssetPath path) {
// keep the old mapping in case the delete fails
auto oldMapping = _fileMappings.take(path);
if (!oldMapping.isNull()) {
// deleted the old mapping, attempt to persist to file
if (writeMappingsToFile()) {
// persistence succeeded we are good to go
return true;
} else {
// we didn't delete the previous mapping, put it back in our in-memory representation
_fileMappings[path] = oldMapping.toString();
}
}
return false;
}

23
assignment-client/src/assets/AssetServer.h
View file

@ -39,14 +39,31 @@ private slots:
void sendStatsPacket();
private:
using Mappings = QVariantHash;
void loadMappingFromFile();
void writeMappingToFile();
void handleGetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket);
void handleGetAllMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket);
void handleSetMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket);
void handleDeleteMappingOperation(ReceivedMessage& message, SharedNodePointer senderNode, NLPacketList& replyPacket);
// Mapping file operations must be called from main assignment thread only
void loadMappingsFromFile();
bool writeMappingsToFile();
/// Set the mapping for path to hash
bool setMapping(AssetPath path, AssetHash hash);
/// Delete mapping `path`. Return `true` if mapping existed, else `false`.
bool deleteMapping(AssetPath path);
static void writeError(NLPacketList* packetList, AssetServerError error);
AssetMapping _fileMapping;
void performMappingMigration();
Mappings _fileMappings;
QDir _resourcesDirectory;
QDir _filesDirectory;
QThreadPool _taskPool;
};

5
assignment-client/src/assets/SendAssetTask.cpp
View file

@ -33,13 +33,10 @@ SendAssetTask::SendAssetTask(QSharedPointer<ReceivedMessage> message, const Shar
void SendAssetTask::run() {
MessageID messageID;
uint8_t extensionLength;
DataOffset start, end;
_message->readPrimitive(&messageID);
QByteArray assetHash = _message->read(SHA256_HASH_LENGTH);
_message->readPrimitive(&extensionLength);
QByteArray extension = _message->read(extensionLength);
// `start` and `end` indicate the range of data to retrieve for the asset identified by `assetHash`.
// `start` is inclusive, `end` is exclusive. Requesting `start` = 1, `end` = 10 will retrieve 9 bytes of data,
@ -61,7 +58,7 @@ void SendAssetTask::run() {
if (end <= start) {
writeError(replyPacketList.get(), AssetServerError::InvalidByteRange);
} else {
QString filePath = _resourcesDir.filePath(QString(hexHash) + "." + QString(extension));
QString filePath = _resourcesDir.filePath(QString(hexHash));
QFile file { filePath };

9
assignment-client/src/assets/UploadAssetTask.cpp
View file

@ -37,15 +37,10 @@ void UploadAssetTask::run() {
MessageID messageID;
buffer.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
uint8_t extensionLength;
buffer.read(reinterpret_cast<char*>(&extensionLength), sizeof(extensionLength));
QByteArray extension = buffer.read(extensionLength);
uint64_t fileSize;
buffer.read(reinterpret_cast<char*>(&fileSize), sizeof(fileSize));
qDebug() << "UploadAssetTask reading a file of " << fileSize << "bytes and extension" << extension << "from"
qDebug() << "UploadAssetTask reading a file of " << fileSize << "bytes from"
<< uuidStringWithoutCurlyBraces(_senderNode->getUUID());
auto replyPacket = NLPacket::create(PacketType::AssetUploadReply);
@ -62,7 +57,7 @@ void UploadAssetTask::run() {
qDebug() << "Hash for uploaded file from" << uuidStringWithoutCurlyBraces(_senderNode->getUUID())
<< "is: (" << hexHash << ") ";
QFile file { _resourcesDir.filePath(QString(hexHash)) + "." + QString(extension) };
QFile file { _resourcesDir.filePath(QString(hexHash)) };
if (file.exists()) {
qDebug() << "[WARNING] This file already exists: " << hexHash;

9
interface/src/Application.cpp
View file

@ -4331,17 +4331,18 @@ bool Application::askToUploadAsset(const QString& filename) {
}
void Application::modelUploadFinished(AssetUpload* upload, const QString& hash) {
auto filename = QFileInfo(upload->getFilename()).fileName();
auto fileInfo = QFileInfo(upload->getFilename());
auto filename = fileInfo.fileName();
if ((upload->getError() == AssetUpload::NoError) &&
(FBX_EXTENSION.endsWith(upload->getExtension(), Qt::CaseInsensitive) ||
OBJ_EXTENSION.endsWith(upload->getExtension(), Qt::CaseInsensitive))) {
(filename.endsWith(FBX_EXTENSION, Qt::CaseInsensitive) ||
filename.endsWith(OBJ_EXTENSION, Qt::CaseInsensitive))) {
auto entities = DependencyManager::get<EntityScriptingInterface>();
EntityItemProperties properties;
properties.setType(EntityTypes::Model);
properties.setModelURL(QString("%1:%2.%3").arg(URL_SCHEME_ATP).arg(hash).arg(upload->getExtension()));
properties.setModelURL(QString("%1:%2.%3").arg(URL_SCHEME_ATP).arg(hash).arg(fileInfo.completeSuffix()));
properties.setPosition(_myCamera.getPosition() + _myCamera.getOrientation() * Vectors::FRONT * 2.0f);
properties.setName(QUrl(upload->getFilename()).fileName());

4
interface/src/assets/ATPAssetMigrator.cpp
View file

@ -142,7 +142,7 @@ void ATPAssetMigrator::migrateResource(ResourceRequest* request) {
QFileInfo assetInfo { request->getUrl().fileName() };
auto upload = assetClient->createUpload(request->getData(), assetInfo.completeSuffix());
auto upload = assetClient->createUpload(request->getData());
if (upload) {
// add this URL to our hash of AssetUpload to original URL
@ -173,7 +173,7 @@ void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& h
auto values = _pendingReplacements.values(modelURL);
QString atpURL = getATPUrl(hash, upload->getExtension()).toString();
QString atpURL = getATPUrl(hash).toString();
for (auto value : values) {
// replace the modelURL in this QJsonValueRef with the hash

2
interface/src/ui/AssetUploadDialogFactory.cpp
View file

@ -87,7 +87,7 @@ void AssetUploadDialogFactory::handleUploadFinished(AssetUpload* upload, const Q
// setup the line edit to hold the copiable text
QLineEdit* lineEdit = new QLineEdit;
QString atpURL = QString("%1:%2.%3").arg(URL_SCHEME_ATP).arg(hash).arg(upload->getExtension());
QString atpURL = QString("%1:%2").arg(URL_SCHEME_ATP).arg(hash);
// set the ATP URL as the text value so it's copiable
lineEdit->insert(atpURL);

49
libraries/networking/src/AssetClient.cpp
View file

@ -71,8 +71,7 @@ void GetMappingRequest::doStart() {
}
if (!_error) {
//_hash = message->read(SHA256_HASH_HEX_LENGTH);
_hash = message->readString();
_hash = message->read(SHA256_HASH_LENGTH).toHex();
assetClient->_mappingCache[_path] = _hash;
}
emit finished(this);
@ -100,7 +99,7 @@ void GetAllMappingsRequest::doStart() {
if (!error) {
size_t numberOfMappings;
int numberOfMappings;
message->readPrimitive(&numberOfMappings);
assetClient->_mappingCache.clear();
for (auto i = 0; i < numberOfMappings; ++i) {
@ -181,6 +180,7 @@ AssetClient::AssetClient() {
auto nodeList = DependencyManager::get<NodeList>();
auto& packetReceiver = nodeList->getPacketReceiver();
packetReceiver.registerListener(PacketType::AssetMappingOperationReply, this, "handleAssetMappingOperationReply");
packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply");
packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply", true);
@ -297,14 +297,14 @@ SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, c
return new SetMappingRequest(path, hash);
}
AssetRequest* AssetClient::createRequest(const AssetHash& hash, const QString& extension) {
AssetRequest* AssetClient::createRequest(const AssetHash& hash) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qCWarning(asset_client) << "Invalid hash size";
return nullptr;
}
if (haveAssetServer()) {
auto request = new AssetRequest(hash, extension);
auto request = new AssetRequest(hash);
// Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
request->moveToThread(thread());
@ -328,9 +328,9 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
}
}
AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& extension) {
AssetUpload* AssetClient::createUpload(const QByteArray& data) {
if (haveAssetServer()) {
auto upload = new AssetUpload(data, extension);
auto upload = new AssetUpload(data);
upload->moveToThread(thread());
@ -340,7 +340,7 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& ex
}
}
bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,
bool AssetClient::getAsset(const QString& hash, DataOffset start, DataOffset end,
ReceivedAssetCallback callback, ProgressCallback progressCallback) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qCWarning(asset_client) << "Invalid hash size";
@ -354,8 +354,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
auto messageID = ++_currentID;
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(uint8_t) + extension.length()
+ sizeof(start) + sizeof(end);
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(start) + sizeof(end);
auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true);
qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
@ -364,9 +363,6 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
packet->writePrimitive(start);
packet->writePrimitive(end);
@ -380,20 +376,18 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
return false;
}
bool AssetClient::getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback) {
bool AssetClient::getAssetInfo(const QString& hash, GetInfoCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto messageID = ++_currentID;
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(uint8_t) + extension.length();
auto payloadSize = sizeof(messageID) + SHA256_HASH_LENGTH;
auto packet = NLPacket::create(PacketType::AssetGetInfo, payloadSize, true);
packet->writePrimitive(messageID);
packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
nodeList->sendPacket(std::move(packet), *assetServer);
@ -493,6 +487,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<ReceivedMessage> message, S
bool AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true);
@ -571,8 +566,8 @@ bool AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, Ma
packetList->writePrimitive(AssetMappingOperationType::Set);
packetList->writeString(path);
packetList->writeString(hash);
packetList->writeString(path.toUtf8());
packetList->write(QByteArray::fromHex(hash.toUtf8()));
nodeList->sendPacketList(std::move(packetList), *assetServer);
@ -584,7 +579,7 @@ bool AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, Ma
return false;
}
bool AssetClient::uploadAsset(const QByteArray& data, const QString& extension, UploadResultCallback callback) {
bool AssetClient::uploadAsset(const QByteArray& data, UploadResultCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
@ -594,9 +589,6 @@ bool AssetClient::uploadAsset(const QByteArray& data, const QString& extension,
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(static_cast<uint8_t>(extension.length()));
packetList->write(extension.toLatin1().constData(), extension.length());
uint64_t size = data.length();
packetList->writePrimitive(size);
packetList->write(data.constData(), size);
@ -697,17 +689,17 @@ void AssetClient::handleNodeKilled(SharedNodePointer node) {
_mappingCache.clear();
}
void AssetScriptingInterface::uploadData(QString data, QString extension, QScriptValue callback) {
void AssetScriptingInterface::uploadData(QString data, QScriptValue callback) {
QByteArray dataByteArray = data.toUtf8();
auto upload = DependencyManager::get<AssetClient>()->createUpload(dataByteArray, extension);
auto upload = DependencyManager::get<AssetClient>()->createUpload(dataByteArray);
if (!upload) {
qCWarning(asset_client) << "Error uploading file to asset server";
return;
}
QObject::connect(upload, &AssetUpload::finished, this, [this, callback, extension](AssetUpload* upload, const QString& hash) mutable {
QObject::connect(upload, &AssetUpload::finished, this, [this, callback](AssetUpload* upload, const QString& hash) mutable {
if (callback.isFunction()) {
QString url = "atp://" + hash + "." + extension;
QString url = "atp://" + hash;
QScriptValueList args { url };
callback.call(_engine->currentContext()->thisObject(), args);
}
@ -731,14 +723,13 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb
auto path = urlString.right(urlString.length() - ATP_SCHEME.length());
auto parts = path.split(".", QString::SkipEmptyParts);
auto hash = parts.length() > 0 ? parts[0] : "";
auto extension = parts.length() > 1 ? parts[1] : "";
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
return;
}
auto assetClient = DependencyManager::get<AssetClient>();
auto assetRequest = assetClient->createRequest(hash, extension);
auto assetRequest = assetClient->createRequest(hash);
if (!assetRequest) {
return;

18
libraries/networking/src/AssetClient.h
View file

@ -104,8 +104,6 @@ class DeleteMappingRequest : public MappingRequest {
public:
DeleteMappingRequest(AssetPath path);
Q_INVOKABLE void start();
signals:
void finished(DeleteMappingRequest* thisRequest);
@ -120,8 +118,6 @@ class GetAllMappingsRequest : public MappingRequest {
public:
GetAllMappingsRequest();
Q_INVOKABLE void start();
AssetMapping getMappings() const { return _mappings; }
signals:
@ -133,8 +129,6 @@ private:
std::map<AssetPath, AssetHash> _mappings;
};
class AssetClient : public QObject, public Dependency {
Q_OBJECT
public:
@ -144,9 +138,9 @@ public:
Q_INVOKABLE GetAllMappingsRequest* createGetAllMappingsRequest();
Q_INVOKABLE DeleteMappingRequest* createDeleteMappingRequest(const AssetPath& path);
Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash);
Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const QString& extension);
Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash);
Q_INVOKABLE AssetUpload* createUpload(const QString& filename);
Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension);
Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data);
public slots:
void init();
@ -168,10 +162,10 @@ private:
bool setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback);
bool deleteAssetMapping(const AssetPath& path, MappingOperationCallback callback);
bool getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback);
bool getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,
bool getAssetInfo(const QString& hash, GetInfoCallback callback);
bool getAsset(const QString& hash, DataOffset start, DataOffset end,
ReceivedAssetCallback callback, ProgressCallback progressCallback);
bool uploadAsset(const QByteArray& data, const QString& extension, UploadResultCallback callback);
bool uploadAsset(const QByteArray& data, UploadResultCallback callback);
struct GetAssetCallbacks {
ReceivedAssetCallback completeCallback;
@ -200,7 +194,7 @@ class AssetScriptingInterface : public QObject {
public:
AssetScriptingInterface(QScriptEngine* engine);
Q_INVOKABLE void uploadData(QString data, QString extension, QScriptValue callback);
Q_INVOKABLE void uploadData(QString data, QScriptValue callback);
Q_INVOKABLE void downloadData(QString url, QScriptValue downloadComplete);
Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback);
Q_INVOKABLE void getMapping(QString path, QScriptValue callback);

9
libraries/networking/src/AssetRequest.cpp
View file

@ -20,9 +20,8 @@
#include "NodeList.h"
#include "ResourceCache.h"
AssetRequest::AssetRequest(const QString& hash, const QString& extension) :
_hash(hash),
_extension(extension)
AssetRequest::AssetRequest(const QString& hash) :
_hash(hash)
{
}
@ -52,7 +51,7 @@ void AssetRequest::start() {
_state = WaitingForInfo;
auto assetClient = DependencyManager::get<AssetClient>();
assetClient->getAssetInfo(_hash, _extension, [this](bool responseReceived, AssetServerError serverError, AssetInfo info) {
assetClient->getAssetInfo(_hash, [this](bool responseReceived, AssetServerError serverError, AssetInfo info) {
_info = info;
if (!responseReceived) {
@ -84,7 +83,7 @@ void AssetRequest::start() {
int start = 0, end = _info.size;
auto assetClient = DependencyManager::get<AssetClient>();
assetClient->getAsset(_hash, _extension, start, end, [this, start, end](bool responseReceived, AssetServerError serverError,
assetClient->getAsset(_hash, start, end, [this, start, end](bool responseReceived, AssetServerError serverError,
const QByteArray& data) {
if (!responseReceived) {
_error = NetworkError;

5
libraries/networking/src/AssetRequest.h
View file

@ -39,14 +39,14 @@ public:
UnknownError
};
AssetRequest(const QString& hash, const QString& extension);
AssetRequest(const QString& hash);
Q_INVOKABLE void start();
const QByteArray& getData() const { return _data; }
const State& getState() const { return _state; }
const Error& getError() const { return _error; }
QUrl getUrl() const { return ::getATPUrl(_hash, _extension); }
QUrl getUrl() const { return ::getATPUrl(_hash); }
signals:
void finished(AssetRequest* thisRequest);
@ -58,7 +58,6 @@ private:
AssetInfo _info;
uint64_t _totalReceived { 0 };
QString _hash;
QString _extension;
QByteArray _data;
int _numPendingRequests { 0 };
};

86
libraries/networking/src/AssetResourceRequest.cpp
View file

@ -15,16 +15,100 @@
#include "AssetUtils.h"
AssetResourceRequest::~AssetResourceRequest() {
if (_assetMappingRequest) {
_assetMappingRequest->deleteLater();
}
if (_assetRequest) {
_assetRequest->deleteLater();
}
}
bool AssetResourceRequest::urlIsAssetPath() const {
static const QString ATP_HASH_REGEX_STRING = "^atp:([A-Fa-f0-9]{64})(\\.[\\w]+)?$";
QRegExp hashRegex { ATP_HASH_REGEX_STRING };
return !hashRegex.exactMatch(_url.toString());
}
void AssetResourceRequest::doSend() {
auto parts = _url.path().split(".", QString::SkipEmptyParts);
auto hash = parts.length() > 0 ? parts[0] : "";
auto extension = parts.length() > 1 ? parts[1] : "";
// We'll either have a hash or an ATP path to a file (that maps to a hash)
if (urlIsAssetPath()) {
// This is an ATP path, we'll need to figure out what the mapping is.
// This may incur a roundtrip to the asset-server, or it may return immediately from the cache in AssetClient.
auto path = _url.path();
requestMappingForPath(path);
} else {
// We've detected that this is a hash - simply use AssetClient to request that asset
auto parts = _url.path().split(".", QString::SkipEmptyParts);
auto hash = parts.length() > 0 ? parts[0] : "";
requestHash(hash);
}
}
void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
auto assetClient = DependencyManager::get<AssetClient>();
_assetMappingRequest = assetClient->createGetMappingRequest(path);
// if we get a nullptr for createGetMappingRequest assume that there is no currently available asset-server
if (!_assetMappingRequest) {
_result = ServerUnavailable;
_state = Finished;
emit finished();
return;
}
// make sure we'll hear about the result of the get mapping request
connect(_assetMappingRequest, &GetMappingRequest::finished, this, [this, path](GetMappingRequest* request){
Q_ASSERT(_state == InProgress);
Q_ASSERT(request == _assetMappingRequest);
switch (request->getError()) {
case MappingRequest::NoError:
// we have no error, we should have a resulting hash - use that to send of a request for that asset
qDebug() << "Got mapping for:" << path << "=>" << request->getHash();
requestHash(request->getHash());
break;
case MappingRequest::NotFound:
// no result for the mapping request, set error to not found
_result = NotFound;
// since we've failed we know we are finished
_state = Finished;
emit finished();
break;
default:
// these are unexpected errors for a GetMappingRequest object
_result = Error;
// since we've failed we know we are finished
_state = Finished;
emit finished();
break;
}
_assetMappingRequest->deleteLater();
_assetMappingRequest = nullptr;
});
_assetMappingRequest->start();
}
void AssetResourceRequest::requestHash(const AssetHash& hash) {
// in case we haven't parsed a valid hash, return an error now
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
_result = InvalidURL;
_state = Finished;
@ -35,7 +119,7 @@ void AssetResourceRequest::doSend() {
// Make request to atp
auto assetClient = DependencyManager::get<AssetClient>();
_assetRequest = assetClient->createRequest(hash, extension);
_assetRequest = assetClient->createRequest(hash);
if (!_assetRequest) {
_result = ServerUnavailable;

6
libraries/networking/src/AssetResourceRequest.h
View file

@ -30,6 +30,12 @@ private slots:
void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
private:
bool urlIsAssetPath() const;
void requestMappingForPath(const AssetPath& path);
void requestHash(const AssetHash& hash);
GetMappingRequest* _assetMappingRequest { nullptr };
AssetRequest* _assetRequest { nullptr };
};

16
libraries/networking/src/AssetUpload.cpp
View file

@ -19,9 +19,8 @@
const QString AssetUpload::PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server.";
AssetUpload::AssetUpload(const QByteArray& data, const QString& extension) :
_data(data),
_extension(extension)
AssetUpload::AssetUpload(const QByteArray& data) :
_data(data)
{
}
@ -59,12 +58,7 @@ void AssetUpload::start() {
// try to open the file at the given filename
QFile file { _filename };
if (file.open(QIODevice::ReadOnly)) {
// file opened, read the data and grab the extension
_extension = QFileInfo(_filename).suffix();
_extension = _extension.toLower();
if (file.open(QIODevice::ReadOnly)) {
_data = file.readAll();
} else {
// we couldn't open the file - set the error result
@ -82,7 +76,7 @@ void AssetUpload::start() {
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
}
assetClient->uploadAsset(_data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
assetClient->uploadAsset(_data, [this](bool responseReceived, AssetServerError error, const QString& hash){
if (!responseReceived) {
_error = NetworkError;
} else {
@ -103,7 +97,7 @@ void AssetUpload::start() {
}
if (_error == NoError && hash == hashData(_data).toHex()) {
saveToCache(getATPUrl(hash, _extension), _data);
saveToCache(getATPUrl(hash), _data);
}
emit finished(this, hash);

4
libraries/networking/src/AssetUpload.h
View file

@ -38,12 +38,11 @@ public:
static const QString PERMISSION_DENIED_ERROR;
AssetUpload(const QString& filename);
AssetUpload(const QByteArray& data, const QString& extension);
AssetUpload(const QByteArray& data);
Q_INVOKABLE void start();
const QString& getFilename() const { return _filename; }
const QString& getExtension() const { return _extension; }
const Error& getError() const { return _error; }
QString getErrorString() const;
@ -54,7 +53,6 @@ signals:
private:
QString _filename;
QByteArray _data;
QString _extension;
Error _error;
};

8
libraries/networking/src/AssetUtils.cpp
View file

@ -19,12 +19,8 @@
#include "ResourceManager.h"
QUrl getATPUrl(const QString& hash, const QString& extension) {
if (!extension.isEmpty()) {
return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, hash, extension));
} else {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, hash));
}
QUrl getATPUrl(const QString& hash) {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, hash));
}
QByteArray hashData(const QByteArray& data) {

5
libraries/networking/src/AssetUtils.h
View file

@ -35,7 +35,8 @@ enum AssetServerError : uint8_t {
AssetNotFound,
InvalidByteRange,
AssetTooLarge,
PermissionDenied
PermissionDenied,
MappingOperationFailed
};
enum AssetMappingOperationType : uint8_t {
@ -45,7 +46,7 @@ enum AssetMappingOperationType : uint8_t {
Delete
};
QUrl getATPUrl(const QString& hash, const QString& extension = QString());
QUrl getATPUrl(const QString& hash);
QByteArray hashData(const QByteArray& data);

5
libraries/networking/src/udt/PacketHeaders.cpp
View file

@ -53,6 +53,11 @@ PacketVersion versionForPacketType(PacketType packetType) {
return static_cast<PacketVersion>(AvatarMixerPacketVersion::SoftAttachmentSupport);
case PacketType::ICEServerHeartbeat:
return 18; // ICE Server Heartbeat signing
case PacketType::AssetGetInfo:
case PacketType::AssetGet:
case PacketType::AssetUpload:
// Removal of extension from Asset requests
return 18;
default:
return 17;
}

4
libraries/script-engine/src/RecordingScriptingInterface.cpp
View file

@ -198,13 +198,13 @@ bool RecordingScriptingInterface::saveRecordingToAsset(QScriptValue getClipAtpUr
return false;
}
if (auto upload = DependencyManager::get<AssetClient>()->createUpload(recording::Clip::toBuffer(_lastClip), HFR_EXTENSION)) {
if (auto upload = DependencyManager::get<AssetClient>()->createUpload(recording::Clip::toBuffer(_lastClip))) {
QObject::connect(upload, &AssetUpload::finished, this, [=](AssetUpload* upload, const QString& hash) mutable {
QString clip_atp_url = "";
if (upload->getError() == AssetUpload::NoError) {
clip_atp_url = QString("%1:%2.%3").arg(URL_SCHEME_ATP, hash, upload->getExtension());
clip_atp_url = QString("%1:%2").arg(URL_SCHEME_ATP, hash);
upload->deleteLater();
} else {
qCWarning(scriptengine) << "Error during the Asset upload.";