Merge pull request #6075 from birarda/default-asset-server

add a migrator to move assets to ATP
This commit is contained in:
Clément Brisset 2015-10-14 17:02:10 -07:00
commit efd106e68c
14 changed files with 491 additions and 111 deletions

View file

@ -24,6 +24,7 @@
#include "Application.h"
#include "AccountManager.h"
#include "assets/ATPAssetMigrator.h"
#include "audio/AudioScope.h"
#include "avatar/AvatarManager.h"
#include "devices/DdeFaceTracker.h"
@ -354,7 +355,7 @@ Menu::Menu() {
MenuWrapper* assetDeveloperMenu = developerMenu->addMenu("Assets");
auto& assetDialogFactory = AssetUploadDialogFactory::getInstance();
assetDialogFactory.setParent(this);
assetDialogFactory.setDialogParent(this);
QAction* assetUpload = addActionToQMenuAndActionHash(assetDeveloperMenu,
MenuOption::UploadAsset,
@ -365,6 +366,13 @@ Menu::Menu() {
// disable the asset upload action by default - it gets enabled only if asset server becomes present
assetUpload->setEnabled(false);
auto& atpMigrator = ATPAssetMigrator::getInstance();
atpMigrator.setDialogParent(this);
QAction* assetMigration = addActionToQMenuAndActionHash(assetDeveloperMenu, MenuOption::AssetMigration,
0, &atpMigrator,
SLOT(loadEntityServerFile()));
MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar");
MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking");

View file

@ -135,6 +135,7 @@ namespace MenuOption {
const QString AnimDebugDrawAnimPose = "Debug Draw Animation";
const QString AnimDebugDrawBindPose = "Debug Draw Bind Pose";
const QString Antialiasing = "Antialiasing";
const QString AssetMigration = "ATP Asset Migration";
const QString Atmosphere = "Atmosphere";
const QString Attachments = "Attachments...";
const QString AudioNoiseReduction = "Audio Noise Reduction";

View file

@ -0,0 +1,273 @@
//
// ATPAssetMigrator.cpp
// interface/src/assets
//
// Created by Stephen Birarda on 2015-10-12.
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "ATPAssetMigrator.h"
#include <QtCore/QDebug>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
#include <QtCore/QLoggingCategory>
#include <QtCore/QTemporaryFile>
#include <QtWidgets/QFileDialog>
#include <QtWidgets/QMessageBox>
#include <Gzip.h>
#include <AssetClient.h>
#include <AssetUpload.h>
#include <ResourceManager.h>
#include "../ui/AssetUploadDialogFactory.h"
Q_DECLARE_LOGGING_CATEGORY(asset_migrator);
Q_LOGGING_CATEGORY(asset_migrator, "hf.asset_migrator");
ATPAssetMigrator& ATPAssetMigrator::getInstance() {
static ATPAssetMigrator instance;
return instance;
}
static const QString ENTITIES_OBJECT_KEY = "Entities";
static const QString MODEL_URL_KEY = "modelURL";
static const QString MESSAGE_BOX_TITLE = "ATP Asset Migration";
void ATPAssetMigrator::loadEntityServerFile() {
auto filename = QFileDialog::getOpenFileName(_dialogParent, "Select an entity-server content file to migrate",
QString(), QString("Entity-Server Content (*.gz)"));
if (!filename.isEmpty()) {
qCDebug(asset_migrator) << "Selected filename for ATP asset migration: " << filename;
static const QString MIGRATION_CONFIRMATION_TEXT {
"The ATP Asset Migration process will scan the selected entity-server file, upload discovered resources to the"\
" current asset-server and then save a new entity-server file with the ATP URLs.\n\nAre you ready to"\
" continue?\n\nMake sure you are connected to the right domain."
};
auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE, MIGRATION_CONFIRMATION_TEXT,
QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);
if (button == QMessageBox::No) {
return;
}
// try to open the file at the given filename
QFile modelsFile { filename };
if (modelsFile.open(QIODevice::ReadOnly)) {
QByteArray compressedJsonData = modelsFile.readAll();
QByteArray jsonData;
if (!gunzip(compressedJsonData, jsonData)) {
QMessageBox::warning(_dialogParent, "Error", "The file at" + filename + "was not in gzip format.");
}
QJsonDocument modelsJSON = QJsonDocument::fromJson(jsonData);
_entitiesArray = modelsJSON.object()["Entities"].toArray();
for (auto jsonValue : _entitiesArray) {
QJsonObject entityObject = jsonValue.toObject();
QString modelURLString = entityObject.value(MODEL_URL_KEY).toString();
if (!modelURLString.isEmpty()) {
QUrl modelURL = QUrl(modelURLString);
if (!_ignoredUrls.contains(modelURL)
&& (modelURL.scheme() == URL_SCHEME_HTTP || modelURL.scheme() == URL_SCHEME_HTTPS
|| modelURL.scheme() == URL_SCHEME_FILE || modelURL.scheme() == URL_SCHEME_FTP)) {
if (_pendingReplacements.contains(modelURL)) {
// we already have a request out for this asset, just store the QJsonValueRef
// so we can do the hash replacement when the request comes back
_pendingReplacements.insert(modelURL, jsonValue);
} else if (_uploadedAssets.contains(modelURL)) {
// we already have a hash for this asset
// so just do the replacement immediately
entityObject[MODEL_URL_KEY] = _uploadedAssets.value(modelURL).toString();
jsonValue = entityObject;
} else if (wantsToMigrateResource(modelURL)) {
auto request = ResourceManager::createResourceRequest(this, modelURL);
if (request) {
qCDebug(asset_migrator) << "Requesting" << modelURL << "for ATP asset migration";
// add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL
// to an ATP one once ready
_pendingReplacements.insert(modelURL, jsonValue);
connect(request, &ResourceRequest::finished, this, [=]() {
if (request->getResult() == ResourceRequest::Success) {
migrateResource(request);
} else {
QMessageBox::warning(_dialogParent, "Error",
QString("Could not retrieve asset at %1").arg(modelURL.toString()));
}
request->deleteLater();
});
request->send();
} else {
QMessageBox::warning(_dialogParent, "Error",
QString("Could not create request for asset at %1").arg(modelURL.toString()));
}
} else {
_ignoredUrls.insert(modelURL);
}
}
}
}
_doneReading = true;
} else {
QMessageBox::warning(_dialogParent, "Error",
"There was a problem loading that entity-server file for ATP asset migration. Please try again");
}
}
}
void ATPAssetMigrator::migrateResource(ResourceRequest* request) {
// use an asset client to upload the asset
auto assetClient = DependencyManager::get<AssetClient>();
QFileInfo assetInfo { request->getUrl().fileName() };
auto upload = assetClient->createUpload(request->getData(), assetInfo.completeSuffix());
if (upload) {
// add this URL to our hash of AssetUpload to original URL
_originalURLs.insert(upload, request->getUrl());
qCDebug(asset_migrator) << "Starting upload of asset from" << request->getUrl();
// connect to the finished signal so we know when the AssetUpload is done
QObject::connect(upload, &AssetUpload::finished, this, &ATPAssetMigrator::assetUploadFinished);
// start the upload now
upload->start();
} else {
// show a QMessageBox to say that there is no local asset server
QString messageBoxText = QString("Could not upload \n\n%1\n\nbecause you are currently not connected" \
" to a local asset-server.").arg(assetInfo.fileName());
QMessageBox::information(_dialogParent, "Failed to Upload", messageBoxText);
}
}
void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& hash) {
if (upload->getError() == AssetUpload::NoError) {
const auto& modelURL = _originalURLs[upload];
// successfully uploaded asset - make any required replacements found in the pending replacements
auto values = _pendingReplacements.values(modelURL);
QString atpURL = getATPUrl(hash, upload->getExtension()).toString();
for (auto value : values) {
// replace the modelURL in this QJsonValueRef with the hash
QJsonObject valueObject = value.toObject();
valueObject[MODEL_URL_KEY] = atpURL;
value = valueObject;
}
// add this URL to our list of uploaded assets
_uploadedAssets.insert(modelURL, atpURL);
// pull the replaced models from _pendingReplacements
_pendingReplacements.remove(modelURL);
// are we out of pending replacements? if so it is time to save the entity-server file
if (_doneReading && _pendingReplacements.empty()) {
saveEntityServerFile();
// reset after the attempted save, success or fail
reset();
}
} else {
AssetUploadDialogFactory::showErrorDialog(upload, _dialogParent);
}
upload->deleteLater();
}
bool ATPAssetMigrator::wantsToMigrateResource(const QUrl& url) {
static bool hasAskedForCompleteMigration { false };
static bool wantsCompleteMigration { false };
if (!hasAskedForCompleteMigration) {
// this is the first resource migration - ask the user if they just want to migrate everything
static const QString COMPLETE_MIGRATION_TEXT { "Do you want to migrate all assets found in this entity-server file?\n\n"\
"Select \"Yes\" to upload all discovered assets to the current asset-server immediately.\n\n"\
"Select \"No\" to be prompted for each discovered asset."
};
auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE, COMPLETE_MIGRATION_TEXT,
QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);
if (button == QMessageBox::Yes) {
wantsCompleteMigration = true;
}
hasAskedForCompleteMigration = true;
}
if (wantsCompleteMigration) {
return true;
} else {
// present a dialog asking the user if they want to migrate this specific resource
auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE,
"Would you like to migrate the following resource?\n" + url.toString(),
QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes);
return button == QMessageBox::Yes;
}
}
void ATPAssetMigrator::saveEntityServerFile() {
// show a dialog to ask the user where they want to save the file
QString saveName = QFileDialog::getSaveFileName(_dialogParent, "Save Migrated Entities File");
QFile saveFile { saveName };
if (saveFile.open(QIODevice::WriteOnly)) {
QJsonObject rootObject;
rootObject[ENTITIES_OBJECT_KEY] = _entitiesArray;
QJsonDocument newDocument { rootObject };
QByteArray jsonDataForFile;
if (gzip(newDocument.toJson(), jsonDataForFile, -1)) {
saveFile.write(jsonDataForFile);
saveFile.close();
QMessageBox::information(_dialogParent, "Success",
QString("Your new entities file has been saved at %1").arg(saveName));
} else {
QMessageBox::warning(_dialogParent, "Error", "Could not gzip JSON data for new entities file.");
}
} else {
QMessageBox::warning(_dialogParent, "Error",
QString("Could not open file at %1 to write new entities file to.").arg(saveName));
}
}
void ATPAssetMigrator::reset() {
_entitiesArray = QJsonArray();
_doneReading = false;
_pendingReplacements.clear();
_uploadedAssets.clear();
_originalURLs.clear();
_ignoredUrls.clear();
}

View file

@ -0,0 +1,55 @@
//
// ATPAssetMigrator.h
// interface/src/assets
//
// Created by Stephen Birarda on 2015-10-12.
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#pragma once
#ifndef hifi_ATPAssetMigrator_h
#define hifi_ATPAssetMigrator_h
#include <QtCore/QJsonArray>
#include <QtCore/QObject>
#include <QtCore/QMultiHash>
#include <QtCore/QSet>
class AssetUpload;
class ResourceRequest;
class ATPAssetMigrator : public QObject {
Q_OBJECT
public:
static ATPAssetMigrator& getInstance();
void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; }
public slots:
void loadEntityServerFile();
private slots:
void assetUploadFinished(AssetUpload* upload, const QString& hash);
private:
void migrateResource(ResourceRequest* request);
void saveEntityServerFile();
void reset();
bool wantsToMigrateResource(const QUrl& url);
QWidget* _dialogParent = nullptr;
QJsonArray _entitiesArray;
bool _doneReading { false };
QMultiHash<QUrl, QJsonValueRef> _pendingReplacements;
QHash<QUrl, QUrl> _uploadedAssets;
QHash<AssetUpload*, QUrl> _originalURLs;
QSet<QUrl> _ignoredUrls;
};
#endif // hifi_ATPAssetMigrator_h

View file

@ -30,7 +30,7 @@ AssetUploadDialogFactory& AssetUploadDialogFactory::getInstance() {
return staticInstance;
}
static const QString PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server.";
void AssetUploadDialogFactory::showDialog() {
auto nodeList = DependencyManager::get<NodeList>();
@ -60,7 +60,7 @@ void AssetUploadDialogFactory::showDialog() {
}
} else {
// we don't have permission to upload to asset server in this domain - show the permission denied error
showErrorDialog(QString(), PERMISSION_DENIED_ERROR);
showErrorDialog(nullptr, _dialogParent, AssetUpload::PERMISSION_DENIED_ERROR);
}
}
@ -118,42 +118,33 @@ void AssetUploadDialogFactory::handleUploadFinished(AssetUpload* upload, const Q
// show the new dialog
hashCopyDialog->show();
} else {
// figure out the right error message for the message box
QString additionalError;
switch (upload->getError()) {
case AssetUpload::PermissionDenied:
additionalError = PERMISSION_DENIED_ERROR;
break;
case AssetUpload::TooLarge:
additionalError = "The uploaded content was too large and could not be stored in the asset-server.";
break;
case AssetUpload::FileOpenError:
additionalError = "The file could not be opened. Please check your permissions and try again.";
break;
case AssetUpload::NetworkError:
additionalError = "The file could not be opened. Please check your network connectivity.";
break;
default:
// not handled, do not show a message box
return;
}
// display a message box with the error
showErrorDialog(QFileInfo(upload->getFilename()).fileName(), additionalError);
showErrorDialog(upload, _dialogParent);
}
upload->deleteLater();
}
void AssetUploadDialogFactory::showErrorDialog(const QString& filename, const QString& additionalError) {
QString errorMessage;
void AssetUploadDialogFactory::showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage) {
QString filename;
if (!filename.isEmpty()) {
errorMessage += QString("Failed to upload %1.\n\n").arg(filename);
if (upload) {
filename = QFileInfo { upload->getFilename() }.fileName();
}
errorMessage += additionalError;
QString errorMessage = overrideMessage;
QMessageBox::warning(_dialogParent, "Failed Upload", errorMessage);
if (errorMessage.isEmpty() && upload) {
errorMessage = upload->getErrorString();
}
QString dialogMessage;
if (upload) {
dialogMessage += QString("Failed to upload %1.\n\n").arg(filename);
}
dialogMessage += errorMessage;
QMessageBox::warning(dialogParent, "Failed Upload", dialogMessage);
}

View file

@ -25,6 +25,7 @@ public:
AssetUploadDialogFactory& operator=(const AssetUploadDialogFactory& rhs) = delete;
static AssetUploadDialogFactory& getInstance();
static void showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage = QString());
void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; }
@ -35,7 +36,7 @@ public slots:
private:
AssetUploadDialogFactory() = default;
void showErrorDialog(const QString& filename, const QString& additionalError);
QWidget* _dialogParent { nullptr };
};

View file

@ -65,43 +65,65 @@ void AssetClient::init() {
}
}
bool haveAssetServer() {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qCWarning(asset_client) << "Could not complete AssetClient operation "
<< "since you are not currently connected to an asset-server.";
return false;
}
return true;
}
AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qCWarning(asset_client) << "Invalid hash size";
return nullptr;
}
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension
<< " since you are not currently connected to an asset-server.";
if (haveAssetServer()) {
auto request = new AssetRequest(hash, extension);
// Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
request->moveToThread(thread());
request->setParent(this);
return request;
} else {
return nullptr;
}
auto request = new AssetRequest(hash, extension);
// Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
request->moveToThread(thread());
return request;
}
AssetUpload* AssetClient::createUpload(const QString& filename) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
if (haveAssetServer()) {
auto upload = new AssetUpload(filename);
upload->moveToThread(thread());
upload->setParent(this);
return upload;
} else {
return nullptr;
}
auto upload = new AssetUpload(this, filename);
}
upload->moveToThread(thread());
return upload;
AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& extension) {
if (haveAssetServer()) {
auto upload = new AssetUpload(data, extension);
upload->moveToThread(thread());
upload->setParent(this);
return upload;
} else {
return nullptr;
}
}
bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,

View file

@ -45,6 +45,7 @@ public:
Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension);
Q_INVOKABLE AssetUpload* createUpload(const QString& filename);
Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension);
private slots:
void handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);

View file

@ -127,8 +127,3 @@ void AssetRequest::start() {
});
});
}
QUrl AssetRequest::getUrl() const {
return ::getUrl(_hash, _extension);
}

View file

@ -46,7 +46,7 @@ public:
const QByteArray& getData() const { return _data; }
const State& getState() const { return _state; }
const Error& getError() const { return _error; }
QUrl getUrl() const;
QUrl getUrl() const { return ::getATPUrl(_hash, _extension); }
signals:
void finished(AssetRequest* thisRequest);

View file

@ -17,65 +17,94 @@
#include "AssetClient.h"
#include "NetworkLogging.h"
AssetUpload::AssetUpload(QObject* object, const QString& filename) :
const QString AssetUpload::PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server.";
AssetUpload::AssetUpload(const QByteArray& data, const QString& extension) :
_data(data),
_extension(extension)
{
}
AssetUpload::AssetUpload(const QString& filename) :
_filename(filename)
{
}
QString AssetUpload::getErrorString() const {
// figure out the right error message for error
switch (_error) {
case AssetUpload::PermissionDenied:
return PERMISSION_DENIED_ERROR;
case AssetUpload::TooLarge:
return "The uploaded content was too large and could not be stored in the asset-server.";
case AssetUpload::FileOpenError:
return "The file could not be opened. Please check your permissions and try again.";
case AssetUpload::NetworkError:
return "The file could not be opened. Please check your network connectivity.";
default:
// not handled, do not show a message box
return QString();
}
}
void AssetUpload::start() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "start");
return;
}
// try to open the file at the given filename
QFile file { _filename };
if (file.open(QIODevice::ReadOnly)) {
if (_data.isEmpty() && !_filename.isEmpty()) {
// try to open the file at the given filename
QFile file { _filename };
// file opened, read the data and grab the extension
_extension = QFileInfo(_filename).suffix();
_data = file.readAll();
// ask the AssetClient to upload the asset and emit the proper signals from the passed callback
auto assetClient = DependencyManager::get<AssetClient>();
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
assetClient->uploadAsset(_data, _extension, [this](bool responseReceived, AssetServerError error,
const QString& hash){
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::AssetTooLarge:
_error = TooLarge;
break;
case AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
_error = FileOpenError;
break;
}
}
if (file.open(QIODevice::ReadOnly)) {
if (_error == NoError && hash == hashData(_data).toHex()) {
saveToCache(getUrl(hash, _extension), _data);
}
// file opened, read the data and grab the extension
_extension = QFileInfo(_filename).suffix();
emit finished(this, hash);
});
} else {
// we couldn't open the file - set the error result
_error = FileOpenError;
// emit that we are done
emit finished(this, QString());
_data = file.readAll();
} else {
// we couldn't open the file - set the error result
_error = FileOpenError;
// emit that we are done
emit finished(this, QString());
}
}
// ask the AssetClient to upload the asset and emit the proper signals from the passed callback
auto assetClient = DependencyManager::get<AssetClient>();
if (!_filename.isEmpty()) {
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
}
assetClient->uploadAsset(_data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
if (!responseReceived) {
_error = NetworkError;
} else {
switch (error) {
case AssetServerError::NoError:
_error = NoError;
break;
case AssetServerError::AssetTooLarge:
_error = TooLarge;
break;
case AssetServerError::PermissionDenied:
_error = PermissionDenied;
break;
default:
_error = FileOpenError;
break;
}
}
if (_error == NoError && hash == hashData(_data).toHex()) {
saveToCache(getATPUrl(hash, _extension), _data);
}
emit finished(this, hash);
});
}

View file

@ -35,13 +35,17 @@ public:
FileOpenError
};
AssetUpload(QObject* parent, const QString& filename);
static const QString PERMISSION_DENIED_ERROR;
AssetUpload(const QString& filename);
AssetUpload(const QByteArray& data, const QString& extension);
Q_INVOKABLE void start();
const QString& getFilename() const { return _filename; }
const QString& getExtension() const { return _extension; }
const Error& getError() const { return _error; }
QString getErrorString() const;
signals:
void finished(AssetUpload* upload, const QString& hash);
@ -49,8 +53,8 @@ signals:
private:
QString _filename;
QString _extension;
QByteArray _data;
QString _extension;
Error _error;
};

View file

@ -19,7 +19,7 @@
#include "ResourceManager.h"
QUrl getUrl(const QString& hash, const QString& extension) {
QUrl getATPUrl(const QString& hash, const QString& extension) {
if (!extension.isEmpty()) {
return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, hash, extension));
} else {
@ -66,4 +66,4 @@ bool saveToCache(const QUrl& url, const QByteArray& file) {
qCWarning(asset_client) << "No disk cache to save assets to.";
}
return false;
}
}

View file

@ -32,7 +32,7 @@ enum AssetServerError : uint8_t {
PermissionDenied
};
QUrl getUrl(const QString& hash, const QString& extension = QString());
QUrl getATPUrl(const QString& hash, const QString& extension = QString());
QByteArray hashData(const QByteArray& data);