From cf74cfb50e48ae6b5bafeb2e397cb25fda31e3d3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 10:38:52 -0700 Subject: [PATCH 01/10] add an ATPAssetMigrator for bulk ATP migration --- interface/src/Menu.cpp | 10 +- interface/src/Menu.h | 1 + interface/src/assets/ATPAssetMigrator.cpp | 177 ++++++++++++++++++ interface/src/assets/ATPAssetMigrator.h | 48 +++++ interface/src/ui/AssetUploadDialogFactory.cpp | 51 +++-- interface/src/ui/AssetUploadDialogFactory.h | 3 +- libraries/networking/src/AssetClient.cpp | 66 ++++--- libraries/networking/src/AssetClient.h | 1 + libraries/networking/src/AssetUpload.cpp | 117 ++++++++---- libraries/networking/src/AssetUpload.h | 7 +- 10 files changed, 385 insertions(+), 96 deletions(-) create mode 100644 interface/src/assets/ATPAssetMigrator.cpp create mode 100644 interface/src/assets/ATPAssetMigrator.h diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index ae41ca2493..cc9b259e06 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -24,6 +24,7 @@ #include "Application.h" #include "AccountManager.h" +#include "assets/ATPAssetMigrator.h" #include "audio/AudioScope.h" #include "avatar/AvatarManager.h" #include "devices/DdeFaceTracker.h" @@ -354,7 +355,7 @@ Menu::Menu() { MenuWrapper* assetDeveloperMenu = developerMenu->addMenu("Assets"); auto& assetDialogFactory = AssetUploadDialogFactory::getInstance(); - assetDialogFactory.setParent(this); + assetDialogFactory.setDialogParent(this); QAction* assetUpload = addActionToQMenuAndActionHash(assetDeveloperMenu, MenuOption::UploadAsset, @@ -365,6 +366,13 @@ Menu::Menu() { // disable the asset upload action by default - it gets enabled only if asset server becomes present assetUpload->setEnabled(false); + auto& atpMigrator = ATPAssetMigrator::getInstance(); + atpMigrator.setDialogParent(this); + + QAction* assetMigration = addActionToQMenuAndActionHash(assetDeveloperMenu, MenuOption::AssetMigration, + 0, &atpMigrator, + SLOT(loadEntityServerFile())); + MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar"); MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking"); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 4c9c3ef7b5..43388fcbd1 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -135,6 +135,7 @@ namespace MenuOption { const QString AnimDebugDrawAnimPose = "Debug Draw Animation"; const QString AnimDebugDrawBindPose = "Debug Draw Bind Pose"; const QString Antialiasing = "Antialiasing"; + const QString AssetMigration = "ATP Asset Migration"; const QString Atmosphere = "Atmosphere"; const QString Attachments = "Attachments..."; const QString AudioNoiseReduction = "Audio Noise Reduction"; diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp new file mode 100644 index 0000000000..d32378085b --- /dev/null +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -0,0 +1,177 @@ +// +// ATPAssetMigrator.cpp +// interface/src/assets +// +// Created by Stephen Birarda on 2015-10-12. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ATPAssetMigrator.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "../ui/AssetUploadDialogFactory.h" + +ATPAssetMigrator& ATPAssetMigrator::getInstance() { + static ATPAssetMigrator instance; + return instance; +} + +static const QString MODEL_URL_KEY = "modelURL"; + +void ATPAssetMigrator::loadEntityServerFile() { + auto filename = QFileDialog::getOpenFileName(_dialogParent, "Select an entity-server content file to migrate", + QString(), QString("Entity-Server Content (*.gz)")); + + if (!filename.isEmpty()) { + qDebug() << "Selected filename for ATP asset migration: " << filename; + + // try to open the file at the given filename + QFile modelsFile { filename }; + + if (modelsFile.open(QIODevice::ReadOnly)) { + QByteArray compressedJsonData = modelsFile.readAll(); + QByteArray jsonData; + + if (!gunzip(compressedJsonData, jsonData)) { + QMessageBox::warning(_dialogParent, "Error", "The file at" + filename + "was not in gzip format."); + } + + QJsonDocument modelsJSON = QJsonDocument::fromJson(jsonData); + _entitiesArray = modelsJSON.object()["Entities"].toArray(); + + for (auto jsonValue : _entitiesArray) { + QJsonObject entityObject = jsonValue.toObject(); + QString modelURLString = entityObject.value(MODEL_URL_KEY).toString(); + + if (!modelURLString.isEmpty()) { + QUrl modelURL = QUrl(modelURLString); + + if (modelURL.scheme() == URL_SCHEME_HTTP || modelURL.scheme() == URL_SCHEME_HTTPS + || modelURL.scheme() == URL_SCHEME_FILE || modelURL.scheme() == URL_SCHEME_FTP) { + + QMessageBox messageBox; + messageBox.setWindowTitle("Asset Migration"); + messageBox.setText("Would you like to migrate the following file to the asset server?"); + messageBox.setInformativeText(modelURL.toDisplayString()); + messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel); + messageBox.setDefaultButton(QMessageBox::Ok); + + if (messageBox.exec() == QMessageBox::Ok) { + // user wants to migrate this asset + + if (_pendingReplacements.contains(modelURL)) { + // we already have a request out for this asset, just store the QJsonValueRef + // so we can do the hash replacement when the request comes back + _pendingReplacements.insert(modelURL, jsonValue); + } else if (_uploadedAssets.contains(modelURL)) { + // we already have a hash for this asset + // so just do the replacement immediately + entityObject[MODEL_URL_KEY] = _uploadedAssets.value(modelURL).toString(); + jsonValue = entityObject; + } else { + auto request = ResourceManager::createResourceRequest(this, modelURL); + + qDebug() << "Requesting" << modelURL << "for ATP asset migration"; + + connect(request, &ResourceRequest::finished, this, [=]() { + if (request->getResult() == ResourceRequest::Success) { + migrateResource(request); + } else { + QMessageBox::warning(_dialogParent, "Error", + QString("Could not retreive asset at %1").arg(modelURL.toString())); + } + request->deleteLater(); + }); + + // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL + // to an ATP one once ready + _pendingReplacements.insert(modelURL, jsonValue); + + request->send(); + } + + } + } + } + } + + _doneReading = true; + + } else { + QMessageBox::warning(_dialogParent, "Error", + "There was a problem loading that entity-server file for ATP asset migration. Please try again"); + } + } +} + +void ATPAssetMigrator::migrateResource(ResourceRequest* request) { + // use an asset client to upload the asset + auto assetClient = DependencyManager::get(); + + QFileInfo assetInfo { request->getUrl().fileName() }; + + auto upload = assetClient->createUpload(request->getData(), assetInfo.completeSuffix()); + + if (upload) { + // add this URL to our hash of AssetUpload to original URL + _originalURLs.insert(upload, request->getUrl()); + + qDebug() << "Starting upload of asset from" << request->getUrl(); + + // connect to the finished signal so we know when the AssetUpload is done + QObject::connect(upload, &AssetUpload::finished, this, &ATPAssetMigrator::assetUploadFinished); + + // start the upload now + upload->start(); + } else { + // show a QMessageBox to say that there is no local asset server + QString messageBoxText = QString("Could not upload \n\n%1\n\nbecause you are currently not connected" \ + " to a local asset-server.").arg(assetInfo.fileName()); + + QMessageBox::information(_dialogParent, "Failed to Upload", messageBoxText); + } +} + +void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& hash) { + if (upload->getError() == AssetUpload::NoError) { + + const auto& modelURL = _originalURLs[upload]; + + // successfully uploaded asset - make any required replacements found in the pending replacements + auto values = _pendingReplacements.values(modelURL); + + QString atpURL = QString("%1:%2.%3").arg(ATP_SCHEME).arg(hash).arg(upload->getExtension()); + + for (auto value : values) { + // replace the modelURL in this QJsonValueRef with the hash + QJsonObject valueObject = value.toObject(); + valueObject[MODEL_URL_KEY] = atpURL; + value = valueObject; + } + + // pull the replaced models from _pendingReplacements + _pendingReplacements.remove(modelURL); + + // are we out of pending replacements? if so it is time to save the entity-server file + if (_doneReading && _pendingReplacements.empty()) { + // show a dialog to ask the user where they want to save the file + } + } else { + AssetUploadDialogFactory::showErrorDialog(upload, _dialogParent); + } +} diff --git a/interface/src/assets/ATPAssetMigrator.h b/interface/src/assets/ATPAssetMigrator.h new file mode 100644 index 0000000000..84f05b600d --- /dev/null +++ b/interface/src/assets/ATPAssetMigrator.h @@ -0,0 +1,48 @@ +// +// ATPAssetMigrator.h +// interface/src/assets +// +// Created by Stephen Birarda on 2015-10-12. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_ATPAssetMigrator_h +#define hifi_ATPAssetMigrator_h + +#include +#include +#include + +class AssetUpload; +class ResourceRequest; + +class ATPAssetMigrator : public QObject { + Q_OBJECT +public: + static ATPAssetMigrator& getInstance(); + + void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; } +public slots: + void loadEntityServerFile(); +private slots: + void assetUploadFinished(AssetUpload* upload, const QString& hash); +private: + void migrateResource(ResourceRequest* request); + + QWidget* _dialogParent = nullptr; + QJsonArray _entitiesArray; + + bool _doneReading { false }; + + QMultiHash _pendingReplacements; + QHash _uploadedAssets; + QHash _originalURLs; +}; + + +#endif // hifi_ATPAssetMigrator_h diff --git a/interface/src/ui/AssetUploadDialogFactory.cpp b/interface/src/ui/AssetUploadDialogFactory.cpp index 4910e6d604..7b1611b616 100644 --- a/interface/src/ui/AssetUploadDialogFactory.cpp +++ b/interface/src/ui/AssetUploadDialogFactory.cpp @@ -29,7 +29,7 @@ AssetUploadDialogFactory& AssetUploadDialogFactory::getInstance() { return staticInstance; } -static const QString PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server."; + void AssetUploadDialogFactory::showDialog() { auto nodeList = DependencyManager::get(); @@ -59,7 +59,7 @@ void AssetUploadDialogFactory::showDialog() { } } else { // we don't have permission to upload to asset server in this domain - show the permission denied error - showErrorDialog(QString(), PERMISSION_DENIED_ERROR); + showErrorDialog(nullptr, _dialogParent, AssetUpload::PERMISSION_DENIED_ERROR); } } @@ -117,42 +117,33 @@ void AssetUploadDialogFactory::handleUploadFinished(AssetUpload* upload, const Q // show the new dialog hashCopyDialog->show(); } else { - // figure out the right error message for the message box - QString additionalError; - - switch (upload->getError()) { - case AssetUpload::PermissionDenied: - additionalError = PERMISSION_DENIED_ERROR; - break; - case AssetUpload::TooLarge: - additionalError = "The uploaded content was too large and could not be stored in the asset-server."; - break; - case AssetUpload::FileOpenError: - additionalError = "The file could not be opened. Please check your permissions and try again."; - break; - case AssetUpload::NetworkError: - additionalError = "The file could not be opened. Please check your network connectivity."; - break; - default: - // not handled, do not show a message box - return; - } - // display a message box with the error - showErrorDialog(QFileInfo(upload->getFilename()).fileName(), additionalError); + showErrorDialog(upload, _dialogParent); } upload->deleteLater(); } -void AssetUploadDialogFactory::showErrorDialog(const QString& filename, const QString& additionalError) { - QString errorMessage; +void AssetUploadDialogFactory::showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage) { + QString filename; - if (!filename.isEmpty()) { - errorMessage += QString("Failed to upload %1.\n\n").arg(filename); + if (upload) { + filename = QFileInfo { upload->getFilename() }.fileName(); } - errorMessage += additionalError; + QString errorMessage = overrideMessage; - QMessageBox::warning(_dialogParent, "Failed Upload", errorMessage); + if (errorMessage.isEmpty() && upload) { + errorMessage = upload->getErrorString(); + } + + QString dialogMessage; + + if (upload) { + dialogMessage += QString("Failed to upload %1.\n\n").arg(filename); + } + + dialogMessage += errorMessage; + + QMessageBox::warning(dialogParent, "Failed Upload", dialogMessage); } diff --git a/interface/src/ui/AssetUploadDialogFactory.h b/interface/src/ui/AssetUploadDialogFactory.h index a8e8765f20..fe3ecf5dc4 100644 --- a/interface/src/ui/AssetUploadDialogFactory.h +++ b/interface/src/ui/AssetUploadDialogFactory.h @@ -25,6 +25,7 @@ public: AssetUploadDialogFactory& operator=(const AssetUploadDialogFactory& rhs) = delete; static AssetUploadDialogFactory& getInstance(); + static void showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage = QString()); void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; } @@ -35,7 +36,7 @@ public slots: private: AssetUploadDialogFactory() = default; - void showErrorDialog(const QString& filename, const QString& additionalError); + QWidget* _dialogParent { nullptr }; }; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 87af2a5cf8..b7f1205847 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -65,43 +65,65 @@ void AssetClient::init() { } } +bool haveAssetServer() { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (!assetServer) { + qCWarning(asset_client) << "Could not complete AssetClient operation " + << "since you are not currently connected to an asset-server."; + return false; + } + + return true; +} + AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { qCWarning(asset_client) << "Invalid hash size"; return nullptr; } - auto nodeList = DependencyManager::get(); - SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); - - if (!assetServer) { - qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension - << " since you are not currently connected to an asset-server."; + if (haveAssetServer()) { + auto request = new AssetRequest(hash, extension); + + // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case) + request->moveToThread(thread()); + request->setParent(this); + + return request; + } else { return nullptr; } - - auto request = new AssetRequest(hash, extension); - - // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case) - request->moveToThread(thread()); - - return request; } + + AssetUpload* AssetClient::createUpload(const QString& filename) { - auto nodeList = DependencyManager::get(); - SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); - if (!assetServer) { - qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server."; + if (haveAssetServer()) { + auto upload = new AssetUpload(filename); + + upload->moveToThread(thread()); + upload->setParent(this); + + return upload; + } else { return nullptr; } - - auto upload = new AssetUpload(this, filename); +} - upload->moveToThread(thread()); - - return upload; +AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& extension) { + if (haveAssetServer()) { + auto upload = new AssetUpload(data, extension); + + upload->moveToThread(thread()); + upload->setParent(this); + + return upload; + } else { + return nullptr; + } } bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 9b82e63b58..22933ea30b 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -45,6 +45,7 @@ public: Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); + Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension); private slots: void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index 92632a43e5..0518b1e7a1 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -17,59 +17,94 @@ #include "AssetClient.h" #include "NetworkLogging.h" -AssetUpload::AssetUpload(QObject* object, const QString& filename) : +const QString AssetUpload::PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server."; + +AssetUpload::AssetUpload(const QByteArray& data, const QString& extension) : + _data(data), + _extension(extension) +{ + +} + +AssetUpload::AssetUpload(const QString& filename) : _filename(filename) { } +QString AssetUpload::getErrorString() const { + // figure out the right error message for error + switch (_error) { + case AssetUpload::PermissionDenied: + return PERMISSION_DENIED_ERROR; + case AssetUpload::TooLarge: + return "The uploaded content was too large and could not be stored in the asset-server."; + case AssetUpload::FileOpenError: + return "The file could not be opened. Please check your permissions and try again."; + case AssetUpload::NetworkError: + return "The file could not be opened. Please check your network connectivity."; + default: + // not handled, do not show a message box + return QString(); + } +} + void AssetUpload::start() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); return; } - // try to open the file at the given filename - QFile file { _filename }; + auto data = _data; - if (file.open(QIODevice::ReadOnly)) { + if (data.isEmpty() && !_filename.isEmpty()) { + // try to open the file at the given filename + QFile file { _filename }; - // file opened, read the data and grab the extension - _extension = QFileInfo(_filename).suffix(); - - auto data = file.readAll(); - - // ask the AssetClient to upload the asset and emit the proper signals from the passed callback - auto assetClient = DependencyManager::get(); - - qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server."; - - assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){ - if (!responseReceived) { - _error = NetworkError; - } else { - switch (error) { - case AssetServerError::NoError: - _error = NoError; - break; - case AssetServerError::AssetTooLarge: - _error = TooLarge; - break; - case AssetServerError::PermissionDenied: - _error = PermissionDenied; - break; - default: - _error = FileOpenError; - break; - } - } - emit finished(this, hash); - }); - } else { - // we couldn't open the file - set the error result - _error = FileOpenError; - - // emit that we are done - emit finished(this, QString()); + if (file.open(QIODevice::ReadOnly)) { + + // file opened, read the data and grab the extension + _extension = QFileInfo(_filename).suffix(); + + data = file.readAll(); + } else { + // we couldn't open the file - set the error result + _error = FileOpenError; + + // emit that we are done + emit finished(this, QString()); + } } + + // ask the AssetClient to upload the asset and emit the proper signals from the passed callback + auto assetClient = DependencyManager::get(); + + if (!_filename.isEmpty()) { + qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server."; + } else { + qCDebug(asset_client) << "Attempting to upload file of" << _data.size() << "bytes to asset-server."; + } + + + assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){ + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + case AssetServerError::AssetTooLarge: + _error = TooLarge; + break; + case AssetServerError::PermissionDenied: + _error = PermissionDenied; + break; + default: + _error = FileOpenError; + break; + } + } + emit finished(this, hash); + }); } diff --git a/libraries/networking/src/AssetUpload.h b/libraries/networking/src/AssetUpload.h index 80faa8a9c1..92f677cbf2 100644 --- a/libraries/networking/src/AssetUpload.h +++ b/libraries/networking/src/AssetUpload.h @@ -35,13 +35,17 @@ public: FileOpenError }; - AssetUpload(QObject* parent, const QString& filename); + static const QString PERMISSION_DENIED_ERROR; + + AssetUpload(const QString& filename); + AssetUpload(const QByteArray& data, const QString& extension); Q_INVOKABLE void start(); const QString& getFilename() const { return _filename; } const QString& getExtension() const { return _extension; } const Error& getError() const { return _error; } + QString getErrorString() const; signals: void finished(AssetUpload* upload, const QString& hash); @@ -49,6 +53,7 @@ signals: private: QString _filename; + QByteArray _data; QString _extension; Error _error; }; From 65976f78917104432ffeaaa1ef0710b91266bb31 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 10:51:57 -0700 Subject: [PATCH 02/10] add saving of entity file to ATPAssetMigrator --- interface/src/assets/ATPAssetMigrator.cpp | 33 ++++++++++++++++++++++- interface/src/assets/ATPAssetMigrator.h | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index d32378085b..6dd1a04434 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -31,6 +31,7 @@ ATPAssetMigrator& ATPAssetMigrator::getInstance() { return instance; } +static const QString ENTITIES_OBJECT_KEY = "Entities"; static const QString MODEL_URL_KEY = "modelURL"; void ATPAssetMigrator::loadEntityServerFile() { @@ -169,9 +170,39 @@ void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& h // are we out of pending replacements? if so it is time to save the entity-server file if (_doneReading && _pendingReplacements.empty()) { - // show a dialog to ask the user where they want to save the file + saveEntityServerFile(); } } else { AssetUploadDialogFactory::showErrorDialog(upload, _dialogParent); } } + +void ATPAssetMigrator::saveEntityServerFile() { + // show a dialog to ask the user where they want to save the file + QString saveName = QFileDialog::getSaveFileName(_dialogParent, "Save Migrated Entities File"); + + QFile saveFile { saveName }; + + if (saveFile.open(QIODevice::WriteOnly)) { + QJsonObject rootObject; + rootObject[ENTITIES_OBJECT_KEY] = _entitiesArray; + + QJsonDocument newDocument { rootObject }; + QByteArray jsonDataForFile; + + if (gzip(newDocument.toJson(), jsonDataForFile, -1)) { + + saveFile.write(jsonDataForFile); + saveFile.close(); + + QMessageBox::information(_dialogParent, "Success", + QString("Your new entities file has been saved at %1").arg(saveName)); + } else { + QMessageBox::warning(_dialogParent, "Error", "Could not gzip JSON data for new entities file."); + } + + } else { + QMessageBox::warning(_dialogParent, "Error", + QString("Could not open file at %1 to write new entities file to.").arg(saveName)); + } +} diff --git a/interface/src/assets/ATPAssetMigrator.h b/interface/src/assets/ATPAssetMigrator.h index 84f05b600d..2c96b55ba2 100644 --- a/interface/src/assets/ATPAssetMigrator.h +++ b/interface/src/assets/ATPAssetMigrator.h @@ -33,6 +33,7 @@ private slots: void assetUploadFinished(AssetUpload* upload, const QString& hash); private: void migrateResource(ResourceRequest* request); + void saveEntityServerFile(); QWidget* _dialogParent = nullptr; QJsonArray _entitiesArray; From 549043f25533ce5506c70b02db11671ac76cd843 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 15:20:37 -0700 Subject: [PATCH 03/10] add an option to bulk upload assets in ATPAssetMigrator --- interface/src/assets/ATPAssetMigrator.cpp | 141 +++++++++++++++------- interface/src/assets/ATPAssetMigrator.h | 6 + 2 files changed, 105 insertions(+), 42 deletions(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index 6dd1a04434..a4325d482f 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -33,6 +33,7 @@ ATPAssetMigrator& ATPAssetMigrator::getInstance() { static const QString ENTITIES_OBJECT_KEY = "Entities"; static const QString MODEL_URL_KEY = "modelURL"; +static const QString MESSAGE_BOX_TITLE = "ATP Asset Migration"; void ATPAssetMigrator::loadEntityServerFile() { auto filename = QFileDialog::getOpenFileName(_dialogParent, "Select an entity-server content file to migrate", @@ -41,6 +42,19 @@ void ATPAssetMigrator::loadEntityServerFile() { if (!filename.isEmpty()) { qDebug() << "Selected filename for ATP asset migration: " << filename; + static const QString MIGRATION_CONFIRMATION_TEXT { + "The ATP Asset Migration process will scan the selected entity-server file, upload discovered resources to the"\ + " current asset-server and then save a new entity-server file with the ATP URLs.\n\nAre you ready to"\ + " continue?\n\nMake sure you are connected to the right domain." + }; + + auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE, MIGRATION_CONFIRMATION_TEXT, + QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes); + + if (button == QMessageBox::No) { + return; + } + // try to open the file at the given filename QFile modelsFile { filename }; @@ -62,50 +76,42 @@ void ATPAssetMigrator::loadEntityServerFile() { if (!modelURLString.isEmpty()) { QUrl modelURL = QUrl(modelURLString); - if (modelURL.scheme() == URL_SCHEME_HTTP || modelURL.scheme() == URL_SCHEME_HTTPS - || modelURL.scheme() == URL_SCHEME_FILE || modelURL.scheme() == URL_SCHEME_FTP) { + if (!_ignoredUrls.contains(modelURL) + && (modelURL.scheme() == URL_SCHEME_HTTP || modelURL.scheme() == URL_SCHEME_HTTPS + || modelURL.scheme() == URL_SCHEME_FILE || modelURL.scheme() == URL_SCHEME_FTP)) { - QMessageBox messageBox; - messageBox.setWindowTitle("Asset Migration"); - messageBox.setText("Would you like to migrate the following file to the asset server?"); - messageBox.setInformativeText(modelURL.toDisplayString()); - messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel); - messageBox.setDefaultButton(QMessageBox::Ok); - - if (messageBox.exec() == QMessageBox::Ok) { - // user wants to migrate this asset + if (_pendingReplacements.contains(modelURL)) { + // we already have a request out for this asset, just store the QJsonValueRef + // so we can do the hash replacement when the request comes back + _pendingReplacements.insert(modelURL, jsonValue); + } else if (_uploadedAssets.contains(modelURL)) { + // we already have a hash for this asset + // so just do the replacement immediately + entityObject[MODEL_URL_KEY] = _uploadedAssets.value(modelURL).toString(); + jsonValue = entityObject; + } else if (wantsToMigrateResource(modelURL)) { + auto request = ResourceManager::createResourceRequest(this, modelURL); - if (_pendingReplacements.contains(modelURL)) { - // we already have a request out for this asset, just store the QJsonValueRef - // so we can do the hash replacement when the request comes back - _pendingReplacements.insert(modelURL, jsonValue); - } else if (_uploadedAssets.contains(modelURL)) { - // we already have a hash for this asset - // so just do the replacement immediately - entityObject[MODEL_URL_KEY] = _uploadedAssets.value(modelURL).toString(); - jsonValue = entityObject; - } else { - auto request = ResourceManager::createResourceRequest(this, modelURL); - - qDebug() << "Requesting" << modelURL << "for ATP asset migration"; - - connect(request, &ResourceRequest::finished, this, [=]() { - if (request->getResult() == ResourceRequest::Success) { - migrateResource(request); - } else { - QMessageBox::warning(_dialogParent, "Error", - QString("Could not retreive asset at %1").arg(modelURL.toString())); - } - request->deleteLater(); - }); - - // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL - // to an ATP one once ready - _pendingReplacements.insert(modelURL, jsonValue); - - request->send(); - } - + qDebug() << "Requesting" << modelURL << "for ATP asset migration"; + + // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL + // to an ATP one once ready + _pendingReplacements.insert(modelURL, jsonValue); + + connect(request, &ResourceRequest::finished, this, [=]() { + if (request->getResult() == ResourceRequest::Success) { + migrateResource(request); + } else { + QMessageBox::warning(_dialogParent, "Error", + QString("Could not retreive asset at %1").arg(modelURL.toString())); + } + request->deleteLater(); + }); + + + request->send(); + } else { + _ignoredUrls.insert(modelURL); } } } @@ -165,16 +171,58 @@ void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& h value = valueObject; } + // add this URL to our list of uploaded assets + _uploadedAssets.insert(modelURL, atpURL); + // pull the replaced models from _pendingReplacements _pendingReplacements.remove(modelURL); // are we out of pending replacements? if so it is time to save the entity-server file if (_doneReading && _pendingReplacements.empty()) { saveEntityServerFile(); + + // reset after the attempted save, success or fail + reset(); } } else { AssetUploadDialogFactory::showErrorDialog(upload, _dialogParent); } + + upload->deleteLater(); +} + +bool ATPAssetMigrator::wantsToMigrateResource(const QUrl& url) { + static bool hasAskedForCompleteMigration { false }; + static bool wantsCompleteMigration { false }; + + + + if (!hasAskedForCompleteMigration) { + // this is the first resource migration - ask the user if they just want to migrate everything + static const QString COMPLETE_MIGRATION_TEXT { "Do you want to migrate all assets found in this entity-server file?\n\n"\ + "Select \"Yes\" to upload all discovered assets to the current asset-server immediately.\n\n"\ + "Select \"No\" to be prompted for each discovered asset." + }; + + auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE, COMPLETE_MIGRATION_TEXT, + QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes); + + if (button == QMessageBox::Yes) { + wantsCompleteMigration = true; + } + + hasAskedForCompleteMigration = true; + } + + if (wantsCompleteMigration) { + return true; + } else { + // present a dialog asking the user if they want to migrate this specific resource + auto button = QMessageBox::question(_dialogParent, MESSAGE_BOX_TITLE, + "Would you like to migrate the following resource?\n" + url.toString(), + QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes); + return button == QMessageBox::Yes; + } } void ATPAssetMigrator::saveEntityServerFile() { @@ -206,3 +254,12 @@ void ATPAssetMigrator::saveEntityServerFile() { QString("Could not open file at %1 to write new entities file to.").arg(saveName)); } } + +void ATPAssetMigrator::reset() { + _entitiesArray = QJsonArray(); + _doneReading = false; + _pendingReplacements.clear(); + _uploadedAssets.clear(); + _originalURLs.clear(); + _ignoredUrls.clear(); +} diff --git a/interface/src/assets/ATPAssetMigrator.h b/interface/src/assets/ATPAssetMigrator.h index 2c96b55ba2..454eb1eac1 100644 --- a/interface/src/assets/ATPAssetMigrator.h +++ b/interface/src/assets/ATPAssetMigrator.h @@ -17,6 +17,7 @@ #include #include #include +#include class AssetUpload; class ResourceRequest; @@ -35,6 +36,10 @@ private: void migrateResource(ResourceRequest* request); void saveEntityServerFile(); + void reset(); + + bool wantsToMigrateResource(const QUrl& url); + QWidget* _dialogParent = nullptr; QJsonArray _entitiesArray; @@ -43,6 +48,7 @@ private: QMultiHash _pendingReplacements; QHash _uploadedAssets; QHash _originalURLs; + QSet _ignoredUrls; }; From 14503053902da26c91daa4e47206ee4cc2495f95 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:13:47 -0700 Subject: [PATCH 04/10] fix ATP url scheme constant --- interface/src/assets/ATPAssetMigrator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index a4325d482f..e5c31d917c 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -162,7 +162,7 @@ void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& h // successfully uploaded asset - make any required replacements found in the pending replacements auto values = _pendingReplacements.values(modelURL); - QString atpURL = QString("%1:%2.%3").arg(ATP_SCHEME).arg(hash).arg(upload->getExtension()); + QString atpURL = QString("%1:%2.%3").arg(URL_SCHEME_ATP).arg(hash).arg(upload->getExtension()); for (auto value : values) { // replace the modelURL in this QJsonValueRef with the hash From 76bfc6218a8e685acb5a09473aefafeb06ffda91 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:36:06 -0700 Subject: [PATCH 05/10] add back block removed in merge --- libraries/networking/src/AssetUpload.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index fdf1c0efc4..67c1049d1a 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -100,6 +100,11 @@ void AssetUpload::start() { break; } } + + if (_error == NoError && hash == hashData(_data).toHex()) { + saveToCache(getUrl(hash, _extension), _data); + } + emit finished(this, hash); }); } From cc9376707744c28511edb506902c9c3bfe3654a2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:38:00 -0700 Subject: [PATCH 06/10] remove some extra spaces --- interface/src/assets/ATPAssetMigrator.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index e5c31d917c..57c0df16ff 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -195,8 +195,6 @@ bool ATPAssetMigrator::wantsToMigrateResource(const QUrl& url) { static bool hasAskedForCompleteMigration { false }; static bool wantsCompleteMigration { false }; - - if (!hasAskedForCompleteMigration) { // this is the first resource migration - ask the user if they just want to migrate everything static const QString COMPLETE_MIGRATION_TEXT { "Do you want to migrate all assets found in this entity-server file?\n\n"\ From d22c60204137b7e62bfdeb87cb42c30a74fd7616 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:44:13 -0700 Subject: [PATCH 07/10] use categorized logging in ATPAssetMigrator --- interface/src/assets/ATPAssetMigrator.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index 57c0df16ff..56adc38f9b 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,9 @@ #include "../ui/AssetUploadDialogFactory.h" +Q_DECLARE_LOGGING_CATEGORY(asset_migrator); +Q_LOGGING_CATEGORY(asset_migrator, "hf.asset_migrator"); + ATPAssetMigrator& ATPAssetMigrator::getInstance() { static ATPAssetMigrator instance; return instance; @@ -40,7 +44,7 @@ void ATPAssetMigrator::loadEntityServerFile() { QString(), QString("Entity-Server Content (*.gz)")); if (!filename.isEmpty()) { - qDebug() << "Selected filename for ATP asset migration: " << filename; + qCDebug(asset_migrator) << "Selected filename for ATP asset migration: " << filename; static const QString MIGRATION_CONFIRMATION_TEXT { "The ATP Asset Migration process will scan the selected entity-server file, upload discovered resources to the"\ @@ -92,7 +96,7 @@ void ATPAssetMigrator::loadEntityServerFile() { } else if (wantsToMigrateResource(modelURL)) { auto request = ResourceManager::createResourceRequest(this, modelURL); - qDebug() << "Requesting" << modelURL << "for ATP asset migration"; + qCDebug(asset_migrator) << "Requesting" << modelURL << "for ATP asset migration"; // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL // to an ATP one once ready @@ -138,7 +142,7 @@ void ATPAssetMigrator::migrateResource(ResourceRequest* request) { // add this URL to our hash of AssetUpload to original URL _originalURLs.insert(upload, request->getUrl()); - qDebug() << "Starting upload of asset from" << request->getUrl(); + qCDebug(asset_migrator) << "Starting upload of asset from" << request->getUrl(); // connect to the finished signal so we know when the AssetUpload is done QObject::connect(upload, &AssetUpload::finished, this, &ATPAssetMigrator::assetUploadFinished); From 574089530a8c8ba040b708c98b6d1d6614cdedbd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:49:12 -0700 Subject: [PATCH 08/10] make sure request is not nullptr before using it --- interface/src/assets/ATPAssetMigrator.cpp | 39 +++++++++++++---------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index 56adc38f9b..c268b351fa 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -96,24 +96,29 @@ void ATPAssetMigrator::loadEntityServerFile() { } else if (wantsToMigrateResource(modelURL)) { auto request = ResourceManager::createResourceRequest(this, modelURL); - qCDebug(asset_migrator) << "Requesting" << modelURL << "for ATP asset migration"; + if (request) { + qCDebug(asset_migrator) << "Requesting" << modelURL << "for ATP asset migration"; + + // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL + // to an ATP one once ready + _pendingReplacements.insert(modelURL, jsonValue); + + connect(request, &ResourceRequest::finished, this, [=]() { + if (request->getResult() == ResourceRequest::Success) { + migrateResource(request); + } else { + QMessageBox::warning(_dialogParent, "Error", + QString("Could not retrieve asset at %1").arg(modelURL.toString())); + } + request->deleteLater(); + }); + + request->send(); + } else { + QMessageBox::warning(_dialogParent, "Error", + QString("Could not create request for asset at %1").arg(modelURL.toString())); + } - // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL - // to an ATP one once ready - _pendingReplacements.insert(modelURL, jsonValue); - - connect(request, &ResourceRequest::finished, this, [=]() { - if (request->getResult() == ResourceRequest::Success) { - migrateResource(request); - } else { - QMessageBox::warning(_dialogParent, "Error", - QString("Could not retreive asset at %1").arg(modelURL.toString())); - } - request->deleteLater(); - }); - - - request->send(); } else { _ignoredUrls.insert(modelURL); } From 2caa7f6d647ab3c75d078caab670388e47cf755b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 16:56:20 -0700 Subject: [PATCH 09/10] change getUrl to getATPUrl, use in ATPAssetMigrator --- interface/src/assets/ATPAssetMigrator.cpp | 3 ++- libraries/networking/src/AssetRequest.cpp | 5 ----- libraries/networking/src/AssetRequest.h | 2 +- libraries/networking/src/AssetUpload.cpp | 2 +- libraries/networking/src/AssetUtils.cpp | 4 ++-- libraries/networking/src/AssetUtils.h | 2 +- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index c268b351fa..861a1df772 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -171,7 +171,8 @@ void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& h // successfully uploaded asset - make any required replacements found in the pending replacements auto values = _pendingReplacements.values(modelURL); - QString atpURL = QString("%1:%2.%3").arg(URL_SCHEME_ATP).arg(hash).arg(upload->getExtension()); + + QString atpURL = getATPUrl(hash, upload->getExtension()).toString(); for (auto value : values) { // replace the modelURL in this QJsonValueRef with the hash diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 121b4cd4fd..7e647e9142 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -127,8 +127,3 @@ void AssetRequest::start() { }); }); } - -QUrl AssetRequest::getUrl() const { - return ::getUrl(_hash, _extension); -} - diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index a5275e718a..3c3459b15d 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -46,7 +46,7 @@ public: const QByteArray& getData() const { return _data; } const State& getState() const { return _state; } const Error& getError() const { return _error; } - QUrl getUrl() const; + QUrl getUrl() const { return ::getATPUrl(_hash, _extension); } signals: void finished(AssetRequest* thisRequest); diff --git a/libraries/networking/src/AssetUpload.cpp b/libraries/networking/src/AssetUpload.cpp index 67c1049d1a..e6f467e717 100644 --- a/libraries/networking/src/AssetUpload.cpp +++ b/libraries/networking/src/AssetUpload.cpp @@ -102,7 +102,7 @@ void AssetUpload::start() { } if (_error == NoError && hash == hashData(_data).toHex()) { - saveToCache(getUrl(hash, _extension), _data); + saveToCache(getATPUrl(hash, _extension), _data); } emit finished(this, hash); diff --git a/libraries/networking/src/AssetUtils.cpp b/libraries/networking/src/AssetUtils.cpp index 7311d73525..f37e0af820 100644 --- a/libraries/networking/src/AssetUtils.cpp +++ b/libraries/networking/src/AssetUtils.cpp @@ -19,7 +19,7 @@ #include "ResourceManager.h" -QUrl getUrl(const QString& hash, const QString& extension) { +QUrl getATPUrl(const QString& hash, const QString& extension) { if (!extension.isEmpty()) { return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, hash, extension)); } else { @@ -66,4 +66,4 @@ bool saveToCache(const QUrl& url, const QByteArray& file) { qCWarning(asset_client) << "No disk cache to save assets to."; } return false; -} \ No newline at end of file +} diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 5fd5c9144d..21b6b3f434 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -32,7 +32,7 @@ enum AssetServerError : uint8_t { PermissionDenied }; -QUrl getUrl(const QString& hash, const QString& extension = QString()); +QUrl getATPUrl(const QString& hash, const QString& extension = QString()); QByteArray hashData(const QByteArray& data); From d84994d8e0a1da6bdef40cdb13951e3281882116 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Oct 2015 17:04:45 -0700 Subject: [PATCH 10/10] fix casing for ubuntu include failure --- interface/src/assets/ATPAssetMigrator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index 861a1df772..fadf4ca7ad 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include