add an ATPAssetMigrator for bulk ATP migration

interface/src/Menu.cpp

@@ -24,6 +24,7 @@
 
 #include "Application.h"
 #include "AccountManager.h"
+#include "assets/ATPAssetMigrator.h"
 #include "audio/AudioScope.h"
 #include "avatar/AvatarManager.h"
 #include "devices/DdeFaceTracker.h"
@@ -354,7 +355,7 @@ Menu::Menu() {
     MenuWrapper* assetDeveloperMenu = developerMenu->addMenu("Assets");
     
     auto& assetDialogFactory = AssetUploadDialogFactory::getInstance();
-    assetDialogFactory.setParent(this);
+    assetDialogFactory.setDialogParent(this);
     
     QAction* assetUpload = addActionToQMenuAndActionHash(assetDeveloperMenu,
                                                          MenuOption::UploadAsset,
@@ -365,6 +366,13 @@ Menu::Menu() {
     // disable the asset upload action by default - it gets enabled only if asset server becomes present
     assetUpload->setEnabled(false);
     
+    auto& atpMigrator = ATPAssetMigrator::getInstance();
+    atpMigrator.setDialogParent(this);
+    
+    QAction* assetMigration = addActionToQMenuAndActionHash(assetDeveloperMenu, MenuOption::AssetMigration,
+                                                            0, &atpMigrator,
+                                                            SLOT(loadEntityServerFile()));
+    
     MenuWrapper* avatarDebugMenu = developerMenu->addMenu("Avatar");
 
     MenuWrapper* faceTrackingMenu = avatarDebugMenu->addMenu("Face Tracking");

interface/src/Menu.h

@@ -135,6 +135,7 @@ namespace MenuOption {
     const QString AnimDebugDrawAnimPose = "Debug Draw Animation";
     const QString AnimDebugDrawBindPose = "Debug Draw Bind Pose";
     const QString Antialiasing = "Antialiasing";
+    const QString AssetMigration = "ATP Asset Migration";
     const QString Atmosphere = "Atmosphere";
     const QString Attachments = "Attachments...";
     const QString AudioNoiseReduction = "Audio Noise Reduction";

interface/src/assets/ATPAssetMigrator.cpp

@@ -0,0 +1,177 @@
+//
+//  ATPAssetMigrator.cpp
+//  interface/src/assets
+//
+//  Created by Stephen Birarda on 2015-10-12.
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ATPAssetMigrator.h"
+
+#include <QtCore/QDebug>
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonObject>
+#include <QtCore/QTemporaryFile>
+#include <QtWidgets/QFileDialog>
+#include <QtWidgets/QMessageBox>
+
+#include <GZip.h>
+
+#include <AssetClient.h>
+#include <AssetUpload.h>
+#include <ResourceManager.h>
+
+#include "../ui/AssetUploadDialogFactory.h"
+
+ATPAssetMigrator& ATPAssetMigrator::getInstance() {
+    static ATPAssetMigrator instance;
+    return instance;
+}
+
+static const QString MODEL_URL_KEY = "modelURL";
+
+void ATPAssetMigrator::loadEntityServerFile() {
+    auto filename = QFileDialog::getOpenFileName(_dialogParent, "Select an entity-server content file to migrate",
+                                                 QString(), QString("Entity-Server Content (*.gz)"));
+    
+    if (!filename.isEmpty()) {
+        qDebug() << "Selected filename for ATP asset migration: " << filename;
+        
+        // try to open the file at the given filename
+        QFile modelsFile { filename };
+        
+        if (modelsFile.open(QIODevice::ReadOnly)) {
+            QByteArray compressedJsonData = modelsFile.readAll();
+            QByteArray jsonData;
+            
+            if (!gunzip(compressedJsonData, jsonData)) {
+                QMessageBox::warning(_dialogParent, "Error", "The file at" + filename + "was not in gzip format.");
+            }
+            
+            QJsonDocument modelsJSON = QJsonDocument::fromJson(jsonData);
+            _entitiesArray = modelsJSON.object()["Entities"].toArray();
+            
+            for (auto jsonValue : _entitiesArray) {
+                QJsonObject entityObject = jsonValue.toObject();
+                QString modelURLString = entityObject.value(MODEL_URL_KEY).toString();
+                
+                if (!modelURLString.isEmpty()) {
+                    QUrl modelURL = QUrl(modelURLString);
+                    
+                    if (modelURL.scheme() == URL_SCHEME_HTTP || modelURL.scheme() == URL_SCHEME_HTTPS
+                        || modelURL.scheme() == URL_SCHEME_FILE || modelURL.scheme() == URL_SCHEME_FTP) {
+                        
+                        QMessageBox messageBox;
+                        messageBox.setWindowTitle("Asset Migration");
+                        messageBox.setText("Would you like to migrate the following file to the asset server?");
+                        messageBox.setInformativeText(modelURL.toDisplayString());
+                        messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel);
+                        messageBox.setDefaultButton(QMessageBox::Ok);
+                        
+                        if (messageBox.exec() == QMessageBox::Ok) {
+                            // user wants to migrate this asset
+                            
+                            if (_pendingReplacements.contains(modelURL)) {
+                                // we already have a request out for this asset, just store the QJsonValueRef
+                                // so we can do the hash replacement when the request comes back
+                                _pendingReplacements.insert(modelURL, jsonValue);
+                            } else if (_uploadedAssets.contains(modelURL)) {
+                                // we already have a hash for this asset
+                                // so just do the replacement immediately
+                                entityObject[MODEL_URL_KEY] = _uploadedAssets.value(modelURL).toString();
+                                jsonValue = entityObject;
+                            } else {
+                                auto request = ResourceManager::createResourceRequest(this, modelURL);
+                                
+                                qDebug() << "Requesting" << modelURL << "for ATP asset migration";
+                                
+                                connect(request, &ResourceRequest::finished, this, [=]() {
+                                    if (request->getResult() == ResourceRequest::Success) {
+                                        migrateResource(request);
+                                    } else {
+                                        QMessageBox::warning(_dialogParent, "Error",
+                                                             QString("Could not retreive asset at %1").arg(modelURL.toString()));
+                                    }
+                                    request->deleteLater();
+                                });
+                                
+                                // add this combination of QUrl and QJsonValueRef to our multi hash so we can change the URL
+                                // to an ATP one once ready
+                                _pendingReplacements.insert(modelURL, jsonValue);
+                                
+                                request->send();
+                            }
+
+                        }
+                    }
+                }
+            }
+            
+            _doneReading = true;
+            
+        } else {
+            QMessageBox::warning(_dialogParent, "Error",
+                                 "There was a problem loading that entity-server file for ATP asset migration. Please try again");
+        }
+    }
+}
+
+void ATPAssetMigrator::migrateResource(ResourceRequest* request) {
+    // use an asset client to upload the asset
+    auto assetClient = DependencyManager::get<AssetClient>();
+    
+    QFileInfo assetInfo { request->getUrl().fileName() };
+    
+    auto upload = assetClient->createUpload(request->getData(), assetInfo.completeSuffix());
+    
+    if (upload) {
+        // add this URL to our hash of AssetUpload to original URL
+        _originalURLs.insert(upload, request->getUrl());
+        
+        qDebug() << "Starting upload of asset from" << request->getUrl();
+        
+        // connect to the finished signal so we know when the AssetUpload is done
+        QObject::connect(upload, &AssetUpload::finished, this, &ATPAssetMigrator::assetUploadFinished);
+        
+        // start the upload now
+        upload->start();
+    } else {
+        // show a QMessageBox to say that there is no local asset server
+        QString messageBoxText = QString("Could not upload \n\n%1\n\nbecause you are currently not connected" \
+                                         " to a local asset-server.").arg(assetInfo.fileName());
+        
+        QMessageBox::information(_dialogParent, "Failed to Upload", messageBoxText);
+    }
+}
+
+void ATPAssetMigrator::assetUploadFinished(AssetUpload *upload, const QString& hash) {
+    if (upload->getError() == AssetUpload::NoError) {
+
+        const auto& modelURL = _originalURLs[upload];
+        
+        // successfully uploaded asset - make any required replacements found in the pending replacements
+        auto values = _pendingReplacements.values(modelURL);
+        
+        QString atpURL = QString("%1:%2.%3").arg(ATP_SCHEME).arg(hash).arg(upload->getExtension());
+        
+        for (auto value : values) {
+            // replace the modelURL in this QJsonValueRef with the hash
+            QJsonObject valueObject = value.toObject();
+            valueObject[MODEL_URL_KEY] = atpURL;
+            value = valueObject;
+        }
+        
+        // pull the replaced models from _pendingReplacements
+        _pendingReplacements.remove(modelURL);
+        
+        // are we out of pending replacements? if so it is time to save the entity-server file
+        if (_doneReading && _pendingReplacements.empty()) {
+            // show a dialog to ask the user where they want to save the file
+        }
+    } else {
+        AssetUploadDialogFactory::showErrorDialog(upload, _dialogParent);
+    }
+}

interface/src/assets/ATPAssetMigrator.h

@@ -0,0 +1,48 @@
+//
+//  ATPAssetMigrator.h
+//  interface/src/assets
+//
+//  Created by Stephen Birarda on 2015-10-12.
+//  Copyright 2015 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+
+#ifndef hifi_ATPAssetMigrator_h
+#define hifi_ATPAssetMigrator_h
+
+#include <QtCore/QJsonArray>
+#include <QtCore/QObject>
+#include <QtCore/QMultiHash>
+
+class AssetUpload;
+class ResourceRequest;
+
+class ATPAssetMigrator : public QObject {
+    Q_OBJECT
+public:
+    static ATPAssetMigrator& getInstance();
+    
+    void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; }
+public slots:
+    void loadEntityServerFile();
+private slots:
+    void assetUploadFinished(AssetUpload* upload, const QString& hash);
+private:
+    void migrateResource(ResourceRequest* request);
+    
+    QWidget* _dialogParent = nullptr;
+    QJsonArray _entitiesArray;
+    
+    bool _doneReading { false };
+    
+    QMultiHash<QUrl, QJsonValueRef> _pendingReplacements;
+    QHash<QUrl, QUrl> _uploadedAssets;
+    QHash<AssetUpload*, QUrl> _originalURLs;
+};
+
+
+#endif // hifi_ATPAssetMigrator_h

interface/src/ui/AssetUploadDialogFactory.cpp

@@ -29,7 +29,7 @@ AssetUploadDialogFactory& AssetUploadDialogFactory::getInstance() {
     return staticInstance;
 }
 
-static const QString PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server.";
+
 
 void AssetUploadDialogFactory::showDialog() {
     auto nodeList = DependencyManager::get<NodeList>();
@@ -59,7 +59,7 @@ void AssetUploadDialogFactory::showDialog() {
         }
     } else {
         // we don't have permission to upload to asset server in this domain - show the permission denied error
-        showErrorDialog(QString(), PERMISSION_DENIED_ERROR);
+        showErrorDialog(nullptr, _dialogParent, AssetUpload::PERMISSION_DENIED_ERROR);
     }
     
 }
@@ -117,42 +117,33 @@ void AssetUploadDialogFactory::handleUploadFinished(AssetUpload* upload, const Q
         // show the new dialog
         hashCopyDialog->show();
     } else {
-        // figure out the right error message for the message box
-        QString additionalError;
-        
-        switch (upload->getError()) {
-            case AssetUpload::PermissionDenied:
-                additionalError = PERMISSION_DENIED_ERROR;
-                break;
-            case AssetUpload::TooLarge:
-                additionalError = "The uploaded content was too large and could not be stored in the asset-server.";
-                break;
-            case AssetUpload::FileOpenError:
-                additionalError = "The file could not be opened. Please check your permissions and try again.";
-                break;
-            case AssetUpload::NetworkError:
-                additionalError = "The file could not be opened. Please check your network connectivity.";
-                break;
-            default:
-                // not handled, do not show a message box
-                return;
-        }
-        
         // display a message box with the error
-        showErrorDialog(QFileInfo(upload->getFilename()).fileName(), additionalError);
+        showErrorDialog(upload, _dialogParent);
     }
     
     upload->deleteLater();
 }
 
-void AssetUploadDialogFactory::showErrorDialog(const QString& filename, const QString& additionalError) {
+void AssetUploadDialogFactory::showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage) {
-    QString errorMessage;
+    QString filename;
     
-    if (!filename.isEmpty()) {
+    if (upload) {
-        errorMessage += QString("Failed to upload %1.\n\n").arg(filename);
+        filename = QFileInfo { upload->getFilename() }.fileName();
     }
     
-    errorMessage += additionalError;
+    QString errorMessage = overrideMessage;
     
-    QMessageBox::warning(_dialogParent, "Failed Upload", errorMessage);
+    if (errorMessage.isEmpty() && upload) {
+        errorMessage = upload->getErrorString();
+    }
+    
+    QString dialogMessage;
+    
+    if (upload) {
+        dialogMessage += QString("Failed to upload %1.\n\n").arg(filename);
+    }
+    
+    dialogMessage += errorMessage;
+    
+    QMessageBox::warning(dialogParent, "Failed Upload", dialogMessage);
 }

interface/src/ui/AssetUploadDialogFactory.h

@@ -25,6 +25,7 @@ public:
     AssetUploadDialogFactory& operator=(const AssetUploadDialogFactory& rhs) = delete;
     
     static AssetUploadDialogFactory& getInstance();
+    static void showErrorDialog(AssetUpload* upload, QWidget* dialogParent, const QString& overrideMessage = QString());
     
     void setDialogParent(QWidget* dialogParent) { _dialogParent = dialogParent; }
     
@@ -35,7 +36,7 @@ public slots:
 private:
     AssetUploadDialogFactory() = default;
     
-    void showErrorDialog(const QString& filename, const QString& additionalError);
+    
     
     QWidget* _dialogParent { nullptr };
 };

libraries/networking/src/AssetClient.cpp

@@ -65,43 +65,65 @@ void AssetClient::init() {
     }
 }
 
+bool haveAssetServer() {
+    auto nodeList = DependencyManager::get<NodeList>();
+    SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
+    
+    if (!assetServer) {
+        qCWarning(asset_client) << "Could not complete AssetClient operation "
+            << "since you are not currently connected to an asset-server.";
+        return false;
+    }
+    
+    return true;
+}
+
 AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) {
     if (hash.length() != SHA256_HASH_HEX_LENGTH) {
         qCWarning(asset_client) << "Invalid hash size";
         return nullptr;
     }
 
-    auto nodeList = DependencyManager::get<NodeList>();
+    if (haveAssetServer()) {
-    SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
+        auto request = new AssetRequest(hash, extension);
-
+        
-    if (!assetServer) {
+        // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
-        qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension
+        request->moveToThread(thread());
-            << " since you are not currently connected to an asset-server.";
+        request->setParent(this);
+        
+        return request;
+    } else {
         return nullptr;
     }
-
-    auto request = new AssetRequest(hash, extension);
-
-    // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case)
-    request->moveToThread(thread());
-
-    return request;
 }
 
+
+
 AssetUpload* AssetClient::createUpload(const QString& filename) {
-    auto nodeList = DependencyManager::get<NodeList>();
-    SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
     
-    if (!assetServer) {
+    if (haveAssetServer()) {
-        qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
+        auto upload = new AssetUpload(filename);
+        
+        upload->moveToThread(thread());
+        upload->setParent(this);
+        
+        return upload;
+    } else {
         return nullptr;
     }
-    
+}
-    auto upload = new AssetUpload(this, filename);
 
-    upload->moveToThread(thread());
+AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& extension) {
-
+    if (haveAssetServer()) {
-    return upload;
+        auto upload = new AssetUpload(data, extension);
+        
+        upload->moveToThread(thread());
+        upload->setParent(this);
+        
+        return upload;
+    } else {
+        return nullptr;
+    }
 }
 
 bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,

libraries/networking/src/AssetClient.h

@@ -45,6 +45,7 @@ public:
 
     Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension);
     Q_INVOKABLE AssetUpload* createUpload(const QString& filename);
+    Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension);
 
 private slots:
     void handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);

libraries/networking/src/AssetUpload.cpp

@@ -17,59 +17,94 @@
 #include "AssetClient.h"
 #include "NetworkLogging.h"
 
-AssetUpload::AssetUpload(QObject* object, const QString& filename) :
+const QString AssetUpload::PERMISSION_DENIED_ERROR = "You do not have permission to upload content to this asset-server.";
+
+AssetUpload::AssetUpload(const QByteArray& data, const QString& extension) :
+    _data(data),
+    _extension(extension)
+{
+    
+}
+
+AssetUpload::AssetUpload(const QString& filename) :
     _filename(filename)
 {
     
 }
 
+QString AssetUpload::getErrorString() const {
+    // figure out the right error message for error
+    switch (_error) {
+        case AssetUpload::PermissionDenied:
+            return PERMISSION_DENIED_ERROR;
+        case AssetUpload::TooLarge:
+            return "The uploaded content was too large and could not be stored in the asset-server.";
+        case AssetUpload::FileOpenError:
+            return "The file could not be opened. Please check your permissions and try again.";
+        case AssetUpload::NetworkError:
+            return "The file could not be opened. Please check your network connectivity.";
+        default:
+            // not handled, do not show a message box
+            return QString();
+    }
+}
+
 void AssetUpload::start() {
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "start", Qt::AutoConnection);
         return;
     }
     
-    // try to open the file at the given filename
+    auto data = _data;
-    QFile file { _filename };
     
-    if (file.open(QIODevice::ReadOnly)) {
+    if (data.isEmpty() && !_filename.isEmpty()) {
+        // try to open the file at the given filename
+        QFile file { _filename };
         
-        // file opened, read the data and grab the extension
+        if (file.open(QIODevice::ReadOnly)) {
-        _extension = QFileInfo(_filename).suffix();
+            
-        
+            // file opened, read the data and grab the extension
-        auto data = file.readAll();
+            _extension = QFileInfo(_filename).suffix();
-        
+            
-        // ask the AssetClient to upload the asset and emit the proper signals from the passed callback
+            data = file.readAll();
-        auto assetClient = DependencyManager::get<AssetClient>();
+        } else {
-        
+            // we couldn't open the file - set the error result
-        qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
+            _error = FileOpenError;
-        
+            
-        assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
+            // emit that we are done
-            if (!responseReceived) {
+            emit finished(this, QString());
-                _error = NetworkError;
+        }
-            } else {
-                switch (error) {
-                    case AssetServerError::NoError:
-                        _error = NoError;
-                        break;
-                    case AssetServerError::AssetTooLarge:
-                        _error = TooLarge;
-                        break;
-                    case AssetServerError::PermissionDenied:
-                        _error = PermissionDenied;
-                        break;
-                    default:
-                        _error = FileOpenError;
-                        break;
-                }
-            }
-            emit finished(this, hash);
-        });
-    } else {
-        // we couldn't open the file - set the error result
-        _error = FileOpenError;
-        
-        // emit that we are done
-        emit finished(this, QString());
     }
+    
+    // ask the AssetClient to upload the asset and emit the proper signals from the passed callback
+    auto assetClient = DependencyManager::get<AssetClient>();
+   
+    if (!_filename.isEmpty()) {
+        qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
+    } else {
+        qCDebug(asset_client) << "Attempting to upload file of" << _data.size() << "bytes to asset-server.";
+    }
+    
+    
+    assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
+        if (!responseReceived) {
+            _error = NetworkError;
+        } else {
+            switch (error) {
+                case AssetServerError::NoError:
+                    _error = NoError;
+                    break;
+                case AssetServerError::AssetTooLarge:
+                    _error = TooLarge;
+                    break;
+                case AssetServerError::PermissionDenied:
+                    _error = PermissionDenied;
+                    break;
+                default:
+                    _error = FileOpenError;
+                    break;
+            }
+        }
+        emit finished(this, hash);
+    });
 }

libraries/networking/src/AssetUpload.h

@@ -35,13 +35,17 @@ public:
         FileOpenError
     };
     
-    AssetUpload(QObject* parent, const QString& filename);
+    static const QString PERMISSION_DENIED_ERROR;
+    
+    AssetUpload(const QString& filename);
+    AssetUpload(const QByteArray& data, const QString& extension);
     
     Q_INVOKABLE void start();
     
     const QString& getFilename() const { return _filename; }
     const QString& getExtension() const  { return _extension; }
     const Error& getError() const { return _error; }
+    QString getErrorString() const;
     
 signals:
     void finished(AssetUpload* upload, const QString& hash);
@@ -49,6 +53,7 @@ signals:
     
 private:
     QString _filename;
+    QByteArray _data;
     QString _extension;
     Error _error;
 };