Merge pull request #2613 from Atlante45/more_upload_work

More upload work

interface/src/Application.cpp

@@ -3248,9 +3248,12 @@ void Application::toggleRunningScriptsWidget()
 
 void Application::uploadFST(bool isHead) {
     ModelUploader* uploader = new ModelUploader(isHead);
-    if (uploader->zip()) {
+    QThread* thread = new QThread();
-        uploader->send();
+    thread->connect(uploader, SIGNAL(destroyed()), SLOT(quit()));
-    }
+    thread->connect(thread, SIGNAL(finished()), SLOT(deleteLater()));
+    uploader->connect(thread, SIGNAL(started()), SLOT(send()));
+    
+    thread->start();
 }
 
 void Application::uploadHead() {

libraries/shared/src/AccountManager.cpp

@@ -34,6 +34,16 @@ Q_DECLARE_METATYPE(JSONCallbackParameters)
 
 const QString ACCOUNTS_GROUP = "accounts";
 
+JSONCallbackParameters::JSONCallbackParameters() :
+    jsonCallbackReceiver(NULL),
+    jsonCallbackMethod(),
+    errorCallbackReceiver(NULL),
+    errorCallbackMethod(),
+    updateReciever(NULL),
+    updateSlot()
+{
+}
+
 AccountManager::AccountManager() :
     _authURL(),
     _networkAccessManager(NULL),
@@ -170,18 +180,31 @@ void AccountManager::invokedRequest(const QString& path, QNetworkAccessManager::
             if (!callbackParams.isEmpty()) {
                 // if we have information for a callback, insert the callbackParams into our local map
                 _pendingCallbackMap.insert(networkReply, callbackParams);
+                
+                if (callbackParams.updateReciever && !callbackParams.updateSlot.isEmpty()) {
+                    callbackParams.updateReciever->connect(networkReply, SIGNAL(uploadProgress(qint64, qint64)),
+                                                            callbackParams.updateSlot.toStdString().c_str());
+                }
             }
             
             // if we ended up firing of a request, hook up to it now
-            connect(networkReply, SIGNAL(finished()), this, SLOT(passSuccessToCallback()));
+            connect(networkReply, SIGNAL(finished()), SLOT(processReply()));
-            connect(networkReply, SIGNAL(error(QNetworkReply::NetworkError)),
-                    this, SLOT(passErrorToCallback(QNetworkReply::NetworkError)));
         }
     }
 }
 
-void AccountManager::passSuccessToCallback() {
+void AccountManager::processReply() {
     QNetworkReply* requestReply = reinterpret_cast<QNetworkReply*>(sender());
+    
+    if (requestReply->error() == QNetworkReply::NoError) {
+        passSuccessToCallback(requestReply);
+    } else {
+        passErrorToCallback(requestReply);
+    }
+    delete requestReply;
+}
+
+void AccountManager::passSuccessToCallback(QNetworkReply* requestReply) {
     QJsonDocument jsonResponse = QJsonDocument::fromJson(requestReply->readAll());
     
     JSONCallbackParameters callbackParams = _pendingCallbackMap.value(requestReply);
@@ -200,17 +223,15 @@ void AccountManager::passSuccessToCallback() {
             qDebug() << jsonResponse;
         }
     }
-    delete requestReply;
 }
 
-void AccountManager::passErrorToCallback(QNetworkReply::NetworkError errorCode) {
+void AccountManager::passErrorToCallback(QNetworkReply* requestReply) {
-    QNetworkReply* requestReply = reinterpret_cast<QNetworkReply*>(sender());
     JSONCallbackParameters callbackParams = _pendingCallbackMap.value(requestReply);
     
     if (callbackParams.errorCallbackReceiver) {
         // invoke the right method on the callback receiver
         QMetaObject::invokeMethod(callbackParams.errorCallbackReceiver, qPrintable(callbackParams.errorCallbackMethod),
-                                  Q_ARG(QNetworkReply::NetworkError, errorCode),
+                                  Q_ARG(QNetworkReply::NetworkError, requestReply->error()),
                                   Q_ARG(const QString&, requestReply->errorString()));
         
         // remove the related reply-callback group from the map
@@ -218,10 +239,9 @@ void AccountManager::passErrorToCallback(QNetworkReply::NetworkError errorCode)
     } else {
         if (VERBOSE_HTTP_REQUEST_DEBUGGING) {
             qDebug() << "Received error response from data-server that has no matching callback.";
-            qDebug() << "Error" << errorCode << "-" << requestReply->errorString();
+            qDebug() << "Error" << requestReply->error() << "-" << requestReply->errorString();
         }
     }
-    delete requestReply;
 }
 
 bool AccountManager::hasValidAccessToken() {

libraries/shared/src/AccountManager.h

@@ -19,9 +19,7 @@
 
 class JSONCallbackParameters {
 public:
-    JSONCallbackParameters() :
+    JSONCallbackParameters();
-        jsonCallbackReceiver(NULL), jsonCallbackMethod(),
-        errorCallbackReceiver(NULL), errorCallbackMethod() {};
     
     bool isEmpty() const { return !jsonCallbackReceiver && !errorCallbackReceiver; }
     
@@ -29,6 +27,8 @@ public:
     QString jsonCallbackMethod;
     QObject* errorCallbackReceiver;
     QString errorCallbackMethod;
+    QObject* updateReciever;
+    QString updateSlot;
 };
 
 class AccountManager : public QObject {
@@ -70,13 +70,15 @@ signals:
     void loginComplete(const QUrl& authURL);
     void logoutComplete();
 private slots:
-    void passSuccessToCallback();
+    void processReply();
-    void passErrorToCallback(QNetworkReply::NetworkError errorCode);
 private:
     AccountManager();
     AccountManager(AccountManager const& other); // not implemented
     void operator=(AccountManager const& other); // not implemented
     
+    void passSuccessToCallback(QNetworkReply* reply);
+    void passErrorToCallback(QNetworkReply* reply);
+    
     Q_INVOKABLE void invokedRequest(const QString& path, QNetworkAccessManager::Operation operation,
                                     const JSONCallbackParameters& callbackParams,
                                     const QByteArray& dataByteArray,

libraries/shared/src/ModelUploader.cpp

@@ -9,13 +9,14 @@
 
 #include <QDebug>
 #include <QFile>
-#include <QTextStream>
 #include <QFileDialog>
-#include <QStandardPaths>
+#include <QGridLayout>
 #include <QHttpMultiPart>
-#include <QTemporaryDir>
-#include <QVariant>
 #include <QMessageBox>
+#include <QProgressBar>
+#include <QStandardPaths>
+#include <QTextStream>
+#include <QVariant>
 
 #include "AccountManager.h"
 #include "ModelUploader.h"
@@ -26,29 +27,26 @@ static const QString FILENAME_FIELD = "filename";
 static const QString TEXDIR_FIELD = "texdir";
 static const QString LOD_FIELD = "lod";
 
+static const QString S3_URL = "http://highfidelity-public.s3-us-west-1.amazonaws.com";
 static const QString MODEL_URL = "/api/v1/models";
 
 static const int MAX_SIZE = 10 * 1024 * 1024; // 10 MB
+static const int TIMEOUT = 1000;
+static const int MAX_CHECK = 30;
 
-// Class providing the QObject parent system to QTemporaryDir
+static const int QCOMPRESS_HEADER_POSITION = 0;
-class TemporaryDir : public QTemporaryDir, public QObject {
+static const int QCOMPRESS_HEADER_SIZE = 4;
-public:
-    virtual ~TemporaryDir() {
-        // ensuring the entire object gets deleted by the QObject parent.
-    }
-};
 
 ModelUploader::ModelUploader(bool isHead) :
-    _zipDir(new TemporaryDir()),
     _lodCount(-1),
     _texturesCount(-1),
     _totalSize(0),
     _isHead(isHead),
     _readyToSend(false),
-    _dataMultiPart(new QHttpMultiPart(QHttpMultiPart::FormDataType))
+    _dataMultiPart(new QHttpMultiPart(QHttpMultiPart::FormDataType)),
+    _numberOfChecks(MAX_CHECK)
 {
-    _zipDir->setParent(_dataMultiPart);
+    connect(&_timer, SIGNAL(timeout()), SLOT(checkS3()));
-    
 }
 
 ModelUploader::~ModelUploader() {
@@ -59,8 +57,9 @@ bool ModelUploader::zip() {
     // File Dialog
     QString filename = QFileDialog::getOpenFileName(NULL,
                                                     "Select your .fst file ...",
-                                                    QStandardPaths::writableLocation(QStandardPaths::DownloadLocation),
+                                                    QStandardPaths::writableLocation(QStandardPaths::HomeLocation),
                                                     "*.fst");
+    qDebug() << QStandardPaths::writableLocation(QStandardPaths::HomeLocation);
     if (filename == "") {
         // If the user canceled we return.
         return false;
@@ -79,11 +78,7 @@ bool ModelUploader::zip() {
     qDebug() << "Reading FST file : " << QFileInfo(fst).filePath();
     
     // Compress and copy the fst
-    if (!compressFile(QFileInfo(fst).filePath(), _zipDir->path() + "/" + QFileInfo(fst).fileName())) {
+    if (!addPart(QFileInfo(fst).filePath(), QString("fst"))) {
-        return false;
-    }
-    if (!addPart(_zipDir->path() + "/" + QFileInfo(fst).fileName(),
-                 QString("fst"))) {
         return false;
     }
     
@@ -103,6 +98,7 @@ bool ModelUploader::zip() {
                                " name=\"model_name\"");
             textPart.setBody(line[1].toUtf8());
             _dataMultiPart->append(textPart);
+            _url = S3_URL + ((_isHead)? "/models/heads/" : "/models/skeletons/") + line[1].toUtf8() + ".fst";
         } else if (line[0] == FILENAME_FIELD) {
             QFileInfo fbx(QFileInfo(fst).path() + "/" + line[1]);
             if (!fbx.exists() || !fbx.isFile()) { // Check existence
@@ -114,10 +110,7 @@ bool ModelUploader::zip() {
                 return false;
             }
             // Compress and copy
-            if (!compressFile(fbx.filePath(), _zipDir->path() + "/" + line[1])) {
+            if (!addPart(fbx.filePath(), "fbx")) {
-                return false;
-            }
-            if (!addPart(_zipDir->path() + "/" + line[1], "fbx")) {
                 return false;
             }
         } else if (line[0] == TEXDIR_FIELD) { // Check existence
@@ -144,10 +137,7 @@ bool ModelUploader::zip() {
                 return false;
             }
             // Compress and copy
-            if (!compressFile(lod.filePath(), _zipDir->path() + "/" + line[1])) {
+            if (!addPart(lod.filePath(), QString("lod%1").arg(++_lodCount))) {
-                return false;
-            }
-            if (!addPart(_zipDir->path() + "/" + line[1], QString("lod%1").arg(++_lodCount))) {
                 return false;
             }
         }
@@ -167,9 +157,9 @@ bool ModelUploader::zip() {
     return true;
 }
 
-bool ModelUploader::send() {
+void ModelUploader::send() {
-    if (!_readyToSend) {
+    if (!zip()) {
-        return false;
+        return;
     }
     
     JSONCallbackParameters callbackParams;
@@ -177,22 +167,51 @@ bool ModelUploader::send() {
     callbackParams.jsonCallbackMethod = "uploadSuccess";
     callbackParams.errorCallbackReceiver = this;
     callbackParams.errorCallbackMethod = "uploadFailed";
+    callbackParams.updateReciever = this;
+    callbackParams.updateSlot = SLOT(uploadUpdate(qint64, qint64));
     
     AccountManager::getInstance().authenticatedRequest(MODEL_URL, QNetworkAccessManager::PostOperation, callbackParams, QByteArray(), _dataMultiPart);
-    _zipDir = NULL;
     _dataMultiPart = NULL;
     qDebug() << "Sending model...";
+    _progressDialog = new QDialog();
+    _progressBar = new QProgressBar(_progressDialog);
+    _progressBar->setRange(0, 100);
+    _progressBar->setValue(0);
     
-    return true;
+    _progressDialog->setWindowTitle("Uploading model...");
+    _progressDialog->setLayout(new QGridLayout(_progressDialog));
+    _progressDialog->layout()->addWidget(_progressBar);
+    
+    _progressDialog->exec();
+
+    delete _progressDialog;
+    _progressDialog = NULL;
+    _progressBar = NULL;
+}
+
+void ModelUploader::uploadUpdate(qint64 bytesSent, qint64 bytesTotal) {
+    if (_progressDialog) {
+        _progressBar->setRange(0, bytesTotal);
+        _progressBar->setValue(bytesSent);
+    }
 }
 
 void ModelUploader::uploadSuccess(const QJsonObject& jsonResponse) {
-    qDebug() << "Model sent with success to the data server.";
+    if (_progressDialog) {
-    qDebug() << "It might take a few minute for it to appear in your model browser.";
+        _progressDialog->accept();
-    deleteLater();
+    }
+    QMessageBox::information(NULL,
+                             QString("ModelUploader::uploadSuccess()"),
+                             QString("Your model is being processed by the system."),
+                             QMessageBox::Ok);
+    qDebug() << "Model sent with success";
+    checkS3();
 }
 
 void ModelUploader::uploadFailed(QNetworkReply::NetworkError errorCode, const QString& errorString) {
+    if (_progressDialog) {
+        _progressDialog->reject();
+    }
     QMessageBox::warning(NULL,
                          QString("ModelUploader::uploadFailed()"),
                          QString("Model could not be sent to the data server."),
@@ -201,6 +220,42 @@ void ModelUploader::uploadFailed(QNetworkReply::NetworkError errorCode, const QS
     deleteLater();
 }
 
+void ModelUploader::checkS3() {
+    qDebug() << "Checking S3 for " << _url;
+    QNetworkRequest request(_url);
+    QNetworkReply* reply = _networkAccessManager.head(request);
+    connect(reply, SIGNAL(finished()), SLOT(processCheck()));
+}
+
+void ModelUploader::processCheck() {
+    QNetworkReply* reply = static_cast<QNetworkReply*>(sender());
+    _timer.stop();
+    
+    switch (reply->error()) {
+        case QNetworkReply::NoError:
+            QMessageBox::information(NULL,
+                                     QString("ModelUploader::processCheck()"),
+                                     QString("Your model is now available in the browser."),
+                                     QMessageBox::Ok);
+            deleteLater();
+            break;
+        case QNetworkReply::ContentNotFoundError:
+            if (--_numberOfChecks) {
+                _timer.start(TIMEOUT);
+                break;
+            }
+        default:
+            QMessageBox::warning(NULL,
+                                 QString("ModelUploader::processCheck()"),
+                                 QString("Could not verify that the model is present on the server."),
+                                 QMessageBox::Ok);
+            deleteLater();
+            break;
+    }
+    
+    delete reply;
+}
+
 bool ModelUploader::addTextures(const QFileInfo& texdir) {
     QStringList filter;
     filter << "*.png" << "*.tif" << "*.jpg" << "*.jpeg";
@@ -213,11 +268,7 @@ bool ModelUploader::addTextures(const QFileInfo& texdir) {
     foreach (QFileInfo info, list) {
         if (info.isFile()) {
             // Compress and copy
-            if (!compressFile(info.filePath(), _zipDir->path() + "/" + info.fileName())) {
+            if (!addPart(info.filePath(), QString("texture%1").arg(++_texturesCount))) {
-                return false;
-            }
-            if (!addPart(_zipDir->path() + "/" + info.fileName(),
-                         QString("texture%1").arg(++_texturesCount))) {
                 return false;
             }
         } else if (info.isDir()) {
@@ -230,54 +281,33 @@ bool ModelUploader::addTextures(const QFileInfo& texdir) {
     return true;
 }
 
-bool ModelUploader::compressFile(const QString &inFileName, const QString &outFileName) {
-    QFile inFile(inFileName);
-    inFile.open(QIODevice::ReadOnly);
-    QByteArray buffer = inFile.readAll();
-    
-    QFile outFile(outFileName);
-    if (!outFile.open(QIODevice::WriteOnly)) {
-        QDir(_zipDir->path()).mkpath(QFileInfo(outFileName).path());
-        if (!outFile.open(QIODevice::WriteOnly)) {
-            QMessageBox::warning(NULL,
-                                 QString("ModelUploader::compressFile()"),
-                                 QString("Could not compress %1").arg(inFileName),
-                                 QMessageBox::Ok);
-            qDebug() << "[Warning] " << QString("Could not compress %1").arg(inFileName);
-            return false;
-        }
-    }
-    QDataStream out(&outFile);
-    out << qCompress(buffer);
-    
-    return true;
-}
-
-
 bool ModelUploader::addPart(const QString &path, const QString& name) {
-    QFile* file = new QFile(path);
+    QFile file(path);
-    if (!file->open(QIODevice::ReadOnly)) {
+    if (!file.open(QIODevice::ReadOnly)) {
         QMessageBox::warning(NULL,
                              QString("ModelUploader::addPart()"),
                              QString("Could not open %1").arg(path),
                              QMessageBox::Ok);
         qDebug() << "[Warning] " << QString("Could not open %1").arg(path);
-        delete file;
         return false;
     }
+    QByteArray buffer = qCompress(file.readAll());
+    
+    // Qt's qCompress() default compression level (-1) is the standard zLib compression.
+    // Here remove Qt's custom header that prevent the data server from uncompressing the files with zLib.
+    buffer.remove(QCOMPRESS_HEADER_POSITION, QCOMPRESS_HEADER_SIZE);
     
     QHttpPart part;
-    part.setHeader(QNetworkRequest::ContentDispositionHeader, "form-data;"
+    part.setHeader(QNetworkRequest::ContentDispositionHeader, QVariant("form-data;"
                    " name=\"" + name.toUtf8() +  "\";"
-                   " filename=\"" + QFileInfo(*file).fileName().toUtf8() +  "\"");
+                   " filename=\"" + QFileInfo(file).fileName().toUtf8() +  "\""));
-    part.setHeader(QNetworkRequest::ContentTypeHeader, "application/octet-stream");
+    part.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("application/octet-stream"));
-    part.setBodyDevice(file);
+    part.setBody(buffer);
     _dataMultiPart->append(part);
-    file->setParent(_dataMultiPart);
     
     
-    qDebug() << "File " << QFileInfo(*file).fileName() << " added to model.";
+    qDebug() << "File " << QFileInfo(file).fileName() << " added to model.";
-    _totalSize += file->size();
+    _totalSize += file.size();
     if (_totalSize > MAX_SIZE) {
         QMessageBox::warning(NULL,
                              QString("ModelUploader::zip()"),

libraries/shared/src/ModelUploader.h

@@ -10,9 +10,14 @@
 #ifndef __hifi__ModelUploader__
 #define __hifi__ModelUploader__
 
-class TemporaryDir;
+#include <QTimer>
-class QHttpMultiPart;
+
+class QDialog;
 class QFileInfo;
+class QHttpMultiPart;
+class QProgressBar;
+
+class TemporaryDir;
 
 class ModelUploader : public QObject {
     Q_OBJECT
@@ -21,15 +26,18 @@ public:
     ModelUploader(bool isHead);
     ~ModelUploader();
     
-    bool zip();
+public slots:
-    bool send();
+    void send();
     
 private slots:
+    void uploadUpdate(qint64 bytesSent, qint64 bytesTotal);
     void uploadSuccess(const QJsonObject& jsonResponse);
     void uploadFailed(QNetworkReply::NetworkError errorCode, const QString& errorString);
+    void checkS3();
+    void processCheck();
     
 private:
-    TemporaryDir* _zipDir;
+    QString _url;
     int _lodCount;
     int _texturesCount;
     int _totalSize;
@@ -37,10 +45,17 @@ private:
     bool _readyToSend;
     
     QHttpMultiPart* _dataMultiPart;
+    QNetworkAccessManager _networkAccessManager;
+    
+    int _numberOfChecks;
+    QTimer _timer;
+    
+    QDialog* _progressDialog;
+    QProgressBar* _progressBar;
     
     
+    bool zip();
     bool addTextures(const QFileInfo& texdir);
-    bool compressFile(const QString& inFileName, const QString& outFileName);
     bool addPart(const QString& path, const QString& name);
 };