From bf9bfb4575730f01f034f981712f845e8544301b Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:07:20 -0700 Subject: [PATCH 01/77] Add AssetServer --- assignment-client/src/assets/AssetServer.cpp | 149 +++++++++++++++++++ assignment-client/src/assets/AssetServer.h | 40 +++++ 2 files changed, 189 insertions(+) create mode 100644 assignment-client/src/assets/AssetServer.cpp create mode 100644 assignment-client/src/assets/AssetServer.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp new file mode 100644 index 0000000000..1a384271e0 --- /dev/null +++ b/assignment-client/src/assets/AssetServer.cpp @@ -0,0 +1,149 @@ +// +// AssetServer.cpp +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#include "AssetServer.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; +const int HASH_HEX_LENGTH = 32; +using MessageID = int; + +AssetServer::AssetServer(NLPacket& packet) : ThreadedAssignment(packet) { + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); + packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); +} + +AssetServer::~AssetServer() { +} + +void AssetServer::run() { + ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer); + + auto nodeList = DependencyManager::get(); + nodeList->addNodeTypeToInterestSet(NodeType::Agent); + + _resourcesDirectory = QDir(QCoreApplication::applicationDirPath()).filePath("resources/assets"); + if (!_resourcesDirectory.exists()) { + qDebug() << "Creating resources directory"; + _resourcesDirectory.mkpath("."); + } + qDebug() << "Serving files from: " << _resourcesDirectory.path(); + + // Scan for new files + qDebug() << "Looking for new files in asset directory"; + auto files = _resourcesDirectory.entryInfoList(QDir::Files); + QRegExp filenameRegex { "^[a-f0-9]{32}$" }; + for (auto fileInfo : files) { + auto filename = fileInfo.fileName(); + if (!filenameRegex.exactMatch(filename)) { + qDebug() << "Found file: " << filename; + if (!fileInfo.isReadable()) { + qDebug() << "\tCan't open file for reading: " << filename; + continue; + } + } + } + + while (!_isFinished) { + // since we're a while loop we need to help Qt's event processing + QCoreApplication::processEvents(); + } +} + +void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { + QByteArray assetHash; + + if (packet->getPayloadSize() < HASH_HEX_LENGTH) { + qDebug() << "ERROR bad file request"; + return; + } + + assetHash = packet->read(HASH_HEX_LENGTH); + qDebug() << "Got a request for the file: " << assetHash; + + // We need to reply... + QFile file { _resourcesDirectory.filePath(QString(assetHash)) }; + qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); + bool found = file.open(QIODevice::ReadOnly); + + auto assetPacket = NLPacket::create(PacketType::AssetGetReply); + + assetPacket->write(assetHash, HASH_HEX_LENGTH); + assetPacket->writePrimitive(found); + + const int MAX_LENGTH = 1024; + + if (found) { + QByteArray data = file.read(MAX_LENGTH); + assetPacket->writePrimitive(data.size()); + assetPacket->write(data, data.size()); + } else { + qDebug() << "File not found"; + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(assetPacket), *senderNode); +} + +void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + packet->readPrimitive(&messageID); + + char extensionLength; + packet->readPrimitive(&extensionLength); + + char extension[extensionLength]; + packet->read(extension, extensionLength); + + qDebug() << "Got extension: " << extension; + + int fileSize; + packet->readPrimitive(&fileSize); + + const int MAX_LENGTH = 1024; + fileSize = std::min(MAX_LENGTH, fileSize); + qDebug() << "Receiving a file of size " << fileSize; + + QByteArray data = packet->read(fileSize); + + QString hash = QString(QCryptographicHash::hash(data, QCryptographicHash::Md5).toHex()); + + qDebug() << "Got data: (" << hash << ") " << data; + + QFile file { _resourcesDirectory.filePath(QString(hash)) }; + + if (file.exists()) { + qDebug() << "[WARNING] This file already exists"; + } else { + file.open(QIODevice::WriteOnly); + file.write(data); + } + + auto replyPacket = NLPacket::create(PacketType::AssetUploadReply); + + replyPacket->writePrimitive(messageID); + + replyPacket->writePrimitive(true); + replyPacket->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *senderNode); +} diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h new file mode 100644 index 0000000000..ef658fae4a --- /dev/null +++ b/assignment-client/src/assets/AssetServer.h @@ -0,0 +1,40 @@ +// +// AssetServer.h +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetServer_h +#define hifi_AssetServer_h + +#include + +#include + +class AssetServer : public ThreadedAssignment { + Q_OBJECT +public: + AssetServer(NLPacket& packet); + ~AssetServer(); + + enum ServerResponse { + ASSET_NOT_FOUND = 0, + ASSET_UPLOADED = 1, + }; + +public slots: + void run(); + +private slots: + void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode); + +private: + QDir _resourcesDirectory; +}; + +#endif From 4c42edf7fabed01521dcb8174763c6f09a3ca1ec Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:07:40 -0700 Subject: [PATCH 02/77] Add AssetServer to AssignmentFactory --- assignment-client/src/AssignmentFactory.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/assignment-client/src/AssignmentFactory.cpp b/assignment-client/src/AssignmentFactory.cpp index c6ad2c9f09..cacc523ebd 100644 --- a/assignment-client/src/AssignmentFactory.cpp +++ b/assignment-client/src/AssignmentFactory.cpp @@ -16,6 +16,7 @@ #include "audio/AudioMixer.h" #include "avatars/AvatarMixer.h" #include "entities/EntityServer.h" +#include "assets/AssetServer.h" ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { @@ -33,6 +34,8 @@ ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { return new Agent(packet); case Assignment::EntityServerType: return new EntityServer(packet); + case Assignment::AssetServerType: + return new AssetServer(packet); default: return NULL; } From ee5a1028f796c6c2a4dfd17f6cab58875d6a729e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:08:38 -0700 Subject: [PATCH 03/77] Add AssetClient --- libraries/networking/src/AssetClient.cpp | 124 +++++++++++++++++++++++ libraries/networking/src/AssetClient.h | 51 ++++++++++ 2 files changed, 175 insertions(+) create mode 100644 libraries/networking/src/AssetClient.cpp create mode 100644 libraries/networking/src/AssetClient.h diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp new file mode 100644 index 0000000000..e8131a40f2 --- /dev/null +++ b/libraries/networking/src/AssetClient.cpp @@ -0,0 +1,124 @@ +// +// AssetClient.cpp +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetClient.h" + +#include "NodeList.h" +#include "PacketReceiver.h" + +const int HASH_HEX_LENGTH = 32; +MessageID AssetClient::_currentID = 0; + +AssetClient::AssetClient() { + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply"); + packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); +} + +bool AssetClient::getAsset(QString hash, ReceivedAssetCallback callback) { + if (hash.length() != 32) { + qDebug() << "Invalid hash size"; + return false; + } + + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packet = NLPacket::create(PacketType::AssetGet); + packet->write(hash.toLatin1().constData(), 32); + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingRequests[hash] = callback; + + return true; + } + + return false; +} + +void AssetClient::handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode) { + auto assetHash = packet->read(HASH_HEX_LENGTH); + qDebug() << "Got reply for asset: " << assetHash; + + bool success; + packet->readPrimitive(&success); + QByteArray data; + + if (success) { + int length; + packet->readPrimitive(&length); + char assetData[length]; + packet->read(assetData, length); + + data = QByteArray(assetData, length); + } else { + qDebug() << "Failure getting asset"; + } + + if (_pendingRequests.contains(assetHash)) { + auto callback = _pendingRequests.take(assetHash); + callback(success, data); + } +} + +bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + if (assetServer) { + auto packet = NLPacket::create(PacketType::AssetUpload); + + auto messageID = _currentID++; + packet->writePrimitive(messageID); + + packet->writePrimitive(static_cast(extension.length())); + packet->write(extension.toLatin1().constData(), extension.length()); + + qDebug() << "Extension length: " << extension.length(); + qDebug() << "Extension: " << extension; + + int size = data.length(); + packet->writePrimitive(size); + packet->write(data.constData(), size); + + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingUploads[messageID] = callback; + + return true; + } + return false; +} + +void AssetClient::handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode) { + qDebug() << "Got asset upload reply"; + MessageID messageID; + packet->readPrimitive(&messageID); + + bool success; + packet->readPrimitive(&success); + + QString hashString { "" }; + + if (success) { + auto hashData = packet->read(HASH_HEX_LENGTH); + + hashString = QString(hashData); + + qDebug() << "Hash: " << hashString; + } else { + qDebug() << "Error uploading file"; + } + + if (_pendingUploads.contains(messageID)) { + auto callback = _pendingUploads.take(messageID); + callback(success, hashString); + } +} diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h new file mode 100644 index 0000000000..87da601563 --- /dev/null +++ b/libraries/networking/src/AssetClient.h @@ -0,0 +1,51 @@ +// +// AssetClient.h +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#ifndef hifi_AssetClient_h +#define hifi_AssetClient_h + +#include +#include + +#include + +#include "LimitedNodeList.h" +#include "NLPacket.h" + +using ReceivedAssetCallback = std::function; +using UploadResultCallback = std::function; +using MessageID = int; + +class AssetClient : public QObject, public Dependency { + Q_OBJECT +public: + AssetClient(); + + enum RequestResult { + SUCCESS = 0, + FAILURE, + TIMEOUT + }; + + bool getAsset(QString hash, ReceivedAssetCallback callback); + bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); + +private slots: + void handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode); + +private: + static MessageID _currentID; + QMap _pendingRequests; + QMap _pendingUploads; +}; + +#endif From 52e422198b338462259c0b1dd47ea5fb196a8df6 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:09:59 -0700 Subject: [PATCH 04/77] Add AssetScriptingInterface --- interface/src/AssetScriptingInterface.cpp | 43 +++++++++++++++++++++++ interface/src/AssetScriptingInterface.h | 31 ++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 interface/src/AssetScriptingInterface.cpp create mode 100644 interface/src/AssetScriptingInterface.h diff --git a/interface/src/AssetScriptingInterface.cpp b/interface/src/AssetScriptingInterface.cpp new file mode 100644 index 0000000000..c03e9ee8c2 --- /dev/null +++ b/interface/src/AssetScriptingInterface.cpp @@ -0,0 +1,43 @@ +// +// AssetScriptingInterface.cpp +// +// Created by Ryan Huffman on 2015/07/22 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetScriptingInterface.h" + +#include + +#include +#include +#include +#include +#include + +#include + +const int HASH_HEX_LENGTH = 32; + +AssetScriptingInterface::AssetScriptingInterface() { +} + +QScriptValue AssetScriptingInterface::getAsset(QString url, QScriptValue callback) { + bool success = AssetManager::getAsset(QUrl(url), [callback](AssetRequestUpdateType type, QByteArray data) mutable { + auto result = callback.engine()->newVariant(data); + QList arguments { type == AssetRequestUpdateType::COMPLETE, result }; + callback.call(QScriptValue(), arguments); + }); + return success; +} + +QScriptValue AssetScriptingInterface::uploadAsset(QString data, QString extension, QScriptValue callback) { + auto assetClient = DependencyManager::get(); + return assetClient->uploadAsset(data.toLatin1(), extension, [callback](bool success, QString hash) mutable { + QList arguments { success, hash }; + auto result = callback.call(QScriptValue(), arguments); + }); +} diff --git a/interface/src/AssetScriptingInterface.h b/interface/src/AssetScriptingInterface.h new file mode 100644 index 0000000000..8c0f7a71a1 --- /dev/null +++ b/interface/src/AssetScriptingInterface.h @@ -0,0 +1,31 @@ +// +// AssetScriptingInterface.h +// +// Created by Ryan Huffman on 2015/07/22 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetScriptingInterface_h +#define hifi_AssetScriptingInterface_h + +#include +#include + +#include +#include +#include + +class AssetScriptingInterface : public QObject, public Dependency { + Q_OBJECT +public: + AssetScriptingInterface(); + +public slots: + QScriptValue getAsset(QString hash, QScriptValue callback); + QScriptValue uploadAsset(QString data, QString extension, QScriptValue callback); +}; + +#endif From 0b2e896ea48b7c0de6ca0e256a8abe24e48f8699 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:13:56 -0700 Subject: [PATCH 05/77] Add Asset-related types and packets --- interface/src/Application.cpp | 9 ++++++++- libraries/networking/src/Assignment.cpp | 2 ++ libraries/networking/src/Assignment.h | 3 ++- libraries/networking/src/NodeType.h | 1 + libraries/networking/src/udt/PacketHeaders.h | 6 +++++- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7de69ef602..137f3ffa83 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -52,6 +52,8 @@ #include #include +#include +#include #include #include #include @@ -287,6 +289,8 @@ bool setupEssentials(int& argc, char** argv) { auto autoUpdater = DependencyManager::set(); auto pathUtils = DependencyManager::set(); auto actionFactory = DependencyManager::set(); + auto assetClient = DependencyManager::set(); + auto assetScriptingInterface = DependencyManager::set(); return true; } @@ -494,7 +498,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : // tell the NodeList instance who to tell the domain server we care about nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer - << NodeType::EntityServer); + << NodeType::EntityServer << NodeType::AssetServer); // connect to the packet sent signal of the _entityEditSender connect(&_entityEditSender, &EntityEditPacketSender::packetSent, this, &Application::packetSent); @@ -1738,6 +1742,7 @@ void Application::sendPingPackets() { case NodeType::AvatarMixer: case NodeType::AudioMixer: case NodeType::EntityServer: + case NodeType::AssetServer: return true; default: return false; @@ -3750,6 +3755,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri qScriptRegisterMetaType(scriptEngine, RayToOverlayIntersectionResultToScriptValue, RayToOverlayIntersectionResultFromScriptValue); + scriptEngine->registerGlobalObject("Assets", DependencyManager::get().data()); + QScriptValue windowValue = scriptEngine->registerGlobalObject("Window", DependencyManager::get().data()); scriptEngine->registerGetterSetter("location", LocationScriptingInterface::locationGetter, LocationScriptingInterface::locationSetter, windowValue); diff --git a/libraries/networking/src/Assignment.cpp b/libraries/networking/src/Assignment.cpp index 293d86475f..795b775d25 100644 --- a/libraries/networking/src/Assignment.cpp +++ b/libraries/networking/src/Assignment.cpp @@ -27,6 +27,8 @@ Assignment::Type Assignment::typeForNodeType(NodeType_t nodeType) { return Assignment::AgentType; case NodeType::EntityServer: return Assignment::EntityServerType; + case NodeType::AssetServer: + return Assignment::AssetServerType; default: return Assignment::AllTypes; } diff --git a/libraries/networking/src/Assignment.h b/libraries/networking/src/Assignment.h index 67f861f850..db20c6d06a 100644 --- a/libraries/networking/src/Assignment.h +++ b/libraries/networking/src/Assignment.h @@ -33,7 +33,8 @@ public: UNUSED_1 = 4, UNUSED_2 = 5, EntityServerType = 6, - AllTypes = 7 + AssetServerType = 7, + AllTypes = 8 }; enum Command { diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 4427b87158..ab3d4a1dbe 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -23,6 +23,7 @@ namespace NodeType { const NodeType_t Agent = 'I'; const NodeType_t AudioMixer = 'M'; const NodeType_t AvatarMixer = 'W'; + const NodeType_t AssetServer = 'A'; const NodeType_t Unassigned = 1; void init(); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 3f3f165e87..b133319a57 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -73,7 +73,11 @@ namespace PacketType { EntityQuery, EntityAdd, EntityErase, - EntityEdit + EntityEdit, + AssetGet, + AssetGetReply, + AssetUpload, + AssetUploadReply, }; }; From cf20701275cb858ed0903ac62ea0e2fee6fe56c7 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:14:25 -0700 Subject: [PATCH 06/77] Update soloNodeOfType to take NodeType_t --- libraries/networking/src/LimitedNodeList.cpp | 2 +- libraries/networking/src/LimitedNodeList.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index fdb3461c1f..2f3be63ba8 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -510,7 +510,7 @@ unsigned int LimitedNodeList::broadcastToNodes(std::unique_ptr packet, return n; } -SharedNodePointer LimitedNodeList::soloNodeOfType(char nodeType) { +SharedNodePointer LimitedNodeList::soloNodeOfType(NodeType_t nodeType) { return nodeMatchingPredicate([&](const SharedNodePointer& node){ return node->getType() == nodeType; }); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 33d490c960..c3980fd099 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -151,7 +151,7 @@ public: int updateNodeWithDataFromPacket(QSharedPointer packet, SharedNodePointer matchingNode); unsigned int broadcastToNodes(std::unique_ptr packet, const NodeSet& destinationNodeTypes); - SharedNodePointer soloNodeOfType(char nodeType); + SharedNodePointer soloNodeOfType(NodeType_t nodeType); void getPacketStats(float &packetsPerSecond, float &bytesPerSecond); void resetPacketStats(); From 260404f1fb30348b90e031b4a1d558d059fa55aa Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:23:57 -0700 Subject: [PATCH 07/77] Add AssetRequest --- libraries/networking/src/AssetRequest.cpp | 78 ++++++++++++++++++++++ libraries/networking/src/AssetRequest.h | 80 +++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 libraries/networking/src/AssetRequest.cpp create mode 100644 libraries/networking/src/AssetRequest.h diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp new file mode 100644 index 0000000000..d3863a5882 --- /dev/null +++ b/libraries/networking/src/AssetRequest.cpp @@ -0,0 +1,78 @@ +// +// AssetRequest.cpp +// +// Created by Ryan Huffman on 2015/07/24 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetRequest.h" + +#include + +#include + +#include "AssetClient.h" +#include "NodeList.h" + + +AssetRequest::AssetRequest(QObject* parent, QString hash) : + QObject(parent), + _hash(hash) +{ +} + +void AssetRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + //(&AssetRequest::start) + return; + } + + if (_state == NOT_STARTED) { + _state = WAITING_FOR_INFO; + + auto assetClient = DependencyManager::get(); + assetClient->getAssetInfo(_hash, [this](bool success, AssetInfo info) { + _info = info; + _data.resize(info.size); + const DataOffset CHUNK_SIZE = 1024; + + qDebug() << "Got size of " << _hash << " : " << info.size << " bytes"; + + // Round up + int numChunks = (info.size + CHUNK_SIZE - 1) / CHUNK_SIZE; + auto assetClient = DependencyManager::get(); + for (int i = 0; i < numChunks; ++i) { + ++_numPendingRequests; + auto start = i * CHUNK_SIZE; + auto end = std::min((i + 1) * CHUNK_SIZE, info.size); + assetClient->getAsset(_hash, start, end, [this, start, end](bool success, QByteArray data) { + Q_ASSERT(data.size() == (end - start)); + + if (success) { + _result = Success; + memcpy((_data.data() + start), data.constData(), end - start); + _totalReceived += data.size(); + emit progress(_totalReceived, _info.size); + } else { + _result = Error; + qDebug() << "Got error retrieving asset"; + } + + --_numPendingRequests; + if (_numPendingRequests == 0) { + _state = FINISHED; + emit finished(this); + } + }); + } + }); + } +} + +const QByteArray& AssetRequest::getData() { + return _data; +} diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h new file mode 100644 index 0000000000..402aacad09 --- /dev/null +++ b/libraries/networking/src/AssetRequest.h @@ -0,0 +1,80 @@ +// +// AssetRequest.h +// +// Created by Ryan Huffman on 2015/07/24 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetRequest_h +#define hifi_AssetRequest_h + +#include +#include +#include + +#include "AssetClient.h" + +#include "AssetUtils.h" + +// You should be able to get an asset from any thread, and handle the responses in a safe way +// on your own thread. Everything should happen on AssetClient's thread, the caller should +// receive events by connecting to signals on an object that lives on AssetClient's threads. + +// Receives parts of an asset and puts them together +// Emits signals: +// Progress +// Completion, success or error +// On finished, the AssetClient is effectively immutable and can be read from +// any thread safely +// +// Will often make multiple requests to the AssetClient to get data +class AssetRequest : public QObject { + Q_OBJECT +public: + enum State { + NOT_STARTED = 0, + WAITING_FOR_INFO, + WAITING_FOR_DATA, + FINISHED + }; + + enum Result { + Success = 0, + Timeout, + NotFound, + Error, + }; + + AssetRequest(QObject* parent, QString hash); + + Q_INVOKABLE void start(); + //AssetRequest* requestAsset(QString hash); + // Create AssetRequest + // Start request for hash + // Store messageID -> AssetRequest + // When complete: + // Update AssetRequest + // AssetRequest emits signal + + void receiveData(DataOffset start, DataOffset end, QByteArray data); + const QByteArray& getData(); + +signals: + void finished(AssetRequest*); + void progress(uint64_t totalReceived, uint64_t total); + +private: + State _state = NOT_STARTED; + Result _result; + AssetInfo _info; + uint64_t _totalReceived { 0 }; + QString _hash; + QByteArray _data; + int _numPendingRequests { 0 }; + // Timeout +}; + +#endif From 86c82a2b2da559068240a64551b47f0dde4021fe Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:24:11 -0700 Subject: [PATCH 08/77] Add generalized ResourceRequest --- libraries/networking/src/ResourceManager.cpp | 138 +++++++++++++++++++ libraries/networking/src/ResourceManager.h | 97 +++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 libraries/networking/src/ResourceManager.cpp create mode 100644 libraries/networking/src/ResourceManager.h diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp new file mode 100644 index 0000000000..7fdac3028a --- /dev/null +++ b/libraries/networking/src/ResourceManager.cpp @@ -0,0 +1,138 @@ +// +// ResourceManager.cpp +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ResourceManager.h" + +#include +#include +#include + +#include "AssetClient.h" +#include "AssetRequest.h" +#include "NetworkAccessManager.h" + +#include + +const QString URL_SCHEME_FILE = "file"; +const QString URL_SCHEME_HTTP = "http"; +const QString URL_SCHEME_HTTPS = "https"; +const QString URL_SCHEME_FTP = "ftp"; +const QString URL_SCHEME_ATP = "atp"; + +ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) : + QObject(parent), + _url(url) { +} + +void HTTPResourceRequest::doSend() { + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkRequest networkRequest = QNetworkRequest(_url); + networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + + if (!_cacheEnabled) { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + } + + QNetworkReply* reply = networkAccessManager.get(networkRequest); + QObject::connect(reply, &QNetworkReply::finished, [=]() { + if (_state != IN_PROGRESS) return; + + _state = FINISHED; + auto error = reply->error(); + qDebug() << "Loaded " << _url; + if (error == QNetworkReply::NoError) { + _data = reply->readAll(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else if (error == QNetworkReply::TimeoutError) { + _result = ResourceRequest::TIMEOUT; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + reply->deleteLater(); + }); +} + +void FileResourceRequest::doSend() { + QString filename = _url.toLocalFile(); + QFile file(filename); + _state = FINISHED; + if (file.open(QFile::ReadOnly)) { + _data = file.readAll(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } +} + +void ATPResourceRequest::doSend() { + // Make request to atp + auto assetClient = DependencyManager::get(); + auto hash = _url.path(); + + auto request = assetClient->create(hash); + + if (!request) { + return; + } + + QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { + if (_state != IN_PROGRESS) return; + _state = FINISHED; + if (true) { + _data = req->getData(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + }); + + request->start(); +} + +const int TIMEOUT_MS = 2000; +void ResourceRequest::send() { + connect(&_sendTimer, &QTimer::timeout, this, &ResourceRequest::timeout); + _sendTimer.setSingleShot(true); + _sendTimer.start(TIMEOUT_MS); + _state = IN_PROGRESS; + doSend(); +} + +void ResourceRequest::timeout() { + // TIMEOUT!! + if (_state == UNSENT) { + qDebug() << "TImed out loading " << _url; + _state = FINISHED; + _result = TIMEOUT; + emit finished(); + } +} + +ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const QUrl& url) { + auto scheme = url.scheme(); + if (scheme == URL_SCHEME_FILE) { + return new FileResourceRequest(parent, url); + } else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) { + return new HTTPResourceRequest(parent, url); + } else if (scheme == URL_SCHEME_ATP) { + return new ATPResourceRequest(parent, url); + } + + qDebug() << "Failed to load: " << url.url(); + + return nullptr; +} diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h new file mode 100644 index 0000000000..4050b5f390 --- /dev/null +++ b/libraries/networking/src/ResourceManager.h @@ -0,0 +1,97 @@ +// +// ResourceManager.h +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#ifndef hifi_ResourceManager_h +#define hifi_ResourceManager_h + +#include + +#include +#include +#include + +class ResourceRequest : public QObject { + Q_OBJECT +public: + ResourceRequest(QObject* parent, const QUrl& url); + + enum State { + UNSENT = 0, + IN_PROGRESS, + FINISHED + }; + + enum Result { + SUCCESS, + ERROR, + TIMEOUT, + NOT_FOUND + }; + + void send(); + QByteArray moveData() { return _data; } + State getState() const { return _state; } + Result getResult() const { return _result; } + QUrl getUrl() const { return _url; } + + void setCacheEnabled(bool value) { _cacheEnabled = value; } + +signals: + void progress(uint64_t bytesReceived, uint64_t bytesTotal); + void finished(); + +protected: + virtual void doSend() = 0; + + QUrl _url; + State _state = UNSENT; + Result _result; + QTimer _sendTimer; + QByteArray _data; + bool _cacheEnabled { true }; + +private slots: + void timeout(); +}; + +class HTTPResourceRequest : public ResourceRequest { + Q_OBJECT +public: + HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; +}; + +class FileResourceRequest : public ResourceRequest { + Q_OBJECT +public: + FileResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; +}; + +class ATPResourceRequest : public ResourceRequest { + Q_OBJECT +public: + ATPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; +}; + +class ResourceManager { +public: + static ResourceRequest* createResourceRequest(QObject* parent, const QUrl& url); +}; + +#endif From 5258474f98b64c0a2c8256ff6e29c7811d73deb2 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:24:34 -0700 Subject: [PATCH 09/77] Update AssetServer with GetInfo packet --- assignment-client/src/assets/AssetServer.cpp | 117 +++++++++++++++---- assignment-client/src/assets/AssetServer.h | 9 +- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 1a384271e0..6ac106191f 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -22,12 +22,11 @@ #include const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; -const int HASH_HEX_LENGTH = 32; -using MessageID = int; AssetServer::AssetServer(NLPacket& packet) : ThreadedAssignment(packet) { auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); + packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); } @@ -59,6 +58,17 @@ void AssetServer::run() { qDebug() << "\tCan't open file for reading: " << filename; continue; } + + // Read file + QFile file { fileInfo.absoluteFilePath() }; + file.open(QFile::ReadOnly); + QByteArray data = file.readAll(); + + auto hash = hashData(data); + + qDebug() << "\tMoving " << filename << " to " << hash; + + file.rename(_resourcesDirectory.absoluteFilePath(hash)); } } @@ -68,39 +78,97 @@ void AssetServer::run() { } } -void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { - QByteArray assetHash; +void AssetServer::writeError(NLPacket* packet, AssetServerError error) { + packet->writePrimitive(error); +} +void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode) { if (packet->getPayloadSize() < HASH_HEX_LENGTH) { qDebug() << "ERROR bad file request"; return; } + QByteArray assetHash; + MessageID messageID; + packet->readPrimitive(&messageID); assetHash = packet->read(HASH_HEX_LENGTH); - qDebug() << "Got a request for the file: " << assetHash; - // We need to reply... - QFile file { _resourcesDirectory.filePath(QString(assetHash)) }; + auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply); + + replyPacket->writePrimitive(messageID); + replyPacket->write(assetHash, HASH_HEX_LENGTH); + + QFileInfo fileInfo { _resourcesDirectory.filePath(QString(assetHash)) }; qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); - bool found = file.open(QIODevice::ReadOnly); - auto assetPacket = NLPacket::create(PacketType::AssetGetReply); - - assetPacket->write(assetHash, HASH_HEX_LENGTH); - assetPacket->writePrimitive(found); - - const int MAX_LENGTH = 1024; - - if (found) { - QByteArray data = file.read(MAX_LENGTH); - assetPacket->writePrimitive(data.size()); - assetPacket->write(data, data.size()); + if (fileInfo.exists() && fileInfo.isReadable()) { + replyPacket->writePrimitive(AssetServerError::NO_ERROR); + replyPacket->writePrimitive(fileInfo.size()); } else { - qDebug() << "File not found"; + qDebug() << "Asset not found: " << assetHash; + replyPacket->writePrimitive(AssetServerError::ASSET_NOT_FOUND); } auto nodeList = DependencyManager::get(); - nodeList->sendPacket(std::move(assetPacket), *senderNode); + nodeList->sendPacket(std::move(replyPacket), *senderNode); +} + +void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { + if (packet->getPayloadSize() < HASH_HEX_LENGTH) { + qDebug() << "ERROR bad file request"; + return; + } + + MessageID messageID; + packet->readPrimitive(&messageID); + + QByteArray assetHash; + assetHash = packet->read(HASH_HEX_LENGTH); + + DataOffset start; + packet->readPrimitive(&start); + + DataOffset end; + packet->readPrimitive(&end); + + qDebug() << "Received a request for the file: " << assetHash << " from " << start << " to " << end; + + // We need to reply... + auto replyPacket = NLPacket::create(PacketType::AssetGetReply); + + replyPacket->write(assetHash, HASH_HEX_LENGTH); + + replyPacket->writePrimitive(messageID); + + const int64_t MAX_LENGTH = 1024; + + if (end <= start) { + writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); + } else if (end - start > MAX_LENGTH) { + writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + QFile file { _resourcesDirectory.filePath(QString(assetHash)) }; + qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); + + if (file.open(QIODevice::ReadOnly)) { + if (file.size() < end) { + writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + auto size = end - start; + file.seek(start); + QByteArray data = file.read(size); + replyPacket->writePrimitive(AssetServerError::NO_ERROR); + replyPacket->writePrimitive(size); + replyPacket->write(data, size); + } + } else { + qDebug() << "Asset not found"; + writeError(replyPacket.get(), AssetServerError::ASSET_NOT_FOUND); + } + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *senderNode); } void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode) { @@ -124,7 +192,7 @@ void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodeP QByteArray data = packet->read(fileSize); - QString hash = QString(QCryptographicHash::hash(data, QCryptographicHash::Md5).toHex()); + QString hash = hashData(data); qDebug() << "Got data: (" << hash << ") " << data; @@ -147,3 +215,8 @@ void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodeP auto nodeList = DependencyManager::get(); nodeList->sendPacket(std::move(replyPacket), *senderNode); } + +QString AssetServer::hashData(const QByteArray& data) { + return QString(QCryptographicHash::hash(data, QCryptographicHash::Md5).toHex()); +} + diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index ef658fae4a..585ecaa259 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -15,25 +15,26 @@ #include +#include "AssetUtils.h" + class AssetServer : public ThreadedAssignment { Q_OBJECT public: AssetServer(NLPacket& packet); ~AssetServer(); - enum ServerResponse { - ASSET_NOT_FOUND = 0, - ASSET_UPLOADED = 1, - }; + static QString hashData(const QByteArray& data); public slots: void run(); private slots: + void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode); private: + static void writeError(NLPacket* packet, AssetServerError error); QDir _resourcesDirectory; }; From 4e1b52a0222e6dd85f7dc4b71e0187dfb37584a5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:25:07 -0700 Subject: [PATCH 10/77] Add thread for AssetClient --- interface/src/Application.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 137f3ffa83..ee785642b1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -428,6 +428,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : audioThread->start(); + + QThread* assetThread = new QThread(); + + assetThread->setObjectName("Asset Thread"); + auto assetClient = DependencyManager::get(); + + assetClient->moveToThread(assetThread); + + assetThread->start(); + + const DomainHandler& domainHandler = nodeList->getDomainHandler(); connect(&domainHandler, SIGNAL(hostnameChanged(const QString&)), SLOT(domainChanged(const QString&))); From fc9cd836cef234911821f3672a5f1ba25c40bd50 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:25:37 -0700 Subject: [PATCH 11/77] Update AssetScriptingInterface --- interface/src/AssetScriptingInterface.cpp | 30 +++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/interface/src/AssetScriptingInterface.cpp b/interface/src/AssetScriptingInterface.cpp index c03e9ee8c2..267f0cc25e 100644 --- a/interface/src/AssetScriptingInterface.cpp +++ b/interface/src/AssetScriptingInterface.cpp @@ -13,25 +13,41 @@ #include #include +#include #include #include #include #include -#include - -const int HASH_HEX_LENGTH = 32; +#include AssetScriptingInterface::AssetScriptingInterface() { } QScriptValue AssetScriptingInterface::getAsset(QString url, QScriptValue callback) { - bool success = AssetManager::getAsset(QUrl(url), [callback](AssetRequestUpdateType type, QByteArray data) mutable { - auto result = callback.engine()->newVariant(data); - QList arguments { type == AssetRequestUpdateType::COMPLETE, result }; + + auto assetClient = DependencyManager::get(); + // auto request = assetClient->requestAsset(url); + auto request = assetClient->create(url); + + if (!request) { + return false; + } + + connect(request, &AssetRequest::finished, [callback](AssetRequest* req) mutable { + auto result = callback.engine()->newVariant(req->getData()); + QList arguments { true, result }; callback.call(QScriptValue(), arguments); }); - return success; + + request->start(); + + // bool success = AssetManager::getAsset(QUrl(url), [callback](AssetRequestUpdateType type, QByteArray data) mutable { + // auto result = callback.engine()->newVariant(data); + // QList arguments { type == AssetRequestUpdateType::COMPLETE, result }; + // callback.call(QScriptValue(), arguments); + // }); + return true; } QScriptValue AssetScriptingInterface::uploadAsset(QString data, QString extension, QScriptValue callback) { From ccc0000fd0cee4ec5eb1feaaf26fe1de94c6e243 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:26:16 -0700 Subject: [PATCH 12/77] Update ResourceCache to use ResourceRequest --- libraries/networking/src/ResourceCache.cpp | 266 ++++++++++++--------- libraries/networking/src/ResourceCache.h | 17 +- 2 files changed, 157 insertions(+), 126 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index e127380630..2903e810f2 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -155,6 +155,7 @@ void ResourceCache::clearUnusedResource() { void ResourceCache::attemptRequest(Resource* resource) { auto sharedItems = DependencyManager::get(); if (_requestLimit <= 0) { + qDebug() << "REQUEST LIMIT REACHED, queueing: " << resource->getURL(); // wait until a slot becomes available sharedItems->_pendingRequests.append(resource); return; @@ -187,6 +188,7 @@ void ResourceCache::requestCompleted(Resource* resource) { i++; } if (highestIndex >= 0) { + qDebug() << "trying to attempt a pending request"; attemptRequest(sharedItems->_pendingRequests.takeAt(highestIndex)); } } @@ -196,24 +198,22 @@ int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT; Resource::Resource(const QUrl& url, bool delayLoad) : _url(url), - _request(url) { + _request(nullptr) { init(); - _request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - _request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); - // start loading immediately unless instructed otherwise if (!(_startedLoading || delayLoad)) { - attemptRequest(); + QTimer::singleShot(1, this, &Resource::attemptRequest); + //attemptRequest(); } } Resource::~Resource() { - if (_reply) { + if (_request) { ResourceCache::requestCompleted(this); - delete _reply; - _reply = nullptr; + _request->deleteLater(); + _request = nullptr; } } @@ -259,21 +259,20 @@ float Resource::getLoadPriority() { } void Resource::refresh() { - if (_reply && !(_loaded || _failedToLoad)) { + if (_request && !(_loaded || _failedToLoad)) { return; } - if (_reply) { + if (_request) { + _request->disconnect(this); + // _requestTimer->disconnect(this); + _request->deleteLater(); + _request = nullptr; + // _requestTimer->deleteLater(); + // _requestTimer = nullptr; ResourceCache::requestCompleted(this); - _reply->disconnect(this); - _replyTimer->disconnect(this); - _reply->deleteLater(); - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; } init(); - _request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); ensureLoading(); emit onRefresh(); } @@ -331,34 +330,34 @@ void Resource::reinsert() { _cache->_resources.insert(_url, _self); } -static const int REPLY_TIMEOUT_MS = 5000; -void Resource::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { - if (!_reply->isFinished()) { - _bytesReceived = bytesReceived; - _bytesTotal = bytesTotal; - _replyTimer->start(REPLY_TIMEOUT_MS); - return; - } - _reply->disconnect(this); - _replyTimer->disconnect(this); - QNetworkReply* reply = _reply; - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; - ResourceCache::requestCompleted(this); - - downloadFinished(reply); -} - -void Resource::handleReplyError() { - handleReplyError(_reply->error(), qDebug() << _reply->errorString()); -} - -void Resource::handleReplyTimeout() { - handleReplyError(QNetworkReply::TimeoutError, qDebug() << "Timed out loading" << _reply->url() << - "received" << _bytesReceived << "total" << _bytesTotal); -} - +// static const int REPLY_TIMEOUT_MS = 5000; +// void Resource::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { +// if (!_reply->isFinished()) { +// _bytesReceived = bytesReceived; +// _bytesTotal = bytesTotal; +// _replyTimer->start(REPLY_TIMEOUT_MS); +// return; +// } +// _reply->disconnect(this); +// _replyTimer->disconnect(this); +// QNetworkReply* reply = _reply; +// _reply = nullptr; +// _replyTimer->deleteLater(); +// _replyTimer = nullptr; +// ResourceCache::requestCompleted(this); +// +// // downloadFinished(reply); +// } +// +// void Resource::handleReplyError() { +// handleReplyError(_reply->error(), qDebug() << _reply->errorString()); +// } +// +// void Resource::handleReplyTimeout() { +// handleReplyError(QNetworkReply::TimeoutError, qDebug() << "Timed out loading" << _reply->url() << +// "received" << _bytesReceived << "total" << _bytesTotal); +// } +// void Resource::maybeRefresh() { if (Q_LIKELY(NetworkAccessManager::getInstance().cache())) { QNetworkReply* reply = qobject_cast(sender()); @@ -383,87 +382,116 @@ void Resource::maybeRefresh() { } } +// #include "AssetManager.h" void Resource::makeRequest() { - _reply = NetworkAccessManager::getInstance().get(_request); - - connect(_reply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleDownloadProgress(qint64,qint64))); - connect(_reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleReplyError())); - connect(_reply, SIGNAL(finished()), SLOT(handleReplyFinished())); - - if (_reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool()) { - // If the file as been updated since it was cached, refresh it - QNetworkRequest request(_request); - request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, false); - QNetworkReply* reply = NetworkAccessManager::getInstance().head(request); - connect(reply, &QNetworkReply::finished, this, &Resource::maybeRefresh); - } else { - if (Q_LIKELY(NetworkAccessManager::getInstance().cache())) { - QNetworkCacheMetaData metaData = NetworkAccessManager::getInstance().cache()->metaData(_url); - bool needUpdate = false; - if (metaData.expirationDate().isNull() || metaData.expirationDate() <= QDateTime::currentDateTime()) { - // If the expiration date is NULL or in the past, - // put one far enough away that it won't be an issue. - metaData.setExpirationDate(QDateTime::currentDateTime().addYears(100)); - needUpdate = true; - } - if (metaData.lastModified().isNull()) { - // If the lastModified date is NULL, set it to now. - metaData.setLastModified(QDateTime::currentDateTime()); - needUpdate = true; - } - if (needUpdate) { - NetworkAccessManager::getInstance().cache()->updateMetaData(metaData); - } - } + auto request = ResourceManager::createResourceRequest(this, _url); + _request = request; + if (!_request) { + qDebug() << "Failed to get request for " << _url; + return; } - _replyTimer = new QTimer(this); - connect(_replyTimer, SIGNAL(timeout()), SLOT(handleReplyTimeout())); - _replyTimer->setSingleShot(true); - _replyTimer->start(REPLY_TIMEOUT_MS); + qDebug() << "Starting request for: " << _url; + + connect(request, &ResourceRequest::finished, this, &Resource::handleReplyFinished); + // connect(_reply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleDownloadProgress(qint64,qint64))); + // connect(_reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleReplyError())); + // connect(_reply, SIGNAL(finished()), SLOT(handleReplyFinished())); + + // AssetManager::getAsset(_url, [this](AssetRequestUpdateType type, QByteArray data) { + // if (type == AssetRequestUpdateType::COMPLETE) { + // downloadFinished(data); + // } else { + // handleReplyError(QNetworkReply::TimeoutError, qDebug()); + // } + // }); + // + request->send(); + + // if (_reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool()) { + // // If the file as been updated since it was cached, refresh it + // QNetworkRequest request(_request); + // request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + // request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, false); + // QNetworkReply* reply = NetworkAccessManager::getInstance().head(request); + // connect(reply, &QNetworkReply::finished, this, &Resource::maybeRefresh); + // } else { + // if (Q_LIKELY(NetworkAccessManager::getInstance().cache())) { + // QNetworkCacheMetaData metaData = NetworkAccessManager::getInstance().cache()->metaData(_url); + // bool needUpdate = false; + // if (metaData.expirationDate().isNull() || metaData.expirationDate() <= QDateTime::currentDateTime()) { + // // If the expiration date is NULL or in the past, + // // put one far enough away that it won't be an issue. + // metaData.setExpirationDate(QDateTime::currentDateTime().addYears(100)); + // needUpdate = true; + // } + // if (metaData.lastModified().isNull()) { + // // If the lastModified date is NULL, set it to now. + // metaData.setLastModified(QDateTime::currentDateTime()); + // needUpdate = true; + // } + // if (needUpdate) { + // NetworkAccessManager::getInstance().cache()->updateMetaData(metaData); + // } + // } + // } + + // _replyTimer = new QTimer(this); + // connect(_replyTimer, SIGNAL(timeout()), SLOT(handleReplyTimeout())); + // _replyTimer->setSingleShot(true); + // _replyTimer->start(REPLY_TIMEOUT_MS); _bytesReceived = _bytesTotal = 0; } -void Resource::handleReplyError(QNetworkReply::NetworkError error, QDebug debug) { - _reply->disconnect(this); - _replyTimer->disconnect(this); - _reply->deleteLater(); - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; - ResourceCache::requestCompleted(this); - - // retry for certain types of failures - switch (error) { - case QNetworkReply::RemoteHostClosedError: - case QNetworkReply::TimeoutError: - case QNetworkReply::TemporaryNetworkFailureError: - case QNetworkReply::ProxyConnectionClosedError: - case QNetworkReply::ProxyTimeoutError: - case QNetworkReply::UnknownNetworkError: - case QNetworkReply::UnknownProxyError: - case QNetworkReply::UnknownContentError: - case QNetworkReply::ProtocolFailure: { - // retry with increasing delays - const int MAX_ATTEMPTS = 8; - const int BASE_DELAY_MS = 1000; - if (++_attempts < MAX_ATTEMPTS) { - QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); - debug << "-- retrying..."; - return; - } - // fall through to final failure - } - default: - finishedLoading(false); - break; - } -} - +// void Resource::handleReplyError(QNetworkReply::NetworkError error, QDebug debug) { +// _reply->disconnect(this); +// _replyTimer->disconnect(this); +// _reply->deleteLater(); +// _reply = nullptr; +// _replyTimer->deleteLater(); +// _replyTimer = nullptr; +// ResourceCache::requestCompleted(this); +// +// // retry for certain types of failures +// switch (error) { +// case QNetworkReply::RemoteHostClosedError: +// case QNetworkReply::TimeoutError: +// case QNetworkReply::TemporaryNetworkFailureError: +// case QNetworkReply::ProxyConnectionClosedError: +// case QNetworkReply::ProxyTimeoutError: +// case QNetworkReply::UnknownNetworkError: +// case QNetworkReply::UnknownProxyError: +// case QNetworkReply::UnknownContentError: +// case QNetworkReply::ProtocolFailure: { +// // retry with increasing delays +// const int MAX_ATTEMPTS = 8; +// const int BASE_DELAY_MS = 1000; +// if (++_attempts < MAX_ATTEMPTS) { +// QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); +// debug << "-- retrying..."; +// return; +// } +// // fall through to final failure +// } +// default: +// finishedLoading(false); +// break; +// } +// } +// void Resource::handleReplyFinished() { - qCDebug(networking) << "Got finished without download progress/error?" << _url; - handleDownloadProgress(0, 0); + // _request->disconnect(this); + // _request->deleteLater(); + // _request = nullptr; + Q_ASSERT(_request); + if (_request->getResult() == ResourceRequest::SUCCESS) { + _data = _request->moveData(); + _request->disconnect(this); + _request->deleteLater(); + _request = nullptr; + downloadFinished(_data); + } + ResourceCache::requestCompleted(this); } uint qHash(const QPointer& value, uint seed) { diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 93ddfe77be..59192449b8 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -26,6 +26,8 @@ #include +#include "ResourceManager.h" + class QNetworkReply; class QTimer; @@ -102,7 +104,7 @@ protected: void reserveUnusedResource(qint64 resourceSize); void clearUnusedResource(); - static void attemptRequest(Resource* resource); + Q_INVOKABLE static void attemptRequest(Resource* resource); static void requestCompleted(Resource* resource); private: @@ -188,7 +190,7 @@ protected: virtual void init(); /// Called when the download has finished. The recipient should delete the reply when done with it. - virtual void downloadFinished(QNetworkReply* reply) = 0; + virtual void downloadFinished(const QByteArray& data) = 0; /// Should be called by subclasses when all the loading that will be done has been done. Q_INVOKABLE void finishedLoading(bool success); @@ -197,19 +199,21 @@ protected: virtual void reinsert(); QUrl _url; - QNetworkRequest _request; + ResourceRequest* _request = nullptr; + //QNetworkRequest _request; bool _startedLoading = false; bool _failedToLoad = false; bool _loaded = false; QHash, float> _loadPriorities; QWeakPointer _self; QPointer _cache; + QByteArray _data; private slots: - void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); - void handleReplyError(); + // void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); + // void handleReplyError(); void handleReplyFinished(); - void handleReplyTimeout(); + // void handleReplyTimeout(); private: void setLRUKey(int lruKey) { _lruKey = lruKey; } @@ -221,7 +225,6 @@ private: friend class ResourceCache; int _lruKey = 0; - QNetworkReply* _reply = nullptr; QTimer* _replyTimer = nullptr; qint64 _bytesReceived = 0; qint64 _bytesTotal = 0; From 3a2ef35a3b76817ca5ccda49b23df995e95a97aa Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:26:51 -0700 Subject: [PATCH 13/77] Update Geometry and Texture caches to use updated ResourceCache --- libraries/render-utils/src/GeometryCache.cpp | 44 +++++++++++--------- libraries/render-utils/src/GeometryCache.h | 2 +- libraries/render-utils/src/TextureCache.cpp | 26 ++++-------- libraries/render-utils/src/TextureCache.h | 2 +- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 6c03d57de3..2e76b97551 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -2030,7 +2030,7 @@ class GeometryReader : public QRunnable { public: GeometryReader(const QWeakPointer& geometry, const QUrl& url, - QNetworkReply* reply, const QVariantHash& mapping); + QByteArray data, const QVariantHash& mapping); virtual void run(); @@ -2038,28 +2038,25 @@ private: QWeakPointer _geometry; QUrl _url; - QNetworkReply* _reply; + QByteArray _data; QVariantHash _mapping; }; GeometryReader::GeometryReader(const QWeakPointer& geometry, const QUrl& url, - QNetworkReply* reply, const QVariantHash& mapping) : + QByteArray data, const QVariantHash& mapping) : _geometry(geometry), _url(url), - _reply(reply), + _data(data), _mapping(mapping) { } void GeometryReader::run() { QSharedPointer geometry = _geometry.toStrongRef(); if (geometry.isNull()) { - _reply->deleteLater(); + // _reply->deleteLater(); return; } try { - if (!_reply) { - throw QString("Reply is NULL ?!"); - } QString urlname = _url.path().toLower(); bool urlValid = true; urlValid &= !urlname.isEmpty(); @@ -2082,9 +2079,11 @@ void GeometryReader::run() { } else if (_url.path().toLower().endsWith("palaceoforinthilian4.fbx")) { lightmapLevel = 3.5f; } - fbxgeo = readFBX(_reply, _mapping, grabLightmaps, lightmapLevel); + fbxgeo = readFBX(_data, _mapping, grabLightmaps, lightmapLevel); } else if (_url.path().toLower().endsWith(".obj")) { - fbxgeo = OBJReader().readOBJ(_reply, _mapping, &_url); + auto d = QByteArray(_data); + QBuffer buffer { &d }; + fbxgeo = OBJReader().readOBJ(&buffer, _mapping, &_url); } QMetaObject::invokeMethod(geometry.data(), "setGeometry", Q_ARG(const FBXGeometry&, fbxgeo)); } else { @@ -2095,7 +2094,6 @@ void GeometryReader::run() { qCDebug(renderutils) << "Error reading " << _url << ": " << error; QMetaObject::invokeMethod(geometry.data(), "finishedLoading", Q_ARG(bool, false)); } - _reply->deleteLater(); } void NetworkGeometry::init() { @@ -2104,16 +2102,15 @@ void NetworkGeometry::init() { _meshes.clear(); _lods.clear(); _pendingTextureChanges.clear(); - _request.setUrl(_url); + //_request.setUrl(_url); Resource::init(); } -void NetworkGeometry::downloadFinished(QNetworkReply* reply) { - QUrl url = reply->url(); +void NetworkGeometry::downloadFinished(const QByteArray& data) { + QUrl url = getURL(); if (url.path().toLower().endsWith(".fst")) { // it's a mapping file; parse it and get the mesh filename - _mapping = FSTReader::readMapping(reply->readAll()); - reply->deleteLater(); + _mapping = FSTReader::readMapping(data); QString filename = _mapping.value("filename").toString(); if (filename.isNull()) { qCDebug(renderutils) << "Mapping file " << url << " has no filename."; @@ -2135,19 +2132,28 @@ void NetworkGeometry::downloadFinished(QNetworkReply* reply) { geometry->setLODParent(_lodParent); _lods.insert(it.value().toFloat(), geometry); } - _request.setUrl(url.resolved(filename)); - + // make the request immediately only if we have no LODs to switch between + // TODO reimplement using ResourceRequest + _url = url.resolved(filename); _startedLoading = false; if (_lods.isEmpty()) { attemptRequest(); } + + // _request.setUrl(url.resolved(filename)); + + // make the request immediately only if we have no LODs to switch between + // _startedLoading = false; + // if (_lods.isEmpty()) { + // attemptRequest(); + // } } return; } // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new GeometryReader(_self, url, reply, _mapping)); + QThreadPool::globalInstance()->start(new GeometryReader(_self, url, data, _mapping)); } void NetworkGeometry::reinsert() { diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 147b4f8093..31bdcd16bf 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -380,7 +380,7 @@ public: protected: virtual void init(); - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& reply) override; virtual void reinsert(); Q_INVOKABLE void setGeometry(const FBXGeometry& geometry); diff --git a/libraries/render-utils/src/TextureCache.cpp b/libraries/render-utils/src/TextureCache.cpp index f86d15079d..0694bf45d4 100644 --- a/libraries/render-utils/src/TextureCache.cpp +++ b/libraries/render-utils/src/TextureCache.cpp @@ -206,8 +206,7 @@ NetworkTexture::NetworkTexture(const QUrl& url, TextureType type, const QByteArr class ImageReader : public QRunnable { public: - ImageReader(const QWeakPointer& texture, TextureType type, QNetworkReply* reply, const QUrl& url = QUrl(), - const QByteArray& content = QByteArray()); + ImageReader(const QWeakPointer& texture, TextureType type, const QByteArray& data, const QUrl& url = QUrl()); virtual void run(); @@ -215,27 +214,25 @@ private: QWeakPointer _texture; TextureType _type; - QNetworkReply* _reply; QUrl _url; QByteArray _content; }; -void NetworkTexture::downloadFinished(QNetworkReply* reply) { +void NetworkTexture::downloadFinished(const QByteArray& data) { // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new ImageReader(_self, _type, reply)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _type, data, _url)); } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, _type, NULL, _url, content)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _type, content)); } -ImageReader::ImageReader(const QWeakPointer& texture, TextureType type, QNetworkReply* reply, - const QUrl& url, const QByteArray& content) : +ImageReader::ImageReader(const QWeakPointer& texture, TextureType type, const QByteArray& data, + const QUrl& url) : _texture(texture), _type(type), - _reply(reply), _url(url), - _content(content) { + _content(data) { } std::once_flag onceListSupportedFormatsflag; @@ -288,16 +285,9 @@ public: void ImageReader::run() { QSharedPointer texture = _texture.toStrongRef(); if (texture.isNull()) { - if (_reply) { - _reply->deleteLater(); - } + qDebug() << "TEXTURE IS NULL"; return; } - if (_reply) { - _url = _reply->url(); - _content = _reply->readAll(); - _reply->deleteLater(); - } listSupportedImageFormats(); diff --git a/libraries/render-utils/src/TextureCache.h b/libraries/render-utils/src/TextureCache.h index 8f60382e9b..c5e8a756a0 100644 --- a/libraries/render-utils/src/TextureCache.h +++ b/libraries/render-utils/src/TextureCache.h @@ -119,7 +119,7 @@ public: TextureType getType() const { return _type; } protected: - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; Q_INVOKABLE void loadContent(const QByteArray& content); // FIXME: This void* should be a gpu::Texture* but i cannot get it to work for now, moving on... From 3843e642e56327f54b5d2c3201b8bc9190c3aa2d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:27:17 -0700 Subject: [PATCH 14/77] Move shared Asset code to AssetUtils.h --- libraries/networking/src/AssetUtils.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 libraries/networking/src/AssetUtils.h diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h new file mode 100644 index 0000000000..61def81ea8 --- /dev/null +++ b/libraries/networking/src/AssetUtils.h @@ -0,0 +1,25 @@ +// +// AssetUtils.h +// +// Created by Ryan Huffman on 2015/07/30 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetUtils_h +#define hifi_AssetUtils_h + +using MessageID = uint32_t; +using DataOffset = int64_t; + +const int HASH_HEX_LENGTH = 32; + +enum AssetServerError : uint8_t { + NO_ERROR = 0, + ASSET_NOT_FOUND, + INVALID_BYTE_RANGE, +}; + +#endif From ae93d74d8b4e06e862abafed8114a99bf8550c40 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:27:34 -0700 Subject: [PATCH 15/77] Update Sound to use new ResourceCache --- libraries/audio/src/Sound.cpp | 17 +++++++++-------- libraries/audio/src/Sound.h | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 7dc6010f8f..5ff0f55f7f 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -56,16 +56,17 @@ Sound::Sound(const QUrl& url, bool isStereo) : } -void Sound::downloadFinished(QNetworkReply* reply) { +void Sound::downloadFinished(const QByteArray& data) { // replace our byte array with the downloaded data - QByteArray rawAudioByteArray = reply->readAll(); - QString fileName = reply->url().fileName(); + QByteArray rawAudioByteArray = QByteArray(data); + QString fileName = getURL().fileName(); const QString WAV_EXTENSION = ".wav"; - if (reply->hasRawHeader("Content-Type") || fileName.endsWith(WAV_EXTENSION)) { + if (fileName.endsWith(WAV_EXTENSION)) { - QByteArray headerContentType = reply->rawHeader("Content-Type"); + QString headerContentType = "audio/x-wav"; + //QByteArray headerContentType = reply->rawHeader("Content-Type"); // WAV audio file encountered if (headerContentType == "audio/x-wav" @@ -80,9 +81,9 @@ void Sound::downloadFinished(QNetworkReply* reply) { } else { // check if this was a stereo raw file // since it's raw the only way for us to know that is if the file was called .stereo.raw - if (reply->url().fileName().toLower().endsWith("stereo.raw")) { + if (fileName.toLower().endsWith("stereo.raw")) { _isStereo = true; - qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << reply->url() << "as stereo audio file."; + qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << getURL() << "as stereo audio file."; } // Process as RAW file @@ -94,7 +95,7 @@ void Sound::downloadFinished(QNetworkReply* reply) { } _isReady = true; - reply->deleteLater(); + _request->deleteLater(); } void Sound::downSample(const QByteArray& rawAudioByteArray) { diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 9aa92feea1..842c395a7d 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -39,7 +39,7 @@ private: void downSample(const QByteArray& rawAudioByteArray); void interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; }; typedef QSharedPointer SharedSoundPointer; From 2d938f2d0a98f1f47cf9386470330337908b134e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:27:48 -0700 Subject: [PATCH 16/77] Update AnimationCache to use new ResourceCache --- libraries/animation/src/AnimationCache.cpp | 15 +++++++-------- libraries/animation/src/AnimationCache.h | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index 99224f7dce..5ac89f5a7e 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -43,28 +43,27 @@ Animation::Animation(const QUrl& url) : Resource(url) {} class AnimationReader : public QRunnable { public: - AnimationReader(const QWeakPointer& animation, QNetworkReply* reply); + AnimationReader(const QWeakPointer& animation, QByteArray data); virtual void run(); private: QWeakPointer _animation; - QNetworkReply* _reply; + QByteArray _data; }; -AnimationReader::AnimationReader(const QWeakPointer& animation, QNetworkReply* reply) : +AnimationReader::AnimationReader(const QWeakPointer& animation, QByteArray data) : _animation(animation), - _reply(reply) { + _data(data) { } void AnimationReader::run() { QSharedPointer animation = _animation.toStrongRef(); if (!animation.isNull()) { QMetaObject::invokeMethod(animation.data(), "setGeometry", - Q_ARG(const FBXGeometry&, readFBX(_reply->readAll(), QVariantHash()))); + Q_ARG(const FBXGeometry&, readFBX(QByteArray(_data), QVariantHash()))); } - _reply->deleteLater(); } QStringList Animation::getJointNames() const { @@ -96,9 +95,9 @@ void Animation::setGeometry(const FBXGeometry& geometry) { finishedLoading(true); } -void Animation::downloadFinished(QNetworkReply* reply) { +void Animation::downloadFinished(const QByteArray& data) { // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new AnimationReader(_self, reply)); + QThreadPool::globalInstance()->start(new AnimationReader(_self, data)); } diff --git a/libraries/animation/src/AnimationCache.h b/libraries/animation/src/AnimationCache.h index c90c4c9225..2509585fb0 100644 --- a/libraries/animation/src/AnimationCache.h +++ b/libraries/animation/src/AnimationCache.h @@ -62,7 +62,7 @@ protected: Q_INVOKABLE void setGeometry(const FBXGeometry& geometry); - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; private: From 6b839488492f05b8ad0f1ec366c3afb421168bc4 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:28:04 -0700 Subject: [PATCH 17/77] Add new packet headers --- libraries/networking/src/udt/PacketHeaders.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index b133319a57..9e29498706 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -78,6 +78,8 @@ namespace PacketType { AssetGetReply, AssetUpload, AssetUploadReply, + AssetGetInfo, + AssetGetInfoReply, }; }; From acff31ee2dcd2101f41e38bfc2f680d4d2528a56 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:28:21 -0700 Subject: [PATCH 18/77] Update script loaders to use ResourceRequest --- libraries/script-engine/src/BatchLoader.cpp | 93 ++++++++++++--------- libraries/script-engine/src/ScriptCache.cpp | 31 +++---- 2 files changed, 65 insertions(+), 59 deletions(-) diff --git a/libraries/script-engine/src/BatchLoader.cpp b/libraries/script-engine/src/BatchLoader.cpp index db0743808f..99916310f4 100644 --- a/libraries/script-engine/src/BatchLoader.cpp +++ b/libraries/script-engine/src/BatchLoader.cpp @@ -17,9 +17,7 @@ #include "BatchLoader.h" #include #include - - - +#include "ResourceManager.h" BatchLoader::BatchLoader(const QList& urls) : QObject(), @@ -27,6 +25,7 @@ BatchLoader::BatchLoader(const QList& urls) _finished(false), _urls(urls.toSet()), _data() { + qRegisterMetaType>("QMap"); } void BatchLoader::start() { @@ -35,45 +34,63 @@ void BatchLoader::start() { } _started = true; - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + // QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); for (QUrl url : _urls) { - if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { - QNetworkRequest request = QNetworkRequest(url); - request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - QNetworkReply* reply = networkAccessManager.get(request); - - qCDebug(scriptengine) << "Downloading file at" << url; - - connect(reply, &QNetworkReply::finished, [=]() { - if (reply->error()) { - _data.insert(url, QString()); - } else { - _data.insert(url, reply->readAll()); - } - reply->deleteLater(); - checkFinished(); - }); - - // If we end up being destroyed before the reply finishes, clean it up - connect(this, &QObject::destroyed, reply, &QObject::deleteLater); - - } else { -#ifdef _WIN32 - QString fileName = url.toString(); -#else - QString fileName = url.toLocalFile(); -#endif - - qCDebug(scriptengine) << "Reading file at " << fileName; - - QFile scriptFile(fileName); - if (scriptFile.open(QFile::ReadOnly | QFile::Text)) { - QTextStream in(&scriptFile); - _data.insert(url, in.readAll()); + auto request = ResourceManager::createResourceRequest(this, url); + if (!request) { + continue; + } + connect(request, &ResourceRequest::finished, [=]() { + if (request->getResult() == ResourceRequest::SUCCESS) { + _data.insert(url, request->moveData()); } else { _data.insert(url, QString()); } - } + request->deleteLater(); + checkFinished(); + }); + + qCDebug(scriptengine) << "Loading script at " << url; + + request->send(); + +// if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { +// QNetworkRequest request = QNetworkRequest(url); +// request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); +// QNetworkReply* reply = networkAccessManager.get(request); +// +// qCDebug(scriptengine) << "Downloading file at" << url; +// +// connect(reply, &QNetworkReply::finished, [=]() { +// if (reply->error()) { +// _data.insert(url, QString()); +// } else { +// _data.insert(url, reply->readAll()); +// } +// reply->deleteLater(); +// checkFinished(); +// }); +// +// // If we end up being destroyed before the reply finishes, clean it up +// connect(this, &QObject::destroyed, reply, &QObject::deleteLater); +// +// } else { +// #ifdef _WIN32 +// QString fileName = url.toString(); +// #else +// QString fileName = url.toLocalFile(); +// #endif +// +// qCDebug(scriptengine) << "Reading file at " << fileName; +// +// QFile scriptFile(fileName); +// if (scriptFile.open(QFile::ReadOnly | QFile::Text)) { +// QTextStream in(&scriptFile); +// _data.insert(url, in.readAll()); +// } else { +// _data.insert(url, QString()); +// } +// } } checkFinished(); } diff --git a/libraries/script-engine/src/ScriptCache.cpp b/libraries/script-engine/src/ScriptCache.cpp index 2047442ce6..87cdae0026 100644 --- a/libraries/script-engine/src/ScriptCache.cpp +++ b/libraries/script-engine/src/ScriptCache.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include "ScriptCache.h" @@ -44,18 +43,10 @@ QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& is if (alreadyWaiting) { qCDebug(scriptengine) << "Already downloading script at:" << url.toString(); } else { - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest networkRequest = QNetworkRequest(url); - networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - if (reload) { - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - qCDebug(scriptengine) << "Redownloading script at:" << url.toString(); - } else { - qCDebug(scriptengine) << "Downloading script at:" << url.toString(); - } - - QNetworkReply* reply = networkAccessManager.get(networkRequest); - connect(reply, &QNetworkReply::finished, this, &ScriptCache::scriptDownloaded); + auto request = ResourceManager::createResourceRequest(this, url); + request->setCacheEnabled(!reload); + connect(request, &ResourceRequest::finished, this, &ScriptCache::scriptDownloaded); + request->send(); } } return scriptContents; @@ -69,27 +60,25 @@ void ScriptCache::deleteScript(const QUrl& url) { } void ScriptCache::scriptDownloaded() { - QNetworkReply* reply = qobject_cast(sender()); - QUrl url = reply->url(); + ResourceRequest* req = qobject_cast(sender()); + QUrl url = req->getUrl(); QList scriptUsers = _scriptUsers.values(url); _scriptUsers.remove(url); - if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) { - _scriptCache[url] = reply->readAll(); + if (req->getResult() == ResourceRequest::SUCCESS) { + _scriptCache[url] = req->moveData(); qCDebug(scriptengine) << "Done downloading script at:" << url.toString(); foreach(ScriptUser* user, scriptUsers) { user->scriptContentsAvailable(url, _scriptCache[url]); } } else { - qCWarning(scriptengine) << "Error loading script from URL " << reply->url().toString() - << "- HTTP status code is" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() - << "and error from QNetworkReply is" << reply->errorString(); + qCWarning(scriptengine) << "Error loading script from URL " << url; foreach(ScriptUser* user, scriptUsers) { user->errorInLoadingScript(url); } } - reply->deleteLater(); + req->deleteLater(); } From a6665612fcbd359f746cdb1ce60a7b9ec69b4f61 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 3 Aug 2015 16:30:22 -0700 Subject: [PATCH 19/77] Add GetInfo packets to AssetClient --- libraries/networking/src/AssetClient.cpp | 110 +++++++++++++++++++---- libraries/networking/src/AssetClient.h | 28 +++--- 2 files changed, 112 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index e8131a40f2..cf8231abeb 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -10,20 +10,52 @@ #include "AssetClient.h" +#include + +#include "AssetRequest.h" #include "NodeList.h" #include "PacketReceiver.h" -const int HASH_HEX_LENGTH = 32; MessageID AssetClient::_currentID = 0; + AssetClient::AssetClient() { auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply"); packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); } -bool AssetClient::getAsset(QString hash, ReceivedAssetCallback callback) { +AssetRequest* AssetClient::create(QString hash) { + if (QThread::currentThread() != thread()) { + AssetRequest* req; + QMetaObject::invokeMethod(this, "create", + Qt::BlockingQueuedConnection, + Q_RETURN_ARG(AssetRequest*, req), + Q_ARG(QString, hash)); + return req; + } + if (hash.length() != 32) { + qDebug() << "Invalid hash size"; + return nullptr; + } + + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto assetClient = DependencyManager::get(); + auto request = new AssetRequest(assetClient.data(), hash); + + return request; + } + + return nullptr; +} + +bool AssetClient::getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { + if (hash.length() != HASH_HEX_LENGTH) { qDebug() << "Invalid hash size"; return false; } @@ -33,10 +65,16 @@ bool AssetClient::getAsset(QString hash, ReceivedAssetCallback callback) { if (assetServer) { auto packet = NLPacket::create(PacketType::AssetGet); + + auto messageID = ++_currentID; + packet->writePrimitive(messageID); packet->write(hash.toLatin1().constData(), 32); + packet->writePrimitive(start); + packet->writePrimitive(end); + nodeList->sendPacket(std::move(packet), *assetServer); - _pendingRequests[hash] = callback; + _pendingRequests[messageID] = callback; return true; } @@ -44,28 +82,70 @@ bool AssetClient::getAsset(QString hash, ReceivedAssetCallback callback) { return false; } +bool AssetClient::getAssetInfo(QString hash, GetInfoCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packet = NLPacket::create(PacketType::AssetGetInfo); + + auto messageID = ++_currentID; + packet->writePrimitive(messageID); + packet->write(hash.toLatin1().constData(), 32); + + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingInfoRequests[messageID] = callback; + + return true; + } + + return false; +} + +void AssetClient::handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + packet->readPrimitive(&messageID); + auto assetHash = QString(packet->read(HASH_HEX_LENGTH)); + + AssetServerError error; + packet->readPrimitive(&error); + + AssetInfo info; + + if (!error) { + packet->readPrimitive(&info.size); + } + + if (_pendingInfoRequests.contains(messageID)) { + auto callback = _pendingInfoRequests.take(messageID); + callback(error != NO_ERROR, info); + } +} + void AssetClient::handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode) { auto assetHash = packet->read(HASH_HEX_LENGTH); qDebug() << "Got reply for asset: " << assetHash; - bool success; - packet->readPrimitive(&success); + MessageID messageID; + packet->readPrimitive(&messageID); + + AssetServerError error; + packet->readPrimitive(&error); QByteArray data; - if (success) { - int length; + if (!error) { + DataOffset length; packet->readPrimitive(&length); - char assetData[length]; - packet->read(assetData, length); - - data = QByteArray(assetData, length); + data = packet->read(length); + qDebug() << "Got data: " << length << ", " << data.toHex(); } else { - qDebug() << "Failure getting asset"; + qDebug() << "Failure getting asset: " << error; } - if (_pendingRequests.contains(assetHash)) { - auto callback = _pendingRequests.take(assetHash); - callback(success, data); + if (_pendingRequests.contains(messageID)) { + auto callback = _pendingRequests.take(messageID); + callback(!error, data); } } diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 87da601563..7ad9c1a638 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -13,39 +13,45 @@ #define hifi_AssetClient_h #include -#include #include +#include "AssetUtils.h" #include "LimitedNodeList.h" #include "NLPacket.h" +class AssetRequest; + +struct AssetInfo { + QString hash; + int64_t size; +}; + using ReceivedAssetCallback = std::function; +using GetInfoCallback = std::function; using UploadResultCallback = std::function; -using MessageID = int; class AssetClient : public QObject, public Dependency { Q_OBJECT public: AssetClient(); - enum RequestResult { - SUCCESS = 0, - FAILURE, - TIMEOUT - }; - - bool getAsset(QString hash, ReceivedAssetCallback callback); + Q_INVOKABLE AssetRequest* create(QString hash); + bool getAssetInfo(QString hash, GetInfoCallback callback); + bool getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback); bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); + bool abortDataRequest(MessageID messageID); private slots: + void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode); private: static MessageID _currentID; - QMap _pendingRequests; - QMap _pendingUploads; + QHash _pendingRequests; + QHash _pendingInfoRequests; + QHash _pendingUploads; }; #endif From a168a1058f8b761c54cb3b445ed71364f1a79854 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:09:46 -0700 Subject: [PATCH 20/77] Update filename regex in AssetServer to allow extensions --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6ac106191f..7dd31eba5f 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -49,7 +49,7 @@ void AssetServer::run() { // Scan for new files qDebug() << "Looking for new files in asset directory"; auto files = _resourcesDirectory.entryInfoList(QDir::Files); - QRegExp filenameRegex { "^[a-f0-9]{32}$" }; + QRegExp filenameRegex { "^[a-f0-9]{32}(\\..+)?$" }; for (auto fileInfo : files) { auto filename = fileInfo.fileName(); if (!filenameRegex.exactMatch(filename)) { From 51d4cf84da9cc92518c56a4ccb0266c3b2f27054 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:10:47 -0700 Subject: [PATCH 21/77] Fix AnimationReader to work with new ResourceCache updates --- libraries/animation/src/AnimationCache.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index 9fc650ecbe..7563e44178 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -43,7 +43,7 @@ Animation::Animation(const QUrl& url) : Resource(url) {} class AnimationReader : public QRunnable { public: - AnimationReader(const QWeakPointer& animation, QByteArray data); + AnimationReader(const QWeakPointer& animation, QByteArray data, QUrl url); virtual void run(); @@ -64,7 +64,7 @@ void AnimationReader::run() { QSharedPointer animation = _animation.toStrongRef(); if (!animation.isNull()) { QMetaObject::invokeMethod(animation.data(), "setGeometry", - Q_ARG(const FBXGeometry&, readFBX(QByteArray(_data), QVariantHash(), _url))); + Q_ARG(const FBXGeometry&, readFBX(QByteArray(_data), QVariantHash(), _url.path()))); } } @@ -99,7 +99,7 @@ void Animation::setGeometry(const FBXGeometry& geometry) { void Animation::downloadFinished(const QByteArray& data) { // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new AnimationReader(_self, data)); + QThreadPool::globalInstance()->start(new AnimationReader(_self, data, _url)); } From 4827d326f4f7d001fc566c5b866051c4f8281b64 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:11:47 -0700 Subject: [PATCH 22/77] Update ResourceCache to allow multiple downloads on the same resource --- libraries/audio/src/Sound.cpp | 1 - libraries/networking/src/ResourceCache.cpp | 171 +++++++++++---------- libraries/networking/src/ResourceCache.h | 9 +- 3 files changed, 99 insertions(+), 82 deletions(-) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 5ff0f55f7f..2457bda74a 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -95,7 +95,6 @@ void Sound::downloadFinished(const QByteArray& data) { } _isReady = true; - _request->deleteLater(); } void Sound::downSample(const QByteArray& rawAudioByteArray) { diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 2903e810f2..018f9b1c36 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -154,12 +154,13 @@ void ResourceCache::clearUnusedResource() { void ResourceCache::attemptRequest(Resource* resource) { auto sharedItems = DependencyManager::get(); - if (_requestLimit <= 0) { - qDebug() << "REQUEST LIMIT REACHED, queueing: " << resource->getURL(); - // wait until a slot becomes available - sharedItems->_pendingRequests.append(resource); - return; - } + // if (_requestLimit <= 0) { + // qDebug() << "REQUEST LIMIT REACHED (" << _requestLimit << "), queueing: " << resource->getURL(); + // // wait until a slot becomes available + // sharedItems->_pendingRequests.append(resource); + // return; + // } + qDebug() << "-- Decreasing limit for : " << resource->getURL(); _requestLimit--; sharedItems->_loadingRequests.append(resource); resource->makeRequest(); @@ -169,6 +170,7 @@ void ResourceCache::requestCompleted(Resource* resource) { auto sharedItems = DependencyManager::get(); sharedItems->_loadingRequests.removeOne(resource); + qDebug() << "++ Increasing limit after finished: " << resource->getURL(); _requestLimit++; // look for the highest priority pending request @@ -198,13 +200,14 @@ int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT; Resource::Resource(const QUrl& url, bool delayLoad) : _url(url), + _activeUrl(url), _request(nullptr) { init(); // start loading immediately unless instructed otherwise if (!(_startedLoading || delayLoad)) { - QTimer::singleShot(1, this, &Resource::attemptRequest); + // QTimer::singleShot(1, this, &Resource::attemptRequest); //attemptRequest(); } } @@ -264,11 +267,8 @@ void Resource::refresh() { } if (_request) { _request->disconnect(this); - // _requestTimer->disconnect(this); _request->deleteLater(); _request = nullptr; - // _requestTimer->deleteLater(); - // _requestTimer = nullptr; ResourceCache::requestCompleted(this); } @@ -302,6 +302,7 @@ void Resource::init() { _failedToLoad = false; _loaded = false; _attempts = 0; + _activeUrl = _url; if (_url.isEmpty()) { _startedLoading = _loaded = true; @@ -317,6 +318,7 @@ void Resource::attemptRequest() { } void Resource::finishedLoading(bool success) { + qDebug() << "Finished loading: " << _url; if (success) { _loaded = true; emit loaded(); @@ -349,15 +351,6 @@ void Resource::reinsert() { // // downloadFinished(reply); // } // -// void Resource::handleReplyError() { -// handleReplyError(_reply->error(), qDebug() << _reply->errorString()); -// } -// -// void Resource::handleReplyTimeout() { -// handleReplyError(QNetworkReply::TimeoutError, qDebug() << "Timed out loading" << _reply->url() << -// "received" << _bytesReceived << "total" << _bytesTotal); -// } -// void Resource::maybeRefresh() { if (Q_LIKELY(NetworkAccessManager::getInstance().cache())) { QNetworkReply* reply = qobject_cast(sender()); @@ -382,31 +375,30 @@ void Resource::maybeRefresh() { } } -// #include "AssetManager.h" void Resource::makeRequest() { - auto request = ResourceManager::createResourceRequest(this, _url); - _request = request; + QString u = _activeUrl.path(); + QString uu = _url.path(); + Q_ASSERT(!_request); + + _request = ResourceManager::createResourceRequest(this, _activeUrl); + if (!_request) { qDebug() << "Failed to get request for " << _url; + ResourceCache::requestCompleted(this); + finishedLoading(false); return; } qDebug() << "Starting request for: " << _url; - connect(request, &ResourceRequest::finished, this, &Resource::handleReplyFinished); + connect(_request, &ResourceRequest::finished, this, &Resource::handleReplyFinished); // connect(_reply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleDownloadProgress(qint64,qint64))); // connect(_reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleReplyError())); // connect(_reply, SIGNAL(finished()), SLOT(handleReplyFinished())); - // AssetManager::getAsset(_url, [this](AssetRequestUpdateType type, QByteArray data) { - // if (type == AssetRequestUpdateType::COMPLETE) { - // downloadFinished(data); - // } else { - // handleReplyError(QNetworkReply::TimeoutError, qDebug()); - // } - // }); - // - request->send(); + _bytesReceived = _bytesTotal = 0; + + _request->send(); // if (_reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool()) { // // If the file as been updated since it was cached, refresh it @@ -435,63 +427,88 @@ void Resource::makeRequest() { // } // } // } - - // _replyTimer = new QTimer(this); - // connect(_replyTimer, SIGNAL(timeout()), SLOT(handleReplyTimeout())); - // _replyTimer->setSingleShot(true); - // _replyTimer->start(REPLY_TIMEOUT_MS); - _bytesReceived = _bytesTotal = 0; } -// void Resource::handleReplyError(QNetworkReply::NetworkError error, QDebug debug) { -// _reply->disconnect(this); -// _replyTimer->disconnect(this); -// _reply->deleteLater(); -// _reply = nullptr; -// _replyTimer->deleteLater(); -// _replyTimer = nullptr; -// ResourceCache::requestCompleted(this); -// -// // retry for certain types of failures -// switch (error) { -// case QNetworkReply::RemoteHostClosedError: -// case QNetworkReply::TimeoutError: -// case QNetworkReply::TemporaryNetworkFailureError: -// case QNetworkReply::ProxyConnectionClosedError: -// case QNetworkReply::ProxyTimeoutError: -// case QNetworkReply::UnknownNetworkError: -// case QNetworkReply::UnknownProxyError: -// case QNetworkReply::UnknownContentError: -// case QNetworkReply::ProtocolFailure: { -// // retry with increasing delays -// const int MAX_ATTEMPTS = 8; -// const int BASE_DELAY_MS = 1000; -// if (++_attempts < MAX_ATTEMPTS) { -// QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); -// debug << "-- retrying..."; -// return; -// } -// // fall through to final failure -// } -// default: -// finishedLoading(false); -// break; -// } +// void Resource::handleReplyError(ResourceRequest::Result result, QDebug debug) { +// +// // // retry for certain types of failures +// // switch (error) { +// // case QNetworkReply::RemoteHostClosedError: +// // case QNetworkReply::TimeoutError: +// // case QNetworkReply::TemporaryNetworkFailureError: +// // case QNetworkReply::ProxyConnectionClosedError: +// // case QNetworkReply::ProxyTimeoutError: +// // case QNetworkReply::UnknownNetworkError: +// // case QNetworkReply::UnknownProxyError: +// // case QNetworkReply::UnknownContentError: +// // case QNetworkReply::ProtocolFailure: { +// // // retry with increasing delays +// // const int MAX_ATTEMPTS = 8; +// // const int BASE_DELAY_MS = 1000; +// // if (++_attempts < MAX_ATTEMPTS) { +// // QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(nttemptRequest())); +// // debug << "-- retrying..."; +// // return; +// // } +// // // fall through to final failure +// // } +// // default: +// // finishedLoading(false); +// // break; +// // } // } // void Resource::handleReplyFinished() { - // _request->disconnect(this); - // _request->deleteLater(); - // _request = nullptr; + QString u = _url.path(); Q_ASSERT(_request); - if (_request->getResult() == ResourceRequest::SUCCESS) { + + auto result = _request->getResult(); + if (result == ResourceRequest::SUCCESS) { _data = _request->moveData(); + qDebug() << "Reqeust finsihed for " << _url << ", " << _activeUrl; + _request->disconnect(this); _request->deleteLater(); _request = nullptr; + + ResourceCache::requestCompleted(this); + downloadFinished(_data); + } else { + _request->disconnect(this); + _request->deleteLater(); + _request = nullptr; + + if (result == ResourceRequest::Result::TIMEOUT) { + qDebug() << "Timed out loading" << _url << + "received" << _bytesReceived << "total" << _bytesTotal; + } else { + qDebug() << "Error loading " << _url; + } + + bool retry = false; + switch (result) { + case ResourceRequest::Result::TIMEOUT: + case ResourceRequest::Result::ERROR: { + // retry with increasing delays + const int MAX_ATTEMPTS = 8; + const int BASE_DELAY_MS = 1000; + if (++_attempts < MAX_ATTEMPTS) { + QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); + retry = true; + break; + } + // fall through to final failure + } + default: + finishedLoading(false); + break; + } + + if (!retry) { + ResourceCache::requestCompleted(this); + } } - ResourceCache::requestCompleted(this); } uint qHash(const QPointer& value, uint seed) { diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 59192449b8..e23b68416a 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -199,8 +199,7 @@ protected: virtual void reinsert(); QUrl _url; - ResourceRequest* _request = nullptr; - //QNetworkRequest _request; + QUrl _activeUrl; bool _startedLoading = false; bool _failedToLoad = false; bool _loaded = false; @@ -211,19 +210,21 @@ protected: private slots: // void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); - // void handleReplyError(); void handleReplyFinished(); + // void handleReplyError(); // void handleReplyTimeout(); private: void setLRUKey(int lruKey) { _lruKey = lruKey; } void makeRequest(); + void retry(); - void handleReplyError(QNetworkReply::NetworkError error, QDebug debug); + // void handleReplyError(ResourceRequest::Result result, QDebug debug); friend class ResourceCache; + ResourceRequest* _request = nullptr; int _lruKey = 0; QTimer* _replyTimer = nullptr; qint64 _bytesReceived = 0; From 7305135125f09b31d2f6ba37eaf919a2a88f85e8 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:16:34 -0700 Subject: [PATCH 23/77] Update ResourceManager --- libraries/networking/src/ResourceManager.cpp | 42 ++++++++++++++------ libraries/networking/src/ResourceManager.h | 10 ++++- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index 7fdac3028a..00348df69b 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -12,7 +12,6 @@ #include #include -#include #include "AssetClient.h" #include "AssetRequest.h" @@ -31,24 +30,38 @@ ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) : _url(url) { } +HTTPResourceRequest::~HTTPResourceRequest() { + if (_reply) { + _reply->disconnect(this); + _reply->deleteLater(); + _reply = nullptr; + } +} + void HTTPResourceRequest::doSend() { QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); QNetworkRequest networkRequest = QNetworkRequest(_url); networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - if (!_cacheEnabled) { + if (_cacheEnabled) { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); + } else { networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - } + } - QNetworkReply* reply = networkAccessManager.get(networkRequest); - QObject::connect(reply, &QNetworkReply::finished, [=]() { - if (_state != IN_PROGRESS) return; + _reply = networkAccessManager.get(networkRequest); + QObject::connect(_reply, &QNetworkReply::finished, [=]() { + Q_ASSERT(_state == IN_PROGRESS); + Q_ASSERT(_reply); _state = FINISHED; - auto error = reply->error(); + auto error = _reply->error(); qDebug() << "Loaded " << _url; + QString u = _url.path(); if (error == QNetworkReply::NoError) { - _data = reply->readAll(); + _data = _reply->readAll(); + qDebug() << "!!!! " << _data.size() << " " << _url.path(); + _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); _result = ResourceRequest::SUCCESS; emit finished(); } else if (error == QNetworkReply::TimeoutError) { @@ -58,7 +71,8 @@ void HTTPResourceRequest::doSend() { _result = ResourceRequest::ERROR; emit finished(); } - reply->deleteLater(); + _reply->deleteLater(); + _reply = nullptr; }); } @@ -105,16 +119,20 @@ void ATPResourceRequest::doSend() { const int TIMEOUT_MS = 2000; void ResourceRequest::send() { + Q_ASSERT(_state == UNSENT); + + _state = IN_PROGRESS; + doSend(); connect(&_sendTimer, &QTimer::timeout, this, &ResourceRequest::timeout); _sendTimer.setSingleShot(true); _sendTimer.start(TIMEOUT_MS); - _state = IN_PROGRESS; - doSend(); } void ResourceRequest::timeout() { + Q_ASSERT(_state != UNSENT); + // TIMEOUT!! - if (_state == UNSENT) { + if (_state == IN_PROGRESS) { qDebug() << "TImed out loading " << _url; _state = FINISHED; _result = TIMEOUT; diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h index 4050b5f390..599399fb15 100644 --- a/libraries/networking/src/ResourceManager.h +++ b/libraries/networking/src/ResourceManager.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -41,6 +42,7 @@ public: State getState() const { return _state; } Result getResult() const { return _result; } QUrl getUrl() const { return _url; } + bool loadedFromCache() const { return _loadedFromCache; } void setCacheEnabled(bool value) { _cacheEnabled = value; } @@ -52,11 +54,12 @@ protected: virtual void doSend() = 0; QUrl _url; - State _state = UNSENT; + State _state { UNSENT }; Result _result; QTimer _sendTimer; QByteArray _data; bool _cacheEnabled { true }; + bool _loadedFromCache { false }; private slots: void timeout(); @@ -65,10 +68,15 @@ private slots: class HTTPResourceRequest : public ResourceRequest { Q_OBJECT public: + ~HTTPResourceRequest(); + HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } protected: virtual void doSend() override; + +private: + QNetworkReply* _reply { nullptr }; }; class FileResourceRequest : public ResourceRequest { From e1f8b6700c025b9d94ccef3f42959eb3e8c370c8 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:17:19 -0700 Subject: [PATCH 24/77] Update GeometryCache to work with ResourceCache updates --- libraries/render-utils/src/GeometryCache.cpp | 14 +++++++++----- libraries/render-utils/src/GeometryCache.h | 2 +- libraries/render-utils/src/Model.cpp | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 5f9de399a6..69d5da939b 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1973,10 +1973,10 @@ GeometryReader::GeometryReader(const QWeakPointer& geometry, const QUr void GeometryReader::run() { QSharedPointer geometry = _geometry.toStrongRef(); if (geometry.isNull()) { - // _reply->deleteLater(); return; } try { + auto url = _url.path(); QString urlname = _url.path().toLower(); bool urlValid = true; urlValid &= !urlname.isEmpty(); @@ -2004,8 +2004,10 @@ void GeometryReader::run() { auto d = QByteArray(_data); QBuffer buffer { &d }; fbxgeo = OBJReader().readOBJ(&buffer, _mapping, &_url); + } else { + throw QString("Unknown file type"); } - QMetaObject::invokeMethod(geometry.data(), "setGeometry", Q_ARG(const FBXGeometry&, fbxgeo)); + QMetaObject::invokeMethod(geometry.data(), "setGeometry", Q_ARG(FBXGeometry, fbxgeo)); } else { throw QString("url is invalid"); } @@ -2027,7 +2029,7 @@ void NetworkGeometry::init() { } void NetworkGeometry::downloadFinished(const QByteArray& data) { - QUrl url = getURL(); + QUrl url = _activeUrl; if (url.path().toLower().endsWith(".fst")) { // it's a mapping file; parse it and get the mesh filename _mapping = FSTReader::readMapping(data); @@ -2055,7 +2057,7 @@ void NetworkGeometry::downloadFinished(const QByteArray& data) { // make the request immediately only if we have no LODs to switch between // TODO reimplement using ResourceRequest - _url = url.resolved(filename); + _activeUrl = url.resolved(filename); _startedLoading = false; if (_lods.isEmpty()) { attemptRequest(); @@ -2072,6 +2074,8 @@ void NetworkGeometry::downloadFinished(const QByteArray& data) { return; } + QString u = url.path(); + // send the reader off to the thread pool QThreadPool::globalInstance()->start(new GeometryReader(_self, url, data, _mapping)); } @@ -2085,7 +2089,7 @@ void NetworkGeometry::reinsert() { } } -void NetworkGeometry::setGeometry(const FBXGeometry& geometry) { +void NetworkGeometry::setGeometry(FBXGeometry geometry) { _geometry = geometry; auto textureCache = DependencyManager::get(); diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 679b67ab8a..e3a1525c55 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -342,7 +342,7 @@ protected: virtual void downloadFinished(const QByteArray& reply) override; virtual void reinsert(); - Q_INVOKABLE void setGeometry(const FBXGeometry& geometry); + Q_INVOKABLE void setGeometry(FBXGeometry geometry); private slots: void replaceTexturesWithPendingChanges(); diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 51f737505f..dbe1dfbe09 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -1405,7 +1405,9 @@ void Model::inverseKinematics(int endIndex, glm::vec3 targetPosition, const glm: } bool Model::restoreJointPosition(int jointIndex, float fraction, float priority) { + QString url = _url.path(); const FBXGeometry& geometry = _geometry->getFBXGeometry(); + return true; const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; return _rig->restoreJointPosition(jointIndex, fraction, priority, freeLineage); } From 330a6bb279ca771b9c367254fdf2c466c7ba4fc8 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:17:36 -0700 Subject: [PATCH 25/77] Update TextureCache to work with ResourceCache updates --- libraries/render-utils/src/TextureCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/TextureCache.cpp b/libraries/render-utils/src/TextureCache.cpp index 8290b7d544..16e8502340 100644 --- a/libraries/render-utils/src/TextureCache.cpp +++ b/libraries/render-utils/src/TextureCache.cpp @@ -223,7 +223,7 @@ void NetworkTexture::downloadFinished(const QByteArray& data) { } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, _type, content)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _type, content, _url)); } ImageReader::ImageReader(const QWeakPointer& texture, TextureType type, const QByteArray& data, From ebb4db134d4feae9d2e7f9e7a39d53441c6332f7 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 19 Aug 2015 12:18:01 -0700 Subject: [PATCH 26/77] Temporarily disable assert in ScriptCache --- libraries/script-engine/src/ScriptCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/script-engine/src/ScriptCache.cpp b/libraries/script-engine/src/ScriptCache.cpp index 87cdae0026..94fff89c35 100644 --- a/libraries/script-engine/src/ScriptCache.cpp +++ b/libraries/script-engine/src/ScriptCache.cpp @@ -27,7 +27,7 @@ ScriptCache::ScriptCache(QObject* parent) { } QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending, bool reload) { - assert(!_scriptCache.contains(url) || !reload); + //assert(!_scriptCache.contains(url) || !reload); QString scriptContents; if (_scriptCache.contains(url) && !reload) { From 4191ccd60de3924e8740839fb2e4c57cb0e872f9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 24 Aug 2015 12:16:20 -0700 Subject: [PATCH 27/77] Move *ResourceRequest objects to separate files --- .../networking/src/AssetResourceRequest.cpp | 44 +++++++ .../networking/src/AssetResourceRequest.h | 30 +++++ .../networking/src/FileResourceRequest.cpp | 27 ++++ .../networking/src/FileResourceRequest.h | 27 ++++ .../networking/src/HTTPResourceRequest.cpp | 96 ++++++++++++++ .../networking/src/HTTPResourceRequest.h | 40 ++++++ libraries/networking/src/ResourceManager.cpp | 124 +----------------- libraries/networking/src/ResourceManager.h | 84 +----------- libraries/networking/src/ResourceRequest.cpp | 23 ++++ libraries/networking/src/ResourceRequest.h | 59 +++++++++ 10 files changed, 350 insertions(+), 204 deletions(-) create mode 100644 libraries/networking/src/AssetResourceRequest.cpp create mode 100644 libraries/networking/src/AssetResourceRequest.h create mode 100644 libraries/networking/src/FileResourceRequest.cpp create mode 100644 libraries/networking/src/FileResourceRequest.h create mode 100644 libraries/networking/src/HTTPResourceRequest.cpp create mode 100644 libraries/networking/src/HTTPResourceRequest.h create mode 100644 libraries/networking/src/ResourceRequest.cpp create mode 100644 libraries/networking/src/ResourceRequest.h diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp new file mode 100644 index 0000000000..f7727336a9 --- /dev/null +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -0,0 +1,44 @@ +// +// AssetResourceRequest.cpp +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetResourceRequest.h" + +#include "AssetClient.h" +#include "AssetRequest.h" + +void ATPResourceRequest::doSend() { + // Make request to atp + auto assetClient = DependencyManager::get(); + auto hash = _url.path(); + + auto request = assetClient->create(hash); + + if (!request) { + return; + } + + QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { + if (_state != IN_PROGRESS) return; + _state = FINISHED; + if (true) { + _data = req->getData(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + }); + + request->start(); +} + +void ATPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { +} diff --git a/libraries/networking/src/AssetResourceRequest.h b/libraries/networking/src/AssetResourceRequest.h new file mode 100644 index 0000000000..34c1878b04 --- /dev/null +++ b/libraries/networking/src/AssetResourceRequest.h @@ -0,0 +1,30 @@ +// +// AssetResourceRequest.h +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetResourceRequest_h +#define hifi_AssetResourceRequest_h + +#include + +#include "ResourceRequest.h" + +class ATPResourceRequest : public ResourceRequest { + Q_OBJECT +public: + ATPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; + +private slots: + void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); +}; + +#endif diff --git a/libraries/networking/src/FileResourceRequest.cpp b/libraries/networking/src/FileResourceRequest.cpp new file mode 100644 index 0000000000..8bd7ecf94e --- /dev/null +++ b/libraries/networking/src/FileResourceRequest.cpp @@ -0,0 +1,27 @@ +// +// FileResourceRequest.cpp +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "FileResourceRequest.h" + +#include + +void FileResourceRequest::doSend() { + QString filename = _url.toLocalFile(); + QFile file(filename); + _state = FINISHED; + if (file.open(QFile::ReadOnly)) { + _data = file.readAll(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } +} diff --git a/libraries/networking/src/FileResourceRequest.h b/libraries/networking/src/FileResourceRequest.h new file mode 100644 index 0000000000..7618311626 --- /dev/null +++ b/libraries/networking/src/FileResourceRequest.h @@ -0,0 +1,27 @@ +// +// FileResourceRequest.h +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FileResourceRequest_h +#define hifi_FileResourceRequest_h + +#include + +#include "ResourceRequest.h" + +class FileResourceRequest : public ResourceRequest { + Q_OBJECT +public: + FileResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; +}; + +#endif diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp new file mode 100644 index 0000000000..f684718bf1 --- /dev/null +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -0,0 +1,96 @@ +// +// HTTPResourceRequest.cpp +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "HTTPResourceRequest.h" + +#include +#include +#include + +#include + +#include "NetworkAccessManager.h" +#include "NetworkLogging.h" + +HTTPResourceRequest::~HTTPResourceRequest() { + if (_reply) { + _reply->disconnect(this); + _reply->deleteLater(); + _reply = nullptr; + } +} + +void HTTPResourceRequest::doSend() { + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkRequest networkRequest = QNetworkRequest(_url); + networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + + if (_cacheEnabled) { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); + } else { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + } + + _reply = networkAccessManager.get(networkRequest); + + connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished); + + static const int TIMEOUT_MS = 10000; + + connect(&_sendTimer, &QTimer::timeout, this, &HTTPResourceRequest::onTimeout); + _sendTimer.setSingleShot(true); + _sendTimer.start(TIMEOUT_MS); +} + +void HTTPResourceRequest::onRequestFinished() { + Q_ASSERT(_state == IN_PROGRESS); + Q_ASSERT(_reply); + + _state = FINISHED; + auto error = _reply->error(); + qDebug() << "Loaded " << _url; + QString u = _url.path(); + if (error == QNetworkReply::NoError) { + _data = _reply->readAll(); + qDebug() << "!!!! " << _data.size() << " " << _url.path(); + _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else if (error == QNetworkReply::TimeoutError) { + _result = ResourceRequest::TIMEOUT; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + _reply->deleteLater(); + _reply = nullptr; +} + +void HTTPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { + if (_state == IN_PROGRESS) { + // Restart timer + _sendTimer.start(); + } +} + +void HTTPResourceRequest::onTimeout() { + Q_ASSERT(_state != UNSENT); + + // TODO Cancel request if timed out, handle properly in + // receive callback + if (_state == IN_PROGRESS) { + qCDebug(networking) << "Timed out loading " << _url; + _state = FINISHED; + _result = TIMEOUT; + _reply->abort(); + emit finished(); + } +} diff --git a/libraries/networking/src/HTTPResourceRequest.h b/libraries/networking/src/HTTPResourceRequest.h new file mode 100644 index 0000000000..c42204f3e9 --- /dev/null +++ b/libraries/networking/src/HTTPResourceRequest.h @@ -0,0 +1,40 @@ +// +// HTTPResourceRequest.h +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_HTTPResourceRequest_h +#define hifi_HTTPResourceRequest_h + +#include +#include +#include + +#include "ResourceRequest.h" + +class HTTPResourceRequest : public ResourceRequest { + Q_OBJECT +public: + ~HTTPResourceRequest(); + + HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; + +private slots: + void onTimeout(); + void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void onRequestFinished(); + +private: + QTimer _sendTimer; + QNetworkReply* _reply { nullptr }; +}; + +#endif diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index 00348df69b..cfa7513a1a 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -10,12 +10,9 @@ #include "ResourceManager.h" -#include -#include - -#include "AssetClient.h" -#include "AssetRequest.h" -#include "NetworkAccessManager.h" +#include "AssetResourceRequest.h" +#include "FileResourceRequest.h" +#include "HTTPResourceRequest.h" #include @@ -25,121 +22,6 @@ const QString URL_SCHEME_HTTPS = "https"; const QString URL_SCHEME_FTP = "ftp"; const QString URL_SCHEME_ATP = "atp"; -ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) : - QObject(parent), - _url(url) { -} - -HTTPResourceRequest::~HTTPResourceRequest() { - if (_reply) { - _reply->disconnect(this); - _reply->deleteLater(); - _reply = nullptr; - } -} - -void HTTPResourceRequest::doSend() { - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest networkRequest = QNetworkRequest(_url); - networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - - if (_cacheEnabled) { - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); - } else { - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - } - - _reply = networkAccessManager.get(networkRequest); - QObject::connect(_reply, &QNetworkReply::finished, [=]() { - Q_ASSERT(_state == IN_PROGRESS); - Q_ASSERT(_reply); - - _state = FINISHED; - auto error = _reply->error(); - qDebug() << "Loaded " << _url; - QString u = _url.path(); - if (error == QNetworkReply::NoError) { - _data = _reply->readAll(); - qDebug() << "!!!! " << _data.size() << " " << _url.path(); - _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); - _result = ResourceRequest::SUCCESS; - emit finished(); - } else if (error == QNetworkReply::TimeoutError) { - _result = ResourceRequest::TIMEOUT; - emit finished(); - } else { - _result = ResourceRequest::ERROR; - emit finished(); - } - _reply->deleteLater(); - _reply = nullptr; - }); -} - -void FileResourceRequest::doSend() { - QString filename = _url.toLocalFile(); - QFile file(filename); - _state = FINISHED; - if (file.open(QFile::ReadOnly)) { - _data = file.readAll(); - _result = ResourceRequest::SUCCESS; - emit finished(); - } else { - _result = ResourceRequest::ERROR; - emit finished(); - } -} - -void ATPResourceRequest::doSend() { - // Make request to atp - auto assetClient = DependencyManager::get(); - auto hash = _url.path(); - - auto request = assetClient->create(hash); - - if (!request) { - return; - } - - QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { - if (_state != IN_PROGRESS) return; - _state = FINISHED; - if (true) { - _data = req->getData(); - _result = ResourceRequest::SUCCESS; - emit finished(); - } else { - _result = ResourceRequest::ERROR; - emit finished(); - } - }); - - request->start(); -} - -const int TIMEOUT_MS = 2000; -void ResourceRequest::send() { - Q_ASSERT(_state == UNSENT); - - _state = IN_PROGRESS; - doSend(); - connect(&_sendTimer, &QTimer::timeout, this, &ResourceRequest::timeout); - _sendTimer.setSingleShot(true); - _sendTimer.start(TIMEOUT_MS); -} - -void ResourceRequest::timeout() { - Q_ASSERT(_state != UNSENT); - - // TIMEOUT!! - if (_state == IN_PROGRESS) { - qDebug() << "TImed out loading " << _url; - _state = FINISHED; - _result = TIMEOUT; - emit finished(); - } -} - ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const QUrl& url) { auto scheme = url.scheme(); if (scheme == URL_SCHEME_FILE) { diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h index 599399fb15..1cb6b08a79 100644 --- a/libraries/networking/src/ResourceManager.h +++ b/libraries/networking/src/ResourceManager.h @@ -8,94 +8,12 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // - #ifndef hifi_ResourceManager_h #define hifi_ResourceManager_h #include -#include -#include -#include -#include - -class ResourceRequest : public QObject { - Q_OBJECT -public: - ResourceRequest(QObject* parent, const QUrl& url); - - enum State { - UNSENT = 0, - IN_PROGRESS, - FINISHED - }; - - enum Result { - SUCCESS, - ERROR, - TIMEOUT, - NOT_FOUND - }; - - void send(); - QByteArray moveData() { return _data; } - State getState() const { return _state; } - Result getResult() const { return _result; } - QUrl getUrl() const { return _url; } - bool loadedFromCache() const { return _loadedFromCache; } - - void setCacheEnabled(bool value) { _cacheEnabled = value; } - -signals: - void progress(uint64_t bytesReceived, uint64_t bytesTotal); - void finished(); - -protected: - virtual void doSend() = 0; - - QUrl _url; - State _state { UNSENT }; - Result _result; - QTimer _sendTimer; - QByteArray _data; - bool _cacheEnabled { true }; - bool _loadedFromCache { false }; - -private slots: - void timeout(); -}; - -class HTTPResourceRequest : public ResourceRequest { - Q_OBJECT -public: - ~HTTPResourceRequest(); - - HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } - -protected: - virtual void doSend() override; - -private: - QNetworkReply* _reply { nullptr }; -}; - -class FileResourceRequest : public ResourceRequest { - Q_OBJECT -public: - FileResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } - -protected: - virtual void doSend() override; -}; - -class ATPResourceRequest : public ResourceRequest { - Q_OBJECT -public: - ATPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } - -protected: - virtual void doSend() override; -}; +#include "ResourceRequest.h" class ResourceManager { public: diff --git a/libraries/networking/src/ResourceRequest.cpp b/libraries/networking/src/ResourceRequest.cpp new file mode 100644 index 0000000000..44bfaae254 --- /dev/null +++ b/libraries/networking/src/ResourceRequest.cpp @@ -0,0 +1,23 @@ +// +// ResourceRequest.cpp +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ResourceRequest.h" + +ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) : + QObject(parent), + _url(url) { +} + +void ResourceRequest::send() { + Q_ASSERT(_state == UNSENT); + + _state = IN_PROGRESS; + doSend(); +} diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h new file mode 100644 index 0000000000..3eb975d8d6 --- /dev/null +++ b/libraries/networking/src/ResourceRequest.h @@ -0,0 +1,59 @@ +// +// ResourceRequest.h +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ResourceRequest_h +#define hifi_ResourceRequest_h + +#include +#include + +class ResourceRequest : public QObject { + Q_OBJECT +public: + ResourceRequest(QObject* parent, const QUrl& url); + + enum State { + UNSENT = 0, + IN_PROGRESS, + FINISHED + }; + + enum Result { + SUCCESS, + ERROR, + TIMEOUT, + NOT_FOUND + }; + + void send(); + QByteArray moveData() { return _data; } + State getState() const { return _state; } + Result getResult() const { return _result; } + QUrl getUrl() const { return _url; } + bool loadedFromCache() const { return _loadedFromCache; } + + void setCacheEnabled(bool value) { _cacheEnabled = value; } + +signals: + void progress(uint64_t bytesReceived, uint64_t bytesTotal); + void finished(); + +protected: + virtual void doSend() = 0; + + QUrl _url; + State _state { UNSENT }; + Result _result; + QByteArray _data; + bool _cacheEnabled { true }; + bool _loadedFromCache { false }; +}; + +#endif From b865a3f8fb548219f554e6a2ff5553b869011d2f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 12:41:55 -0700 Subject: [PATCH 28/77] Add thread pool to AssetGet requests --- assignment-client/src/assets/AssetServer.cpp | 94 ++++++++++++++++++-- assignment-client/src/assets/AssetServer.h | 6 +- 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 7dd31eba5f..f97769593d 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -11,23 +11,108 @@ #include "AssetServer.h" +#include #include #include #include #include #include #include +#include #include #include const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; -AssetServer::AssetServer(NLPacket& packet) : ThreadedAssignment(packet) { +void writeError(NLPacketList* packetList, AssetServerError error) { + packetList->writePrimitive(error); +} + +class SendAssetTask : public QRunnable { +public: + SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, const SharedNodePointer& sendToNode) : + QRunnable(), + _messageID(messageID), + _assetHash(assetHash), + _filePath(filePath), + _start(start), + _end(end), + _sendToNode(sendToNode) + { + } + + void run(); + +signals: + void finished(); + +private: + MessageID _messageID; + QByteArray _assetHash; + QString _filePath; + DataOffset _start; + DataOffset _end; + SharedNodePointer _sendToNode; +}; + +void SendAssetTask::run() { + qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; + auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); + + replyPacketList->write(_assetHash, HASH_HEX_LENGTH); + + replyPacketList->writePrimitive(_messageID); + + const int64_t MAX_LENGTH = 4294967296; + + if (_end <= _start) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else if (_end - _start > MAX_LENGTH) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + QFile file { _filePath }; + qDebug() << "Opening file: " << QString(QFileInfo(_assetHash).fileName()); + + if (file.open(QIODevice::ReadOnly)) { + if (file.size() < _end) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + auto size = _end - _start; + file.seek(_start); + replyPacketList->writePrimitive(AssetServerError::NO_ERROR); + replyPacketList->writePrimitive(size); + while (file.pos() < file.size()) { + static const int chunkSize = 20000; + QByteArray data = file.read(chunkSize); + replyPacketList->write(data, chunkSize); + } + qDebug() << "Done reading"; + } + file.close(); + } else { + qDebug() << "Asset not found"; + writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND); + } + } + + qDebug() << "Sending asset"; + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(replyPacketList), *_sendToNode); +} + +AssetServer::AssetServer(NLPacket& packet) : + ThreadedAssignment(packet), + _taskPool(this) { + + // Most of the work will be I/O bound, reading from disk and constructing packet objects, + // so the ideal is greater than the number of cores on the system. + _taskPool.setMaxThreadCount(20); + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); - packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); + packetReceiver.registerMessageListener(PacketType::AssetUpload, this, "handleAssetUpload"); } AssetServer::~AssetServer() { @@ -75,13 +160,10 @@ void AssetServer::run() { while (!_isFinished) { // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); + // QCoreApplication::processEvents(QEventLoop::WaitForMoreEvents); } } -void AssetServer::writeError(NLPacket* packet, AssetServerError error) { - packet->writePrimitive(error); -} - void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode) { if (packet->getPayloadSize() < HASH_HEX_LENGTH) { qDebug() << "ERROR bad file request"; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 585ecaa259..94116b191e 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -14,6 +14,7 @@ #include #include +#include #include "AssetUtils.h" @@ -31,11 +32,12 @@ public slots: private slots: void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); - void handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode); private: - static void writeError(NLPacket* packet, AssetServerError error); + static void writeError(NLPacketList* packetList, AssetServerError error); QDir _resourcesDirectory; + QThreadPool _taskPool; }; #endif From 0352782e4675a9c3a9eb3ca77f9717a4350a0c46 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 12:46:44 -0700 Subject: [PATCH 29/77] Add SendAssetTask --- assignment-client/src/assets/AssetServer.cpp | 40 ++------------------ 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f97769593d..1617307659 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -215,42 +215,10 @@ void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePoin qDebug() << "Received a request for the file: " << assetHash << " from " << start << " to " << end; - // We need to reply... - auto replyPacket = NLPacket::create(PacketType::AssetGetReply); - - replyPacket->write(assetHash, HASH_HEX_LENGTH); - - replyPacket->writePrimitive(messageID); - - const int64_t MAX_LENGTH = 1024; - - if (end <= start) { - writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); - } else if (end - start > MAX_LENGTH) { - writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); - } else { - QFile file { _resourcesDirectory.filePath(QString(assetHash)) }; - qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); - - if (file.open(QIODevice::ReadOnly)) { - if (file.size() < end) { - writeError(replyPacket.get(), AssetServerError::INVALID_BYTE_RANGE); - } else { - auto size = end - start; - file.seek(start); - QByteArray data = file.read(size); - replyPacket->writePrimitive(AssetServerError::NO_ERROR); - replyPacket->writePrimitive(size); - replyPacket->write(data, size); - } - } else { - qDebug() << "Asset not found"; - writeError(replyPacket.get(), AssetServerError::ASSET_NOT_FOUND); - } - } - - auto nodeList = DependencyManager::get(); - nodeList->sendPacket(std::move(replyPacket), *senderNode); + // Queue task + QString filePath = _resourcesDirectory.filePath(QString(assetHash)); + auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode); + _taskPool.start(task); } void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode) { From c1b9613a309c53b973cc5ddb05da31f5ab00152c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 14:58:56 -0700 Subject: [PATCH 30/77] Update PacketList::writeData() to be non-recursive --- libraries/networking/src/udt/PacketList.cpp | 148 +++++++++++--------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 2e9bef09e1..648736f6f9 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -68,7 +68,7 @@ std::unique_ptr PacketList::createPacket() { std::unique_ptr PacketList::createPacketWithExtendedHeader() { // use the static create method to create a new packet auto packet = createPacket(); - + if (!_extendedHeader.isEmpty()) { // add the extended header to the front of the packet if (packet->write(_extendedHeader) == -1) { @@ -76,7 +76,7 @@ std::unique_ptr PacketList::createPacketWithExtendedHeader() { << "- make sure that _extendedHeader is not larger than the payload capacity."; } } - + return packet; } @@ -98,83 +98,91 @@ QByteArray PacketList::getMessage() { } qint64 PacketList::writeData(const char* data, qint64 maxSize) { - if (!_currentPacket) { - // we don't have a current packet, time to set one up - _currentPacket = createPacketWithExtendedHeader(); - } - - // check if this block of data can fit into the currentPacket - if (maxSize <= _currentPacket->bytesAvailableForWrite()) { - // it fits, just write it to the current packet - _currentPacket->write(data, maxSize); - - // return the number of bytes written - return maxSize; - } else { - // it does not fit - this may need to be in the next packet - - if (!_isOrdered) { - auto newPacket = createPacketWithExtendedHeader(); - - if (_segmentStartIndex >= 0) { - // We in the process of writing a segment for an unordered PacketList. - // We need to try and pull the first part of the segment out to our new packet - - // check now to see if this is an unsupported write - int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); - - if ((newPacket->size() - numBytesToEnd) < maxSize) { - // this is an unsupported case - the segment is bigger than the size of an individual packet - // but the PacketList is not going to be sent ordered - qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is" - << "larger than the payload size."; - Q_ASSERT(false); - } - - int segmentSize = _currentPacket->pos() - _segmentStartIndex; - - // copy from currentPacket where the segment started to the beginning of the newPacket - newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize); - - // the current segment now starts at the beginning of the new packet - _segmentStartIndex = _extendedHeader.size(); - - // shrink the current payload to the actual size of the packet - _currentPacket->setPayloadSize(_segmentStartIndex); - } - - // move the current packet to our list of packets - _packets.push_back(std::move(_currentPacket)); - - // write the data to the newPacket - newPacket->write(data, maxSize); - - // swap our current packet with the new packet - _currentPacket.swap(newPacket); - - // return the number of bytes written to the new packet - return maxSize; + auto sizeRemaining = maxSize; + + while (sizeRemaining > 0) { + if (!_currentPacket) { + // we don't have a current packet, time to set one up + _currentPacket = createPacketWithExtendedHeader(); + } + + // check if this block of data can fit into the currentPacket + if (sizeRemaining <= _currentPacket->bytesAvailableForWrite()) { + // it fits, just write it to the current packet + _currentPacket->write(data, sizeRemaining); + + sizeRemaining = 0; } else { - // we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover - // into a new packet - - int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); - _currentPacket->write(data, numBytesToEnd); - - // move the current packet to our list of packets - _packets.push_back(std::move(_currentPacket)); - - // recursively call our writeData method for the remaining data to write to a new packet - return numBytesToEnd + writeData(data + numBytesToEnd, maxSize - numBytesToEnd); + // it does not fit - this may need to be in the next packet + + if (!_isOrdered) { + auto newPacket = createPacketWithExtendedHeader(); + + if (_segmentStartIndex >= 0) { + // We in the process of writing a segment for an unordered PacketList. + // We need to try and pull the first part of the segment out to our new packet + + // check now to see if this is an unsupported write + int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); + + if ((newPacket->size() - numBytesToEnd) < sizeRemaining) { + // this is an unsupported case - the segment is bigger than the size of an individual packet + // but the PacketList is not going to be sent ordered + qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is" + << "larger than the payload size."; + Q_ASSERT(false); + } + + int segmentSize = _currentPacket->pos() - _segmentStartIndex; + + // copy from currentPacket where the segment started to the beginning of the newPacket + newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize); + + // the current segment now starts at the beginning of the new packet + _segmentStartIndex = _extendedHeader.size(); + + // shrink the current payload to the actual size of the packet + _currentPacket->setPayloadSize(_segmentStartIndex); + } + + // move the current packet to our list of packets + _packets.push_back(std::move(_currentPacket)); + + // write the data to the newPacket + newPacket->write(data, sizeRemaining); + + // swap our current packet with the new packet + _currentPacket.swap(newPacket); + + // We've written all of the data, so set sizeRemaining to 0 + sizeRemaining = 0; + } else { + // we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover + // into a new packet + + int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); + _currentPacket->write(data, numBytesToEnd); + + // Remove number of bytes written from sizeRemaining + sizeRemaining -= numBytesToEnd; + + // Move the data pointer forward + data += numBytesToEnd; + + // move the current packet to our list of packets + _packets.push_back(std::move(_currentPacket)); + } } } + + return maxSize; } void PacketList::closeCurrentPacket(bool shouldSendEmpty) { if (shouldSendEmpty && !_currentPacket) { _currentPacket = createPacketWithExtendedHeader(); } - + if (_currentPacket) { // move the current packet to our list of packets _packets.push_back(std::move(_currentPacket)); From 85f80ff6f9af62e6b4edc75201692c580415b325 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:05:35 -0700 Subject: [PATCH 31/77] Update AssetServer to use PacketList --- assignment-client/src/assets/AssetServer.cpp | 37 ++++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 1617307659..43387e7e66 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -221,38 +221,45 @@ void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePoin _taskPool.start(task); } -void AssetServer::handleAssetUpload(QSharedPointer packet, SharedNodePointer senderNode) { +void AssetServer::handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode) { + auto data = packetList->getMessage(); + QBuffer buffer { &data }; + buffer.open(QIODevice::ReadOnly); + MessageID messageID; - packet->readPrimitive(&messageID); + buffer.read(reinterpret_cast(&messageID), sizeof(messageID)); + // buffer.readPrimitive(&messageID); - char extensionLength; - packet->readPrimitive(&extensionLength); + uint8_t extensionLength; + buffer.read(reinterpret_cast(&extensionLength), sizeof(extensionLength)); + // buffer.readPrimitive(&extensionLength); - char extension[extensionLength]; - packet->read(extension, extensionLength); + QByteArray extension = buffer.read(extensionLength); qDebug() << "Got extension: " << extension; - int fileSize; - packet->readPrimitive(&fileSize); + uint64_t fileSize; + buffer.read(reinterpret_cast(&fileSize), sizeof(fileSize)); + // buffer.readPrimitive(&fileSize); - const int MAX_LENGTH = 1024; - fileSize = std::min(MAX_LENGTH, fileSize); + // const uint64_t MAX_LENGTH = 1024; + // fileSize = std::min(MAX_LENGTH, fileSize); qDebug() << "Receiving a file of size " << fileSize; - QByteArray data = packet->read(fileSize); + QByteArray fileData = buffer.read(fileSize); - QString hash = hashData(data); + QString hash = hashData(fileData); - qDebug() << "Got data: (" << hash << ") " << data; + qDebug() << "Got data: (" << hash << ") "; - QFile file { _resourcesDirectory.filePath(QString(hash)) }; + QFile file { _resourcesDirectory.filePath(QString(hash)) }; //+ "." + QString(extension) }; if (file.exists()) { qDebug() << "[WARNING] This file already exists"; } else { file.open(QIODevice::WriteOnly); - file.write(data); + file.write(fileData); + file.close(); } auto replyPacket = NLPacket::create(PacketType::AssetUploadReply); From dc5d5cbb63ca70b0957b65c0cb6091c79486760a Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:06:40 -0700 Subject: [PATCH 32/77] Add asset server ping to stats --- interface/resources/qml/Stats.qml | 149 +++++++++++++++--------------- interface/src/ui/Stats.cpp | 2 + interface/src/ui/Stats.h | 2 + 3 files changed, 81 insertions(+), 72 deletions(-) diff --git a/interface/resources/qml/Stats.qml b/interface/resources/qml/Stats.qml index b7081afadf..878d784bdf 100644 --- a/interface/resources/qml/Stats.qml +++ b/interface/resources/qml/Stats.qml @@ -28,39 +28,39 @@ Item { anchors.fill: parent onClicked: { root.expanded = !root.expanded; } } - + Column { id: generalCol spacing: 4; x: 4; y: 4; - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize text: "Servers: " + root.serverCount } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Avatars: " + root.avatarCount + text: "Avatars: " + root.avatarCount } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Framerate: " + root.framerate + text: "Framerate: " + root.framerate } Text { color: root.fontColor; font.pixelSize: root.fontSize text: "Simrate: " + root.simrate } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Packets In/Out: " + root.packetInCount + "/" + root.packetOutCount + text: "Packets In/Out: " + root.packetInCount + "/" + root.packetOutCount } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Mbps In/Out: " + root.mbpsIn.toFixed(2) + "/" + root.mbpsOut.toFixed(2) + text: "Mbps In/Out: " + root.mbpsIn.toFixed(2) + "/" + root.mbpsOut.toFixed(2) } } } @@ -77,30 +77,35 @@ Item { Column { id: pingCol spacing: 4; x: 4; y: 4; - Text { + Text { color: root.fontColor font.pixelSize: root.fontSize - text: "Audio ping: " + root.audioPing + text: "Audio ping: " + root.audioPing } - Text { + Text { color: root.fontColor font.pixelSize: root.fontSize - text: "Avatar ping: " + root.avatarPing + text: "Avatar ping: " + root.avatarPing } - Text { + Text { color: root.fontColor font.pixelSize: root.fontSize - text: "Entities avg ping: " + root.entitiesPing + text: "Entities avg ping: " + root.entitiesPing } - Text { + Text { color: root.fontColor font.pixelSize: root.fontSize - visible: root.expanded; + text: "Asset ping: " + root.assetPing + } + Text { + color: root.fontColor + font.pixelSize: root.fontSize + visible: root.expanded; text: "Voxel max ping: " + 0 } } } - + Rectangle { width: geoCol.width + 8 height: geoCol.height + 8 @@ -112,34 +117,34 @@ Item { Column { id: geoCol spacing: 4; x: 4; y: 4; - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Position: " + root.position.x.toFixed(1) + ", " + - root.position.y.toFixed(1) + ", " + root.position.z.toFixed(1) + text: "Position: " + root.position.x.toFixed(1) + ", " + + root.position.y.toFixed(1) + ", " + root.position.z.toFixed(1) } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Velocity: " + root.velocity.toFixed(1) + text: "Velocity: " + root.velocity.toFixed(1) } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Yaw: " + root.yaw.toFixed(1) + text: "Yaw: " + root.yaw.toFixed(1) } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded; - text: "Avatar Mixer: " + root.avatarMixerKbps + " kbps, " + - root.avatarMixerPps + "pps"; + visible: root.expanded; + text: "Avatar Mixer: " + root.avatarMixerKbps + " kbps, " + + root.avatarMixerPps + "pps"; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded; - text: "Downloads: "; + visible: root.expanded; + text: "Downloads: "; } } } @@ -154,72 +159,72 @@ Item { Column { id: octreeCol spacing: 4; x: 4; y: 4; - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - text: "Triangles: " + root.triangles + - " / Quads: " + root.quads + " / Material Switches: " + root.materialSwitches + text: "Triangles: " + root.triangles + + " / Quads: " + root.quads + " / Material Switches: " + root.materialSwitches } - Text { - color: root.fontColor; - font.pixelSize: root.fontSize - visible: root.expanded; - text: "\tMesh Parts Rendered Opaque: " + root.meshOpaque + - " / Translucent: " + root.meshTranslucent; - } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize visible: root.expanded; - text: "\tOpaque considered: " + root.opaqueConsidered + - " / Out of view: " + root.opaqueOutOfView + " / Too small: " + root.opaqueTooSmall; + text: "\tMesh Parts Rendered Opaque: " + root.meshOpaque + + " / Translucent: " + root.meshTranslucent; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: !root.expanded - text: "Octree Elements Server: " + root.serverElements + - " Local: " + root.localElements; + visible: root.expanded; + text: "\tOpaque considered: " + root.opaqueConsidered + + " / Out of view: " + root.opaqueOutOfView + " / Too small: " + root.opaqueTooSmall; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "Octree Sending Mode: " + root.sendingMode; + visible: !root.expanded + text: "Octree Elements Server: " + root.serverElements + + " Local: " + root.localElements; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "Octree Packets to Process: " + root.packetStats; + visible: root.expanded + text: "Octree Sending Mode: " + root.sendingMode; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "Octree Elements - "; + visible: root.expanded + text: "Octree Packets to Process: " + root.packetStats; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "\tServer: " + root.serverElements + - " Internal: " + root.serverInternal + - " Leaves: " + root.serverLeaves; + visible: root.expanded + text: "Octree Elements - "; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "\tLocal: " + root.localElements + - " Internal: " + root.localInternal + - " Leaves: " + root.localLeaves; + visible: root.expanded + text: "\tServer: " + root.serverElements + + " Internal: " + root.serverInternal + + " Leaves: " + root.serverLeaves; } - Text { + Text { color: root.fontColor; font.pixelSize: root.fontSize - visible: root.expanded - text: "LOD: " + root.lodStatus; + visible: root.expanded + text: "\tLocal: " + root.localElements + + " Internal: " + root.localInternal + + " Leaves: " + root.localLeaves; + } + Text { + color: root.fontColor; + font.pixelSize: root.fontSize + visible: root.expanded + text: "LOD: " + root.lodStatus; } } } diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 74e3a8f44f..74e6452d7f 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -126,8 +126,10 @@ void Stats::updateStats() { if (Menu::getInstance()->isOptionChecked(MenuOption::TestPing)) { SharedNodePointer audioMixerNode = nodeList->soloNodeOfType(NodeType::AudioMixer); SharedNodePointer avatarMixerNode = nodeList->soloNodeOfType(NodeType::AvatarMixer); + SharedNodePointer assetServerNode = nodeList->soloNodeOfType(NodeType::AssetServer); STAT_UPDATE(audioPing, audioMixerNode ? audioMixerNode->getPingMs() : -1); STAT_UPDATE(avatarPing, avatarMixerNode ? avatarMixerNode->getPingMs() : -1); + STAT_UPDATE(assetPing, assetServerNode ? assetServerNode->getPingMs() : -1); //// Now handle voxel servers, since there could be more than one, we average their ping times unsigned long totalPingOctree = 0; diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 4f0619d9c8..096469a84d 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -39,6 +39,7 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, audioPing, 0) STATS_PROPERTY(int, avatarPing, 0) STATS_PROPERTY(int, entitiesPing, 0) + STATS_PROPERTY(int, assetPing, 0) STATS_PROPERTY(QVector3D, position, QVector3D(0, 0, 0) ) STATS_PROPERTY(float, velocity, 0) STATS_PROPERTY(float, yaw, 0) @@ -105,6 +106,7 @@ signals: void audioPingChanged(); void avatarPingChanged(); void entitiesPingChanged(); + void assetPingChanged(); void positionChanged(); void velocityChanged(); void yawChanged(); From 4e3d0758e5c342615a5bd231937588b40578f2c3 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:07:07 -0700 Subject: [PATCH 33/77] Remove UNUSED_2 --- domain-server/src/DomainServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 410b58d17f..f2f1f3dee9 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -571,7 +571,6 @@ void DomainServer::populateDefaultStaticAssignmentsExcludingTypes(const QSet Date: Tue, 25 Aug 2015 15:18:40 -0700 Subject: [PATCH 34/77] Update Packet to properly copy packet position and message number --- libraries/networking/src/udt/Packet.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 2cc0624f7b..56c65e0657 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -93,6 +93,8 @@ Packet::Packet(Packet&& other) : _isReliable = other._isReliable; _isPartOfMessage = other._isPartOfMessage; _sequenceNumber = other._sequenceNumber; + _packetPosition = other._packetPosition; + _messageNumber = other._messageNumber; } Packet& Packet::operator=(Packet&& other) { @@ -101,6 +103,8 @@ Packet& Packet::operator=(Packet&& other) { _isReliable = other._isReliable; _isPartOfMessage = other._isPartOfMessage; _sequenceNumber = other._sequenceNumber; + _packetPosition = other._packetPosition; + _messageNumber = other._messageNumber; return *this; } From 2efba00e3e076f35bc68a788f7a2733ee8e63252 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:19:07 -0700 Subject: [PATCH 35/77] Update BasePacket::read to do a copy --- libraries/networking/src/udt/BasePacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 2bfa2b6805..aad547faa7 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -142,7 +142,7 @@ void BasePacket::setPayloadSize(qint64 payloadSize) { QByteArray BasePacket::read(qint64 maxSize) { qint64 sizeToRead = std::min(size() - pos(), maxSize); - QByteArray data { QByteArray::fromRawData(getPayload() + pos(), sizeToRead) }; + QByteArray data { QByteArray(getPayload() + pos(), sizeToRead) }; seek(pos() + sizeToRead); return data; } From 3959928bbb7e15003fffa32fd4e2cf37a3a79314 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:20:00 -0700 Subject: [PATCH 36/77] Update ResourceRequest::moveData to getData --- libraries/networking/src/ResourceCache.cpp | 2 +- libraries/networking/src/ResourceRequest.h | 2 +- libraries/script-engine/src/BatchLoader.cpp | 2 +- libraries/script-engine/src/ScriptCache.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 018f9b1c36..bb4e4aeeb9 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -464,7 +464,7 @@ void Resource::handleReplyFinished() { auto result = _request->getResult(); if (result == ResourceRequest::SUCCESS) { - _data = _request->moveData(); + _data = _request->getData(); qDebug() << "Reqeust finsihed for " << _url << ", " << _activeUrl; _request->disconnect(this); diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 3eb975d8d6..458a859d16 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -33,7 +33,7 @@ public: }; void send(); - QByteArray moveData() { return _data; } + QByteArray getData() { return _data; } State getState() const { return _state; } Result getResult() const { return _result; } QUrl getUrl() const { return _url; } diff --git a/libraries/script-engine/src/BatchLoader.cpp b/libraries/script-engine/src/BatchLoader.cpp index 99916310f4..58145c2ef3 100644 --- a/libraries/script-engine/src/BatchLoader.cpp +++ b/libraries/script-engine/src/BatchLoader.cpp @@ -42,7 +42,7 @@ void BatchLoader::start() { } connect(request, &ResourceRequest::finished, [=]() { if (request->getResult() == ResourceRequest::SUCCESS) { - _data.insert(url, request->moveData()); + _data.insert(url, request->getData()); } else { _data.insert(url, QString()); } diff --git a/libraries/script-engine/src/ScriptCache.cpp b/libraries/script-engine/src/ScriptCache.cpp index 94fff89c35..9409c10652 100644 --- a/libraries/script-engine/src/ScriptCache.cpp +++ b/libraries/script-engine/src/ScriptCache.cpp @@ -66,7 +66,7 @@ void ScriptCache::scriptDownloaded() { _scriptUsers.remove(url); if (req->getResult() == ResourceRequest::SUCCESS) { - _scriptCache[url] = req->moveData(); + _scriptCache[url] = req->getData(); qCDebug(scriptengine) << "Done downloading script at:" << url.toString(); foreach(ScriptUser* user, scriptUsers) { From 3eff605d72660ddf1c5e63fe400d695954a88f91 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:44:37 -0700 Subject: [PATCH 37/77] Update asset server number --- libraries/networking/src/Assignment.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/Assignment.h b/libraries/networking/src/Assignment.h index fe0cd72fb6..0d7738d22b 100644 --- a/libraries/networking/src/Assignment.h +++ b/libraries/networking/src/Assignment.h @@ -29,12 +29,11 @@ public: AudioMixerType = 0, AvatarMixerType = 1, AgentType = 2, - UNUSED_0 = 3, - UNUSED_1 = 4, - UNUSED_2 = 5, + AssetServerType = 3, + UNUSED_0 = 4, + UNUSED_1 = 5, EntityServerType = 6, - AssetServerType = 7, - AllTypes = 8 + AllTypes = 7 }; enum Command { From 12b2dfb2f96a2a3060ebd27a81594c601edd5aa0 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 15:45:00 -0700 Subject: [PATCH 38/77] Clean up HTTPResourceRequest --- libraries/networking/src/HTTPResourceRequest.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index f684718bf1..cf074eb98b 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -54,12 +54,10 @@ void HTTPResourceRequest::onRequestFinished() { Q_ASSERT(_reply); _state = FINISHED; + auto error = _reply->error(); - qDebug() << "Loaded " << _url; - QString u = _url.path(); if (error == QNetworkReply::NoError) { _data = _reply->readAll(); - qDebug() << "!!!! " << _data.size() << " " << _url.path(); _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); _result = ResourceRequest::SUCCESS; emit finished(); @@ -70,27 +68,28 @@ void HTTPResourceRequest::onRequestFinished() { _result = ResourceRequest::ERROR; emit finished(); } + _reply->deleteLater(); _reply = nullptr; } void HTTPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { if (_state == IN_PROGRESS) { - // Restart timer + // We've received data, so reset the timer _sendTimer.start(); } + + emit progress(bytesReceived, bytesTotal); } void HTTPResourceRequest::onTimeout() { Q_ASSERT(_state != UNSENT); - // TODO Cancel request if timed out, handle properly in - // receive callback if (_state == IN_PROGRESS) { qCDebug(networking) << "Timed out loading " << _url; + _reply->abort(); _state = FINISHED; _result = TIMEOUT; - _reply->abort(); emit finished(); } } From 0e98b179ddb8b7aaa7e66b1beb80266cb2a7f839 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 16:34:44 -0700 Subject: [PATCH 39/77] Update Asset gettings and uploading in AssetClient --- libraries/networking/src/AssetClient.cpp | 38 +++++++++++++----------- libraries/networking/src/AssetClient.h | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index cf8231abeb..3877f05fe9 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -10,6 +10,7 @@ #include "AssetClient.h" +#include #include #include "AssetRequest.h" @@ -22,7 +23,7 @@ MessageID AssetClient::_currentID = 0; AssetClient::AssetClient() { auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); - packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply"); + packetReceiver.registerMessageListener(PacketType::AssetGetReply, this, "handleAssetGetReply"); packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); } @@ -123,22 +124,25 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer packet, Share } } -void AssetClient::handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode) { - auto assetHash = packet->read(HASH_HEX_LENGTH); +void AssetClient::handleAssetGetReply(QSharedPointer packetList, SharedNodePointer senderNode) { + QByteArray data = packetList->getMessage(); + QBuffer packet { &data }; + packet.open(QIODevice::ReadOnly); + + auto assetHash = packet.read(HASH_HEX_LENGTH); qDebug() << "Got reply for asset: " << assetHash; MessageID messageID; - packet->readPrimitive(&messageID); + packet.read(reinterpret_cast(&messageID), sizeof(messageID)); AssetServerError error; - packet->readPrimitive(&error); - QByteArray data; + packet.read(reinterpret_cast(&error), sizeof(AssetServerError)); + QByteArray assetData; if (!error) { DataOffset length; - packet->readPrimitive(&length); - data = packet->read(length); - qDebug() << "Got data: " << length << ", " << data.toHex(); + packet.read(reinterpret_cast(&length), sizeof(DataOffset)); + data = packet.read(length); } else { qDebug() << "Failure getting asset: " << error; } @@ -153,22 +157,22 @@ bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCa auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (assetServer) { - auto packet = NLPacket::create(PacketType::AssetUpload); + auto packetList = std::unique_ptr(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true)); auto messageID = _currentID++; - packet->writePrimitive(messageID); + packetList->writePrimitive(messageID); - packet->writePrimitive(static_cast(extension.length())); - packet->write(extension.toLatin1().constData(), extension.length()); + packetList->writePrimitive(static_cast(extension.length())); + packetList->write(extension.toLatin1().constData(), extension.length()); qDebug() << "Extension length: " << extension.length(); qDebug() << "Extension: " << extension; - int size = data.length(); - packet->writePrimitive(size); - packet->write(data.constData(), size); + uint64_t size = data.length(); + packetList->writePrimitive(size); + packetList->write(data.constData(), size); - nodeList->sendPacket(std::move(packet), *assetServer); + nodeList->sendPacketList(std::move(packetList), *assetServer); _pendingUploads[messageID] = callback; diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 7ad9c1a638..5e26ff18b6 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -44,7 +44,7 @@ public: private slots: void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); - void handleAssetGetReply(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetGetReply(QSharedPointer packetList, SharedNodePointer senderNode); void handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode); private: From 0bfa0c4942238925a849ddbae08ba7e34b043b1d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 25 Aug 2015 16:35:41 -0700 Subject: [PATCH 40/77] Add progress to AssetResourceRequest --- libraries/networking/src/AssetResourceRequest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index f7727336a9..c29a54a787 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -24,6 +24,7 @@ void ATPResourceRequest::doSend() { return; } + connect(request, &AssetRequest::progress, this, &ATPResourceRequest::progress); QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { if (_state != IN_PROGRESS) return; _state = FINISHED; @@ -41,4 +42,6 @@ void ATPResourceRequest::doSend() { } void ATPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { + qDebug() << "Got asset data: " << bytesReceived << " / " << bytesTotal; + emit progress(bytesReceived, bytesTotal); } From d90ceb200a0fcc25624960b8381639e2aa307c77 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:27:50 -0700 Subject: [PATCH 41/77] Add asset uploader to Menu --- interface/src/Menu.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 939d733a13..9f38999853 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -21,6 +21,7 @@ #include #include +#include #include "Application.h" #include "AccountManager.h" #include "audio/AudioScope.h" @@ -87,6 +88,36 @@ Menu::Menu() { addActionToQMenuAndActionHash(fileMenu, MenuOption::RunningScripts, Qt::CTRL | Qt::Key_J, qApp, SLOT(toggleRunningScriptsWidget())); + // Asset uploading + { + auto action = new QAction("Upload File", fileMenu); + fileMenu->addAction(action); + action->setMenuRole(QAction::NoRole); + _actionHash.insert("Upload File", action); + + connect(action, &QAction::triggered, [this](bool checked) { + qDebug() << "Clicked upload file"; + auto filename = QFileDialog::getOpenFileUrl(nullptr, "Select a file to upload"); + if (!filename.isEmpty()) { + qDebug() << "Selected: " << filename; + QFile file { filename.path() }; + if (file.open(QIODevice::ReadOnly)) { + auto fileInfo { filename.path() }; + auto extension = fileInfo.suffix(); + auto data = file.readAll(); + auto assetClient = DependencyManager::get(); + assetClient->uploadAsset(data, extension, [this](bool result, QString hash) mutable { + if (result) { + QMessageBox::information(this, "Upload Successful", "URL: apt:/" + hash + "." + extension); + } else { + QMessageBox::warning(this, "Upload Failed", "There was an error uploading the file."); + } + }); + } + } + }); + } + auto addressManager = DependencyManager::get(); addDisabledActionAndSeparator(fileMenu, "History"); From d503c5293aa667780d8af461073db080748cceea Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:28:06 -0700 Subject: [PATCH 42/77] Remove commented out line --- assignment-client/src/assets/AssetServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 43387e7e66..0ed5b12b61 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -160,7 +160,6 @@ void AssetServer::run() { while (!_isFinished) { // since we're a while loop we need to help Qt's event processing QCoreApplication::processEvents(); - // QCoreApplication::processEvents(QEventLoop::WaitForMoreEvents); } } From 9b174bfbf443936177a29eeadbca5e130121da07 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:28:19 -0700 Subject: [PATCH 43/77] Remove extension from filename --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 0ed5b12b61..ad5d2a1d63 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -251,7 +251,7 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha qDebug() << "Got data: (" << hash << ") "; - QFile file { _resourcesDirectory.filePath(QString(hash)) }; //+ "." + QString(extension) }; + QFile file { _resourcesDirectory.filePath(QString(hash)) }; if (file.exists()) { qDebug() << "[WARNING] This file already exists"; From 24faab2089fb66b447b49d3781be292860d47ada Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:29:15 -0700 Subject: [PATCH 44/77] Add sendPacketList that take Node --- libraries/networking/src/LimitedNodeList.cpp | 14 ++++++++++++++ libraries/networking/src/LimitedNodeList.h | 1 + 2 files changed, 15 insertions(+) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index b45a37d002..4dd1a2399d 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -32,6 +32,7 @@ #include "HifiSockAddr.h" #include "UUID.h" #include "NetworkLogging.h" +#include "udt/Packet.h" const char SOLO_NODE_TYPES[2] = { NodeType::AvatarMixer, @@ -344,6 +345,19 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, return _nodeSocket.writePacketList(std::move(packetList), sockAddr); } +qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, const Node& destinationNode) { + // close the last packet in the list + packetList->closeCurrentPacket(); + + for (std::unique_ptr& packet : packetList->_packets) { + NLPacket* nlPacket = static_cast(packet.get()); + collectPacketStats(*nlPacket); + fillPacketHeader(*nlPacket, destinationNode.getConnectionSecret()); + } + + return _nodeSocket.writePacketList(std::move(packetList), *destinationNode.getActiveSocket()); +} + qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr) { // use the node's active socket as the destination socket if there is no overriden socket address diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 5f3e40fe11..1e94492e93 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -128,6 +128,7 @@ public: qint64 sendPacketList(NLPacketList& packetList, const HifiSockAddr& sockAddr, const QUuid& connectionSecret = QUuid()); qint64 sendPacketList(std::unique_ptr packetList, const HifiSockAddr& sockAddr); + qint64 sendPacketList(std::unique_ptr packetList, const Node& destinationNode); void (*linkedDataCreateCallback)(Node *); From 868558b4f0eb2aaa56d55a3bb7bdd953749a9dc9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:29:46 -0700 Subject: [PATCH 45/77] Add registration of QSharedPointer meta type --- libraries/networking/src/PacketReceiver.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index a086949ac8..002ecc2c6f 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -19,8 +19,10 @@ #include "NodeList.h" #include "SharedUtil.h" +Q_DECLARE_METATYPE(QSharedPointer); PacketReceiver::PacketReceiver(QObject* parent) : QObject(parent) { qRegisterMetaType>(); + qRegisterMetaType>(); } bool PacketReceiver::registerListenerForTypes(PacketTypeList types, QObject* listener, const char* slot) { From c878eb0cd99c0ed5ae005795c22f25247b8e9d62 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 09:31:03 -0700 Subject: [PATCH 46/77] Fix atomics not being initialized in SendQueue --- libraries/networking/src/udt/SendQueue.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/SendQueue.h b/libraries/networking/src/udt/SendQueue.h index 901a9f7a87..2557d920c7 100644 --- a/libraries/networking/src/udt/SendQueue.h +++ b/libraries/networking/src/udt/SendQueue.h @@ -85,13 +85,13 @@ private: Socket* _socket { nullptr }; // Socket to send packet on HifiSockAddr _destination; // Destination addr - std::atomic _lastACKSequenceNumber; // Last ACKed sequence number + std::atomic _lastACKSequenceNumber { 0 }; // Last ACKed sequence number MessageNumber _currentMessageNumber { 0 }; SequenceNumber _currentSequenceNumber; // Last sequence number sent out - std::atomic _atomicCurrentSequenceNumber;// Atomic for last sequence number sent out + std::atomic _atomicCurrentSequenceNumber { 0 };// Atomic for last sequence number sent out - std::atomic _packetSendPeriod; // Interval between two packet send event in microseconds, set from CC + std::atomic _packetSendPeriod { 1 }; // Interval between two packet send event in microseconds, set from CC std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure std::atomic _isRunning { false }; From aed1ba31c4a242d36c8cd73773f0bf028f16496d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 12:26:24 -0700 Subject: [PATCH 47/77] Fix bugs in Menu.cpp --- interface/src/Menu.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 3349171320..3305da5ba7 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -104,11 +104,11 @@ Menu::Menu() { qDebug() << "Selected: " << filename; QFile file { filename.path() }; if (file.open(QIODevice::ReadOnly)) { - auto fileInfo { filename.path() }; + QFileInfo fileInfo { filename.path() }; auto extension = fileInfo.suffix(); auto data = file.readAll(); auto assetClient = DependencyManager::get(); - assetClient->uploadAsset(data, extension, [this](bool result, QString hash) mutable { + assetClient->uploadAsset(data, extension, [this, extension](bool result, QString hash) mutable { if (result) { QMessageBox::information(this, "Upload Successful", "URL: apt:/" + hash + "." + extension); } else { From 269d5bb6da810b9928aecbc812dd51f4712edc89 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 14:04:55 -0700 Subject: [PATCH 48/77] Resolve bugs with *Cache changes --- libraries/networking/src/ResourceCache.cpp | 7 ++-- libraries/networking/src/ResourceCache.h | 4 +-- libraries/render-utils/src/GeometryCache.cpp | 36 +++++++++++--------- libraries/render-utils/src/GeometryCache.h | 10 +++--- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 994a023a75..71e3de8254 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -207,8 +207,7 @@ Resource::Resource(const QUrl& url, bool delayLoad) : // start loading immediately unless instructed otherwise if (!(_startedLoading || delayLoad)) { - // QTimer::singleShot(1, this, &Resource::attemptRequest); - //attemptRequest(); + QTimer::singleShot(1, this, &Resource::attemptRequest); } } @@ -376,6 +375,8 @@ void Resource::handleReplyFinished() { ResourceCache::requestCompleted(this); + emit loaded(_data); + downloadFinished(_data); } else { _request->disconnect(this); @@ -408,6 +409,8 @@ void Resource::handleReplyFinished() { break; } + emit failed(QNetworkReply::UnknownNetworkError); + if (!retry) { ResourceCache::requestCompleted(this); } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 3f40d32460..08d38db9a4 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -176,7 +176,7 @@ public: signals: /// Fired when the resource has been loaded. - void loaded(QNetworkReply& request); + void loaded(const QByteArray& request); /// Fired when resource failed to load. void failed(QNetworkReply::NetworkError error); @@ -219,8 +219,6 @@ private: void makeRequest(); void retry(); - void handleReplyErrorInternal(QNetworkReply::NetworkError error); - friend class ResourceCache; ResourceRequest* _request = nullptr; diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 2f81fe8b84..db7c25012c 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1678,15 +1678,15 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { } } -GeometryReader::GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping) : +GeometryReader::GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping) : _url(url), - _reply(reply), + _data(data), _mapping(mapping) { } void GeometryReader::run() { try { - if (!_reply) { + if (_data.isEmpty()) { throw QString("Reply is NULL ?!"); } QString urlname = _url.path().toLower(); @@ -1701,9 +1701,9 @@ void GeometryReader::run() { if (_url.path().toLower().endsWith(".fbx")) { const bool grabLightmaps = true; const float lightmapLevel = 1.0f; - fbxgeo = readFBX(_reply, _mapping, _url.path(), grabLightmaps, lightmapLevel); + fbxgeo = readFBX(_data, _mapping, _url.path(), grabLightmaps, lightmapLevel); } else if (_url.path().toLower().endsWith(".obj")) { - fbxgeo = OBJReader().readOBJ(_reply, _mapping, &_url); + fbxgeo = OBJReader().readOBJ(_data, _mapping); } else { QString errorStr("usupported format"); emit onError(NetworkGeometry::ModelParseError, errorStr); @@ -1717,7 +1717,6 @@ void GeometryReader::run() { qCDebug(renderutils) << "Error reading " << _url << ": " << error; emit onError(NetworkGeometry::ModelParseError, error); } - _reply->deleteLater(); } NetworkGeometry::NetworkGeometry(const QUrl& url, bool delayLoad, const QVariantHash& mapping, const QUrl& textureBaseUrl) : @@ -1746,8 +1745,10 @@ void NetworkGeometry::attemptRequest() { void NetworkGeometry::attemptRequestInternal() { if (_url.path().toLower().endsWith(".fst")) { + _mappingUrl = _url; requestMapping(_url); } else { + _modelUrl = _url; requestModel(_url); } } @@ -1838,8 +1839,8 @@ void NetworkGeometry::requestMapping(const QUrl& url) { _resource->deleteLater(); } _resource = new Resource(url, false); - connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(mappingRequestDone(QNetworkReply&))); - connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(mappingRequestError(QNetworkReply::NetworkError))); + connect(_resource, &Resource::loaded, this, &NetworkGeometry::mappingRequestDone); + connect(_resource, &Resource::failed, this, &NetworkGeometry::mappingRequestError); } void NetworkGeometry::requestModel(const QUrl& url) { @@ -1847,18 +1848,19 @@ void NetworkGeometry::requestModel(const QUrl& url) { if (_resource) { _resource->deleteLater(); } + _modelUrl = url; _resource = new Resource(url, false); - connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(modelRequestDone(QNetworkReply&))); - connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(modelRequestError(QNetworkReply::NetworkError))); + connect(_resource, &Resource::loaded, this, &NetworkGeometry::modelRequestDone); + connect(_resource, &Resource::failed, this, &NetworkGeometry::modelRequestError); } -void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) { +void NetworkGeometry::mappingRequestDone(const QByteArray& data) { assert(_state == RequestMappingState); // parse the mapping file - _mapping = FSTReader::readMapping(reply.readAll()); + _mapping = FSTReader::readMapping(data); - QUrl replyUrl = reply.url(); + QUrl replyUrl = _mappingUrl; QString modelUrlStr = _mapping.value("filename").toString(); if (modelUrlStr.isNull()) { qCDebug(renderutils) << "Mapping file " << _url << "has no \"filename\" entry"; @@ -1873,8 +1875,8 @@ void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) { _textureBaseUrl = replyUrl.resolved(texdir); } - QUrl modelUrl = replyUrl.resolved(modelUrlStr); - requestModel(modelUrl); + _modelUrl = replyUrl.resolved(modelUrlStr); + requestModel(_modelUrl); } } @@ -1884,13 +1886,13 @@ void NetworkGeometry::mappingRequestError(QNetworkReply::NetworkError error) { emit onFailure(*this, MappingRequestError); } -void NetworkGeometry::modelRequestDone(QNetworkReply& reply) { +void NetworkGeometry::modelRequestDone(const QByteArray& data) { assert(_state == RequestModelState); _state = ParsingModelState; // asynchronously parse the model file. - GeometryReader* geometryReader = new GeometryReader(reply.url(), &reply, _mapping); + GeometryReader* geometryReader = new GeometryReader(_modelUrl, data, _mapping); connect(geometryReader, SIGNAL(onSuccess(FBXGeometry*)), SLOT(modelParseSuccess(FBXGeometry*))); connect(geometryReader, SIGNAL(onError(int, QString)), SLOT(modelParseError(int, QString))); diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 71fa35c054..3820b58baf 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -348,10 +348,10 @@ signals: void onFailure(NetworkGeometry& networkGeometry, Error error); protected slots: - void mappingRequestDone(QNetworkReply& reply); + void mappingRequestDone(const QByteArray& data); void mappingRequestError(QNetworkReply::NetworkError error); - void modelRequestDone(QNetworkReply& reply); + void modelRequestDone(const QByteArray& data); void modelRequestError(QNetworkReply::NetworkError error); void modelParseSuccess(FBXGeometry* geometry); @@ -371,6 +371,8 @@ protected: State _state; QUrl _url; + QUrl _mappingUrl; + QUrl _modelUrl; QVariantHash _mapping; QUrl _textureBaseUrl; @@ -386,14 +388,14 @@ protected: class GeometryReader : public QObject, public QRunnable { Q_OBJECT public: - GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping); + GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping); virtual void run(); signals: void onSuccess(FBXGeometry* geometry); void onError(int error, QString str); private: QUrl _url; - QNetworkReply* _reply; + QByteArray _data; QVariantHash _mapping; }; From 2e585256ffc169b04b856e0b858ad5f3bac6a0c9 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 14:20:24 -0700 Subject: [PATCH 49/77] Update asset hash to Sha256 --- assignment-client/src/assets/AssetServer.cpp | 2 +- libraries/networking/src/AssetUtils.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index ad5d2a1d63..c7f61abdd3 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -273,6 +273,6 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha } QString AssetServer::hashData(const QByteArray& data) { - return QString(QCryptographicHash::hash(data, QCryptographicHash::Md5).toHex()); + return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex()); } diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 61def81ea8..904fa71da2 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -14,7 +14,7 @@ using MessageID = uint32_t; using DataOffset = int64_t; -const int HASH_HEX_LENGTH = 32; +const int HASH_HEX_LENGTH = 64; enum AssetServerError : uint8_t { NO_ERROR = 0, From 34f7aa74f365516a4864638a00acf58a199be793 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 14:26:09 -0700 Subject: [PATCH 50/77] Move SendAssetTask declaration to header --- assignment-client/src/assets/AssetServer.cpp | 37 ++++++-------------- assignment-client/src/assets/AssetServer.h | 19 ++++++++++ 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c7f61abdd3..697b4b21ec 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -29,32 +29,17 @@ void writeError(NLPacketList* packetList, AssetServerError error) { packetList->writePrimitive(error); } -class SendAssetTask : public QRunnable { -public: - SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, const SharedNodePointer& sendToNode) : - QRunnable(), - _messageID(messageID), - _assetHash(assetHash), - _filePath(filePath), - _start(start), - _end(end), - _sendToNode(sendToNode) - { - } - - void run(); - -signals: - void finished(); - -private: - MessageID _messageID; - QByteArray _assetHash; - QString _filePath; - DataOffset _start; - DataOffset _end; - SharedNodePointer _sendToNode; -}; +SendAssetTask::SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode) : + QRunnable(), + _messageID(messageID), + _assetHash(assetHash), + _filePath(filePath), + _start(start), + _end(end), + _sendToNode(sendToNode) +{ +} void SendAssetTask::run() { qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 94116b191e..485614d984 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -40,4 +40,23 @@ private: QThreadPool _taskPool; }; +class SendAssetTask : public QRunnable { +public: + SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode); + + void run(); + +signals: + void finished(); + +private: + MessageID _messageID; + QByteArray _assetHash; + QString _filePath; + DataOffset _start; + DataOffset _end; + SharedNodePointer _sendToNode; +}; + #endif From d66c5212009270b447fa925b94266f81d4307869 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 15:37:23 -0700 Subject: [PATCH 51/77] Move SendAssetTask to separate file --- assignment-client/src/assets/AssetServer.cpp | 70 ++---------------- assignment-client/src/assets/AssetServer.h | 21 +----- .../src/assets/SendAssetTask.cpp | 73 +++++++++++++++++++ assignment-client/src/assets/SendAssetTask.h | 41 +++++++++++ libraries/networking/src/AssetUtils.h | 2 + 5 files changed, 125 insertions(+), 82 deletions(-) create mode 100644 assignment-client/src/assets/SendAssetTask.cpp create mode 100644 assignment-client/src/assets/SendAssetTask.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 697b4b21ec..f8f3aa8d4c 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -21,74 +21,16 @@ #include #include -#include +#include "NetworkLogging.h" +#include "NodeType.h" +#include "SendAssetTask.h" const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; -void writeError(NLPacketList* packetList, AssetServerError error) { - packetList->writePrimitive(error); -} - -SendAssetTask::SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, - const SharedNodePointer& sendToNode) : - QRunnable(), - _messageID(messageID), - _assetHash(assetHash), - _filePath(filePath), - _start(start), - _end(end), - _sendToNode(sendToNode) -{ -} - -void SendAssetTask::run() { - qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; - auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); - - replyPacketList->write(_assetHash, HASH_HEX_LENGTH); - - replyPacketList->writePrimitive(_messageID); - - const int64_t MAX_LENGTH = 4294967296; - - if (_end <= _start) { - writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); - } else if (_end - _start > MAX_LENGTH) { - writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); - } else { - QFile file { _filePath }; - qDebug() << "Opening file: " << QString(QFileInfo(_assetHash).fileName()); - - if (file.open(QIODevice::ReadOnly)) { - if (file.size() < _end) { - writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); - } else { - auto size = _end - _start; - file.seek(_start); - replyPacketList->writePrimitive(AssetServerError::NO_ERROR); - replyPacketList->writePrimitive(size); - while (file.pos() < file.size()) { - static const int chunkSize = 20000; - QByteArray data = file.read(chunkSize); - replyPacketList->write(data, chunkSize); - } - qDebug() << "Done reading"; - } - file.close(); - } else { - qDebug() << "Asset not found"; - writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND); - } - } - - qDebug() << "Sending asset"; - auto nodeList = DependencyManager::get(); - nodeList->sendPacketList(std::move(replyPacketList), *_sendToNode); -} - AssetServer::AssetServer(NLPacket& packet) : - ThreadedAssignment(packet), - _taskPool(this) { + ThreadedAssignment(packet), + _taskPool(this) +{ // Most of the work will be I/O bound, reading from disk and constructing packet objects, // so the ideal is greater than the number of cores on the system. diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 485614d984..7a9cbd36ef 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -40,23 +40,8 @@ private: QThreadPool _taskPool; }; -class SendAssetTask : public QRunnable { -public: - SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, - const SharedNodePointer& sendToNode); - - void run(); - -signals: - void finished(); - -private: - MessageID _messageID; - QByteArray _assetHash; - QString _filePath; - DataOffset _start; - DataOffset _end; - SharedNodePointer _sendToNode; -}; +inline void writeError(NLPacketList* packetList, AssetServerError error) { + packetList->writePrimitive(error); +} #endif diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp new file mode 100644 index 0000000000..559331342f --- /dev/null +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -0,0 +1,73 @@ +// +// SendAssetTask.cpp +// +// Created by Ryan Huffman on 2015/08/26 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SendAssetTask.h" + +#include + +#include +#include +#include +#include +#include + +#include "AssetUtils.h" + +SendAssetTask::SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode) : + QRunnable(), + _messageID(messageID), + _assetHash(assetHash), + _filePath(filePath), + _start(start), + _end(end), + _sendToNode(sendToNode) +{ +} + +void SendAssetTask::run() { + qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; + auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); + + replyPacketList->write(_assetHash, HASH_HEX_LENGTH); + + replyPacketList->writePrimitive(_messageID); + + const int64_t MAX_LENGTH = 4294967296; + + if (_end <= _start) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else if (_end - _start > MAX_LENGTH) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + QFile file { _filePath }; + + if (file.open(QIODevice::ReadOnly)) { + if (file.size() < _end) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end; + } else { + auto size = _end - _start; + file.seek(_start); + replyPacketList->writePrimitive(AssetServerError::NO_ERROR); + replyPacketList->writePrimitive(size); + replyPacketList->write(file.read(size)); + } + file.close(); + qCDebug(networking) << "Sending asset: " << _assetHash; + } else { + qCDebug(networking) << "Asset not found: " << _assetHash; + writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND); + } + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(replyPacketList), *_sendToNode); +} diff --git a/assignment-client/src/assets/SendAssetTask.h b/assignment-client/src/assets/SendAssetTask.h new file mode 100644 index 0000000000..7dd3616f83 --- /dev/null +++ b/assignment-client/src/assets/SendAssetTask.h @@ -0,0 +1,41 @@ +// +// SendAssetTask.h +// +// Created by Ryan Huffman on 2015/08/26 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SendAssetTask_h +#define hifi_SendAssetTask_h + +#include +#include +#include + +#include "AssetUtils.h" +#include "AssetServer.h" +#include "Node.h" + +class SendAssetTask : public QRunnable { +public: + SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode); + + void run(); + +signals: + void finished(); + +private: + MessageID _messageID; + QByteArray _assetHash; + QString _filePath; + DataOffset _start; + DataOffset _end; + SharedNodePointer _sendToNode; +}; + +#endif diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 904fa71da2..c23b54de5d 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -11,6 +11,8 @@ #ifndef hifi_AssetUtils_h #define hifi_AssetUtils_h +#include "NLPacketList.h" + using MessageID = uint32_t; using DataOffset = int64_t; From 287f2533eb32958ffd595f979dd99e35e5e01337 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 15:37:38 -0700 Subject: [PATCH 52/77] Remove AssetServer dtor --- assignment-client/src/assets/AssetServer.cpp | 3 --- assignment-client/src/assets/AssetServer.h | 1 - 2 files changed, 4 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f8f3aa8d4c..f89b288f9c 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -42,9 +42,6 @@ AssetServer::AssetServer(NLPacket& packet) : packetReceiver.registerMessageListener(PacketType::AssetUpload, this, "handleAssetUpload"); } -AssetServer::~AssetServer() { -} - void AssetServer::run() { ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer); diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 7a9cbd36ef..07e0680c4f 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -22,7 +22,6 @@ class AssetServer : public ThreadedAssignment { Q_OBJECT public: AssetServer(NLPacket& packet); - ~AssetServer(); static QString hashData(const QByteArray& data); From 4ed5748afaaadc3fa419861461514e4466c78900 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 15:37:58 -0700 Subject: [PATCH 53/77] Fix new asset loading in directory --- assignment-client/src/assets/AssetServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f89b288f9c..2c294e652c 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -58,8 +58,8 @@ void AssetServer::run() { // Scan for new files qDebug() << "Looking for new files in asset directory"; auto files = _resourcesDirectory.entryInfoList(QDir::Files); - QRegExp filenameRegex { "^[a-f0-9]{32}(\\..+)?$" }; - for (auto fileInfo : files) { + QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" }; + for (const auto& fileInfo : files) { auto filename = fileInfo.fileName(); if (!filenameRegex.exactMatch(filename)) { qDebug() << "Found file: " << filename; From 59b32751f0efc2266335498d1886dcc398da4acd Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 15:54:02 -0700 Subject: [PATCH 54/77] Cleanup AssetScriptingInterface --- interface/src/AssetScriptingInterface.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/interface/src/AssetScriptingInterface.cpp b/interface/src/AssetScriptingInterface.cpp index 267f0cc25e..863d2ae3e7 100644 --- a/interface/src/AssetScriptingInterface.cpp +++ b/interface/src/AssetScriptingInterface.cpp @@ -27,7 +27,6 @@ AssetScriptingInterface::AssetScriptingInterface() { QScriptValue AssetScriptingInterface::getAsset(QString url, QScriptValue callback) { auto assetClient = DependencyManager::get(); - // auto request = assetClient->requestAsset(url); auto request = assetClient->create(url); if (!request) { @@ -42,11 +41,6 @@ QScriptValue AssetScriptingInterface::getAsset(QString url, QScriptValue callbac request->start(); - // bool success = AssetManager::getAsset(QUrl(url), [callback](AssetRequestUpdateType type, QByteArray data) mutable { - // auto result = callback.engine()->newVariant(data); - // QList arguments { type == AssetRequestUpdateType::COMPLETE, result }; - // callback.call(QScriptValue(), arguments); - // }); return true; } From 4105b7369cd4815e5c3ead6797edb373bb9732c5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 15:55:29 -0700 Subject: [PATCH 55/77] Remove AssetScriptingInterface --- interface/src/Application.cpp | 4 -- interface/src/AssetScriptingInterface.cpp | 53 ----------------------- interface/src/AssetScriptingInterface.h | 31 ------------- 3 files changed, 88 deletions(-) delete mode 100644 interface/src/AssetScriptingInterface.cpp delete mode 100644 interface/src/AssetScriptingInterface.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5974a33230..d94cff19f1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -300,7 +299,6 @@ bool setupEssentials(int& argc, char** argv) { auto pathUtils = DependencyManager::set(); auto actionFactory = DependencyManager::set(); auto assetClient = DependencyManager::set(); - auto assetScriptingInterface = DependencyManager::set(); auto userInputMapper = DependencyManager::set(); return true; @@ -3987,8 +3985,6 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri qScriptRegisterMetaType(scriptEngine, RayToOverlayIntersectionResultToScriptValue, RayToOverlayIntersectionResultFromScriptValue); - scriptEngine->registerGlobalObject("Assets", DependencyManager::get().data()); - scriptEngine->registerGlobalObject("Desktop", DependencyManager::get().data()); QScriptValue windowValue = scriptEngine->registerGlobalObject("Window", DependencyManager::get().data()); diff --git a/interface/src/AssetScriptingInterface.cpp b/interface/src/AssetScriptingInterface.cpp deleted file mode 100644 index 863d2ae3e7..0000000000 --- a/interface/src/AssetScriptingInterface.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// -// AssetScriptingInterface.cpp -// -// Created by Ryan Huffman on 2015/07/22 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "AssetScriptingInterface.h" - -#include - -#include -#include -#include -#include -#include -#include - -#include - -AssetScriptingInterface::AssetScriptingInterface() { -} - -QScriptValue AssetScriptingInterface::getAsset(QString url, QScriptValue callback) { - - auto assetClient = DependencyManager::get(); - auto request = assetClient->create(url); - - if (!request) { - return false; - } - - connect(request, &AssetRequest::finished, [callback](AssetRequest* req) mutable { - auto result = callback.engine()->newVariant(req->getData()); - QList arguments { true, result }; - callback.call(QScriptValue(), arguments); - }); - - request->start(); - - return true; -} - -QScriptValue AssetScriptingInterface::uploadAsset(QString data, QString extension, QScriptValue callback) { - auto assetClient = DependencyManager::get(); - return assetClient->uploadAsset(data.toLatin1(), extension, [callback](bool success, QString hash) mutable { - QList arguments { success, hash }; - auto result = callback.call(QScriptValue(), arguments); - }); -} diff --git a/interface/src/AssetScriptingInterface.h b/interface/src/AssetScriptingInterface.h deleted file mode 100644 index 8c0f7a71a1..0000000000 --- a/interface/src/AssetScriptingInterface.h +++ /dev/null @@ -1,31 +0,0 @@ -// -// AssetScriptingInterface.h -// -// Created by Ryan Huffman on 2015/07/22 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_AssetScriptingInterface_h -#define hifi_AssetScriptingInterface_h - -#include -#include - -#include -#include -#include - -class AssetScriptingInterface : public QObject, public Dependency { - Q_OBJECT -public: - AssetScriptingInterface(); - -public slots: - QScriptValue getAsset(QString hash, QScriptValue callback); - QScriptValue uploadAsset(QString data, QString extension, QScriptValue callback); -}; - -#endif From c2fcf5865a5b7d1e7b5871e091d7dcf27063c3f3 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 26 Aug 2015 16:02:47 -0700 Subject: [PATCH 56/77] Update log message for uploading assets --- libraries/networking/src/AssetClient.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 3877f05fe9..0c16b82774 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -182,7 +182,6 @@ bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCa } void AssetClient::handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode) { - qDebug() << "Got asset upload reply"; MessageID messageID; packet->readPrimitive(&messageID); @@ -196,9 +195,9 @@ void AssetClient::handleAssetUploadReply(QSharedPointer packet, Shared hashString = QString(hashData); - qDebug() << "Hash: " << hashString; + qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; } else { - qDebug() << "Error uploading file"; + qDebug() << "Error uploading file to asset server"; } if (_pendingUploads.contains(messageID)) { From f95957a2718988656ab4fdcae77ed713963f487d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:30:34 -0700 Subject: [PATCH 57/77] Cleanup AssetServer --- assignment-client/src/assets/AssetServer.cpp | 75 +++++++++----------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 2c294e652c..6b50edbf2f 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -80,28 +80,24 @@ void AssetServer::run() { file.rename(_resourcesDirectory.absoluteFilePath(hash)); } } - - while (!_isFinished) { - // since we're a while loop we need to help Qt's event processing - QCoreApplication::processEvents(); - } } void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode) { - if (packet->getPayloadSize() < HASH_HEX_LENGTH) { + QByteArray assetHash; + MessageID messageID; + + if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID))) { qDebug() << "ERROR bad file request"; return; } - QByteArray assetHash; - MessageID messageID; packet->readPrimitive(&messageID); - assetHash = packet->read(HASH_HEX_LENGTH); + assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH); auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply); replyPacket->writePrimitive(messageID); - replyPacket->write(assetHash, HASH_HEX_LENGTH); + replyPacket->write(assetHash); QFileInfo fileInfo { _resourcesDirectory.filePath(QString(assetHash)) }; qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); @@ -119,21 +115,19 @@ void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNode } void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { - if (packet->getPayloadSize() < HASH_HEX_LENGTH) { + MessageID messageID; + QByteArray assetHash; + DataOffset start; + DataOffset end; + + if (packet->getPayloadSize() < qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(start) + sizeof(end))) { qDebug() << "ERROR bad file request"; return; } - MessageID messageID; packet->readPrimitive(&messageID); - - QByteArray assetHash; assetHash = packet->read(HASH_HEX_LENGTH); - - DataOffset start; packet->readPrimitive(&start); - - DataOffset end; packet->readPrimitive(&end); qDebug() << "Received a request for the file: " << assetHash << " from " << start << " to " << end; @@ -151,11 +145,9 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha MessageID messageID; buffer.read(reinterpret_cast(&messageID), sizeof(messageID)); - // buffer.readPrimitive(&messageID); uint8_t extensionLength; buffer.read(reinterpret_cast(&extensionLength), sizeof(extensionLength)); - // buffer.readPrimitive(&extensionLength); QByteArray extension = buffer.read(extensionLength); @@ -163,34 +155,33 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha uint64_t fileSize; buffer.read(reinterpret_cast(&fileSize), sizeof(fileSize)); - // buffer.readPrimitive(&fileSize); - // const uint64_t MAX_LENGTH = 1024; - // fileSize = std::min(MAX_LENGTH, fileSize); qDebug() << "Receiving a file of size " << fileSize; - QByteArray fileData = buffer.read(fileSize); - - QString hash = hashData(fileData); - - qDebug() << "Got data: (" << hash << ") "; - - QFile file { _resourcesDirectory.filePath(QString(hash)) }; - - if (file.exists()) { - qDebug() << "[WARNING] This file already exists"; - } else { - file.open(QIODevice::WriteOnly); - file.write(fileData); - file.close(); - } - auto replyPacket = NLPacket::create(PacketType::AssetUploadReply); - replyPacket->writePrimitive(messageID); - replyPacket->writePrimitive(true); - replyPacket->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + if (fileSize > MAX_UPLOAD_SIZE) { + replyPacket->writePrimitive(false); + } else { + QByteArray fileData = buffer.read(fileSize); + + QString hash = hashData(fileData); + + qDebug() << "Got data: (" << hash << ") "; + + QFile file { _resourcesDirectory.filePath(QString(hash)) }; + + if (file.exists()) { + qDebug() << "[WARNING] This file already exists: " << hash; + } else { + file.open(QIODevice::WriteOnly); + file.write(fileData); + file.close(); + } + replyPacket->writePrimitive(true); + replyPacket->write(hash.toLatin1()); + } auto nodeList = DependencyManager::get(); nodeList->sendPacket(std::move(replyPacket), *senderNode); From 3731091d5e6b83f87fa88e83c24cd3d83867e27f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:30:47 -0700 Subject: [PATCH 58/77] Remove max chunk requesting from SendAssetTask --- assignment-client/src/assets/SendAssetTask.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index 559331342f..84c3a33c9f 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -40,12 +40,8 @@ void SendAssetTask::run() { replyPacketList->writePrimitive(_messageID); - const int64_t MAX_LENGTH = 4294967296; - if (_end <= _start) { writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); - } else if (_end - _start > MAX_LENGTH) { - writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); } else { QFile file { _filePath }; From b72e7e356d18c1cb2c5271a748e1a277043acfa6 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:31:17 -0700 Subject: [PATCH 59/77] Clean up AssertClient --- libraries/networking/src/AssetClient.cpp | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 0c16b82774..e72e05d5ff 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -16,6 +16,7 @@ #include "AssetRequest.h" #include "NodeList.h" #include "PacketReceiver.h" +#include "AssetUtils.h" MessageID AssetClient::_currentID = 0; @@ -37,7 +38,7 @@ AssetRequest* AssetClient::create(QString hash) { return req; } - if (hash.length() != 32) { + if (hash.length() != HASH_HEX_LENGTH) { qDebug() << "Invalid hash size"; return nullptr; } @@ -69,7 +70,7 @@ bool AssetClient::getAsset(QString hash, DataOffset start, DataOffset end, Recei auto messageID = ++_currentID; packet->writePrimitive(messageID); - packet->write(hash.toLatin1().constData(), 32); + packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); packet->writePrimitive(start); packet->writePrimitive(end); @@ -92,7 +93,7 @@ bool AssetClient::getAssetInfo(QString hash, GetInfoCallback callback) { auto messageID = ++_currentID; packet->writePrimitive(messageID); - packet->write(hash.toLatin1().constData(), 32); + packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); nodeList->sendPacket(std::move(packet), *assetServer); @@ -112,15 +113,15 @@ void AssetClient::handleAssetGetInfoReply(QSharedPointer packet, Share AssetServerError error; packet->readPrimitive(&error); - AssetInfo info; + AssetInfo info { assetHash, 0 }; - if (!error) { + if (error == NO_ERROR) { packet->readPrimitive(&info.size); } if (_pendingInfoRequests.contains(messageID)) { auto callback = _pendingInfoRequests.take(messageID); - callback(error != NO_ERROR, info); + callback(error == NO_ERROR, info); } } @@ -149,7 +150,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S if (_pendingRequests.contains(messageID)) { auto callback = _pendingRequests.take(messageID); - callback(!error, data); + callback(error == NO_ERROR, data); } } @@ -159,7 +160,7 @@ bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCa if (assetServer) { auto packetList = std::unique_ptr(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true)); - auto messageID = _currentID++; + auto messageID = ++_currentID; packetList->writePrimitive(messageID); packetList->writePrimitive(static_cast(extension.length())); @@ -185,23 +186,23 @@ void AssetClient::handleAssetUploadReply(QSharedPointer packet, Shared MessageID messageID; packet->readPrimitive(&messageID); - bool success; - packet->readPrimitive(&success); + AssetServerError error; + packet->readPrimitive(&error); QString hashString { "" }; - if (success) { + if (error) { + qDebug() << "Error uploading file to asset server"; + } else { auto hashData = packet->read(HASH_HEX_LENGTH); hashString = QString(hashData); qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; - } else { - qDebug() << "Error uploading file to asset server"; } if (_pendingUploads.contains(messageID)) { auto callback = _pendingUploads.take(messageID); - callback(success, hashString); + callback(error == NO_ERROR, hashString); } } From 15854c6715bb20f02f325aed98eba5f60d31a4d2 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:31:34 -0700 Subject: [PATCH 60/77] Make AssetClient methods private --- libraries/networking/src/AssetClient.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 5e26ff18b6..ef10cc2ea8 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -37,10 +37,6 @@ public: AssetClient(); Q_INVOKABLE AssetRequest* create(QString hash); - bool getAssetInfo(QString hash, GetInfoCallback callback); - bool getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback); - bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); - bool abortDataRequest(MessageID messageID); private slots: void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); @@ -48,6 +44,13 @@ private slots: void handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode); private: + friend class AssetRequest; + friend class Menu; + + bool getAssetInfo(QString hash, GetInfoCallback callback); + bool getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback); + bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); + static MessageID _currentID; QHash _pendingRequests; QHash _pendingInfoRequests; From 92dbe9997c52a2aa44cd069927ed909dd5b37211 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:31:50 -0700 Subject: [PATCH 61/77] Clean up AssetRequest --- libraries/networking/src/AssetRequest.cpp | 7 +------ libraries/networking/src/AssetRequest.h | 23 +---------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index d3863a5882..9e77899961 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -27,7 +27,6 @@ AssetRequest::AssetRequest(QObject* parent, QString hash) : void AssetRequest::start() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); - //(&AssetRequest::start) return; } @@ -38,7 +37,7 @@ void AssetRequest::start() { assetClient->getAssetInfo(_hash, [this](bool success, AssetInfo info) { _info = info; _data.resize(info.size); - const DataOffset CHUNK_SIZE = 1024; + const DataOffset CHUNK_SIZE = 1024000000; qDebug() << "Got size of " << _hash << " : " << info.size << " bytes"; @@ -72,7 +71,3 @@ void AssetRequest::start() { }); } } - -const QByteArray& AssetRequest::getData() { - return _data; -} diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 402aacad09..1e3e3e2f27 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -19,18 +19,6 @@ #include "AssetUtils.h" -// You should be able to get an asset from any thread, and handle the responses in a safe way -// on your own thread. Everything should happen on AssetClient's thread, the caller should -// receive events by connecting to signals on an object that lives on AssetClient's threads. - -// Receives parts of an asset and puts them together -// Emits signals: -// Progress -// Completion, success or error -// On finished, the AssetClient is effectively immutable and can be read from -// any thread safely -// -// Will often make multiple requests to the AssetClient to get data class AssetRequest : public QObject { Q_OBJECT public: @@ -51,16 +39,8 @@ public: AssetRequest(QObject* parent, QString hash); Q_INVOKABLE void start(); - //AssetRequest* requestAsset(QString hash); - // Create AssetRequest - // Start request for hash - // Store messageID -> AssetRequest - // When complete: - // Update AssetRequest - // AssetRequest emits signal - void receiveData(DataOffset start, DataOffset end, QByteArray data); - const QByteArray& getData(); + const QByteArray& getData() { return _data; } signals: void finished(AssetRequest*); @@ -74,7 +54,6 @@ private: QString _hash; QByteArray _data; int _numPendingRequests { 0 }; - // Timeout }; #endif From 8765329a46a882ddf3e71a8dcf528321da2e2434 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:32:39 -0700 Subject: [PATCH 62/77] Add ASSET_TOO_LARGE to AssetUtils --- libraries/networking/src/AssetUtils.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index c23b54de5d..cc5b2a281f 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -16,12 +16,14 @@ using MessageID = uint32_t; using DataOffset = int64_t; -const int HASH_HEX_LENGTH = 64; +const size_t HASH_HEX_LENGTH = 64; +const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB enum AssetServerError : uint8_t { NO_ERROR = 0, ASSET_NOT_FOUND, INVALID_BYTE_RANGE, + ASSET_TOO_LARGE, }; #endif From 667db200c032456569cd090cc6155f3413cbb992 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:39:12 -0700 Subject: [PATCH 63/77] Cleanup BatchLoader --- libraries/networking/src/ResourceCache.cpp | 1 - libraries/script-engine/src/BatchLoader.cpp | 41 ++------------------- 2 files changed, 3 insertions(+), 39 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 71e3de8254..c24c2f5075 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -190,7 +190,6 @@ void ResourceCache::requestCompleted(Resource* resource) { i++; } if (highestIndex >= 0) { - qDebug() << "trying to attempt a pending request"; attemptRequest(sharedItems->_pendingRequests.takeAt(highestIndex)); } } diff --git a/libraries/script-engine/src/BatchLoader.cpp b/libraries/script-engine/src/BatchLoader.cpp index 58145c2ef3..109b715df6 100644 --- a/libraries/script-engine/src/BatchLoader.cpp +++ b/libraries/script-engine/src/BatchLoader.cpp @@ -50,47 +50,12 @@ void BatchLoader::start() { checkFinished(); }); + // If we end up being destroyed before the reply finishes, clean it up + connect(this, &QObject::destroyed, request, &QObject::deleteLater); + qCDebug(scriptengine) << "Loading script at " << url; request->send(); - -// if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { -// QNetworkRequest request = QNetworkRequest(url); -// request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); -// QNetworkReply* reply = networkAccessManager.get(request); -// -// qCDebug(scriptengine) << "Downloading file at" << url; -// -// connect(reply, &QNetworkReply::finished, [=]() { -// if (reply->error()) { -// _data.insert(url, QString()); -// } else { -// _data.insert(url, reply->readAll()); -// } -// reply->deleteLater(); -// checkFinished(); -// }); -// -// // If we end up being destroyed before the reply finishes, clean it up -// connect(this, &QObject::destroyed, reply, &QObject::deleteLater); -// -// } else { -// #ifdef _WIN32 -// QString fileName = url.toString(); -// #else -// QString fileName = url.toLocalFile(); -// #endif -// -// qCDebug(scriptengine) << "Reading file at " << fileName; -// -// QFile scriptFile(fileName); -// if (scriptFile.open(QFile::ReadOnly | QFile::Text)) { -// QTextStream in(&scriptFile); -// _data.insert(url, in.readAll()); -// } else { -// _data.insert(url, QString()); -// } -// } } checkFinished(); } From 1d45293184e152e09e4be030a21e8093a19b67cb Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 09:56:26 -0700 Subject: [PATCH 64/77] Fix lambda callback affinity in BatchLoader --- libraries/script-engine/src/BatchLoader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/script-engine/src/BatchLoader.cpp b/libraries/script-engine/src/BatchLoader.cpp index 109b715df6..01e4a5e869 100644 --- a/libraries/script-engine/src/BatchLoader.cpp +++ b/libraries/script-engine/src/BatchLoader.cpp @@ -34,13 +34,12 @@ void BatchLoader::start() { } _started = true; - // QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); for (QUrl url : _urls) { auto request = ResourceManager::createResourceRequest(this, url); if (!request) { continue; } - connect(request, &ResourceRequest::finished, [=]() { + connect(request, &ResourceRequest::finished, this, [=]() { if (request->getResult() == ResourceRequest::SUCCESS) { _data.insert(url, request->getData()); } else { From 628f275013a3ac226e413aaab91a3456d6bbd348 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 10:07:53 -0700 Subject: [PATCH 65/77] Fix asset server upload errors --- assignment-client/src/assets/AssetServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 6b50edbf2f..d1bea55f36 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -162,7 +162,7 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha replyPacket->writePrimitive(messageID); if (fileSize > MAX_UPLOAD_SIZE) { - replyPacket->writePrimitive(false); + replyPacket->writePrimitive(AssetServerError::ASSET_TOO_LARGE); } else { QByteArray fileData = buffer.read(fileSize); @@ -179,7 +179,7 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha file.write(fileData); file.close(); } - replyPacket->writePrimitive(true); + replyPacket->writePrimitive(AssetServerError::NO_ERROR); replyPacket->write(hash.toLatin1()); } From c6493b1720e9a3c26fa322a28396de62a2ae4bab Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 10:27:12 -0700 Subject: [PATCH 66/77] Remove extension from hash in AssetResourceRequest --- libraries/networking/src/AssetResourceRequest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index c29a54a787..5cf1faad88 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -16,7 +16,7 @@ void ATPResourceRequest::doSend() { // Make request to atp auto assetClient = DependencyManager::get(); - auto hash = _url.path(); + auto hash = _url.path().split(".", QString::SkipEmptyParts)[0]; auto request = assetClient->create(hash); From a49a1eb0d6db9b061dfc2ba8290b6820f7c6d8e1 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 11:51:45 -0700 Subject: [PATCH 67/77] Add extension to atp requests --- assignment-client/src/assets/AssetServer.cpp | 24 ++++++++++++------- .../src/assets/SendAssetTask.cpp | 6 ++--- libraries/networking/src/AssetClient.cpp | 20 +++++++++++----- libraries/networking/src/AssetClient.h | 6 ++--- libraries/networking/src/AssetRequest.cpp | 9 +++---- libraries/networking/src/AssetRequest.h | 7 +++--- .../networking/src/AssetResourceRequest.cpp | 12 ++++++---- .../networking/src/AssetResourceRequest.h | 4 ++-- libraries/networking/src/ResourceManager.cpp | 2 +- 9 files changed, 55 insertions(+), 35 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index d1bea55f36..c6ad9f0f4d 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -77,7 +77,7 @@ void AssetServer::run() { qDebug() << "\tMoving " << filename << " to " << hash; - file.rename(_resourcesDirectory.absoluteFilePath(hash)); + file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix()); } } } @@ -85,24 +85,28 @@ void AssetServer::run() { void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode) { QByteArray assetHash; MessageID messageID; + uint8_t extensionLength; - if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID))) { + if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) { qDebug() << "ERROR bad file request"; return; } packet->readPrimitive(&messageID); assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH); + packet->readPrimitive(&extensionLength); + QByteArray extension = packet->read(extensionLength); auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply); replyPacket->writePrimitive(messageID); replyPacket->write(assetHash); - QFileInfo fileInfo { _resourcesDirectory.filePath(QString(assetHash)) }; - qDebug() << "Opening file: " << QString(QFileInfo(assetHash).fileName()); + QString fileName = QString(assetHash) + "." + extension; + QFileInfo fileInfo { _resourcesDirectory.filePath(fileName) }; if (fileInfo.exists() && fileInfo.isReadable()) { + qDebug() << "Opening file: " << fileInfo.filePath(); replyPacket->writePrimitive(AssetServerError::NO_ERROR); replyPacket->writePrimitive(fileInfo.size()); } else { @@ -117,23 +121,27 @@ void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNode void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { MessageID messageID; QByteArray assetHash; + uint8_t extensionLength; DataOffset start; DataOffset end; - if (packet->getPayloadSize() < qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(start) + sizeof(end))) { + auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end)); + if (packet->getPayloadSize() < minSize) { qDebug() << "ERROR bad file request"; return; } packet->readPrimitive(&messageID); assetHash = packet->read(HASH_HEX_LENGTH); + packet->readPrimitive(&extensionLength); + QByteArray extension = packet->read(extensionLength); packet->readPrimitive(&start); packet->readPrimitive(&end); - qDebug() << "Received a request for the file: " << assetHash << " from " << start << " to " << end; + qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end; // Queue task - QString filePath = _resourcesDirectory.filePath(QString(assetHash)); + QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension)); auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode); _taskPool.start(task); } @@ -170,7 +178,7 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha qDebug() << "Got data: (" << hash << ") "; - QFile file { _resourcesDirectory.filePath(QString(hash)) }; + QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) }; if (file.exists()) { qDebug() << "[WARNING] This file already exists: " << hash; diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index 84c3a33c9f..f30689b042 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -36,7 +36,7 @@ void SendAssetTask::run() { qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); - replyPacketList->write(_assetHash, HASH_HEX_LENGTH); + replyPacketList->write(_assetHash); replyPacketList->writePrimitive(_messageID); @@ -55,11 +55,11 @@ void SendAssetTask::run() { replyPacketList->writePrimitive(AssetServerError::NO_ERROR); replyPacketList->writePrimitive(size); replyPacketList->write(file.read(size)); + qCDebug(networking) << "Sending asset: " << _assetHash; } file.close(); - qCDebug(networking) << "Sending asset: " << _assetHash; } else { - qCDebug(networking) << "Asset not found: " << _assetHash; + qCDebug(networking) << "Asset not found: " << _filePath << "(" << _assetHash << ")"; writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND); } } diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index e72e05d5ff..f424171766 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -28,13 +28,14 @@ AssetClient::AssetClient() { packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); } -AssetRequest* AssetClient::create(QString hash) { +AssetRequest* AssetClient::create(QString hash, QString extension) { if (QThread::currentThread() != thread()) { AssetRequest* req; QMetaObject::invokeMethod(this, "create", Qt::BlockingQueuedConnection, Q_RETURN_ARG(AssetRequest*, req), - Q_ARG(QString, hash)); + Q_ARG(QString, hash), + Q_ARG(QString, extension)); return req; } @@ -48,7 +49,7 @@ AssetRequest* AssetClient::create(QString hash) { if (assetServer) { auto assetClient = DependencyManager::get(); - auto request = new AssetRequest(assetClient.data(), hash); + auto request = new AssetRequest(assetClient.data(), hash, extension); return request; } @@ -56,7 +57,7 @@ AssetRequest* AssetClient::create(QString hash) { return nullptr; } -bool AssetClient::getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { +bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { if (hash.length() != HASH_HEX_LENGTH) { qDebug() << "Invalid hash size"; return false; @@ -70,7 +71,12 @@ bool AssetClient::getAsset(QString hash, DataOffset start, DataOffset end, Recei auto messageID = ++_currentID; packet->writePrimitive(messageID); - packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + + packet->write(hash.toLatin1()); + + packet->writePrimitive(uint8_t(extension.length())); + packet->write(extension.toLatin1()); + packet->writePrimitive(start); packet->writePrimitive(end); @@ -84,7 +90,7 @@ bool AssetClient::getAsset(QString hash, DataOffset start, DataOffset end, Recei return false; } -bool AssetClient::getAssetInfo(QString hash, GetInfoCallback callback) { +bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -94,6 +100,8 @@ bool AssetClient::getAssetInfo(QString hash, GetInfoCallback callback) { auto messageID = ++_currentID; packet->writePrimitive(messageID); packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + packet->writePrimitive(uint8_t(extension.length())); + packet->write(extension.toLatin1()); nodeList->sendPacket(std::move(packet), *assetServer); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index ef10cc2ea8..0e81c6cfa2 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -36,7 +36,7 @@ class AssetClient : public QObject, public Dependency { public: AssetClient(); - Q_INVOKABLE AssetRequest* create(QString hash); + Q_INVOKABLE AssetRequest* create(QString hash, QString extension); private slots: void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); @@ -47,8 +47,8 @@ private: friend class AssetRequest; friend class Menu; - bool getAssetInfo(QString hash, GetInfoCallback callback); - bool getAsset(QString hash, DataOffset start, DataOffset end, ReceivedAssetCallback callback); + bool getAssetInfo(QString hash, QString extension, GetInfoCallback callback); + bool getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback); bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); static MessageID _currentID; diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 9e77899961..87e39e1edd 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -18,9 +18,10 @@ #include "NodeList.h" -AssetRequest::AssetRequest(QObject* parent, QString hash) : +AssetRequest::AssetRequest(QObject* parent, QString hash, QString extension) : QObject(parent), - _hash(hash) + _hash(hash), + _extension(extension) { } @@ -34,7 +35,7 @@ void AssetRequest::start() { _state = WAITING_FOR_INFO; auto assetClient = DependencyManager::get(); - assetClient->getAssetInfo(_hash, [this](bool success, AssetInfo info) { + assetClient->getAssetInfo(_hash, _extension, [this](bool success, AssetInfo info) { _info = info; _data.resize(info.size); const DataOffset CHUNK_SIZE = 1024000000; @@ -48,7 +49,7 @@ void AssetRequest::start() { ++_numPendingRequests; auto start = i * CHUNK_SIZE; auto end = std::min((i + 1) * CHUNK_SIZE, info.size); - assetClient->getAsset(_hash, start, end, [this, start, end](bool success, QByteArray data) { + assetClient->getAsset(_hash, _extension, start, end, [this, start, end](bool success, QByteArray data) { Q_ASSERT(data.size() == (end - start)); if (success) { diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 1e3e3e2f27..d1cc9c9b33 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -20,7 +20,7 @@ #include "AssetUtils.h" class AssetRequest : public QObject { - Q_OBJECT + Q_OBJECT public: enum State { NOT_STARTED = 0, @@ -36,7 +36,7 @@ public: Error, }; - AssetRequest(QObject* parent, QString hash); + AssetRequest(QObject* parent, QString hash, QString extension); Q_INVOKABLE void start(); @@ -44,7 +44,7 @@ public: signals: void finished(AssetRequest*); - void progress(uint64_t totalReceived, uint64_t total); + void progress(qint64 totalReceived, qint64 total); private: State _state = NOT_STARTED; @@ -52,6 +52,7 @@ private: AssetInfo _info; uint64_t _totalReceived { 0 }; QString _hash; + QString _extension; QByteArray _data; int _numPendingRequests { 0 }; }; diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 5cf1faad88..5a0af8ef1f 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -13,18 +13,20 @@ #include "AssetClient.h" #include "AssetRequest.h" -void ATPResourceRequest::doSend() { +void AssetResourceRequest::doSend() { // Make request to atp auto assetClient = DependencyManager::get(); - auto hash = _url.path().split(".", QString::SkipEmptyParts)[0]; + auto parts = _url.path().split(".", QString::SkipEmptyParts); + auto hash = parts[0]; + auto extension = parts.length() > 1 ? parts[1] : ""; - auto request = assetClient->create(hash); + auto request = assetClient->create(hash, extension); if (!request) { return; } - connect(request, &AssetRequest::progress, this, &ATPResourceRequest::progress); + connect(request, &AssetRequest::progress, this, &AssetResourceRequest::progress); QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { if (_state != IN_PROGRESS) return; _state = FINISHED; @@ -41,7 +43,7 @@ void ATPResourceRequest::doSend() { request->start(); } -void ATPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { +void AssetResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { qDebug() << "Got asset data: " << bytesReceived << " / " << bytesTotal; emit progress(bytesReceived, bytesTotal); } diff --git a/libraries/networking/src/AssetResourceRequest.h b/libraries/networking/src/AssetResourceRequest.h index 34c1878b04..2cfd1d28b7 100644 --- a/libraries/networking/src/AssetResourceRequest.h +++ b/libraries/networking/src/AssetResourceRequest.h @@ -15,10 +15,10 @@ #include "ResourceRequest.h" -class ATPResourceRequest : public ResourceRequest { +class AssetResourceRequest : public ResourceRequest { Q_OBJECT public: - ATPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + AssetResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } protected: virtual void doSend() override; diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index cfa7513a1a..b8e3eeeae7 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -29,7 +29,7 @@ ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const Q } else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) { return new HTTPResourceRequest(parent, url); } else if (scheme == URL_SCHEME_ATP) { - return new ATPResourceRequest(parent, url); + return new AssetResourceRequest(parent, url); } qDebug() << "Failed to load: " << url.url(); From b30a008162b08a0e7ad0bfdbe1bb57d1c496e384 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 12:48:55 -0700 Subject: [PATCH 68/77] Fix bug with NetworkTexture occasionally breaking on assert --- libraries/networking/src/ResourceCache.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index c24c2f5075..7ed5866707 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -206,7 +206,7 @@ Resource::Resource(const QUrl& url, bool delayLoad) : // start loading immediately unless instructed otherwise if (!(_startedLoading || delayLoad)) { - QTimer::singleShot(1, this, &Resource::attemptRequest); + QTimer::singleShot(0, this, &Resource::ensureLoading); } } @@ -333,8 +333,6 @@ void Resource::reinsert() { void Resource::makeRequest() { Q_ASSERT(!_request); - static const int REPLY_TIMEOUT_MS = 5000; - _request = ResourceManager::createResourceRequest(this, _activeUrl); if (!_request) { @@ -366,7 +364,7 @@ void Resource::handleReplyFinished() { auto result = _request->getResult(); if (result == ResourceRequest::SUCCESS) { _data = _request->getData(); - qDebug() << "Reqeust finsihed for " << _url << ", " << _activeUrl; + qDebug() << "Request finished for " << _url << ", " << _activeUrl; _request->disconnect(this); _request->deleteLater(); @@ -408,7 +406,8 @@ void Resource::handleReplyFinished() { break; } - emit failed(QNetworkReply::UnknownNetworkError); + auto error = result == ResourceRequest::TIMEOUT ? QNetworkReply::TimeoutError : QNetworkReply::UnknownNetworkError; + emit failed(error); if (!retry) { ResourceCache::requestCompleted(this); From 750080fe66582ddf6aee15b42e5df5b2f5a19624 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 12:52:01 -0700 Subject: [PATCH 69/77] Remove early return in Model::restoreJointPosition --- libraries/render-utils/src/Model.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index b4f3949a93..9c6dcdfc9c 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -1356,7 +1356,6 @@ void Model::inverseKinematics(int endIndex, glm::vec3 targetPosition, const glm: bool Model::restoreJointPosition(int jointIndex, float fraction, float priority) { QString url = _url.path(); const FBXGeometry& geometry = _geometry->getFBXGeometry(); - return true; const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; return _rig->restoreJointPosition(jointIndex, fraction, priority, freeLineage); } From 33d6f561eb94238b31b5436990057eacae75a797 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 12:52:11 -0700 Subject: [PATCH 70/77] Reintroduce request limit in ResourceCache --- libraries/networking/src/ResourceCache.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 7ed5866707..e4ff88b9e3 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -154,12 +154,12 @@ void ResourceCache::clearUnusedResource() { void ResourceCache::attemptRequest(Resource* resource) { auto sharedItems = DependencyManager::get(); - // if (_requestLimit <= 0) { - // qDebug() << "REQUEST LIMIT REACHED (" << _requestLimit << "), queueing: " << resource->getURL(); - // // wait until a slot becomes available - // sharedItems->_pendingRequests.append(resource); - // return; - // } + if (_requestLimit <= 0) { + qDebug() << "REQUEST LIMIT REACHED (" << _requestLimit << "), queueing: " << resource->getURL(); + // wait until a slot becomes available + sharedItems->_pendingRequests.append(resource); + return; + } qDebug() << "-- Decreasing limit for : " << resource->getURL(); _requestLimit--; sharedItems->_loadingRequests.append(resource); From 12d3cf557abe62c2ce1923f9f7cab0606bd457de Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 12:58:57 -0700 Subject: [PATCH 71/77] Remove assert in ScriptCache --- libraries/script-engine/src/ScriptCache.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/script-engine/src/ScriptCache.cpp b/libraries/script-engine/src/ScriptCache.cpp index 9409c10652..e3d12313dc 100644 --- a/libraries/script-engine/src/ScriptCache.cpp +++ b/libraries/script-engine/src/ScriptCache.cpp @@ -27,8 +27,6 @@ ScriptCache::ScriptCache(QObject* parent) { } QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending, bool reload) { - //assert(!_scriptCache.contains(url) || !reload); - QString scriptContents; if (_scriptCache.contains(url) && !reload) { qCDebug(scriptengine) << "Found script in cache:" << url.toString(); From b401a62d7f006feb7a5d98f4df5e2d4ae89a8c11 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 13:57:00 -0700 Subject: [PATCH 72/77] Disable request limiting for ATP requests in ResourceCache --- libraries/networking/src/ResourceCache.cpp | 10 ++++++++-- libraries/networking/src/ResourceManager.cpp | 6 ------ libraries/networking/src/ResourceManager.h | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index e4ff88b9e3..db506e5320 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -161,7 +161,11 @@ void ResourceCache::attemptRequest(Resource* resource) { return; } qDebug() << "-- Decreasing limit for : " << resource->getURL(); - _requestLimit--; + + // Disable request limiting for ATP + if (resource->getURL() != URL_SCHEME_ATP) { + _requestLimit--; + } sharedItems->_loadingRequests.append(resource); resource->makeRequest(); } @@ -171,7 +175,9 @@ void ResourceCache::requestCompleted(Resource* resource) { auto sharedItems = DependencyManager::get(); sharedItems->_loadingRequests.removeOne(resource); qDebug() << "++ Increasing limit after finished: " << resource->getURL(); - _requestLimit++; + if (resource->getURL() != URL_SCHEME_ATP) { + _requestLimit++; + } // look for the highest priority pending request int highestIndex = -1; diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index b8e3eeeae7..648ce8376e 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -16,12 +16,6 @@ #include -const QString URL_SCHEME_FILE = "file"; -const QString URL_SCHEME_HTTP = "http"; -const QString URL_SCHEME_HTTPS = "https"; -const QString URL_SCHEME_FTP = "ftp"; -const QString URL_SCHEME_ATP = "atp"; - ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const QUrl& url) { auto scheme = url.scheme(); if (scheme == URL_SCHEME_FILE) { diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h index 1cb6b08a79..e31577423b 100644 --- a/libraries/networking/src/ResourceManager.h +++ b/libraries/networking/src/ResourceManager.h @@ -15,6 +15,12 @@ #include "ResourceRequest.h" +const QString URL_SCHEME_FILE = "file"; +const QString URL_SCHEME_HTTP = "http"; +const QString URL_SCHEME_HTTPS = "https"; +const QString URL_SCHEME_FTP = "ftp"; +const QString URL_SCHEME_ATP = "atp"; + class ResourceManager { public: static ResourceRequest* createResourceRequest(QObject* parent, const QUrl& url); From 4fcfd5b8f8379b3868e9f32282778cc3ceb004ac Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 13:58:28 -0700 Subject: [PATCH 73/77] Fix indentation --- libraries/networking/src/udt/PacketList.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 98316ee0ab..7b90276b62 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -143,7 +143,7 @@ qint64 PacketList::writeData(const char* data, qint64 maxSize) { // this is an unsupported case - the segment is bigger than the size of an individual packet // but the PacketList is not going to be sent ordered qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is" - << "larger than the payload size."; + << "larger than the payload size."; Q_ASSERT(false); } From 24eef654e0e64f754956eda0d4bf4cc968695e2e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 14:11:29 -0700 Subject: [PATCH 74/77] Remove unnecessary debug --- libraries/render-utils/src/TextureCache.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/render-utils/src/TextureCache.cpp b/libraries/render-utils/src/TextureCache.cpp index 1fba515cbe..4954629d86 100644 --- a/libraries/render-utils/src/TextureCache.cpp +++ b/libraries/render-utils/src/TextureCache.cpp @@ -294,7 +294,6 @@ public: void ImageReader::run() { QSharedPointer texture = _texture.toStrongRef(); if (texture.isNull()) { - qDebug() << "TEXTURE IS NULL"; return; } From 755c33e5516af0fb021fa0c35890a5f291e22ecc Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 14:11:40 -0700 Subject: [PATCH 75/77] Remove unused variable --- libraries/render-utils/src/Model.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/render-utils/src/Model.cpp b/libraries/render-utils/src/Model.cpp index 9c6dcdfc9c..c2d723a323 100644 --- a/libraries/render-utils/src/Model.cpp +++ b/libraries/render-utils/src/Model.cpp @@ -1354,7 +1354,6 @@ void Model::inverseKinematics(int endIndex, glm::vec3 targetPosition, const glm: } bool Model::restoreJointPosition(int jointIndex, float fraction, float priority) { - QString url = _url.path(); const FBXGeometry& geometry = _geometry->getFBXGeometry(); const QVector& freeLineage = geometry.joints.at(jointIndex).freeLineage; return _rig->restoreJointPosition(jointIndex, fraction, priority, freeLineage); From 5aa5bca8f7d566b42d48faba7e365c6a947059bf Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 14:29:54 -0700 Subject: [PATCH 76/77] Add path to headers --- assignment-client/src/assets/AssetServer.cpp | 1 + assignment-client/src/assets/AssetServer.h | 1 + assignment-client/src/assets/SendAssetTask.cpp | 1 + assignment-client/src/assets/SendAssetTask.h | 1 + libraries/networking/src/AssetClient.cpp | 1 + libraries/networking/src/AssetClient.h | 1 + libraries/networking/src/AssetRequest.cpp | 1 + libraries/networking/src/AssetRequest.h | 1 + libraries/networking/src/AssetResourceRequest.cpp | 1 + libraries/networking/src/AssetResourceRequest.h | 1 + libraries/networking/src/AssetUtils.h | 1 + libraries/networking/src/FileResourceRequest.cpp | 1 + libraries/networking/src/FileResourceRequest.h | 1 + libraries/networking/src/HTTPResourceRequest.cpp | 1 + libraries/networking/src/HTTPResourceRequest.h | 1 + libraries/networking/src/ResourceManager.cpp | 1 + libraries/networking/src/ResourceManager.h | 1 + libraries/networking/src/ResourceRequest.cpp | 1 + libraries/networking/src/ResourceRequest.h | 1 + 19 files changed, 19 insertions(+) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c6ad9f0f4d..a5ad2962f8 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -1,5 +1,6 @@ // // AssetServer.cpp +// assignment-client/src/assets // // Created by Ryan Huffman on 2015/07/21 // Copyright 2015 High Fidelity, Inc. diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 07e0680c4f..e214be1eda 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -1,5 +1,6 @@ // // AssetServer.h +// assignment-client/src/assets // // Created by Ryan Huffman on 2015/07/21 // Copyright 2015 High Fidelity, Inc. diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index f30689b042..8516218681 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -1,5 +1,6 @@ // // SendAssetTask.cpp +// assignment-client/src/assets // // Created by Ryan Huffman on 2015/08/26 // Copyright 2015 High Fidelity, Inc. diff --git a/assignment-client/src/assets/SendAssetTask.h b/assignment-client/src/assets/SendAssetTask.h index 7dd3616f83..6b6c555326 100644 --- a/assignment-client/src/assets/SendAssetTask.h +++ b/assignment-client/src/assets/SendAssetTask.h @@ -1,5 +1,6 @@ // // SendAssetTask.h +// assignment-client/src/assets // // Created by Ryan Huffman on 2015/08/26 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index f424171766..fa277d72f4 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -1,5 +1,6 @@ // // AssetClient.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/21 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 0e81c6cfa2..a397c500e3 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -1,5 +1,6 @@ // // AssetClient.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/21 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 87e39e1edd..c49d178412 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -1,5 +1,6 @@ // // AssetRequest.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/24 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index d1cc9c9b33..6e0738c333 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -1,5 +1,6 @@ // // AssetRequest.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/24 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 5a0af8ef1f..bdb6b51378 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -1,5 +1,6 @@ // // AssetResourceRequest.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetResourceRequest.h b/libraries/networking/src/AssetResourceRequest.h index 2cfd1d28b7..fb9c25e092 100644 --- a/libraries/networking/src/AssetResourceRequest.h +++ b/libraries/networking/src/AssetResourceRequest.h @@ -1,5 +1,6 @@ // // AssetResourceRequest.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index cc5b2a281f..4018b5c96b 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -1,5 +1,6 @@ // // AssetUtils.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/30 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/FileResourceRequest.cpp b/libraries/networking/src/FileResourceRequest.cpp index 8bd7ecf94e..d8b8d962f5 100644 --- a/libraries/networking/src/FileResourceRequest.cpp +++ b/libraries/networking/src/FileResourceRequest.cpp @@ -1,5 +1,6 @@ // // FileResourceRequest.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/FileResourceRequest.h b/libraries/networking/src/FileResourceRequest.h index 7618311626..4ff0d2ecf2 100644 --- a/libraries/networking/src/FileResourceRequest.h +++ b/libraries/networking/src/FileResourceRequest.h @@ -1,5 +1,6 @@ // // FileResourceRequest.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index cf074eb98b..22f55cd641 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -1,5 +1,6 @@ // // HTTPResourceRequest.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/HTTPResourceRequest.h b/libraries/networking/src/HTTPResourceRequest.h index c42204f3e9..09c94314d6 100644 --- a/libraries/networking/src/HTTPResourceRequest.h +++ b/libraries/networking/src/HTTPResourceRequest.h @@ -1,5 +1,6 @@ // // HTTPResourceRequest.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index 648ce8376e..ded3dfe222 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -1,5 +1,6 @@ // // ResourceManager.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h index e31577423b..3748036c8e 100644 --- a/libraries/networking/src/ResourceManager.h +++ b/libraries/networking/src/ResourceManager.h @@ -1,5 +1,6 @@ // // ResourceManager.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/ResourceRequest.cpp b/libraries/networking/src/ResourceRequest.cpp index 44bfaae254..d56033670b 100644 --- a/libraries/networking/src/ResourceRequest.cpp +++ b/libraries/networking/src/ResourceRequest.cpp @@ -1,5 +1,6 @@ // // ResourceRequest.cpp +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 458a859d16..ac13523e96 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -1,5 +1,6 @@ // // ResourceRequest.h +// libraries/networking/src // // Created by Ryan Huffman on 2015/07/23 // Copyright 2015 High Fidelity, Inc. From dabea21d669920b2d0d99ee3e71ddb7e9ba6027f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 27 Aug 2015 14:30:48 -0700 Subject: [PATCH 77/77] Remove unused variable --- libraries/networking/src/ResourceCache.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index db506e5320..b07c3cc60e 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -364,7 +364,6 @@ void Resource::handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTota } void Resource::handleReplyFinished() { - QString u = _url.path(); Q_ASSERT(_request); auto result = _request->getResult(); @@ -387,8 +386,7 @@ void Resource::handleReplyFinished() { _request = nullptr; if (result == ResourceRequest::Result::TIMEOUT) { - qDebug() << "Timed out loading" << _url << - "received" << _bytesReceived << "total" << _bytesTotal; + qDebug() << "Timed out loading" << _url << "received" << _bytesReceived << "total" << _bytesTotal; } else { qDebug() << "Error loading " << _url; }