diff --git a/assignment-client/src/AssignmentFactory.cpp b/assignment-client/src/AssignmentFactory.cpp index c6ad2c9f09..cacc523ebd 100644 --- a/assignment-client/src/AssignmentFactory.cpp +++ b/assignment-client/src/AssignmentFactory.cpp @@ -16,6 +16,7 @@ #include "audio/AudioMixer.h" #include "avatars/AvatarMixer.h" #include "entities/EntityServer.h" +#include "assets/AssetServer.h" ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { @@ -33,6 +34,8 @@ ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { return new Agent(packet); case Assignment::EntityServerType: return new EntityServer(packet); + case Assignment::AssetServerType: + return new AssetServer(packet); default: return NULL; } diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp new file mode 100644 index 0000000000..a5ad2962f8 --- /dev/null +++ b/assignment-client/src/assets/AssetServer.cpp @@ -0,0 +1,202 @@ +// +// AssetServer.cpp +// assignment-client/src/assets +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#include "AssetServer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "NetworkLogging.h" +#include "NodeType.h" +#include "SendAssetTask.h" + +const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server"; + +AssetServer::AssetServer(NLPacket& packet) : + ThreadedAssignment(packet), + _taskPool(this) +{ + + // Most of the work will be I/O bound, reading from disk and constructing packet objects, + // so the ideal is greater than the number of cores on the system. + _taskPool.setMaxThreadCount(20); + + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); + packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); + packetReceiver.registerMessageListener(PacketType::AssetUpload, this, "handleAssetUpload"); +} + +void AssetServer::run() { + ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer); + + auto nodeList = DependencyManager::get(); + nodeList->addNodeTypeToInterestSet(NodeType::Agent); + + _resourcesDirectory = QDir(QCoreApplication::applicationDirPath()).filePath("resources/assets"); + if (!_resourcesDirectory.exists()) { + qDebug() << "Creating resources directory"; + _resourcesDirectory.mkpath("."); + } + qDebug() << "Serving files from: " << _resourcesDirectory.path(); + + // Scan for new files + qDebug() << "Looking for new files in asset directory"; + auto files = _resourcesDirectory.entryInfoList(QDir::Files); + QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" }; + for (const auto& fileInfo : files) { + auto filename = fileInfo.fileName(); + if (!filenameRegex.exactMatch(filename)) { + qDebug() << "Found file: " << filename; + if (!fileInfo.isReadable()) { + qDebug() << "\tCan't open file for reading: " << filename; + continue; + } + + // Read file + QFile file { fileInfo.absoluteFilePath() }; + file.open(QFile::ReadOnly); + QByteArray data = file.readAll(); + + auto hash = hashData(data); + + qDebug() << "\tMoving " << filename << " to " << hash; + + file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix()); + } + } +} + +void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode) { + QByteArray assetHash; + MessageID messageID; + uint8_t extensionLength; + + if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) { + qDebug() << "ERROR bad file request"; + return; + } + + packet->readPrimitive(&messageID); + assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH); + packet->readPrimitive(&extensionLength); + QByteArray extension = packet->read(extensionLength); + + auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply); + + replyPacket->writePrimitive(messageID); + replyPacket->write(assetHash); + + QString fileName = QString(assetHash) + "." + extension; + QFileInfo fileInfo { _resourcesDirectory.filePath(fileName) }; + + if (fileInfo.exists() && fileInfo.isReadable()) { + qDebug() << "Opening file: " << fileInfo.filePath(); + replyPacket->writePrimitive(AssetServerError::NO_ERROR); + replyPacket->writePrimitive(fileInfo.size()); + } else { + qDebug() << "Asset not found: " << assetHash; + replyPacket->writePrimitive(AssetServerError::ASSET_NOT_FOUND); + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *senderNode); +} + +void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + QByteArray assetHash; + uint8_t extensionLength; + DataOffset start; + DataOffset end; + + auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end)); + if (packet->getPayloadSize() < minSize) { + qDebug() << "ERROR bad file request"; + return; + } + + packet->readPrimitive(&messageID); + assetHash = packet->read(HASH_HEX_LENGTH); + packet->readPrimitive(&extensionLength); + QByteArray extension = packet->read(extensionLength); + packet->readPrimitive(&start); + packet->readPrimitive(&end); + + qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end; + + // Queue task + QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension)); + auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode); + _taskPool.start(task); +} + +void AssetServer::handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode) { + auto data = packetList->getMessage(); + QBuffer buffer { &data }; + buffer.open(QIODevice::ReadOnly); + + MessageID messageID; + buffer.read(reinterpret_cast(&messageID), sizeof(messageID)); + + uint8_t extensionLength; + buffer.read(reinterpret_cast(&extensionLength), sizeof(extensionLength)); + + QByteArray extension = buffer.read(extensionLength); + + qDebug() << "Got extension: " << extension; + + uint64_t fileSize; + buffer.read(reinterpret_cast(&fileSize), sizeof(fileSize)); + + qDebug() << "Receiving a file of size " << fileSize; + + auto replyPacket = NLPacket::create(PacketType::AssetUploadReply); + replyPacket->writePrimitive(messageID); + + if (fileSize > MAX_UPLOAD_SIZE) { + replyPacket->writePrimitive(AssetServerError::ASSET_TOO_LARGE); + } else { + QByteArray fileData = buffer.read(fileSize); + + QString hash = hashData(fileData); + + qDebug() << "Got data: (" << hash << ") "; + + QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) }; + + if (file.exists()) { + qDebug() << "[WARNING] This file already exists: " << hash; + } else { + file.open(QIODevice::WriteOnly); + file.write(fileData); + file.close(); + } + replyPacket->writePrimitive(AssetServerError::NO_ERROR); + replyPacket->write(hash.toLatin1()); + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *senderNode); +} + +QString AssetServer::hashData(const QByteArray& data) { + return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex()); +} + diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h new file mode 100644 index 0000000000..e214be1eda --- /dev/null +++ b/assignment-client/src/assets/AssetServer.h @@ -0,0 +1,47 @@ +// +// AssetServer.h +// assignment-client/src/assets +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetServer_h +#define hifi_AssetServer_h + +#include + +#include +#include + +#include "AssetUtils.h" + +class AssetServer : public ThreadedAssignment { + Q_OBJECT +public: + AssetServer(NLPacket& packet); + + static QString hashData(const QByteArray& data); + +public slots: + void run(); + +private slots: + void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode); + +private: + static void writeError(NLPacketList* packetList, AssetServerError error); + QDir _resourcesDirectory; + QThreadPool _taskPool; +}; + +inline void writeError(NLPacketList* packetList, AssetServerError error) { + packetList->writePrimitive(error); +} + +#endif diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp new file mode 100644 index 0000000000..8516218681 --- /dev/null +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -0,0 +1,70 @@ +// +// SendAssetTask.cpp +// assignment-client/src/assets +// +// Created by Ryan Huffman on 2015/08/26 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SendAssetTask.h" + +#include + +#include +#include +#include +#include +#include + +#include "AssetUtils.h" + +SendAssetTask::SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode) : + QRunnable(), + _messageID(messageID), + _assetHash(assetHash), + _filePath(filePath), + _start(start), + _end(end), + _sendToNode(sendToNode) +{ +} + +void SendAssetTask::run() { + qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID; + auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); + + replyPacketList->write(_assetHash); + + replyPacketList->writePrimitive(_messageID); + + if (_end <= _start) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + } else { + QFile file { _filePath }; + + if (file.open(QIODevice::ReadOnly)) { + if (file.size() < _end) { + writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); + qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end; + } else { + auto size = _end - _start; + file.seek(_start); + replyPacketList->writePrimitive(AssetServerError::NO_ERROR); + replyPacketList->writePrimitive(size); + replyPacketList->write(file.read(size)); + qCDebug(networking) << "Sending asset: " << _assetHash; + } + file.close(); + } else { + qCDebug(networking) << "Asset not found: " << _filePath << "(" << _assetHash << ")"; + writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND); + } + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(replyPacketList), *_sendToNode); +} diff --git a/assignment-client/src/assets/SendAssetTask.h b/assignment-client/src/assets/SendAssetTask.h new file mode 100644 index 0000000000..6b6c555326 --- /dev/null +++ b/assignment-client/src/assets/SendAssetTask.h @@ -0,0 +1,42 @@ +// +// SendAssetTask.h +// assignment-client/src/assets +// +// Created by Ryan Huffman on 2015/08/26 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SendAssetTask_h +#define hifi_SendAssetTask_h + +#include +#include +#include + +#include "AssetUtils.h" +#include "AssetServer.h" +#include "Node.h" + +class SendAssetTask : public QRunnable { +public: + SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end, + const SharedNodePointer& sendToNode); + + void run(); + +signals: + void finished(); + +private: + MessageID _messageID; + QByteArray _assetHash; + QString _filePath; + DataOffset _start; + DataOffset _end; + SharedNodePointer _sendToNode; +}; + +#endif diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index e0bd874385..52b20d721f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -573,7 +573,6 @@ void DomainServer::populateDefaultStaticAssignmentsExcludingTypes(const QSet #include +#include #include #include #include @@ -297,6 +298,7 @@ bool setupEssentials(int& argc, char** argv) { auto autoUpdater = DependencyManager::set(); auto pathUtils = DependencyManager::set(); auto actionFactory = DependencyManager::set(); + auto assetClient = DependencyManager::set(); auto userInputMapper = DependencyManager::set(); return true; @@ -444,6 +446,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : audioThread->start(); + + QThread* assetThread = new QThread(); + + assetThread->setObjectName("Asset Thread"); + auto assetClient = DependencyManager::get(); + + assetClient->moveToThread(assetThread); + + assetThread->start(); + + const DomainHandler& domainHandler = nodeList->getDomainHandler(); connect(&domainHandler, SIGNAL(hostnameChanged(const QString&)), SLOT(domainChanged(const QString&))); @@ -513,7 +526,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : // tell the NodeList instance who to tell the domain server we care about nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer - << NodeType::EntityServer); + << NodeType::EntityServer << NodeType::AssetServer); // connect to the packet sent signal of the _entityEditSender connect(&_entityEditSender, &EntityEditPacketSender::packetSent, this, &Application::packetSent); @@ -2010,6 +2023,7 @@ void Application::sendPingPackets() { case NodeType::AvatarMixer: case NodeType::AudioMixer: case NodeType::EntityServer: + case NodeType::AssetServer: return true; default: return false; diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 1c2cdbaa26..3305da5ba7 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -22,6 +22,7 @@ #include #include +#include #include "Application.h" #include "AccountManager.h" @@ -89,6 +90,36 @@ Menu::Menu() { addActionToQMenuAndActionHash(fileMenu, MenuOption::RunningScripts, Qt::CTRL | Qt::Key_J, qApp, SLOT(toggleRunningScriptsWidget())); + // Asset uploading + { + auto action = new QAction("Upload File", fileMenu); + fileMenu->addAction(action); + action->setMenuRole(QAction::NoRole); + _actionHash.insert("Upload File", action); + + connect(action, &QAction::triggered, [this](bool checked) { + qDebug() << "Clicked upload file"; + auto filename = QFileDialog::getOpenFileUrl(nullptr, "Select a file to upload"); + if (!filename.isEmpty()) { + qDebug() << "Selected: " << filename; + QFile file { filename.path() }; + if (file.open(QIODevice::ReadOnly)) { + QFileInfo fileInfo { filename.path() }; + auto extension = fileInfo.suffix(); + auto data = file.readAll(); + auto assetClient = DependencyManager::get(); + assetClient->uploadAsset(data, extension, [this, extension](bool result, QString hash) mutable { + if (result) { + QMessageBox::information(this, "Upload Successful", "URL: apt:/" + hash + "." + extension); + } else { + QMessageBox::warning(this, "Upload Failed", "There was an error uploading the file."); + } + }); + } + } + }); + } + auto addressManager = DependencyManager::get(); addDisabledActionAndSeparator(fileMenu, "History"); diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 048f0d0ef9..465b24ff80 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -126,8 +126,10 @@ void Stats::updateStats() { if (Menu::getInstance()->isOptionChecked(MenuOption::TestPing)) { SharedNodePointer audioMixerNode = nodeList->soloNodeOfType(NodeType::AudioMixer); SharedNodePointer avatarMixerNode = nodeList->soloNodeOfType(NodeType::AvatarMixer); + SharedNodePointer assetServerNode = nodeList->soloNodeOfType(NodeType::AssetServer); STAT_UPDATE(audioPing, audioMixerNode ? audioMixerNode->getPingMs() : -1); STAT_UPDATE(avatarPing, avatarMixerNode ? avatarMixerNode->getPingMs() : -1); + STAT_UPDATE(assetPing, assetServerNode ? assetServerNode->getPingMs() : -1); //// Now handle entity servers, since there could be more than one, we average their ping times int totalPingOctree = 0; diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 4f0619d9c8..096469a84d 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -39,6 +39,7 @@ class Stats : public QQuickItem { STATS_PROPERTY(int, audioPing, 0) STATS_PROPERTY(int, avatarPing, 0) STATS_PROPERTY(int, entitiesPing, 0) + STATS_PROPERTY(int, assetPing, 0) STATS_PROPERTY(QVector3D, position, QVector3D(0, 0, 0) ) STATS_PROPERTY(float, velocity, 0) STATS_PROPERTY(float, yaw, 0) @@ -105,6 +106,7 @@ signals: void audioPingChanged(); void avatarPingChanged(); void entitiesPingChanged(); + void assetPingChanged(); void positionChanged(); void velocityChanged(); void yawChanged(); diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index e7a4fb50a3..dfe1f2cbed 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -39,14 +39,16 @@ QSharedPointer AnimationCache::createResource(const QUrl& url, const Q return QSharedPointer(new Animation(url), &Resource::allReferencesCleared); } -AnimationReader::AnimationReader(const QUrl& url, QNetworkReply* reply) : +Animation::Animation(const QUrl& url) : Resource(url) {} + +AnimationReader::AnimationReader(const QUrl& url, const QByteArray& data) : _url(url), - _reply(reply) { + _data(data) { } void AnimationReader::run() { try { - if (!_reply) { + if (_data.isEmpty()) { throw QString("Reply is NULL ?!"); } QString urlname = _url.path().toLower(); @@ -58,7 +60,7 @@ void AnimationReader::run() { // Parse the FBX directly from the QNetworkReply FBXGeometry* fbxgeo = nullptr; if (_url.path().toLower().endsWith(".fbx")) { - fbxgeo = readFBX(_reply, QVariantHash(), _url.path()); + fbxgeo = readFBX(_data, QVariantHash(), _url.path()); } else { QString errorStr("usupported format"); emit onError(299, errorStr); @@ -71,11 +73,8 @@ void AnimationReader::run() { } catch (const QString& error) { emit onError(299, error); } - _reply->deleteLater(); } -Animation::Animation(const QUrl& url) : Resource(url) {} - bool Animation::isLoaded() const { return _loaded && _geometry; } @@ -108,9 +107,9 @@ const QVector& Animation::getFramesReference() const { return _geometry->animationFrames; } -void Animation::downloadFinished(QNetworkReply* reply) { +void Animation::downloadFinished(const QByteArray& data) { // parse the animation/fbx file on a background thread. - AnimationReader* animationReader = new AnimationReader(reply->url(), reply); + AnimationReader* animationReader = new AnimationReader(_url, data); connect(animationReader, SIGNAL(onSuccess(FBXGeometry*)), SLOT(animationParseSuccess(FBXGeometry*))); connect(animationReader, SIGNAL(onError(int, QString)), SLOT(animationParseError(int, QString))); QThreadPool::globalInstance()->start(animationReader); diff --git a/libraries/animation/src/AnimationCache.h b/libraries/animation/src/AnimationCache.h index af07eda9a4..50184862eb 100644 --- a/libraries/animation/src/AnimationCache.h +++ b/libraries/animation/src/AnimationCache.h @@ -65,7 +65,7 @@ public: const QVector& getFramesReference() const; protected: - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; protected slots: void animationParseSuccess(FBXGeometry* geometry); @@ -81,7 +81,7 @@ class AnimationReader : public QObject, public QRunnable { Q_OBJECT public: - AnimationReader(const QUrl& url, QNetworkReply* reply); + AnimationReader(const QUrl& url, const QByteArray& data); virtual void run(); signals: @@ -90,7 +90,7 @@ signals: private: QUrl _url; - QNetworkReply* _reply; + QByteArray _data; }; class AnimationDetails { diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 7dc6010f8f..2457bda74a 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -56,16 +56,17 @@ Sound::Sound(const QUrl& url, bool isStereo) : } -void Sound::downloadFinished(QNetworkReply* reply) { +void Sound::downloadFinished(const QByteArray& data) { // replace our byte array with the downloaded data - QByteArray rawAudioByteArray = reply->readAll(); - QString fileName = reply->url().fileName(); + QByteArray rawAudioByteArray = QByteArray(data); + QString fileName = getURL().fileName(); const QString WAV_EXTENSION = ".wav"; - if (reply->hasRawHeader("Content-Type") || fileName.endsWith(WAV_EXTENSION)) { + if (fileName.endsWith(WAV_EXTENSION)) { - QByteArray headerContentType = reply->rawHeader("Content-Type"); + QString headerContentType = "audio/x-wav"; + //QByteArray headerContentType = reply->rawHeader("Content-Type"); // WAV audio file encountered if (headerContentType == "audio/x-wav" @@ -80,9 +81,9 @@ void Sound::downloadFinished(QNetworkReply* reply) { } else { // check if this was a stereo raw file // since it's raw the only way for us to know that is if the file was called .stereo.raw - if (reply->url().fileName().toLower().endsWith("stereo.raw")) { + if (fileName.toLower().endsWith("stereo.raw")) { _isStereo = true; - qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << reply->url() << "as stereo audio file."; + qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << getURL() << "as stereo audio file."; } // Process as RAW file @@ -94,7 +95,6 @@ void Sound::downloadFinished(QNetworkReply* reply) { } _isReady = true; - reply->deleteLater(); } void Sound::downSample(const QByteArray& rawAudioByteArray) { diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 9aa92feea1..842c395a7d 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -39,7 +39,7 @@ private: void downSample(const QByteArray& rawAudioByteArray); void interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; }; typedef QSharedPointer SharedSoundPointer; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp new file mode 100644 index 0000000000..fa277d72f4 --- /dev/null +++ b/libraries/networking/src/AssetClient.cpp @@ -0,0 +1,217 @@ +// +// AssetClient.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetClient.h" + +#include +#include + +#include "AssetRequest.h" +#include "NodeList.h" +#include "PacketReceiver.h" +#include "AssetUtils.h" + +MessageID AssetClient::_currentID = 0; + + +AssetClient::AssetClient() { + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); + packetReceiver.registerMessageListener(PacketType::AssetGetReply, this, "handleAssetGetReply"); + packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); +} + +AssetRequest* AssetClient::create(QString hash, QString extension) { + if (QThread::currentThread() != thread()) { + AssetRequest* req; + QMetaObject::invokeMethod(this, "create", + Qt::BlockingQueuedConnection, + Q_RETURN_ARG(AssetRequest*, req), + Q_ARG(QString, hash), + Q_ARG(QString, extension)); + return req; + } + + if (hash.length() != HASH_HEX_LENGTH) { + qDebug() << "Invalid hash size"; + return nullptr; + } + + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto assetClient = DependencyManager::get(); + auto request = new AssetRequest(assetClient.data(), hash, extension); + + return request; + } + + return nullptr; +} + +bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { + if (hash.length() != HASH_HEX_LENGTH) { + qDebug() << "Invalid hash size"; + return false; + } + + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packet = NLPacket::create(PacketType::AssetGet); + + auto messageID = ++_currentID; + packet->writePrimitive(messageID); + + packet->write(hash.toLatin1()); + + packet->writePrimitive(uint8_t(extension.length())); + packet->write(extension.toLatin1()); + + packet->writePrimitive(start); + packet->writePrimitive(end); + + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingRequests[messageID] = callback; + + return true; + } + + return false; +} + +bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packet = NLPacket::create(PacketType::AssetGetInfo); + + auto messageID = ++_currentID; + packet->writePrimitive(messageID); + packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + packet->writePrimitive(uint8_t(extension.length())); + packet->write(extension.toLatin1()); + + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingInfoRequests[messageID] = callback; + + return true; + } + + return false; +} + +void AssetClient::handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + packet->readPrimitive(&messageID); + auto assetHash = QString(packet->read(HASH_HEX_LENGTH)); + + AssetServerError error; + packet->readPrimitive(&error); + + AssetInfo info { assetHash, 0 }; + + if (error == NO_ERROR) { + packet->readPrimitive(&info.size); + } + + if (_pendingInfoRequests.contains(messageID)) { + auto callback = _pendingInfoRequests.take(messageID); + callback(error == NO_ERROR, info); + } +} + +void AssetClient::handleAssetGetReply(QSharedPointer packetList, SharedNodePointer senderNode) { + QByteArray data = packetList->getMessage(); + QBuffer packet { &data }; + packet.open(QIODevice::ReadOnly); + + auto assetHash = packet.read(HASH_HEX_LENGTH); + qDebug() << "Got reply for asset: " << assetHash; + + MessageID messageID; + packet.read(reinterpret_cast(&messageID), sizeof(messageID)); + + AssetServerError error; + packet.read(reinterpret_cast(&error), sizeof(AssetServerError)); + QByteArray assetData; + + if (!error) { + DataOffset length; + packet.read(reinterpret_cast(&length), sizeof(DataOffset)); + data = packet.read(length); + } else { + qDebug() << "Failure getting asset: " << error; + } + + if (_pendingRequests.contains(messageID)) { + auto callback = _pendingRequests.take(messageID); + callback(error == NO_ERROR, data); + } +} + +bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + if (assetServer) { + auto packetList = std::unique_ptr(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true)); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(static_cast(extension.length())); + packetList->write(extension.toLatin1().constData(), extension.length()); + + qDebug() << "Extension length: " << extension.length(); + qDebug() << "Extension: " << extension; + + uint64_t size = data.length(); + packetList->writePrimitive(size); + packetList->write(data.constData(), size); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + + _pendingUploads[messageID] = callback; + + return true; + } + return false; +} + +void AssetClient::handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + packet->readPrimitive(&messageID); + + AssetServerError error; + packet->readPrimitive(&error); + + QString hashString { "" }; + + if (error) { + qDebug() << "Error uploading file to asset server"; + } else { + auto hashData = packet->read(HASH_HEX_LENGTH); + + hashString = QString(hashData); + + qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; + } + + if (_pendingUploads.contains(messageID)) { + auto callback = _pendingUploads.take(messageID); + callback(error == NO_ERROR, hashString); + } +} diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h new file mode 100644 index 0000000000..a397c500e3 --- /dev/null +++ b/libraries/networking/src/AssetClient.h @@ -0,0 +1,61 @@ +// +// AssetClient.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/21 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#ifndef hifi_AssetClient_h +#define hifi_AssetClient_h + +#include + +#include + +#include "AssetUtils.h" +#include "LimitedNodeList.h" +#include "NLPacket.h" + +class AssetRequest; + +struct AssetInfo { + QString hash; + int64_t size; +}; + +using ReceivedAssetCallback = std::function; +using GetInfoCallback = std::function; +using UploadResultCallback = std::function; + +class AssetClient : public QObject, public Dependency { + Q_OBJECT +public: + AssetClient(); + + Q_INVOKABLE AssetRequest* create(QString hash, QString extension); + +private slots: + void handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode); + void handleAssetGetReply(QSharedPointer packetList, SharedNodePointer senderNode); + void handleAssetUploadReply(QSharedPointer packet, SharedNodePointer senderNode); + +private: + friend class AssetRequest; + friend class Menu; + + bool getAssetInfo(QString hash, QString extension, GetInfoCallback callback); + bool getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback); + bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback); + + static MessageID _currentID; + QHash _pendingRequests; + QHash _pendingInfoRequests; + QHash _pendingUploads; +}; + +#endif diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp new file mode 100644 index 0000000000..c49d178412 --- /dev/null +++ b/libraries/networking/src/AssetRequest.cpp @@ -0,0 +1,75 @@ +// +// AssetRequest.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/24 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetRequest.h" + +#include + +#include + +#include "AssetClient.h" +#include "NodeList.h" + + +AssetRequest::AssetRequest(QObject* parent, QString hash, QString extension) : + QObject(parent), + _hash(hash), + _extension(extension) +{ +} + +void AssetRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + return; + } + + if (_state == NOT_STARTED) { + _state = WAITING_FOR_INFO; + + auto assetClient = DependencyManager::get(); + assetClient->getAssetInfo(_hash, _extension, [this](bool success, AssetInfo info) { + _info = info; + _data.resize(info.size); + const DataOffset CHUNK_SIZE = 1024000000; + + qDebug() << "Got size of " << _hash << " : " << info.size << " bytes"; + + // Round up + int numChunks = (info.size + CHUNK_SIZE - 1) / CHUNK_SIZE; + auto assetClient = DependencyManager::get(); + for (int i = 0; i < numChunks; ++i) { + ++_numPendingRequests; + auto start = i * CHUNK_SIZE; + auto end = std::min((i + 1) * CHUNK_SIZE, info.size); + assetClient->getAsset(_hash, _extension, start, end, [this, start, end](bool success, QByteArray data) { + Q_ASSERT(data.size() == (end - start)); + + if (success) { + _result = Success; + memcpy((_data.data() + start), data.constData(), end - start); + _totalReceived += data.size(); + emit progress(_totalReceived, _info.size); + } else { + _result = Error; + qDebug() << "Got error retrieving asset"; + } + + --_numPendingRequests; + if (_numPendingRequests == 0) { + _state = FINISHED; + emit finished(this); + } + }); + } + }); + } +} diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h new file mode 100644 index 0000000000..6e0738c333 --- /dev/null +++ b/libraries/networking/src/AssetRequest.h @@ -0,0 +1,61 @@ +// +// AssetRequest.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/24 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetRequest_h +#define hifi_AssetRequest_h + +#include +#include +#include + +#include "AssetClient.h" + +#include "AssetUtils.h" + +class AssetRequest : public QObject { + Q_OBJECT +public: + enum State { + NOT_STARTED = 0, + WAITING_FOR_INFO, + WAITING_FOR_DATA, + FINISHED + }; + + enum Result { + Success = 0, + Timeout, + NotFound, + Error, + }; + + AssetRequest(QObject* parent, QString hash, QString extension); + + Q_INVOKABLE void start(); + + const QByteArray& getData() { return _data; } + +signals: + void finished(AssetRequest*); + void progress(qint64 totalReceived, qint64 total); + +private: + State _state = NOT_STARTED; + Result _result; + AssetInfo _info; + uint64_t _totalReceived { 0 }; + QString _hash; + QString _extension; + QByteArray _data; + int _numPendingRequests { 0 }; +}; + +#endif diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp new file mode 100644 index 0000000000..bdb6b51378 --- /dev/null +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -0,0 +1,50 @@ +// +// AssetResourceRequest.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AssetResourceRequest.h" + +#include "AssetClient.h" +#include "AssetRequest.h" + +void AssetResourceRequest::doSend() { + // Make request to atp + auto assetClient = DependencyManager::get(); + auto parts = _url.path().split(".", QString::SkipEmptyParts); + auto hash = parts[0]; + auto extension = parts.length() > 1 ? parts[1] : ""; + + auto request = assetClient->create(hash, extension); + + if (!request) { + return; + } + + connect(request, &AssetRequest::progress, this, &AssetResourceRequest::progress); + QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable { + if (_state != IN_PROGRESS) return; + _state = FINISHED; + if (true) { + _data = req->getData(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + }); + + request->start(); +} + +void AssetResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { + qDebug() << "Got asset data: " << bytesReceived << " / " << bytesTotal; + emit progress(bytesReceived, bytesTotal); +} diff --git a/libraries/networking/src/AssetResourceRequest.h b/libraries/networking/src/AssetResourceRequest.h new file mode 100644 index 0000000000..fb9c25e092 --- /dev/null +++ b/libraries/networking/src/AssetResourceRequest.h @@ -0,0 +1,31 @@ +// +// AssetResourceRequest.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetResourceRequest_h +#define hifi_AssetResourceRequest_h + +#include + +#include "ResourceRequest.h" + +class AssetResourceRequest : public ResourceRequest { + Q_OBJECT +public: + AssetResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; + +private slots: + void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); +}; + +#endif diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h new file mode 100644 index 0000000000..4018b5c96b --- /dev/null +++ b/libraries/networking/src/AssetUtils.h @@ -0,0 +1,30 @@ +// +// AssetUtils.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/30 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AssetUtils_h +#define hifi_AssetUtils_h + +#include "NLPacketList.h" + +using MessageID = uint32_t; +using DataOffset = int64_t; + +const size_t HASH_HEX_LENGTH = 64; +const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB + +enum AssetServerError : uint8_t { + NO_ERROR = 0, + ASSET_NOT_FOUND, + INVALID_BYTE_RANGE, + ASSET_TOO_LARGE, +}; + +#endif diff --git a/libraries/networking/src/Assignment.cpp b/libraries/networking/src/Assignment.cpp index 71111d541d..36df79f65c 100644 --- a/libraries/networking/src/Assignment.cpp +++ b/libraries/networking/src/Assignment.cpp @@ -28,6 +28,8 @@ Assignment::Type Assignment::typeForNodeType(NodeType_t nodeType) { return Assignment::AgentType; case NodeType::EntityServer: return Assignment::EntityServerType; + case NodeType::AssetServer: + return Assignment::AssetServerType; default: return Assignment::AllTypes; } diff --git a/libraries/networking/src/Assignment.h b/libraries/networking/src/Assignment.h index 7f2d4beec4..ee3d9cb5fd 100644 --- a/libraries/networking/src/Assignment.h +++ b/libraries/networking/src/Assignment.h @@ -29,9 +29,9 @@ public: AudioMixerType = 0, AvatarMixerType = 1, AgentType = 2, - UNUSED_0 = 3, - UNUSED_1 = 4, - UNUSED_2 = 5, + AssetServerType = 3, + UNUSED_0 = 4, + UNUSED_1 = 5, EntityServerType = 6, AllTypes = 7 }; diff --git a/libraries/networking/src/FileResourceRequest.cpp b/libraries/networking/src/FileResourceRequest.cpp new file mode 100644 index 0000000000..d8b8d962f5 --- /dev/null +++ b/libraries/networking/src/FileResourceRequest.cpp @@ -0,0 +1,28 @@ +// +// FileResourceRequest.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "FileResourceRequest.h" + +#include + +void FileResourceRequest::doSend() { + QString filename = _url.toLocalFile(); + QFile file(filename); + _state = FINISHED; + if (file.open(QFile::ReadOnly)) { + _data = file.readAll(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } +} diff --git a/libraries/networking/src/FileResourceRequest.h b/libraries/networking/src/FileResourceRequest.h new file mode 100644 index 0000000000..4ff0d2ecf2 --- /dev/null +++ b/libraries/networking/src/FileResourceRequest.h @@ -0,0 +1,28 @@ +// +// FileResourceRequest.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_FileResourceRequest_h +#define hifi_FileResourceRequest_h + +#include + +#include "ResourceRequest.h" + +class FileResourceRequest : public ResourceRequest { + Q_OBJECT +public: + FileResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; +}; + +#endif diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp new file mode 100644 index 0000000000..22f55cd641 --- /dev/null +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -0,0 +1,96 @@ +// +// HTTPResourceRequest.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "HTTPResourceRequest.h" + +#include +#include +#include + +#include + +#include "NetworkAccessManager.h" +#include "NetworkLogging.h" + +HTTPResourceRequest::~HTTPResourceRequest() { + if (_reply) { + _reply->disconnect(this); + _reply->deleteLater(); + _reply = nullptr; + } +} + +void HTTPResourceRequest::doSend() { + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkRequest networkRequest = QNetworkRequest(_url); + networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + + if (_cacheEnabled) { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); + } else { + networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + } + + _reply = networkAccessManager.get(networkRequest); + + connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished); + + static const int TIMEOUT_MS = 10000; + + connect(&_sendTimer, &QTimer::timeout, this, &HTTPResourceRequest::onTimeout); + _sendTimer.setSingleShot(true); + _sendTimer.start(TIMEOUT_MS); +} + +void HTTPResourceRequest::onRequestFinished() { + Q_ASSERT(_state == IN_PROGRESS); + Q_ASSERT(_reply); + + _state = FINISHED; + + auto error = _reply->error(); + if (error == QNetworkReply::NoError) { + _data = _reply->readAll(); + _loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); + _result = ResourceRequest::SUCCESS; + emit finished(); + } else if (error == QNetworkReply::TimeoutError) { + _result = ResourceRequest::TIMEOUT; + emit finished(); + } else { + _result = ResourceRequest::ERROR; + emit finished(); + } + + _reply->deleteLater(); + _reply = nullptr; +} + +void HTTPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { + if (_state == IN_PROGRESS) { + // We've received data, so reset the timer + _sendTimer.start(); + } + + emit progress(bytesReceived, bytesTotal); +} + +void HTTPResourceRequest::onTimeout() { + Q_ASSERT(_state != UNSENT); + + if (_state == IN_PROGRESS) { + qCDebug(networking) << "Timed out loading " << _url; + _reply->abort(); + _state = FINISHED; + _result = TIMEOUT; + emit finished(); + } +} diff --git a/libraries/networking/src/HTTPResourceRequest.h b/libraries/networking/src/HTTPResourceRequest.h new file mode 100644 index 0000000000..09c94314d6 --- /dev/null +++ b/libraries/networking/src/HTTPResourceRequest.h @@ -0,0 +1,41 @@ +// +// HTTPResourceRequest.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_HTTPResourceRequest_h +#define hifi_HTTPResourceRequest_h + +#include +#include +#include + +#include "ResourceRequest.h" + +class HTTPResourceRequest : public ResourceRequest { + Q_OBJECT +public: + ~HTTPResourceRequest(); + + HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { } + +protected: + virtual void doSend() override; + +private slots: + void onTimeout(); + void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void onRequestFinished(); + +private: + QTimer _sendTimer; + QNetworkReply* _reply { nullptr }; +}; + +#endif diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 830224ff94..1bc5f10069 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -32,6 +32,7 @@ #include "HifiSockAddr.h" #include "UUID.h" #include "NetworkLogging.h" +#include "udt/Packet.h" const char SOLO_NODE_TYPES[2] = { NodeType::AvatarMixer, @@ -344,6 +345,19 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, return _nodeSocket.writePacketList(std::move(packetList), sockAddr); } +qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, const Node& destinationNode) { + // close the last packet in the list + packetList->closeCurrentPacket(); + + for (std::unique_ptr& packet : packetList->_packets) { + NLPacket* nlPacket = static_cast(packet.get()); + collectPacketStats(*nlPacket); + fillPacketHeader(*nlPacket, destinationNode.getConnectionSecret()); + } + + return _nodeSocket.writePacketList(std::move(packetList), *destinationNode.getActiveSocket()); +} + qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr) { // use the node's active socket as the destination socket if there is no overriden socket address @@ -531,7 +545,7 @@ unsigned int LimitedNodeList::broadcastToNodes(std::unique_ptr packet, return n; } -SharedNodePointer LimitedNodeList::soloNodeOfType(char nodeType) { +SharedNodePointer LimitedNodeList::soloNodeOfType(NodeType_t nodeType) { return nodeMatchingPredicate([&](const SharedNodePointer& node){ return node->getType() == nodeType; }); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 5371831e3e..1e94492e93 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -128,6 +128,7 @@ public: qint64 sendPacketList(NLPacketList& packetList, const HifiSockAddr& sockAddr, const QUuid& connectionSecret = QUuid()); qint64 sendPacketList(std::unique_ptr packetList, const HifiSockAddr& sockAddr); + qint64 sendPacketList(std::unique_ptr packetList, const Node& destinationNode); void (*linkedDataCreateCallback)(Node *); @@ -150,7 +151,7 @@ public: int updateNodeWithDataFromPacket(QSharedPointer packet, SharedNodePointer matchingNode); unsigned int broadcastToNodes(std::unique_ptr packet, const NodeSet& destinationNodeTypes); - SharedNodePointer soloNodeOfType(char nodeType); + SharedNodePointer soloNodeOfType(NodeType_t nodeType); void getPacketStats(float &packetsPerSecond, float &bytesPerSecond); void resetPacketStats(); diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 38590e1b03..e680f218db 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -22,6 +22,7 @@ namespace NodeType { const NodeType_t Agent = 'I'; const NodeType_t AudioMixer = 'M'; const NodeType_t AvatarMixer = 'W'; + const NodeType_t AssetServer = 'A'; const NodeType_t Unassigned = 1; void init(); diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index a086949ac8..002ecc2c6f 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -19,8 +19,10 @@ #include "NodeList.h" #include "SharedUtil.h" +Q_DECLARE_METATYPE(QSharedPointer); PacketReceiver::PacketReceiver(QObject* parent) : QObject(parent) { qRegisterMetaType>(); + qRegisterMetaType>(); } bool PacketReceiver::registerListenerForTypes(PacketTypeList types, QObject* listener, const char* slot) { diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index 75028abe93..b07c3cc60e 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -155,11 +155,17 @@ void ResourceCache::clearUnusedResource() { void ResourceCache::attemptRequest(Resource* resource) { auto sharedItems = DependencyManager::get(); if (_requestLimit <= 0) { + qDebug() << "REQUEST LIMIT REACHED (" << _requestLimit << "), queueing: " << resource->getURL(); // wait until a slot becomes available sharedItems->_pendingRequests.append(resource); return; } - _requestLimit--; + qDebug() << "-- Decreasing limit for : " << resource->getURL(); + + // Disable request limiting for ATP + if (resource->getURL() != URL_SCHEME_ATP) { + _requestLimit--; + } sharedItems->_loadingRequests.append(resource); resource->makeRequest(); } @@ -168,7 +174,10 @@ void ResourceCache::requestCompleted(Resource* resource) { auto sharedItems = DependencyManager::get(); sharedItems->_loadingRequests.removeOne(resource); - _requestLimit++; + qDebug() << "++ Increasing limit after finished: " << resource->getURL(); + if (resource->getURL() != URL_SCHEME_ATP) { + _requestLimit++; + } // look for the highest priority pending request int highestIndex = -1; @@ -196,24 +205,22 @@ int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT; Resource::Resource(const QUrl& url, bool delayLoad) : _url(url), - _request(url) { + _activeUrl(url), + _request(nullptr) { init(); - _request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - _request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); - // start loading immediately unless instructed otherwise if (!(_startedLoading || delayLoad)) { - attemptRequest(); + QTimer::singleShot(0, this, &Resource::ensureLoading); } } Resource::~Resource() { - if (_reply) { + if (_request) { ResourceCache::requestCompleted(this); - delete _reply; - _reply = nullptr; + _request->deleteLater(); + _request = nullptr; } } @@ -259,21 +266,17 @@ float Resource::getLoadPriority() { } void Resource::refresh() { - if (_reply && !(_loaded || _failedToLoad)) { + if (_request && !(_loaded || _failedToLoad)) { return; } - if (_reply) { + if (_request) { + _request->disconnect(this); + _request->deleteLater(); + _request = nullptr; ResourceCache::requestCompleted(this); - _reply->disconnect(this); - _replyTimer->disconnect(this); - _reply->deleteLater(); - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; } init(); - _request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); ensureLoading(); emit onRefresh(); } @@ -303,6 +306,7 @@ void Resource::init() { _failedToLoad = false; _loaded = false; _attempts = 0; + _activeUrl = _url; if (_url.isEmpty()) { _startedLoading = _loaded = true; @@ -318,6 +322,7 @@ void Resource::attemptRequest() { } void Resource::finishedLoading(bool success) { + qDebug() << "Finished loading: " << _url; if (success) { _loaded = true; } else { @@ -330,94 +335,91 @@ void Resource::reinsert() { _cache->_resources.insert(_url, _self); } -static const int REPLY_TIMEOUT_MS = 5000; -void Resource::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) { - _bytesReceived = bytesReceived; - _bytesTotal = bytesTotal; - _replyTimer->start(REPLY_TIMEOUT_MS); -} - -void Resource::handleReplyError() { - handleReplyErrorInternal(_reply->error()); -} - -void Resource::handleReplyTimeout() { - handleReplyErrorInternal(QNetworkReply::TimeoutError); -} void Resource::makeRequest() { - _reply = NetworkAccessManager::getInstance().get(_request); + Q_ASSERT(!_request); - connect(_reply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleDownloadProgress(qint64,qint64))); - connect(_reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleReplyError())); - connect(_reply, SIGNAL(finished()), SLOT(handleReplyFinished())); + _request = ResourceManager::createResourceRequest(this, _activeUrl); + + if (!_request) { + qDebug() << "Failed to get request for " << _url; + ResourceCache::requestCompleted(this); + finishedLoading(false); + return; + } + + qDebug() << "Starting request for: " << _url; + + connect(_request, &ResourceRequest::progress, this, &Resource::handleDownloadProgress); + connect(_request, &ResourceRequest::finished, this, &Resource::handleReplyFinished); - _replyTimer = new QTimer(this); - connect(_replyTimer, SIGNAL(timeout()), SLOT(handleReplyTimeout())); - _replyTimer->setSingleShot(true); - _replyTimer->start(REPLY_TIMEOUT_MS); _bytesReceived = _bytesTotal = 0; + + _request->send(); } -void Resource::handleReplyErrorInternal(QNetworkReply::NetworkError error) { - - _reply->disconnect(this); - _replyTimer->disconnect(this); - _reply->deleteLater(); - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; - ResourceCache::requestCompleted(this); - - // retry for certain types of failures - switch (error) { - case QNetworkReply::RemoteHostClosedError: - case QNetworkReply::TimeoutError: - case QNetworkReply::TemporaryNetworkFailureError: - case QNetworkReply::ProxyConnectionClosedError: - case QNetworkReply::ProxyTimeoutError: - case QNetworkReply::UnknownNetworkError: - case QNetworkReply::UnknownProxyError: - case QNetworkReply::UnknownContentError: - case QNetworkReply::ProtocolFailure: { - // retry with increasing delays - const int MAX_ATTEMPTS = 8; - const int BASE_DELAY_MS = 1000; - if (++_attempts < MAX_ATTEMPTS) { - QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); - qCWarning(networking) << "error downloading url =" << _url.toDisplayString() << ", error =" << error << ", retrying (" << _attempts << "/" << MAX_ATTEMPTS << ")"; - return; - } - // fall through to final failure - } - default: - qCCritical(networking) << "error downloading, url =" << _url.toDisplayString() << ", error =" << error; - emit failed(error); - finishedLoading(false); - break; - } +void Resource::handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal) { + _bytesReceived = bytesReceived; + _bytesTotal = bytesTotal; } void Resource::handleReplyFinished() { + Q_ASSERT(_request); - bool fromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); - qCDebug(networking) << "success downloading url =" << _url.toDisplayString() << (fromCache ? "from cache" : ""); + auto result = _request->getResult(); + if (result == ResourceRequest::SUCCESS) { + _data = _request->getData(); + qDebug() << "Request finished for " << _url << ", " << _activeUrl; - _reply->disconnect(this); - _replyTimer->disconnect(this); - QNetworkReply* reply = _reply; - _reply = nullptr; - _replyTimer->deleteLater(); - _replyTimer = nullptr; - ResourceCache::requestCompleted(this); + _request->disconnect(this); + _request->deleteLater(); + _request = nullptr; - finishedLoading(true); - emit loaded(*reply); - downloadFinished(reply); + ResourceCache::requestCompleted(this); + + emit loaded(_data); + + downloadFinished(_data); + } else { + _request->disconnect(this); + _request->deleteLater(); + _request = nullptr; + + if (result == ResourceRequest::Result::TIMEOUT) { + qDebug() << "Timed out loading" << _url << "received" << _bytesReceived << "total" << _bytesTotal; + } else { + qDebug() << "Error loading " << _url; + } + + bool retry = false; + switch (result) { + case ResourceRequest::Result::TIMEOUT: + case ResourceRequest::Result::ERROR: { + // retry with increasing delays + const int MAX_ATTEMPTS = 8; + const int BASE_DELAY_MS = 1000; + if (++_attempts < MAX_ATTEMPTS) { + QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest())); + retry = true; + break; + } + // fall through to final failure + } + default: + finishedLoading(false); + break; + } + + auto error = result == ResourceRequest::TIMEOUT ? QNetworkReply::TimeoutError : QNetworkReply::UnknownNetworkError; + emit failed(error); + + if (!retry) { + ResourceCache::requestCompleted(this); + } + } } - -void Resource::downloadFinished(QNetworkReply* reply) { +void Resource::downloadFinished(const QByteArray& data) { ; } diff --git a/libraries/networking/src/ResourceCache.h b/libraries/networking/src/ResourceCache.h index 9a88c434e1..08d38db9a4 100644 --- a/libraries/networking/src/ResourceCache.h +++ b/libraries/networking/src/ResourceCache.h @@ -26,6 +26,8 @@ #include +#include "ResourceManager.h" + class QNetworkReply; class QTimer; @@ -102,7 +104,7 @@ protected: void reserveUnusedResource(qint64 resourceSize); void clearUnusedResource(); - static void attemptRequest(Resource* resource); + Q_INVOKABLE static void attemptRequest(Resource* resource); static void requestCompleted(Resource* resource); private: @@ -174,7 +176,7 @@ public: signals: /// Fired when the resource has been loaded. - void loaded(QNetworkReply& request); + void loaded(const QByteArray& request); /// Fired when resource failed to load. void failed(QNetworkReply::NetworkError error); @@ -189,7 +191,7 @@ protected: virtual void init(); /// Called when the download has finished. The recipient should delete the reply when done with it. - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data); /// Should be called by subclasses when all the loading that will be done has been done. Q_INVOKABLE void finishedLoading(bool success); @@ -198,31 +200,29 @@ protected: virtual void reinsert(); QUrl _url; - QNetworkRequest _request; + QUrl _activeUrl; bool _startedLoading = false; bool _failedToLoad = false; bool _loaded = false; QHash, float> _loadPriorities; QWeakPointer _self; QPointer _cache; + QByteArray _data; private slots: - void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); - void handleReplyError(); + void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal); void handleReplyFinished(); - void handleReplyTimeout(); private: void setLRUKey(int lruKey) { _lruKey = lruKey; } void makeRequest(); - - void handleReplyErrorInternal(QNetworkReply::NetworkError error); + void retry(); friend class ResourceCache; + ResourceRequest* _request = nullptr; int _lruKey = 0; - QNetworkReply* _reply = nullptr; QTimer* _replyTimer = nullptr; qint64 _bytesReceived = 0; qint64 _bytesTotal = 0; diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp new file mode 100644 index 0000000000..ded3dfe222 --- /dev/null +++ b/libraries/networking/src/ResourceManager.cpp @@ -0,0 +1,33 @@ +// +// ResourceManager.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ResourceManager.h" + +#include "AssetResourceRequest.h" +#include "FileResourceRequest.h" +#include "HTTPResourceRequest.h" + +#include + +ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const QUrl& url) { + auto scheme = url.scheme(); + if (scheme == URL_SCHEME_FILE) { + return new FileResourceRequest(parent, url); + } else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) { + return new HTTPResourceRequest(parent, url); + } else if (scheme == URL_SCHEME_ATP) { + return new AssetResourceRequest(parent, url); + } + + qDebug() << "Failed to load: " << url.url(); + + return nullptr; +} diff --git a/libraries/networking/src/ResourceManager.h b/libraries/networking/src/ResourceManager.h new file mode 100644 index 0000000000..3748036c8e --- /dev/null +++ b/libraries/networking/src/ResourceManager.h @@ -0,0 +1,30 @@ +// +// ResourceManager.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ResourceManager_h +#define hifi_ResourceManager_h + +#include + +#include "ResourceRequest.h" + +const QString URL_SCHEME_FILE = "file"; +const QString URL_SCHEME_HTTP = "http"; +const QString URL_SCHEME_HTTPS = "https"; +const QString URL_SCHEME_FTP = "ftp"; +const QString URL_SCHEME_ATP = "atp"; + +class ResourceManager { +public: + static ResourceRequest* createResourceRequest(QObject* parent, const QUrl& url); +}; + +#endif diff --git a/libraries/networking/src/ResourceRequest.cpp b/libraries/networking/src/ResourceRequest.cpp new file mode 100644 index 0000000000..d56033670b --- /dev/null +++ b/libraries/networking/src/ResourceRequest.cpp @@ -0,0 +1,24 @@ +// +// ResourceRequest.cpp +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ResourceRequest.h" + +ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) : + QObject(parent), + _url(url) { +} + +void ResourceRequest::send() { + Q_ASSERT(_state == UNSENT); + + _state = IN_PROGRESS; + doSend(); +} diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h new file mode 100644 index 0000000000..ac13523e96 --- /dev/null +++ b/libraries/networking/src/ResourceRequest.h @@ -0,0 +1,60 @@ +// +// ResourceRequest.h +// libraries/networking/src +// +// Created by Ryan Huffman on 2015/07/23 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ResourceRequest_h +#define hifi_ResourceRequest_h + +#include +#include + +class ResourceRequest : public QObject { + Q_OBJECT +public: + ResourceRequest(QObject* parent, const QUrl& url); + + enum State { + UNSENT = 0, + IN_PROGRESS, + FINISHED + }; + + enum Result { + SUCCESS, + ERROR, + TIMEOUT, + NOT_FOUND + }; + + void send(); + QByteArray getData() { return _data; } + State getState() const { return _state; } + Result getResult() const { return _result; } + QUrl getUrl() const { return _url; } + bool loadedFromCache() const { return _loadedFromCache; } + + void setCacheEnabled(bool value) { _cacheEnabled = value; } + +signals: + void progress(uint64_t bytesReceived, uint64_t bytesTotal); + void finished(); + +protected: + virtual void doSend() = 0; + + QUrl _url; + State _state { UNSENT }; + Result _result; + QByteArray _data; + bool _cacheEnabled { true }; + bool _loadedFromCache { false }; +}; + +#endif diff --git a/libraries/networking/src/udt/Packet.cpp b/libraries/networking/src/udt/Packet.cpp index 2cc0624f7b..56c65e0657 100644 --- a/libraries/networking/src/udt/Packet.cpp +++ b/libraries/networking/src/udt/Packet.cpp @@ -93,6 +93,8 @@ Packet::Packet(Packet&& other) : _isReliable = other._isReliable; _isPartOfMessage = other._isPartOfMessage; _sequenceNumber = other._sequenceNumber; + _packetPosition = other._packetPosition; + _messageNumber = other._messageNumber; } Packet& Packet::operator=(Packet&& other) { @@ -101,6 +103,8 @@ Packet& Packet::operator=(Packet&& other) { _isReliable = other._isReliable; _isPartOfMessage = other._isPartOfMessage; _sequenceNumber = other._sequenceNumber; + _packetPosition = other._packetPosition; + _messageNumber = other._messageNumber; return *this; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 16969a6804..18394f4c87 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -73,7 +73,13 @@ enum class PacketType : uint8_t { EntityEdit, DomainServerConnectionToken, DomainSettingsRequest, - DomainSettings + DomainSettings, + AssetGet, + AssetGetReply, + AssetUpload, + AssetUploadReply, + AssetGetInfo, + AssetGetInfoReply }; const int NUM_BYTES_MD5_HASH = 16; diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 6dad7e9133..7b90276b62 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -82,7 +82,7 @@ std::unique_ptr PacketList::createPacket() { std::unique_ptr PacketList::createPacketWithExtendedHeader() { // use the static create method to create a new packet auto packet = createPacket(); - + if (!_extendedHeader.isEmpty()) { // add the extended header to the front of the packet if (packet->write(_extendedHeader) == -1) { @@ -90,7 +90,7 @@ std::unique_ptr PacketList::createPacketWithExtendedHeader() { << "- make sure that _extendedHeader is not larger than the payload capacity."; } } - + return packet; } @@ -112,83 +112,91 @@ QByteArray PacketList::getMessage() { } qint64 PacketList::writeData(const char* data, qint64 maxSize) { - if (!_currentPacket) { - // we don't have a current packet, time to set one up - _currentPacket = createPacketWithExtendedHeader(); - } - - // check if this block of data can fit into the currentPacket - if (maxSize <= _currentPacket->bytesAvailableForWrite()) { - // it fits, just write it to the current packet - _currentPacket->write(data, maxSize); - - // return the number of bytes written - return maxSize; - } else { - // it does not fit - this may need to be in the next packet - - if (!_isOrdered) { - auto newPacket = createPacketWithExtendedHeader(); - - if (_segmentStartIndex >= 0) { - // We in the process of writing a segment for an unordered PacketList. - // We need to try and pull the first part of the segment out to our new packet - - // check now to see if this is an unsupported write - int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); - - if ((newPacket->size() - numBytesToEnd) < maxSize) { - // this is an unsupported case - the segment is bigger than the size of an individual packet - // but the PacketList is not going to be sent ordered - qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is" - << "larger than the payload size."; - Q_ASSERT(false); - } - - int segmentSize = _currentPacket->pos() - _segmentStartIndex; - - // copy from currentPacket where the segment started to the beginning of the newPacket - newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize); - - // the current segment now starts at the beginning of the new packet - _segmentStartIndex = _extendedHeader.size(); - - // shrink the current payload to the actual size of the packet - _currentPacket->setPayloadSize(_segmentStartIndex); - } - - // move the current packet to our list of packets - _packets.push_back(std::move(_currentPacket)); - - // write the data to the newPacket - newPacket->write(data, maxSize); - - // swap our current packet with the new packet - _currentPacket.swap(newPacket); - - // return the number of bytes written to the new packet - return maxSize; + auto sizeRemaining = maxSize; + + while (sizeRemaining > 0) { + if (!_currentPacket) { + // we don't have a current packet, time to set one up + _currentPacket = createPacketWithExtendedHeader(); + } + + // check if this block of data can fit into the currentPacket + if (sizeRemaining <= _currentPacket->bytesAvailableForWrite()) { + // it fits, just write it to the current packet + _currentPacket->write(data, sizeRemaining); + + sizeRemaining = 0; } else { - // we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover - // into a new packet - - int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); - _currentPacket->write(data, numBytesToEnd); - - // move the current packet to our list of packets - _packets.push_back(std::move(_currentPacket)); - - // recursively call our writeData method for the remaining data to write to a new packet - return numBytesToEnd + writeData(data + numBytesToEnd, maxSize - numBytesToEnd); + // it does not fit - this may need to be in the next packet + + if (!_isOrdered) { + auto newPacket = createPacketWithExtendedHeader(); + + if (_segmentStartIndex >= 0) { + // We in the process of writing a segment for an unordered PacketList. + // We need to try and pull the first part of the segment out to our new packet + + // check now to see if this is an unsupported write + int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); + + if ((newPacket->size() - numBytesToEnd) < sizeRemaining) { + // this is an unsupported case - the segment is bigger than the size of an individual packet + // but the PacketList is not going to be sent ordered + qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is" + << "larger than the payload size."; + Q_ASSERT(false); + } + + int segmentSize = _currentPacket->pos() - _segmentStartIndex; + + // copy from currentPacket where the segment started to the beginning of the newPacket + newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize); + + // the current segment now starts at the beginning of the new packet + _segmentStartIndex = _extendedHeader.size(); + + // shrink the current payload to the actual size of the packet + _currentPacket->setPayloadSize(_segmentStartIndex); + } + + // move the current packet to our list of packets + _packets.push_back(std::move(_currentPacket)); + + // write the data to the newPacket + newPacket->write(data, sizeRemaining); + + // swap our current packet with the new packet + _currentPacket.swap(newPacket); + + // We've written all of the data, so set sizeRemaining to 0 + sizeRemaining = 0; + } else { + // we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover + // into a new packet + + int numBytesToEnd = _currentPacket->bytesAvailableForWrite(); + _currentPacket->write(data, numBytesToEnd); + + // Remove number of bytes written from sizeRemaining + sizeRemaining -= numBytesToEnd; + + // Move the data pointer forward + data += numBytesToEnd; + + // move the current packet to our list of packets + _packets.push_back(std::move(_currentPacket)); + } } } + + return maxSize; } void PacketList::closeCurrentPacket(bool shouldSendEmpty) { if (shouldSendEmpty && !_currentPacket) { _currentPacket = createPacketWithExtendedHeader(); } - + if (_currentPacket) { // move the current packet to our list of packets _packets.push_back(std::move(_currentPacket)); diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 2f81fe8b84..db7c25012c 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1678,15 +1678,15 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { } } -GeometryReader::GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping) : +GeometryReader::GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping) : _url(url), - _reply(reply), + _data(data), _mapping(mapping) { } void GeometryReader::run() { try { - if (!_reply) { + if (_data.isEmpty()) { throw QString("Reply is NULL ?!"); } QString urlname = _url.path().toLower(); @@ -1701,9 +1701,9 @@ void GeometryReader::run() { if (_url.path().toLower().endsWith(".fbx")) { const bool grabLightmaps = true; const float lightmapLevel = 1.0f; - fbxgeo = readFBX(_reply, _mapping, _url.path(), grabLightmaps, lightmapLevel); + fbxgeo = readFBX(_data, _mapping, _url.path(), grabLightmaps, lightmapLevel); } else if (_url.path().toLower().endsWith(".obj")) { - fbxgeo = OBJReader().readOBJ(_reply, _mapping, &_url); + fbxgeo = OBJReader().readOBJ(_data, _mapping); } else { QString errorStr("usupported format"); emit onError(NetworkGeometry::ModelParseError, errorStr); @@ -1717,7 +1717,6 @@ void GeometryReader::run() { qCDebug(renderutils) << "Error reading " << _url << ": " << error; emit onError(NetworkGeometry::ModelParseError, error); } - _reply->deleteLater(); } NetworkGeometry::NetworkGeometry(const QUrl& url, bool delayLoad, const QVariantHash& mapping, const QUrl& textureBaseUrl) : @@ -1746,8 +1745,10 @@ void NetworkGeometry::attemptRequest() { void NetworkGeometry::attemptRequestInternal() { if (_url.path().toLower().endsWith(".fst")) { + _mappingUrl = _url; requestMapping(_url); } else { + _modelUrl = _url; requestModel(_url); } } @@ -1838,8 +1839,8 @@ void NetworkGeometry::requestMapping(const QUrl& url) { _resource->deleteLater(); } _resource = new Resource(url, false); - connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(mappingRequestDone(QNetworkReply&))); - connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(mappingRequestError(QNetworkReply::NetworkError))); + connect(_resource, &Resource::loaded, this, &NetworkGeometry::mappingRequestDone); + connect(_resource, &Resource::failed, this, &NetworkGeometry::mappingRequestError); } void NetworkGeometry::requestModel(const QUrl& url) { @@ -1847,18 +1848,19 @@ void NetworkGeometry::requestModel(const QUrl& url) { if (_resource) { _resource->deleteLater(); } + _modelUrl = url; _resource = new Resource(url, false); - connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(modelRequestDone(QNetworkReply&))); - connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(modelRequestError(QNetworkReply::NetworkError))); + connect(_resource, &Resource::loaded, this, &NetworkGeometry::modelRequestDone); + connect(_resource, &Resource::failed, this, &NetworkGeometry::modelRequestError); } -void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) { +void NetworkGeometry::mappingRequestDone(const QByteArray& data) { assert(_state == RequestMappingState); // parse the mapping file - _mapping = FSTReader::readMapping(reply.readAll()); + _mapping = FSTReader::readMapping(data); - QUrl replyUrl = reply.url(); + QUrl replyUrl = _mappingUrl; QString modelUrlStr = _mapping.value("filename").toString(); if (modelUrlStr.isNull()) { qCDebug(renderutils) << "Mapping file " << _url << "has no \"filename\" entry"; @@ -1873,8 +1875,8 @@ void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) { _textureBaseUrl = replyUrl.resolved(texdir); } - QUrl modelUrl = replyUrl.resolved(modelUrlStr); - requestModel(modelUrl); + _modelUrl = replyUrl.resolved(modelUrlStr); + requestModel(_modelUrl); } } @@ -1884,13 +1886,13 @@ void NetworkGeometry::mappingRequestError(QNetworkReply::NetworkError error) { emit onFailure(*this, MappingRequestError); } -void NetworkGeometry::modelRequestDone(QNetworkReply& reply) { +void NetworkGeometry::modelRequestDone(const QByteArray& data) { assert(_state == RequestModelState); _state = ParsingModelState; // asynchronously parse the model file. - GeometryReader* geometryReader = new GeometryReader(reply.url(), &reply, _mapping); + GeometryReader* geometryReader = new GeometryReader(_modelUrl, data, _mapping); connect(geometryReader, SIGNAL(onSuccess(FBXGeometry*)), SLOT(modelParseSuccess(FBXGeometry*))); connect(geometryReader, SIGNAL(onError(int, QString)), SLOT(modelParseError(int, QString))); diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 71fa35c054..3820b58baf 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -348,10 +348,10 @@ signals: void onFailure(NetworkGeometry& networkGeometry, Error error); protected slots: - void mappingRequestDone(QNetworkReply& reply); + void mappingRequestDone(const QByteArray& data); void mappingRequestError(QNetworkReply::NetworkError error); - void modelRequestDone(QNetworkReply& reply); + void modelRequestDone(const QByteArray& data); void modelRequestError(QNetworkReply::NetworkError error); void modelParseSuccess(FBXGeometry* geometry); @@ -371,6 +371,8 @@ protected: State _state; QUrl _url; + QUrl _mappingUrl; + QUrl _modelUrl; QVariantHash _mapping; QUrl _textureBaseUrl; @@ -386,14 +388,14 @@ protected: class GeometryReader : public QObject, public QRunnable { Q_OBJECT public: - GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping); + GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping); virtual void run(); signals: void onSuccess(FBXGeometry* geometry); void onError(int error, QString str); private: QUrl _url; - QNetworkReply* _reply; + QByteArray _data; QVariantHash _mapping; }; diff --git a/libraries/render-utils/src/TextureCache.cpp b/libraries/render-utils/src/TextureCache.cpp index deeec58f49..4954629d86 100644 --- a/libraries/render-utils/src/TextureCache.cpp +++ b/libraries/render-utils/src/TextureCache.cpp @@ -215,8 +215,7 @@ NetworkTexture::NetworkTexture(const QUrl& url, TextureType type, const QByteArr class ImageReader : public QRunnable { public: - ImageReader(const QWeakPointer& texture, TextureType type, QNetworkReply* reply, const QUrl& url = QUrl(), - const QByteArray& content = QByteArray()); + ImageReader(const QWeakPointer& texture, TextureType type, const QByteArray& data, const QUrl& url = QUrl()); virtual void run(); @@ -224,27 +223,25 @@ private: QWeakPointer _texture; TextureType _type; - QNetworkReply* _reply; QUrl _url; QByteArray _content; }; -void NetworkTexture::downloadFinished(QNetworkReply* reply) { +void NetworkTexture::downloadFinished(const QByteArray& data) { // send the reader off to the thread pool - QThreadPool::globalInstance()->start(new ImageReader(_self, _type, reply)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _type, data, _url)); } void NetworkTexture::loadContent(const QByteArray& content) { - QThreadPool::globalInstance()->start(new ImageReader(_self, _type, NULL, _url, content)); + QThreadPool::globalInstance()->start(new ImageReader(_self, _type, content, _url)); } -ImageReader::ImageReader(const QWeakPointer& texture, TextureType type, QNetworkReply* reply, - const QUrl& url, const QByteArray& content) : +ImageReader::ImageReader(const QWeakPointer& texture, TextureType type, const QByteArray& data, + const QUrl& url) : _texture(texture), _type(type), - _reply(reply), _url(url), - _content(content) { + _content(data) { } std::once_flag onceListSupportedFormatsflag; @@ -297,16 +294,8 @@ public: void ImageReader::run() { QSharedPointer texture = _texture.toStrongRef(); if (texture.isNull()) { - if (_reply) { - _reply->deleteLater(); - } return; } - if (_reply) { - _url = _reply->url(); - _content = _reply->readAll(); - _reply->deleteLater(); - } listSupportedImageFormats(); diff --git a/libraries/render-utils/src/TextureCache.h b/libraries/render-utils/src/TextureCache.h index eeb17f07b9..4f98adb157 100644 --- a/libraries/render-utils/src/TextureCache.h +++ b/libraries/render-utils/src/TextureCache.h @@ -123,7 +123,7 @@ public: TextureType getType() const { return _type; } protected: - virtual void downloadFinished(QNetworkReply* reply); + virtual void downloadFinished(const QByteArray& data) override; Q_INVOKABLE void loadContent(const QByteArray& content); // FIXME: This void* should be a gpu::Texture* but i cannot get it to work for now, moving on... diff --git a/libraries/script-engine/src/BatchLoader.cpp b/libraries/script-engine/src/BatchLoader.cpp index db0743808f..01e4a5e869 100644 --- a/libraries/script-engine/src/BatchLoader.cpp +++ b/libraries/script-engine/src/BatchLoader.cpp @@ -17,9 +17,7 @@ #include "BatchLoader.h" #include #include - - - +#include "ResourceManager.h" BatchLoader::BatchLoader(const QList& urls) : QObject(), @@ -27,6 +25,7 @@ BatchLoader::BatchLoader(const QList& urls) _finished(false), _urls(urls.toSet()), _data() { + qRegisterMetaType>("QMap"); } void BatchLoader::start() { @@ -35,45 +34,27 @@ void BatchLoader::start() { } _started = true; - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); for (QUrl url : _urls) { - if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { - QNetworkRequest request = QNetworkRequest(url); - request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - QNetworkReply* reply = networkAccessManager.get(request); - - qCDebug(scriptengine) << "Downloading file at" << url; - - connect(reply, &QNetworkReply::finished, [=]() { - if (reply->error()) { - _data.insert(url, QString()); - } else { - _data.insert(url, reply->readAll()); - } - reply->deleteLater(); - checkFinished(); - }); - - // If we end up being destroyed before the reply finishes, clean it up - connect(this, &QObject::destroyed, reply, &QObject::deleteLater); - - } else { -#ifdef _WIN32 - QString fileName = url.toString(); -#else - QString fileName = url.toLocalFile(); -#endif - - qCDebug(scriptengine) << "Reading file at " << fileName; - - QFile scriptFile(fileName); - if (scriptFile.open(QFile::ReadOnly | QFile::Text)) { - QTextStream in(&scriptFile); - _data.insert(url, in.readAll()); + auto request = ResourceManager::createResourceRequest(this, url); + if (!request) { + continue; + } + connect(request, &ResourceRequest::finished, this, [=]() { + if (request->getResult() == ResourceRequest::SUCCESS) { + _data.insert(url, request->getData()); } else { _data.insert(url, QString()); } - } + request->deleteLater(); + checkFinished(); + }); + + // If we end up being destroyed before the reply finishes, clean it up + connect(this, &QObject::destroyed, request, &QObject::deleteLater); + + qCDebug(scriptengine) << "Loading script at " << url; + + request->send(); } checkFinished(); } diff --git a/libraries/script-engine/src/ScriptCache.cpp b/libraries/script-engine/src/ScriptCache.cpp index 2047442ce6..e3d12313dc 100644 --- a/libraries/script-engine/src/ScriptCache.cpp +++ b/libraries/script-engine/src/ScriptCache.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include "ScriptCache.h" @@ -28,8 +27,6 @@ ScriptCache::ScriptCache(QObject* parent) { } QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending, bool reload) { - assert(!_scriptCache.contains(url) || !reload); - QString scriptContents; if (_scriptCache.contains(url) && !reload) { qCDebug(scriptengine) << "Found script in cache:" << url.toString(); @@ -44,18 +41,10 @@ QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& is if (alreadyWaiting) { qCDebug(scriptengine) << "Already downloading script at:" << url.toString(); } else { - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest networkRequest = QNetworkRequest(url); - networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); - if (reload) { - networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); - qCDebug(scriptengine) << "Redownloading script at:" << url.toString(); - } else { - qCDebug(scriptengine) << "Downloading script at:" << url.toString(); - } - - QNetworkReply* reply = networkAccessManager.get(networkRequest); - connect(reply, &QNetworkReply::finished, this, &ScriptCache::scriptDownloaded); + auto request = ResourceManager::createResourceRequest(this, url); + request->setCacheEnabled(!reload); + connect(request, &ResourceRequest::finished, this, &ScriptCache::scriptDownloaded); + request->send(); } } return scriptContents; @@ -69,27 +58,25 @@ void ScriptCache::deleteScript(const QUrl& url) { } void ScriptCache::scriptDownloaded() { - QNetworkReply* reply = qobject_cast(sender()); - QUrl url = reply->url(); + ResourceRequest* req = qobject_cast(sender()); + QUrl url = req->getUrl(); QList scriptUsers = _scriptUsers.values(url); _scriptUsers.remove(url); - if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) { - _scriptCache[url] = reply->readAll(); + if (req->getResult() == ResourceRequest::SUCCESS) { + _scriptCache[url] = req->getData(); qCDebug(scriptengine) << "Done downloading script at:" << url.toString(); foreach(ScriptUser* user, scriptUsers) { user->scriptContentsAvailable(url, _scriptCache[url]); } } else { - qCWarning(scriptengine) << "Error loading script from URL " << reply->url().toString() - << "- HTTP status code is" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() - << "and error from QNetworkReply is" << reply->errorString(); + qCWarning(scriptengine) << "Error loading script from URL " << url; foreach(ScriptUser* user, scriptUsers) { user->errorInLoadingScript(url); } } - reply->deleteLater(); + req->deleteLater(); }