From 7b179ab91aac3ec14b26cb17ccf7b46f348985fc Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 29 Feb 2016 16:57:24 -0800 Subject: [PATCH 01/20] First draft for mapping requests --- assignment-client/src/assets/AssetServer.cpp | 27 +++++++ assignment-client/src/assets/AssetServer.h | 7 ++ libraries/networking/src/AssetClient.cpp | 79 ++++++++++++++++++- libraries/networking/src/AssetClient.h | 6 +- libraries/networking/src/AssetRequest.cpp | 5 +- libraries/networking/src/AssetRequest.h | 7 +- .../networking/src/AssetResourceRequest.cpp | 14 +--- libraries/networking/src/udt/PacketHeaders.h | 4 +- 8 files changed, 128 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 4ca2439996..ab86787769 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -39,6 +39,7 @@ AssetServer::AssetServer(ReceivedMessage& message) : _taskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGetMapping, this, "handleAssetGetMapping"); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); @@ -161,6 +162,32 @@ void AssetServer::completeSetup() { nodeList->addNodeTypeToInterestSet(NodeType::Agent); } + +void AssetServer::handleAssetGetMapping(QSharedPointer packet, SharedNodePointer senderNode) { + MessageID messageID; + message->readPrimitive(&messageID); + QString assetPath = message->readAll(); + + auto replyPacket = NLPacket::create(PacketType::AssetGetMappingReply); + + replyPacket->writePrimitive(messageID); + + auto it = _fileMapping.find(assetPath.toStdString()); + + if (it != _fileMapping.end()) { + auto assetHash = QString::fromStdString(it.second); + qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash; + replyPacket->writePrimitive(AssetServerError::NoError); + replyPacket->write(assetHash.toHex()); + } else { + qDebug() << "Mapping not found for: " << assetPath; + replyPacket->writePrimitive(AssetServerError::AssetNotFound); + } + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *senderNode); +} + void AssetServer::handleAssetGetInfo(QSharedPointer message, SharedNodePointer senderNode) { QByteArray assetHash; MessageID messageID; diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 7d6e26af08..90e1eb8f34 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -31,6 +31,7 @@ public slots: private slots: void completeSetup(); + void handleAssetGetMapping(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode); @@ -39,6 +40,12 @@ private slots: private: static void writeError(NLPacketList* packetList, AssetServerError error); + + using Path = std::string; + using Hash = std::string; + using Mapping = std::map; + + Mapping _fileMapping; QDir _resourcesDirectory; QThreadPool _taskPool; }; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 9591828fef..a61e30a8c1 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -39,6 +39,7 @@ AssetClient::AssetClient() { auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AssetGetMappingReply, this, "handleAssetGetMappingReply"); packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply", true); packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); @@ -110,7 +111,22 @@ bool haveAssetServer() { return true; } -AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) { +AssetRequest* AssetClient::createRequest(const QUrl& url) { + + auto parts = _url.path().split(".", QString::SkipEmptyParts); + auto hash = parts.length() > 0 ? parts[0] : ""; + auto extension = parts.length() > 1 ? parts[1] : ""; + + if (hash.length() != SHA256_HASH_HEX_LENGTH) { + _result = InvalidURL; + _state = Finished; + + emit finished(); + return; + } + + + if (hash.length() != SHA256_HASH_HEX_LENGTH) { qCWarning(asset_client) << "Invalid hash size"; return nullptr; @@ -155,6 +171,34 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& ex } } +bool AssetClient::getAssetMapping(const QString& path, GetMappingCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto messageID = ++_currentID; + + auto payload = path.toLatin1(); + auto payloadSize = sizeof(messageID) + payload.size(); + auto packet = NLPacket::create(PacketType::AssetGetMapping, payloadSize, true); + + qCDebug(asset_client) << "Requesting mapping for" << path << "from asset-server."; + + packet->writePrimitive(messageID); + + auto bytesWritten = packet->write(payload); + Q_ASSERT(bytesWritten == payload.size()); + + nodeList->sendPacket(std::move(packet), *assetServer); + + _pendingMappingRequests[assetServer][messageID] = callback; + + return true; + } + + return false; +} + bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { @@ -220,6 +264,39 @@ bool AssetClient::getAssetInfo(const QString& hash, const QString& extension, Ge return false; } + +void AssetClient::handleAssetGetMappingReply(QSharedPointer message, SharedNodePointer senderNode) { + MessageID messageID; + message->readPrimitive(&messageID); + + AssetServerError error; + message->readPrimitive(&error); + + QString assetHash; + if (error == AssetServerError::NoError) { + assetHash = message->read(SHA256_HASH_LENGTH); + } + + // Check if we have any pending requests for this node + auto messageMapIt = _pendingMappingRequests.find(senderNode); + if (messageMapIt != _pendingMappingRequests.end()) { + + // Found the node, get the MessageID -> Callback map + auto& messageCallbackMap = messageMapIt->second; + + // Check if we have this pending request + auto requestIt = messageCallbackMap.find(messageID); + if (requestIt != messageCallbackMap.end()) { + auto callback = requestIt->second; + callback(true, error, assetHash); + messageCallbackMap.erase(requestIt); + } + + // Although the messageCallbackMap may now be empty, we won't delete the node until we have disconnected from + // it to avoid constantly creating/deleting the map on subsequent requests. + } +} + void AssetClient::handleAssetGetInfoReply(QSharedPointer message, SharedNodePointer senderNode) { MessageID messageID; message->readPrimitive(&messageID); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index f66fe8adcc..cbbf603de6 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -34,6 +34,7 @@ struct AssetInfo { }; using ReceivedAssetCallback = std::function; +using GetMappingCallback = std::function; using GetInfoCallback = std::function; using UploadResultCallback = std::function; using ProgressCallback = std::function; @@ -44,7 +45,7 @@ class AssetClient : public QObject, public Dependency { public: AssetClient(); - Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension); + Q_INVOKABLE AssetRequest* createRequest(const QUrl& url); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension); @@ -55,6 +56,7 @@ public slots: void clearCache(); private slots: + void handleAssetGetMappingReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetGetInfoReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetGetReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetUploadReply(QSharedPointer message, SharedNodePointer senderNode); @@ -62,6 +64,7 @@ private slots: void handleNodeKilled(SharedNodePointer node); private: + bool getAssetMapping(const QString& path, GetMappingCallback callback); bool getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback); bool getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback); @@ -73,6 +76,7 @@ private: }; static MessageID _currentID; + std::unordered_map> _pendingMappingRequests; std::unordered_map> _pendingRequests; std::unordered_map> _pendingInfoRequests; std::unordered_map> _pendingUploads; diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 19f91349fb..505b64e080 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -20,9 +20,8 @@ #include "NodeList.h" #include "ResourceCache.h" -AssetRequest::AssetRequest(const QString& hash, const QString& extension) : - QObject(), - _hash(hash), +AssetRequest::AssetRequest(const QString& url, const QString& extension) : + _url(url), _extension(extension) { } diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 3c3459b15d..288caac6df 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -39,7 +39,7 @@ public: UnknownError }; - AssetRequest(const QString& hash, const QString& extension); + AssetRequest(const QString& url, const QString& extension); Q_INVOKABLE void start(); @@ -52,12 +52,15 @@ signals: void finished(AssetRequest* thisRequest); void progress(qint64 totalReceived, qint64 total); +private slots: + void getAssetCallback(bool responseReceived, AssetServerError serverError, const QByteArray& data); + private: State _state = NotStarted; Error _error = NoError; AssetInfo _info; uint64_t _totalReceived { 0 }; - QString _hash; + QString _url; QString _extension; QByteArray _data; int _numPendingRequests { 0 }; diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index dac3329153..0025ba565a 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -23,19 +23,7 @@ AssetResourceRequest::~AssetResourceRequest() { void AssetResourceRequest::doSend() { // Make request to atp auto assetClient = DependencyManager::get(); - auto parts = _url.path().split(".", QString::SkipEmptyParts); - auto hash = parts.length() > 0 ? parts[0] : ""; - auto extension = parts.length() > 1 ? parts[1] : ""; - - if (hash.length() != SHA256_HASH_HEX_LENGTH) { - _result = InvalidURL; - _state = Finished; - - emit finished(); - return; - } - - _assetRequest = assetClient->createRequest(hash, extension); + _assetRequest = assetClient->createRequest(_url); if (!_assetRequest) { _result = ServerUnavailable; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 0f586018db..3d92aa1285 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -90,7 +90,9 @@ public: DomainServerRemovedNode, MessagesData, MessagesSubscribe, - MessagesUnsubscribe + MessagesUnsubscribe, + AssetGetMapping, + AssetGetMappingReply }; }; From db98e46bb8f3379bff5d938d4b15592c00fcec7b Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 1 Mar 2016 16:18:35 -0800 Subject: [PATCH 02/20] Update atp mapping operations to use a single packet --- assignment-client/src/assets/AssetServer.cpp | 58 +++++++++++++++----- assignment-client/src/assets/AssetServer.h | 19 +++++-- libraries/networking/src/AssetClient.cpp | 2 +- libraries/networking/src/AssetClient.h | 6 +- libraries/networking/src/AssetUtils.h | 6 ++ libraries/networking/src/ReceivedMessage.cpp | 9 +++ libraries/networking/src/ReceivedMessage.h | 2 + libraries/networking/src/udt/PacketHeaders.h | 4 +- 8 files changed, 82 insertions(+), 24 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index ab86787769..50723c77c0 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -39,10 +39,10 @@ AssetServer::AssetServer(ReceivedMessage& message) : _taskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.registerListener(PacketType::AssetGetMapping, this, "handleAssetGetMapping"); packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); + packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOpertation"); } void AssetServer::run() { @@ -163,25 +163,45 @@ void AssetServer::completeSetup() { } -void AssetServer::handleAssetGetMapping(QSharedPointer packet, SharedNodePointer senderNode) { +void AssetServer::handleAssetMappingOperation(QSharedPointer message, SharedNodePointer senderNode) { MessageID messageID; message->readPrimitive(&messageID); - QString assetPath = message->readAll(); - auto replyPacket = NLPacket::create(PacketType::AssetGetMappingReply); + AssetMappingOperationType operationType; + message->readPrimitive(&operationType); + auto replyPacket = NLPacket::create(PacketType::AssetMappingOperationReply); replyPacket->writePrimitive(messageID); - auto it = _fileMapping.find(assetPath.toStdString()); + switch (operationType) { + case AssetMappingOperationType::Get: { + QString assetPath = message->readString(); - if (it != _fileMapping.end()) { - auto assetHash = QString::fromStdString(it.second); - qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash; - replyPacket->writePrimitive(AssetServerError::NoError); - replyPacket->write(assetHash.toHex()); - } else { - qDebug() << "Mapping not found for: " << assetPath; - replyPacket->writePrimitive(AssetServerError::AssetNotFound); + auto it = _fileMapping.find(assetPath); + if (it != _fileMapping.end()) { + auto assetHash = it->second; + qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash; + replyPacket->writePrimitive(AssetServerError::NoError); + replyPacket->write(assetHash.toLatin1().toHex()); + } + else { + qDebug() << "Mapping not found for: " << assetPath; + replyPacket->writePrimitive(AssetServerError::AssetNotFound); + } + } + case AssetMappingOperationType::Set: { + QString assetPath = message->readString(); + auto assetHash = message->read(SHA256_HASH_LENGTH); + _fileMapping[assetPath] = assetHash; + + replyPacket->writePrimitive(AssetServerError::NoError); + } + case AssetMappingOperationType::Delete: { + QString assetPath = message->readString(); + bool removed = _fileMapping.erase(assetPath) > 0; + + replyPacket->writePrimitive(AssetServerError::NoError); + } } auto nodeList = DependencyManager::get(); @@ -335,3 +355,15 @@ void AssetServer::sendStatsPacket() { // send off the stats packets ThreadedAssignment::addPacketStatsAndSendStatsPacket(serverStats); } + +AssetServer::Hash AssetServer::getMapping(Path path) { + return _fileMapping[path]; +} + +void AssetServer::setMapping(Path path, Hash hash) { + _fileMapping[path] = hash; +} + +bool AssetServer::deleteMapping(Path path) { + return _fileMapping.erase(path) > 0; +} diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 90e1eb8f34..5287c222aa 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -31,19 +31,28 @@ public slots: private slots: void completeSetup(); - void handleAssetGetMapping(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode); + void handleAssetMappingOperation(QSharedPointer message, SharedNodePointer senderNode); void sendStatsPacket(); private: - static void writeError(NLPacketList* packetList, AssetServerError error); + using Path = QString; + using Hash = QString; + using Mapping = std::unordered_map; - using Path = std::string; - using Hash = std::string; - using Mapping = std::map; + /// Return the hash mapping for Path `path` + Hash getMapping(Path path); + + /// Set the mapping for path to hash + void setMapping(Path path, Hash hash); + + /// Delete mapping `path`. Return `true` if mapping existed, else `false`. + bool deleteMapping(Path path); + + static void writeError(NLPacketList* packetList, AssetServerError error); Mapping _fileMapping; QDir _resourcesDirectory; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index a61e30a8c1..22671b729d 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -39,7 +39,7 @@ AssetClient::AssetClient() { auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); - packetReceiver.registerListener(PacketType::AssetGetMappingReply, this, "handleAssetGetMappingReply"); + packetReceiver.registerListener(PacketType::AssetMappingOperationReply, this, "handleAssetMappingReply"); packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply", true); packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index cbbf603de6..1b74a47f99 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -34,7 +34,7 @@ struct AssetInfo { }; using ReceivedAssetCallback = std::function; -using GetMappingCallback = std::function; +using MappingOperationCallback = std::function; using GetInfoCallback = std::function; using UploadResultCallback = std::function; using ProgressCallback = std::function; @@ -64,7 +64,7 @@ private slots: void handleNodeKilled(SharedNodePointer node); private: - bool getAssetMapping(const QString& path, GetMappingCallback callback); + bool getAssetMapping(const QString& path, MappingOperationCallback callback); bool getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback); bool getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback); @@ -76,7 +76,7 @@ private: }; static MessageID _currentID; - std::unordered_map> _pendingMappingRequests; + std::unordered_map> _pendingMappingRequests; std::unordered_map> _pendingRequests; std::unordered_map> _pendingInfoRequests; std::unordered_map> _pendingUploads; diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 21b6b3f434..58e7aa6146 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -32,6 +32,12 @@ enum AssetServerError : uint8_t { PermissionDenied }; +enum AssetMappingOperationType : uint8_t { + Get = 0, + Set, + Delete +}; + QUrl getATPUrl(const QString& hash, const QString& extension = QString()); QByteArray hashData(const QByteArray& data); diff --git a/libraries/networking/src/ReceivedMessage.cpp b/libraries/networking/src/ReceivedMessage.cpp index 651b856923..eed275dc1a 100644 --- a/libraries/networking/src/ReceivedMessage.cpp +++ b/libraries/networking/src/ReceivedMessage.cpp @@ -107,6 +107,15 @@ QByteArray ReceivedMessage::readAll() { return read(getBytesLeftToRead()); } +QString ReceivedMessage::readString() { + uint32_t size; + readPrimitive(&size); + Q_ASSERT(size <= _size - _position); + auto string = QString::fromUtf8(_data.constData(), size); + _position += size; + return string; +} + QByteArray ReceivedMessage::readWithoutCopy(qint64 size) { QByteArray data { QByteArray::fromRawData(_data.constData() + _position, size) }; _position += size; diff --git a/libraries/networking/src/ReceivedMessage.h b/libraries/networking/src/ReceivedMessage.h index 1f797f1262..b813149bed 100644 --- a/libraries/networking/src/ReceivedMessage.h +++ b/libraries/networking/src/ReceivedMessage.h @@ -63,6 +63,8 @@ public: QByteArray read(qint64 size); QByteArray readAll(); + QString readString(); + QByteArray readHead(qint64 size); // This will return a QByteArray referencing the underlying data _without_ refcounting that data. diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 3d92aa1285..c830978e92 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -91,8 +91,8 @@ public: MessagesData, MessagesSubscribe, MessagesUnsubscribe, - AssetGetMapping, - AssetGetMappingReply + AssetMappingOperation, + AssetMappingOperationReply }; }; From 9148db2612a4b801ba2579b93a03af89bbc4515d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 13:22:23 -0800 Subject: [PATCH 03/20] Clean up AssetServer mapping handling --- assignment-client/src/assets/AssetServer.cpp | 23 +++++++++++++++----- assignment-client/src/assets/AssetServer.h | 15 +++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 50723c77c0..f685bbe1a9 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -42,7 +42,7 @@ AssetServer::AssetServer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet"); packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo"); packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload"); - packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOpertation"); + packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation"); } void AssetServer::run() { @@ -182,25 +182,30 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me auto assetHash = it->second; qDebug() << "Found mapping for: " << assetPath << "=>" << assetHash; replyPacket->writePrimitive(AssetServerError::NoError); - replyPacket->write(assetHash.toLatin1().toHex()); + //replyPacket->write(assetHash.toLatin1().toHex()); + replyPacket->writeString(assetHash.toLatin1()); } else { qDebug() << "Mapping not found for: " << assetPath; replyPacket->writePrimitive(AssetServerError::AssetNotFound); } + break; } case AssetMappingOperationType::Set: { QString assetPath = message->readString(); - auto assetHash = message->read(SHA256_HASH_LENGTH); + //auto assetHash = message->read(SHA256_HASH_LENGTH); + auto assetHash = message->readString(); _fileMapping[assetPath] = assetHash; replyPacket->writePrimitive(AssetServerError::NoError); + break; } case AssetMappingOperationType::Delete: { QString assetPath = message->readString(); bool removed = _fileMapping.erase(assetPath) > 0; replyPacket->writePrimitive(AssetServerError::NoError); + break; } } @@ -356,14 +361,20 @@ void AssetServer::sendStatsPacket() { ThreadedAssignment::addPacketStatsAndSendStatsPacket(serverStats); } -AssetServer::Hash AssetServer::getMapping(Path path) { +void AssetServer::loadMappingFromFile() { +} + +void AssetServer::writeMappingToFile() { +} + +AssetHash AssetServer::getMapping(AssetPath path) { return _fileMapping[path]; } -void AssetServer::setMapping(Path path, Hash hash) { +void AssetServer::setMapping(AssetPath path, AssetHash hash) { _fileMapping[path] = hash; } -bool AssetServer::deleteMapping(Path path) { +bool AssetServer::deleteMapping(AssetPath path) { return _fileMapping.erase(path) > 0; } diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 5287c222aa..1eee4feb94 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -39,18 +39,19 @@ private slots: void sendStatsPacket(); private: - using Path = QString; - using Hash = QString; - using Mapping = std::unordered_map; + using Mapping = std::map; - /// Return the hash mapping for Path `path` - Hash getMapping(Path path); + void loadMappingFromFile(); + void writeMappingToFile(); + + /// Return the hash mapping for AssetPath `path` + AssetHash getMapping(AssetPath path); /// Set the mapping for path to hash - void setMapping(Path path, Hash hash); + void setMapping(AssetPath path, AssetHash hash); /// Delete mapping `path`. Return `true` if mapping existed, else `false`. - bool deleteMapping(Path path); + bool deleteMapping(AssetPath path); static void writeError(NLPacketList* packetList, AssetServerError error); From 4e7b63c72e363579a6ebc6bc568fc317f7bd74d6 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 13:23:00 -0800 Subject: [PATCH 04/20] Add get/set mapping request --- libraries/networking/src/AssetClient.cpp | 227 +++++++++++++++-------- 1 file changed, 149 insertions(+), 78 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 22671b729d..f3d677a98a 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -30,6 +30,43 @@ MessageID AssetClient::_currentID = 0; +GetMappingRequest::GetMappingRequest(AssetPath path) : _path(path) { +}; + +void GetMappingRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + return; + } + + auto assetClient = DependencyManager::get(); + assetClient->getAssetMapping(_path, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + // read message + _error = error; + if (!error) { + //_hash = message->read(SHA256_HASH_HEX_LENGTH); + _hash = message->readString(); + } + emit finished(this); + }); +}; +SetMappingRequest::SetMappingRequest(AssetPath path, AssetHash hash) : _path(path), _hash(hash) { +}; + +void SetMappingRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + return; + } + + auto assetClient = DependencyManager::get(); + assetClient->setAssetMapping(_path, _hash, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + // read message + _error = error; + emit finished(this); + }); +}; + AssetClient::AssetClient() { @@ -39,7 +76,7 @@ AssetClient::AssetClient() { auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); - packetReceiver.registerListener(PacketType::AssetMappingOperationReply, this, "handleAssetMappingReply"); + packetReceiver.registerListener(PacketType::AssetMappingOperationReply, this, "handleAssetMappingOperationReply"); packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply"); packetReceiver.registerListener(PacketType::AssetGetReply, this, "handleAssetGetReply", true); packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply"); @@ -98,6 +135,33 @@ void AssetClient::clearCache() { } } +void AssetClient::handleAssetMappingOperationReply(QSharedPointer message, SharedNodePointer senderNode) { + MessageID messageID; + message->readPrimitive(&messageID); + + AssetServerError error; + message->readPrimitive(&error); + + // Check if we have any pending requests for this node + auto messageMapIt = _pendingMappingRequests.find(senderNode); + if (messageMapIt != _pendingMappingRequests.end()) { + + // Found the node, get the MessageID -> Callback map + auto& messageCallbackMap = messageMapIt->second; + + // Check if we have this pending request + auto requestIt = messageCallbackMap.find(messageID); + if (requestIt != messageCallbackMap.end()) { + auto callback = requestIt->second; + callback(true, error, message); + messageCallbackMap.erase(requestIt); + } + + // Although the messageCallbackMap may now be empty, we won't delete the node until we have disconnected from + // it to avoid constantly creating/deleting the map on subsequent requests. + } +} + bool haveAssetServer() { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -111,22 +175,15 @@ bool haveAssetServer() { return true; } -AssetRequest* AssetClient::createRequest(const QUrl& url) { - - auto parts = _url.path().split(".", QString::SkipEmptyParts); - auto hash = parts.length() > 0 ? parts[0] : ""; - auto extension = parts.length() > 1 ? parts[1] : ""; - - if (hash.length() != SHA256_HASH_HEX_LENGTH) { - _result = InvalidURL; - _state = Finished; - - emit finished(); - return; - } - +GetMappingRequest* AssetClient::createGetMappingRequest(const AssetPath& path) { + return new GetMappingRequest(path); +} +SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, const AssetHash& hash) { + return new SetMappingRequest(path, hash); +} +AssetRequest* AssetClient::createRequest(const AssetHash& hash, const QString& extension) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { qCWarning(asset_client) << "Invalid hash size"; return nullptr; @@ -144,8 +201,6 @@ AssetRequest* AssetClient::createRequest(const QUrl& url) { } } - - AssetUpload* AssetClient::createUpload(const QString& filename) { if (haveAssetServer()) { @@ -171,34 +226,34 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& ex } } -bool AssetClient::getAssetMapping(const QString& path, GetMappingCallback callback) { - auto nodeList = DependencyManager::get(); - SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); - - if (assetServer) { - auto messageID = ++_currentID; - - auto payload = path.toLatin1(); - auto payloadSize = sizeof(messageID) + payload.size(); - auto packet = NLPacket::create(PacketType::AssetGetMapping, payloadSize, true); - - qCDebug(asset_client) << "Requesting mapping for" << path << "from asset-server."; - - packet->writePrimitive(messageID); - - auto bytesWritten = packet->write(payload); - Q_ASSERT(bytesWritten == payload.size()); - - nodeList->sendPacket(std::move(packet), *assetServer); - - _pendingMappingRequests[assetServer][messageID] = callback; - - return true; - } - - return false; -} - +//bool AssetClient::setAssetMapping(const QString& path, MappingOperationCallback callback) { +// auto nodeList = DependencyManager::get(); +// SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); +// +// if (assetServer) { +// auto messageID = ++_currentID; +// +// auto payload = path.toLatin1(); +// auto payloadSize = sizeof(messageID) + payload.size(); +// auto packet = NLPacket::create(PacketType::AssetMappingOperation, payloadSize, true); +// +// qCDebug(asset_client) << "Requesting mapping for" << path << "from asset-server."; +// +// packet->writePrimitive(messageID); +// +// auto bytesWritten = packet->write(payload); +// Q_ASSERT(bytesWritten == payload.size()); +// +// nodeList->sendPacket(std::move(packet), *assetServer); +// +// _pendingMappingRequests[assetServer][messageID] = callback; +// +// return true; +// } +// +// return false; +//} +// bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { @@ -264,39 +319,6 @@ bool AssetClient::getAssetInfo(const QString& hash, const QString& extension, Ge return false; } - -void AssetClient::handleAssetGetMappingReply(QSharedPointer message, SharedNodePointer senderNode) { - MessageID messageID; - message->readPrimitive(&messageID); - - AssetServerError error; - message->readPrimitive(&error); - - QString assetHash; - if (error == AssetServerError::NoError) { - assetHash = message->read(SHA256_HASH_LENGTH); - } - - // Check if we have any pending requests for this node - auto messageMapIt = _pendingMappingRequests.find(senderNode); - if (messageMapIt != _pendingMappingRequests.end()) { - - // Found the node, get the MessageID -> Callback map - auto& messageCallbackMap = messageMapIt->second; - - // Check if we have this pending request - auto requestIt = messageCallbackMap.find(messageID); - if (requestIt != messageCallbackMap.end()) { - auto callback = requestIt->second; - callback(true, error, assetHash); - messageCallbackMap.erase(requestIt); - } - - // Although the messageCallbackMap may now be empty, we won't delete the node until we have disconnected from - // it to avoid constantly creating/deleting the map on subsequent requests. - } -} - void AssetClient::handleAssetGetInfoReply(QSharedPointer message, SharedNodePointer senderNode) { MessageID messageID; message->readPrimitive(&messageID); @@ -382,6 +404,55 @@ void AssetClient::handleAssetGetReply(QSharedPointer message, S } } +bool AssetClient::getAssetMapping(const AssetHash& hash, MappingOperationCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(AssetMappingOperationType::Get); + + packetList->writeString(hash.toUtf8()); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + + _pendingMappingRequests[assetServer][messageID] = callback; + + return true; + } + + return false; +} + +bool AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(AssetMappingOperationType::Set); + + packetList->writeString(path.toUtf8()); + packetList->writeString(hash.toUtf8()); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + + _pendingMappingRequests[assetServer][messageID] = callback; + + return true; + } + + return false; +} + bool AssetClient::uploadAsset(const QByteArray& data, const QString& extension, UploadResultCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); From 6dd3b1b4ee642117dd4f49e493d5639b6eeff513 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 13:23:27 -0800 Subject: [PATCH 05/20] Add AssetScriptingInterface methods for mapping --- libraries/networking/src/AssetClient.cpp | 37 ++++++++++++++- libraries/networking/src/AssetClient.h | 60 ++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index f3d677a98a..93139af2d0 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -619,4 +619,39 @@ void AssetScriptingInterface::downloadData(QString urlString, QScriptValue callb }); assetRequest->start(); -} \ No newline at end of file +} + +void AssetScriptingInterface::setMapping(QString path, QString hash, QScriptValue callback) { + auto assetClient = DependencyManager::get(); + auto request = assetClient->createSetMappingRequest(path, hash); + + connect(request, &SetMappingRequest::finished, this, [this, callback](SetMappingRequest* request) mutable { + QScriptValueList args { uint8_t(request->getError()) }; + + callback.call(_engine->currentContext()->thisObject(), args); + + request->deleteLater(); + + }); + + request->start(); +} + +void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) { + auto assetClient = DependencyManager::get(); + auto request = assetClient->createGetMappingRequest(path); + + connect(request, &GetMappingRequest::finished, this, [this, callback](GetMappingRequest* request) mutable { + QScriptValueList args { uint8_t(request->getError()), request->getHash() }; + + callback.call(_engine->currentContext()->thisObject(), args); + + request->deleteLater(); + + }); + + request->start(); +} + +void AssetScriptingInterface::getAllMappings(QString path, QScriptValue callback) { +} diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 1b74a47f99..0face50367 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -25,6 +25,8 @@ #include "ReceivedMessage.h" #include "ResourceCache.h" +class GetMappingRequest; +class SetMappingRequest; class AssetRequest; class AssetUpload; @@ -33,19 +35,61 @@ struct AssetInfo { int64_t size; }; +using MappingOperationCallback = std::function message)>; using ReceivedAssetCallback = std::function; -using MappingOperationCallback = std::function; using GetInfoCallback = std::function; using UploadResultCallback = std::function; using ProgressCallback = std::function; +class GetMappingRequest : public QObject { + Q_OBJECT +public: + GetMappingRequest(AssetPath path); + + Q_INVOKABLE void start(); + + AssetHash getHash() { return _hash; } + AssetServerError getError() { return _error; } + +signals: + void finished(GetMappingRequest* thisRequest); + +private: + AssetPath _path; + AssetHash _hash; + AssetServerError _error { AssetServerError::NoError }; +}; + + +class SetMappingRequest : public QObject { + Q_OBJECT +public: + SetMappingRequest(AssetPath path, AssetHash hash); + + Q_INVOKABLE void start(); + + AssetHash getHash() { return _hash; } + AssetServerError getError() { return _error; } + +signals: + void finished(SetMappingRequest* thisRequest); + +private: + AssetPath _path; + AssetHash _hash; + AssetServerError _error { AssetServerError::NoError }; +}; + + class AssetClient : public QObject, public Dependency { Q_OBJECT public: AssetClient(); - Q_INVOKABLE AssetRequest* createRequest(const QUrl& url); + Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetPath& path); + Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash); + Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const QString& extension); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data, const QString& extension); @@ -56,7 +100,7 @@ public slots: void clearCache(); private slots: - void handleAssetGetMappingReply(QSharedPointer message, SharedNodePointer senderNode); + void handleAssetMappingOperationReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetGetInfoReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetGetReply(QSharedPointer message, SharedNodePointer senderNode); void handleAssetUploadReply(QSharedPointer message, SharedNodePointer senderNode); @@ -64,7 +108,10 @@ private slots: void handleNodeKilled(SharedNodePointer node); private: - bool getAssetMapping(const QString& path, MappingOperationCallback callback); + bool getAssetMapping(const AssetHash& hash, MappingOperationCallback callback); + bool setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback); + bool deleteAssetMapping(const AssetHash& hash, MappingOperationCallback callback); + bool getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback); bool getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback); @@ -83,6 +130,8 @@ private: friend class AssetRequest; friend class AssetUpload; + friend class GetMappingRequest; + friend class SetMappingRequest; }; @@ -93,6 +142,9 @@ public: Q_INVOKABLE void uploadData(QString data, QString extension, QScriptValue callback); Q_INVOKABLE void downloadData(QString url, QScriptValue downloadComplete); + Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); + Q_INVOKABLE void getMapping(QString path, QScriptValue callback); + Q_INVOKABLE void getAllMappings(QString path, QScriptValue callback); protected: QSet _pendingRequests; QScriptEngine* _engine; From 2986aa1937e23203b1db21ce42279b3d6c08230f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 13:23:47 -0800 Subject: [PATCH 06/20] Revert AssetRequest back to using hash --- libraries/networking/src/AssetRequest.cpp | 4 ++-- libraries/networking/src/AssetRequest.h | 8 ++++---- libraries/networking/src/AssetResourceRequest.cpp | 14 +++++++++++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 505b64e080..0665590902 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -20,8 +20,8 @@ #include "NodeList.h" #include "ResourceCache.h" -AssetRequest::AssetRequest(const QString& url, const QString& extension) : - _url(url), +AssetRequest::AssetRequest(const QString& hash, const QString& extension) : + _hash(hash), _extension(extension) { } diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 288caac6df..049ab19ef4 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -39,7 +39,7 @@ public: UnknownError }; - AssetRequest(const QString& url, const QString& extension); + AssetRequest(const QString& hash, const QString& extension); Q_INVOKABLE void start(); @@ -52,15 +52,15 @@ signals: void finished(AssetRequest* thisRequest); void progress(qint64 totalReceived, qint64 total); -private slots: - void getAssetCallback(bool responseReceived, AssetServerError serverError, const QByteArray& data); +//private slots: + //void getAssetCallback(bool responseReceived, AssetServerError serverError, const QByteArray& data); private: State _state = NotStarted; Error _error = NoError; AssetInfo _info; uint64_t _totalReceived { 0 }; - QString _url; + QString _hash; QString _extension; QByteArray _data; int _numPendingRequests { 0 }; diff --git a/libraries/networking/src/AssetResourceRequest.cpp b/libraries/networking/src/AssetResourceRequest.cpp index 0025ba565a..92a804d421 100644 --- a/libraries/networking/src/AssetResourceRequest.cpp +++ b/libraries/networking/src/AssetResourceRequest.cpp @@ -21,9 +21,21 @@ AssetResourceRequest::~AssetResourceRequest() { } void AssetResourceRequest::doSend() { + auto parts = _url.path().split(".", QString::SkipEmptyParts); + auto hash = parts.length() > 0 ? parts[0] : ""; + auto extension = parts.length() > 1 ? parts[1] : ""; + + if (hash.length() != SHA256_HASH_HEX_LENGTH) { + _result = InvalidURL; + _state = Finished; + + emit finished(); + return; + } + // Make request to atp auto assetClient = DependencyManager::get(); - _assetRequest = assetClient->createRequest(_url); + _assetRequest = assetClient->createRequest(hash, extension); if (!_assetRequest) { _result = ServerUnavailable; From 292bc12f3740d8ca76bd0633b538204cb46addfa Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 13:24:12 -0800 Subject: [PATCH 07/20] Add read/writeString to PacketList and Packet --- libraries/networking/src/AssetUtils.h | 3 +++ libraries/networking/src/ReceivedMessage.cpp | 4 ++-- libraries/networking/src/ReceivedMessage.h | 1 - libraries/networking/src/udt/BasePacket.cpp | 14 ++++++++++++++ libraries/networking/src/udt/BasePacket.h | 5 ++++- libraries/networking/src/udt/PacketList.cpp | 6 ++++++ libraries/networking/src/udt/PacketList.h | 2 ++ 7 files changed, 31 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 58e7aa6146..f13c9ac729 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -20,6 +20,9 @@ using MessageID = uint32_t; using DataOffset = int64_t; +using AssetPath = QString; +using AssetHash = QString; + const size_t SHA256_HASH_LENGTH = 32; const size_t SHA256_HASH_HEX_LENGTH = 64; const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB diff --git a/libraries/networking/src/ReceivedMessage.cpp b/libraries/networking/src/ReceivedMessage.cpp index eed275dc1a..6ef2b7c7d4 100644 --- a/libraries/networking/src/ReceivedMessage.cpp +++ b/libraries/networking/src/ReceivedMessage.cpp @@ -110,8 +110,8 @@ QByteArray ReceivedMessage::readAll() { QString ReceivedMessage::readString() { uint32_t size; readPrimitive(&size); - Q_ASSERT(size <= _size - _position); - auto string = QString::fromUtf8(_data.constData(), size); + //Q_ASSERT(size <= _size - _position); + auto string = QString::fromUtf8(_data.constData() + _position, size); _position += size; return string; } diff --git a/libraries/networking/src/ReceivedMessage.h b/libraries/networking/src/ReceivedMessage.h index b813149bed..e9131c074f 100644 --- a/libraries/networking/src/ReceivedMessage.h +++ b/libraries/networking/src/ReceivedMessage.h @@ -88,7 +88,6 @@ private: QByteArray _data; QByteArray _headData; - std::atomic _size { true }; std::atomic _position { 0 }; std::atomic _numPackets { 0 }; diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 000573d241..e0b2a8ba3a 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -150,6 +150,20 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) { return data; } +qint64 BasePacket::writeString(const QString& string) { + QByteArray data = string.toUtf8(); + writePrimitive(static_cast(data.length())); + return writeData(data.constData(), data.length()); +} + +QString BasePacket::readString() { + uint32_t size {}; + readPrimitive(&size); + auto string = QString::fromUtf8(getPayload() + pos(), size); + seek(pos() + size); + return string; +} + bool BasePacket::reset() { if (isWritable()) { _payloadSize = 0; diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index 392a7c4384..846c05ecbf 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -77,7 +77,10 @@ public: using QIODevice::read; // Bring QIODevice::read methods to scope, otherwise they are hidden by folling method QByteArray read(qint64 maxSize); QByteArray readWithoutCopy(qint64 maxSize); // this can only be used if packet will stay in scope - + + qint64 writeString(const QString& string); + QString readString(); + template qint64 peekPrimitive(T* data); template qint64 readPrimitive(T* data); template qint64 writePrimitive(const T& data); diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 978df9b9bb..b12b7e6938 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -158,6 +158,12 @@ void PacketList::preparePackets(MessageNumber messageNumber) { const qint64 PACKET_LIST_WRITE_ERROR = -1; +qint64 PacketList::writeString(const QString& string) { + QByteArray data = string.toUtf8(); + writePrimitive(static_cast(data.length())); + return writeData(data.constData(), data.length()); +} + qint64 PacketList::writeData(const char* data, qint64 maxSize) { auto sizeRemaining = maxSize; diff --git a/libraries/networking/src/udt/PacketList.h b/libraries/networking/src/udt/PacketList.h index 4f1e46e616..875eb70e45 100644 --- a/libraries/networking/src/udt/PacketList.h +++ b/libraries/networking/src/udt/PacketList.h @@ -59,6 +59,8 @@ public: template qint64 readPrimitive(T* data); template qint64 writePrimitive(const T& data); + + qint64 writeString(const QString& string); protected: PacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false); From 37c0417ea607a5131541b13b9e36130450f10f85 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 16:31:49 -0800 Subject: [PATCH 08/20] Add GetAll and Delete mapping requests --- assignment-client/src/assets/AssetServer.cpp | 14 +- assignment-client/src/assets/AssetServer.h | 3 +- libraries/networking/src/AssetClient.cpp | 130 ++++++++++++++++++- libraries/networking/src/AssetClient.h | 48 ++++++- libraries/networking/src/AssetUtils.h | 4 + 5 files changed, 186 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index f685bbe1a9..109ded1b73 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -170,7 +170,7 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me AssetMappingOperationType operationType; message->readPrimitive(&operationType); - auto replyPacket = NLPacket::create(PacketType::AssetMappingOperationReply); + auto replyPacket = NLPacketList::create(PacketType::AssetMappingOperationReply); replyPacket->writePrimitive(messageID); switch (operationType) { @@ -191,6 +191,16 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me } break; } + case AssetMappingOperationType::GetAll: { + replyPacket->writePrimitive(AssetServerError::NoError); + auto count = _fileMapping.size(); + replyPacket->writePrimitive(count); + for (auto& kv : _fileMapping) { + replyPacket->writeString(kv.first); + replyPacket->writeString(kv.second); + } + break; + } case AssetMappingOperationType::Set: { QString assetPath = message->readString(); //auto assetHash = message->read(SHA256_HASH_LENGTH); @@ -210,7 +220,7 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me } auto nodeList = DependencyManager::get(); - nodeList->sendPacket(std::move(replyPacket), *senderNode); + nodeList->sendPacketList(std::move(replyPacket), *senderNode); } void AssetServer::handleAssetGetInfo(QSharedPointer message, SharedNodePointer senderNode) { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 1eee4feb94..545faa23fb 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -39,7 +39,6 @@ private slots: void sendStatsPacket(); private: - using Mapping = std::map; void loadMappingFromFile(); void writeMappingToFile(); @@ -55,7 +54,7 @@ private: static void writeError(NLPacketList* packetList, AssetServerError error); - Mapping _fileMapping; + AssetMapping _fileMapping; QDir _resourcesDirectory; QThreadPool _taskPool; }; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 93139af2d0..6f5e2eee03 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -50,6 +50,34 @@ void GetMappingRequest::start() { emit finished(this); }); }; + +GetAllMappingsRequest::GetAllMappingsRequest() { +}; + +void GetAllMappingsRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + return; + } + + auto assetClient = DependencyManager::get(); + assetClient->getAllAssetMappings([this](bool responseReceived, AssetServerError error, QSharedPointer message) { + // read message + _error = error; + if (!error) { + size_t numberOfMappings; + message->readPrimitive(&numberOfMappings); + AssetMapping mapping; + for (auto i = 0; i < numberOfMappings; ++i) { + auto path = message->readString(); + auto hash = message->readString(); + mapping[path] = hash; + } + } + emit finished(this); + }); +}; + SetMappingRequest::SetMappingRequest(AssetPath path, AssetHash hash) : _path(path), _hash(hash) { }; @@ -67,6 +95,23 @@ void SetMappingRequest::start() { }); }; +DeleteMappingRequest::DeleteMappingRequest(AssetPath path) : _path(path) { +}; + +void DeleteMappingRequest::start() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); + return; + } + + auto assetClient = DependencyManager::get(); + assetClient->deleteAssetMapping(_path, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + // read message + _error = error; + emit finished(this); + }); +}; + AssetClient::AssetClient() { @@ -174,11 +219,18 @@ bool haveAssetServer() { return true; } - GetMappingRequest* AssetClient::createGetMappingRequest(const AssetPath& path) { return new GetMappingRequest(path); } +GetAllMappingsRequest* AssetClient::createGetAllMappingsRequest() { + return new GetAllMappingsRequest(); +} + +DeleteMappingRequest* AssetClient::createDeleteMappingRequest(const AssetPath& path) { + return new DeleteMappingRequest(path); +} + SetMappingRequest* AssetClient::createSetMappingRequest(const AssetPath& path, const AssetHash& hash) { return new SetMappingRequest(path, hash); } @@ -404,7 +456,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer message, S } } -bool AssetClient::getAssetMapping(const AssetHash& hash, MappingOperationCallback callback) { +bool AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); @@ -416,7 +468,53 @@ bool AssetClient::getAssetMapping(const AssetHash& hash, MappingOperationCallbac packetList->writePrimitive(AssetMappingOperationType::Get); - packetList->writeString(hash.toUtf8()); + packetList->writeString(path); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + + _pendingMappingRequests[assetServer][messageID] = callback; + + return true; + } + + return false; +} + +bool AssetClient::getAllAssetMappings(MappingOperationCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(AssetMappingOperationType::GetAll); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + + _pendingMappingRequests[assetServer][messageID] = callback; + + return true; + } + + return false; +} + +bool AssetClient::deleteAssetMapping(const AssetPath& path, MappingOperationCallback callback) { + auto nodeList = DependencyManager::get(); + SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + + if (assetServer) { + auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); + + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(AssetMappingOperationType::Delete); + + packetList->writeString(path); nodeList->sendPacketList(std::move(packetList), *assetServer); @@ -440,8 +538,8 @@ bool AssetClient::setAssetMapping(const QString& path, const AssetHash& hash, Ma packetList->writePrimitive(AssetMappingOperationType::Set); - packetList->writeString(path.toUtf8()); - packetList->writeString(hash.toUtf8()); + packetList->writeString(path); + packetList->writeString(hash); nodeList->sendPacketList(std::move(packetList), *assetServer); @@ -653,5 +751,25 @@ void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) { request->start(); } -void AssetScriptingInterface::getAllMappings(QString path, QScriptValue callback) { +void AssetScriptingInterface::getAllMappings(QScriptValue callback) { + auto assetClient = DependencyManager::get(); + auto request = assetClient->createGetAllMappingsRequest(); + + connect(request, &GetAllMappingsRequest::finished, this, [this, callback](GetAllMappingsRequest* request) mutable { + auto mappings = request->getMappings(); + auto map = callback.engine()->newObject(); + + for (auto& kv : mappings ) { + map.setProperty(kv.first, kv.second); + } + + QScriptValueList args { uint8_t(request->getError()), map }; + + callback.call(_engine->currentContext()->thisObject(), args); + + request->deleteLater(); + + }); + + request->start(); } diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 0face50367..5e6a071ad8 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -16,6 +16,8 @@ #include #include +#include + #include #include "AssetUtils.h" @@ -61,7 +63,6 @@ private: AssetServerError _error { AssetServerError::NoError }; }; - class SetMappingRequest : public QObject { Q_OBJECT public: @@ -81,6 +82,42 @@ private: AssetServerError _error { AssetServerError::NoError }; }; +class DeleteMappingRequest : public QObject { + Q_OBJECT +public: + DeleteMappingRequest(AssetPath path); + + Q_INVOKABLE void start(); + + AssetServerError getError() { return _error; } + +signals: + void finished(DeleteMappingRequest* thisRequest); + +private: + AssetPath _path; + AssetServerError _error { AssetServerError::NoError }; +}; + +class GetAllMappingsRequest : public QObject { + Q_OBJECT +public: + GetAllMappingsRequest(); + + Q_INVOKABLE void start(); + + AssetServerError getError() { return _error; } + AssetMapping getMappings() { return _mappings; } + +signals: + void finished(GetAllMappingsRequest* thisRequest); + +private: + std::map _mappings; + AssetServerError _error { AssetServerError::NoError }; +}; + + class AssetClient : public QObject, public Dependency { Q_OBJECT @@ -88,6 +125,8 @@ public: AssetClient(); Q_INVOKABLE GetMappingRequest* createGetMappingRequest(const AssetPath& path); + Q_INVOKABLE GetAllMappingsRequest* createGetAllMappingsRequest(); + Q_INVOKABLE DeleteMappingRequest* createDeleteMappingRequest(const AssetPath& path); Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash); Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, const QString& extension); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); @@ -109,8 +148,9 @@ private slots: private: bool getAssetMapping(const AssetHash& hash, MappingOperationCallback callback); + bool getAllAssetMappings(MappingOperationCallback callback); bool setAssetMapping(const QString& path, const AssetHash& hash, MappingOperationCallback callback); - bool deleteAssetMapping(const AssetHash& hash, MappingOperationCallback callback); + bool deleteAssetMapping(const AssetPath& path, MappingOperationCallback callback); bool getAssetInfo(const QString& hash, const QString& extension, GetInfoCallback callback); bool getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, @@ -131,7 +171,9 @@ private: friend class AssetRequest; friend class AssetUpload; friend class GetMappingRequest; + friend class GetAllMappingsRequest; friend class SetMappingRequest; + friend class DeleteMappingRequest; }; @@ -144,7 +186,7 @@ public: Q_INVOKABLE void downloadData(QString url, QScriptValue downloadComplete); Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); Q_INVOKABLE void getMapping(QString path, QScriptValue callback); - Q_INVOKABLE void getAllMappings(QString path, QScriptValue callback); + Q_INVOKABLE void getAllMappings(QScriptValue callback); protected: QSet _pendingRequests; QScriptEngine* _engine; diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index f13c9ac729..99e8fdc386 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -14,6 +14,8 @@ #include +#include + #include #include @@ -22,6 +24,7 @@ using DataOffset = int64_t; using AssetPath = QString; using AssetHash = QString; +using AssetMapping = std::map; const size_t SHA256_HASH_LENGTH = 32; const size_t SHA256_HASH_HEX_LENGTH = 64; @@ -37,6 +40,7 @@ enum AssetServerError : uint8_t { enum AssetMappingOperationType : uint8_t { Get = 0, + GetAll, Set, Delete }; From bc99e871e16fa77955760b1cf8b775f99d8073ad Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 7 Mar 2016 16:51:49 -0800 Subject: [PATCH 09/20] Add asset mapping cache --- libraries/networking/src/AssetClient.cpp | 31 +++++++++++++++++++++--- libraries/networking/src/AssetClient.h | 2 ++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 6f5e2eee03..d518961216 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -40,12 +40,13 @@ void GetMappingRequest::start() { } auto assetClient = DependencyManager::get(); - assetClient->getAssetMapping(_path, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + assetClient->getAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { // read message _error = error; if (!error) { //_hash = message->read(SHA256_HASH_HEX_LENGTH); _hash = message->readString(); + assetClient->_mappingCache[_path] = _hash; } emit finished(this); }); @@ -61,17 +62,19 @@ void GetAllMappingsRequest::start() { } auto assetClient = DependencyManager::get(); - assetClient->getAllAssetMappings([this](bool responseReceived, AssetServerError error, QSharedPointer message) { + assetClient->getAllAssetMappings([this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { // read message _error = error; if (!error) { size_t numberOfMappings; message->readPrimitive(&numberOfMappings); AssetMapping mapping; + assetClient->_mappingCache.clear(); for (auto i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); auto hash = message->readString(); mapping[path] = hash; + assetClient->_mappingCache[path] = hash; } } emit finished(this); @@ -88,9 +91,12 @@ void SetMappingRequest::start() { } auto assetClient = DependencyManager::get(); - assetClient->setAssetMapping(_path, _hash, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + assetClient->setAssetMapping(_path, _hash, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { // read message _error = error; + if (!error) { + assetClient->_mappingCache[_path] = _hash; + } emit finished(this); }); }; @@ -105,9 +111,12 @@ void DeleteMappingRequest::start() { } auto assetClient = DependencyManager::get(); - assetClient->deleteAssetMapping(_path, [this](bool responseReceived, AssetServerError error, QSharedPointer message) { + assetClient->deleteAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { // read message _error = error; + if (!error) { + assetClient->_mappingCache.remove(_path); + } emit finished(this); }); }; @@ -172,6 +181,8 @@ void AssetClient::clearCache() { return; } + _mappingCache.clear(); + if (auto cache = NetworkAccessManager::getInstance().cache()) { qDebug() << "AssetClient::clearCache(): Clearing disk cache."; cache->clear(); @@ -650,6 +661,18 @@ void AssetClient::handleNodeKilled(SharedNodePointer node) { messageMapIt->second.clear(); } } + + { + auto messageMapIt = _pendingMappingRequests.find(node); + if (messageMapIt != _pendingMappingRequests.end()) { + for (const auto& value : messageMapIt->second) { + value.second(false, AssetServerError::PermissionDenied, QSharedPointer()); + } + messageMapIt->second.clear(); + } + } + + _mappingCache.clear(); } void AssetScriptingInterface::uploadData(QString data, QString extension, QScriptValue callback) { diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 5e6a071ad8..10fad14ee9 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -167,6 +167,8 @@ private: std::unordered_map> _pendingRequests; std::unordered_map> _pendingInfoRequests; std::unordered_map> _pendingUploads; + + QHash _mappingCache; friend class AssetRequest; friend class AssetUpload; From cf9d4c256ad6db9202db18e67e47de1e1618aaec Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:10:48 -0800 Subject: [PATCH 10/20] Add proper cache checking to GetAssetMapping --- libraries/networking/src/AssetClient.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index d518961216..f690c17cbd 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -40,6 +40,15 @@ void GetMappingRequest::start() { } auto assetClient = DependencyManager::get(); + + // Check cache + auto it = assetClient->_mappingCache.constFind(_path); + if (it != assetClient->_mappingCache.constEnd()) { + _hash = it.value(); + emit finished(this); + return; + } + assetClient->getAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { // read message _error = error; From a18486d97db8c13baf11b99d380d44eb575d2873 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:11:37 -0800 Subject: [PATCH 11/20] Fix GetAllMappingsRequest not storing mappings --- libraries/networking/src/AssetClient.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index f690c17cbd..70064c3f69 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -77,12 +77,11 @@ void GetAllMappingsRequest::start() { if (!error) { size_t numberOfMappings; message->readPrimitive(&numberOfMappings); - AssetMapping mapping; assetClient->_mappingCache.clear(); for (auto i = 0; i < numberOfMappings; ++i) { auto path = message->readString(); auto hash = message->readString(); - mapping[path] = hash; + _mappings[path] = hash; assetClient->_mappingCache[path] = hash; } } From 6ca4b72479cf75e35f53c0f064912b5bdc200d6c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:11:57 -0800 Subject: [PATCH 12/20] Add deleteMapping to asset scripting interface --- libraries/networking/src/AssetClient.cpp | 16 ++++++++++++++++ libraries/networking/src/AssetClient.h | 1 + 2 files changed, 17 insertions(+) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 70064c3f69..d8c356aa07 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -782,6 +782,22 @@ void AssetScriptingInterface::getMapping(QString path, QScriptValue callback) { request->start(); } +void AssetScriptingInterface::deleteMapping(QString path, QScriptValue callback) { + auto assetClient = DependencyManager::get(); + auto request = assetClient->createDeleteMappingRequest(path); + + connect(request, &DeleteMappingRequest::finished, this, [this, callback](DeleteMappingRequest* request) mutable { + QScriptValueList args { uint8_t(request->getError()) }; + + callback.call(_engine->currentContext()->thisObject(), args); + + request->deleteLater(); + + }); + + request->start(); +} + void AssetScriptingInterface::getAllMappings(QScriptValue callback) { auto assetClient = DependencyManager::get(); auto request = assetClient->createGetAllMappingsRequest(); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index 10fad14ee9..f95f4bc9bc 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -188,6 +188,7 @@ public: Q_INVOKABLE void downloadData(QString url, QScriptValue downloadComplete); Q_INVOKABLE void setMapping(QString path, QString hash, QScriptValue callback); Q_INVOKABLE void getMapping(QString path, QScriptValue callback); + Q_INVOKABLE void deleteMapping(QString path, QScriptValue callback); Q_INVOKABLE void getAllMappings(QScriptValue callback); protected: QSet _pendingRequests; From 7db296fa69f3e16143c927416758b10f58725190 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:23:48 -0800 Subject: [PATCH 13/20] Remove unused get/set/deleteMapping in AssetServer --- assignment-client/src/assets/AssetServer.cpp | 12 ------------ assignment-client/src/assets/AssetServer.h | 9 --------- libraries/networking/src/AssetClient.cpp | 1 - 3 files changed, 22 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 109ded1b73..d3198c0f30 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -376,15 +376,3 @@ void AssetServer::loadMappingFromFile() { void AssetServer::writeMappingToFile() { } - -AssetHash AssetServer::getMapping(AssetPath path) { - return _fileMapping[path]; -} - -void AssetServer::setMapping(AssetPath path, AssetHash hash) { - _fileMapping[path] = hash; -} - -bool AssetServer::deleteMapping(AssetPath path) { - return _fileMapping.erase(path) > 0; -} diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 545faa23fb..e87c0797c8 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -43,15 +43,6 @@ private: void loadMappingFromFile(); void writeMappingToFile(); - /// Return the hash mapping for AssetPath `path` - AssetHash getMapping(AssetPath path); - - /// Set the mapping for path to hash - void setMapping(AssetPath path, AssetHash hash); - - /// Delete mapping `path`. Return `true` if mapping existed, else `false`. - bool deleteMapping(AssetPath path); - static void writeError(NLPacketList* packetList, AssetServerError error); AssetMapping _fileMapping; diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index d8c356aa07..27a3acc00b 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -478,7 +478,6 @@ void AssetClient::handleAssetGetReply(QSharedPointer message, S bool AssetClient::getAssetMapping(const AssetPath& path, MappingOperationCallback callback) { auto nodeList = DependencyManager::get(); SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); - if (assetServer) { auto packetList = NLPacketList::create(PacketType::AssetMappingOperation, QByteArray(), true, true); From 89b37f78f03f971aa0916b72e9761edf75bdc449 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:26:50 -0800 Subject: [PATCH 14/20] Make AssetServer response reliable and ordered --- assignment-client/src/assets/AssetServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index d3198c0f30..b7d02cdb85 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -170,7 +170,7 @@ void AssetServer::handleAssetMappingOperation(QSharedPointer me AssetMappingOperationType operationType; message->readPrimitive(&operationType); - auto replyPacket = NLPacketList::create(PacketType::AssetMappingOperationReply); + auto replyPacket = NLPacketList::create(PacketType::AssetMappingOperationReply, QByteArray(), true, true); replyPacket->writePrimitive(messageID); switch (operationType) { From e3523c86046a6459e78ed0de5350d19ecebfe168 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:28:31 -0800 Subject: [PATCH 15/20] Remove commented out function --- libraries/networking/src/AssetClient.cpp | 28 ------------------------ 1 file changed, 28 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 27a3acc00b..d40aa741a9 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -297,34 +297,6 @@ AssetUpload* AssetClient::createUpload(const QByteArray& data, const QString& ex } } -//bool AssetClient::setAssetMapping(const QString& path, MappingOperationCallback callback) { -// auto nodeList = DependencyManager::get(); -// SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); -// -// if (assetServer) { -// auto messageID = ++_currentID; -// -// auto payload = path.toLatin1(); -// auto payloadSize = sizeof(messageID) + payload.size(); -// auto packet = NLPacket::create(PacketType::AssetMappingOperation, payloadSize, true); -// -// qCDebug(asset_client) << "Requesting mapping for" << path << "from asset-server."; -// -// packet->writePrimitive(messageID); -// -// auto bytesWritten = packet->write(payload); -// Q_ASSERT(bytesWritten == payload.size()); -// -// nodeList->sendPacket(std::move(packet), *assetServer); -// -// _pendingMappingRequests[assetServer][messageID] = callback; -// -// return true; -// } -// -// return false; -//} -// bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback, ProgressCallback progressCallback) { if (hash.length() != SHA256_HASH_HEX_LENGTH) { From 0a75a9479014737e72cc7e194d06697dea71196c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:31:40 -0800 Subject: [PATCH 16/20] Remove commented out private slot --- libraries/networking/src/AssetRequest.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 049ab19ef4..3c3459b15d 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -52,9 +52,6 @@ signals: void finished(AssetRequest* thisRequest); void progress(qint64 totalReceived, qint64 total); -//private slots: - //void getAssetCallback(bool responseReceived, AssetServerError serverError, const QByteArray& data); - private: State _state = NotStarted; Error _error = NoError; From e06c1591fbfb7aa86c7e5dbc7bf114338194a833 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 10:43:26 -0800 Subject: [PATCH 17/20] Remove initializer --- libraries/networking/src/udt/BasePacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index e0b2a8ba3a..dba241f221 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -157,7 +157,7 @@ qint64 BasePacket::writeString(const QString& string) { } QString BasePacket::readString() { - uint32_t size {}; + uint32_t size; readPrimitive(&size); auto string = QString::fromUtf8(getPayload() + pos(), size); seek(pos() + size); From c9bd67bc3e9755add744c1ad0e9c34a5c1ead1cd Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 11:18:10 -0800 Subject: [PATCH 18/20] Add MappingRequest superclass and mapping specific errors --- libraries/networking/src/AssetClient.cpp | 90 +++++++++++++++++++----- libraries/networking/src/AssetClient.h | 56 +++++++++------ 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index d40aa741a9..b447380e5c 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -30,14 +30,18 @@ MessageID AssetClient::_currentID = 0; -GetMappingRequest::GetMappingRequest(AssetPath path) : _path(path) { -}; - -void GetMappingRequest::start() { +void MappingRequest::start() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); return; } + doStart(); +}; + +GetMappingRequest::GetMappingRequest(AssetPath path) : _path(path) { +}; + +void GetMappingRequest::doStart() { auto assetClient = DependencyManager::get(); @@ -50,9 +54,23 @@ void GetMappingRequest::start() { } assetClient->getAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { - // read message - _error = error; - if (!error) { + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + case AssetServerError::AssetNotFound: + _error = NotFound; + break; + default: + _error = UnknownError; + break; + } + } + + if (!_error) { //_hash = message->read(SHA256_HASH_HEX_LENGTH); _hash = message->readString(); assetClient->_mappingCache[_path] = _hash; @@ -64,7 +82,7 @@ void GetMappingRequest::start() { GetAllMappingsRequest::GetAllMappingsRequest() { }; -void GetAllMappingsRequest::start() { +void GetAllMappingsRequest::doStart() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); return; @@ -72,8 +90,20 @@ void GetAllMappingsRequest::start() { auto assetClient = DependencyManager::get(); assetClient->getAllAssetMappings([this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { - // read message - _error = error; + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + default: + _error = UnknownError; + break; + } + } + + if (!error) { size_t numberOfMappings; message->readPrimitive(&numberOfMappings); @@ -92,7 +122,7 @@ void GetAllMappingsRequest::start() { SetMappingRequest::SetMappingRequest(AssetPath path, AssetHash hash) : _path(path), _hash(hash) { }; -void SetMappingRequest::start() { +void SetMappingRequest::doStart() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); return; @@ -100,8 +130,22 @@ void SetMappingRequest::start() { auto assetClient = DependencyManager::get(); assetClient->setAssetMapping(_path, _hash, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { - // read message - _error = error; + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + case AssetServerError::PermissionDenied: + _error = PermissionDenied; + break; + default: + _error = UnknownError; + break; + } + } + if (!error) { assetClient->_mappingCache[_path] = _hash; } @@ -112,7 +156,7 @@ void SetMappingRequest::start() { DeleteMappingRequest::DeleteMappingRequest(AssetPath path) : _path(path) { }; -void DeleteMappingRequest::start() { +void DeleteMappingRequest::doStart() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); return; @@ -120,8 +164,22 @@ void DeleteMappingRequest::start() { auto assetClient = DependencyManager::get(); assetClient->deleteAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { - // read message - _error = error; + if (!responseReceived) { + _error = NetworkError; + } else { + switch (error) { + case AssetServerError::NoError: + _error = NoError; + break; + case AssetServerError::PermissionDenied: + _error = PermissionDenied; + break; + default: + _error = UnknownError; + break; + } + } + if (!error) { assetClient->_mappingCache.remove(_path); } diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index f95f4bc9bc..b51e7f5316 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -43,78 +43,94 @@ using GetInfoCallback = std::function; using ProgressCallback = std::function; +class MappingRequest : public QObject { + Q_OBJECT +public: + enum Error { + NoError, + NotFound, + NetworkError, + PermissionDenied, + UnknownError + }; -class GetMappingRequest : public QObject { + Q_INVOKABLE void start(); + Error getError() const { return _error; } + +protected: + Error _error { NoError }; + +private: + virtual void doStart() = 0; +}; + + +class GetMappingRequest : public MappingRequest { Q_OBJECT public: GetMappingRequest(AssetPath path); - Q_INVOKABLE void start(); - - AssetHash getHash() { return _hash; } - AssetServerError getError() { return _error; } + AssetHash getHash() const { return _hash; } signals: void finished(GetMappingRequest* thisRequest); private: + virtual void doStart() override; + AssetPath _path; AssetHash _hash; - AssetServerError _error { AssetServerError::NoError }; }; -class SetMappingRequest : public QObject { +class SetMappingRequest : public MappingRequest { Q_OBJECT public: SetMappingRequest(AssetPath path, AssetHash hash); - Q_INVOKABLE void start(); - - AssetHash getHash() { return _hash; } - AssetServerError getError() { return _error; } + AssetHash getHash() const { return _hash; } signals: void finished(SetMappingRequest* thisRequest); private: + virtual void doStart() override; + AssetPath _path; AssetHash _hash; - AssetServerError _error { AssetServerError::NoError }; }; -class DeleteMappingRequest : public QObject { +class DeleteMappingRequest : public MappingRequest { Q_OBJECT public: DeleteMappingRequest(AssetPath path); Q_INVOKABLE void start(); - AssetServerError getError() { return _error; } - signals: void finished(DeleteMappingRequest* thisRequest); private: + virtual void doStart() override; + AssetPath _path; - AssetServerError _error { AssetServerError::NoError }; }; -class GetAllMappingsRequest : public QObject { +class GetAllMappingsRequest : public MappingRequest { Q_OBJECT public: GetAllMappingsRequest(); Q_INVOKABLE void start(); - AssetServerError getError() { return _error; } - AssetMapping getMappings() { return _mappings; } + AssetMapping getMappings() const { return _mappings; } signals: void finished(GetAllMappingsRequest* thisRequest); private: + virtual void doStart() override; + std::map _mappings; - AssetServerError _error { AssetServerError::NoError }; }; From 93d3fb5ccfc983c8c3e154baa3da6ef774ea5688 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 11:19:08 -0800 Subject: [PATCH 19/20] Fix network error response for mapping requests --- libraries/networking/src/AssetClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index b447380e5c..1df9fae192 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -703,7 +703,7 @@ void AssetClient::handleNodeKilled(SharedNodePointer node) { auto messageMapIt = _pendingMappingRequests.find(node); if (messageMapIt != _pendingMappingRequests.end()) { for (const auto& value : messageMapIt->second) { - value.second(false, AssetServerError::PermissionDenied, QSharedPointer()); + value.second(false, AssetServerError::NoError, QSharedPointer()); } messageMapIt->second.clear(); } From 5368ae3f9144c1b2cb01c7330eb29433c642e998 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 8 Mar 2016 11:20:15 -0800 Subject: [PATCH 20/20] Remove extraneous thread checks in MappingRequest subclasses --- libraries/networking/src/AssetClient.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 1df9fae192..7631ec5292 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -83,11 +83,6 @@ GetAllMappingsRequest::GetAllMappingsRequest() { }; void GetAllMappingsRequest::doStart() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); - return; - } - auto assetClient = DependencyManager::get(); assetClient->getAllAssetMappings([this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { if (!responseReceived) { @@ -123,11 +118,6 @@ SetMappingRequest::SetMappingRequest(AssetPath path, AssetHash hash) : _path(pat }; void SetMappingRequest::doStart() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); - return; - } - auto assetClient = DependencyManager::get(); assetClient->setAssetMapping(_path, _hash, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { if (!responseReceived) { @@ -157,11 +147,6 @@ DeleteMappingRequest::DeleteMappingRequest(AssetPath path) : _path(path) { }; void DeleteMappingRequest::doStart() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "start", Qt::AutoConnection); - return; - } - auto assetClient = DependencyManager::get(); assetClient->deleteAssetMapping(_path, [this, assetClient](bool responseReceived, AssetServerError error, QSharedPointer message) { if (!responseReceived) {