From 397a29039effa0768fba2220488144d58d05d524 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 17 Apr 2017 19:08:06 -0700 Subject: [PATCH] add support for byte range requests to ATP --- .../src/assets/SendAssetTask.cpp | 62 ++++++-- libraries/networking/src/AssetClient.cpp | 5 +- libraries/networking/src/AssetClient.h | 3 +- libraries/networking/src/AssetRequest.cpp | 133 +++++++----------- libraries/networking/src/AssetRequest.h | 10 +- libraries/networking/src/ByteRange.h | 24 ++++ libraries/networking/src/ResourceRequest.h | 7 +- 7 files changed, 132 insertions(+), 112 deletions(-) create mode 100644 libraries/networking/src/ByteRange.h diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index ca8733d660..4a49deabde 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -11,6 +11,8 @@ #include "SendAssetTask.h" +#include + #include #include @@ -21,6 +23,7 @@ #include #include "AssetUtils.h" +#include "ByteRange.h" #include "ClientServerUtils.h" SendAssetTask::SendAssetTask(QSharedPointer message, const SharedNodePointer& sendToNode, const QDir& resourcesDir) : @@ -34,20 +37,21 @@ SendAssetTask::SendAssetTask(QSharedPointer message, const Shar void SendAssetTask::run() { MessageID messageID; - DataOffset start, end; - + ByteRange byteRange; + _message->readPrimitive(&messageID); QByteArray assetHash = _message->read(SHA256_HASH_LENGTH); // `start` and `end` indicate the range of data to retrieve for the asset identified by `assetHash`. // `start` is inclusive, `end` is exclusive. Requesting `start` = 1, `end` = 10 will retrieve 9 bytes of data, // starting at index 1. - _message->readPrimitive(&start); - _message->readPrimitive(&end); + _message->readPrimitive(&byteRange.fromInclusive); + _message->readPrimitive(&byteRange.toExclusive); QString hexHash = assetHash.toHex(); - qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end; + qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " + << byteRange.fromInclusive << " to " << byteRange.toExclusive; qDebug() << "Starting task to send asset: " << hexHash << " for messageID " << messageID; auto replyPacketList = NLPacketList::create(PacketType::AssetGetReply, QByteArray(), true, true); @@ -56,7 +60,7 @@ void SendAssetTask::run() { replyPacketList->writePrimitive(messageID); - if (end <= start) { + if (byteRange.toExclusive < byteRange.fromInclusive) { replyPacketList->writePrimitive(AssetServerError::InvalidByteRange); } else { QString filePath = _resourcesDir.filePath(QString(hexHash)); @@ -64,15 +68,47 @@ void SendAssetTask::run() { QFile file { filePath }; if (file.open(QIODevice::ReadOnly)) { - if (file.size() < end) { + if (byteRange.isSet()) { + // if the byte range is not set, force it to be from 0 to the end of the file + byteRange.fromInclusive = 0; + byteRange.toExclusive = file.size(); + } + + if (file.size() < std::abs(byteRange.toExclusive)) { replyPacketList->writePrimitive(AssetServerError::InvalidByteRange); - qCDebug(networking) << "Bad byte range: " << hexHash << " " << start << ":" << end; + qCDebug(networking) << "Bad byte range: " << hexHash << " " + << byteRange.fromInclusive << ":" << byteRange.toExclusive; } else { - auto size = end - start; - file.seek(start); - replyPacketList->writePrimitive(AssetServerError::NoError); - replyPacketList->writePrimitive(size); - replyPacketList->write(file.read(size)); + // we have a valid byte range, handle it and send the asset + auto size = byteRange.size(); + + if (byteRange.fromInclusive > 0) { + // this range is positive, meaning we just need to seek into the file and then read from there + file.seek(byteRange.fromInclusive); + replyPacketList->writePrimitive(AssetServerError::NoError); + replyPacketList->writePrimitive(size); + replyPacketList->write(file.read(size)); + } else { + // this range is negative, at least the first part of the read will be back into the end of the file + + // seek to the part of the file where the negative range begins + file.seek(file.size() + byteRange.fromInclusive); + + replyPacketList->writePrimitive(AssetServerError::NoError); + replyPacketList->writePrimitive(size); + + // first write everything from the negative range to the end of the file + replyPacketList->write(file.read(-byteRange.fromInclusive)); + + if (byteRange.toExclusive != 0) { + // this range has a portion that is at the front of the file + + // seek to the beginning and read what is left over + file.seek(0); + replyPacketList->write(file.read(byteRange.toExclusive)); + } + } + qCDebug(networking) << "Sending asset: " << hexHash; } file.close(); diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 37b1af0996..48f8bb87f9 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -67,7 +67,6 @@ void AssetClient::init() { } } - void AssetClient::cacheInfoRequest(QObject* reciever, QString slot) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "cacheInfoRequest", Qt::QueuedConnection, @@ -182,8 +181,8 @@ RenameMappingRequest* AssetClient::createRenameMappingRequest(const AssetPath& o return request; } -AssetRequest* AssetClient::createRequest(const AssetHash& hash) { - auto request = new AssetRequest(hash); +AssetRequest* AssetClient::createRequest(const AssetHash& hash, ByteRange byteRange) { + auto request = new AssetRequest(hash, byteRange); // Move to the AssetClient thread in case we are not currently on that thread (which will usually be the case) request->moveToThread(thread()); diff --git a/libraries/networking/src/AssetClient.h b/libraries/networking/src/AssetClient.h index c0d58cd8e6..b204fab47e 100644 --- a/libraries/networking/src/AssetClient.h +++ b/libraries/networking/src/AssetClient.h @@ -21,6 +21,7 @@ #include #include "AssetUtils.h" +#include "ByteRange.h" #include "ClientServerUtils.h" #include "LimitedNodeList.h" #include "Node.h" @@ -55,7 +56,7 @@ public: Q_INVOKABLE DeleteMappingsRequest* createDeleteMappingsRequest(const AssetPathList& paths); Q_INVOKABLE SetMappingRequest* createSetMappingRequest(const AssetPath& path, const AssetHash& hash); Q_INVOKABLE RenameMappingRequest* createRenameMappingRequest(const AssetPath& oldPath, const AssetPath& newPath); - Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash); + Q_INVOKABLE AssetRequest* createRequest(const AssetHash& hash, ByteRange byteRange = ByteRange()); Q_INVOKABLE AssetUpload* createUpload(const QString& filename); Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data); diff --git a/libraries/networking/src/AssetRequest.cpp b/libraries/networking/src/AssetRequest.cpp index 8d663933ca..e54a058ac2 100644 --- a/libraries/networking/src/AssetRequest.cpp +++ b/libraries/networking/src/AssetRequest.cpp @@ -23,10 +23,12 @@ static int requestID = 0; -AssetRequest::AssetRequest(const QString& hash) : +AssetRequest::AssetRequest(const QString& hash, ByteRange byteRange) : _requestID(++requestID), - _hash(hash) + _hash(hash), + _byteRange(byteRange) { + } AssetRequest::~AssetRequest() { @@ -34,9 +36,6 @@ AssetRequest::~AssetRequest() { if (_assetRequestID) { assetClient->cancelGetAssetRequest(_assetRequestID); } - if (_assetInfoRequestID) { - assetClient->cancelGetAssetInfoRequest(_assetInfoRequestID); - } } void AssetRequest::start() { @@ -62,108 +61,74 @@ void AssetRequest::start() { // Try to load from cache _data = loadFromCache(getUrl()); if (!_data.isNull()) { - _info.hash = _hash; - _info.size = _data.size(); _error = NoError; _state = Finished; emit finished(this); return; } - - _state = WaitingForInfo; - + + _state = WaitingForData; + auto assetClient = DependencyManager::get(); - _assetInfoRequestID = assetClient->getAssetInfo(_hash, - [this](bool responseReceived, AssetServerError serverError, AssetInfo info) { + auto that = QPointer(this); // Used to track the request's lifetime + auto hash = _hash; - _assetInfoRequestID = INVALID_MESSAGE_ID; + _assetRequestID = assetClient->getAsset(_hash, _byteRange.fromInclusive, _byteRange.toExclusive, + [this, that, hash](bool responseReceived, AssetServerError serverError, const QByteArray& data) { - _info = info; + if (!that) { + qCWarning(asset_client) << "Got reply for dead asset request " << hash << "- error code" << _error; + // If the request is dead, return + return; + } + _assetRequestID = INVALID_MESSAGE_ID; if (!responseReceived) { _error = NetworkError; } else if (serverError != AssetServerError::NoError) { - switch(serverError) { + switch (serverError) { case AssetServerError::AssetNotFound: _error = NotFound; break; + case AssetServerError::InvalidByteRange: + _error = InvalidByteRange; + break; default: _error = UnknownError; break; } - } + } else { + if (_byteRange.isSet()) { + // we had a byte range, the size of the data does not match what we expect, so we return an error + if (data.size() != _byteRange.size()) { + _error = SizeVerificationFailed; + } + } else if (hashData(data).toHex() != _hash) { + // the hash of the received data does not match what we expect, so we return an error + _error = HashVerificationFailed; + } + if (_error == NoError) { + _data = data; + _totalReceived += data.size(); + emit progress(_totalReceived, data.size()); + + saveToCache(getUrl(), data); + } + } + if (_error != NoError) { - qCWarning(asset_client) << "Got error retrieving asset info for" << _hash; - _state = Finished; - emit finished(this); - + qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error; + } + + _state = Finished; + emit finished(this); + }, [this, that](qint64 totalReceived, qint64 total) { + if (!that) { + // If the request is dead, return return; } - - _state = WaitingForData; - _data.resize(info.size); - - qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes"; - - int start = 0, end = _info.size; - - auto assetClient = DependencyManager::get(); - auto that = QPointer(this); // Used to track the request's lifetime - auto hash = _hash; - _assetRequestID = assetClient->getAsset(_hash, start, end, - [this, that, hash, start, end](bool responseReceived, AssetServerError serverError, const QByteArray& data) { - if (!that) { - qCWarning(asset_client) << "Got reply for dead asset request " << hash << "- error code" << _error; - // If the request is dead, return - return; - } - _assetRequestID = INVALID_MESSAGE_ID; - - if (!responseReceived) { - _error = NetworkError; - } else if (serverError != AssetServerError::NoError) { - switch (serverError) { - case AssetServerError::AssetNotFound: - _error = NotFound; - break; - case AssetServerError::InvalidByteRange: - _error = InvalidByteRange; - break; - default: - _error = UnknownError; - break; - } - } else { - Q_ASSERT(data.size() == (end - start)); - - // we need to check the hash of the received data to make sure it matches what we expect - if (hashData(data).toHex() == _hash) { - memcpy(_data.data() + start, data.constData(), data.size()); - _totalReceived += data.size(); - emit progress(_totalReceived, _info.size); - - saveToCache(getUrl(), data); - } else { - // hash doesn't match - we have an error - _error = HashVerificationFailed; - } - - } - - if (_error != NoError) { - qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error; - } - - _state = Finished; - emit finished(this); - }, [this, that](qint64 totalReceived, qint64 total) { - if (!that) { - // If the request is dead, return - return; - } - emit progress(totalReceived, total); - }); + emit progress(totalReceived, total); }); } diff --git a/libraries/networking/src/AssetRequest.h b/libraries/networking/src/AssetRequest.h index 1632a55336..5120b8066e 100644 --- a/libraries/networking/src/AssetRequest.h +++ b/libraries/networking/src/AssetRequest.h @@ -17,15 +17,15 @@ #include #include "AssetClient.h" - #include "AssetUtils.h" +#include "ByteRange.h" + class AssetRequest : public QObject { Q_OBJECT public: enum State { NotStarted = 0, - WaitingForInfo, WaitingForData, Finished }; @@ -36,11 +36,12 @@ public: InvalidByteRange, InvalidHash, HashVerificationFailed, + SizeVerificationFailed, NetworkError, UnknownError }; - AssetRequest(const QString& hash); + AssetRequest(const QString& hash, ByteRange byteRange); virtual ~AssetRequest() override; Q_INVOKABLE void start(); @@ -59,13 +60,12 @@ private: int _requestID; State _state = NotStarted; Error _error = NoError; - AssetInfo _info; uint64_t _totalReceived { 0 }; QString _hash; QByteArray _data; int _numPendingRequests { 0 }; MessageID _assetRequestID { INVALID_MESSAGE_ID }; - MessageID _assetInfoRequestID { INVALID_MESSAGE_ID }; + ByteRange _byteRange; }; #endif diff --git a/libraries/networking/src/ByteRange.h b/libraries/networking/src/ByteRange.h new file mode 100644 index 0000000000..2fe5264a49 --- /dev/null +++ b/libraries/networking/src/ByteRange.h @@ -0,0 +1,24 @@ +// +// ByteRange.h +// libraries/networking/src +// +// Created by Stephen Birarda on 4/17/17. +// Copyright 2017 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ByteRange_h +#define hifi_ByteRange_h + +struct ByteRange { + int64_t fromInclusive { 0 }; + int64_t toExclusive { 0 }; + + bool isSet() { return fromInclusive < 0 || fromInclusive < toExclusive; } + int64_t size() { return toExclusive - fromInclusive; } +}; + + +#endif // hifi_ByteRange_h diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index 01ca62cf05..77fc8d2591 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -17,12 +17,7 @@ #include -struct ByteRange { - int64_t fromInclusive { 0 }; - int64_t toExclusive { 0 }; - - bool isSet() { return fromInclusive < 0 || fromInclusive < toExclusive; } -}; +#include "ByteRange.h" class ResourceRequest : public QObject { Q_OBJECT