From 86a7a42d24fb1512bf0deca0961e871110118538 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 27 Aug 2015 14:22:21 -0700 Subject: [PATCH] use compact hash representation for asset-server --- assignment-client/src/assets/AssetServer.cpp | 36 ++++++++++--------- assignment-client/src/assets/AssetServer.h | 2 +- .../src/assets/SendAssetTask.cpp | 4 +-- libraries/networking/src/AssetClient.cpp | 23 ++++++------ libraries/networking/src/AssetUtils.h | 3 +- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index b2a4eefabf..181b45d464 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -58,7 +58,7 @@ void AssetServer::run() { // Scan for new files qDebug() << "Looking for new files in asset directory"; auto files = _resourcesDirectory.entryInfoList(QDir::Files); - QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" }; + QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" }; for (const auto& fileInfo : files) { auto filename = fileInfo.fileName(); if (!filenameRegex.exactMatch(filename)) { @@ -74,10 +74,11 @@ void AssetServer::run() { QByteArray data = file.readAll(); auto hash = hashData(data); + auto hexHash = hash.toHex(); - qDebug() << "\tMoving " << filename << " to " << hash; + qDebug() << "\tMoving " << filename << " to " << hexHash; - file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix()); + file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix()); } } } @@ -87,13 +88,13 @@ void AssetServer::handleAssetGetInfo(QSharedPointer packet, SharedNode MessageID messageID; uint8_t extensionLength; - if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) { + if (packet->getPayloadSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID) + sizeof(extensionLength))) { qDebug() << "ERROR bad file request"; return; } packet->readPrimitive(&messageID); - assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH); + assetHash = packet->readWithoutCopy(SHA256_HASH_LENGTH); packet->readPrimitive(&extensionLength); QByteArray extension = packet->read(extensionLength); @@ -125,23 +126,25 @@ void AssetServer::handleAssetGet(QSharedPointer packet, SharedNodePoin DataOffset start; DataOffset end; - auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end)); + auto minSize = qint64(sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end)); if (packet->getPayloadSize() < minSize) { qDebug() << "ERROR bad file request"; return; } packet->readPrimitive(&messageID); - assetHash = packet->read(HASH_HEX_LENGTH); + assetHash = packet->read(SHA256_HASH_LENGTH); packet->readPrimitive(&extensionLength); QByteArray extension = packet->read(extensionLength); packet->readPrimitive(&start); packet->readPrimitive(&end); + + QByteArray hexHash = assetHash.toHex(); - qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end; + qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end; // Queue task - QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension)); + QString filePath = _resourcesDirectory.filePath(QString(hexHash) + "." + QString(extension)); auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode); _taskPool.start(task); } @@ -194,28 +197,29 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha } else { QByteArray fileData = buffer.read(fileSize); - QString hash = hashData(fileData); + auto hash = hashData(fileData); + auto hexHash = hash.toHex(); - qDebug() << "Got data: (" << hash << ") "; + qDebug() << "Got data: (" << hexHash << ") "; - QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) }; + QFile file { _resourcesDirectory.filePath(QString(hexHash)) + "." + QString(extension) }; if (file.exists()) { - qDebug() << "[WARNING] This file already exists: " << hash; + qDebug() << "[WARNING] This file already exists: " << hexHash; } else { file.open(QIODevice::WriteOnly); file.write(fileData); file.close(); } replyPacket->writePrimitive(AssetServerError::NO_ERROR); - replyPacket->write(hash.toLatin1()); + replyPacket->write(hash); } auto nodeList = DependencyManager::get(); nodeList->sendPacket(std::move(replyPacket), *senderNode); } -QString AssetServer::hashData(const QByteArray& data) { - return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex()); +QByteArray AssetServer::hashData(const QByteArray& data) { + return QCryptographicHash::hash(data, QCryptographicHash::Sha256); } diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 07e0680c4f..204a4869bb 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -23,7 +23,7 @@ class AssetServer : public ThreadedAssignment { public: AssetServer(NLPacket& packet); - static QString hashData(const QByteArray& data); + static QByteArray hashData(const QByteArray& data); public slots: void run(); diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index f30689b042..0c3b4cd02d 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -48,14 +48,14 @@ void SendAssetTask::run() { if (file.open(QIODevice::ReadOnly)) { if (file.size() < _end) { writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); - qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end; + qCDebug(networking) << "Bad byte range: " << _assetHash.toHex() << " " << _start << ":" << _end; } else { auto size = _end - _start; file.seek(_start); replyPacketList->writePrimitive(AssetServerError::NO_ERROR); replyPacketList->writePrimitive(size); replyPacketList->write(file.read(size)); - qCDebug(networking) << "Sending asset: " << _assetHash; + qCDebug(networking) << "Sending asset: " << _assetHash.toHex(); } file.close(); } else { diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index d179977d20..d8207ae364 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -40,7 +40,7 @@ AssetRequest* AssetClient::createRequest(QString hash, QString extension) { return req; } - if (hash.length() != HASH_HEX_LENGTH) { + if (hash.length() != SHA256_HASH_HEX_LENGTH) { qDebug() << "Invalid hash size"; return nullptr; } @@ -76,7 +76,7 @@ AssetUpload* AssetClient::createUpload(QString filename) { } bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { - if (hash.length() != HASH_HEX_LENGTH) { + if (hash.length() != SHA256_HASH_HEX_LENGTH) { qDebug() << "Invalid hash size"; return false; } @@ -93,7 +93,7 @@ bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, Da packet->writePrimitive(messageID); - packet->write(hash.toLatin1()); + packet->write(QByteArray::fromHex(hash.toLatin1())); packet->writePrimitive(uint8_t(extension.length())); packet->write(extension.toLatin1()); @@ -120,7 +120,7 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback auto messageID = ++_currentID; packet->writePrimitive(messageID); - packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); + packet->write(QByteArray::fromHex(hash.toLatin1())); packet->writePrimitive(uint8_t(extension.length())); packet->write(extension.toLatin1()); @@ -137,12 +137,12 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback void AssetClient::handleAssetGetInfoReply(QSharedPointer packet, SharedNodePointer senderNode) { MessageID messageID; packet->readPrimitive(&messageID); - auto assetHash = QString(packet->read(HASH_HEX_LENGTH)); + auto assetHash = packet->read(SHA256_HASH_LENGTH); AssetServerError error; packet->readPrimitive(&error); - AssetInfo info { assetHash, 0 }; + AssetInfo info { assetHash.toHex(), 0 }; if (error == NO_ERROR) { packet->readPrimitive(&info.size); @@ -159,8 +159,8 @@ void AssetClient::handleAssetGetReply(QSharedPointer packetList, S QBuffer packet { &data }; packet.open(QIODevice::ReadOnly); - auto assetHash = packet.read(HASH_HEX_LENGTH); - qDebug() << "Got reply for asset: " << assetHash; + auto assetHash = packet.read(SHA256_HASH_LENGTH); + qDebug() << "Got reply for asset: " << assetHash.toHex(); MessageID messageID; packet.read(reinterpret_cast(&messageID), sizeof(messageID)); @@ -224,10 +224,9 @@ void AssetClient::handleAssetUploadReply(QSharedPointer packet, Shared if (error) { qDebug() << "Error uploading file to asset server"; } else { - auto hashData = packet->read(HASH_HEX_LENGTH); - - hashString = QString(hashData); - + auto hash = packet->read(SHA256_HASH_LENGTH); + hashString = hash.toHex(); + qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; } diff --git a/libraries/networking/src/AssetUtils.h b/libraries/networking/src/AssetUtils.h index 0cc71dab70..0447e8b303 100644 --- a/libraries/networking/src/AssetUtils.h +++ b/libraries/networking/src/AssetUtils.h @@ -16,7 +16,8 @@ using MessageID = uint32_t; using DataOffset = int64_t; -const size_t HASH_HEX_LENGTH = 64; +const size_t SHA256_HASH_LENGTH = 32; +const size_t SHA256_HASH_HEX_LENGTH = 64; const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB enum AssetServerError : uint8_t {