Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-08-09 17:49:27 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

use compact hash representation for asset-server

Browse source
This commit is contained in:
Stephen Birarda 2015-08-27 14:22:21 -07:00
parent 229015a5da
commit 86a7a42d24
5 changed files with 36 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
assignment-client/src/assets/AssetServer.cpp
View file

@ -58,7 +58,7 @@ void AssetServer::run() {
// Scan for new files // Scan for new files
qDebug() << "Looking for new files in asset directory"; qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files); auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" }; QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) { for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName(); auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) { if (!filenameRegex.exactMatch(filename)) {
@ -74,10 +74,11 @@ void AssetServer::run() {
QByteArray data = file.readAll(); QByteArray data = file.readAll();
auto hash = hashData(data); auto hash = hashData(data);
auto hexHash = hash.toHex();
qDebug() << "\tMoving " << filename << " to " << hash; qDebug() << "\tMoving " << filename << " to " << hexHash;
file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix()); file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix());
} }
} }
} }
@ -87,13 +88,13 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNode
MessageID messageID; MessageID messageID;
uint8_t extensionLength; uint8_t extensionLength;
if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) { if (packet->getPayloadSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID) + sizeof(extensionLength))) {
qDebug() << "ERROR bad file request"; qDebug() << "ERROR bad file request";
return; return;
} }
packet->readPrimitive(&messageID); packet->readPrimitive(&messageID);
assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH); assetHash = packet->readWithoutCopy(SHA256_HASH_LENGTH);
packet->readPrimitive(&extensionLength); packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength); QByteArray extension = packet->read(extensionLength);
@ -125,23 +126,25 @@ void AssetServer::handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePoin
DataOffset start; DataOffset start;
DataOffset end; DataOffset end;
auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end)); auto minSize = qint64(sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end));
if (packet->getPayloadSize() < minSize) { if (packet->getPayloadSize() < minSize) {
qDebug() << "ERROR bad file request"; qDebug() << "ERROR bad file request";
return; return;
} }
packet->readPrimitive(&messageID); packet->readPrimitive(&messageID);
assetHash = packet->read(HASH_HEX_LENGTH); assetHash = packet->read(SHA256_HASH_LENGTH);
packet->readPrimitive(&extensionLength); packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength); QByteArray extension = packet->read(extensionLength);
packet->readPrimitive(&start); packet->readPrimitive(&start);
packet->readPrimitive(&end); packet->readPrimitive(&end);
QByteArray hexHash = assetHash.toHex();
qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end; qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end;
// Queue task // Queue task
QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension)); QString filePath = _resourcesDirectory.filePath(QString(hexHash) + "." + QString(extension));
auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode); auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode);
_taskPool.start(task); _taskPool.start(task);
} }
@ -194,28 +197,29 @@ void AssetServer::handleAssetUpload(QSharedPointer<NLPacketList> packetList, Sha
} else { } else {
QByteArray fileData = buffer.read(fileSize); QByteArray fileData = buffer.read(fileSize);
QString hash = hashData(fileData); auto hash = hashData(fileData);
auto hexHash = hash.toHex();
qDebug() << "Got data: (" << hash << ") "; qDebug() << "Got data: (" << hexHash << ") ";
QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) }; QFile file { _resourcesDirectory.filePath(QString(hexHash)) + "." + QString(extension) };
if (file.exists()) { if (file.exists()) {
qDebug() << "[WARNING] This file already exists: " << hash; qDebug() << "[WARNING] This file already exists: " << hexHash;
} else { } else {
file.open(QIODevice::WriteOnly); file.open(QIODevice::WriteOnly);
file.write(fileData); file.write(fileData);
file.close(); file.close();
} }
replyPacket->writePrimitive(AssetServerError::NO_ERROR); replyPacket->writePrimitive(AssetServerError::NO_ERROR);
replyPacket->write(hash.toLatin1()); replyPacket->write(hash);
} }
auto nodeList = DependencyManager::get<NodeList>(); auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *senderNode); nodeList->sendPacket(std::move(replyPacket), *senderNode);
} }
QString AssetServer::hashData(const QByteArray& data) { QByteArray AssetServer::hashData(const QByteArray& data) {
return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex()); return QCryptographicHash::hash(data, QCryptographicHash::Sha256);
} }

2
assignment-client/src/assets/AssetServer.h
View file

@ -23,7 +23,7 @@ class AssetServer : public ThreadedAssignment {
public: public:
AssetServer(NLPacket& packet); AssetServer(NLPacket& packet);
static QString hashData(const QByteArray& data); static QByteArray hashData(const QByteArray& data);
public slots: public slots:
void run(); void run();

4
assignment-client/src/assets/SendAssetTask.cpp
View file

@ -48,14 +48,14 @@ void SendAssetTask::run() {
if (file.open(QIODevice::ReadOnly)) { if (file.open(QIODevice::ReadOnly)) {
if (file.size() < _end) { if (file.size() < _end) {
writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE); writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE);
qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end; qCDebug(networking) << "Bad byte range: " << _assetHash.toHex() << " " << _start << ":" << _end;
} else { } else {
auto size = _end - _start; auto size = _end - _start;
file.seek(_start); file.seek(_start);
replyPacketList->writePrimitive(AssetServerError::NO_ERROR); replyPacketList->writePrimitive(AssetServerError::NO_ERROR);
replyPacketList->writePrimitive(size); replyPacketList->writePrimitive(size);
replyPacketList->write(file.read(size)); replyPacketList->write(file.read(size));
qCDebug(networking) << "Sending asset: " << _assetHash; qCDebug(networking) << "Sending asset: " << _assetHash.toHex();
} }
file.close(); file.close();
} else { } else {

23
libraries/networking/src/AssetClient.cpp
View file

@ -40,7 +40,7 @@ AssetRequest* AssetClient::createRequest(QString hash, QString extension) {
return req; return req;
} }
if (hash.length() != HASH_HEX_LENGTH) { if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size"; qDebug() << "Invalid hash size";
return nullptr; return nullptr;
} }
@ -76,7 +76,7 @@ AssetUpload* AssetClient::createUpload(QString filename) {
} }
bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) { bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) {
if (hash.length() != HASH_HEX_LENGTH) { if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size"; qDebug() << "Invalid hash size";
return false; return false;
} }
@ -93,7 +93,7 @@ bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, Da
packet->writePrimitive(messageID); packet->writePrimitive(messageID);
packet->write(hash.toLatin1()); packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length())); packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1()); packet->write(extension.toLatin1());
@ -120,7 +120,7 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback
auto messageID = ++_currentID; auto messageID = ++_currentID;
packet->writePrimitive(messageID); packet->writePrimitive(messageID);
packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH); packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length())); packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1()); packet->write(extension.toLatin1());
@ -137,12 +137,12 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback
void AssetClient::handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) { void AssetClient::handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
MessageID messageID; MessageID messageID;
packet->readPrimitive(&messageID); packet->readPrimitive(&messageID);
auto assetHash = QString(packet->read(HASH_HEX_LENGTH)); auto assetHash = packet->read(SHA256_HASH_LENGTH);
AssetServerError error; AssetServerError error;
packet->readPrimitive(&error); packet->readPrimitive(&error);
AssetInfo info { assetHash, 0 }; AssetInfo info { assetHash.toHex(), 0 };
if (error == NO_ERROR) { if (error == NO_ERROR) {
packet->readPrimitive(&info.size); packet->readPrimitive(&info.size);
@ -159,8 +159,8 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
QBuffer packet { &data }; QBuffer packet { &data };
packet.open(QIODevice::ReadOnly); packet.open(QIODevice::ReadOnly);
auto assetHash = packet.read(HASH_HEX_LENGTH); auto assetHash = packet.read(SHA256_HASH_LENGTH);
qDebug() << "Got reply for asset: " << assetHash; qDebug() << "Got reply for asset: " << assetHash.toHex();
MessageID messageID; MessageID messageID;
packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID)); packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
@ -224,10 +224,9 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, Shared
if (error) { if (error) {
qDebug() << "Error uploading file to asset server"; qDebug() << "Error uploading file to asset server";
} else { } else {
auto hashData = packet->read(HASH_HEX_LENGTH); auto hash = packet->read(SHA256_HASH_LENGTH);
hashString = hash.toHex();
hashString = QString(hashData);
qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString; qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
} }

3
libraries/networking/src/AssetUtils.h
View file

@ -16,7 +16,8 @@
using MessageID = uint32_t; using MessageID = uint32_t;
using DataOffset = int64_t; using DataOffset = int64_t;
const size_t HASH_HEX_LENGTH = 64; const size_t SHA256_HASH_LENGTH = 32;
const size_t SHA256_HASH_HEX_LENGTH = 64;
const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB
enum AssetServerError : uint8_t { enum AssetServerError : uint8_t {