Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-21 06:44:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

use compact hash representation for asset-server

Browse source
This commit is contained in:
Stephen Birarda 2015-08-27 14:22:21 -07:00
parent 229015a5da
commit 86a7a42d24
5 changed files with 36 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
assignment-client/src/assets/AssetServer.cpp
View file

@ -58,7 +58,7 @@ void AssetServer::run() {
// Scan for new files
qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" };
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(SHA256_HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) {
@ -74,10 +74,11 @@ void AssetServer::run() {
QByteArray data = file.readAll();
auto hash = hashData(data);
auto hexHash = hash.toHex();
qDebug() << "\tMoving " << filename << " to " << hash;
qDebug() << "\tMoving " << filename << " to " << hexHash;
file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix());
file.rename(_resourcesDirectory.absoluteFilePath(hexHash) + "." + fileInfo.suffix());
}
}
}
@ -87,13 +88,13 @@ void AssetServer::handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNode
MessageID messageID;
uint8_t extensionLength;
if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) {
if (packet->getPayloadSize() < qint64(SHA256_HASH_LENGTH + sizeof(messageID) + sizeof(extensionLength))) {
qDebug() << "ERROR bad file request";
return;
}
packet->readPrimitive(&messageID);
assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH);
assetHash = packet->readWithoutCopy(SHA256_HASH_LENGTH);
packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength);
@ -125,23 +126,25 @@ void AssetServer::handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePoin
DataOffset start;
DataOffset end;
auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end));
auto minSize = qint64(sizeof(messageID) + SHA256_HASH_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end));
if (packet->getPayloadSize() < minSize) {
qDebug() << "ERROR bad file request";
return;
}
packet->readPrimitive(&messageID);
assetHash = packet->read(HASH_HEX_LENGTH);
assetHash = packet->read(SHA256_HASH_LENGTH);
packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength);
packet->readPrimitive(&start);
packet->readPrimitive(&end);
QByteArray hexHash = assetHash.toHex();
qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end;
qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end;
// Queue task
QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension));
QString filePath = _resourcesDirectory.filePath(QString(hexHash) + "." + QString(extension));
auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode);
_taskPool.start(task);
}
@ -194,28 +197,29 @@ void AssetServer::handleAssetUpload(QSharedPointer<NLPacketList> packetList, Sha
} else {
QByteArray fileData = buffer.read(fileSize);
QString hash = hashData(fileData);
auto hash = hashData(fileData);
auto hexHash = hash.toHex();
qDebug() << "Got data: (" << hash << ") ";
qDebug() << "Got data: (" << hexHash << ") ";
QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) };
QFile file { _resourcesDirectory.filePath(QString(hexHash)) + "." + QString(extension) };
if (file.exists()) {
qDebug() << "[WARNING] This file already exists: " << hash;
qDebug() << "[WARNING] This file already exists: " << hexHash;
} else {
file.open(QIODevice::WriteOnly);
file.write(fileData);
file.close();
}
replyPacket->writePrimitive(AssetServerError::NO_ERROR);
replyPacket->write(hash.toLatin1());
replyPacket->write(hash);
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *senderNode);
}
QString AssetServer::hashData(const QByteArray& data) {
return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex());
QByteArray AssetServer::hashData(const QByteArray& data) {
return QCryptographicHash::hash(data, QCryptographicHash::Sha256);
}

2
assignment-client/src/assets/AssetServer.h
View file

@ -23,7 +23,7 @@ class AssetServer : public ThreadedAssignment {
public:
AssetServer(NLPacket& packet);
static QString hashData(const QByteArray& data);
static QByteArray hashData(const QByteArray& data);
public slots:
void run();

4
assignment-client/src/assets/SendAssetTask.cpp
View file

@ -48,14 +48,14 @@ void SendAssetTask::run() {
if (file.open(QIODevice::ReadOnly)) {
if (file.size() < _end) {
writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE);
qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end;
qCDebug(networking) << "Bad byte range: " << _assetHash.toHex() << " " << _start << ":" << _end;
} else {
auto size = _end - _start;
file.seek(_start);
replyPacketList->writePrimitive(AssetServerError::NO_ERROR);
replyPacketList->writePrimitive(size);
replyPacketList->write(file.read(size));
qCDebug(networking) << "Sending asset: " << _assetHash;
qCDebug(networking) << "Sending asset: " << _assetHash.toHex();
}
file.close();
} else {

23
libraries/networking/src/AssetClient.cpp
View file

@ -40,7 +40,7 @@ AssetRequest* AssetClient::createRequest(QString hash, QString extension) {
return req;
}
if (hash.length() != HASH_HEX_LENGTH) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
return nullptr;
}
@ -76,7 +76,7 @@ AssetUpload* AssetClient::createUpload(QString filename) {
}
bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) {
if (hash.length() != HASH_HEX_LENGTH) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
return false;
}
@ -93,7 +93,7 @@ bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, Da
packet->writePrimitive(messageID);
packet->write(hash.toLatin1());
packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
@ -120,7 +120,7 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback
auto messageID = ++_currentID;
packet->writePrimitive(messageID);
packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH);
packet->write(QByteArray::fromHex(hash.toLatin1()));
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
@ -137,12 +137,12 @@ bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback
void AssetClient::handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
MessageID messageID;
packet->readPrimitive(&messageID);
auto assetHash = QString(packet->read(HASH_HEX_LENGTH));
auto assetHash = packet->read(SHA256_HASH_LENGTH);
AssetServerError error;
packet->readPrimitive(&error);
AssetInfo info { assetHash, 0 };
AssetInfo info { assetHash.toHex(), 0 };
if (error == NO_ERROR) {
packet->readPrimitive(&info.size);
@ -159,8 +159,8 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
QBuffer packet { &data };
packet.open(QIODevice::ReadOnly);
auto assetHash = packet.read(HASH_HEX_LENGTH);
qDebug() << "Got reply for asset: " << assetHash;
auto assetHash = packet.read(SHA256_HASH_LENGTH);
qDebug() << "Got reply for asset: " << assetHash.toHex();
MessageID messageID;
packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
@ -224,10 +224,9 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, Shared
if (error) {
qDebug() << "Error uploading file to asset server";
} else {
auto hashData = packet->read(HASH_HEX_LENGTH);
hashString = QString(hashData);
auto hash = packet->read(SHA256_HASH_LENGTH);
hashString = hash.toHex();
qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
}

3
libraries/networking/src/AssetUtils.h
View file

@ -16,7 +16,8 @@
using MessageID = uint32_t;
using DataOffset = int64_t;
const size_t HASH_HEX_LENGTH = 64;
const size_t SHA256_HASH_LENGTH = 32;
const size_t SHA256_HASH_HEX_LENGTH = 64;
const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB
enum AssetServerError : uint8_t {