Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 18:23:54 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Asset caching

Browse source
This commit is contained in:
Atlante45 2015-09-15 15:27:38 +02:00
parent eabee9857a
commit 9678e70444
11 changed files with 118 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
interface/src/Application.cpp
View file

@ -182,9 +182,6 @@ public:
using namespace std;
// Starfield information
const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB
static QTimer* locationUpdateTimer = NULL;
static QTimer* balanceUpdateTimer = NULL;
static QTimer* identityPacketTimer = NULL;
@ -462,13 +459,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
audioThread->start();
QThread* assetThread = new QThread;
assetThread->setObjectName("Asset Thread");
// Setup AssetClient
auto assetClient = DependencyManager::get<AssetClient>();
QThread* assetThread = new QThread;
assetThread->setObjectName("Asset Thread");
assetClient->moveToThread(assetThread);
connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init);
assetThread->start();
const DomainHandler& domainHandler = nodeList->getDomainHandler();
@ -857,8 +853,7 @@ void Application::cleanupBeforeQuit() {
}
void Application::emptyLocalCache() {
QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
if (cache) {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
cache->clear();
}

3
interface/src/ui/DiskCacheEditor.cpp
View file

@ -140,8 +140,7 @@ void DiskCacheEditor::clear() {
"You are about to erase all the content of the disk cache,"
"are you sure you want to do that?");
if (buttonClicked == QMessageBox::Yes) {
QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
if (cache) {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
cache->clear();
}

41
libraries/networking/src/AssetClient.cpp
View file

@ -14,13 +14,18 @@
#include <cstdint>
#include <QtCore/QBuffer>
#include <QtCore/QStandardPaths>
#include <QtCore/QThread>
#include <QtNetwork/QNetworkDiskCache>
#include "AssetRequest.h"
#include "AssetUpload.h"
#include "AssetUtils.h"
#include "NetworkAccessManager.h"
#include "NetworkLogging.h"
#include "NodeList.h"
#include "PacketReceiver.h"
#include "ResourceCache.h"
MessageID AssetClient::_currentID = 0;
@ -40,9 +45,25 @@ AssetClient::AssetClient() {
connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &AssetClient::handleNodeKilled);
}
void AssetClient::init() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection);
}
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
if (!networkAccessManager.cache()) {
QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
QNetworkDiskCache* cache = new QNetworkDiskCache();
cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE);
cache->setCacheDirectory(!cachePath.isEmpty() ? cachePath : "interfaceCache");
networkAccessManager.setCache(cache);
qCDebug(asset_client) << "AssetClient disk cache setup.";
}
}
AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
qCWarning(asset_client) << "Invalid hash size";
return nullptr;
}
@ -50,7 +71,7 @@ AssetRequest* AssetClient::createRequest(const QString& hash, const QString& ext
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qDebug().nospace() << "Could not request " << hash << "." << extension
qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension
<< " since you are not currently connected to an asset-server.";
return nullptr;
}
@ -68,7 +89,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qDebug() << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
return nullptr;
}
@ -82,7 +103,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,
ReceivedAssetCallback callback) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
qCWarning(asset_client) << "Invalid hash size";
return false;
}
@ -97,7 +118,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
+ sizeof(start) + sizeof(end);
auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true);
qDebug() << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
packet->writePrimitive(messageID);
@ -184,7 +205,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
packet.open(QIODevice::ReadOnly);
auto assetHash = packet.read(SHA256_HASH_LENGTH);
qDebug() << "Got reply for asset: " << assetHash.toHex();
qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex();
MessageID messageID;
packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
@ -198,7 +219,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
packet.read(reinterpret_cast<char*>(&length), sizeof(DataOffset));
data = packet.read(length);
} else {
qDebug() << "Failure getting asset: " << error;
qCWarning(asset_client) << "Failure getting asset: " << error;
}
// Check if we have any pending requests for this node
@ -254,15 +275,15 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, Shared
AssetServerError error;
packet->readPrimitive(&error);
QString hashString { "" };
QString hashString;
if (error) {
qDebug() << "Error uploading file to asset server";
qCWarning(asset_client) << "Error uploading file to asset server";
} else {
auto hash = packet->read(SHA256_HASH_LENGTH);
hashString = hash.toHex();
qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
}
// Check if we have any pending requests for this node

2
libraries/networking/src/AssetClient.h
View file

@ -40,6 +40,8 @@ class AssetClient : public QObject, public Dependency {
Q_OBJECT
public:
AssetClient();
Q_INVOKABLE void init();
Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension);
Q_INVOKABLE AssetUpload* createUpload(const QString& filename);

75
libraries/networking/src/AssetRequest.cpp
View file

@ -14,17 +14,19 @@
#include <algorithm>
#include <QtCore/QThread>
#include <QtNetwork/QAbstractNetworkCache>
#include "AssetClient.h"
#include "NetworkAccessManager.h"
#include "NetworkLogging.h"
#include "NodeList.h"
#include "ResourceCache.h"
AssetRequest::AssetRequest(const QString& hash, const QString& extension) :
QObject(),
_hash(hash),
_extension(extension)
{
}
void AssetRequest::start() {
@ -34,7 +36,18 @@ void AssetRequest::start() {
}
if (_state != NotStarted) {
qCWarning(networking) << "AssetRequest already started.";
qCWarning(asset_client) << "AssetRequest already started.";
return;
}
if (loadFromCache()) {
_info.hash = _hash;
_info.size = _data.size();
_error = NoError;
_state = Finished;
emit finished(this);
qCDebug(asset_client) << getUrl().toDisplayString() << "loaded from disk cache.";
return;
}
@ -58,7 +71,7 @@ void AssetRequest::start() {
}
if (_error != NoError) {
qCDebug(networking) << "Got error retrieving asset info for" << _hash;
qCWarning(asset_client) << "Got error retrieving asset info for" << _hash;
_state = Finished;
emit finished(this);
@ -68,7 +81,7 @@ void AssetRequest::start() {
_state = WaitingForData;
_data.resize(info.size);
qCDebug(networking) << "Got size of " << _hash << " : " << info.size << " bytes";
qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes";
int start = 0, end = _info.size;
@ -97,6 +110,10 @@ void AssetRequest::start() {
memcpy(_data.data() + start, data.constData(), data.size());
_totalReceived += data.size();
emit progress(_totalReceived, _info.size);
if (saveToCache(data)) {
qCDebug(asset_client) << getUrl().toDisplayString() << "saved to disk cache";
}
} else {
// hash doesn't match - we have an error
_error = HashVerificationFailed;
@ -105,7 +122,7 @@ void AssetRequest::start() {
}
if (_error != NoError) {
qCDebug(networking) << "Got error retrieving asset" << _hash << "- error code" << _error;
qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error;
}
_state = Finished;
@ -113,3 +130,51 @@ void AssetRequest::start() {
});
});
}
QUrl AssetRequest::getUrl() const {
if (!_extension.isEmpty()) {
return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, _hash, _extension));
} else {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, _hash));
}
}
bool AssetRequest::loadFromCache() {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
auto url = getUrl();
if (auto ioDevice = cache->data(url)) {
_data = ioDevice->readAll();
return true;
} else {
qCDebug(asset_client) << url.toDisplayString() << "not in disk cache";
}
} else {
qCWarning(asset_client) << "No disk cache to load assets from.";
}
return false;
}
bool AssetRequest::saveToCache(const QByteArray& file) const {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
auto url = getUrl();
if (!cache->metaData(url).isValid()) {
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData.setSaveToDisk(true);
metaData.setLastModified(QDateTime::currentDateTime());
metaData.setExpirationDate(QDateTime()); // Never expires
if (auto ioDevice = cache->prepare(metaData)) {
ioDevice->write(file);
cache->insert(ioDevice);
return true;
}
qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
}
} else {
qCWarning(asset_client) << "No disk cache to save assets to.";
}
return false;
}

4
libraries/networking/src/AssetRequest.h
View file

@ -46,12 +46,16 @@ public:
const QByteArray& getData() const { return _data; }
const State& getState() const { return _state; }
const Error& getError() const { return _error; }
QUrl getUrl() const;
signals:
void finished(AssetRequest* thisRequest);
void progress(qint64 totalReceived, qint64 total);
private:
bool loadFromCache();
bool saveToCache(const QByteArray& file) const;
State _state = NotStarted;
Error _error = NoError;
AssetInfo _info;

3
libraries/networking/src/AssetUpload.cpp
View file

@ -15,6 +15,7 @@
#include <QtCore/QThread>
#include "AssetClient.h"
#include "NetworkLogging.h"
AssetUpload::AssetUpload(QObject* object, const QString& filename) :
_filename(filename)
@ -41,7 +42,7 @@ void AssetUpload::start() {
// ask the AssetClient to upload the asset and emit the proper signals from the passed callback
auto assetClient = DependencyManager::get<AssetClient>();
qDebug() << "Attempting to upload" << _filename << "to asset-server.";
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
if (!responseReceived) {

1
libraries/networking/src/NetworkLogging.cpp
View file

@ -12,3 +12,4 @@
#include "NetworkLogging.h"
Q_LOGGING_CATEGORY(networking, "hifi.networking")
Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client")

1
libraries/networking/src/NetworkLogging.h
View file

@ -15,5 +15,6 @@
#include <QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(networking)
Q_DECLARE_LOGGING_CATEGORY(asset_client)
#endif // hifi_NetworkLogging_h

2
libraries/networking/src/ResourceCache.cpp
View file

@ -12,8 +12,6 @@
#include <cfloat>
#include <cmath>
#include <QDebug>
#include <QNetworkDiskCache>
#include <QThread>
#include <QTimer>

1
libraries/networking/src/ResourceCache.h
View file

@ -35,6 +35,7 @@ class Resource;
static const qint64 BYTES_PER_MEGABYTES = 1024 * 1024;
static const qint64 BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES;
static const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB
// Windows can have troubles allocating that much memory in ram sometimes
// so default cache size at 100 MB on windows (1GB otherwise)