Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 12:04:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #5806 from Atlante45/as_caching

Browse source 
Asset Server caching
This commit is contained in:
Stephen Birarda 2015-09-16 10:06:40 -07:00
commit 13728a7a96
15 changed files with 135 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
interface/src/Application.cpp
View file

@ -182,9 +182,6 @@ public:
using namespace std;
// Starfield information
const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB
static QTimer* locationUpdateTimer = NULL;
static QTimer* balanceUpdateTimer = NULL;
static QTimer* identityPacketTimer = NULL;
@ -370,6 +367,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
_lastFaceTrackerUpdate(0),
_applicationOverlay()
{
thread()->setObjectName("Main Thread");
setInstance(this);
_entityClipboard->createRootElement();
@ -460,13 +459,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
audioThread->start();
QThread* assetThread = new QThread;
assetThread->setObjectName("Asset Thread");
// Setup AssetClient
auto assetClient = DependencyManager::get<AssetClient>();
QThread* assetThread = new QThread;
assetThread->setObjectName("Asset Thread");
assetClient->moveToThread(assetThread);
connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init);
assetThread->start();
const DomainHandler& domainHandler = nodeList->getDomainHandler();
@ -855,8 +853,7 @@ void Application::cleanupBeforeQuit() {
}
void Application::emptyLocalCache() {
QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
if (cache) {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
cache->clear();
}

3
interface/src/ui/DiskCacheEditor.cpp
View file

@ -140,8 +140,7 @@ void DiskCacheEditor::clear() {
"You are about to erase all the content of the disk cache,"
"are you sure you want to do that?");
if (buttonClicked == QMessageBox::Yes) {
QNetworkDiskCache* cache = qobject_cast<QNetworkDiskCache*>(NetworkAccessManager::getInstance().cache());
if (cache) {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
qDebug() << "DiskCacheEditor::clear(): Clearing disk cache.";
cache->clear();
}

53
libraries/networking/src/AssetClient.cpp
View file

@ -11,16 +11,21 @@
#include "AssetClient.h"
#include <QBuffer>
#include <QThread>
#include <cstdint>
#include <QtCore/QBuffer>
#include <QtCore/QStandardPaths>
#include <QtCore/QThread>
#include <QtNetwork/QNetworkDiskCache>
#include "AssetRequest.h"
#include "AssetUpload.h"
#include "AssetUtils.h"
#include "NetworkAccessManager.h"
#include "NetworkLogging.h"
#include "NodeList.h"
#include "PacketReceiver.h"
#include "AssetUtils.h"
#include "ResourceCache.h"
MessageID AssetClient::_currentID = 0;
@ -40,9 +45,29 @@ AssetClient::AssetClient() {
connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &AssetClient::handleNodeKilled);
}
void AssetClient::init() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection);
}
// Setup disk cache if not already
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
if (!networkAccessManager.cache()) {
QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation);
cachePath = !cachePath.isEmpty() ? cachePath : "interfaceCache";
QNetworkDiskCache* cache = new QNetworkDiskCache();
cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE);
cache->setCacheDirectory(cachePath);
networkAccessManager.setCache(cache);
qCDebug(asset_client) << "AssetClient disk cache setup at" << cachePath
<< "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)";
}
}
AssetRequest* AssetClient::createRequest(const QString& hash, const QString& extension) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
qCWarning(asset_client) << "Invalid hash size";
return nullptr;
}
@ -50,7 +75,7 @@ AssetRequest* AssetClient::createRequest(const QString& hash, const QString& ext
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qDebug().nospace() << "Could not request " << hash << "." << extension
qCWarning(asset_client).nospace() << "Could not request " << hash << "." << extension
<< " since you are not currently connected to an asset-server.";
return nullptr;
}
@ -68,7 +93,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (!assetServer) {
qDebug() << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
qCWarning(asset_client) << "Could not upload" << filename << "since you are not currently connected to an asset-server.";
return nullptr;
}
@ -82,7 +107,7 @@ AssetUpload* AssetClient::createUpload(const QString& filename) {
bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOffset start, DataOffset end,
ReceivedAssetCallback callback) {
if (hash.length() != SHA256_HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
qCWarning(asset_client) << "Invalid hash size";
return false;
}
@ -97,7 +122,7 @@ bool AssetClient::getAsset(const QString& hash, const QString& extension, DataOf
+ sizeof(start) + sizeof(end);
auto packet = NLPacket::create(PacketType::AssetGet, payloadSize, true);
qDebug() << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
qCDebug(asset_client) << "Requesting data from" << start << "to" << end << "of" << hash << "from asset-server.";
packet->writePrimitive(messageID);
@ -184,7 +209,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
packet.open(QIODevice::ReadOnly);
auto assetHash = packet.read(SHA256_HASH_LENGTH);
qDebug() << "Got reply for asset: " << assetHash.toHex();
qCDebug(asset_client) << "Got reply for asset: " << assetHash.toHex();
MessageID messageID;
packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
@ -198,7 +223,7 @@ void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, S
packet.read(reinterpret_cast<char*>(&length), sizeof(DataOffset));
data = packet.read(length);
} else {
qDebug() << "Failure getting asset: " << error;
qCWarning(asset_client) << "Failure getting asset: " << error;
}
// Check if we have any pending requests for this node
@ -254,15 +279,15 @@ void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, Shared
AssetServerError error;
packet->readPrimitive(&error);
QString hashString { "" };
QString hashString;
if (error) {
qDebug() << "Error uploading file to asset server";
qCWarning(asset_client) << "Error uploading file to asset server";
} else {
auto hash = packet->read(SHA256_HASH_LENGTH);
hashString = hash.toHex();
qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
qCDebug(asset_client) << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
}
// Check if we have any pending requests for this node

2
libraries/networking/src/AssetClient.h
View file

@ -40,6 +40,8 @@ class AssetClient : public QObject, public Dependency {
Q_OBJECT
public:
AssetClient();
Q_INVOKABLE void init();
Q_INVOKABLE AssetRequest* createRequest(const QString& hash, const QString& extension);
Q_INVOKABLE AssetUpload* createUpload(const QString& filename);

79
libraries/networking/src/AssetRequest.cpp
View file

@ -13,18 +13,20 @@
#include <algorithm>
#include <QThread>
#include <QtCore/QThread>
#include <QtNetwork/QAbstractNetworkCache>
#include "AssetClient.h"
#include "NetworkAccessManager.h"
#include "NetworkLogging.h"
#include "NodeList.h"
#include "ResourceCache.h"
AssetRequest::AssetRequest(const QString& hash, const QString& extension) :
QObject(),
_hash(hash),
_extension(extension)
{
}
void AssetRequest::start() {
@ -34,7 +36,19 @@ void AssetRequest::start() {
}
if (_state != NotStarted) {
qCWarning(networking) << "AssetRequest already started.";
qCWarning(asset_client) << "AssetRequest already started.";
return;
}
// Try to load from cache
if (loadFromCache()) {
_info.hash = _hash;
_info.size = _data.size();
_error = NoError;
_state = Finished;
emit finished(this);
qCDebug(asset_client) << getUrl().toDisplayString() << "loaded from disk cache.";
return;
}
@ -58,8 +72,7 @@ void AssetRequest::start() {
}
if (_error != NoError) {
qCDebug(networking) << "Got error retrieving asset info for" << _hash;
qCWarning(asset_client) << "Got error retrieving asset info for" << _hash;
_state = Finished;
emit finished(this);
@ -69,7 +82,7 @@ void AssetRequest::start() {
_state = WaitingForData;
_data.resize(info.size);
qCDebug(networking) << "Got size of " << _hash << " : " << info.size << " bytes";
qCDebug(asset_client) << "Got size of " << _hash << " : " << info.size << " bytes";
int start = 0, end = _info.size;
@ -98,6 +111,10 @@ void AssetRequest::start() {
memcpy(_data.data() + start, data.constData(), data.size());
_totalReceived += data.size();
emit progress(_totalReceived, _info.size);
if (saveToCache(data)) {
qCDebug(asset_client) << getUrl().toDisplayString() << "saved to disk cache";
}
} else {
// hash doesn't match - we have an error
_error = HashVerificationFailed;
@ -106,7 +123,7 @@ void AssetRequest::start() {
}
if (_error != NoError) {
qCDebug(networking) << "Got error retrieving asset" << _hash << "- error code" << _error;
qCWarning(asset_client) << "Got error retrieving asset" << _hash << "- error code" << _error;
}
_state = Finished;
@ -114,3 +131,51 @@ void AssetRequest::start() {
});
});
}
QUrl AssetRequest::getUrl() const {
if (!_extension.isEmpty()) {
return QUrl(QString("%1:%2.%3").arg(URL_SCHEME_ATP, _hash, _extension));
} else {
return QUrl(QString("%1:%2").arg(URL_SCHEME_ATP, _hash));
}
}
bool AssetRequest::loadFromCache() {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
auto url = getUrl();
if (auto ioDevice = cache->data(url)) {
_data = ioDevice->readAll();
return true;
} else {
qCDebug(asset_client) << url.toDisplayString() << "not in disk cache";
}
} else {
qCWarning(asset_client) << "No disk cache to load assets from.";
}
return false;
}
bool AssetRequest::saveToCache(const QByteArray& file) const {
if (auto cache = NetworkAccessManager::getInstance().cache()) {
auto url = getUrl();
if (!cache->metaData(url).isValid()) {
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData.setSaveToDisk(true);
metaData.setLastModified(QDateTime::currentDateTime());
metaData.setExpirationDate(QDateTime()); // Never expires
if (auto ioDevice = cache->prepare(metaData)) {
ioDevice->write(file);
cache->insert(ioDevice);
return true;
}
qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
}
} else {
qCWarning(asset_client) << "No disk cache to save assets to.";
}
return false;
}

4
libraries/networking/src/AssetRequest.h
View file

@ -46,12 +46,16 @@ public:
const QByteArray& getData() const { return _data; }
const State& getState() const { return _state; }
const Error& getError() const { return _error; }
QUrl getUrl() const;
signals:
void finished(AssetRequest* thisRequest);
void progress(qint64 totalReceived, qint64 total);
private:
bool loadFromCache();
bool saveToCache(const QByteArray& file) const;
State _state = NotStarted;
Error _error = NoError;
AssetInfo _info;

4
libraries/networking/src/AssetResourceRequest.cpp
View file

@ -32,7 +32,6 @@ void AssetResourceRequest::doSend() {
_state = Finished;
emit finished();
return;
}
@ -43,12 +42,11 @@ void AssetResourceRequest::doSend() {
_state = Finished;
emit finished();
return;
}
connect(_assetRequest, &AssetRequest::progress, this, &AssetResourceRequest::progress);
QObject::connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) mutable {
connect(_assetRequest, &AssetRequest::finished, [this](AssetRequest* req) {
Q_ASSERT(_state == InProgress);
Q_ASSERT(req == _assetRequest);
Q_ASSERT(req->getState() == AssetRequest::Finished);

3
libraries/networking/src/AssetUpload.cpp
View file

@ -15,6 +15,7 @@
#include <QtCore/QThread>
#include "AssetClient.h"
#include "NetworkLogging.h"
AssetUpload::AssetUpload(QObject* object, const QString& filename) :
_filename(filename)
@ -41,7 +42,7 @@ void AssetUpload::start() {
// ask the AssetClient to upload the asset and emit the proper signals from the passed callback
auto assetClient = DependencyManager::get<AssetClient>();
qDebug() << "Attempting to upload" << _filename << "to asset-server.";
qCDebug(asset_client) << "Attempting to upload" << _filename << "to asset-server.";
assetClient->uploadAsset(data, _extension, [this](bool responseReceived, AssetServerError error, const QString& hash){
if (!responseReceived) {

5
libraries/networking/src/HTTPResourceRequest.cpp
View file

@ -29,8 +29,7 @@ HTTPResourceRequest::~HTTPResourceRequest() {
}
void HTTPResourceRequest::doSend() {
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
QNetworkRequest networkRequest = QNetworkRequest(_url);
QNetworkRequest networkRequest(_url);
networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
if (_cacheEnabled) {
@ -39,7 +38,7 @@ void HTTPResourceRequest::doSend() {
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
}
_reply = networkAccessManager.get(networkRequest);
_reply = NetworkAccessManager::getInstance().get(networkRequest);
connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished);
connect(_reply, &QNetworkReply::downloadProgress, this, &HTTPResourceRequest::onDownloadProgress);

1
libraries/networking/src/NetworkAccessManager.cpp
View file

@ -18,6 +18,7 @@ QThreadStorage<QNetworkAccessManager*> networkAccessManagers;
QNetworkAccessManager& NetworkAccessManager::getInstance() {
if (!networkAccessManagers.hasLocalData()) {
networkAccessManagers.setLocalData(new QNetworkAccessManager());
}
return *networkAccessManagers.localData();

2
libraries/networking/src/NetworkAccessManager.h
View file

@ -12,7 +12,7 @@
#ifndef hifi_NetworkAccessManager_h
#define hifi_NetworkAccessManager_h
#include <QtNetwork/qnetworkaccessmanager.h>
#include <QtNetwork/QNetworkAccessManager>
/// Wrapper around QNetworkAccessManager to restrict at one instance by thread
class NetworkAccessManager : public QObject {

1
libraries/networking/src/NetworkLogging.cpp
View file

@ -12,3 +12,4 @@
#include "NetworkLogging.h"
Q_LOGGING_CATEGORY(networking, "hifi.networking")
Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client")

1
libraries/networking/src/NetworkLogging.h
View file

@ -15,5 +15,6 @@
#include <QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(networking)
Q_DECLARE_LOGGING_CATEGORY(asset_client)
#endif // hifi_NetworkLogging_h

2
libraries/networking/src/ResourceCache.cpp
View file

@ -12,8 +12,6 @@
#include <cfloat>
#include <cmath>
#include <QDebug>
#include <QNetworkDiskCache>
#include <QThread>
#include <QTimer>

1
libraries/networking/src/ResourceCache.h
View file

@ -35,6 +35,7 @@ class Resource;
static const qint64 BYTES_PER_MEGABYTES = 1024 * 1024;
static const qint64 BYTES_PER_GIGABYTES = 1024 * BYTES_PER_MEGABYTES;
static const qint64 MAXIMUM_CACHE_SIZE = 10 * BYTES_PER_GIGABYTES; // 10GB
// Windows can have troubles allocating that much memory in ram sometimes
// so default cache size at 100 MB on windows (1GB otherwise)