Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-08 15:43:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix various resourcecache bugs

Browse source
This commit is contained in:
SamGondelman 2018-10-04 10:33:08 -07:00
parent 130e8dfbe9
commit 4024f2a180
7 changed files with 119 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
interface/src/Application.cpp
View file

@ -151,6 +151,8 @@
#include <avatars-renderer/ScriptAvatar.h>
#include <RenderableEntityItem.h>
#include <procedural/ProceduralSkybox.h>
#include <model-networking/MaterialCache.h>
#include "recording/ClipCache.h"
#include "AudioClient.h"
#include "audio/AudioScope.h"
@ -326,9 +328,9 @@ static bool DISABLE_DEFERRED = QProcessEnvironment::systemEnvironment().contains
#endif
#if !defined(Q_OS_ANDROID)
static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16;
static const uint32_t MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16;
#else
static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 4;
static const uint32_t MAX_CONCURRENT_RESOURCE_DOWNLOADS = 4;
#endif
// For processing on QThreadPool, we target a number of threads after reserving some
@ -1315,7 +1317,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
QString concurrentDownloadsStr = getCmdOption(argc, constArgv, "--concurrent-downloads");
bool success;
int concurrentDownloads = concurrentDownloadsStr.toInt(&success);
uint32_t concurrentDownloads = concurrentDownloadsStr.toUInt(&success);
if (!success) {
concurrentDownloads = MAX_CONCURRENT_RESOURCE_DOWNLOADS;
}
@ -2036,7 +2038,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
}
properties["active_downloads"] = loadingRequests.size();
properties["pending_downloads"] = ResourceCache::getPendingRequestCount();
properties["pending_downloads"] = (int)ResourceCache::getPendingRequestCount();
properties["active_downloads_details"] = loadingRequestsStats;
auto statTracker = DependencyManager::get<StatTracker>();
@ -4631,8 +4633,8 @@ void Application::idle() {
PROFILE_COUNTER_IF_CHANGED(app, "present", float, displayPlugin->presentRate());
}
PROFILE_COUNTER_IF_CHANGED(app, "renderLoopRate", float, _renderLoopCounter.rate());
PROFILE_COUNTER_IF_CHANGED(app, "currentDownloads", int, ResourceCache::getLoadingRequests().length());
PROFILE_COUNTER_IF_CHANGED(app, "pendingDownloads", int, ResourceCache::getPendingRequestCount());
PROFILE_COUNTER_IF_CHANGED(app, "currentDownloads", uint32_t, ResourceCache::getLoadingRequestCount());
PROFILE_COUNTER_IF_CHANGED(app, "pendingDownloads", uint32_t, ResourceCache::getPendingRequestCount());
PROFILE_COUNTER_IF_CHANGED(app, "currentProcessing", int, DependencyManager::get<StatTracker>()->getStat("Processing").toInt());
PROFILE_COUNTER_IF_CHANGED(app, "pendingProcessing", int, DependencyManager::get<StatTracker>()->getStat("PendingProcessing").toInt());
auto renderConfig = _renderEngine->getConfiguration();
@ -5371,13 +5373,21 @@ void Application::reloadResourceCaches() {
queryOctree(NodeType::EntityServer, PacketType::EntityQuery);
// Clear the entities and their renderables
getEntities()->clear();
DependencyManager::get<AssetClient>()->clearCache();
DependencyManager::get<ScriptCache>()->clearCache();
// Clear all the resource caches
DependencyManager::get<ResourceCacheSharedItems>()->clear();
DependencyManager::get<AnimationCache>()->refreshAll();
DependencyManager::get<ModelCache>()->refreshAll();
DependencyManager::get<SoundCache>()->refreshAll();
MaterialCache::instance().refreshAll();
DependencyManager::get<ModelCache>()->refreshAll();
ShaderCache::instance().refreshAll();
DependencyManager::get<TextureCache>()->refreshAll();
DependencyManager::get<recording::ClipCache>()->refreshAll();
DependencyManager::get<NodeList>()->reset(); // Force redownload of .fst models
@ -6423,9 +6433,12 @@ void Application::clearDomainOctreeDetails() {
skyStage->setBackgroundMode(graphics::SunSkyStage::SKY_DEFAULT);
DependencyManager::get<AnimationCache>()->clearUnusedResources();
DependencyManager::get<ModelCache>()->clearUnusedResources();
DependencyManager::get<SoundCache>()->clearUnusedResources();
MaterialCache::instance().clearUnusedResources();
DependencyManager::get<ModelCache>()->clearUnusedResources();
ShaderCache::instance().clearUnusedResources();
DependencyManager::get<TextureCache>()->clearUnusedResources();
DependencyManager::get<recording::ClipCache>()->clearUnusedResources();
getMyAvatar()->setAvatarEntityDataChanged(true);
}

4
interface/src/ui/Stats.cpp
View file

@ -267,8 +267,8 @@ void Stats::updateStats(bool force) {
auto loadingRequests = ResourceCache::getLoadingRequests();
STAT_UPDATE(downloads, loadingRequests.size());
STAT_UPDATE(downloadLimit, ResourceCache::getRequestLimit())
STAT_UPDATE(downloadsPending, ResourceCache::getPendingRequestCount());
STAT_UPDATE(downloadLimit, (int)ResourceCache::getRequestLimit())
STAT_UPDATE(downloadsPending, (int)ResourceCache::getPendingRequestCount());
STAT_UPDATE(processing, DependencyManager::get<StatTracker>()->getStat("Processing").toInt());
STAT_UPDATE(processingPending, DependencyManager::get<StatTracker>()->getStat("PendingProcessing").toInt());

6
libraries/animation/src/AnimationCache.cpp
View file

@ -32,12 +32,6 @@ AnimationCache::AnimationCache(QObject* parent) :
}
AnimationPointer AnimationCache::getAnimation(const QUrl& url) {
if (QThread::currentThread() != thread()) {
AnimationPointer result;
BLOCKING_INVOKE_METHOD(this, "getAnimation",
Q_RETURN_ARG(AnimationPointer, result), Q_ARG(const QUrl&, url));
return result;
}
return getResource(url).staticCast<Animation>();
}

6
libraries/audio/src/SoundCache.cpp
View file

@ -30,12 +30,6 @@ SoundCache::SoundCache(QObject* parent) :
}
SharedSoundPointer SoundCache::getSound(const QUrl& url) {
if (QThread::currentThread() != thread()) {
SharedSoundPointer result;
BLOCKING_INVOKE_METHOD(this, "getSound",
Q_RETURN_ARG(SharedSoundPointer, result), Q_ARG(const QUrl&, url));
return result;
}
return getResource(url).staticCast<Sound>();
}

2
libraries/model-networking/src/model-networking/TextureCache.h
View file

@ -75,8 +75,6 @@ protected:
void makeLocalRequest();
Q_INVOKABLE void handleLocalRequestCompleted();
virtual bool isCacheable() const override { return _loaded; }
Q_INVOKABLE virtual void downloadFinished(const QByteArray& data) override;
bool handleFailedRequest(ResourceRequest::Result result) override;

165
libraries/networking/src/ResourceCache.cpp
View file

@ -27,28 +27,35 @@
#include "NetworkLogging.h"
#include "NodeList.h"
#define clamp(x, min, max) (((x) < (min)) ? (min) :\
(((x) > (max)) ? (max) :\
(x)))
void ResourceCacheSharedItems::appendActiveRequest(QWeakPointer<Resource> resource) {
bool ResourceCacheSharedItems::appendRequest(QWeakPointer<Resource> resource) {
Lock lock(_mutex);
_loadingRequests.append(resource);
if ((uint32_t)_loadingRequests.size() < _requestLimit) {
_loadingRequests.append(resource);
return true;
} else {
_pendingRequests.append(resource);
return false;
}
}
void ResourceCacheSharedItems::appendPendingRequest(QWeakPointer<Resource> resource) {
void ResourceCacheSharedItems::setRequestLimit(uint32_t limit) {
Lock lock(_mutex);
_pendingRequests.append(resource);
_requestLimit = limit;
}
QList<QSharedPointer<Resource>> ResourceCacheSharedItems::getPendingRequests() {
uint32_t ResourceCacheSharedItems::getRequestLimit() const {
Lock lock(_mutex);
return _requestLimit;
}
QList<QSharedPointer<Resource>> ResourceCacheSharedItems::getPendingRequests() const {
QList<QSharedPointer<Resource>> result;
Lock lock(_mutex);
foreach(QSharedPointer<Resource> resource, _pendingRequests) {
if (resource) {
result.append(resource);
foreach (QWeakPointer<Resource> resource, _pendingRequests) {
auto locked = resource.lock();
if (locked) {
result.append(locked);
}
}
@ -60,16 +67,12 @@ uint32_t ResourceCacheSharedItems::getPendingRequestsCount() const {
return _pendingRequests.size();
}
QList<QSharedPointer<Resource>> ResourceCacheSharedItems::getLoadingRequests() {
QList<QSharedPointer<Resource>> ResourceCacheSharedItems::getLoadingRequests() const {
QList<QSharedPointer<Resource>> result;
Lock lock(_mutex);
foreach(QSharedPointer<Resource> resource, _loadingRequests) {
if (resource) {
result.append(resource);
}
{
Lock lock(_mutex);
result = _loadingRequests;
}
return result;
}
@ -131,6 +134,11 @@ QSharedPointer<Resource> ResourceCacheSharedItems::getHighestPendingRequest() {
return highestResource;
}
void ResourceCacheSharedItems::clear() {
Lock lock(_mutex);
_pendingRequests.clear();
_loadingRequests.clear();
}
ScriptableResourceCache::ScriptableResourceCache(QSharedPointer<ResourceCache> resourceCache) {
_resourceCache = resourceCache;
@ -244,9 +252,7 @@ ResourceCache::ResourceCache(QObject* parent) : QObject(parent) {
}
}
ResourceCache::~ResourceCache() {
clearUnusedResources();
}
ResourceCache::~ResourceCache() {}
void ResourceCache::clearATPAssets() {
{
@ -260,6 +266,7 @@ void ResourceCache::clearATPAssets() {
if (auto strongRef = resource.lock()) {
// Make sure the resource won't reinsert itself
strongRef->setCache(nullptr);
_totalResourcesSize -= strongRef->getBytes();
}
}
}
@ -269,28 +276,18 @@ void ResourceCache::clearATPAssets() {
for (auto& resource : _unusedResources.values()) {
if (resource->getURL().scheme() == URL_SCHEME_ATP) {
_unusedResources.remove(resource->getLRUKey());
}
}
}
{
QWriteLocker locker(&_resourcesToBeGottenLock);
auto it = _resourcesToBeGotten.begin();
while (it != _resourcesToBeGotten.end()) {
if (it->scheme() == URL_SCHEME_ATP) {
it = _resourcesToBeGotten.erase(it);
} else {
++it;
_unusedResourcesSize -= resource->getBytes();
}
}
}
resetResourceCounters();
}
void ResourceCache::refreshAll() {
// Clear all unused resources so we don't have to reload them
clearUnusedResources();
resetResourceCounters();
resetUnusedResourceCounter();
QHash<QUrl, QWeakPointer<Resource>> resources;
{
@ -306,21 +303,6 @@ void ResourceCache::refreshAll() {
}
}
void ResourceCache::refresh(const QUrl& url) {
QSharedPointer<Resource> resource;
{
QReadLocker locker(&_resourcesLock);
resource = _resources.value(url).lock();
}
if (resource) {
resource->refresh();
} else {
removeResource(url);
resetResourceCounters();
}
}
QVariantList ResourceCache::getResourceList() {
QVariantList list;
if (QThread::currentThread() != thread()) {
@ -338,12 +320,13 @@ QVariantList ResourceCache::getResourceList() {
return list;
}
void ResourceCache::setRequestLimit(int limit) {
_requestLimit = limit;
void ResourceCache::setRequestLimit(uint32_t limit) {
auto sharedItems = DependencyManager::get<ResourceCacheSharedItems>();
sharedItems->setRequestLimit(limit);
// Now go fill any new request spots
while (attemptHighestPriorityRequest()) {
// just keep looping until we reach the new limit or no more pending requests
while (sharedItems->getLoadingRequestsCount() < limit && sharedItems->getPendingRequestsCount() > 0) {
attemptHighestPriorityRequest();
}
}
@ -381,9 +364,9 @@ QSharedPointer<Resource> ResourceCache::getResource(const QUrl& url, const QUrl&
}
void ResourceCache::setUnusedResourceCacheSize(qint64 unusedResourcesMaxSize) {
_unusedResourcesMaxSize = clamp(unusedResourcesMaxSize, MIN_UNUSED_MAX_SIZE, MAX_UNUSED_MAX_SIZE);
_unusedResourcesMaxSize = glm::clamp(unusedResourcesMaxSize, MIN_UNUSED_MAX_SIZE, MAX_UNUSED_MAX_SIZE);
reserveUnusedResource(0);
resetResourceCounters();
resetUnusedResourceCounter();
}
void ResourceCache::addUnusedResource(const QSharedPointer<Resource>& resource) {
@ -391,18 +374,20 @@ void ResourceCache::addUnusedResource(const QSharedPointer<Resource>& resource)
if (resource->getBytes() == 0 || resource->getBytes() > _unusedResourcesMaxSize) {
resource->setCache(nullptr);
removeResource(resource->getURL(), resource->getBytes());
resetResourceCounters();
resetTotalResourceCounter();
return;
}
reserveUnusedResource(resource->getBytes());
resource->setLRUKey(++_lastLRUKey);
_unusedResourcesSize += resource->getBytes();
resetResourceCounters();
{
QWriteLocker locker(&_unusedResourcesLock);
_unusedResources.insert(resource->getLRUKey(), resource);
_unusedResourcesSize += resource->getBytes();
}
QWriteLocker locker(&_unusedResourcesLock);
_unusedResources.insert(resource->getLRUKey(), resource);
resetUnusedResourceCounter();
}
void ResourceCache::removeUnusedResource(const QSharedPointer<Resource>& resource) {
@ -412,7 +397,7 @@ void ResourceCache::removeUnusedResource(const QSharedPointer<Resource>& resourc
_unusedResourcesSize -= resource->getBytes();
locker.unlock();
resetResourceCounters();
resetUnusedResourceCounter();
}
}
@ -445,14 +430,19 @@ void ResourceCache::clearUnusedResources() {
}
_unusedResources.clear();
}
_unusedResourcesSize = 0;
}
void ResourceCache::resetResourceCounters() {
void ResourceCache::resetTotalResourceCounter() {
{
QReadLocker locker(&_resourcesLock);
_numTotalResources = _resources.size();
}
emit dirty();
}
void ResourceCache::resetUnusedResourceCounter() {
{
QReadLocker locker(&_unusedResourcesLock);
_numUnusedResources = _unusedResources.size();
@ -461,6 +451,13 @@ void ResourceCache::resetResourceCounters() {
emit dirty();
}
void ResourceCache::resetResourceCounters() {
resetTotalResourceCounter();
resetUnusedResourceCounter();
emit dirty();
}
void ResourceCache::removeResource(const QUrl& url, qint64 size) {
QWriteLocker locker(&_resourcesLock);
_resources.remove(url);
@ -481,38 +478,34 @@ QList<QSharedPointer<Resource>> ResourceCache::getLoadingRequests() {
return DependencyManager::get<ResourceCacheSharedItems>()->getLoadingRequests();
}
int ResourceCache::getPendingRequestCount() {
uint32_t ResourceCache::getPendingRequestCount() {
return DependencyManager::get<ResourceCacheSharedItems>()->getPendingRequestsCount();
}
int ResourceCache::getLoadingRequestCount() {
uint32_t ResourceCache::getLoadingRequestCount() {
return DependencyManager::get<ResourceCacheSharedItems>()->getLoadingRequestsCount();
}
bool ResourceCache::attemptRequest(QSharedPointer<Resource> resource) {
Q_ASSERT(!resource.isNull());
auto sharedItems = DependencyManager::get<ResourceCacheSharedItems>();
if (_requestsActive >= _requestLimit) {
// wait until a slot becomes available
sharedItems->appendPendingRequest(resource);
return false;
if (sharedItems->appendRequest(resource)) {
resource->makeRequest();
return true;
}
++_requestsActive;
sharedItems->appendActiveRequest(resource);
resource->makeRequest();
return true;
return false;
}
void ResourceCache::requestCompleted(QWeakPointer<Resource> resource) {
auto sharedItems = DependencyManager::get<ResourceCacheSharedItems>();
sharedItems->removeRequest(resource);
--_requestsActive;
attemptHighestPriorityRequest();
// Now go fill any new request spots
while (sharedItems->getLoadingRequestsCount() < sharedItems->getRequestLimit() && sharedItems->getPendingRequestsCount() > 0) {
attemptHighestPriorityRequest();
}
}
bool ResourceCache::attemptHighestPriorityRequest() {
@ -521,10 +514,6 @@ bool ResourceCache::attemptHighestPriorityRequest() {
return (resource && attemptRequest(resource));
}
const int DEFAULT_REQUEST_LIMIT = 10;
int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT;
int ResourceCache::_requestsActive = 0;
static int requestID = 0;
Resource::Resource(const QUrl& url) :
@ -550,7 +539,7 @@ void Resource::ensureLoading() {
}
void Resource::setLoadPriority(const QPointer<QObject>& owner, float priority) {
if (!(_failedToLoad)) {
if (!_failedToLoad) {
_loadPriorities.insert(owner, priority);
}
}
@ -566,7 +555,7 @@ void Resource::setLoadPriorities(const QHash<QPointer<QObject>, float>& prioriti
}
void Resource::clearLoadPriority(const QPointer<QObject>& owner) {
if (!(_failedToLoad)) {
if (!_failedToLoad) {
_loadPriorities.remove(owner);
}
}
@ -592,6 +581,7 @@ void Resource::refresh() {
if (_request && !(_loaded || _failedToLoad)) {
return;
}
if (_request) {
_request->disconnect(this);
_request->deleteLater();
@ -613,7 +603,7 @@ void Resource::allReferencesCleared() {
if (_cache && isCacheable()) {
// create and reinsert new shared pointer
QSharedPointer<Resource> self(this, &Resource::allReferencesCleared);
QSharedPointer<Resource> self(this, &Resource::deleter);
setSelf(self);
reinsert();
@ -623,7 +613,7 @@ void Resource::allReferencesCleared() {
if (_cache) {
// remove from the cache
_cache->removeResource(getURL(), getBytes());
_cache->resetResourceCounters();
_cache->resetTotalResourceCounter();
}
deleteLater();
@ -732,6 +722,7 @@ void Resource::handleReplyFinished() {
{ "from_cache", false },
{ "size_mb", _bytesTotal / 1000000.0 }
});
ResourceCache::requestCompleted(_self);
return;
}

41
libraries/networking/src/ResourceCache.h
View file

@ -66,21 +66,25 @@ class ResourceCacheSharedItems : public Dependency {
using Lock = std::unique_lock<Mutex>;
public:
void appendPendingRequest(QWeakPointer<Resource> newRequest);
void appendActiveRequest(QWeakPointer<Resource> newRequest);
bool appendRequest(QWeakPointer<Resource> newRequest);
void removeRequest(QWeakPointer<Resource> doneRequest);
QList<QSharedPointer<Resource>> getPendingRequests();
uint32_t getPendingRequestsCount() const;
QList<QSharedPointer<Resource>> getLoadingRequests();
void setRequestLimit(uint32_t limit);
uint32_t getRequestLimit() const;
QList<QSharedPointer<Resource>> getPendingRequests() const;
QSharedPointer<Resource> getHighestPendingRequest();
uint32_t getPendingRequestsCount() const;
QList<QSharedPointer<Resource>> getLoadingRequests() const;
uint32_t getLoadingRequestsCount() const;
void clear();
private:
ResourceCacheSharedItems() = default;
mutable Mutex _mutex;
QList<QWeakPointer<Resource>> _pendingRequests;
QList<QWeakPointer<Resource>> _loadingRequests;
QList<QSharedPointer<Resource>> _loadingRequests;
const uint32_t DEFAULT_REQUEST_LIMIT = 10;
uint32_t _requestLimit { DEFAULT_REQUEST_LIMIT };
};
/// Wrapper to expose resources to JS/QML
@ -200,25 +204,20 @@ public:
Q_INVOKABLE QVariantList getResourceList();
static void setRequestLimit(int limit);
static int getRequestLimit() { return _requestLimit; }
static int getRequestsActive() { return _requestsActive; }
static void setRequestLimit(uint32_t limit);
static uint32_t getRequestLimit() { return DependencyManager::get<ResourceCacheSharedItems>()->getRequestLimit(); }
void setUnusedResourceCacheSize(qint64 unusedResourcesMaxSize);
qint64 getUnusedResourceCacheSize() const { return _unusedResourcesMaxSize; }
static QList<QSharedPointer<Resource>> getLoadingRequests();
static int getPendingRequestCount();
static int getLoadingRequestCount();
static uint32_t getPendingRequestCount();
static uint32_t getLoadingRequestCount();
ResourceCache(QObject* parent = nullptr);
virtual ~ResourceCache();
void refreshAll();
void refresh(const QUrl& url);
void clearUnusedResources();
signals:
@ -272,11 +271,11 @@ private:
friend class ScriptableResourceCache;
void reserveUnusedResource(qint64 resourceSize);
void resetResourceCounters();
void removeResource(const QUrl& url, qint64 size = 0);
static int _requestLimit;
static int _requestsActive;
void resetTotalResourceCounter();
void resetUnusedResourceCounter();
void resetResourceCounters();
// Resources
QHash<QUrl, QWeakPointer<Resource>> _resources;
@ -293,10 +292,6 @@ private:
std::atomic<size_t> _numUnusedResources { 0 };
std::atomic<qint64> _unusedResourcesSize { 0 };
// Pending resources
QQueue<QUrl> _resourcesToBeGotten;
QReadWriteLock _resourcesToBeGottenLock { QReadWriteLock::Recursive };
};
/// Wrapper to expose resource caches to JS/QML
@ -455,7 +450,7 @@ protected:
virtual void makeRequest();
/// Checks whether the resource is cacheable.
virtual bool isCacheable() const { return true; }
virtual bool isCacheable() const { return _loaded; }
/// Called when the download has finished.
/// This should be overridden by subclasses that need to process the data once it is downloaded.