Merge pull request #10621 from highfidelity/stable

Merge Stable back into Master

interface/src/Application.cpp

@@ -72,6 +72,7 @@
 #include <ErrorDialog.h>
 #include <FileScriptingInterface.h>
 #include <Finally.h>
+#include <FingerprintUtils.h>
 #include <FramebufferCache.h>
 #include <gpu/Batch.h>
 #include <gpu/Context.h>
@@ -114,6 +115,7 @@
 #include <RenderDeferredTask.h>
 #include <RenderForwardTask.h>
 #include <ResourceCache.h>
+#include <ResourceRequest.h>
 #include <SandboxUtils.h>
 #include <SceneScriptingInterface.h>
 #include <ScriptEngines.h>
@@ -951,6 +953,13 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         properties["processor_l3_cache_count"] = procInfo.numProcessorCachesL3;
     }
 
+    // add firstRun flag from settings to launch event
+    Setting::Handle<bool> firstRun { Settings::firstRun, true };
+    properties["first_run"] = firstRun.get();
+
+    // add the user's machine ID to the launch event
+    properties["machine_fingerprint"] = uuidStringWithoutCurlyBraces(FingerprintUtils::getMachineFingerprint());
+
     UserActivityLogger::getInstance().logAction("launch", properties);
 
     // Tell our entity edit sender about our known jurisdictions
@@ -1310,6 +1319,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         properties["kbps_in"] = bandwidthRecorder->getCachedTotalAverageInputKilobitsPerSecond();
         properties["kbps_out"] = bandwidthRecorder->getCachedTotalAverageOutputKilobitsPerSecond();
 
+        properties["atp_in_kbps"] = bandwidthRecorder->getAverageInputKilobitsPerSecond(NodeType::AssetServer);
+
         auto nodeList = DependencyManager::get<NodeList>();
         SharedNodePointer entityServerNode = nodeList->soloNodeOfType(NodeType::EntityServer);
         SharedNodePointer audioMixerNode = nodeList->soloNodeOfType(NodeType::AudioMixer);
@@ -1323,8 +1334,61 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         properties["messages_ping"] = messagesMixerNode ? messagesMixerNode->getPingMs() : -1;
 
         auto loadingRequests = ResourceCache::getLoadingRequests();
+
+        QJsonArray loadingRequestsStats;
+        for (const auto& request : loadingRequests) {
+            QJsonObject requestStats;
+            requestStats["filename"] = request->getURL().fileName();
+            requestStats["received"] = request->getBytesReceived();
+            requestStats["total"] = request->getBytesTotal();
+            requestStats["attempts"] = (int)request->getDownloadAttempts();
+            loadingRequestsStats.append(requestStats);
+        }
+
         properties["active_downloads"] = loadingRequests.size();
         properties["pending_downloads"] = ResourceCache::getPendingRequestCount();
+        properties["active_downloads_details"] = loadingRequestsStats;
+
+        auto statTracker = DependencyManager::get<StatTracker>();
+
+        properties["processing_resources"] = statTracker->getStat("Processing").toInt();
+        properties["pending_processing_resources"] = statTracker->getStat("PendingProcessing").toInt();
+
+        QJsonObject startedRequests;
+        startedRequests["atp"] = statTracker->getStat(STAT_ATP_REQUEST_STARTED).toInt();
+        startedRequests["http"] = statTracker->getStat(STAT_HTTP_REQUEST_STARTED).toInt();
+        startedRequests["file"] = statTracker->getStat(STAT_FILE_REQUEST_STARTED).toInt();
+        startedRequests["total"] = startedRequests["atp"].toInt() + startedRequests["http"].toInt()
+            + startedRequests["file"].toInt();
+        properties["started_requests"] = startedRequests;
+
+        QJsonObject successfulRequests;
+        successfulRequests["atp"] = statTracker->getStat(STAT_ATP_REQUEST_SUCCESS).toInt();
+        successfulRequests["http"] = statTracker->getStat(STAT_HTTP_REQUEST_SUCCESS).toInt();
+        successfulRequests["file"] = statTracker->getStat(STAT_FILE_REQUEST_SUCCESS).toInt();
+        successfulRequests["total"] = successfulRequests["atp"].toInt() + successfulRequests["http"].toInt()
+            + successfulRequests["file"].toInt();
+        properties["successful_requests"] = successfulRequests;
+
+        QJsonObject failedRequests;
+        failedRequests["atp"] = statTracker->getStat(STAT_ATP_REQUEST_FAILED).toInt();
+        failedRequests["http"] = statTracker->getStat(STAT_HTTP_REQUEST_FAILED).toInt();
+        failedRequests["file"] = statTracker->getStat(STAT_FILE_REQUEST_FAILED).toInt();
+        failedRequests["total"] = failedRequests["atp"].toInt() + failedRequests["http"].toInt()
+            + failedRequests["file"].toInt();
+        properties["failed_requests"] = failedRequests;
+
+        QJsonObject cacheRequests;
+        cacheRequests["atp"] = statTracker->getStat(STAT_ATP_REQUEST_CACHE).toInt();
+        cacheRequests["http"] = statTracker->getStat(STAT_HTTP_REQUEST_CACHE).toInt();
+        cacheRequests["total"] = cacheRequests["atp"].toInt() + cacheRequests["http"].toInt();
+        properties["cache_requests"] = cacheRequests;
+
+        QJsonObject atpMappingRequests;
+        atpMappingRequests["started"] = statTracker->getStat(STAT_ATP_MAPPING_REQUEST_STARTED).toInt();
+        atpMappingRequests["failed"] = statTracker->getStat(STAT_ATP_MAPPING_REQUEST_FAILED).toInt();
+        atpMappingRequests["successful"] = statTracker->getStat(STAT_ATP_MAPPING_REQUEST_SUCCESS).toInt();
+        properties["atp_mapping_requests"] = atpMappingRequests;
 
         properties["throttled"] = _displayPlugin ? _displayPlugin->isThrottled() : false;
 
@@ -1340,6 +1404,15 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         properties["deleted_entity_cnt"] = entityActivityTracking.deletedEntityCount;
         properties["edited_entity_cnt"] = entityActivityTracking.editedEntityCount;
 
+        NodeToOctreeSceneStats* octreeServerSceneStats = getOcteeSceneStats();
+        unsigned long totalServerOctreeElements = 0;
+        for (NodeToOctreeSceneStatsIterator i = octreeServerSceneStats->begin(); i != octreeServerSceneStats->end(); i++) {
+            totalServerOctreeElements += i->second.getTotalElements();
+        }
+
+        properties["local_octree_elements"] = (qint64) OctreeElement::getInternalNodeCount();
+        properties["server_octree_elements"] = (qint64) totalServerOctreeElements;
+
         properties["active_display_plugin"] = getActiveDisplayPlugin()->getName();
         properties["using_hmd"] = isHMDMode();
 
@@ -1365,6 +1438,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         lastLeftHandPose = leftHandPose;
         lastRightHandPose = rightHandPose;
 
+        properties["local_socket_changes"] = DependencyManager::get<StatTracker>()->getStat(LOCAL_SOCKET_CHANGE_STAT).toInt();
+
         UserActivityLogger::getInstance().logAction("stats", properties);
     });
     sendStatsTimer->start();

libraries/networking/src/AssetRequest.cpp

@@ -15,11 +15,13 @@
 
 #include <QtCore/QThread>
 
+#include <StatTracker.h>
+#include <Trace.h>
+
 #include "AssetClient.h"
 #include "NetworkLogging.h"
 #include "NodeList.h"
 #include "ResourceCache.h"
-#include <Trace.h>
 
 static int requestID = 0;
 
@@ -62,9 +64,12 @@ void AssetRequest::start() {
     _data = loadFromCache(getUrl());
     if (!_data.isNull()) {
         _error = NoError;
-        
+
+        _loadedFromCache = true;
+
         _state = Finished;
         emit finished(this);
+
         return;
     }
 

libraries/networking/src/AssetRequest.h

@@ -52,6 +52,8 @@ public:
     QUrl getUrl() const { return ::getATPUrl(_hash); }
     QString getHash() const { return _hash; }
 
+    bool loadedFromCache() const { return _loadedFromCache; }
+
 signals:
     void finished(AssetRequest* thisRequest);
     void progress(qint64 totalReceived, qint64 total);
@@ -66,6 +68,7 @@ private:
     int _numPendingRequests { 0 };
     MessageID _assetRequestID { INVALID_MESSAGE_ID };
     const ByteRange _byteRange;
+    bool _loadedFromCache { false };
 };
 
 #endif

libraries/networking/src/AssetResourceRequest.cpp

@@ -13,12 +13,14 @@
 
 #include <QtCore/QLoggingCategory>
 
+#include <Trace.h>
+#include <Profile.h>
+#include <StatTracker.h>
+
 #include "AssetClient.h"
 #include "AssetUtils.h"
 #include "MappingRequest.h"
 #include "NetworkLogging.h"
-#include <Trace.h>
-#include <Profile.h>
 
 static const int DOWNLOAD_PROGRESS_LOG_INTERVAL_SECONDS = 5;
 
@@ -48,6 +50,8 @@ bool AssetResourceRequest::urlIsAssetHash(const QUrl& url) {
 }
 
 void AssetResourceRequest::doSend() {
+    DependencyManager::get<StatTracker>()->incrementStat(STAT_ATP_REQUEST_STARTED);
+
     // We'll either have a hash or an ATP path to a file (that maps to a hash)
     if (urlIsAssetHash(_url)) {
         // We've detected that this is a hash - simply use AssetClient to request that asset
@@ -65,11 +69,16 @@ void AssetResourceRequest::doSend() {
 }
 
 void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
+    auto statTracker = DependencyManager::get<StatTracker>();
+    statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_STARTED);
+
     auto assetClient = DependencyManager::get<AssetClient>();
     _assetMappingRequest = assetClient->createGetMappingRequest(path);
 
     // make sure we'll hear about the result of the get mapping request
     connect(_assetMappingRequest, &GetMappingRequest::finished, this, [this, path](GetMappingRequest* request){
+        auto statTracker = DependencyManager::get<StatTracker>();
+        
         Q_ASSERT(_state == InProgress);
         Q_ASSERT(request == _assetMappingRequest);
 
@@ -80,6 +89,8 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
 
                 requestHash(request->getHash());
 
+                statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_SUCCESS);
+
                 break;
             default: {
                 switch (request->getError()) {
@@ -100,6 +111,9 @@ void AssetResourceRequest::requestMappingForPath(const AssetPath& path) {
                 _state = Finished;
                 emit finished();
 
+                statTracker->incrementStat(STAT_ATP_MAPPING_REQUEST_FAILED);
+                statTracker->incrementStat(STAT_ATP_REQUEST_FAILED);
+
                 break;
             }
         }
@@ -140,10 +154,26 @@ void AssetResourceRequest::requestHash(const AssetHash& hash) {
                 _result = Error;
                 break;
         }
-        
+
+        auto statTracker = DependencyManager::get<StatTracker>();
+
+        if (_assetRequest->loadedFromCache()) {
+            _loadedFromCache = true;
+        }
+
         _state = Finished;
         emit finished();
 
+        if (_result == Success) {
+            statTracker->incrementStat(STAT_ATP_REQUEST_SUCCESS);
+
+            if (loadedFromCache()) {
+                statTracker->incrementStat(STAT_ATP_REQUEST_CACHE);
+            }
+        } else {
+            statTracker->incrementStat(STAT_ATP_REQUEST_FAILED);
+        }
+
         _assetRequest->deleteLater();
         _assetRequest = nullptr;
     });

libraries/networking/src/FileResourceRequest.cpp

@@ -15,7 +15,12 @@
 
 #include <QFile>
 
+#include <StatTracker.h>
+
 void FileResourceRequest::doSend() {
+    auto statTracker = DependencyManager::get<StatTracker>();
+    statTracker->incrementStat(STAT_FILE_REQUEST_STARTED);
+
     QString filename = _url.toLocalFile();
     
     // sometimes on windows, we see the toLocalFile() return null,
@@ -60,4 +65,10 @@ void FileResourceRequest::doSend() {
     
     _state = Finished;
     emit finished();
+
+    if (_result == ResourceRequest::Success) {
+        statTracker->incrementStat(STAT_FILE_REQUEST_SUCCESS);
+    } else {
+        statTracker->incrementStat(STAT_FILE_REQUEST_FAILED);
+    }
 }

libraries/networking/src/HTTPResourceRequest.cpp

@@ -17,6 +17,7 @@
 #include <QMetaEnum>
 
 #include <SharedUtil.h>
+#include <StatTracker.h>
 
 #include "NetworkAccessManager.h"
 #include "NetworkLogging.h"
@@ -49,6 +50,8 @@ void HTTPResourceRequest::cleanupTimer() {
 }
 
 void HTTPResourceRequest::doSend() {
+    DependencyManager::get<StatTracker>()->incrementStat(STAT_HTTP_REQUEST_STARTED);
+
     QNetworkRequest networkRequest(_url);
     networkRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
     networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
@@ -178,6 +181,17 @@ void HTTPResourceRequest::onRequestFinished() {
     
     _state = Finished;
     emit finished();
+
+    auto statTracker = DependencyManager::get<StatTracker>();
+    if (_result == Success) {
+        statTracker->incrementStat(STAT_HTTP_REQUEST_SUCCESS);
+
+        if (loadedFromCache()) {
+            statTracker->incrementStat(STAT_HTTP_REQUEST_CACHE);
+        }
+    } else {
+        statTracker->incrementStat(STAT_HTTP_REQUEST_FAILED);
+    }
 }
 
 void HTTPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) {
@@ -202,4 +216,6 @@ void HTTPResourceRequest::onTimeout() {
     _result = Timeout;
     _state = Finished;
     emit finished();
+
+    DependencyManager::get<StatTracker>()->incrementStat(STAT_HTTP_REQUEST_FAILED);
 }

libraries/networking/src/LimitedNodeList.cpp

@@ -29,6 +29,7 @@
 #include <NumericalConstants.h>
 #include <SettingHandle.h>
 #include <SharedUtil.h>
+#include <StatTracker.h>
 #include <UUID.h>
 
 #include "AccountManager.h"
@@ -37,7 +38,6 @@
 #include "HifiSockAddr.h"
 #include "NetworkLogging.h"
 #include "udt/Packet.h"
-#include <Trace.h>
 
 static Setting::Handle<quint16> LIMITED_NODELIST_LOCAL_PORT("LimitedNodeList.LocalPort", 0);
 
@@ -1044,10 +1044,10 @@ void LimitedNodeList::setLocalSocket(const HifiSockAddr& sockAddr) {
             qCInfo(networking) << "Local socket is" << sockAddr;
         } else {
             qCInfo(networking) << "Local socket has changed from" << _localSockAddr << "to" << sockAddr;
+            DependencyManager::get<StatTracker>()->incrementStat(LOCAL_SOCKET_CHANGE_STAT);
         }
 
         _localSockAddr = sockAddr;
-
         emit localSockAddrChanged(_localSockAddr);
     }
 }

libraries/networking/src/LimitedNodeList.h

@@ -66,6 +66,8 @@ const QHostAddress DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME = QHostAddress::Lo
 
 const QString USERNAME_UUID_REPLACEMENT_STATS_KEY = "$username";
 
+const QString LOCAL_SOCKET_CHANGE_STAT = "LocalSocketChanges";
+
 using namespace tbb;
 typedef std::pair<QUuid, SharedNodePointer> UUIDNodePair;
 typedef concurrent_unordered_map<QUuid, SharedNodePointer, UUIDHasher> NodeHash;

libraries/networking/src/ResourceRequest.h

@@ -19,6 +19,21 @@
 
 #include "ByteRange.h"
 
+const QString STAT_ATP_REQUEST_STARTED = "StartedATPRequest";
+const QString STAT_HTTP_REQUEST_STARTED = "StartedHTTPRequest";
+const QString STAT_FILE_REQUEST_STARTED = "StartedFileRequest";
+const QString STAT_ATP_REQUEST_SUCCESS = "SuccessfulATPRequest";
+const QString STAT_HTTP_REQUEST_SUCCESS = "SuccessfulHTTPRequest";
+const QString STAT_FILE_REQUEST_SUCCESS = "SuccessfulFileRequest";
+const QString STAT_ATP_REQUEST_FAILED = "FailedATPRequest";
+const QString STAT_HTTP_REQUEST_FAILED = "FailedHTTPRequest";
+const QString STAT_FILE_REQUEST_FAILED = "FailedFileRequest";
+const QString STAT_ATP_REQUEST_CACHE = "CacheATPRequest";
+const QString STAT_HTTP_REQUEST_CACHE = "CacheHTTPRequest";
+const QString STAT_ATP_MAPPING_REQUEST_STARTED = "StartedATPMappingRequest";
+const QString STAT_ATP_MAPPING_REQUEST_FAILED = "FailedATPMappingRequest";
+const QString STAT_ATP_MAPPING_REQUEST_SUCCESS = "SuccessfulATPMappingRequest";
+
 class ResourceRequest : public QObject {
     Q_OBJECT
 public: