From 58086589583e2480f108b5cb6098db4ecd57eee1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Sep 2015 16:26:43 +0200 Subject: [PATCH 01/10] Make NodeJSONStats reliable --- .../src/AssignmentClientMonitor.cpp | 1 - assignment-client/src/assets/AssetServer.cpp | 6 +-- .../src/assets/SendAssetTask.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 14 +++--- .../octree/OctreeInboundPacketProcessor.cpp | 10 ++-- domain-server/src/DomainServer.cpp | 18 +++---- domain-server/src/DomainServer.h | 2 +- domain-server/src/DomainServerNodeData.cpp | 6 +-- domain-server/src/DomainServerNodeData.h | 2 +- .../src/DomainServerSettingsManager.cpp | 2 +- interface/src/Application.cpp | 11 ++-- interface/src/ui/AddressBarDialog.cpp | 1 + interface/src/ui/AddressBarDialog.h | 1 + interface/src/ui/DialogsManager.h | 6 --- libraries/networking/src/AssetClient.cpp | 2 +- libraries/networking/src/LimitedNodeList.cpp | 6 +++ libraries/networking/src/LimitedNodeList.h | 2 +- libraries/networking/src/NLPacketList.cpp | 22 ++++++-- libraries/networking/src/NLPacketList.h | 9 +++- libraries/networking/src/NodeList.cpp | 15 ++---- libraries/networking/src/PacketReceiver.cpp | 10 ++-- libraries/networking/src/udt/BasePacket.h | 12 ++--- libraries/networking/src/udt/PacketList.cpp | 50 +++++++++++-------- libraries/networking/src/udt/PacketList.h | 29 ++++++----- 24 files changed, 130 insertions(+), 109 deletions(-) diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index 5bb31c8c6f..7c08d01996 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 9d60b74508..b010ce704a 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -12,14 +12,12 @@ #include "AssetServer.h" -#include +#include #include +#include #include #include #include -#include -#include -#include #include #include "NetworkLogging.h" diff --git a/assignment-client/src/assets/SendAssetTask.cpp b/assignment-client/src/assets/SendAssetTask.cpp index f12603d6c7..9211aa1256 100644 --- a/assignment-client/src/assets/SendAssetTask.cpp +++ b/assignment-client/src/assets/SendAssetTask.cpp @@ -52,7 +52,7 @@ void SendAssetTask::run() { qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end; qDebug() << "Starting task to send asset: " << hexHash << " for messageID " << messageID; - auto replyPacketList = std::unique_ptr(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true)); + auto replyPacketList = NLPacketList::create(PacketType::AssetGetReply, QByteArray(), true, true); replyPacketList->write(assetHash); diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 195bd53a97..66b924b8ab 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -219,7 +219,7 @@ void AvatarMixer::broadcastAvatarData() { } // setup a PacketList for the avatarPackets - NLPacketList avatarPacketList(PacketType::BulkAvatarData); + auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); // this is an AGENT we have received head data from // send back a packet with other active node data to this node @@ -292,13 +292,13 @@ void AvatarMixer::broadcastAvatarData() { otherNodeData->getLastReceivedSequenceNumber()); // start a new segment in the PacketList for this avatar - avatarPacketList.startSegment(); + avatarPacketList->startSegment(); - numAvatarDataBytes += avatarPacketList.write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); numAvatarDataBytes += - avatarPacketList.write(otherAvatar.toByteArray(false, randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO)); + avatarPacketList->write(otherAvatar.toByteArray(false, randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO)); - avatarPacketList.endSegment(); + avatarPacketList->endSegment(); // if the receiving avatar has just connected make sure we send out the mesh and billboard // for this avatar (assuming they exist) @@ -344,10 +344,10 @@ void AvatarMixer::broadcastAvatarData() { }); // close the current packet so that we're always sending something - avatarPacketList.closeCurrentPacket(true); + avatarPacketList->closeCurrentPacket(true); // send the avatar data PacketList - nodeList->sendPacketList(avatarPacketList, *node); + nodeList->sendPacketList(std::move(avatarPacketList), *node); // record the bytes sent for other avatar data in the AvatarMixerClientData nodeData->recordSentAvatarData(numAvatarDataBytes); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index a023b9a7fd..4fde1970c7 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -238,7 +238,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { return 0; } - NLPacketList nackPacketList(_myServer->getMyEditNackType()); + auto nackPacketList = NLPacketList::create(_myServer->getMyEditNackType()); auto nodeList = DependencyManager::get(); int packetsSent = 0; @@ -274,18 +274,18 @@ int OctreeInboundPacketProcessor::sendNackPackets() { while (it != missingSequenceNumbers.constEnd()) { unsigned short int sequenceNumber = *it; - nackPacketList.writePrimitive(sequenceNumber); + nackPacketList->writePrimitive(sequenceNumber); ++it; } - if (nackPacketList.getNumPackets()) { + if (nackPacketList->getNumPackets()) { qDebug() << "NACK Sent back to editor/client... destinationNode=" << nodeUUID; - packetsSent += nackPacketList.getNumPackets(); + packetsSent += nackPacketList->getNumPackets(); // send the list of nack packets - nodeList->sendPacketList(nackPacketList, *destinationNode); + nodeList->sendPacketList(std::move(nackPacketList), *destinationNode); } ++i; diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 2b59ba7510..2f5e553c9f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -287,7 +287,7 @@ void DomainServer::setupNodeListAndAssignments(const QUuid& sessionUUID) { packetReceiver.registerListener(PacketType::RequestAssignment, this, "processRequestAssignmentPacket"); packetReceiver.registerListener(PacketType::DomainListRequest, this, "processListRequestPacket"); packetReceiver.registerListener(PacketType::DomainServerPathQuery, this, "processPathQueryPacket"); - packetReceiver.registerListener(PacketType::NodeJsonStats, this, "processNodeJSONStatsPacket"); + packetReceiver.registerMessageListener(PacketType::NodeJsonStats, this, "processNodeJSONStatsPacket"); // NodeList won't be available to the settings manager when it is created, so call registerListener here packetReceiver.registerListener(PacketType::DomainSettingsRequest, &_settingsManager, "processSettingsRequestPacket"); @@ -679,10 +679,10 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif extendedHeaderStream << (quint8) node->getCanAdjustLocks(); extendedHeaderStream << (quint8) node->getCanRez(); - NLPacketList domainListPackets(PacketType::DomainList, extendedHeader); + auto domainListPackets = NLPacketList::create(PacketType::DomainList, extendedHeader); // always send the node their own UUID back - QDataStream domainListStream(&domainListPackets); + QDataStream domainListStream(domainListPackets.get()); DomainServerNodeData* nodeData = reinterpret_cast(node->getLinkedData()); @@ -698,7 +698,7 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif if (otherNode->getUUID() != node->getUUID() && nodeInterestSet.contains(otherNode->getType())) { // since we're about to add a node to the packet we start a segment - domainListPackets.startSegment(); + domainListPackets->startSegment(); // don't send avatar nodes to other avatars, that will come from avatar mixer domainListStream << *otherNode.data(); @@ -707,17 +707,17 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif domainListStream << connectionSecretForNodes(node, otherNode); // we've added the node we wanted so end the segment now - domainListPackets.endSegment(); + domainListPackets->endSegment(); } }); } } // send an empty list to the node, in case there were no other nodes - domainListPackets.closeCurrentPacket(true); + domainListPackets->closeCurrentPacket(true); // write the PacketList to this node - limitedNodeList->sendPacketList(domainListPackets, *node); + limitedNodeList->sendPacketList(std::move(domainListPackets), *node); } QUuid DomainServer::connectionSecretForNodes(const SharedNodePointer& nodeA, const SharedNodePointer& nodeB) { @@ -1007,10 +1007,10 @@ void DomainServer::sendHeartbeatToIceServer() { DependencyManager::get()->sendHeartbeatToIceServer(_iceServerSocket); } -void DomainServer::processNodeJSONStatsPacket(QSharedPointer packet, SharedNodePointer sendingNode) { +void DomainServer::processNodeJSONStatsPacket(QSharedPointer packetList, SharedNodePointer sendingNode) { auto nodeData = dynamic_cast(sendingNode->getLinkedData()); if (nodeData) { - nodeData->processJSONStatsPacket(*packet); + nodeData->processJSONStatsPacket(packetList->getMessage()); } } diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index a66192b44e..df42bf3ad9 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -58,7 +58,7 @@ public slots: void processRequestAssignmentPacket(QSharedPointer packet); void processListRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); - void processNodeJSONStatsPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void processNodeJSONStatsPacket(QSharedPointer packetList, SharedNodePointer sendingNode); void processPathQueryPacket(QSharedPointer packet); private slots: diff --git a/domain-server/src/DomainServerNodeData.cpp b/domain-server/src/DomainServerNodeData.cpp index a66eec5705..ecbe6bd36e 100644 --- a/domain-server/src/DomainServerNodeData.cpp +++ b/domain-server/src/DomainServerNodeData.cpp @@ -31,9 +31,9 @@ DomainServerNodeData::DomainServerNodeData() : _paymentIntervalTimer.start(); } -void DomainServerNodeData::processJSONStatsPacket(NLPacket& statsPacket) { - QVariantMap packetVariantMap = JSONBreakableMarshal::fromStringBuffer(statsPacket.readAll()); - _statsJSONObject = mergeJSONStatsFromNewObject(QJsonObject::fromVariantMap(packetVariantMap), _statsJSONObject); +void DomainServerNodeData::processJSONStatsPacket(QByteArray statsByteArray) { + QJsonObject packetJsonObject = JSONBreakableMarshal::fromByteArray(statsByteArray); + _statsJSONObject = mergeJSONStatsFromNewObject(packetJsonObject, _statsJSONObject); } QJsonObject DomainServerNodeData::mergeJSONStatsFromNewObject(const QJsonObject& newObject, QJsonObject destinationObject) { diff --git a/domain-server/src/DomainServerNodeData.h b/domain-server/src/DomainServerNodeData.h index 2c57368653..73003f1a7c 100644 --- a/domain-server/src/DomainServerNodeData.h +++ b/domain-server/src/DomainServerNodeData.h @@ -27,7 +27,7 @@ public: const QJsonObject& getStatsJSONObject() const { return _statsJSONObject; } - void processJSONStatsPacket(NLPacket& packet); + void processJSONStatsPacket(QByteArray statsByteArray); void setAssignmentUUID(const QUuid& assignmentUUID) { _assignmentUUID = assignmentUUID; } const QUuid& getAssignmentUUID() const { return _assignmentUUID; } diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index fc0ed95b92..f650089486 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -74,7 +74,7 @@ void DomainServerSettingsManager::processSettingsRequestPacket(QSharedPointer(new NLPacketList(PacketType::DomainSettings, QByteArray(), true, true)); + auto packetList = NLPacketList::create(PacketType::DomainSettings, QByteArray(), true, true); packetList->write(json); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 07c1b4e38d..b51ba91246 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -142,14 +142,15 @@ #include "SpeechRecognizer.h" #endif +#include "ui/AddressBarDialog.h" #include "ui/AvatarInputs.h" #include "ui/DataWebDialog.h" #include "ui/DialogsManager.h" +#include "ui/FileDialog.h" #include "ui/LoginDialog.h" #include "ui/Snapshot.h" #include "ui/StandAloneJSConsole.h" #include "ui/Stats.h" -#include "ui/AddressBarDialog.h" #include "ui/UpdateDialog.h" #include "ui/overlays/Cube3DOverlay.h" @@ -3009,7 +3010,7 @@ int Application::sendNackPackets() { if (node->getActiveSocket() && node->getType() == NodeType::EntityServer) { - NLPacketList nackPacketList(PacketType::OctreeDataNack); + auto nackPacketList = NLPacketList::create(PacketType::OctreeDataNack); QUuid nodeUUID = node->getUUID(); @@ -3038,15 +3039,15 @@ int Application::sendNackPackets() { auto it = missingSequenceNumbers.constBegin(); while (it != missingSequenceNumbers.constEnd()) { OCTREE_PACKET_SEQUENCE missingNumber = *it; - nackPacketList.writePrimitive(missingNumber); + nackPacketList->writePrimitive(missingNumber); ++it; } if (nackPacketList.getNumPackets()) { - packetsSent += nackPacketList.getNumPackets(); + packetsSent += nackPacketList->getNumPackets(); // send the packet list - nodeList->sendPacketList(nackPacketList, *node); + nodeList->sendPacketList(std::move(nackPacketList), *node); } } }); diff --git a/interface/src/ui/AddressBarDialog.cpp b/interface/src/ui/AddressBarDialog.cpp index 5d27af444a..b208c15264 100644 --- a/interface/src/ui/AddressBarDialog.cpp +++ b/interface/src/ui/AddressBarDialog.cpp @@ -1,5 +1,6 @@ // // AddressBarDialog.cpp +// interface/src/ui // // Created by Bradley Austin Davis on 2015/04/14 // Copyright 2015 High Fidelity, Inc. diff --git a/interface/src/ui/AddressBarDialog.h b/interface/src/ui/AddressBarDialog.h index eec3acdbfc..811e2f5b12 100644 --- a/interface/src/ui/AddressBarDialog.h +++ b/interface/src/ui/AddressBarDialog.h @@ -1,5 +1,6 @@ // // AddressBarDialog.h +// interface/src/ui // // Created by Bradley Austin Davis on 2015/04/14 // Copyright 2015 High Fidelity, Inc. diff --git a/interface/src/ui/DialogsManager.h b/interface/src/ui/DialogsManager.h index 68d371021a..d54e91b9d6 100644 --- a/interface/src/ui/DialogsManager.h +++ b/interface/src/ui/DialogsManager.h @@ -21,7 +21,6 @@ class QAction; -class AddressBarDialog; class AnimationsDialog; class AttachmentsDialog; class AudioStatsDialog; @@ -29,13 +28,11 @@ class BandwidthDialog; class CachesSizeDialog; class DiskCacheEditor; class LodToolsDialog; -class LoginDialog; class OctreeStatsDialog; class PreferencesDialog; class ScriptEditorWindow; class QMessageBox; class DomainConnectionDialog; -class UpdateDialog; class DialogsManager : public QObject, public Dependency { Q_OBJECT @@ -94,7 +91,6 @@ private: } } - QPointer _addressBarDialog; QPointer _animationsDialog; QPointer _attachmentsDialog; QPointer _audioStatsDialog; @@ -104,12 +100,10 @@ private: QPointer _ircInfoBox; QPointer _hmdToolsDialog; QPointer _lodToolsDialog; - QPointer _loginDialog; QPointer _octreeStatsDialog; QPointer _preferencesDialog; QPointer _scriptEditor; QPointer _domainConnectionDialog; - QPointer _updateDialog; }; #endif // hifi_DialogsManager_h diff --git a/libraries/networking/src/AssetClient.cpp b/libraries/networking/src/AssetClient.cpp index 9931bab5ed..2de5173b76 100644 --- a/libraries/networking/src/AssetClient.cpp +++ b/libraries/networking/src/AssetClient.cpp @@ -205,7 +205,7 @@ bool AssetClient::uploadAsset(const QByteArray& data, const QString& extension, SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); if (assetServer) { - auto packetList = std::unique_ptr(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true)); + auto packetList = NLPacketList::create(PacketType::AssetUpload, QByteArray(), true, true); auto messageID = ++_currentID; packetList->writePrimitive(messageID); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index b6c3db3562..e91e8a7c7e 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -355,6 +355,12 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, // close the last packet in the list packetList->closeCurrentPacket(); + for (std::unique_ptr& packet : packetList->_packets) { + NLPacket* nlPacket = static_cast(packet.get()); + collectPacketStats(*nlPacket); + fillPacketHeader(*nlPacket); + } + return _nodeSocket.writePacketList(std::move(packetList), sockAddr); } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index f1d4887b70..9d4e426d47 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -254,7 +254,7 @@ protected: qint64 writePacket(const NLPacket& packet, const HifiSockAddr& destinationSockAddr, const QUuid& connectionSecret = QUuid()); void collectPacketStats(const NLPacket& packet); - void fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret); + void fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret = QUuid()); bool isPacketVerified(const udt::Packet& packet); bool packetVersionMatch(const udt::Packet& packet); diff --git a/libraries/networking/src/NLPacketList.cpp b/libraries/networking/src/NLPacketList.cpp index 6dfd627271..3b115c558b 100644 --- a/libraries/networking/src/NLPacketList.cpp +++ b/libraries/networking/src/NLPacketList.cpp @@ -13,16 +13,30 @@ #include "udt/Packet.h" + +std::unique_ptr NLPacketList::create(PacketType packetType, QByteArray extendedHeader, + bool isReliable, bool isOrdered) { + auto nlPacketList = std::unique_ptr(new NLPacketList(packetType, extendedHeader, + isReliable, isOrdered)); + nlPacketList->open(WriteOnly); + return nlPacketList; +} + +std::unique_ptr NLPacketList::fromPacketList(std::unique_ptr packetList) { + auto nlPacketList = std::unique_ptr(new NLPacketList(std::move(*packetList.release()))); nlPacketList->open(ReadOnly); + return nlPacketList; +} + + NLPacketList::NLPacketList(PacketType packetType, QByteArray extendedHeader, bool isReliable, bool isOrdered) : PacketList(packetType, extendedHeader, isReliable, isOrdered) { } -NLPacketList::NLPacketList(PacketList&& other) : PacketList(other.getType(), other.getExtendedHeader(), other.isReliable(), other.isOrdered()) { +NLPacketList::NLPacketList(PacketList&& other) : PacketList(std::move(other)) { // Update _packets - for (auto& packet : other._packets) { - auto nlPacket = NLPacket::fromBase(std::move(packet)); - _packets.push_back(std::move(nlPacket)); + for (auto& packet : _packets) { + packet = NLPacket::fromBase(std::move(packet)); } if (_packets.size() > 0) { diff --git a/libraries/networking/src/NLPacketList.h b/libraries/networking/src/NLPacketList.h index 5391e49488..c705e6ece3 100644 --- a/libraries/networking/src/NLPacketList.h +++ b/libraries/networking/src/NLPacketList.h @@ -18,12 +18,17 @@ class NLPacketList : public udt::PacketList { public: - NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false); - NLPacketList(PacketList&& packetList); + static std::unique_ptr create(PacketType packetType, QByteArray extendedHeader = QByteArray(), + bool isReliable = false, bool isOrdered = false); + + static std::unique_ptr fromPacketList(std::unique_ptr); const QUuid& getSourceID() const { return _sourceID; } private: + NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, + bool isOrdered = false); + NLPacketList(PacketList&& packetList); NLPacketList(const NLPacketList& other) = delete; NLPacketList& operator=(const NLPacketList& other) = delete; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index cf202fa83a..6a27726a6b 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -107,20 +107,11 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned } qint64 NodeList::sendStats(const QJsonObject& statsObject, const HifiSockAddr& destination) { - NLPacketList statsPacketList(PacketType::NodeJsonStats); + auto statsPacketList = NLPacketList::create(PacketType::NodeJsonStats, QByteArray(), true, true); - // get a QStringList using JSONBreakableMarshal - QStringList statsStringList = JSONBreakableMarshal::toStringList(statsObject, ""); + statsPacketList->write(JSONBreakableMarshal::toByteArray(statsObject)); - // enumerate the resulting strings - pack them and send off packets via NLPacketList - foreach(const QString& statsItem, statsStringList) { - QByteArray utf8String = statsItem.toUtf8(); - utf8String.append('\0'); - - statsPacketList.write(utf8String); - } - - sendPacketList(statsPacketList, destination); + sendPacketList(std::move(statsPacketList), destination); // enumerate the resulting strings, breaking them into MTU sized packets return 0; diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index fb1e2d896f..0efb8bba7c 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -247,7 +247,7 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr p } // setup an NLPacketList from the PacketList we were passed - auto nlPacketList = new NLPacketList(std::move(*packetList)); + auto nlPacketList = NLPacketList::fromPacketList(std::move(packetList)); auto nodeList = DependencyManager::get(); @@ -297,21 +297,21 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr p success = metaMethod.invoke(listener.first, connectionType, Q_ARG(QSharedPointer, - QSharedPointer(nlPacketList)), + QSharedPointer(nlPacketList.release())), Q_ARG(SharedNodePointer, matchingNode)); } else if (metaMethod.parameterTypes().contains(QSHAREDPOINTER_NODE_NORMALIZED)) { success = metaMethod.invoke(listener.first, connectionType, Q_ARG(QSharedPointer, - QSharedPointer(nlPacketList)), + QSharedPointer(nlPacketList.release())), Q_ARG(QSharedPointer, matchingNode)); } else { success = metaMethod.invoke(listener.first, connectionType, Q_ARG(QSharedPointer, - QSharedPointer(nlPacketList))); + QSharedPointer(nlPacketList.release()))); } } else { listenerIsDead = true; @@ -323,7 +323,7 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr p if (listener.first) { success = listener.second.invoke(listener.first, Q_ARG(QSharedPointer, - QSharedPointer(nlPacketList))); + QSharedPointer(nlPacketList.release()))); } else { listenerIsDead = true; } diff --git a/libraries/networking/src/udt/BasePacket.h b/libraries/networking/src/udt/BasePacket.h index 4e13d99013..cc18c75a80 100644 --- a/libraries/networking/src/udt/BasePacket.h +++ b/libraries/networking/src/udt/BasePacket.h @@ -31,8 +31,6 @@ public: static std::unique_ptr create(qint64 size = -1); static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); - - // Current level's header size static int localHeaderSize(); @@ -72,8 +70,8 @@ public: virtual bool isSequential() const { return false; } virtual bool reset(); virtual qint64 size() const { return _payloadCapacity; } - - using QIODevice::read; + + using QIODevice::read; // Bring QIODevice::read methods to scope, otherwise they are hidden by folling method QByteArray read(qint64 maxSize); QByteArray readWithoutCopy(qint64 maxSize); // this can only be used if packet will stay in scope @@ -107,16 +105,16 @@ protected: }; template qint64 BasePacket::peekPrimitive(T* data) { - return QIODevice::peek(reinterpret_cast(data), sizeof(T)); + return peek(reinterpret_cast(data), sizeof(T)); } template qint64 BasePacket::readPrimitive(T* data) { - return QIODevice::read(reinterpret_cast(data), sizeof(T)); + return read(reinterpret_cast(data), sizeof(T)); } template qint64 BasePacket::writePrimitive(const T& data) { static_assert(!std::is_pointer::value, "T must not be a pointer"); - return QIODevice::write(reinterpret_cast(&data), sizeof(T)); + return write(reinterpret_cast(&data), sizeof(T)); } } // namespace udt diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 7b90276b62..53edfb32f1 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -15,6 +15,21 @@ using namespace udt; +std::unique_ptr PacketList::create(PacketType packetType, QByteArray extendedHeader, + bool isReliable, bool isOrdered) { + auto packetList = std::unique_ptr(new PacketList(packetType, extendedHeader, + isReliable, isOrdered)); + packetList->open(WriteOnly); + return packetList; +} + +std::unique_ptr PacketList::fromReceivedPackets(std::list>&& packets) { + auto packetList = std::unique_ptr(new PacketList(PacketType::Unknown, QByteArray(), true, true)); + packetList->_packets = std::move(packets); + packetList->open(ReadOnly); + return packetList; +} + PacketList::PacketList(PacketType packetType, QByteArray extendedHeader, bool isReliable, bool isOrdered) : _packetType(packetType), _isReliable(isReliable), @@ -22,14 +37,15 @@ PacketList::PacketList(PacketType packetType, QByteArray extendedHeader, bool is _extendedHeader(extendedHeader) { Q_ASSERT_X(!(!_isReliable && _isOrdered), "PacketList", "Unreliable ordered PacketLists are not currently supported"); - QIODevice::open(WriteOnly); } PacketList::PacketList(PacketList&& other) : + _packetType(other._packetType), _packets(std::move(other._packets)), - _packetType(other._packetType) + _isReliable(other._isReliable), + _isOrdered(other._isOrdered), + _extendedHeader(std::move(other._extendedHeader)) { - } void PacketList::startSegment() { @@ -66,12 +82,6 @@ size_t PacketList::getMessageSize() const { return totalBytes; } -std::unique_ptr PacketList::fromReceivedPackets(std::list>&& packets) { - auto packetList = std::unique_ptr(new PacketList(PacketType::Unknown, QByteArray(), true, true)); - packetList->_packets = std::move(packets); - return packetList; -} - std::unique_ptr PacketList::createPacket() { // use the static create method to create a new packet // If this packet list is supposed to be ordered then we consider this to be part of a message @@ -94,6 +104,17 @@ std::unique_ptr PacketList::createPacketWithExtendedHeader() { return packet; } +void PacketList::closeCurrentPacket(bool shouldSendEmpty) { + if (shouldSendEmpty && !_currentPacket) { + _currentPacket = createPacketWithExtendedHeader(); + } + + if (_currentPacket) { + // move the current packet to our list of packets + _packets.push_back(std::move(_currentPacket)); + } +} + QByteArray PacketList::getMessage() { size_t sizeBytes = 0; @@ -191,14 +212,3 @@ qint64 PacketList::writeData(const char* data, qint64 maxSize) { return maxSize; } - -void PacketList::closeCurrentPacket(bool shouldSendEmpty) { - if (shouldSendEmpty && !_currentPacket) { - _currentPacket = createPacketWithExtendedHeader(); - } - - if (_currentPacket) { - // move the current packet to our list of packets - _packets.push_back(std::move(_currentPacket)); - } -} diff --git a/libraries/networking/src/udt/PacketList.h b/libraries/networking/src/udt/PacketList.h index 05800a1b26..bcf13ad954 100644 --- a/libraries/networking/src/udt/PacketList.h +++ b/libraries/networking/src/udt/PacketList.h @@ -28,13 +28,9 @@ class Packet; class PacketList : public QIODevice { Q_OBJECT public: - PacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false); - PacketList(PacketList&& other); - + static std::unique_ptr create(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false); static std::unique_ptr fromReceivedPackets(std::list>&& packets); - virtual bool isSequential() const { return true; } - bool isReliable() const { return _isReliable; } bool isOrdered() const { return _isOrdered; } @@ -53,20 +49,27 @@ public: QByteArray getMessage(); + // QIODevice virtual functions + virtual bool isSequential() const { return false; } + virtual qint64 size() const { return getDataSize(); } + template qint64 readPrimitive(T* data); template qint64 writePrimitive(const T& data); - std::list> _packets; + protected: - virtual qint64 writeData(const char* data, qint64 maxSize); - virtual qint64 readData(char* data, qint64 maxSize) { return 0; } - PacketType _packetType; + PacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false); + PacketList(PacketList&& other); + virtual qint64 writeData(const char* data, qint64 maxSize); + virtual qint64 readData(char* data, qint64 maxSize) { Q_ASSERT(false); } + + PacketType _packetType; + std::list> _packets; private: friend class ::LimitedNodeList; - friend class Socket; friend class SendQueue; - friend class NLPacketList; + friend class Socket; PacketList(const PacketList& other) = delete; PacketList& operator=(const PacketList& other) = delete; @@ -91,12 +94,12 @@ private: template qint64 PacketList::readPrimitive(T* data) { static_assert(!std::is_pointer::value, "T must not be a pointer"); - return QIODevice::read(reinterpret_cast(data), sizeof(T)); + return read(reinterpret_cast(data), sizeof(T)); } template qint64 PacketList::writePrimitive(const T& data) { static_assert(!std::is_pointer::value, "T must not be a pointer"); - return QIODevice::write(reinterpret_cast(&data), sizeof(T)); + return write(reinterpret_cast(&data), sizeof(T)); } template std::unique_ptr PacketList::takeFront() { From 08e0a8d705b9d778d699e356c61733fc8a436d2f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Sep 2015 16:27:37 +0200 Subject: [PATCH 02/10] AssetServer JSON stats --- assignment-client/src/assets/AssetServer.cpp | 51 +++ assignment-client/src/assets/AssetServer.h | 4 +- .../networking/src/JSONBreakableMarshal.cpp | 356 +++--------------- .../networking/src/JSONBreakableMarshal.h | 14 +- libraries/networking/src/LimitedNodeList.cpp | 7 + libraries/networking/src/LimitedNodeList.h | 3 + .../networking/src/udt/ConnectionStats.cpp | 4 +- .../networking/src/udt/ConnectionStats.h | 6 +- libraries/networking/src/udt/Socket.cpp | 10 + libraries/networking/src/udt/Socket.h | 5 + 10 files changed, 144 insertions(+), 316 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index b010ce704a..bf93755bf8 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -163,3 +163,54 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha } } +void AssetServer::sendStatsPacket() { + QJsonObject statsObject; + + auto stats = DependencyManager::get()->sampleStatsForAllConnections(); + + QString baseName("AssetServer"); + + statsObject[baseName + ".num_connections"] = (int)stats.size(); + for (const auto& stat : stats) { + QString uuid = QUuid().toString(); + if (auto node = DependencyManager::get()->findNodeWithAddr(stat.first)) { + uuid = node->getUUID().toString(); + } + + QString connKey = baseName + "." + uuid + ".connection."; + QString sendKey = baseName + "." + uuid + ".sending."; + QString recvKey = baseName + "." + uuid + ".receiving."; + qDebug() << "Testing" << sendKey << recvKey; + + auto endTimeMs = std::chrono::duration_cast(stat.second.endTime); + QDateTime date = QDateTime::fromMSecsSinceEpoch(endTimeMs.count()); + + statsObject[connKey + "lastHeard"] = date.toString(); + statsObject[connKey + "estimatedBandwith"] = stat.second.estimatedBandwith; + statsObject[connKey + "rtt"] = stat.second.rtt; + statsObject[connKey + "congestionWindowSize"] = stat.second.congestionWindowSize; + statsObject[connKey + "packetSendPeriod"] = stat.second.packetSendPeriod; + + statsObject[sendKey + "sendRate"] = stat.second.sendRate; + statsObject[sendKey + "sentPackets"] = stat.second.sentPackets; + statsObject[sendKey + "receivedACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK]; + statsObject[sendKey + "processedACK"] = stat.second.events[udt::ConnectionStats::Stats::ProcessedACK]; + statsObject[sendKey + "receivedLightACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedLightACK]; + statsObject[sendKey + "receivedNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedNAK]; + statsObject[sendKey + "receivedTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]; + statsObject[sendKey + "sentACK2"] = stat.second.events[udt::ConnectionStats::Stats::SentACK2]; + statsObject[sendKey + "retransmission"] = stat.second.events[udt::ConnectionStats::Stats::Retransmission]; + + statsObject[recvKey + "receiveRate"] = stat.second.receiveRate; + statsObject[recvKey + "receivedPackets"] = stat.second.receivedPackets; + statsObject[recvKey + "SentACK"] = stat.second.events[udt::ConnectionStats::Stats::SentACK]; + statsObject[recvKey + "SentLightACK"] = stat.second.events[udt::ConnectionStats::Stats::SentLightACK]; + statsObject[recvKey + "SentNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentNAK]; + statsObject[recvKey + "SentTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentTimeoutNAK]; + statsObject[recvKey + "ReceivedACK2"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK2]; + statsObject[recvKey + "Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate]; + } + + // send off the stats packets + ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); +} diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 1975f746a9..09144660ec 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -31,7 +31,9 @@ private slots: void handleAssetGetInfo(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetGet(QSharedPointer packet, SharedNodePointer senderNode); void handleAssetUpload(QSharedPointer packetList, SharedNodePointer senderNode); - + + void sendStatsPacket(); + private: static void writeError(NLPacketList* packetList, AssetServerError error); QDir _resourcesDirectory; diff --git a/libraries/networking/src/JSONBreakableMarshal.cpp b/libraries/networking/src/JSONBreakableMarshal.cpp index d71d173947..f0a3ff54ef 100644 --- a/libraries/networking/src/JSONBreakableMarshal.cpp +++ b/libraries/networking/src/JSONBreakableMarshal.cpp @@ -13,326 +13,76 @@ #include #include +#include #include QVariantMap JSONBreakableMarshal::_interpolationMap = QVariantMap(); -QStringList JSONBreakableMarshal::toStringList(const QJsonValue& jsonValue, const QString& keypath) { - // setup the string list that will hold our result - QStringList result; - - // figure out what type of value this is so we know how to act on it - if (jsonValue.isObject()) { - - QJsonObject jsonObject = jsonValue.toObject(); - - // enumerate the keys of the QJsonObject - foreach(const QString& key, jsonObject.keys()) { - QJsonValue childValue = jsonObject[key]; - - // setup the keypath for this key - QString valueKeypath = (keypath.isEmpty() ? "" : keypath + ".") + key; - - if (childValue.isObject() || childValue.isArray()) { - // recursion is required since the value is a QJsonObject or QJsonArray - result << toStringList(childValue, valueKeypath); - } else { - // no recursion required, call our toString method to get the string representation - // append the QStringList resulting from that to our QStringList - result << toString(childValue, valueKeypath); - } - } - } else if (jsonValue.isArray()) { - QJsonArray jsonArray = jsonValue.toArray(); - - // enumerate the elements in this QJsonArray - for (int i = 0; i < jsonArray.size(); i++) { - QJsonValue arrayValue = jsonArray[i]; - - // setup the keypath for this object with the array index - QString valueKeypath = QString("%1[%2]").arg(keypath).arg(i); - - if (arrayValue.isObject() || arrayValue.isArray()) { - // recursion is required since the value is a QJsonObject or QJsonArray - // append the QStringList resulting from that to our QStringList - result << toStringList(arrayValue, valueKeypath); - } else { - result << toString(arrayValue, valueKeypath); - } - } - } else { - // this is a basic value, so set result to whatever toString reports in a QStringList - result = QStringList() << toString(jsonValue, keypath); - } - - return result; +QByteArray JSONBreakableMarshal::toByteArray(const QJsonObject& jsonObject) { + return QJsonDocument(jsonObject).toBinaryData(); } -const QString JSON_NULL_AS_STRING = "null"; -const QString JSON_TRUE_AS_STRING = "true"; -const QString JSON_FALSE_AS_STRING = "false"; -const QString JSON_UNDEFINED_AS_STRING = "undefined"; -const QString JSON_UNKNOWN_AS_STRING = "unknown"; - -QString JSONBreakableMarshal::toString(const QJsonValue& jsonValue, const QString& keypath) { - // default the value as a string to unknown in case conversion fails - QString valueAsString = JSON_UNKNOWN_AS_STRING; - - // ask the QJsonValue what type it is and format its value as a string accordingly - if (jsonValue.isNull()) { - valueAsString = JSON_NULL_AS_STRING; - } else if (jsonValue.isBool()) { - valueAsString = jsonValue.toBool() ? JSON_TRUE_AS_STRING : JSON_FALSE_AS_STRING; - } else if (jsonValue.isDouble()) { - valueAsString = QString::number(jsonValue.toDouble()); - } else if (jsonValue.isString()) { - valueAsString = QString("\"%1\"").arg(jsonValue.toString()); - } else if (jsonValue.isUndefined()) { - valueAsString = JSON_UNDEFINED_AS_STRING; - } else if (jsonValue.isArray() || jsonValue.isObject()) { - qDebug() << "JSONBreakableMarshal::toString does not handle conversion of a QJsonObject or QJsonArray." - << "You should call JSONBreakableMarshal::toStringList instead."; - } else { - qDebug() << "Unrecognized QJsonValue - JSONBreakableMarshal cannot convert to string."; +QJsonObject JSONBreakableMarshal::fromByteArray(const QByteArray& buffer) { + auto document = QJsonDocument::fromBinaryData(buffer); + Q_ASSERT(document.isObject()); + auto object = document.object(); + + QStringList currentKey; + for (auto key : object.keys()) { + interpolate(object[key], key); } - - return QString("%1=%2").arg(keypath, valueAsString); -} - -QVariant JSONBreakableMarshal::fromString(const QString& marshalValue) { - // default the value to null - QVariant result; - - // attempt to match the value with our expected strings - if (marshalValue == JSON_NULL_AS_STRING) { - // this is already our default, we don't need to do anything here - } else if (marshalValue == JSON_TRUE_AS_STRING || marshalValue == JSON_FALSE_AS_STRING) { - result = QVariant(marshalValue == JSON_TRUE_AS_STRING ? true : false); - } else if (marshalValue == JSON_UNDEFINED_AS_STRING) { - result = JSON_UNDEFINED_AS_STRING; - } else if (marshalValue == JSON_UNKNOWN_AS_STRING) { - // we weren't able to marshal this value at the other end, set it as our unknown string - result = JSON_UNKNOWN_AS_STRING; - } else { - // this might be a double, see if it converts - bool didConvert = false; - double convertResult = marshalValue.toDouble(&didConvert); - - if (didConvert) { - result = convertResult; - } else { - // we need to figure out if this is a string - // use a regex to look for surrounding quotes first - const QString JSON_STRING_REGEX = "^\"([\\s\\S]*)\"$"; - QRegExp stringRegex(JSON_STRING_REGEX); - - if (stringRegex.indexIn(marshalValue) != -1) { - // set the result to the string value - result = stringRegex.cap(1); - } else { - // we failed to convert the value to anything, set the result to our unknown value - qDebug() << "Unrecognized output from JSONBreakableMarshal - could not convert" - << marshalValue << "to QVariant."; - result = JSON_UNKNOWN_AS_STRING; - } - } - } - - return result; -} - -QVariantMap JSONBreakableMarshal::fromStringList(const QStringList& stringList) { - QVariant result = QVariantMap(); - - foreach(const QString& marshalString, stringList) { - - // find the equality operator - int equalityIndex = marshalString.indexOf('='); - - // bail on parsing if we didn't find the equality operator - if (equalityIndex != -1) { - - QVariant* currentValue = &result; - - // pull the key (everything left of the equality sign) - QString parentKeypath; - QString keypath = marshalString.left(equalityIndex); - - // setup for array index checking - const QString ARRAY_INDEX_REGEX_STRING = "\\[(\\d+)\\]"; - QRegExp arrayRegex(ARRAY_INDEX_REGEX_STRING); - - // as long as we have a keypath we need to recurse downwards - while (!keypath.isEmpty()) { - parentKeypath = keypath; - - int arrayBracketIndex = arrayRegex.indexIn(keypath); - - // is this an array index - if (arrayBracketIndex == 0) { - // we're here because we think the current value should be an array - // if it isn't then make one - if (!currentValue->canConvert(QMetaType::QVariantList) || currentValue->isNull()) { - *currentValue = QVariantList(); - } - - QVariantList& currentList = *static_cast(currentValue->data()); - - // figure out what index we want to get the QJsonValue& for - bool didConvert = false; - int arrayIndex = arrayRegex.cap(1).toInt(&didConvert); - - if (didConvert) { - // check if we need to resize the array - if (currentList.size() < arrayIndex + 1) { - - for (int i = currentList.size(); i < arrayIndex + 1; i++) { - // add the null QJsonValue at this array index to get the array to the right size - currentList.push_back(QJsonValue()); - } - } - - // set our currentValue to the QJsonValue& from the array at this index - currentValue = ¤tList[arrayIndex]; - - // update the keypath by bumping past the array index - keypath = keypath.mid(keypath.indexOf(']') + 1); - - // check if there is a key after the array index - if so push the keypath forward by a char - if (keypath.startsWith(".")) { - keypath = keypath.mid(1); - } - } else { - qDebug() << "Failed to convert array index from keypath" << keypath << "to int. Will not add" - << "value to resulting QJsonObject."; - break; - } - - } else { - int keySeparatorIndex = keypath.indexOf('.'); - - // we need to figure out what the key to look at is - QString subKey = keypath; - - if (keySeparatorIndex != -1 || arrayBracketIndex != -1) { - int nextBreakIndex = -1; - int nextKeypathStartIndex = -1; - - if (arrayBracketIndex == -1 || (keySeparatorIndex != -1 && keySeparatorIndex < arrayBracketIndex)) { - nextBreakIndex = keySeparatorIndex; - nextKeypathStartIndex = keySeparatorIndex + 1; - } else if (keySeparatorIndex == -1 || (arrayBracketIndex != -1 - && arrayBracketIndex < keySeparatorIndex)) { - nextBreakIndex = arrayBracketIndex; - nextKeypathStartIndex = arrayBracketIndex; - } else { - qDebug() << "Unrecognized key format while trying to parse " << keypath << " - will not add" - << "value to resulting QJsonObject."; - break; - } - - // set the current key from the determined index - subKey = keypath.left(nextBreakIndex); - - // update the keypath being processed - keypath = keypath.mid(nextKeypathStartIndex); - - } else { - // update the keypath being processed, since we have no more separators in the keypath, it should - // be an empty string - keypath = ""; - } - - // we're here becuase we know the current value should be an object - // if it isn't then make it one - - if (!currentValue->canConvert(QMetaType::QVariantMap) || currentValue->isNull()) { - *currentValue = QVariantMap(); - } - - QVariantMap& currentMap = *static_cast(currentValue->data()); - - // is there a QJsonObject for this key yet? - // if not then we make it now - if (!currentMap.contains(subKey)) { - currentMap[subKey] = QVariant(); - } - - // change the currentValue to the QJsonValue for this key - currentValue = ¤tMap[subKey]; - } - } - - *currentValue = fromString(marshalString.mid(equalityIndex + 1)); - - if (_interpolationMap.contains(parentKeypath)) { - // we expect the currentValue here to be a string, that's the key we use for interpolation - // bail if it isn't - if (currentValue->canConvert(QMetaType::QString) - && _interpolationMap[parentKeypath].canConvert(QMetaType::QVariantMap)) { - *currentValue = _interpolationMap[parentKeypath].toMap()[currentValue->toString()]; - } - } - } - } - - return result.toMap(); -} - -QVariantMap JSONBreakableMarshal::fromStringBuffer(const QByteArray& buffer) { - // this is a packet of strings sep by null terminators - pull out each string and create a stringlist - QStringList packetList; - int currentIndex = 0; - int currentSeparator = buffer.indexOf('\0'); - - while (currentIndex < buffer.size() - 1) { - packetList << QString::fromUtf8(buffer.mid(currentIndex, currentSeparator)); - - if (currentSeparator == -1) { - // no more separators to be found, break out of here so we're not looping for nothing - break; - } - - // bump the currentIndex up to the last found separator - currentIndex = currentSeparator + 1; - - // find the index of the next separator, assuming this one wasn't the last one in the packet - if (currentSeparator < buffer.size() - 1) { - currentSeparator = buffer.indexOf('\0', currentIndex); - } - } - - // now that we have a QStringList we use our static method to turn that into a QJsonObject - return fromStringList(packetList); + + return object; } void JSONBreakableMarshal::addInterpolationForKey(const QString& rootKey, const QString& interpolationKey, - const QJsonValue& interpolationValue) { + const QString& interpolationValue) { // if there is no map already beneath this key in our _interpolationMap create a QVariantMap there now - - if (!_interpolationMap.contains(rootKey)) { - _interpolationMap.insert(rootKey, QVariantMap()); - } - - if (_interpolationMap[rootKey].canConvert(QMetaType::QVariantMap)) { - QVariantMap& mapForRootKey = *static_cast(_interpolationMap[rootKey].data()); - - mapForRootKey.insert(interpolationKey, QVariant(interpolationValue)); - } else { - qDebug() << "JSONBreakableMarshal::addInterpolationForKey could not convert variant at key" << rootKey - << "to a QVariantMap. Can not add interpolation."; + auto it = _interpolationMap.find(rootKey); + if (it == _interpolationMap.end()) { + it = _interpolationMap.insert(rootKey, QVariantMap()); } + Q_ASSERT(it->canConvert(QMetaType::QVariantMap)); + static_cast(it->data())->insert(interpolationKey, interpolationValue); } void JSONBreakableMarshal::removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey) { // make sure the interpolation map contains this root key and that the value is a map + auto it = _interpolationMap.find(rootKey); + if (it != _interpolationMap.end() && !it->isNull()) { + // remove the value at the interpolationKey + Q_ASSERT(it->canConvert(QMetaType::QVariantMap)); + static_cast(it->data())->remove(interpolationKey); + } +} - if (_interpolationMap.contains(rootKey)) { - QVariant& rootValue = _interpolationMap[rootKey]; - - if (!rootValue.isNull() && rootValue.canConvert(QMetaType::QVariantMap)) { - // remove the value at the interpolationKey - static_cast(rootValue.data())->remove(interpolationKey); +void JSONBreakableMarshal::interpolate(QJsonValueRef currentValue, QString lastKey) { + if (currentValue.isArray()) { + auto array = currentValue.toArray(); + for (int i = 0; i < array.size(); ++i) { + // pass last key and recurse array + interpolate(array[i], QString::number(i)); + } + currentValue = array; + } else if (currentValue.isObject()) { + auto object = currentValue.toObject(); + for (auto key : object.keys()) { + // pass last key and recurse object + interpolate(object[key], key); + } + currentValue = object; + } else if (currentValue.isString()) { + // Maybe interpolate + auto mapIt = _interpolationMap.find(lastKey); + if (mapIt != _interpolationMap.end()) { + Q_ASSERT(mapIt->canConvert(QMetaType::QVariantMap)); + auto interpolationMap = mapIt->toMap(); + + auto result = interpolationMap.find(currentValue.toString()); + if (result != interpolationMap.end()) { + // Replace value + currentValue = result->toString(); + } } } } diff --git a/libraries/networking/src/JSONBreakableMarshal.h b/libraries/networking/src/JSONBreakableMarshal.h index 287fed675a..28b7c205f0 100644 --- a/libraries/networking/src/JSONBreakableMarshal.h +++ b/libraries/networking/src/JSONBreakableMarshal.h @@ -21,18 +21,16 @@ class JSONBreakableMarshal { public: - static QStringList toStringList(const QJsonValue& jsonValue, const QString& keypath); - static QString toString(const QJsonValue& jsonValue, const QString& keyPath); - - static QVariant fromString(const QString& marshalValue); - static QVariantMap fromStringList(const QStringList& stringList); - static QVariantMap fromStringBuffer(const QByteArray& buffer); + static QByteArray toByteArray(const QJsonObject& jsonObject); + static QJsonObject fromByteArray(const QByteArray& buffer); - static void addInterpolationForKey(const QString& rootKey, - const QString& interpolationKey, const QJsonValue& interpolationValue); + static void addInterpolationForKey(const QString& rootKey, const QString& interpolationKey, + const QString& interpolationValue); static void removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey); private: + static void interpolate(QJsonValueRef currentValue, QString lastKey); + static QVariantMap _interpolationMap; }; diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index e91e8a7c7e..763bad677f 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -845,6 +845,13 @@ void LimitedNodeList::sendPeerQueryToIceServer(const HifiSockAddr& iceServerSock sendPacketToIceServer(PacketType::ICEServerQuery, iceServerSockAddr, clientID, peerID); } +SharedNodePointer LimitedNodeList::findNodeWithAddr(const HifiSockAddr& addr) { + auto it = std::find_if(std::begin(_nodeHash), std::end(_nodeHash), [&](const UUIDNodePair& pair) { + return pair.second->getActiveSocket() ? (*pair.second->getActiveSocket() == addr) : false; + }); + return (it != std::end(_nodeHash)) ? it->second : SharedNodePointer(); +} + void LimitedNodeList::sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerID) { auto icePacket = NLPacket::create(packetType); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 9d4e426d47..719387ace1 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -165,6 +165,8 @@ public: void sendHeartbeatToIceServer(const HifiSockAddr& iceServerSockAddr); void sendPeerQueryToIceServer(const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerID); + SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr); + template void eachNode(NodeLambda functor) { QReadLocker readLock(&_nodeMutex); @@ -216,6 +218,7 @@ public: { QReadLocker readLock(&_connectionTimeLock); return _lastConnectionTimes; } void flagTimeForConnectionStep(ConnectionStep connectionStep); + udt::Socket::StatsVector sampleStatsForAllConnections() { return _nodeSocket.sampleStatsForAllConnections(); } public slots: void reset(); diff --git a/libraries/networking/src/udt/ConnectionStats.cpp b/libraries/networking/src/udt/ConnectionStats.cpp index 52188c29c3..4cc9b17e64 100644 --- a/libraries/networking/src/udt/ConnectionStats.cpp +++ b/libraries/networking/src/udt/ConnectionStats.cpp @@ -15,7 +15,7 @@ using namespace udt; using namespace std::chrono; ConnectionStats::ConnectionStats() { - auto now = duration_cast(high_resolution_clock::now().time_since_epoch()); + auto now = duration_cast(system_clock::now().time_since_epoch()); _currentSample.startTime = now; _total.startTime = now; } @@ -24,7 +24,7 @@ ConnectionStats::Stats ConnectionStats::sample() { Stats sample = _currentSample; _currentSample = Stats(); - auto now = duration_cast(high_resolution_clock::now().time_since_epoch()); + auto now = duration_cast(system_clock::now().time_since_epoch()); sample.endTime = now; _currentSample.startTime = now; diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index 494d433bca..2adc422a47 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -36,11 +36,13 @@ public: SentTimeoutNAK, ReceivedTimeoutNAK, Retransmission, - Duplicate + Duplicate, + + NumEvent }; // construct a vector for the events of the size of our Enum - default value is zero - std::vector events = std::vector((int) Event::Duplicate + 1, 0); + std::vector events = std::vector((int) Event::NumEvent, 0); // packet counts and sizes int sentPackets { 0 }; diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 37bc88ac49..a0aa767de1 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -325,6 +325,16 @@ ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& dest } } +Socket::StatsVector Socket::sampleStatsForAllConnections() { + StatsVector result; + result.reserve(_connectionsHash.size()); + for (const auto& connectionPair : _connectionsHash) { + result.emplace_back(connectionPair.first, connectionPair.second->sampleStats()); + } + return result; +} + + std::vector Socket::getConnectionSockAddrs() { std::vector addr; addr.reserve(_connectionsHash.size()); diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 923248caad..457d3feffc 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -46,6 +46,8 @@ using PacketListHandler = std::function)>; class Socket : public QObject { Q_OBJECT public: + using StatsVector = std::vector>; + Socket(QObject* object = 0); quint16 localPort() const { return _udpSocket.localPort(); } @@ -71,6 +73,8 @@ public: void setCongestionControlFactory(std::unique_ptr ccFactory); void messageReceived(std::unique_ptr packetList); + + StatsVector sampleStatsForAllConnections(); public slots: void cleanupConnection(HifiSockAddr sockAddr); @@ -86,6 +90,7 @@ private: // privatized methods used by UDTTest - they are private since they must be called on the Socket thread ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination); + std::vector getConnectionSockAddrs(); void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot); From 801f32295d22783bbc89c19fdd3fe225cdea2c0e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Sep 2015 17:38:31 +0200 Subject: [PATCH 03/10] Update JSON stats building --- assignment-client/src/assets/AssetServer.cpp | 77 ++++----- assignment-client/src/octree/OctreeServer.cpp | 146 ++++++++++-------- 2 files changed, 119 insertions(+), 104 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index bf93755bf8..c7732a64f9 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -164,53 +164,58 @@ void AssetServer::handleAssetUpload(QSharedPointer packetList, Sha } void AssetServer::sendStatsPacket() { - QJsonObject statsObject; + QJsonObject serverStats; auto stats = DependencyManager::get()->sampleStatsForAllConnections(); - QString baseName("AssetServer"); - - statsObject[baseName + ".num_connections"] = (int)stats.size(); for (const auto& stat : stats) { - QString uuid = QUuid().toString(); - if (auto node = DependencyManager::get()->findNodeWithAddr(stat.first)) { - uuid = node->getUUID().toString(); - } - - QString connKey = baseName + "." + uuid + ".connection."; - QString sendKey = baseName + "." + uuid + ".sending."; - QString recvKey = baseName + "." + uuid + ".receiving."; - qDebug() << "Testing" << sendKey << recvKey; - + QJsonObject nodeStats; auto endTimeMs = std::chrono::duration_cast(stat.second.endTime); QDateTime date = QDateTime::fromMSecsSinceEpoch(endTimeMs.count()); - statsObject[connKey + "lastHeard"] = date.toString(); - statsObject[connKey + "estimatedBandwith"] = stat.second.estimatedBandwith; - statsObject[connKey + "rtt"] = stat.second.rtt; - statsObject[connKey + "congestionWindowSize"] = stat.second.congestionWindowSize; - statsObject[connKey + "packetSendPeriod"] = stat.second.packetSendPeriod; + QString uuid = QUuid().toString(); + if (auto node = DependencyManager::get()->findNodeWithAddr(stat.first)) { + uuid = uuidStringWithoutCurlyBraces(node->getUUID().toString()); + } - statsObject[sendKey + "sendRate"] = stat.second.sendRate; - statsObject[sendKey + "sentPackets"] = stat.second.sentPackets; - statsObject[sendKey + "receivedACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK]; - statsObject[sendKey + "processedACK"] = stat.second.events[udt::ConnectionStats::Stats::ProcessedACK]; - statsObject[sendKey + "receivedLightACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedLightACK]; - statsObject[sendKey + "receivedNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedNAK]; - statsObject[sendKey + "receivedTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]; - statsObject[sendKey + "sentACK2"] = stat.second.events[udt::ConnectionStats::Stats::SentACK2]; - statsObject[sendKey + "retransmission"] = stat.second.events[udt::ConnectionStats::Stats::Retransmission]; + nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuid; - statsObject[recvKey + "receiveRate"] = stat.second.receiveRate; - statsObject[recvKey + "receivedPackets"] = stat.second.receivedPackets; - statsObject[recvKey + "SentACK"] = stat.second.events[udt::ConnectionStats::Stats::SentACK]; - statsObject[recvKey + "SentLightACK"] = stat.second.events[udt::ConnectionStats::Stats::SentLightACK]; - statsObject[recvKey + "SentNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentNAK]; - statsObject[recvKey + "SentTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentTimeoutNAK]; - statsObject[recvKey + "ReceivedACK2"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK2]; - statsObject[recvKey + "Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate]; + QJsonObject connectionStats; + connectionStats["lastHeard"] = date.toString(); + connectionStats["estimatedBandwith"] = stat.second.estimatedBandwith; + connectionStats["rtt"] = stat.second.rtt; + connectionStats["congestionWindowSize"] = stat.second.congestionWindowSize; + connectionStats["packetSendPeriod"] = stat.second.packetSendPeriod; + nodeStats["connection"] = connectionStats; + + QJsonObject sendingStats; + sendingStats["sendRate"] = stat.second.sendRate; + sendingStats["sentPackets"] = stat.second.sentPackets; + sendingStats["receivedACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK]; + sendingStats["processedACK"] = stat.second.events[udt::ConnectionStats::Stats::ProcessedACK]; + sendingStats["receivedLightACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedLightACK]; + sendingStats["receivedNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedNAK]; + sendingStats["receivedTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK]; + sendingStats["sentACK2"] = stat.second.events[udt::ConnectionStats::Stats::SentACK2]; + sendingStats["retransmission"] = stat.second.events[udt::ConnectionStats::Stats::Retransmission]; + nodeStats["sending"] = sendingStats; + + QJsonObject receivingStats; + receivingStats["receiveRate"] = stat.second.receiveRate; + receivingStats["receivedPackets"] = stat.second.receivedPackets; + receivingStats["SentACK"] = stat.second.events[udt::ConnectionStats::Stats::SentACK]; + receivingStats["SentLightACK"] = stat.second.events[udt::ConnectionStats::Stats::SentLightACK]; + receivingStats["SentNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentNAK]; + receivingStats["SentTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentTimeoutNAK]; + receivingStats["ReceivedACK2"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK2]; + receivingStats["Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate]; + nodeStats["receiving"] = receivingStats; + + serverStats[uuid] = nodeStats; } // send off the stats packets + QJsonObject statsObject; + statsObject["AssetServer"] = serverStats; ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); } diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index a0350622db..e02690a367 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -1298,76 +1298,86 @@ void OctreeServer::sendStatsPacket() { // the following features: // 1) remember last state sent // 2) only send new data - // 3) automatically break up into multiple packets - static QJsonObject statsObject1; - - QString baseName = getMyServerName() + QString("Server"); - - statsObject1[baseName + QString(".0.1.configuration")] = getConfiguration(); - - statsObject1[baseName + QString(".0.2.detailed_stats_url")] = getStatusLink(); - - statsObject1[baseName + QString(".0.3.uptime")] = getUptime(); - statsObject1[baseName + QString(".0.4.persistFileLoadTime")] = getFileLoadTime(); - statsObject1[baseName + QString(".0.5.clients")] = getCurrentClientCount(); - + + // Stats Array 1 + QJsonArray threadsStats; quint64 oneSecondAgo = usecTimestampNow() - USECS_PER_SECOND; + threadsStats.push_back(QJsonObject({{"processing", (double)howManyThreadsDidProcess(oneSecondAgo)}})); + threadsStats.push_back(QJsonObject({{"packetDistributor", (double)howManyThreadsDidPacketDistributor(oneSecondAgo)}})); + threadsStats.push_back(QJsonObject({{"handlePacektSend", (double)howManyThreadsDidHandlePacketSend(oneSecondAgo)}})); + threadsStats.push_back(QJsonObject({{"writeDatagram", (double)howManyThreadsDidCallWriteDatagram(oneSecondAgo)}})); + + QJsonArray statsArray1; + statsArray1.push_back(QJsonObject({{"configuration", getConfiguration()}})); + statsArray1.push_back(QJsonObject({{"detailed_stats_url", getStatusLink()}})); + statsArray1.push_back(QJsonObject({{"uptime", getUptime()}})); + statsArray1.push_back(QJsonObject({{"persistFileLoadTime", getFileLoadTime()}})); + statsArray1.push_back(QJsonObject({{"clients", getCurrentClientCount()}})); + statsArray1.push_back(QJsonObject({{"threads", threadsStats}})); + + // Octree Stats + QJsonArray octreeStats; + octreeStats.push_back(QJsonObject({{"elementCount", (double)OctreeElement::getNodeCount()}})); + octreeStats.push_back(QJsonObject({{"internalElementCount", (double)OctreeElement::getInternalNodeCount()}})); + octreeStats.push_back(QJsonObject({{"leafElementCount", (double)OctreeElement::getLeafNodeCount()}})); + + // Stats Object 2 + QJsonObject dataObject1; + dataObject1["totalPackets"] = (double)OctreeSendThread::_totalPackets; + dataObject1["totalBytes"] = (double)OctreeSendThread::_totalBytes; + dataObject1["totalBytesWasted"] = (double)OctreeSendThread::_totalWastedBytes; + dataObject1["totalBytesOctalCodes"] = (double)OctreePacketData::getTotalBytesOfOctalCodes(); + dataObject1["totalBytesBitMasks"] = (double)OctreePacketData::getTotalBytesOfBitMasks(); + dataObject1["totalBytesBitMasks"] = (double)OctreePacketData::getTotalBytesOfColor(); - statsObject1[baseName + QString(".0.6.threads.1.processing")] = (double)howManyThreadsDidProcess(oneSecondAgo); - statsObject1[baseName + QString(".0.6.threads.2.packetDistributor")] = - (double)howManyThreadsDidPacketDistributor(oneSecondAgo); - statsObject1[baseName + QString(".0.6.threads.3.handlePacektSend")] = - (double)howManyThreadsDidHandlePacketSend(oneSecondAgo); - statsObject1[baseName + QString(".0.6.threads.4.writeDatagram")] = - (double)howManyThreadsDidCallWriteDatagram(oneSecondAgo); - - statsObject1[baseName + QString(".1.1.octree.elementCount")] = (double)OctreeElement::getNodeCount(); - statsObject1[baseName + QString(".1.2.octree.internalElementCount")] = (double)OctreeElement::getInternalNodeCount(); - statsObject1[baseName + QString(".1.3.octree.leafElementCount")] = (double)OctreeElement::getLeafNodeCount(); - - ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject1); - - static QJsonObject statsObject2; - - statsObject2[baseName + QString(".2.outbound.data.totalPackets")] = (double)OctreeSendThread::_totalPackets; - statsObject2[baseName + QString(".2.outbound.data.totalBytes")] = (double)OctreeSendThread::_totalBytes; - statsObject2[baseName + QString(".2.outbound.data.totalBytesWasted")] = (double)OctreeSendThread::_totalWastedBytes; - statsObject2[baseName + QString(".2.outbound.data.totalBytesOctalCodes")] = - (double)OctreePacketData::getTotalBytesOfOctalCodes(); - statsObject2[baseName + QString(".2.outbound.data.totalBytesBitMasks")] = - (double)OctreePacketData::getTotalBytesOfBitMasks(); - statsObject2[baseName + QString(".2.outbound.data.totalBytesBitMasks")] = (double)OctreePacketData::getTotalBytesOfColor(); - - statsObject2[baseName + QString(".2.outbound.timing.1.avgLoopTime")] = getAverageLoopTime(); - statsObject2[baseName + QString(".2.outbound.timing.2.avgInsideTime")] = getAverageInsideTime(); - statsObject2[baseName + QString(".2.outbound.timing.3.avgTreeLockTime")] = getAverageTreeWaitTime(); - statsObject2[baseName + QString(".2.outbound.timing.4.avgEncodeTime")] = getAverageEncodeTime(); - statsObject2[baseName + QString(".2.outbound.timing.5.avgCompressAndWriteTime")] = getAverageCompressAndWriteTime(); - statsObject2[baseName + QString(".2.outbound.timing.5.avgSendTime")] = getAveragePacketSendingTime(); - statsObject2[baseName + QString(".2.outbound.timing.5.nodeWaitTime")] = getAverageNodeWaitTime(); - - DependencyManager::get()->sendStatsToDomainServer(statsObject2); - - static QJsonObject statsObject3; - - statsObject3[baseName + QString(".3.inbound.data.1.packetQueue")] = - (double)_octreeInboundPacketProcessor->packetsToProcessCount(); - statsObject3[baseName + QString(".3.inbound.data.1.totalPackets")] = - (double)_octreeInboundPacketProcessor->getTotalPacketsProcessed(); - statsObject3[baseName + QString(".3.inbound.data.2.totalElements")] = - (double)_octreeInboundPacketProcessor->getTotalElementsProcessed(); - statsObject3[baseName + QString(".3.inbound.timing.1.avgTransitTimePerPacket")] = - (double)_octreeInboundPacketProcessor->getAverageTransitTimePerPacket(); - statsObject3[baseName + QString(".3.inbound.timing.2.avgProcessTimePerPacket")] = - (double)_octreeInboundPacketProcessor->getAverageProcessTimePerPacket(); - statsObject3[baseName + QString(".3.inbound.timing.3.avgLockWaitTimePerPacket")] = - (double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerPacket(); - statsObject3[baseName + QString(".3.inbound.timing.4.avgProcessTimePerElement")] = - (double)_octreeInboundPacketProcessor->getAverageProcessTimePerElement(); - statsObject3[baseName + QString(".3.inbound.timing.5.avgLockWaitTimePerElement")] = - (double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerElement(); - - DependencyManager::get()->sendStatsToDomainServer(statsObject3); + QJsonArray timingArray1; + timingArray1.push_back(QJsonObject({{"avgLoopTime", getAverageLoopTime()}})); + timingArray1.push_back(QJsonObject({{"avgInsideTime", getAverageInsideTime()}})); + timingArray1.push_back(QJsonObject({{"avgTreeLockTime", getAverageTreeWaitTime()}})); + timingArray1.push_back(QJsonObject({{"avgEncodeTime", getAverageEncodeTime()}})); + timingArray1.push_back(QJsonObject({{"avgCompressAndWriteTime", getAverageCompressAndWriteTime()}})); + timingArray1.push_back(QJsonObject({{"avgSendTime", getAveragePacketSendingTime()}})); + timingArray1.push_back(QJsonObject({{"nodeWaitTime", getAverageNodeWaitTime()}})); + + QJsonObject statsObject2; + statsObject2["data"] = dataObject1; + statsObject2["timing"] = timingArray1; + + // Stats Object 3 + QJsonArray dataArray2; + dataArray2.push_back(QJsonObject({{"packetQueue", + (double)_octreeInboundPacketProcessor->packetsToProcessCount()}})); + dataArray2.push_back(QJsonObject({{"totalPackets", + (double)_octreeInboundPacketProcessor->getTotalPacketsProcessed()}})); + dataArray2.push_back(QJsonObject({{"totalElements", + (double)_octreeInboundPacketProcessor->getTotalElementsProcessed()}})); + + QJsonArray timingArray2; + timingArray2.push_back(QJsonObject({{"avgTransitTimePerPacket", + (double)_octreeInboundPacketProcessor->getAverageTransitTimePerPacket()}})); + timingArray2.push_back(QJsonObject({{"avgProcessTimePerPacket", + (double)_octreeInboundPacketProcessor->getAverageProcessTimePerPacket()}})); + timingArray2.push_back(QJsonObject({{"avgLockWaitTimePerPacket", + (double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerPacket()}})); + timingArray2.push_back(QJsonObject({{"avgProcessTimePerElement", + (double)_octreeInboundPacketProcessor->getAverageProcessTimePerElement()}})); + timingArray2.push_back(QJsonObject({{"avgLockWaitTimePerElement", + (double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerElement()}})); + + QJsonObject statsObject3; + statsObject3["data"] = dataArray2; + statsObject3["timing"] = timingArray2; + + // Merge everything + QJsonArray jsonArray; + jsonArray.push_back(statsArray1); + jsonArray.push_back(QJsonObject({{"octree", octreeStats}})); + jsonArray.push_back(QJsonObject({{"outbound", statsObject2}})); + jsonArray.push_back(QJsonObject({{"inbound", statsObject3}})); + + QJsonObject statsObject; + statsObject[QString(getMyServerName()) + "Server"] = jsonArray; + addPacketStatsAndSendStatsPacket(statsObject); } QMap OctreeServer::_threadsDidProcess; From 15f7745bcc25c22d490dae906136b543c6b31d31 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Sep 2015 23:43:58 +0200 Subject: [PATCH 04/10] JSON stats reliable and ordered --- assignment-client/src/octree/OctreeServer.cpp | 6 -- domain-server/src/DomainGatekeeper.cpp | 7 +- domain-server/src/DomainServer.cpp | 9 +- domain-server/src/DomainServerNodeData.cpp | 74 +++++++++++----- domain-server/src/DomainServerNodeData.h | 15 +++- .../networking/src/JSONBreakableMarshal.cpp | 88 ------------------- .../networking/src/JSONBreakableMarshal.h | 37 -------- libraries/networking/src/NodeList.cpp | 4 +- tools/udt-test/src/UDTTest.cpp | 4 +- 9 files changed, 74 insertions(+), 170 deletions(-) delete mode 100644 libraries/networking/src/JSONBreakableMarshal.cpp delete mode 100644 libraries/networking/src/JSONBreakableMarshal.h diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index e02690a367..e0d0f9eeac 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -1293,12 +1293,6 @@ QString OctreeServer::getStatusLink() { } void OctreeServer::sendStatsPacket() { - // TODO: we have too many stats to fit in a single MTU... so for now, we break it into multiple JSON objects and - // send them separately. What we really should do is change the NodeList::sendStatsToDomainServer() to handle the - // the following features: - // 1) remember last state sent - // 2) only send new data - // Stats Array 1 QJsonArray threadsStats; quint64 oneSecondAgo = usecTimestampNow() - USECS_PER_SECOND; diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index ae827f3a4a..90b22ffdd8 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -17,7 +17,6 @@ #include #include -#include #include "DomainServer.h" #include "DomainServerNodeData.h" @@ -272,9 +271,9 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect // if we have a username from the connect request, set it on the DomainServerNodeData nodeData->setUsername(username); - // also add an interpolation to JSONBreakableMarshal so that servers can get username in stats - JSONBreakableMarshal::addInterpolationForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY, - uuidStringWithoutCurlyBraces(newNode->getUUID()), username); + // also add an interpolation to DomainServerNodeData so that servers can get username in stats + nodeData->addOverrideForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY, + uuidStringWithoutCurlyBraces(newNode->getUUID()), username); return newNode; } diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 2f5e553c9f..49b7f2e183 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -1010,7 +1009,7 @@ void DomainServer::sendHeartbeatToIceServer() { void DomainServer::processNodeJSONStatsPacket(QSharedPointer packetList, SharedNodePointer sendingNode) { auto nodeData = dynamic_cast(sendingNode->getLinkedData()); if (nodeData) { - nodeData->processJSONStatsPacket(packetList->getMessage()); + nodeData->updateJSONStats(packetList->getMessage()); } } @@ -1676,9 +1675,9 @@ void DomainServer::nodeKilled(SharedNodePointer node) { } } - // If this node was an Agent ask JSONBreakableMarshal to potentially remove the interpolation we stored - JSONBreakableMarshal::removeInterpolationForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY, - uuidStringWithoutCurlyBraces(node->getUUID())); + // If this node was an Agent ask DomainServerNodeData to potentially remove the interpolation we stored + nodeData->removeOverrideForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY, + uuidStringWithoutCurlyBraces(node->getUUID())); // cleanup the connection secrets that we set up for this node (on the other nodes) foreach (const QUuid& otherNodeSessionUUID, nodeData->getSessionSecretHash().keys()) { diff --git a/domain-server/src/DomainServerNodeData.cpp b/domain-server/src/DomainServerNodeData.cpp index ecbe6bd36e..9b86697d2f 100644 --- a/domain-server/src/DomainServerNodeData.cpp +++ b/domain-server/src/DomainServerNodeData.cpp @@ -10,40 +10,72 @@ // #include +#include +#include #include #include -#include #include #include "DomainServerNodeData.h" -DomainServerNodeData::DomainServerNodeData() : - _sessionSecretHash(), - _assignmentUUID(), - _walletUUID(), - _username(), - _paymentIntervalTimer(), - _statsJSONObject(), - _sendingSockAddr(), - _isAuthenticated(true) -{ +QHash, QString> DomainServerNodeData::_overrideHash; + +DomainServerNodeData::DomainServerNodeData() { _paymentIntervalTimer.start(); } -void DomainServerNodeData::processJSONStatsPacket(QByteArray statsByteArray) { - QJsonObject packetJsonObject = JSONBreakableMarshal::fromByteArray(statsByteArray); - _statsJSONObject = mergeJSONStatsFromNewObject(packetJsonObject, _statsJSONObject); +void DomainServerNodeData::updateJSONStats(QByteArray statsByteArray) { + auto document = QJsonDocument::fromBinaryData(statsByteArray); + Q_ASSERT(document.isObject()); + _statsJSONObject = overrideValuesIfNeeded(document.object()); } -QJsonObject DomainServerNodeData::mergeJSONStatsFromNewObject(const QJsonObject& newObject, QJsonObject destinationObject) { - foreach(const QString& key, newObject.keys()) { - if (newObject[key].isObject() && destinationObject.contains(key)) { - destinationObject[key] = mergeJSONStatsFromNewObject(newObject[key].toObject(), destinationObject[key].toObject()); +QJsonObject DomainServerNodeData::overrideValuesIfNeeded(const QJsonObject& newStats) { + QJsonObject result; + for(auto it = newStats.constBegin(); it != newStats.constEnd(); ++it) { + const auto& key = it.key(); + const auto& value = it.value(); + + auto overrideIt = value.isString() ? _overrideHash.find({key, value.toString()}) : _overrideHash.end(); + if (overrideIt != _overrideHash.end()) { + // We have a match, override the value + result[key] = *overrideIt; + } else if (value.isObject()) { + result[key] = overrideValuesIfNeeded(value.toObject()); + } else if (value.isArray()) { + result[key] = overrideValuesIfNeeded(value.toArray()); } else { - destinationObject[key] = newObject[key]; + result[key] = newStats[key]; } } - - return destinationObject; + return result; +} + +QJsonArray DomainServerNodeData::overrideValuesIfNeeded(const QJsonArray& newStats) { + QJsonArray result; + for (int i = 0; i < newStats.size(); ++i) { + const auto& value = newStats[i]; + + if (value.isObject()) { + + result.push_back(overrideValuesIfNeeded(value.toObject())); + } else if (value.isArray()) { + result.push_back(overrideValuesIfNeeded(value.toArray())); + } else { + result.push_back(newStats[i]); + } + } + return result; +} + +void DomainServerNodeData::addOverrideForKey(const QString& key, const QString& value, + const QString& overrideValue) { + // Insert override value + _overrideHash.insert({key, value}, overrideValue); +} + +void DomainServerNodeData::removeOverrideForKey(const QString& key, const QString& value) { + // Remove override value + _overrideHash.remove({key, value}); } diff --git a/domain-server/src/DomainServerNodeData.h b/domain-server/src/DomainServerNodeData.h index 73003f1a7c..fcdc09e248 100644 --- a/domain-server/src/DomainServerNodeData.h +++ b/domain-server/src/DomainServerNodeData.h @@ -27,7 +27,7 @@ public: const QJsonObject& getStatsJSONObject() const { return _statsJSONObject; } - void processJSONStatsPacket(QByteArray statsByteArray); + void updateJSONStats(QByteArray statsByteArray); void setAssignmentUUID(const QUuid& assignmentUUID) { _assignmentUUID = assignmentUUID; } const QUuid& getAssignmentUUID() const { return _assignmentUUID; } @@ -54,17 +54,24 @@ public: void setNodeVersion(const QString& nodeVersion) { _nodeVersion = nodeVersion; } const QString& getNodeVersion() { return _nodeVersion; } + void addOverrideForKey(const QString& key, const QString& value, const QString& overrideValue); + void removeOverrideForKey(const QString& key, const QString& value); + private: - QJsonObject mergeJSONStatsFromNewObject(const QJsonObject& newObject, QJsonObject destinationObject); - + QJsonObject overrideValuesIfNeeded(const QJsonObject& newStats); + QJsonArray overrideValuesIfNeeded(const QJsonArray& newStats); + QHash _sessionSecretHash; QUuid _assignmentUUID; QUuid _walletUUID; QString _username; QElapsedTimer _paymentIntervalTimer; + QJsonObject _statsJSONObject; + static QHash, QString> _overrideHash; + HifiSockAddr _sendingSockAddr; - bool _isAuthenticated; + bool _isAuthenticated = true; NodeSet _nodeInterestSet; QString _nodeVersion; }; diff --git a/libraries/networking/src/JSONBreakableMarshal.cpp b/libraries/networking/src/JSONBreakableMarshal.cpp deleted file mode 100644 index f0a3ff54ef..0000000000 --- a/libraries/networking/src/JSONBreakableMarshal.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// JSONBreakableMarshal.cpp -// libraries/networking/src -// -// Created by Stephen Birarda on 04/28/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "JSONBreakableMarshal.h" - -#include -#include -#include -#include - -QVariantMap JSONBreakableMarshal::_interpolationMap = QVariantMap(); - -QByteArray JSONBreakableMarshal::toByteArray(const QJsonObject& jsonObject) { - return QJsonDocument(jsonObject).toBinaryData(); -} - -QJsonObject JSONBreakableMarshal::fromByteArray(const QByteArray& buffer) { - auto document = QJsonDocument::fromBinaryData(buffer); - Q_ASSERT(document.isObject()); - auto object = document.object(); - - QStringList currentKey; - for (auto key : object.keys()) { - interpolate(object[key], key); - } - - return object; -} - -void JSONBreakableMarshal::addInterpolationForKey(const QString& rootKey, const QString& interpolationKey, - const QString& interpolationValue) { - // if there is no map already beneath this key in our _interpolationMap create a QVariantMap there now - auto it = _interpolationMap.find(rootKey); - if (it == _interpolationMap.end()) { - it = _interpolationMap.insert(rootKey, QVariantMap()); - } - Q_ASSERT(it->canConvert(QMetaType::QVariantMap)); - static_cast(it->data())->insert(interpolationKey, interpolationValue); -} - -void JSONBreakableMarshal::removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey) { - // make sure the interpolation map contains this root key and that the value is a map - auto it = _interpolationMap.find(rootKey); - if (it != _interpolationMap.end() && !it->isNull()) { - // remove the value at the interpolationKey - Q_ASSERT(it->canConvert(QMetaType::QVariantMap)); - static_cast(it->data())->remove(interpolationKey); - } -} - -void JSONBreakableMarshal::interpolate(QJsonValueRef currentValue, QString lastKey) { - if (currentValue.isArray()) { - auto array = currentValue.toArray(); - for (int i = 0; i < array.size(); ++i) { - // pass last key and recurse array - interpolate(array[i], QString::number(i)); - } - currentValue = array; - } else if (currentValue.isObject()) { - auto object = currentValue.toObject(); - for (auto key : object.keys()) { - // pass last key and recurse object - interpolate(object[key], key); - } - currentValue = object; - } else if (currentValue.isString()) { - // Maybe interpolate - auto mapIt = _interpolationMap.find(lastKey); - if (mapIt != _interpolationMap.end()) { - Q_ASSERT(mapIt->canConvert(QMetaType::QVariantMap)); - auto interpolationMap = mapIt->toMap(); - - auto result = interpolationMap.find(currentValue.toString()); - if (result != interpolationMap.end()) { - // Replace value - currentValue = result->toString(); - } - } - } -} diff --git a/libraries/networking/src/JSONBreakableMarshal.h b/libraries/networking/src/JSONBreakableMarshal.h deleted file mode 100644 index 28b7c205f0..0000000000 --- a/libraries/networking/src/JSONBreakableMarshal.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// JSONBreakableMarshal.h -// libraries/networking/src -// -// Created by Stephen Birarda on 04/28/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_JSONBreakableMarshal_h -#define hifi_JSONBreakableMarshal_h - -#pragma once - -#include -#include -#include -#include - -class JSONBreakableMarshal { -public: - static QByteArray toByteArray(const QJsonObject& jsonObject); - static QJsonObject fromByteArray(const QByteArray& buffer); - - static void addInterpolationForKey(const QString& rootKey, const QString& interpolationKey, - const QString& interpolationValue); - static void removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey); - -private: - static void interpolate(QJsonValueRef currentValue, QString lastKey); - - static QVariantMap _interpolationMap; -}; - -#endif // hifi_JSONBreakableMarshal_h diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 6a27726a6b..b262904c63 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -27,7 +27,6 @@ #include "AddressManager.h" #include "Assignment.h" #include "HifiSockAddr.h" -#include "JSONBreakableMarshal.h" #include "NetworkLogging.h" #include "udt/PacketHeaders.h" @@ -109,7 +108,8 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned qint64 NodeList::sendStats(const QJsonObject& statsObject, const HifiSockAddr& destination) { auto statsPacketList = NLPacketList::create(PacketType::NodeJsonStats, QByteArray(), true, true); - statsPacketList->write(JSONBreakableMarshal::toByteArray(statsObject)); + QJsonDocument jsonDocument(statsObject); + statsPacketList->write(jsonDocument.toBinaryData()); sendPacketList(std::move(statsPacketList), destination); diff --git a/tools/udt-test/src/UDTTest.cpp b/tools/udt-test/src/UDTTest.cpp index 9a828ebafb..d57ff7a874 100644 --- a/tools/udt-test/src/UDTTest.cpp +++ b/tools/udt-test/src/UDTTest.cpp @@ -267,9 +267,7 @@ void UDTTest::sendPacket() { if (call++ % refillCount == 0) { // construct a reliable and ordered packet list - auto packetList = std::unique_ptr({ - new udt::PacketList(PacketType::BulkAvatarData, QByteArray(), true, true) - }); + auto packetList = udt::PacketList::create(PacketType::BulkAvatarData, QByteArray(), true, true); // fill the packet list with random data according to the constant seed (so receiver can verify) for (int i = 0; i < messageSizePackets; ++i) { From 18a696d88cfde5ada35872f7ed7937040d7f2c1e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 10 Sep 2015 23:49:41 +0200 Subject: [PATCH 05/10] Type cleanup --- domain-server/src/DomainServerNodeData.cpp | 2 +- domain-server/src/DomainServerNodeData.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/domain-server/src/DomainServerNodeData.cpp b/domain-server/src/DomainServerNodeData.cpp index 9b86697d2f..3dd9a1998a 100644 --- a/domain-server/src/DomainServerNodeData.cpp +++ b/domain-server/src/DomainServerNodeData.cpp @@ -19,7 +19,7 @@ #include "DomainServerNodeData.h" -QHash, QString> DomainServerNodeData::_overrideHash; +DomainServerNodeData::StringPairHash DomainServerNodeData::_overrideHash; DomainServerNodeData::DomainServerNodeData() { _paymentIntervalTimer.start(); diff --git a/domain-server/src/DomainServerNodeData.h b/domain-server/src/DomainServerNodeData.h index fcdc09e248..cd68aa3006 100644 --- a/domain-server/src/DomainServerNodeData.h +++ b/domain-server/src/DomainServerNodeData.h @@ -67,8 +67,9 @@ private: QString _username; QElapsedTimer _paymentIntervalTimer; + using StringPairHash = QHash, QString>; QJsonObject _statsJSONObject; - static QHash, QString> _overrideHash; + static StringPairHash _overrideHash; HifiSockAddr _sendingSockAddr; bool _isAuthenticated = true; From 0037a1f4a7186062cbabbdbbe9b158e626275711 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Sep 2015 00:09:55 +0200 Subject: [PATCH 06/10] Win build fix --- domain-server/src/DomainServerNodeData.cpp | 7 ++----- libraries/networking/src/udt/PacketList.h | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/domain-server/src/DomainServerNodeData.cpp b/domain-server/src/DomainServerNodeData.cpp index 3dd9a1998a..521406ad5c 100644 --- a/domain-server/src/DomainServerNodeData.cpp +++ b/domain-server/src/DomainServerNodeData.cpp @@ -54,16 +54,13 @@ QJsonObject DomainServerNodeData::overrideValuesIfNeeded(const QJsonObject& newS QJsonArray DomainServerNodeData::overrideValuesIfNeeded(const QJsonArray& newStats) { QJsonArray result; - for (int i = 0; i < newStats.size(); ++i) { - const auto& value = newStats[i]; - + for (const auto& value : newStats) { if (value.isObject()) { - result.push_back(overrideValuesIfNeeded(value.toObject())); } else if (value.isArray()) { result.push_back(overrideValuesIfNeeded(value.toArray())); } else { - result.push_back(newStats[i]); + result.push_back(value); } } return result; diff --git a/libraries/networking/src/udt/PacketList.h b/libraries/networking/src/udt/PacketList.h index bcf13ad954..ae783dabe3 100644 --- a/libraries/networking/src/udt/PacketList.h +++ b/libraries/networking/src/udt/PacketList.h @@ -61,7 +61,8 @@ protected: PacketList(PacketList&& other); virtual qint64 writeData(const char* data, qint64 maxSize); - virtual qint64 readData(char* data, qint64 maxSize) { Q_ASSERT(false); } + // Not implemented, added an assert so that it doesn't get used by accident + virtual qint64 readData(char* data, qint64 maxSize) { Q_ASSERT(false); return 0; } PacketType _packetType; std::list> _packets; From 42d1ab37453200dfd46948864d0115c820fd47bb Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Sep 2015 00:22:25 +0200 Subject: [PATCH 07/10] Remove accidentally added header --- interface/src/Application.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b51ba91246..394b582556 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -146,7 +146,6 @@ #include "ui/AvatarInputs.h" #include "ui/DataWebDialog.h" #include "ui/DialogsManager.h" -#include "ui/FileDialog.h" #include "ui/LoginDialog.h" #include "ui/Snapshot.h" #include "ui/StandAloneJSConsole.h" From 8c1fb67825c7424dc9cc75ea4e33f4b3a19fc794 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Sep 2015 00:29:08 +0200 Subject: [PATCH 08/10] Lock nodeHash mutex for node lookup --- libraries/networking/src/LimitedNodeList.cpp | 29 ++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 763bad677f..c13a82f821 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -416,11 +416,12 @@ void LimitedNodeList::eraseAllNodes() { killedNodes.insert(node); }); - // iterate the current nodes, emit that they are dying and remove them from the hash - _nodeMutex.lockForWrite(); - _nodeHash.clear(); - _nodeMutex.unlock(); - + { + // iterate the current nodes, emit that they are dying and remove them from the hash + QWriteLocker writeLocker(&_nodeMutex); + _nodeHash.clear(); + } + foreach(const SharedNodePointer& killedNode, killedNodes) { handleNodeKill(killedNode); } @@ -434,21 +435,20 @@ void LimitedNodeList::reset() { } void LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { - _nodeMutex.lockForRead(); + QReadLocker readLocker(&_nodeMutex); NodeHash::iterator it = _nodeHash.find(nodeUUID); if (it != _nodeHash.end()) { SharedNodePointer matchingNode = it->second; - _nodeMutex.unlock(); - - _nodeMutex.lockForWrite(); - _nodeHash.unsafe_erase(it); - _nodeMutex.unlock(); - + readLocker.unlock(); + + { + QWriteLocker writeLocker(&_nodeMutex); + _nodeHash.unsafe_erase(it); + } + handleNodeKill(matchingNode); - } else { - _nodeMutex.unlock(); } } @@ -846,6 +846,7 @@ void LimitedNodeList::sendPeerQueryToIceServer(const HifiSockAddr& iceServerSock } SharedNodePointer LimitedNodeList::findNodeWithAddr(const HifiSockAddr& addr) { + QReadLocker locker(&_nodeMutex); auto it = std::find_if(std::begin(_nodeHash), std::end(_nodeHash), [&](const UUIDNodePair& pair) { return pair.second->getActiveSocket() ? (*pair.second->getActiveSocket() == addr) : false; }); From 449d6a04d84c53e8a483bb8695f1651dec152c14 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Sep 2015 00:49:57 +0200 Subject: [PATCH 09/10] CR --- domain-server/src/DomainServerNodeData.cpp | 4 ++-- interface/src/Application.cpp | 2 +- libraries/networking/src/udt/ConnectionStats.h | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/domain-server/src/DomainServerNodeData.cpp b/domain-server/src/DomainServerNodeData.cpp index 521406ad5c..974d4a59c3 100644 --- a/domain-server/src/DomainServerNodeData.cpp +++ b/domain-server/src/DomainServerNodeData.cpp @@ -33,7 +33,7 @@ void DomainServerNodeData::updateJSONStats(QByteArray statsByteArray) { QJsonObject DomainServerNodeData::overrideValuesIfNeeded(const QJsonObject& newStats) { QJsonObject result; - for(auto it = newStats.constBegin(); it != newStats.constEnd(); ++it) { + for (auto it = newStats.constBegin(); it != newStats.constEnd(); ++it) { const auto& key = it.key(); const auto& value = it.value(); @@ -67,7 +67,7 @@ QJsonArray DomainServerNodeData::overrideValuesIfNeeded(const QJsonArray& newSta } void DomainServerNodeData::addOverrideForKey(const QString& key, const QString& value, - const QString& overrideValue) { + const QString& overrideValue) { // Insert override value _overrideHash.insert({key, value}, overrideValue); } diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 394b582556..e4f902ef1d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3042,7 +3042,7 @@ int Application::sendNackPackets() { ++it; } - if (nackPacketList.getNumPackets()) { + if (nackPacketList->getNumPackets()) { packetsSent += nackPacketList->getNumPackets(); // send the packet list diff --git a/libraries/networking/src/udt/ConnectionStats.h b/libraries/networking/src/udt/ConnectionStats.h index 2adc422a47..e04a836ca8 100644 --- a/libraries/networking/src/udt/ConnectionStats.h +++ b/libraries/networking/src/udt/ConnectionStats.h @@ -38,11 +38,11 @@ public: Retransmission, Duplicate, - NumEvent + NumEvents }; // construct a vector for the events of the size of our Enum - default value is zero - std::vector events = std::vector((int) Event::NumEvent, 0); + std::vector events = std::vector((int) Event::NumEvents, 0); // packet counts and sizes int sentPackets { 0 }; From 05df84ee67b4d8648ac302d6c4c9811a9143b32b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 11 Sep 2015 15:20:01 +0200 Subject: [PATCH 10/10] Remove AS stats namespace/Filter DS --- assignment-client/src/assets/AssetServer.cpp | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index c7732a64f9..138adb4716 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -173,13 +173,6 @@ void AssetServer::sendStatsPacket() { auto endTimeMs = std::chrono::duration_cast(stat.second.endTime); QDateTime date = QDateTime::fromMSecsSinceEpoch(endTimeMs.count()); - QString uuid = QUuid().toString(); - if (auto node = DependencyManager::get()->findNodeWithAddr(stat.first)) { - uuid = uuidStringWithoutCurlyBraces(node->getUUID().toString()); - } - - nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuid; - QJsonObject connectionStats; connectionStats["lastHeard"] = date.toString(); connectionStats["estimatedBandwith"] = stat.second.estimatedBandwith; @@ -211,11 +204,20 @@ void AssetServer::sendStatsPacket() { receivingStats["Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate]; nodeStats["receiving"] = receivingStats; + QString uuid; + auto nodelist = DependencyManager::get(); + if (stat.first == nodelist->getDomainHandler().getSockAddr()) { + uuid = uuidStringWithoutCurlyBraces(nodelist->getDomainHandler().getUUID()); + nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = "DomainServer"; + } else { + auto node = nodelist->findNodeWithAddr(stat.first); + uuid = uuidStringWithoutCurlyBraces(node ? node->getUUID() : QUuid()); + nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuid; + } + serverStats[uuid] = nodeStats; } // send off the stats packets - QJsonObject statsObject; - statsObject["AssetServer"] = serverStats; - ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); + ThreadedAssignment::addPacketStatsAndSendStatsPacket(serverStats); }