Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-21 06:44:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #5772 from Atlante45/protocol

Browse source 
Asset server node JSON stats
This commit is contained in:
Stephen Birarda 2015-09-11 09:35:39 -07:00
commit a286f4e7d0
34 changed files with 381 additions and 612 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
assignment-client/src/AssignmentClientMonitor.cpp
View file

@ -12,7 +12,6 @@
#include <signal.h>
#include <AddressManager.h>
#include <JSONBreakableMarshal.h>
#include <LogHandler.h>
#include <udt/PacketHeaders.h>

64
assignment-client/src/assets/AssetServer.cpp
View file

@ -12,14 +12,12 @@
#include "AssetServer.h"
#include <QBuffer>
#include <QCoreApplication>
#include <QCryptographicHash>
#include <QDateTime>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QCoreApplication>
#include <QEventLoop>
#include <QRunnable>
#include <QString>
#include "NetworkLogging.h"
@ -165,3 +163,61 @@ void AssetServer::handleAssetUpload(QSharedPointer<NLPacketList> packetList, Sha
}
}
void AssetServer::sendStatsPacket() {
QJsonObject serverStats;
auto stats = DependencyManager::get<NodeList>()->sampleStatsForAllConnections();
for (const auto& stat : stats) {
QJsonObject nodeStats;
auto endTimeMs = std::chrono::duration_cast<std::chrono::milliseconds>(stat.second.endTime);
QDateTime date = QDateTime::fromMSecsSinceEpoch(endTimeMs.count());
QJsonObject connectionStats;
connectionStats["lastHeard"] = date.toString();
connectionStats["estimatedBandwith"] = stat.second.estimatedBandwith;
connectionStats["rtt"] = stat.second.rtt;
connectionStats["congestionWindowSize"] = stat.second.congestionWindowSize;
connectionStats["packetSendPeriod"] = stat.second.packetSendPeriod;
nodeStats["connection"] = connectionStats;
QJsonObject sendingStats;
sendingStats["sendRate"] = stat.second.sendRate;
sendingStats["sentPackets"] = stat.second.sentPackets;
sendingStats["receivedACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK];
sendingStats["processedACK"] = stat.second.events[udt::ConnectionStats::Stats::ProcessedACK];
sendingStats["receivedLightACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedLightACK];
sendingStats["receivedNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedNAK];
sendingStats["receivedTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK];
sendingStats["sentACK2"] = stat.second.events[udt::ConnectionStats::Stats::SentACK2];
sendingStats["retransmission"] = stat.second.events[udt::ConnectionStats::Stats::Retransmission];
nodeStats["sending"] = sendingStats;
QJsonObject receivingStats;
receivingStats["receiveRate"] = stat.second.receiveRate;
receivingStats["receivedPackets"] = stat.second.receivedPackets;
receivingStats["SentACK"] = stat.second.events[udt::ConnectionStats::Stats::SentACK];
receivingStats["SentLightACK"] = stat.second.events[udt::ConnectionStats::Stats::SentLightACK];
receivingStats["SentNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentNAK];
receivingStats["SentTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentTimeoutNAK];
receivingStats["ReceivedACK2"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK2];
receivingStats["Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate];
nodeStats["receiving"] = receivingStats;
QString uuid;
auto nodelist = DependencyManager::get<NodeList>();
if (stat.first == nodelist->getDomainHandler().getSockAddr()) {
uuid = uuidStringWithoutCurlyBraces(nodelist->getDomainHandler().getUUID());
nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = "DomainServer";
} else {
auto node = nodelist->findNodeWithAddr(stat.first);
uuid = uuidStringWithoutCurlyBraces(node ? node->getUUID() : QUuid());
nodeStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuid;
}
serverStats[uuid] = nodeStats;
}
// send off the stats packets
ThreadedAssignment::addPacketStatsAndSendStatsPacket(serverStats);
}

4
assignment-client/src/assets/AssetServer.h
View file

@ -31,7 +31,9 @@ private slots:
void handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetUpload(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode);
void sendStatsPacket();
private:
static void writeError(NLPacketList* packetList, AssetServerError error);
QDir _resourcesDirectory;

2
assignment-client/src/assets/SendAssetTask.cpp
View file

@ -52,7 +52,7 @@ void SendAssetTask::run() {
qDebug() << "Received a request for the file (" << messageID << "): " << hexHash << " from " << start << " to " << end;
qDebug() << "Starting task to send asset: " << hexHash << " for messageID " << messageID;
auto replyPacketList = std::unique_ptr<NLPacketList>(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true));
auto replyPacketList = NLPacketList::create(PacketType::AssetGetReply, QByteArray(), true, true);
replyPacketList->write(assetHash);

14
assignment-client/src/avatars/AvatarMixer.cpp
View file

@ -219,7 +219,7 @@ void AvatarMixer::broadcastAvatarData() {
}
// setup a PacketList for the avatarPackets
NLPacketList avatarPacketList(PacketType::BulkAvatarData);
auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData);
// this is an AGENT we have received head data from
// send back a packet with other active node data to this node
@ -292,13 +292,13 @@ void AvatarMixer::broadcastAvatarData() {
otherNodeData->getLastReceivedSequenceNumber());
// start a new segment in the PacketList for this avatar
avatarPacketList.startSegment();
avatarPacketList->startSegment();
numAvatarDataBytes += avatarPacketList.write(otherNode->getUUID().toRfc4122());
numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122());
numAvatarDataBytes +=
avatarPacketList.write(otherAvatar.toByteArray(false, randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO));
avatarPacketList->write(otherAvatar.toByteArray(false, randFloat() < AVATAR_SEND_FULL_UPDATE_RATIO));
avatarPacketList.endSegment();
avatarPacketList->endSegment();
// if the receiving avatar has just connected make sure we send out the mesh and billboard
// for this avatar (assuming they exist)
@ -344,10 +344,10 @@ void AvatarMixer::broadcastAvatarData() {
});
// close the current packet so that we're always sending something
avatarPacketList.closeCurrentPacket(true);
avatarPacketList->closeCurrentPacket(true);
// send the avatar data PacketList
nodeList->sendPacketList(avatarPacketList, *node);
nodeList->sendPacketList(std::move(avatarPacketList), *node);
// record the bytes sent for other avatar data in the AvatarMixerClientData
nodeData->recordSentAvatarData(numAvatarDataBytes);

10
assignment-client/src/octree/OctreeInboundPacketProcessor.cpp
View file

@ -238,7 +238,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() {
return 0;
}
NLPacketList nackPacketList(_myServer->getMyEditNackType());
auto nackPacketList = NLPacketList::create(_myServer->getMyEditNackType());
auto nodeList = DependencyManager::get<NodeList>();
int packetsSent = 0;
@ -274,18 +274,18 @@ int OctreeInboundPacketProcessor::sendNackPackets() {
while (it != missingSequenceNumbers.constEnd()) {
unsigned short int sequenceNumber = *it;
nackPacketList.writePrimitive(sequenceNumber);
nackPacketList->writePrimitive(sequenceNumber);
++it;
}
if (nackPacketList.getNumPackets()) {
if (nackPacketList->getNumPackets()) {
qDebug() << "NACK Sent back to editor/client... destinationNode=" << nodeUUID;
packetsSent += nackPacketList.getNumPackets();
packetsSent += nackPacketList->getNumPackets();
// send the list of nack packets
nodeList->sendPacketList(nackPacketList, *destinationNode);
nodeList->sendPacketList(std::move(nackPacketList), *destinationNode);
}
++i;

150
assignment-client/src/octree/OctreeServer.cpp
View file

@ -1293,81 +1293,85 @@ QString OctreeServer::getStatusLink() {
}
void OctreeServer::sendStatsPacket() {
// TODO: we have too many stats to fit in a single MTU... so for now, we break it into multiple JSON objects and
// send them separately. What we really should do is change the NodeList::sendStatsToDomainServer() to handle the
// the following features:
// 1) remember last state sent
// 2) only send new data
// 3) automatically break up into multiple packets
static QJsonObject statsObject1;
QString baseName = getMyServerName() + QString("Server");
statsObject1[baseName + QString(".0.1.configuration")] = getConfiguration();
statsObject1[baseName + QString(".0.2.detailed_stats_url")] = getStatusLink();
statsObject1[baseName + QString(".0.3.uptime")] = getUptime();
statsObject1[baseName + QString(".0.4.persistFileLoadTime")] = getFileLoadTime();
statsObject1[baseName + QString(".0.5.clients")] = getCurrentClientCount();
// Stats Array 1
QJsonArray threadsStats;
quint64 oneSecondAgo = usecTimestampNow() - USECS_PER_SECOND;
threadsStats.push_back(QJsonObject({{"processing", (double)howManyThreadsDidProcess(oneSecondAgo)}}));
threadsStats.push_back(QJsonObject({{"packetDistributor", (double)howManyThreadsDidPacketDistributor(oneSecondAgo)}}));
threadsStats.push_back(QJsonObject({{"handlePacektSend", (double)howManyThreadsDidHandlePacketSend(oneSecondAgo)}}));
threadsStats.push_back(QJsonObject({{"writeDatagram", (double)howManyThreadsDidCallWriteDatagram(oneSecondAgo)}}));
QJsonArray statsArray1;
statsArray1.push_back(QJsonObject({{"configuration", getConfiguration()}}));
statsArray1.push_back(QJsonObject({{"detailed_stats_url", getStatusLink()}}));
statsArray1.push_back(QJsonObject({{"uptime", getUptime()}}));
statsArray1.push_back(QJsonObject({{"persistFileLoadTime", getFileLoadTime()}}));
statsArray1.push_back(QJsonObject({{"clients", getCurrentClientCount()}}));
statsArray1.push_back(QJsonObject({{"threads", threadsStats}}));
// Octree Stats
QJsonArray octreeStats;
octreeStats.push_back(QJsonObject({{"elementCount", (double)OctreeElement::getNodeCount()}}));
octreeStats.push_back(QJsonObject({{"internalElementCount", (double)OctreeElement::getInternalNodeCount()}}));
octreeStats.push_back(QJsonObject({{"leafElementCount", (double)OctreeElement::getLeafNodeCount()}}));
// Stats Object 2
QJsonObject dataObject1;
dataObject1["totalPackets"] = (double)OctreeSendThread::_totalPackets;
dataObject1["totalBytes"] = (double)OctreeSendThread::_totalBytes;
dataObject1["totalBytesWasted"] = (double)OctreeSendThread::_totalWastedBytes;
dataObject1["totalBytesOctalCodes"] = (double)OctreePacketData::getTotalBytesOfOctalCodes();
dataObject1["totalBytesBitMasks"] = (double)OctreePacketData::getTotalBytesOfBitMasks();
dataObject1["totalBytesBitMasks"] = (double)OctreePacketData::getTotalBytesOfColor();
statsObject1[baseName + QString(".0.6.threads.1.processing")] = (double)howManyThreadsDidProcess(oneSecondAgo);
statsObject1[baseName + QString(".0.6.threads.2.packetDistributor")] =
(double)howManyThreadsDidPacketDistributor(oneSecondAgo);
statsObject1[baseName + QString(".0.6.threads.3.handlePacektSend")] =
(double)howManyThreadsDidHandlePacketSend(oneSecondAgo);
statsObject1[baseName + QString(".0.6.threads.4.writeDatagram")] =
(double)howManyThreadsDidCallWriteDatagram(oneSecondAgo);
statsObject1[baseName + QString(".1.1.octree.elementCount")] = (double)OctreeElement::getNodeCount();
statsObject1[baseName + QString(".1.2.octree.internalElementCount")] = (double)OctreeElement::getInternalNodeCount();
statsObject1[baseName + QString(".1.3.octree.leafElementCount")] = (double)OctreeElement::getLeafNodeCount();
ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject1);
static QJsonObject statsObject2;
statsObject2[baseName + QString(".2.outbound.data.totalPackets")] = (double)OctreeSendThread::_totalPackets;
statsObject2[baseName + QString(".2.outbound.data.totalBytes")] = (double)OctreeSendThread::_totalBytes;
statsObject2[baseName + QString(".2.outbound.data.totalBytesWasted")] = (double)OctreeSendThread::_totalWastedBytes;
statsObject2[baseName + QString(".2.outbound.data.totalBytesOctalCodes")] =
(double)OctreePacketData::getTotalBytesOfOctalCodes();
statsObject2[baseName + QString(".2.outbound.data.totalBytesBitMasks")] =
(double)OctreePacketData::getTotalBytesOfBitMasks();
statsObject2[baseName + QString(".2.outbound.data.totalBytesBitMasks")] = (double)OctreePacketData::getTotalBytesOfColor();
statsObject2[baseName + QString(".2.outbound.timing.1.avgLoopTime")] = getAverageLoopTime();
statsObject2[baseName + QString(".2.outbound.timing.2.avgInsideTime")] = getAverageInsideTime();
statsObject2[baseName + QString(".2.outbound.timing.3.avgTreeLockTime")] = getAverageTreeWaitTime();
statsObject2[baseName + QString(".2.outbound.timing.4.avgEncodeTime")] = getAverageEncodeTime();
statsObject2[baseName + QString(".2.outbound.timing.5.avgCompressAndWriteTime")] = getAverageCompressAndWriteTime();
statsObject2[baseName + QString(".2.outbound.timing.5.avgSendTime")] = getAveragePacketSendingTime();
statsObject2[baseName + QString(".2.outbound.timing.5.nodeWaitTime")] = getAverageNodeWaitTime();
DependencyManager::get<NodeList>()->sendStatsToDomainServer(statsObject2);
static QJsonObject statsObject3;
statsObject3[baseName + QString(".3.inbound.data.1.packetQueue")] =
(double)_octreeInboundPacketProcessor->packetsToProcessCount();
statsObject3[baseName + QString(".3.inbound.data.1.totalPackets")] =
(double)_octreeInboundPacketProcessor->getTotalPacketsProcessed();
statsObject3[baseName + QString(".3.inbound.data.2.totalElements")] =
(double)_octreeInboundPacketProcessor->getTotalElementsProcessed();
statsObject3[baseName + QString(".3.inbound.timing.1.avgTransitTimePerPacket")] =
(double)_octreeInboundPacketProcessor->getAverageTransitTimePerPacket();
statsObject3[baseName + QString(".3.inbound.timing.2.avgProcessTimePerPacket")] =
(double)_octreeInboundPacketProcessor->getAverageProcessTimePerPacket();
statsObject3[baseName + QString(".3.inbound.timing.3.avgLockWaitTimePerPacket")] =
(double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerPacket();
statsObject3[baseName + QString(".3.inbound.timing.4.avgProcessTimePerElement")] =
(double)_octreeInboundPacketProcessor->getAverageProcessTimePerElement();
statsObject3[baseName + QString(".3.inbound.timing.5.avgLockWaitTimePerElement")] =
(double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerElement();
DependencyManager::get<NodeList>()->sendStatsToDomainServer(statsObject3);
QJsonArray timingArray1;
timingArray1.push_back(QJsonObject({{"avgLoopTime", getAverageLoopTime()}}));
timingArray1.push_back(QJsonObject({{"avgInsideTime", getAverageInsideTime()}}));
timingArray1.push_back(QJsonObject({{"avgTreeLockTime", getAverageTreeWaitTime()}}));
timingArray1.push_back(QJsonObject({{"avgEncodeTime", getAverageEncodeTime()}}));
timingArray1.push_back(QJsonObject({{"avgCompressAndWriteTime", getAverageCompressAndWriteTime()}}));
timingArray1.push_back(QJsonObject({{"avgSendTime", getAveragePacketSendingTime()}}));
timingArray1.push_back(QJsonObject({{"nodeWaitTime", getAverageNodeWaitTime()}}));
QJsonObject statsObject2;
statsObject2["data"] = dataObject1;
statsObject2["timing"] = timingArray1;
// Stats Object 3
QJsonArray dataArray2;
dataArray2.push_back(QJsonObject({{"packetQueue",
(double)_octreeInboundPacketProcessor->packetsToProcessCount()}}));
dataArray2.push_back(QJsonObject({{"totalPackets",
(double)_octreeInboundPacketProcessor->getTotalPacketsProcessed()}}));
dataArray2.push_back(QJsonObject({{"totalElements",
(double)_octreeInboundPacketProcessor->getTotalElementsProcessed()}}));
QJsonArray timingArray2;
timingArray2.push_back(QJsonObject({{"avgTransitTimePerPacket",
(double)_octreeInboundPacketProcessor->getAverageTransitTimePerPacket()}}));
timingArray2.push_back(QJsonObject({{"avgProcessTimePerPacket",
(double)_octreeInboundPacketProcessor->getAverageProcessTimePerPacket()}}));
timingArray2.push_back(QJsonObject({{"avgLockWaitTimePerPacket",
(double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerPacket()}}));
timingArray2.push_back(QJsonObject({{"avgProcessTimePerElement",
(double)_octreeInboundPacketProcessor->getAverageProcessTimePerElement()}}));
timingArray2.push_back(QJsonObject({{"avgLockWaitTimePerElement",
(double)_octreeInboundPacketProcessor->getAverageLockWaitTimePerElement()}}));
QJsonObject statsObject3;
statsObject3["data"] = dataArray2;
statsObject3["timing"] = timingArray2;
// Merge everything
QJsonArray jsonArray;
jsonArray.push_back(statsArray1);
jsonArray.push_back(QJsonObject({{"octree", octreeStats}}));
jsonArray.push_back(QJsonObject({{"outbound", statsObject2}}));
jsonArray.push_back(QJsonObject({{"inbound", statsObject3}}));
QJsonObject statsObject;
statsObject[QString(getMyServerName()) + "Server"] = jsonArray;
addPacketStatsAndSendStatsPacket(statsObject);
}
QMap<OctreeSendThread*, quint64> OctreeServer::_threadsDidProcess;

7
domain-server/src/DomainGatekeeper.cpp
View file

@ -17,7 +17,6 @@
#include <AccountManager.h>
#include <Assignment.h>
#include <JSONBreakableMarshal.h>
#include "DomainServer.h"
#include "DomainServerNodeData.h"
@ -272,9 +271,9 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect
// if we have a username from the connect request, set it on the DomainServerNodeData
nodeData->setUsername(username);
// also add an interpolation to JSONBreakableMarshal so that servers can get username in stats
JSONBreakableMarshal::addInterpolationForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY,
uuidStringWithoutCurlyBraces(newNode->getUUID()), username);
// also add an interpolation to DomainServerNodeData so that servers can get username in stats
nodeData->addOverrideForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY,
uuidStringWithoutCurlyBraces(newNode->getUUID()), username);
return newNode;
}

25
domain-server/src/DomainServer.cpp
View file

@ -25,7 +25,6 @@
#include <ApplicationVersion.h>
#include <HifiConfigVariantMap.h>
#include <HTTPConnection.h>
#include <JSONBreakableMarshal.h>
#include <LogUtils.h>
#include <NetworkingConstants.h>
#include <udt/PacketHeaders.h>
@ -287,7 +286,7 @@ void DomainServer::setupNodeListAndAssignments(const QUuid& sessionUUID) {
packetReceiver.registerListener(PacketType::RequestAssignment, this, "processRequestAssignmentPacket");
packetReceiver.registerListener(PacketType::DomainListRequest, this, "processListRequestPacket");
packetReceiver.registerListener(PacketType::DomainServerPathQuery, this, "processPathQueryPacket");
packetReceiver.registerListener(PacketType::NodeJsonStats, this, "processNodeJSONStatsPacket");
packetReceiver.registerMessageListener(PacketType::NodeJsonStats, this, "processNodeJSONStatsPacket");
// NodeList won't be available to the settings manager when it is created, so call registerListener here
packetReceiver.registerListener(PacketType::DomainSettingsRequest, &_settingsManager, "processSettingsRequestPacket");
@ -679,10 +678,10 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif
extendedHeaderStream << (quint8) node->getCanAdjustLocks();
extendedHeaderStream << (quint8) node->getCanRez();
NLPacketList domainListPackets(PacketType::DomainList, extendedHeader);
auto domainListPackets = NLPacketList::create(PacketType::DomainList, extendedHeader);
// always send the node their own UUID back
QDataStream domainListStream(&domainListPackets);
QDataStream domainListStream(domainListPackets.get());
DomainServerNodeData* nodeData = reinterpret_cast<DomainServerNodeData*>(node->getLinkedData());
@ -698,7 +697,7 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif
if (otherNode->getUUID() != node->getUUID() && nodeInterestSet.contains(otherNode->getType())) {
// since we're about to add a node to the packet we start a segment
domainListPackets.startSegment();
domainListPackets->startSegment();
// don't send avatar nodes to other avatars, that will come from avatar mixer
domainListStream << *otherNode.data();
@ -707,17 +706,17 @@ void DomainServer::sendDomainListToNode(const SharedNodePointer& node, const Hif
domainListStream << connectionSecretForNodes(node, otherNode);
// we've added the node we wanted so end the segment now
domainListPackets.endSegment();
domainListPackets->endSegment();
}
});
}
}
// send an empty list to the node, in case there were no other nodes
domainListPackets.closeCurrentPacket(true);
domainListPackets->closeCurrentPacket(true);
// write the PacketList to this node
limitedNodeList->sendPacketList(domainListPackets, *node);
limitedNodeList->sendPacketList(std::move(domainListPackets), *node);
}
QUuid DomainServer::connectionSecretForNodes(const SharedNodePointer& nodeA, const SharedNodePointer& nodeB) {
@ -1007,10 +1006,10 @@ void DomainServer::sendHeartbeatToIceServer() {
DependencyManager::get<LimitedNodeList>()->sendHeartbeatToIceServer(_iceServerSocket);
}
void DomainServer::processNodeJSONStatsPacket(QSharedPointer<NLPacket> packet, SharedNodePointer sendingNode) {
void DomainServer::processNodeJSONStatsPacket(QSharedPointer<NLPacketList> packetList, SharedNodePointer sendingNode) {
auto nodeData = dynamic_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
if (nodeData) {
nodeData->processJSONStatsPacket(*packet);
nodeData->updateJSONStats(packetList->getMessage());
}
}
@ -1676,9 +1675,9 @@ void DomainServer::nodeKilled(SharedNodePointer node) {
}
}
// If this node was an Agent ask JSONBreakableMarshal to potentially remove the interpolation we stored
JSONBreakableMarshal::removeInterpolationForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY,
uuidStringWithoutCurlyBraces(node->getUUID()));
// If this node was an Agent ask DomainServerNodeData to potentially remove the interpolation we stored
nodeData->removeOverrideForKey(USERNAME_UUID_REPLACEMENT_STATS_KEY,
uuidStringWithoutCurlyBraces(node->getUUID()));
// cleanup the connection secrets that we set up for this node (on the other nodes)
foreach (const QUuid& otherNodeSessionUUID, nodeData->getSessionSecretHash().keys()) {

2
domain-server/src/DomainServer.h
View file

@ -58,7 +58,7 @@ public slots:
void processRequestAssignmentPacket(QSharedPointer<NLPacket> packet);
void processListRequestPacket(QSharedPointer<NLPacket> packet, SharedNodePointer sendingNode);
void processNodeJSONStatsPacket(QSharedPointer<NLPacket> packet, SharedNodePointer sendingNode);
void processNodeJSONStatsPacket(QSharedPointer<NLPacketList> packetList, SharedNodePointer sendingNode);
void processPathQueryPacket(QSharedPointer<NLPacket> packet);
private slots:

71
domain-server/src/DomainServerNodeData.cpp
View file

@ -10,40 +10,69 @@
//
#include <QtCore/QDataStream>
#include <QtCore/QJsonArray>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
#include <QtCore/QVariant>
#include <JSONBreakableMarshal.h>
#include <udt/PacketHeaders.h>
#include "DomainServerNodeData.h"
DomainServerNodeData::DomainServerNodeData() :
_sessionSecretHash(),
_assignmentUUID(),
_walletUUID(),
_username(),
_paymentIntervalTimer(),
_statsJSONObject(),
_sendingSockAddr(),
_isAuthenticated(true)
{
DomainServerNodeData::StringPairHash DomainServerNodeData::_overrideHash;
DomainServerNodeData::DomainServerNodeData() {
_paymentIntervalTimer.start();
}
void DomainServerNodeData::processJSONStatsPacket(NLPacket& statsPacket) {
QVariantMap packetVariantMap = JSONBreakableMarshal::fromStringBuffer(statsPacket.readAll());
_statsJSONObject = mergeJSONStatsFromNewObject(QJsonObject::fromVariantMap(packetVariantMap), _statsJSONObject);
void DomainServerNodeData::updateJSONStats(QByteArray statsByteArray) {
auto document = QJsonDocument::fromBinaryData(statsByteArray);
Q_ASSERT(document.isObject());
_statsJSONObject = overrideValuesIfNeeded(document.object());
}
QJsonObject DomainServerNodeData::mergeJSONStatsFromNewObject(const QJsonObject& newObject, QJsonObject destinationObject) {
foreach(const QString& key, newObject.keys()) {
if (newObject[key].isObject() && destinationObject.contains(key)) {
destinationObject[key] = mergeJSONStatsFromNewObject(newObject[key].toObject(), destinationObject[key].toObject());
QJsonObject DomainServerNodeData::overrideValuesIfNeeded(const QJsonObject& newStats) {
QJsonObject result;
for (auto it = newStats.constBegin(); it != newStats.constEnd(); ++it) {
const auto& key = it.key();
const auto& value = it.value();
auto overrideIt = value.isString() ? _overrideHash.find({key, value.toString()}) : _overrideHash.end();
if (overrideIt != _overrideHash.end()) {
// We have a match, override the value
result[key] = *overrideIt;
} else if (value.isObject()) {
result[key] = overrideValuesIfNeeded(value.toObject());
} else if (value.isArray()) {
result[key] = overrideValuesIfNeeded(value.toArray());
} else {
destinationObject[key] = newObject[key];
result[key] = newStats[key];
}
}
return destinationObject;
return result;
}
QJsonArray DomainServerNodeData::overrideValuesIfNeeded(const QJsonArray& newStats) {
QJsonArray result;
for (const auto& value : newStats) {
if (value.isObject()) {
result.push_back(overrideValuesIfNeeded(value.toObject()));
} else if (value.isArray()) {
result.push_back(overrideValuesIfNeeded(value.toArray()));
} else {
result.push_back(value);
}
}
return result;
}
void DomainServerNodeData::addOverrideForKey(const QString& key, const QString& value,
const QString& overrideValue) {
// Insert override value
_overrideHash.insert({key, value}, overrideValue);
}
void DomainServerNodeData::removeOverrideForKey(const QString& key, const QString& value) {
// Remove override value
_overrideHash.remove({key, value});
}

16
domain-server/src/DomainServerNodeData.h
View file

@ -27,7 +27,7 @@ public:
const QJsonObject& getStatsJSONObject() const { return _statsJSONObject; }
void processJSONStatsPacket(NLPacket& packet);
void updateJSONStats(QByteArray statsByteArray);
void setAssignmentUUID(const QUuid& assignmentUUID) { _assignmentUUID = assignmentUUID; }
const QUuid& getAssignmentUUID() const { return _assignmentUUID; }
@ -54,17 +54,25 @@ public:
void setNodeVersion(const QString& nodeVersion) { _nodeVersion = nodeVersion; }
const QString& getNodeVersion() { return _nodeVersion; }
void addOverrideForKey(const QString& key, const QString& value, const QString& overrideValue);
void removeOverrideForKey(const QString& key, const QString& value);
private:
QJsonObject mergeJSONStatsFromNewObject(const QJsonObject& newObject, QJsonObject destinationObject);
QJsonObject overrideValuesIfNeeded(const QJsonObject& newStats);
QJsonArray overrideValuesIfNeeded(const QJsonArray& newStats);
QHash<QUuid, QUuid> _sessionSecretHash;
QUuid _assignmentUUID;
QUuid _walletUUID;
QString _username;
QElapsedTimer _paymentIntervalTimer;
using StringPairHash = QHash<QPair<QString, QString>, QString>;
QJsonObject _statsJSONObject;
static StringPairHash _overrideHash;
HifiSockAddr _sendingSockAddr;
bool _isAuthenticated;
bool _isAuthenticated = true;
NodeSet _nodeInterestSet;
QString _nodeVersion;
};

2
domain-server/src/DomainServerSettingsManager.cpp
View file

@ -74,7 +74,7 @@ void DomainServerSettingsManager::processSettingsRequestPacket(QSharedPointer<NL
QJsonObject responseObject = responseObjectForType(QString::number(type));
auto json = QJsonDocument(responseObject).toJson();
auto packetList = std::unique_ptr<NLPacketList>(new NLPacketList(PacketType::DomainSettings, QByteArray(), true, true));
auto packetList = NLPacketList::create(PacketType::DomainSettings, QByteArray(), true, true);
packetList->write(json);

12
interface/src/Application.cpp
View file

@ -142,6 +142,7 @@
#include "SpeechRecognizer.h"
#endif
#include "ui/AddressBarDialog.h"
#include "ui/AvatarInputs.h"
#include "ui/DataWebDialog.h"
#include "ui/DialogsManager.h"
@ -149,7 +150,6 @@
#include "ui/Snapshot.h"
#include "ui/StandAloneJSConsole.h"
#include "ui/Stats.h"
#include "ui/AddressBarDialog.h"
#include "ui/UpdateDialog.h"
#include "ui/overlays/Cube3DOverlay.h"
@ -3009,7 +3009,7 @@ int Application::sendNackPackets() {
if (node->getActiveSocket() && node->getType() == NodeType::EntityServer) {
NLPacketList nackPacketList(PacketType::OctreeDataNack);
auto nackPacketList = NLPacketList::create(PacketType::OctreeDataNack);
QUuid nodeUUID = node->getUUID();
@ -3038,15 +3038,15 @@ int Application::sendNackPackets() {
auto it = missingSequenceNumbers.constBegin();
while (it != missingSequenceNumbers.constEnd()) {
OCTREE_PACKET_SEQUENCE missingNumber = *it;
nackPacketList.writePrimitive(missingNumber);
nackPacketList->writePrimitive(missingNumber);
++it;
}
if (nackPacketList.getNumPackets()) {
packetsSent += nackPacketList.getNumPackets();
if (nackPacketList->getNumPackets()) {
packetsSent += nackPacketList->getNumPackets();
// send the packet list
nodeList->sendPacketList(nackPacketList, *node);
nodeList->sendPacketList(std::move(nackPacketList), *node);
}
}
});

1
interface/src/ui/AddressBarDialog.cpp
View file

@ -1,5 +1,6 @@
//
// AddressBarDialog.cpp
// interface/src/ui
//
// Created by Bradley Austin Davis on 2015/04/14
// Copyright 2015 High Fidelity, Inc.

1
interface/src/ui/AddressBarDialog.h
View file

@ -1,5 +1,6 @@
//
// AddressBarDialog.h
// interface/src/ui
//
// Created by Bradley Austin Davis on 2015/04/14
// Copyright 2015 High Fidelity, Inc.

6
interface/src/ui/DialogsManager.h
View file

@ -21,7 +21,6 @@
class QAction;
class AddressBarDialog;
class AnimationsDialog;
class AttachmentsDialog;
class AudioStatsDialog;
@ -29,13 +28,11 @@ class BandwidthDialog;
class CachesSizeDialog;
class DiskCacheEditor;
class LodToolsDialog;
class LoginDialog;
class OctreeStatsDialog;
class PreferencesDialog;
class ScriptEditorWindow;
class QMessageBox;
class DomainConnectionDialog;
class UpdateDialog;
class DialogsManager : public QObject, public Dependency {
Q_OBJECT
@ -94,7 +91,6 @@ private:
}
}
QPointer<AddressBarDialog> _addressBarDialog;
QPointer<AnimationsDialog> _animationsDialog;
QPointer<AttachmentsDialog> _attachmentsDialog;
QPointer<AudioStatsDialog> _audioStatsDialog;
@ -104,12 +100,10 @@ private:
QPointer<QMessageBox> _ircInfoBox;
QPointer<HMDToolsDialog> _hmdToolsDialog;
QPointer<LodToolsDialog> _lodToolsDialog;
QPointer<LoginDialog> _loginDialog;
QPointer<OctreeStatsDialog> _octreeStatsDialog;
QPointer<PreferencesDialog> _preferencesDialog;
QPointer<ScriptEditorWindow> _scriptEditor;
QPointer<DomainConnectionDialog> _domainConnectionDialog;
QPointer<UpdateDialog> _updateDialog;
};
#endif // hifi_DialogsManager_h

2
libraries/networking/src/AssetClient.cpp
View file

@ -205,7 +205,7 @@ bool AssetClient::uploadAsset(const QByteArray& data, const QString& extension,
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto packetList = std::unique_ptr<NLPacketList>(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true));
auto packetList = NLPacketList::create(PacketType::AssetUpload, QByteArray(), true, true);
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);

338
libraries/networking/src/JSONBreakableMarshal.cpp
View file

@ -1,338 +0,0 @@
//
// JSONBreakableMarshal.cpp
// libraries/networking/src
//
// Created by Stephen Birarda on 04/28/15.
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "JSONBreakableMarshal.h"
#include <QtCore/QDebug>
#include <QtCore/QJsonArray>
#include <QtCore/QJsonObject>
QVariantMap JSONBreakableMarshal::_interpolationMap = QVariantMap();
QStringList JSONBreakableMarshal::toStringList(const QJsonValue& jsonValue, const QString& keypath) {
// setup the string list that will hold our result
QStringList result;
// figure out what type of value this is so we know how to act on it
if (jsonValue.isObject()) {
QJsonObject jsonObject = jsonValue.toObject();
// enumerate the keys of the QJsonObject
foreach(const QString& key, jsonObject.keys()) {
QJsonValue childValue = jsonObject[key];
// setup the keypath for this key
QString valueKeypath = (keypath.isEmpty() ? "" : keypath + ".") + key;
if (childValue.isObject() || childValue.isArray()) {
// recursion is required since the value is a QJsonObject or QJsonArray
result << toStringList(childValue, valueKeypath);
} else {
// no recursion required, call our toString method to get the string representation
// append the QStringList resulting from that to our QStringList
result << toString(childValue, valueKeypath);
}
}
} else if (jsonValue.isArray()) {
QJsonArray jsonArray = jsonValue.toArray();
// enumerate the elements in this QJsonArray
for (int i = 0; i < jsonArray.size(); i++) {
QJsonValue arrayValue = jsonArray[i];
// setup the keypath for this object with the array index
QString valueKeypath = QString("%1[%2]").arg(keypath).arg(i);
if (arrayValue.isObject() || arrayValue.isArray()) {
// recursion is required since the value is a QJsonObject or QJsonArray
// append the QStringList resulting from that to our QStringList
result << toStringList(arrayValue, valueKeypath);
} else {
result << toString(arrayValue, valueKeypath);
}
}
} else {
// this is a basic value, so set result to whatever toString reports in a QStringList
result = QStringList() << toString(jsonValue, keypath);
}
return result;
}
const QString JSON_NULL_AS_STRING = "null";
const QString JSON_TRUE_AS_STRING = "true";
const QString JSON_FALSE_AS_STRING = "false";
const QString JSON_UNDEFINED_AS_STRING = "undefined";
const QString JSON_UNKNOWN_AS_STRING = "unknown";
QString JSONBreakableMarshal::toString(const QJsonValue& jsonValue, const QString& keypath) {
// default the value as a string to unknown in case conversion fails
QString valueAsString = JSON_UNKNOWN_AS_STRING;
// ask the QJsonValue what type it is and format its value as a string accordingly
if (jsonValue.isNull()) {
valueAsString = JSON_NULL_AS_STRING;
} else if (jsonValue.isBool()) {
valueAsString = jsonValue.toBool() ? JSON_TRUE_AS_STRING : JSON_FALSE_AS_STRING;
} else if (jsonValue.isDouble()) {
valueAsString = QString::number(jsonValue.toDouble());
} else if (jsonValue.isString()) {
valueAsString = QString("\"%1\"").arg(jsonValue.toString());
} else if (jsonValue.isUndefined()) {
valueAsString = JSON_UNDEFINED_AS_STRING;
} else if (jsonValue.isArray() || jsonValue.isObject()) {
qDebug() << "JSONBreakableMarshal::toString does not handle conversion of a QJsonObject or QJsonArray."
<< "You should call JSONBreakableMarshal::toStringList instead.";
} else {
qDebug() << "Unrecognized QJsonValue - JSONBreakableMarshal cannot convert to string.";
}
return QString("%1=%2").arg(keypath, valueAsString);
}
QVariant JSONBreakableMarshal::fromString(const QString& marshalValue) {
// default the value to null
QVariant result;
// attempt to match the value with our expected strings
if (marshalValue == JSON_NULL_AS_STRING) {
// this is already our default, we don't need to do anything here
} else if (marshalValue == JSON_TRUE_AS_STRING || marshalValue == JSON_FALSE_AS_STRING) {
result = QVariant(marshalValue == JSON_TRUE_AS_STRING ? true : false);
} else if (marshalValue == JSON_UNDEFINED_AS_STRING) {
result = JSON_UNDEFINED_AS_STRING;
} else if (marshalValue == JSON_UNKNOWN_AS_STRING) {
// we weren't able to marshal this value at the other end, set it as our unknown string
result = JSON_UNKNOWN_AS_STRING;
} else {
// this might be a double, see if it converts
bool didConvert = false;
double convertResult = marshalValue.toDouble(&didConvert);
if (didConvert) {
result = convertResult;
} else {
// we need to figure out if this is a string
// use a regex to look for surrounding quotes first
const QString JSON_STRING_REGEX = "^\"([\\s\\S]*)\"$";
QRegExp stringRegex(JSON_STRING_REGEX);
if (stringRegex.indexIn(marshalValue) != -1) {
// set the result to the string value
result = stringRegex.cap(1);
} else {
// we failed to convert the value to anything, set the result to our unknown value
qDebug() << "Unrecognized output from JSONBreakableMarshal - could not convert"
<< marshalValue << "to QVariant.";
result = JSON_UNKNOWN_AS_STRING;
}
}
}
return result;
}
QVariantMap JSONBreakableMarshal::fromStringList(const QStringList& stringList) {
QVariant result = QVariantMap();
foreach(const QString& marshalString, stringList) {
// find the equality operator
int equalityIndex = marshalString.indexOf('=');
// bail on parsing if we didn't find the equality operator
if (equalityIndex != -1) {
QVariant* currentValue = &result;
// pull the key (everything left of the equality sign)
QString parentKeypath;
QString keypath = marshalString.left(equalityIndex);
// setup for array index checking
const QString ARRAY_INDEX_REGEX_STRING = "\\[(\\d+)\\]";
QRegExp arrayRegex(ARRAY_INDEX_REGEX_STRING);
// as long as we have a keypath we need to recurse downwards
while (!keypath.isEmpty()) {
parentKeypath = keypath;
int arrayBracketIndex = arrayRegex.indexIn(keypath);
// is this an array index
if (arrayBracketIndex == 0) {
// we're here because we think the current value should be an array
// if it isn't then make one
if (!currentValue->canConvert(QMetaType::QVariantList) || currentValue->isNull()) {
*currentValue = QVariantList();
}
QVariantList& currentList = *static_cast<QVariantList*>(currentValue->data());
// figure out what index we want to get the QJsonValue& for
bool didConvert = false;
int arrayIndex = arrayRegex.cap(1).toInt(&didConvert);
if (didConvert) {
// check if we need to resize the array
if (currentList.size() < arrayIndex + 1) {
for (int i = currentList.size(); i < arrayIndex + 1; i++) {
// add the null QJsonValue at this array index to get the array to the right size
currentList.push_back(QJsonValue());
}
}
// set our currentValue to the QJsonValue& from the array at this index
currentValue = &currentList[arrayIndex];
// update the keypath by bumping past the array index
keypath = keypath.mid(keypath.indexOf(']') + 1);
// check if there is a key after the array index - if so push the keypath forward by a char
if (keypath.startsWith(".")) {
keypath = keypath.mid(1);
}
} else {
qDebug() << "Failed to convert array index from keypath" << keypath << "to int. Will not add"
<< "value to resulting QJsonObject.";
break;
}
} else {
int keySeparatorIndex = keypath.indexOf('.');
// we need to figure out what the key to look at is
QString subKey = keypath;
if (keySeparatorIndex != -1 || arrayBracketIndex != -1) {
int nextBreakIndex = -1;
int nextKeypathStartIndex = -1;
if (arrayBracketIndex == -1 || (keySeparatorIndex != -1 && keySeparatorIndex < arrayBracketIndex)) {
nextBreakIndex = keySeparatorIndex;
nextKeypathStartIndex = keySeparatorIndex + 1;
} else if (keySeparatorIndex == -1 || (arrayBracketIndex != -1
&& arrayBracketIndex < keySeparatorIndex)) {
nextBreakIndex = arrayBracketIndex;
nextKeypathStartIndex = arrayBracketIndex;
} else {
qDebug() << "Unrecognized key format while trying to parse " << keypath << " - will not add"
<< "value to resulting QJsonObject.";
break;
}
// set the current key from the determined index
subKey = keypath.left(nextBreakIndex);
// update the keypath being processed
keypath = keypath.mid(nextKeypathStartIndex);
} else {
// update the keypath being processed, since we have no more separators in the keypath, it should
// be an empty string
keypath = "";
}
// we're here becuase we know the current value should be an object
// if it isn't then make it one
if (!currentValue->canConvert(QMetaType::QVariantMap) || currentValue->isNull()) {
*currentValue = QVariantMap();
}
QVariantMap& currentMap = *static_cast<QVariantMap*>(currentValue->data());
// is there a QJsonObject for this key yet?
// if not then we make it now
if (!currentMap.contains(subKey)) {
currentMap[subKey] = QVariant();
}
// change the currentValue to the QJsonValue for this key
currentValue = &currentMap[subKey];
}
}
*currentValue = fromString(marshalString.mid(equalityIndex + 1));
if (_interpolationMap.contains(parentKeypath)) {
// we expect the currentValue here to be a string, that's the key we use for interpolation
// bail if it isn't
if (currentValue->canConvert(QMetaType::QString)
&& _interpolationMap[parentKeypath].canConvert(QMetaType::QVariantMap)) {
*currentValue = _interpolationMap[parentKeypath].toMap()[currentValue->toString()];
}
}
}
}
return result.toMap();
}
QVariantMap JSONBreakableMarshal::fromStringBuffer(const QByteArray& buffer) {
// this is a packet of strings sep by null terminators - pull out each string and create a stringlist
QStringList packetList;
int currentIndex = 0;
int currentSeparator = buffer.indexOf('\0');
while (currentIndex < buffer.size() - 1) {
packetList << QString::fromUtf8(buffer.mid(currentIndex, currentSeparator));
if (currentSeparator == -1) {
// no more separators to be found, break out of here so we're not looping for nothing
break;
}
// bump the currentIndex up to the last found separator
currentIndex = currentSeparator + 1;
// find the index of the next separator, assuming this one wasn't the last one in the packet
if (currentSeparator < buffer.size() - 1) {
currentSeparator = buffer.indexOf('\0', currentIndex);
}
}
// now that we have a QStringList we use our static method to turn that into a QJsonObject
return fromStringList(packetList);
}
void JSONBreakableMarshal::addInterpolationForKey(const QString& rootKey, const QString& interpolationKey,
const QJsonValue& interpolationValue) {
// if there is no map already beneath this key in our _interpolationMap create a QVariantMap there now
if (!_interpolationMap.contains(rootKey)) {
_interpolationMap.insert(rootKey, QVariantMap());
}
if (_interpolationMap[rootKey].canConvert(QMetaType::QVariantMap)) {
QVariantMap& mapForRootKey = *static_cast<QVariantMap*>(_interpolationMap[rootKey].data());
mapForRootKey.insert(interpolationKey, QVariant(interpolationValue));
} else {
qDebug() << "JSONBreakableMarshal::addInterpolationForKey could not convert variant at key" << rootKey
<< "to a QVariantMap. Can not add interpolation.";
}
}
void JSONBreakableMarshal::removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey) {
// make sure the interpolation map contains this root key and that the value is a map
if (_interpolationMap.contains(rootKey)) {
QVariant& rootValue = _interpolationMap[rootKey];
if (!rootValue.isNull() && rootValue.canConvert(QMetaType::QVariantMap)) {
// remove the value at the interpolationKey
static_cast<QVariantMap*>(rootValue.data())->remove(interpolationKey);
}
}
}

39
libraries/networking/src/JSONBreakableMarshal.h
View file

@ -1,39 +0,0 @@
//
// JSONBreakableMarshal.h
// libraries/networking/src
//
// Created by Stephen Birarda on 04/28/15.
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_JSONBreakableMarshal_h
#define hifi_JSONBreakableMarshal_h
#pragma once
#include <QtCore/QJsonValue>
#include <QtCore/QString>
#include <QtCore/QStringList>
#include <QtCore/QVariantMap>
class JSONBreakableMarshal {
public:
static QStringList toStringList(const QJsonValue& jsonValue, const QString& keypath);
static QString toString(const QJsonValue& jsonValue, const QString& keyPath);
static QVariant fromString(const QString& marshalValue);
static QVariantMap fromStringList(const QStringList& stringList);
static QVariantMap fromStringBuffer(const QByteArray& buffer);
static void addInterpolationForKey(const QString& rootKey,
const QString& interpolationKey, const QJsonValue& interpolationValue);
static void removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey);
private:
static QVariantMap _interpolationMap;
};
#endif // hifi_JSONBreakableMarshal_h

42
libraries/networking/src/LimitedNodeList.cpp
View file

@ -355,6 +355,12 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr<NLPacketList> packetList,
// close the last packet in the list
packetList->closeCurrentPacket();
for (std::unique_ptr<udt::Packet>& packet : packetList->_packets) {
NLPacket* nlPacket = static_cast<NLPacket*>(packet.get());
collectPacketStats(*nlPacket);
fillPacketHeader(*nlPacket);
}
return _nodeSocket.writePacketList(std::move(packetList), sockAddr);
}
@ -410,11 +416,12 @@ void LimitedNodeList::eraseAllNodes() {
killedNodes.insert(node);
});
// iterate the current nodes, emit that they are dying and remove them from the hash
_nodeMutex.lockForWrite();
_nodeHash.clear();
_nodeMutex.unlock();
{
// iterate the current nodes, emit that they are dying and remove them from the hash
QWriteLocker writeLocker(&_nodeMutex);
_nodeHash.clear();
}
foreach(const SharedNodePointer& killedNode, killedNodes) {
handleNodeKill(killedNode);
}
@ -428,21 +435,20 @@ void LimitedNodeList::reset() {
}
void LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) {
_nodeMutex.lockForRead();
QReadLocker readLocker(&_nodeMutex);
NodeHash::iterator it = _nodeHash.find(nodeUUID);
if (it != _nodeHash.end()) {
SharedNodePointer matchingNode = it->second;
_nodeMutex.unlock();
_nodeMutex.lockForWrite();
_nodeHash.unsafe_erase(it);
_nodeMutex.unlock();
readLocker.unlock();
{
QWriteLocker writeLocker(&_nodeMutex);
_nodeHash.unsafe_erase(it);
}
handleNodeKill(matchingNode);
} else {
_nodeMutex.unlock();
}
}
@ -839,6 +845,14 @@ void LimitedNodeList::sendPeerQueryToIceServer(const HifiSockAddr& iceServerSock
sendPacketToIceServer(PacketType::ICEServerQuery, iceServerSockAddr, clientID, peerID);
}
SharedNodePointer LimitedNodeList::findNodeWithAddr(const HifiSockAddr& addr) {
QReadLocker locker(&_nodeMutex);
auto it = std::find_if(std::begin(_nodeHash), std::end(_nodeHash), [&](const UUIDNodePair& pair) {
return pair.second->getActiveSocket() ? (*pair.second->getActiveSocket() == addr) : false;
});
return (it != std::end(_nodeHash)) ? it->second : SharedNodePointer();
}
void LimitedNodeList::sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr,
const QUuid& clientID, const QUuid& peerID) {
auto icePacket = NLPacket::create(packetType);

5
libraries/networking/src/LimitedNodeList.h
View file

@ -165,6 +165,8 @@ public:
void sendHeartbeatToIceServer(const HifiSockAddr& iceServerSockAddr);
void sendPeerQueryToIceServer(const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerID);
SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr);
template<typename NodeLambda>
void eachNode(NodeLambda functor) {
QReadLocker readLock(&_nodeMutex);
@ -216,6 +218,7 @@ public:
{ QReadLocker readLock(&_connectionTimeLock); return _lastConnectionTimes; }
void flagTimeForConnectionStep(ConnectionStep connectionStep);
udt::Socket::StatsVector sampleStatsForAllConnections() { return _nodeSocket.sampleStatsForAllConnections(); }
public slots:
void reset();
@ -254,7 +257,7 @@ protected:
qint64 writePacket(const NLPacket& packet, const HifiSockAddr& destinationSockAddr,
const QUuid& connectionSecret = QUuid());
void collectPacketStats(const NLPacket& packet);
void fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret);
void fillPacketHeader(const NLPacket& packet, const QUuid& connectionSecret = QUuid());
bool isPacketVerified(const udt::Packet& packet);
bool packetVersionMatch(const udt::Packet& packet);

22
libraries/networking/src/NLPacketList.cpp
View file

@ -13,16 +13,30 @@
#include "udt/Packet.h"
std::unique_ptr<NLPacketList> NLPacketList::create(PacketType packetType, QByteArray extendedHeader,
bool isReliable, bool isOrdered) {
auto nlPacketList = std::unique_ptr<NLPacketList>(new NLPacketList(packetType, extendedHeader,
isReliable, isOrdered));
nlPacketList->open(WriteOnly);
return nlPacketList;
}
std::unique_ptr<NLPacketList> NLPacketList::fromPacketList(std::unique_ptr<PacketList> packetList) {
auto nlPacketList = std::unique_ptr<NLPacketList>(new NLPacketList(std::move(*packetList.release()))); nlPacketList->open(ReadOnly);
return nlPacketList;
}
NLPacketList::NLPacketList(PacketType packetType, QByteArray extendedHeader, bool isReliable, bool isOrdered) :
PacketList(packetType, extendedHeader, isReliable, isOrdered)
{
}
NLPacketList::NLPacketList(PacketList&& other) : PacketList(other.getType(), other.getExtendedHeader(), other.isReliable(), other.isOrdered()) {
NLPacketList::NLPacketList(PacketList&& other) : PacketList(std::move(other)) {
// Update _packets
for (auto& packet : other._packets) {
auto nlPacket = NLPacket::fromBase(std::move(packet));
_packets.push_back(std::move(nlPacket));
for (auto& packet : _packets) {
packet = NLPacket::fromBase(std::move(packet));
}
if (_packets.size() > 0) {

9
libraries/networking/src/NLPacketList.h
View file

@ -18,12 +18,17 @@
class NLPacketList : public udt::PacketList {
public:
NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false);
NLPacketList(PacketList&& packetList);
static std::unique_ptr<NLPacketList> create(PacketType packetType, QByteArray extendedHeader = QByteArray(),
bool isReliable = false, bool isOrdered = false);
static std::unique_ptr<NLPacketList> fromPacketList(std::unique_ptr<PacketList>);
const QUuid& getSourceID() const { return _sourceID; }
private:
NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false,
bool isOrdered = false);
NLPacketList(PacketList&& packetList);
NLPacketList(const NLPacketList& other) = delete;
NLPacketList& operator=(const NLPacketList& other) = delete;

17
libraries/networking/src/NodeList.cpp
View file

@ -27,7 +27,6 @@
#include "AddressManager.h"
#include "Assignment.h"
#include "HifiSockAddr.h"
#include "JSONBreakableMarshal.h"
#include "NetworkLogging.h"
#include "udt/PacketHeaders.h"
@ -107,20 +106,12 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned
}
qint64 NodeList::sendStats(const QJsonObject& statsObject, const HifiSockAddr& destination) {
NLPacketList statsPacketList(PacketType::NodeJsonStats);
auto statsPacketList = NLPacketList::create(PacketType::NodeJsonStats, QByteArray(), true, true);
// get a QStringList using JSONBreakableMarshal
QStringList statsStringList = JSONBreakableMarshal::toStringList(statsObject, "");
QJsonDocument jsonDocument(statsObject);
statsPacketList->write(jsonDocument.toBinaryData());
// enumerate the resulting strings - pack them and send off packets via NLPacketList
foreach(const QString& statsItem, statsStringList) {
QByteArray utf8String = statsItem.toUtf8();
utf8String.append('\0');
statsPacketList.write(utf8String);
}
sendPacketList(statsPacketList, destination);
sendPacketList(std::move(statsPacketList), destination);
// enumerate the resulting strings, breaking them into MTU sized packets
return 0;

10
libraries/networking/src/PacketReceiver.cpp
View file

@ -247,7 +247,7 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr<udt::PacketList> p
}
// setup an NLPacketList from the PacketList we were passed
auto nlPacketList = new NLPacketList(std::move(*packetList));
auto nlPacketList = NLPacketList::fromPacketList(std::move(packetList));
auto nodeList = DependencyManager::get<LimitedNodeList>();
@ -297,21 +297,21 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr<udt::PacketList> p
success = metaMethod.invoke(listener.first,
connectionType,
Q_ARG(QSharedPointer<NLPacketList>,
QSharedPointer<NLPacketList>(nlPacketList)),
QSharedPointer<NLPacketList>(nlPacketList.release())),
Q_ARG(SharedNodePointer, matchingNode));
} else if (metaMethod.parameterTypes().contains(QSHAREDPOINTER_NODE_NORMALIZED)) {
success = metaMethod.invoke(listener.first,
connectionType,
Q_ARG(QSharedPointer<NLPacketList>,
QSharedPointer<NLPacketList>(nlPacketList)),
QSharedPointer<NLPacketList>(nlPacketList.release())),
Q_ARG(QSharedPointer<Node>, matchingNode));
} else {
success = metaMethod.invoke(listener.first,
connectionType,
Q_ARG(QSharedPointer<NLPacketList>,
QSharedPointer<NLPacketList>(nlPacketList)));
QSharedPointer<NLPacketList>(nlPacketList.release())));
}
} else {
listenerIsDead = true;
@ -323,7 +323,7 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr<udt::PacketList> p
if (listener.first) {
success = listener.second.invoke(listener.first,
Q_ARG(QSharedPointer<NLPacketList>,
QSharedPointer<NLPacketList>(nlPacketList)));
QSharedPointer<NLPacketList>(nlPacketList.release())));
} else {
listenerIsDead = true;
}

12
libraries/networking/src/udt/BasePacket.h
View file

@ -31,8 +31,6 @@ public:
static std::unique_ptr<BasePacket> create(qint64 size = -1);
static std::unique_ptr<BasePacket> fromReceivedPacket(std::unique_ptr<char[]> data, qint64 size,
const HifiSockAddr& senderSockAddr);
// Current level's header size
static int localHeaderSize();
@ -72,8 +70,8 @@ public:
virtual bool isSequential() const { return false; }
virtual bool reset();
virtual qint64 size() const { return _payloadCapacity; }
using QIODevice::read;
using QIODevice::read; // Bring QIODevice::read methods to scope, otherwise they are hidden by folling method
QByteArray read(qint64 maxSize);
QByteArray readWithoutCopy(qint64 maxSize); // this can only be used if packet will stay in scope
@ -107,16 +105,16 @@ protected:
};
template<typename T> qint64 BasePacket::peekPrimitive(T* data) {
return QIODevice::peek(reinterpret_cast<char*>(data), sizeof(T));
return peek(reinterpret_cast<char*>(data), sizeof(T));
}
template<typename T> qint64 BasePacket::readPrimitive(T* data) {
return QIODevice::read(reinterpret_cast<char*>(data), sizeof(T));
return read(reinterpret_cast<char*>(data), sizeof(T));
}
template<typename T> qint64 BasePacket::writePrimitive(const T& data) {
static_assert(!std::is_pointer<T>::value, "T must not be a pointer");
return QIODevice::write(reinterpret_cast<const char*>(&data), sizeof(T));
return write(reinterpret_cast<const char*>(&data), sizeof(T));
}
} // namespace udt

4
libraries/networking/src/udt/ConnectionStats.cpp
View file

@ -15,7 +15,7 @@ using namespace udt;
using namespace std::chrono;
ConnectionStats::ConnectionStats() {
auto now = duration_cast<microseconds>(high_resolution_clock::now().time_since_epoch());
auto now = duration_cast<microseconds>(system_clock::now().time_since_epoch());
_currentSample.startTime = now;
_total.startTime = now;
}
@ -24,7 +24,7 @@ ConnectionStats::Stats ConnectionStats::sample() {
Stats sample = _currentSample;
_currentSample = Stats();
auto now = duration_cast<microseconds>(high_resolution_clock::now().time_since_epoch());
auto now = duration_cast<microseconds>(system_clock::now().time_since_epoch());
sample.endTime = now;
_currentSample.startTime = now;

6
libraries/networking/src/udt/ConnectionStats.h
View file

@ -36,11 +36,13 @@ public:
SentTimeoutNAK,
ReceivedTimeoutNAK,
Retransmission,
Duplicate
Duplicate,
NumEvents
};
// construct a vector for the events of the size of our Enum - default value is zero
std::vector<int> events = std::vector<int>((int) Event::Duplicate + 1, 0);
std::vector<int> events = std::vector<int>((int) Event::NumEvents, 0);
// packet counts and sizes
int sentPackets { 0 };

50
libraries/networking/src/udt/PacketList.cpp
View file

@ -15,6 +15,21 @@
using namespace udt;
std::unique_ptr<PacketList> PacketList::create(PacketType packetType, QByteArray extendedHeader,
bool isReliable, bool isOrdered) {
auto packetList = std::unique_ptr<PacketList>(new PacketList(packetType, extendedHeader,
isReliable, isOrdered));
packetList->open(WriteOnly);
return packetList;
}
std::unique_ptr<PacketList> PacketList::fromReceivedPackets(std::list<std::unique_ptr<Packet>>&& packets) {
auto packetList = std::unique_ptr<PacketList>(new PacketList(PacketType::Unknown, QByteArray(), true, true));
packetList->_packets = std::move(packets);
packetList->open(ReadOnly);
return packetList;
}
PacketList::PacketList(PacketType packetType, QByteArray extendedHeader, bool isReliable, bool isOrdered) :
_packetType(packetType),
_isReliable(isReliable),
@ -22,14 +37,15 @@ PacketList::PacketList(PacketType packetType, QByteArray extendedHeader, bool is
_extendedHeader(extendedHeader)
{
Q_ASSERT_X(!(!_isReliable && _isOrdered), "PacketList", "Unreliable ordered PacketLists are not currently supported");
QIODevice::open(WriteOnly);
}
PacketList::PacketList(PacketList&& other) :
_packetType(other._packetType),
_packets(std::move(other._packets)),
_packetType(other._packetType)
_isReliable(other._isReliable),
_isOrdered(other._isOrdered),
_extendedHeader(std::move(other._extendedHeader))
{
}
void PacketList::startSegment() {
@ -66,12 +82,6 @@ size_t PacketList::getMessageSize() const {
return totalBytes;
}
std::unique_ptr<PacketList> PacketList::fromReceivedPackets(std::list<std::unique_ptr<Packet>>&& packets) {
auto packetList = std::unique_ptr<PacketList>(new PacketList(PacketType::Unknown, QByteArray(), true, true));
packetList->_packets = std::move(packets);
return packetList;
}
std::unique_ptr<Packet> PacketList::createPacket() {
// use the static create method to create a new packet
// If this packet list is supposed to be ordered then we consider this to be part of a message
@ -94,6 +104,17 @@ std::unique_ptr<Packet> PacketList::createPacketWithExtendedHeader() {
return packet;
}
void PacketList::closeCurrentPacket(bool shouldSendEmpty) {
if (shouldSendEmpty && !_currentPacket) {
_currentPacket = createPacketWithExtendedHeader();
}
if (_currentPacket) {
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
}
}
QByteArray PacketList::getMessage() {
size_t sizeBytes = 0;
@ -191,14 +212,3 @@ qint64 PacketList::writeData(const char* data, qint64 maxSize) {
return maxSize;
}
void PacketList::closeCurrentPacket(bool shouldSendEmpty) {
if (shouldSendEmpty && !_currentPacket) {
_currentPacket = createPacketWithExtendedHeader();
}
if (_currentPacket) {
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
}
}

30
libraries/networking/src/udt/PacketList.h
View file

@ -28,13 +28,9 @@ class Packet;
class PacketList : public QIODevice {
Q_OBJECT
public:
PacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false);
PacketList(PacketList&& other);
static std::unique_ptr<PacketList> create(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false);
static std::unique_ptr<PacketList> fromReceivedPackets(std::list<std::unique_ptr<Packet>>&& packets);
virtual bool isSequential() const { return true; }
bool isReliable() const { return _isReliable; }
bool isOrdered() const { return _isOrdered; }
@ -53,20 +49,28 @@ public:
QByteArray getMessage();
// QIODevice virtual functions
virtual bool isSequential() const { return false; }
virtual qint64 size() const { return getDataSize(); }
template<typename T> qint64 readPrimitive(T* data);
template<typename T> qint64 writePrimitive(const T& data);
std::list<std::unique_ptr<Packet>> _packets;
protected:
virtual qint64 writeData(const char* data, qint64 maxSize);
virtual qint64 readData(char* data, qint64 maxSize) { return 0; }
PacketType _packetType;
PacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, bool isOrdered = false);
PacketList(PacketList&& other);
virtual qint64 writeData(const char* data, qint64 maxSize);
// Not implemented, added an assert so that it doesn't get used by accident
virtual qint64 readData(char* data, qint64 maxSize) { Q_ASSERT(false); return 0; }
PacketType _packetType;
std::list<std::unique_ptr<Packet>> _packets;
private:
friend class ::LimitedNodeList;
friend class Socket;
friend class SendQueue;
friend class NLPacketList;
friend class Socket;
PacketList(const PacketList& other) = delete;
PacketList& operator=(const PacketList& other) = delete;
@ -91,12 +95,12 @@ private:
template <typename T> qint64 PacketList::readPrimitive(T* data) {
static_assert(!std::is_pointer<T>::value, "T must not be a pointer");
return QIODevice::read(reinterpret_cast<char*>(data), sizeof(T));
return read(reinterpret_cast<char*>(data), sizeof(T));
}
template <typename T> qint64 PacketList::writePrimitive(const T& data) {
static_assert(!std::is_pointer<T>::value, "T must not be a pointer");
return QIODevice::write(reinterpret_cast<const char*>(&data), sizeof(T));
return write(reinterpret_cast<const char*>(&data), sizeof(T));
}
template<typename T> std::unique_ptr<T> PacketList::takeFront() {

10
libraries/networking/src/udt/Socket.cpp
View file

@ -325,6 +325,16 @@ ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& dest
}
}
Socket::StatsVector Socket::sampleStatsForAllConnections() {
StatsVector result;
result.reserve(_connectionsHash.size());
for (const auto& connectionPair : _connectionsHash) {
result.emplace_back(connectionPair.first, connectionPair.second->sampleStats());
}
return result;
}
std::vector<HifiSockAddr> Socket::getConnectionSockAddrs() {
std::vector<HifiSockAddr> addr;
addr.reserve(_connectionsHash.size());

5
libraries/networking/src/udt/Socket.h
View file

@ -46,6 +46,8 @@ using PacketListHandler = std::function<void(std::unique_ptr<PacketList>)>;
class Socket : public QObject {
Q_OBJECT
public:
using StatsVector = std::vector<std::pair<HifiSockAddr, ConnectionStats::Stats>>;
Socket(QObject* object = 0);
quint16 localPort() const { return _udpSocket.localPort(); }
@ -71,6 +73,8 @@ public:
void setCongestionControlFactory(std::unique_ptr<CongestionControlVirtualFactory> ccFactory);
void messageReceived(std::unique_ptr<PacketList> packetList);
StatsVector sampleStatsForAllConnections();
public slots:
void cleanupConnection(HifiSockAddr sockAddr);
@ -86,6 +90,7 @@ private:
// privatized methods used by UDTTest - they are private since they must be called on the Socket thread
ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination);
std::vector<HifiSockAddr> getConnectionSockAddrs();
void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot);

4
tools/udt-test/src/UDTTest.cpp
View file

@ -267,9 +267,7 @@ void UDTTest::sendPacket() {
if (call++ % refillCount == 0) {
// construct a reliable and ordered packet list
auto packetList = std::unique_ptr<udt::PacketList>({
new udt::PacketList(PacketType::BulkAvatarData, QByteArray(), true, true)
});
auto packetList = udt::PacketList::create(PacketType::BulkAvatarData, QByteArray(), true, true);
// fill the packet list with random data according to the constant seed (so receiver can verify)
for (int i = 0; i < messageSizePackets; ++i) {