Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-25 18:55:01 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

AssetServer JSON stats

Browse source
This commit is contained in:
Atlante45 2015-09-10 16:27:37 +02:00
parent 5808658958
commit 08e0a8d705
10 changed files with 144 additions and 316 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
assignment-client/src/assets/AssetServer.cpp
View file

@ -163,3 +163,54 @@ void AssetServer::handleAssetUpload(QSharedPointer<NLPacketList> packetList, Sha
}
}
void AssetServer::sendStatsPacket() {
QJsonObject statsObject;
auto stats = DependencyManager::get<NodeList>()->sampleStatsForAllConnections();
QString baseName("AssetServer");
statsObject[baseName + ".num_connections"] = (int)stats.size();
for (const auto& stat : stats) {
QString uuid = QUuid().toString();
if (auto node = DependencyManager::get<NodeList>()->findNodeWithAddr(stat.first)) {
uuid = node->getUUID().toString();
}
QString connKey = baseName + "." + uuid + ".connection.";
QString sendKey = baseName + "." + uuid + ".sending.";
QString recvKey = baseName + "." + uuid + ".receiving.";
qDebug() << "Testing" << sendKey << recvKey;
auto endTimeMs = std::chrono::duration_cast<std::chrono::milliseconds>(stat.second.endTime);
QDateTime date = QDateTime::fromMSecsSinceEpoch(endTimeMs.count());
statsObject[connKey + "lastHeard"] = date.toString();
statsObject[connKey + "estimatedBandwith"] = stat.second.estimatedBandwith;
statsObject[connKey + "rtt"] = stat.second.rtt;
statsObject[connKey + "congestionWindowSize"] = stat.second.congestionWindowSize;
statsObject[connKey + "packetSendPeriod"] = stat.second.packetSendPeriod;
statsObject[sendKey + "sendRate"] = stat.second.sendRate;
statsObject[sendKey + "sentPackets"] = stat.second.sentPackets;
statsObject[sendKey + "receivedACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK];
statsObject[sendKey + "processedACK"] = stat.second.events[udt::ConnectionStats::Stats::ProcessedACK];
statsObject[sendKey + "receivedLightACK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedLightACK];
statsObject[sendKey + "receivedNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedNAK];
statsObject[sendKey + "receivedTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedTimeoutNAK];
statsObject[sendKey + "sentACK2"] = stat.second.events[udt::ConnectionStats::Stats::SentACK2];
statsObject[sendKey + "retransmission"] = stat.second.events[udt::ConnectionStats::Stats::Retransmission];
statsObject[recvKey + "receiveRate"] = stat.second.receiveRate;
statsObject[recvKey + "receivedPackets"] = stat.second.receivedPackets;
statsObject[recvKey + "SentACK"] = stat.second.events[udt::ConnectionStats::Stats::SentACK];
statsObject[recvKey + "SentLightACK"] = stat.second.events[udt::ConnectionStats::Stats::SentLightACK];
statsObject[recvKey + "SentNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentNAK];
statsObject[recvKey + "SentTimeoutNAK"] = stat.second.events[udt::ConnectionStats::Stats::SentTimeoutNAK];
statsObject[recvKey + "ReceivedACK2"] = stat.second.events[udt::ConnectionStats::Stats::ReceivedACK2];
statsObject[recvKey + "Duplicate"] = stat.second.events[udt::ConnectionStats::Stats::Duplicate];
}
// send off the stats packets
ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject);
}

4
assignment-client/src/assets/AssetServer.h
View file

@ -31,7 +31,9 @@ private slots:
void handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetUpload(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode);
void sendStatsPacket();
private:
static void writeError(NLPacketList* packetList, AssetServerError error);
QDir _resourcesDirectory;

356
libraries/networking/src/JSONBreakableMarshal.cpp
View file

@ -13,326 +13,76 @@
#include <QtCore/QDebug>
#include <QtCore/QJsonArray>
#include <QtCore/QJsonDocument>
#include <QtCore/QJsonObject>
QVariantMap JSONBreakableMarshal::_interpolationMap = QVariantMap();
QStringList JSONBreakableMarshal::toStringList(const QJsonValue& jsonValue, const QString& keypath) {
// setup the string list that will hold our result
QStringList result;
// figure out what type of value this is so we know how to act on it
if (jsonValue.isObject()) {
QJsonObject jsonObject = jsonValue.toObject();
// enumerate the keys of the QJsonObject
foreach(const QString& key, jsonObject.keys()) {
QJsonValue childValue = jsonObject[key];
// setup the keypath for this key
QString valueKeypath = (keypath.isEmpty() ? "" : keypath + ".") + key;
if (childValue.isObject() || childValue.isArray()) {
// recursion is required since the value is a QJsonObject or QJsonArray
result << toStringList(childValue, valueKeypath);
} else {
// no recursion required, call our toString method to get the string representation
// append the QStringList resulting from that to our QStringList
result << toString(childValue, valueKeypath);
}
}
} else if (jsonValue.isArray()) {
QJsonArray jsonArray = jsonValue.toArray();
// enumerate the elements in this QJsonArray
for (int i = 0; i < jsonArray.size(); i++) {
QJsonValue arrayValue = jsonArray[i];
// setup the keypath for this object with the array index
QString valueKeypath = QString("%1[%2]").arg(keypath).arg(i);
if (arrayValue.isObject() || arrayValue.isArray()) {
// recursion is required since the value is a QJsonObject or QJsonArray
// append the QStringList resulting from that to our QStringList
result << toStringList(arrayValue, valueKeypath);
} else {
result << toString(arrayValue, valueKeypath);
}
}
} else {
// this is a basic value, so set result to whatever toString reports in a QStringList
result = QStringList() << toString(jsonValue, keypath);
}
return result;
QByteArray JSONBreakableMarshal::toByteArray(const QJsonObject& jsonObject) {
return QJsonDocument(jsonObject).toBinaryData();
}
const QString JSON_NULL_AS_STRING = "null";
const QString JSON_TRUE_AS_STRING = "true";
const QString JSON_FALSE_AS_STRING = "false";
const QString JSON_UNDEFINED_AS_STRING = "undefined";
const QString JSON_UNKNOWN_AS_STRING = "unknown";
QString JSONBreakableMarshal::toString(const QJsonValue& jsonValue, const QString& keypath) {
// default the value as a string to unknown in case conversion fails
QString valueAsString = JSON_UNKNOWN_AS_STRING;
// ask the QJsonValue what type it is and format its value as a string accordingly
if (jsonValue.isNull()) {
valueAsString = JSON_NULL_AS_STRING;
} else if (jsonValue.isBool()) {
valueAsString = jsonValue.toBool() ? JSON_TRUE_AS_STRING : JSON_FALSE_AS_STRING;
} else if (jsonValue.isDouble()) {
valueAsString = QString::number(jsonValue.toDouble());
} else if (jsonValue.isString()) {
valueAsString = QString("\"%1\"").arg(jsonValue.toString());
} else if (jsonValue.isUndefined()) {
valueAsString = JSON_UNDEFINED_AS_STRING;
} else if (jsonValue.isArray() || jsonValue.isObject()) {
qDebug() << "JSONBreakableMarshal::toString does not handle conversion of a QJsonObject or QJsonArray."
<< "You should call JSONBreakableMarshal::toStringList instead.";
} else {
qDebug() << "Unrecognized QJsonValue - JSONBreakableMarshal cannot convert to string.";
QJsonObject JSONBreakableMarshal::fromByteArray(const QByteArray& buffer) {
auto document = QJsonDocument::fromBinaryData(buffer);
Q_ASSERT(document.isObject());
auto object = document.object();
QStringList currentKey;
for (auto key : object.keys()) {
interpolate(object[key], key);
}
return QString("%1=%2").arg(keypath, valueAsString);
}
QVariant JSONBreakableMarshal::fromString(const QString& marshalValue) {
// default the value to null
QVariant result;
// attempt to match the value with our expected strings
if (marshalValue == JSON_NULL_AS_STRING) {
// this is already our default, we don't need to do anything here
} else if (marshalValue == JSON_TRUE_AS_STRING || marshalValue == JSON_FALSE_AS_STRING) {
result = QVariant(marshalValue == JSON_TRUE_AS_STRING ? true : false);
} else if (marshalValue == JSON_UNDEFINED_AS_STRING) {
result = JSON_UNDEFINED_AS_STRING;
} else if (marshalValue == JSON_UNKNOWN_AS_STRING) {
// we weren't able to marshal this value at the other end, set it as our unknown string
result = JSON_UNKNOWN_AS_STRING;
} else {
// this might be a double, see if it converts
bool didConvert = false;
double convertResult = marshalValue.toDouble(&didConvert);
if (didConvert) {
result = convertResult;
} else {
// we need to figure out if this is a string
// use a regex to look for surrounding quotes first
const QString JSON_STRING_REGEX = "^\"([\\s\\S]*)\"$";
QRegExp stringRegex(JSON_STRING_REGEX);
if (stringRegex.indexIn(marshalValue) != -1) {
// set the result to the string value
result = stringRegex.cap(1);
} else {
// we failed to convert the value to anything, set the result to our unknown value
qDebug() << "Unrecognized output from JSONBreakableMarshal - could not convert"
<< marshalValue << "to QVariant.";
result = JSON_UNKNOWN_AS_STRING;
}
}
}
return result;
}
QVariantMap JSONBreakableMarshal::fromStringList(const QStringList& stringList) {
QVariant result = QVariantMap();
foreach(const QString& marshalString, stringList) {
// find the equality operator
int equalityIndex = marshalString.indexOf('=');
// bail on parsing if we didn't find the equality operator
if (equalityIndex != -1) {
QVariant* currentValue = &result;
// pull the key (everything left of the equality sign)
QString parentKeypath;
QString keypath = marshalString.left(equalityIndex);
// setup for array index checking
const QString ARRAY_INDEX_REGEX_STRING = "\\[(\\d+)\\]";
QRegExp arrayRegex(ARRAY_INDEX_REGEX_STRING);
// as long as we have a keypath we need to recurse downwards
while (!keypath.isEmpty()) {
parentKeypath = keypath;
int arrayBracketIndex = arrayRegex.indexIn(keypath);
// is this an array index
if (arrayBracketIndex == 0) {
// we're here because we think the current value should be an array
// if it isn't then make one
if (!currentValue->canConvert(QMetaType::QVariantList) || currentValue->isNull()) {
*currentValue = QVariantList();
}
QVariantList& currentList = *static_cast<QVariantList*>(currentValue->data());
// figure out what index we want to get the QJsonValue& for
bool didConvert = false;
int arrayIndex = arrayRegex.cap(1).toInt(&didConvert);
if (didConvert) {
// check if we need to resize the array
if (currentList.size() < arrayIndex + 1) {
for (int i = currentList.size(); i < arrayIndex + 1; i++) {
// add the null QJsonValue at this array index to get the array to the right size
currentList.push_back(QJsonValue());
}
}
// set our currentValue to the QJsonValue& from the array at this index
currentValue = &currentList[arrayIndex];
// update the keypath by bumping past the array index
keypath = keypath.mid(keypath.indexOf(']') + 1);
// check if there is a key after the array index - if so push the keypath forward by a char
if (keypath.startsWith(".")) {
keypath = keypath.mid(1);
}
} else {
qDebug() << "Failed to convert array index from keypath" << keypath << "to int. Will not add"
<< "value to resulting QJsonObject.";
break;
}
} else {
int keySeparatorIndex = keypath.indexOf('.');
// we need to figure out what the key to look at is
QString subKey = keypath;
if (keySeparatorIndex != -1 || arrayBracketIndex != -1) {
int nextBreakIndex = -1;
int nextKeypathStartIndex = -1;
if (arrayBracketIndex == -1 || (keySeparatorIndex != -1 && keySeparatorIndex < arrayBracketIndex)) {
nextBreakIndex = keySeparatorIndex;
nextKeypathStartIndex = keySeparatorIndex + 1;
} else if (keySeparatorIndex == -1 || (arrayBracketIndex != -1
&& arrayBracketIndex < keySeparatorIndex)) {
nextBreakIndex = arrayBracketIndex;
nextKeypathStartIndex = arrayBracketIndex;
} else {
qDebug() << "Unrecognized key format while trying to parse " << keypath << " - will not add"
<< "value to resulting QJsonObject.";
break;
}
// set the current key from the determined index
subKey = keypath.left(nextBreakIndex);
// update the keypath being processed
keypath = keypath.mid(nextKeypathStartIndex);
} else {
// update the keypath being processed, since we have no more separators in the keypath, it should
// be an empty string
keypath = "";
}
// we're here becuase we know the current value should be an object
// if it isn't then make it one
if (!currentValue->canConvert(QMetaType::QVariantMap) || currentValue->isNull()) {
*currentValue = QVariantMap();
}
QVariantMap& currentMap = *static_cast<QVariantMap*>(currentValue->data());
// is there a QJsonObject for this key yet?
// if not then we make it now
if (!currentMap.contains(subKey)) {
currentMap[subKey] = QVariant();
}
// change the currentValue to the QJsonValue for this key
currentValue = &currentMap[subKey];
}
}
*currentValue = fromString(marshalString.mid(equalityIndex + 1));
if (_interpolationMap.contains(parentKeypath)) {
// we expect the currentValue here to be a string, that's the key we use for interpolation
// bail if it isn't
if (currentValue->canConvert(QMetaType::QString)
&& _interpolationMap[parentKeypath].canConvert(QMetaType::QVariantMap)) {
*currentValue = _interpolationMap[parentKeypath].toMap()[currentValue->toString()];
}
}
}
}
return result.toMap();
}
QVariantMap JSONBreakableMarshal::fromStringBuffer(const QByteArray& buffer) {
// this is a packet of strings sep by null terminators - pull out each string and create a stringlist
QStringList packetList;
int currentIndex = 0;
int currentSeparator = buffer.indexOf('\0');
while (currentIndex < buffer.size() - 1) {
packetList << QString::fromUtf8(buffer.mid(currentIndex, currentSeparator));
if (currentSeparator == -1) {
// no more separators to be found, break out of here so we're not looping for nothing
break;
}
// bump the currentIndex up to the last found separator
currentIndex = currentSeparator + 1;
// find the index of the next separator, assuming this one wasn't the last one in the packet
if (currentSeparator < buffer.size() - 1) {
currentSeparator = buffer.indexOf('\0', currentIndex);
}
}
// now that we have a QStringList we use our static method to turn that into a QJsonObject
return fromStringList(packetList);
return object;
}
void JSONBreakableMarshal::addInterpolationForKey(const QString& rootKey, const QString& interpolationKey,
const QJsonValue& interpolationValue) {
const QString& interpolationValue) {
// if there is no map already beneath this key in our _interpolationMap create a QVariantMap there now
if (!_interpolationMap.contains(rootKey)) {
_interpolationMap.insert(rootKey, QVariantMap());
}
if (_interpolationMap[rootKey].canConvert(QMetaType::QVariantMap)) {
QVariantMap& mapForRootKey = *static_cast<QVariantMap*>(_interpolationMap[rootKey].data());
mapForRootKey.insert(interpolationKey, QVariant(interpolationValue));
} else {
qDebug() << "JSONBreakableMarshal::addInterpolationForKey could not convert variant at key" << rootKey
<< "to a QVariantMap. Can not add interpolation.";
auto it = _interpolationMap.find(rootKey);
if (it == _interpolationMap.end()) {
it = _interpolationMap.insert(rootKey, QVariantMap());
}
Q_ASSERT(it->canConvert(QMetaType::QVariantMap));
static_cast<QVariantMap*>(it->data())->insert(interpolationKey, interpolationValue);
}
void JSONBreakableMarshal::removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey) {
// make sure the interpolation map contains this root key and that the value is a map
auto it = _interpolationMap.find(rootKey);
if (it != _interpolationMap.end() && !it->isNull()) {
// remove the value at the interpolationKey
Q_ASSERT(it->canConvert(QMetaType::QVariantMap));
static_cast<QVariantMap*>(it->data())->remove(interpolationKey);
}
}
if (_interpolationMap.contains(rootKey)) {
QVariant& rootValue = _interpolationMap[rootKey];
if (!rootValue.isNull() && rootValue.canConvert(QMetaType::QVariantMap)) {
// remove the value at the interpolationKey
static_cast<QVariantMap*>(rootValue.data())->remove(interpolationKey);
void JSONBreakableMarshal::interpolate(QJsonValueRef currentValue, QString lastKey) {
if (currentValue.isArray()) {
auto array = currentValue.toArray();
for (int i = 0; i < array.size(); ++i) {
// pass last key and recurse array
interpolate(array[i], QString::number(i));
}
currentValue = array;
} else if (currentValue.isObject()) {
auto object = currentValue.toObject();
for (auto key : object.keys()) {
// pass last key and recurse object
interpolate(object[key], key);
}
currentValue = object;
} else if (currentValue.isString()) {
// Maybe interpolate
auto mapIt = _interpolationMap.find(lastKey);
if (mapIt != _interpolationMap.end()) {
Q_ASSERT(mapIt->canConvert(QMetaType::QVariantMap));
auto interpolationMap = mapIt->toMap();
auto result = interpolationMap.find(currentValue.toString());
if (result != interpolationMap.end()) {
// Replace value
currentValue = result->toString();
}
}
}
}

14
libraries/networking/src/JSONBreakableMarshal.h
View file

@ -21,18 +21,16 @@
class JSONBreakableMarshal {
public:
static QStringList toStringList(const QJsonValue& jsonValue, const QString& keypath);
static QString toString(const QJsonValue& jsonValue, const QString& keyPath);
static QVariant fromString(const QString& marshalValue);
static QVariantMap fromStringList(const QStringList& stringList);
static QVariantMap fromStringBuffer(const QByteArray& buffer);
static QByteArray toByteArray(const QJsonObject& jsonObject);
static QJsonObject fromByteArray(const QByteArray& buffer);
static void addInterpolationForKey(const QString& rootKey,
const QString& interpolationKey, const QJsonValue& interpolationValue);
static void addInterpolationForKey(const QString& rootKey, const QString& interpolationKey,
const QString& interpolationValue);
static void removeInterpolationForKey(const QString& rootKey, const QString& interpolationKey);
private:
static void interpolate(QJsonValueRef currentValue, QString lastKey);
static QVariantMap _interpolationMap;
};

7
libraries/networking/src/LimitedNodeList.cpp
View file

@ -845,6 +845,13 @@ void LimitedNodeList::sendPeerQueryToIceServer(const HifiSockAddr& iceServerSock
sendPacketToIceServer(PacketType::ICEServerQuery, iceServerSockAddr, clientID, peerID);
}
SharedNodePointer LimitedNodeList::findNodeWithAddr(const HifiSockAddr& addr) {
auto it = std::find_if(std::begin(_nodeHash), std::end(_nodeHash), [&](const UUIDNodePair& pair) {
return pair.second->getActiveSocket() ? (*pair.second->getActiveSocket() == addr) : false;
});
return (it != std::end(_nodeHash)) ? it->second : SharedNodePointer();
}
void LimitedNodeList::sendPacketToIceServer(PacketType packetType, const HifiSockAddr& iceServerSockAddr,
const QUuid& clientID, const QUuid& peerID) {
auto icePacket = NLPacket::create(packetType);

3
libraries/networking/src/LimitedNodeList.h
View file

@ -165,6 +165,8 @@ public:
void sendHeartbeatToIceServer(const HifiSockAddr& iceServerSockAddr);
void sendPeerQueryToIceServer(const HifiSockAddr& iceServerSockAddr, const QUuid& clientID, const QUuid& peerID);
SharedNodePointer findNodeWithAddr(const HifiSockAddr& addr);
template<typename NodeLambda>
void eachNode(NodeLambda functor) {
QReadLocker readLock(&_nodeMutex);
@ -216,6 +218,7 @@ public:
{ QReadLocker readLock(&_connectionTimeLock); return _lastConnectionTimes; }
void flagTimeForConnectionStep(ConnectionStep connectionStep);
udt::Socket::StatsVector sampleStatsForAllConnections() { return _nodeSocket.sampleStatsForAllConnections(); }
public slots:
void reset();

4
libraries/networking/src/udt/ConnectionStats.cpp
View file

@ -15,7 +15,7 @@ using namespace udt;
using namespace std::chrono;
ConnectionStats::ConnectionStats() {
auto now = duration_cast<microseconds>(high_resolution_clock::now().time_since_epoch());
auto now = duration_cast<microseconds>(system_clock::now().time_since_epoch());
_currentSample.startTime = now;
_total.startTime = now;
}
@ -24,7 +24,7 @@ ConnectionStats::Stats ConnectionStats::sample() {
Stats sample = _currentSample;
_currentSample = Stats();
auto now = duration_cast<microseconds>(high_resolution_clock::now().time_since_epoch());
auto now = duration_cast<microseconds>(system_clock::now().time_since_epoch());
sample.endTime = now;
_currentSample.startTime = now;

6
libraries/networking/src/udt/ConnectionStats.h
View file

@ -36,11 +36,13 @@ public:
SentTimeoutNAK,
ReceivedTimeoutNAK,
Retransmission,
Duplicate
Duplicate,
NumEvent
};
// construct a vector for the events of the size of our Enum - default value is zero
std::vector<int> events = std::vector<int>((int) Event::Duplicate + 1, 0);
std::vector<int> events = std::vector<int>((int) Event::NumEvent, 0);
// packet counts and sizes
int sentPackets { 0 };

10
libraries/networking/src/udt/Socket.cpp
View file

@ -325,6 +325,16 @@ ConnectionStats::Stats Socket::sampleStatsForConnection(const HifiSockAddr& dest
}
}
Socket::StatsVector Socket::sampleStatsForAllConnections() {
StatsVector result;
result.reserve(_connectionsHash.size());
for (const auto& connectionPair : _connectionsHash) {
result.emplace_back(connectionPair.first, connectionPair.second->sampleStats());
}
return result;
}
std::vector<HifiSockAddr> Socket::getConnectionSockAddrs() {
std::vector<HifiSockAddr> addr;
addr.reserve(_connectionsHash.size());

5
libraries/networking/src/udt/Socket.h
View file

@ -46,6 +46,8 @@ using PacketListHandler = std::function<void(std::unique_ptr<PacketList>)>;
class Socket : public QObject {
Q_OBJECT
public:
using StatsVector = std::vector<std::pair<HifiSockAddr, ConnectionStats::Stats>>;
Socket(QObject* object = 0);
quint16 localPort() const { return _udpSocket.localPort(); }
@ -71,6 +73,8 @@ public:
void setCongestionControlFactory(std::unique_ptr<CongestionControlVirtualFactory> ccFactory);
void messageReceived(std::unique_ptr<PacketList> packetList);
StatsVector sampleStatsForAllConnections();
public slots:
void cleanupConnection(HifiSockAddr sockAddr);
@ -86,6 +90,7 @@ private:
// privatized methods used by UDTTest - they are private since they must be called on the Socket thread
ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination);
std::vector<HifiSockAddr> getConnectionSockAddrs();
void connectToSendSignal(const HifiSockAddr& destinationAddr, QObject* receiver, const char* slot);