From c851edd0efbf003e844309960b7e6f6513bdf673 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 3 Jun 2014 16:18:40 -0700 Subject: [PATCH 01/12] added MovingMedian class, but not using it added packet type/time data collection to DatagramProcessor and post-execution prints of that info. --- libraries/networking/src/Node.cpp | 11 +++- libraries/networking/src/Node.h | 6 +- libraries/networking/src/NodeList.cpp | 18 ++++-- libraries/shared/src/MovingMedian.cpp | 87 +++++++++++++++++++++++++++ libraries/shared/src/MovingMedian.h | 26 ++++++++ 5 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 libraries/shared/src/MovingMedian.cpp create mode 100644 libraries/shared/src/MovingMedian.h diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 05f425374b..9c482b9d58 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -57,7 +57,9 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, _linkedData(NULL), _isAlive(true), _clockSkewUsec(0), - _mutex() + _mutex(), + + _clockSkewMovingMedian(31) { } @@ -133,6 +135,13 @@ float Node::getAverageKilobitsPerSecond() { } } +void Node::setClockSkewUsec(int clockSkew) { + //_clockSkewMovingMedian.updateMedian((float)clockSkew); + //_clockSkewUsec = (int)_clockSkewMovingMedian.getMedian(); + +_clockSkewUsec = clockSkew; +} + QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._type; out << node._uuid; diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index f52cda0d0d..7a8fcad0bb 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -24,6 +24,8 @@ #include "NodeData.h" #include "SimpleMovingAverage.h" +#include "MovingMedian.h" + typedef quint8 NodeType_t; namespace NodeType { @@ -94,7 +96,7 @@ public: void setPingMs(int pingMs) { _pingMs = pingMs; } int getClockSkewUsec() const { return _clockSkewUsec; } - void setClockSkewUsec(int clockSkew) { _clockSkewUsec = clockSkew; } + void setClockSkewUsec(int clockSkew); QMutex& getMutex() { return _mutex; } friend QDataStream& operator<<(QDataStream& out, const Node& node); @@ -120,6 +122,8 @@ private: int _pingMs; int _clockSkewUsec; QMutex _mutex; + + MovingMedian _clockSkewMovingMedian; }; QDebug operator<<(QDebug debug, const Node &message); diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index ebd33ef132..f91ee3df28 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -77,7 +77,7 @@ qint64 NodeList::sendStatsToDomainServer(const QJsonObject& statsObject) { return writeUnverifiedDatagram(statsPacket, _domainHandler.getSockAddr()); } -void NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode) { +int NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode) { QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); @@ -97,7 +97,10 @@ void NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode->setPingMs(pingTime / 1000); sendingNode->setClockSkewUsec(clockSkew); - + +//printf("\t\t clock skew sample: %d median: %d\n", clockSkew, sendingNode->getClockSkewUsec()); + + const bool wantDebug = false; if (wantDebug) { @@ -110,9 +113,14 @@ void NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& " othersExprectedReply: " << othersExprectedReply << "\n" << " clockSkew: " << clockSkew; } + + if (abs(clockSkew) > 1000) + printf("clockskew = %d \n", clockSkew); + +return clockSkew; } -void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet) { +int NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet) { switch (packetTypeForPacket(packet)) { case PacketTypeDomainList: { processDomainServerList(packet); @@ -152,7 +160,8 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr activateSocketFromNodeCommunication(packet, sendingNode); // set the ping time for this node for stat collection - timePingReply(packet, sendingNode); + return timePingReply(packet, sendingNode); + } break; @@ -167,6 +176,7 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr LimitedNodeList::processNodeData(senderSockAddr, packet); break; } +return 1234567890; } void NodeList::reset() { diff --git a/libraries/shared/src/MovingMedian.cpp b/libraries/shared/src/MovingMedian.cpp new file mode 100644 index 0000000000..f3cd4cbd16 --- /dev/null +++ b/libraries/shared/src/MovingMedian.cpp @@ -0,0 +1,87 @@ +#include "MovingMedian.h" +//#include "stdio.h"// DEBUG + +MovingMedian::MovingMedian(int numSamples) + : _numSamples(numSamples), + _numExistingSamples(0), + _median(0.0f) +{ + _samplesSorted = new float[numSamples]; + _sampleAges = new int[numSamples]; +} + +MovingMedian::~MovingMedian() { + delete[] _samplesSorted; + delete[] _sampleAges; +} + +void MovingMedian::updateMedian(float sample) { + +//printf("\nnew sample: %2.2f ", sample); + + // find index in _samplesSorted to insert new sample. + // if samples have not been filled yet, this will be the next empty spot + // otherwise, it will be the spot of the oldest sample + int newSampleIndex; + if (_numExistingSamples < _numSamples) { + newSampleIndex = _numExistingSamples; + _numExistingSamples++; + } + else { + for (int i = 0; i < _numExistingSamples; i++) { + if (_sampleAges[i] == _numExistingSamples - 1) { + newSampleIndex = i; + break; + } + } + } + +//printf("will be inserted at index %d\n", newSampleIndex); + + // update _sampleAges to reflect new sample (age all samples by 1) + for (int i = 0; i < _numExistingSamples; i++) { + _sampleAges[i]++; + } + + // insert new sample at that index + _samplesSorted[newSampleIndex] = sample; + _sampleAges[newSampleIndex] = 0; + + // swap new sample with neighboring elements in _samplesSorted until it's in sorted order + // try swapping up first, then down. element will only be swapped one direction. + + float neighborSample; + while (newSampleIndex < _numExistingSamples-1 && sample > (neighborSample = _samplesSorted[newSampleIndex+1])) { +//printf("\t swapping up...\n"); + _samplesSorted[newSampleIndex] = neighborSample; + _samplesSorted[newSampleIndex+1] = sample; + + _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex+1]; + _sampleAges[newSampleIndex+1] = 0; + + newSampleIndex++; + } + while (newSampleIndex > 0 && sample < (neighborSample = _samplesSorted[newSampleIndex - 1])) { +//printf("\t swapping down...\n"); + _samplesSorted[newSampleIndex] = neighborSample; + _samplesSorted[newSampleIndex - 1] = sample; + + _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex - 1]; + _sampleAges[newSampleIndex - 1] = 0; + + newSampleIndex--; + } + + + // find new median + _median = _samplesSorted[_numExistingSamples/2]; +/* +printf(" new median: %f\n", _median); + +// debug: +for (int i = 0; i < _numExistingSamples; i++) { + printf("%2.2f (%d), ", _samplesSorted[i], _sampleAges[i]); +} +printf("\n\n"); +*/ +} \ No newline at end of file diff --git a/libraries/shared/src/MovingMedian.h b/libraries/shared/src/MovingMedian.h new file mode 100644 index 0000000000..813fd42235 --- /dev/null +++ b/libraries/shared/src/MovingMedian.h @@ -0,0 +1,26 @@ + +#ifndef hifi_MovingMedian_h +#define hifi_MovingMedian_h + + +class MovingMedian { + +public: + MovingMedian(int numSamples = 11); + ~MovingMedian(); + + void updateMedian(float sample); + float getMedian() const { return _median; } + + +private: + float* _samplesSorted; + int _numSamples; + + int* _sampleAges; // _sampleAges[i] is the "age" of the sample _sampleSorted[i] (higher means older) + int _numExistingSamples; + + float _median; +}; + +#endif \ No newline at end of file From 50746a65402b3cec60f191eb0318c96961ed53a1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 3 Jun 2014 16:19:12 -0700 Subject: [PATCH 02/12] extra commit of previous commit? --- .../src/octree/OctreeSendThread.cpp | 2 +- interface/src/DatagramProcessor.cpp | 112 +++++++++++++++++- interface/src/DatagramProcessor.h | 26 ++++ interface/src/main.cpp | 23 ++++ libraries/networking/src/NodeList.h | 4 +- 5 files changed, 162 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 9e4dbcd347..f5f9a90339 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -149,7 +149,7 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes int piggyBackSize = nodeData->getPacketLength() + statsMessageLength; // If the size of the stats message and the voxel message will fit in a packet, then piggyback them - if (piggyBackSize < MAX_PACKET_SIZE) { +if (false && piggyBackSize < MAX_PACKET_SIZE) { // copy voxel message to back of stats message memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength()); diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index 56078c1a8d..47836638dd 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -25,7 +25,39 @@ DatagramProcessor::DatagramProcessor(QObject* parent) : } + +// DEBUG + +int DatagramProcessor::skewsI[10000]; +int DatagramProcessor::S = 0; + +unsigned char DatagramProcessor::typesI[10000]; +int DatagramProcessor::diffsI[10000]; +int DatagramProcessor::I = 0; + + + +quint64 DatagramProcessor::prevTime = 0; + +unsigned char DatagramProcessor::typesA[100]; +quint64 DatagramProcessor::timesA[100]; +int DatagramProcessor::A = 1; + +unsigned char DatagramProcessor::typesB[100]; +quint64 DatagramProcessor::timesB[100]; +int DatagramProcessor::B = 1; + +unsigned char* DatagramProcessor::currTypes = typesA; +unsigned char* DatagramProcessor::prevTypes = typesB; +quint64* DatagramProcessor::currTimes = timesA; +quint64* DatagramProcessor::prevTimes = timesB; +int* DatagramProcessor::currN = &A; +int* DatagramProcessor::prevN = &B; + + + void DatagramProcessor::processDatagrams() { + PerformanceWarning warn(Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings), "DatagramProcessor::processDatagrams()"); @@ -35,6 +67,30 @@ void DatagramProcessor::processDatagrams() { Application* application = Application::getInstance(); NodeList* nodeList = NodeList::getInstance(); + + + +prevTime = prevTimes[*prevN-1]; + +// swap +unsigned char* temp = currTypes; +currTypes = prevTypes; +prevTypes = temp; +// swap +quint64* temp2 = currTimes; +currTimes = prevTimes; +prevTimes = temp2; +// swap +int* temp3 = currN; +currN = prevN; +prevN = temp3; + +// reset +*currN = 0; + +int skew = 0; + + while (NodeList::getInstance()->getNodeSocket().hasPendingDatagrams()) { incomingPacket.resize(nodeList->getNodeSocket().pendingDatagramSize()); @@ -45,8 +101,15 @@ void DatagramProcessor::processDatagrams() { _byteCount += incomingPacket.size(); if (nodeList->packetVersionAndHashMatch(incomingPacket)) { + +PacketType type = packetTypeForPacket(incomingPacket); +currTimes[*currN] = usecTimestampNow(); +currTypes[*currN] = (unsigned char)type; +(*currN)++; + + // only process this packet if we have a match on the packet version - switch (packetTypeForPacket(incomingPacket)) { + switch (type) { //packetTypeForPacket(incomingPacket)) { case PacketTypeMixedAudio: QMetaObject::invokeMethod(&application->_audio, "addReceivedAudioToBuffer", Qt::QueuedConnection, Q_ARG(QByteArray, incomingPacket)); @@ -146,9 +209,54 @@ void DatagramProcessor::processDatagrams() { break; } default: - nodeList->processNodeData(senderSockAddr, incomingPacket); + int s = nodeList->processNodeData(senderSockAddr, incomingPacket); + if (s!=1234567890) + skew = s; break; } } } + + if (abs(skew) > 1000) { + + printf("large skew! %d ----------------------------\n", skew); + + skewsI[S++] = skew; + + /* + printf("prev:::::::::::::::::::::::::::::::::::::\n"); + + printf("\t type: %d time: %llu diff: %llu\n", prevTypes[0], prevTimes[0] % 100000000, prevTimes[0] - prevTime); + for (int i = 1; i < *prevN; i++) { + printf("\t type: %d time: %llu diff: %llu\n", prevTypes[i], prevTimes[i] % 100000000, prevTimes[i] - prevTimes[i - 1]); + } + + printf("curr:::::::::::::::::::::::::::::::::::::\n"); + + printf("\t type: %d time: %llu diff: %llu\n", currTypes[0], currTimes[0] % 100000000, currTimes[0] - prevTimes[*prevN - 1]); + for (int i = 1; i < *currN; i++) { + printf("\t type: %d time: %llu diff: %llu\n", currTypes[i], currTimes[i] % 100000000, currTimes[i] - currTimes[i - 1]); + }*/ + + diffsI[I++] = -2; // prev marker + + typesI[I] = prevTypes[0]; + diffsI[I++] = prevTimes[0] - prevTime; + for (int i = 1; i < *prevN; i++) { + typesI[I] = prevTypes[i]; + diffsI[I++] = prevTimes[i] - prevTimes[i - 1]; + } + + + diffsI[I++] = -1; // curr marker + + typesI[I] = currTypes[0]; + diffsI[I++] = currTimes[0] - prevTimes[*prevN - 1]; + for (int i = 1; i < *currN; i++) { + typesI[I] = currTypes[i]; + diffsI[I++] = currTimes[i] - currTimes[i - 1]; + } + } + + skew = 0; } diff --git a/interface/src/DatagramProcessor.h b/interface/src/DatagramProcessor.h index 7d337ec02b..245b2235b4 100644 --- a/interface/src/DatagramProcessor.h +++ b/interface/src/DatagramProcessor.h @@ -29,6 +29,32 @@ public slots: private: int _packetCount; int _byteCount; + +public: + // DEBUG + + static int skewsI[]; + static int S; + + static unsigned char typesI[]; + static int diffsI[]; + static int I; + + + static quint64 prevTime; + + static unsigned char typesA[]; + static quint64 timesA[]; + static int A; + + static unsigned char typesB[]; + static quint64 timesB[]; + static int B; + + + static unsigned char* currTypes, *prevTypes; + static quint64* currTimes, *prevTimes; + static int* currN, *prevN; }; #endif // hifi_DatagramProcessor_h diff --git a/interface/src/main.cpp b/interface/src/main.cpp index 2bb0633f24..59e0b0b3dd 100644 --- a/interface/src/main.cpp +++ b/interface/src/main.cpp @@ -15,6 +15,9 @@ #include #include +// DEBUG!!!!!! +#include "DatagramProcessor.h" + int main(int argc, const char * argv[]) { QElapsedTimer startupTime; startupTime.start(); @@ -43,5 +46,25 @@ int main(int argc, const char * argv[]) { exitCode = app.exec(); } qDebug("Normal exit."); + +int s = 0; +for (int i = 0; i < DatagramProcessor::I; i++) { + + switch (DatagramProcessor::diffsI[i]) { + case -2: + printf("\nskew: %d\n", DatagramProcessor::skewsI[s++]); + printf("prev:::::::::::::::::::::::::::::::\n"); + break; + case -1: + printf("curr:::::::::::::::::::::::::::::::\n"); + break; + default: + printf("\t type: %d diff: %d\n", DatagramProcessor::typesI[i], DatagramProcessor::diffsI[i]); + break; + } +} + + + return exitCode; } diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index af0bfeb368..8609dce589 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -63,7 +63,7 @@ public: void addSetOfNodeTypesToNodeInterestSet(const NodeSet& setOfNodeTypes); void resetNodeInterestSet() { _nodeTypesOfInterest.clear(); } - void processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet); +int processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet); int processDomainServerList(const QByteArray& packet); @@ -95,7 +95,7 @@ private: void processDomainServerAuthRequest(const QByteArray& packet); void requestAuthForDomainServer(); void activateSocketFromNodeCommunication(const QByteArray& packet, const SharedNodePointer& sendingNode); - void timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode); +int timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode); NodeType_t _ownerType; NodeSet _nodeTypesOfInterest; From 12ac6105f1fe373c74339b9d3c2fad7e2b7466b1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 10:59:32 -0700 Subject: [PATCH 03/12] changed MovingMedian to MovingPercentile --- interface/src/DatagramProcessor.cpp | 6 ++-- libraries/networking/src/LimitedNodeList.cpp | 25 +++++++++++++++- libraries/networking/src/Node.cpp | 9 +++--- libraries/networking/src/Node.h | 4 +-- libraries/networking/src/NodeList.cpp | 11 ++++--- libraries/shared/src/MovingMedian.h | 26 ----------------- ...{MovingMedian.cpp => MovingPercentile.cpp} | 22 +++++++++----- libraries/shared/src/MovingPercentile.h | 29 +++++++++++++++++++ 8 files changed, 83 insertions(+), 49 deletions(-) delete mode 100644 libraries/shared/src/MovingMedian.h rename libraries/shared/src/{MovingMedian.cpp => MovingPercentile.cpp} (81%) create mode 100644 libraries/shared/src/MovingPercentile.h diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index 47836638dd..898c4dcd53 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -90,7 +90,6 @@ prevN = temp3; int skew = 0; - while (NodeList::getInstance()->getNodeSocket().hasPendingDatagrams()) { incomingPacket.resize(nodeList->getNodeSocket().pendingDatagramSize()); @@ -210,14 +209,15 @@ currTypes[*currN] = (unsigned char)type; } default: int s = nodeList->processNodeData(senderSockAddr, incomingPacket); - if (s!=1234567890) + if (s!=1234567890 && abs(s) > abs(skew)) skew = s; break; } } } - if (abs(skew) > 1000) { + + if (abs(skew) > 3000) { printf("large skew! %d ----------------------------\n", skew); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index c0d7941edf..cc11c2d322 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -151,6 +151,10 @@ void LimitedNodeList::changeSendSocketBufferSize(int numSendBytes) { } bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { + +quint64 start = usecTimestampNow(); +quint64 end; + PacketType checkType = packetTypeForPacket(packet); int numPacketTypeBytes = numBytesArithmeticCodingFromBuffer(packet.data()); @@ -169,6 +173,11 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { versionDebugSuppressMap.insert(senderUUID, checkType); } + + if ((end=usecTimestampNow()) - start > 100) { + printf("\t\t\t\t version and hash match long diff: %d\n", end-start); + } + return false; } @@ -178,6 +187,11 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { if (sendingNode) { // check if the md5 hash in the header matches the hash we would expect if (hashFromPacketHeader(packet) == hashForPacketAndConnectionUUID(packet, sendingNode->getConnectionSecret())) { + + if ((end = usecTimestampNow()) - start > 100) { + printf("\t\t\t\t version and hash match long diff: %d\n", end - start); + } + return true; } else { qDebug() << "Packet hash mismatch on" << checkType << "- Sender" @@ -188,9 +202,18 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { << uuidFromPacketHeader(packet); } } else { + + if ((end = usecTimestampNow()) - start > 100) { + printf("\t\t\t\t version and hash match long diff: %d\n", end - start); + } + return true; } - + + + if ((end = usecTimestampNow()) - start > 100) { + printf("\t\t\t\t version and hash match long diff: %d\n", end - start); + } return false; } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 9c482b9d58..b241031e2d 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -58,8 +58,7 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, _isAlive(true), _clockSkewUsec(0), _mutex(), - - _clockSkewMovingMedian(31) + _clockSkewMovingPercentile(30, 0.8f) // moving upper quartile of 21 samples { } @@ -136,10 +135,10 @@ float Node::getAverageKilobitsPerSecond() { } void Node::setClockSkewUsec(int clockSkew) { - //_clockSkewMovingMedian.updateMedian((float)clockSkew); - //_clockSkewUsec = (int)_clockSkewMovingMedian.getMedian(); + _clockSkewMovingPercentile.updatePercentile((float)clockSkew); + _clockSkewUsec = (int)_clockSkewMovingPercentile.getValueAtPercentile(); -_clockSkewUsec = clockSkew; + //_clockSkewUsec = clockSkew; } QDataStream& operator<<(QDataStream& out, const Node& node) { diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 7a8fcad0bb..033ae01380 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -24,7 +24,7 @@ #include "NodeData.h" #include "SimpleMovingAverage.h" -#include "MovingMedian.h" +#include "MovingPercentile.h" typedef quint8 NodeType_t; @@ -123,7 +123,7 @@ private: int _clockSkewUsec; QMutex _mutex; - MovingMedian _clockSkewMovingMedian; + MovingPercentile _clockSkewMovingPercentile; }; QDebug operator<<(QDebug debug, const Node &message); diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index f91ee3df28..43b4f56361 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -98,7 +98,7 @@ int NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& s sendingNode->setPingMs(pingTime / 1000); sendingNode->setClockSkewUsec(clockSkew); -//printf("\t\t clock skew sample: %d median: %d\n", clockSkew, sendingNode->getClockSkewUsec()); +printf("\t\t clock skew sample: %d val at percentile: %d\n", clockSkew, sendingNode->getClockSkewUsec()); const bool wantDebug = false; @@ -114,8 +114,8 @@ int NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& s " clockSkew: " << clockSkew; } - if (abs(clockSkew) > 1000) - printf("clockskew = %d \n", clockSkew); +///if (abs(clockSkew) > 1000) +//printf("clockskew = %d \n", clockSkew); return clockSkew; } @@ -489,8 +489,11 @@ QByteArray NodeList::constructPingReplyPacket(const QByteArray& pingPacket) { QByteArray replyPacket = byteArrayWithPopulatedHeader(PacketTypePingReply); QDataStream packetStream(&replyPacket, QIODevice::Append); - packetStream << typeFromOriginalPing << timeFromOriginalPing << usecTimestampNow(); +quint64 now; + packetStream << typeFromOriginalPing << timeFromOriginalPing << (now = usecTimestampNow()); + +printf("\n>>>>>>>> recv ping: %llu reply: %llu diff: %lld\n", timeFromOriginalPing, now, (qint64)now-(qint64)timeFromOriginalPing); return replyPacket; } diff --git a/libraries/shared/src/MovingMedian.h b/libraries/shared/src/MovingMedian.h deleted file mode 100644 index 813fd42235..0000000000 --- a/libraries/shared/src/MovingMedian.h +++ /dev/null @@ -1,26 +0,0 @@ - -#ifndef hifi_MovingMedian_h -#define hifi_MovingMedian_h - - -class MovingMedian { - -public: - MovingMedian(int numSamples = 11); - ~MovingMedian(); - - void updateMedian(float sample); - float getMedian() const { return _median; } - - -private: - float* _samplesSorted; - int _numSamples; - - int* _sampleAges; // _sampleAges[i] is the "age" of the sample _sampleSorted[i] (higher means older) - int _numExistingSamples; - - float _median; -}; - -#endif \ No newline at end of file diff --git a/libraries/shared/src/MovingMedian.cpp b/libraries/shared/src/MovingPercentile.cpp similarity index 81% rename from libraries/shared/src/MovingMedian.cpp rename to libraries/shared/src/MovingPercentile.cpp index f3cd4cbd16..063fd641e9 100644 --- a/libraries/shared/src/MovingMedian.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -1,21 +1,23 @@ -#include "MovingMedian.h" +#include "MovingPercentile.h" //#include "stdio.h"// DEBUG -MovingMedian::MovingMedian(int numSamples) +MovingPercentile::MovingPercentile(int numSamples, float percentile) : _numSamples(numSamples), + _percentile(percentile), _numExistingSamples(0), - _median(0.0f) + _valueAtPercentile(0.0f), + _indexOfPercentile(0) { _samplesSorted = new float[numSamples]; _sampleAges = new int[numSamples]; } -MovingMedian::~MovingMedian() { +MovingPercentile::~MovingPercentile() { delete[] _samplesSorted; delete[] _sampleAges; } -void MovingMedian::updateMedian(float sample) { +void MovingPercentile::updatePercentile(float sample) { //printf("\nnew sample: %2.2f ", sample); @@ -24,8 +26,13 @@ void MovingMedian::updateMedian(float sample) { // otherwise, it will be the spot of the oldest sample int newSampleIndex; if (_numExistingSamples < _numSamples) { + newSampleIndex = _numExistingSamples; _numExistingSamples++; + + // update _indexOfPercentile + float index = _percentile * (float)(_numExistingSamples - 1); + _indexOfPercentile = (int)(index + 0.5f); // round to int } else { for (int i = 0; i < _numExistingSamples; i++) { @@ -72,9 +79,8 @@ void MovingMedian::updateMedian(float sample) { newSampleIndex--; } - - // find new median - _median = _samplesSorted[_numExistingSamples/2]; + // find new value at percentile + _valueAtPercentile = _samplesSorted[_indexOfPercentile]; /* printf(" new median: %f\n", _median); diff --git a/libraries/shared/src/MovingPercentile.h b/libraries/shared/src/MovingPercentile.h new file mode 100644 index 0000000000..374b039985 --- /dev/null +++ b/libraries/shared/src/MovingPercentile.h @@ -0,0 +1,29 @@ + +#ifndef hifi_MovingPercentile_h +#define hifi_MovingPercentile_h + + +class MovingPercentile { + +public: + MovingPercentile(int numSamples, float percentile = 0.5f); + ~MovingPercentile(); + + void updatePercentile(float sample); + float getValueAtPercentile() const { return _valueAtPercentile; } + + +private: + const int _numSamples; + const float _percentile; + + float* _samplesSorted; + int* _sampleAges; // _sampleAges[i] is the "age" of the sample at _sampleSorted[i] (higher means older) + int _numExistingSamples; + + float _valueAtPercentile; + + int _indexOfPercentile; +}; + +#endif \ No newline at end of file From 6f4593911365f1e55f14e7f385158a51caf57a4f Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 11:02:31 -0700 Subject: [PATCH 04/12] re-enabled piggy-backing in OctreeSendThread --- assignment-client/src/octree/OctreeSendThread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index f5f9a90339..9e4dbcd347 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -149,7 +149,7 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes int piggyBackSize = nodeData->getPacketLength() + statsMessageLength; // If the size of the stats message and the voxel message will fit in a packet, then piggyback them -if (false && piggyBackSize < MAX_PACKET_SIZE) { + if (piggyBackSize < MAX_PACKET_SIZE) { // copy voxel message to back of stats message memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength()); From b452d084955a4be20135e8c141af9d9f94136196 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 11:20:29 -0700 Subject: [PATCH 05/12] cleaned up debug statements also reorganized MovingPercentile a little bit --- interface/src/DatagramProcessor.cpp | 111 +------------------ interface/src/DatagramProcessor.h | 26 ----- interface/src/main.cpp | 20 ---- libraries/networking/src/LimitedNodeList.cpp | 14 --- libraries/networking/src/Node.cpp | 8 +- libraries/networking/src/Node.h | 3 +- libraries/networking/src/NodeList.cpp | 23 +--- libraries/networking/src/NodeList.h | 4 +- libraries/shared/src/MovingPercentile.cpp | 47 ++------ libraries/shared/src/MovingPercentile.h | 2 +- 10 files changed, 26 insertions(+), 232 deletions(-) diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index 898c4dcd53..ccbd793e6b 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -25,39 +25,7 @@ DatagramProcessor::DatagramProcessor(QObject* parent) : } - -// DEBUG - -int DatagramProcessor::skewsI[10000]; -int DatagramProcessor::S = 0; - -unsigned char DatagramProcessor::typesI[10000]; -int DatagramProcessor::diffsI[10000]; -int DatagramProcessor::I = 0; - - - -quint64 DatagramProcessor::prevTime = 0; - -unsigned char DatagramProcessor::typesA[100]; -quint64 DatagramProcessor::timesA[100]; -int DatagramProcessor::A = 1; - -unsigned char DatagramProcessor::typesB[100]; -quint64 DatagramProcessor::timesB[100]; -int DatagramProcessor::B = 1; - -unsigned char* DatagramProcessor::currTypes = typesA; -unsigned char* DatagramProcessor::prevTypes = typesB; -quint64* DatagramProcessor::currTimes = timesA; -quint64* DatagramProcessor::prevTimes = timesB; -int* DatagramProcessor::currN = &A; -int* DatagramProcessor::prevN = &B; - - - void DatagramProcessor::processDatagrams() { - PerformanceWarning warn(Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings), "DatagramProcessor::processDatagrams()"); @@ -68,29 +36,6 @@ void DatagramProcessor::processDatagrams() { Application* application = Application::getInstance(); NodeList* nodeList = NodeList::getInstance(); - - -prevTime = prevTimes[*prevN-1]; - -// swap -unsigned char* temp = currTypes; -currTypes = prevTypes; -prevTypes = temp; -// swap -quint64* temp2 = currTimes; -currTimes = prevTimes; -prevTimes = temp2; -// swap -int* temp3 = currN; -currN = prevN; -prevN = temp3; - -// reset -*currN = 0; - -int skew = 0; - - while (NodeList::getInstance()->getNodeSocket().hasPendingDatagrams()) { incomingPacket.resize(nodeList->getNodeSocket().pendingDatagramSize()); nodeList->getNodeSocket().readDatagram(incomingPacket.data(), incomingPacket.size(), @@ -101,14 +46,8 @@ int skew = 0; if (nodeList->packetVersionAndHashMatch(incomingPacket)) { -PacketType type = packetTypeForPacket(incomingPacket); -currTimes[*currN] = usecTimestampNow(); -currTypes[*currN] = (unsigned char)type; -(*currN)++; - - // only process this packet if we have a match on the packet version - switch (type) { //packetTypeForPacket(incomingPacket)) { + switch (packetTypeForPacket(incomingPacket)) { case PacketTypeMixedAudio: QMetaObject::invokeMethod(&application->_audio, "addReceivedAudioToBuffer", Qt::QueuedConnection, Q_ARG(QByteArray, incomingPacket)); @@ -208,55 +147,9 @@ currTypes[*currN] = (unsigned char)type; break; } default: - int s = nodeList->processNodeData(senderSockAddr, incomingPacket); - if (s!=1234567890 && abs(s) > abs(skew)) - skew = s; + nodeList->processNodeData(senderSockAddr, incomingPacket); break; } } } - - - if (abs(skew) > 3000) { - - printf("large skew! %d ----------------------------\n", skew); - - skewsI[S++] = skew; - - /* - printf("prev:::::::::::::::::::::::::::::::::::::\n"); - - printf("\t type: %d time: %llu diff: %llu\n", prevTypes[0], prevTimes[0] % 100000000, prevTimes[0] - prevTime); - for (int i = 1; i < *prevN; i++) { - printf("\t type: %d time: %llu diff: %llu\n", prevTypes[i], prevTimes[i] % 100000000, prevTimes[i] - prevTimes[i - 1]); - } - - printf("curr:::::::::::::::::::::::::::::::::::::\n"); - - printf("\t type: %d time: %llu diff: %llu\n", currTypes[0], currTimes[0] % 100000000, currTimes[0] - prevTimes[*prevN - 1]); - for (int i = 1; i < *currN; i++) { - printf("\t type: %d time: %llu diff: %llu\n", currTypes[i], currTimes[i] % 100000000, currTimes[i] - currTimes[i - 1]); - }*/ - - diffsI[I++] = -2; // prev marker - - typesI[I] = prevTypes[0]; - diffsI[I++] = prevTimes[0] - prevTime; - for (int i = 1; i < *prevN; i++) { - typesI[I] = prevTypes[i]; - diffsI[I++] = prevTimes[i] - prevTimes[i - 1]; - } - - - diffsI[I++] = -1; // curr marker - - typesI[I] = currTypes[0]; - diffsI[I++] = currTimes[0] - prevTimes[*prevN - 1]; - for (int i = 1; i < *currN; i++) { - typesI[I] = currTypes[i]; - diffsI[I++] = currTimes[i] - currTimes[i - 1]; - } - } - - skew = 0; } diff --git a/interface/src/DatagramProcessor.h b/interface/src/DatagramProcessor.h index 245b2235b4..7d337ec02b 100644 --- a/interface/src/DatagramProcessor.h +++ b/interface/src/DatagramProcessor.h @@ -29,32 +29,6 @@ public slots: private: int _packetCount; int _byteCount; - -public: - // DEBUG - - static int skewsI[]; - static int S; - - static unsigned char typesI[]; - static int diffsI[]; - static int I; - - - static quint64 prevTime; - - static unsigned char typesA[]; - static quint64 timesA[]; - static int A; - - static unsigned char typesB[]; - static quint64 timesB[]; - static int B; - - - static unsigned char* currTypes, *prevTypes; - static quint64* currTimes, *prevTimes; - static int* currN, *prevN; }; #endif // hifi_DatagramProcessor_h diff --git a/interface/src/main.cpp b/interface/src/main.cpp index 59e0b0b3dd..7e5b539eb1 100644 --- a/interface/src/main.cpp +++ b/interface/src/main.cpp @@ -46,25 +46,5 @@ int main(int argc, const char * argv[]) { exitCode = app.exec(); } qDebug("Normal exit."); - -int s = 0; -for (int i = 0; i < DatagramProcessor::I; i++) { - - switch (DatagramProcessor::diffsI[i]) { - case -2: - printf("\nskew: %d\n", DatagramProcessor::skewsI[s++]); - printf("prev:::::::::::::::::::::::::::::::\n"); - break; - case -1: - printf("curr:::::::::::::::::::::::::::::::\n"); - break; - default: - printf("\t type: %d diff: %d\n", DatagramProcessor::typesI[i], DatagramProcessor::diffsI[i]); - break; - } -} - - - return exitCode; } diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index cc11c2d322..3823ddc426 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -172,11 +172,6 @@ quint64 end; versionDebugSuppressMap.insert(senderUUID, checkType); } - - - if ((end=usecTimestampNow()) - start > 100) { - printf("\t\t\t\t version and hash match long diff: %d\n", end-start); - } return false; } @@ -187,11 +182,6 @@ quint64 end; if (sendingNode) { // check if the md5 hash in the header matches the hash we would expect if (hashFromPacketHeader(packet) == hashForPacketAndConnectionUUID(packet, sendingNode->getConnectionSecret())) { - - if ((end = usecTimestampNow()) - start > 100) { - printf("\t\t\t\t version and hash match long diff: %d\n", end - start); - } - return true; } else { qDebug() << "Packet hash mismatch on" << checkType << "- Sender" @@ -210,10 +200,6 @@ quint64 end; return true; } - - if ((end = usecTimestampNow()) - start > 100) { - printf("\t\t\t\t version and hash match long diff: %d\n", end - start); - } return false; } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index b241031e2d..360172efde 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -58,7 +58,7 @@ Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, _isAlive(true), _clockSkewUsec(0), _mutex(), - _clockSkewMovingPercentile(30, 0.8f) // moving upper quartile of 21 samples + _clockSkewMovingPercentile(30, 0.8f) // moving 80th percentile of 30 samples { } @@ -134,11 +134,9 @@ float Node::getAverageKilobitsPerSecond() { } } -void Node::setClockSkewUsec(int clockSkew) { - _clockSkewMovingPercentile.updatePercentile((float)clockSkew); +void Node::updateClockSkewUsec(int clockSkewSample) { + _clockSkewMovingPercentile.updatePercentile((float)clockSkewSample); _clockSkewUsec = (int)_clockSkewMovingPercentile.getValueAtPercentile(); - - //_clockSkewUsec = clockSkew; } QDataStream& operator<<(QDataStream& out, const Node& node) { diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 033ae01380..74f2fdfb70 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -96,7 +96,7 @@ public: void setPingMs(int pingMs) { _pingMs = pingMs; } int getClockSkewUsec() const { return _clockSkewUsec; } - void setClockSkewUsec(int clockSkew); + void updateClockSkewUsec(int clockSkewSample); QMutex& getMutex() { return _mutex; } friend QDataStream& operator<<(QDataStream& out, const Node& node); @@ -122,7 +122,6 @@ private: int _pingMs; int _clockSkewUsec; QMutex _mutex; - MovingPercentile _clockSkewMovingPercentile; }; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 43b4f56361..9a298ce26c 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -77,7 +77,7 @@ qint64 NodeList::sendStatsToDomainServer(const QJsonObject& statsObject) { return writeUnverifiedDatagram(statsPacket, _domainHandler.getSockAddr()); } -int NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode) { +void NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode) { QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); @@ -96,10 +96,7 @@ int NodeList::timePingReply(const QByteArray& packet, const SharedNodePointer& s int clockSkew = othersReplyTime - othersExprectedReply; sendingNode->setPingMs(pingTime / 1000); - sendingNode->setClockSkewUsec(clockSkew); - -printf("\t\t clock skew sample: %d val at percentile: %d\n", clockSkew, sendingNode->getClockSkewUsec()); - + sendingNode->updateClockSkewUsec(clockSkew); const bool wantDebug = false; @@ -113,14 +110,9 @@ printf("\t\t clock skew sample: %d val at percentile: %d\n", clockSkew, sending " othersExprectedReply: " << othersExprectedReply << "\n" << " clockSkew: " << clockSkew; } - -///if (abs(clockSkew) > 1000) -//printf("clockskew = %d \n", clockSkew); - -return clockSkew; } -int NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet) { +void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet) { switch (packetTypeForPacket(packet)) { case PacketTypeDomainList: { processDomainServerList(packet); @@ -160,8 +152,7 @@ int NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArr activateSocketFromNodeCommunication(packet, sendingNode); // set the ping time for this node for stat collection - return timePingReply(packet, sendingNode); - + timePingReply(packet, sendingNode); } break; @@ -176,7 +167,6 @@ int NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteArr LimitedNodeList::processNodeData(senderSockAddr, packet); break; } -return 1234567890; } void NodeList::reset() { @@ -489,11 +479,8 @@ QByteArray NodeList::constructPingReplyPacket(const QByteArray& pingPacket) { QByteArray replyPacket = byteArrayWithPopulatedHeader(PacketTypePingReply); QDataStream packetStream(&replyPacket, QIODevice::Append); -quint64 now; - packetStream << typeFromOriginalPing << timeFromOriginalPing << (now = usecTimestampNow()); + packetStream << typeFromOriginalPing << timeFromOriginalPing << usecTimestampNow(); - -printf("\n>>>>>>>> recv ping: %llu reply: %llu diff: %lld\n", timeFromOriginalPing, now, (qint64)now-(qint64)timeFromOriginalPing); return replyPacket; } diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index 8609dce589..af0bfeb368 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -63,7 +63,7 @@ public: void addSetOfNodeTypesToNodeInterestSet(const NodeSet& setOfNodeTypes); void resetNodeInterestSet() { _nodeTypesOfInterest.clear(); } -int processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet); + void processNodeData(const HifiSockAddr& senderSockAddr, const QByteArray& packet); int processDomainServerList(const QByteArray& packet); @@ -95,7 +95,7 @@ private: void processDomainServerAuthRequest(const QByteArray& packet); void requestAuthForDomainServer(); void activateSocketFromNodeCommunication(const QByteArray& packet, const SharedNodePointer& sendingNode); -int timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode); + void timePingReply(const QByteArray& packet, const SharedNodePointer& sendingNode); NodeType_t _ownerType; NodeSet _nodeTypesOfInterest; diff --git a/libraries/shared/src/MovingPercentile.cpp b/libraries/shared/src/MovingPercentile.cpp index 063fd641e9..ba2b919d32 100644 --- a/libraries/shared/src/MovingPercentile.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -1,5 +1,4 @@ #include "MovingPercentile.h" -//#include "stdio.h"// DEBUG MovingPercentile::MovingPercentile(int numSamples, float percentile) : _numSamples(numSamples), @@ -19,14 +18,15 @@ MovingPercentile::~MovingPercentile() { void MovingPercentile::updatePercentile(float sample) { -//printf("\nnew sample: %2.2f ", sample); + // age all current samples by 1 + for (int i = 0; i < _numExistingSamples; i++) { + _sampleAges[i]++; + } // find index in _samplesSorted to insert new sample. - // if samples have not been filled yet, this will be the next empty spot - // otherwise, it will be the spot of the oldest sample int newSampleIndex; if (_numExistingSamples < _numSamples) { - + // if samples have not been filled yet, this will be the next empty spot newSampleIndex = _numExistingSamples; _numExistingSamples++; @@ -35,19 +35,9 @@ void MovingPercentile::updatePercentile(float sample) { _indexOfPercentile = (int)(index + 0.5f); // round to int } else { - for (int i = 0; i < _numExistingSamples; i++) { - if (_sampleAges[i] == _numExistingSamples - 1) { - newSampleIndex = i; - break; - } - } - } - -//printf("will be inserted at index %d\n", newSampleIndex); - - // update _sampleAges to reflect new sample (age all samples by 1) - for (int i = 0; i < _numExistingSamples; i++) { - _sampleAges[i]++; + // if samples have been filled, it will be the spot of the oldest sample + newSampleIndex = 0; + while (_sampleAges[newSampleIndex] != _numExistingSamples) { newSampleIndex++; } } // insert new sample at that index @@ -56,11 +46,8 @@ void MovingPercentile::updatePercentile(float sample) { // swap new sample with neighboring elements in _samplesSorted until it's in sorted order // try swapping up first, then down. element will only be swapped one direction. - - float neighborSample; - while (newSampleIndex < _numExistingSamples-1 && sample > (neighborSample = _samplesSorted[newSampleIndex+1])) { -//printf("\t swapping up...\n"); - _samplesSorted[newSampleIndex] = neighborSample; + while (newSampleIndex < _numExistingSamples-1 && sample > _samplesSorted[newSampleIndex+1]) { + _samplesSorted[newSampleIndex] = _samplesSorted[newSampleIndex + 1]; _samplesSorted[newSampleIndex+1] = sample; _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex+1]; @@ -68,9 +55,8 @@ void MovingPercentile::updatePercentile(float sample) { newSampleIndex++; } - while (newSampleIndex > 0 && sample < (neighborSample = _samplesSorted[newSampleIndex - 1])) { -//printf("\t swapping down...\n"); - _samplesSorted[newSampleIndex] = neighborSample; + while (newSampleIndex > 0 && sample < _samplesSorted[newSampleIndex - 1]) { + _samplesSorted[newSampleIndex] = _samplesSorted[newSampleIndex - 1]; _samplesSorted[newSampleIndex - 1] = sample; _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex - 1]; @@ -81,13 +67,4 @@ void MovingPercentile::updatePercentile(float sample) { // find new value at percentile _valueAtPercentile = _samplesSorted[_indexOfPercentile]; -/* -printf(" new median: %f\n", _median); - -// debug: -for (int i = 0; i < _numExistingSamples; i++) { - printf("%2.2f (%d), ", _samplesSorted[i], _sampleAges[i]); } -printf("\n\n"); -*/ -} \ No newline at end of file diff --git a/libraries/shared/src/MovingPercentile.h b/libraries/shared/src/MovingPercentile.h index 374b039985..d4a43363e1 100644 --- a/libraries/shared/src/MovingPercentile.h +++ b/libraries/shared/src/MovingPercentile.h @@ -26,4 +26,4 @@ private: int _indexOfPercentile; }; -#endif \ No newline at end of file +#endif From 25e2f28b024f15967940ab56a338359da2034a05 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 11:30:35 -0700 Subject: [PATCH 06/12] more cleanup --- interface/src/DatagramProcessor.cpp | 3 +-- interface/src/main.cpp | 3 --- libraries/networking/src/LimitedNodeList.cpp | 11 +---------- libraries/shared/src/MovingPercentile.cpp | 14 ++++++++++++-- libraries/shared/src/MovingPercentile.h | 10 +++++++++- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index ccbd793e6b..56078c1a8d 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -35,7 +35,7 @@ void DatagramProcessor::processDatagrams() { Application* application = Application::getInstance(); NodeList* nodeList = NodeList::getInstance(); - + while (NodeList::getInstance()->getNodeSocket().hasPendingDatagrams()) { incomingPacket.resize(nodeList->getNodeSocket().pendingDatagramSize()); nodeList->getNodeSocket().readDatagram(incomingPacket.data(), incomingPacket.size(), @@ -45,7 +45,6 @@ void DatagramProcessor::processDatagrams() { _byteCount += incomingPacket.size(); if (nodeList->packetVersionAndHashMatch(incomingPacket)) { - // only process this packet if we have a match on the packet version switch (packetTypeForPacket(incomingPacket)) { case PacketTypeMixedAudio: diff --git a/interface/src/main.cpp b/interface/src/main.cpp index 7e5b539eb1..2bb0633f24 100644 --- a/interface/src/main.cpp +++ b/interface/src/main.cpp @@ -15,9 +15,6 @@ #include #include -// DEBUG!!!!!! -#include "DatagramProcessor.h" - int main(int argc, const char * argv[]) { QElapsedTimer startupTime; startupTime.start(); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 3823ddc426..8c24ef25a4 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -151,10 +151,6 @@ void LimitedNodeList::changeSendSocketBufferSize(int numSendBytes) { } bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { - -quint64 start = usecTimestampNow(); -quint64 end; - PacketType checkType = packetTypeForPacket(packet); int numPacketTypeBytes = numBytesArithmeticCodingFromBuffer(packet.data()); @@ -192,14 +188,9 @@ quint64 end; << uuidFromPacketHeader(packet); } } else { - - if ((end = usecTimestampNow()) - start > 100) { - printf("\t\t\t\t version and hash match long diff: %d\n", end - start); - } - return true; } - + return false; } diff --git a/libraries/shared/src/MovingPercentile.cpp b/libraries/shared/src/MovingPercentile.cpp index ba2b919d32..07b633e93e 100644 --- a/libraries/shared/src/MovingPercentile.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -1,3 +1,13 @@ +// +// MovingPercentile.cpp +// libraries/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #include "MovingPercentile.h" MovingPercentile::MovingPercentile(int numSamples, float percentile) @@ -23,7 +33,7 @@ void MovingPercentile::updatePercentile(float sample) { _sampleAges[i]++; } - // find index in _samplesSorted to insert new sample. + // find index at which to insert new sample in _samplesSorted int newSampleIndex; if (_numExistingSamples < _numSamples) { // if samples have not been filled yet, this will be the next empty spot @@ -40,7 +50,7 @@ void MovingPercentile::updatePercentile(float sample) { while (_sampleAges[newSampleIndex] != _numExistingSamples) { newSampleIndex++; } } - // insert new sample at that index + // insert new sample _samplesSorted[newSampleIndex] = sample; _sampleAges[newSampleIndex] = 0; diff --git a/libraries/shared/src/MovingPercentile.h b/libraries/shared/src/MovingPercentile.h index d4a43363e1..94a4f36da5 100644 --- a/libraries/shared/src/MovingPercentile.h +++ b/libraries/shared/src/MovingPercentile.h @@ -1,8 +1,16 @@ +// +// MovingPercentile.h +// libraries/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// #ifndef hifi_MovingPercentile_h #define hifi_MovingPercentile_h - class MovingPercentile { public: From 480efcbe76c5153bad3954814bad1d879ab7236d Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 11:31:44 -0700 Subject: [PATCH 07/12] more cleanup again --- libraries/networking/src/Node.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 74f2fdfb70..85fe2e4458 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -23,7 +23,6 @@ #include "HifiSockAddr.h" #include "NodeData.h" #include "SimpleMovingAverage.h" - #include "MovingPercentile.h" typedef quint8 NodeType_t; From ebfc405d9dc0717d8ff625478b26962e22db64cb Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 11:55:20 -0700 Subject: [PATCH 08/12] more cleanup 3 --- libraries/networking/src/LimitedNodeList.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 8c24ef25a4..c0d7941edf 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -168,7 +168,7 @@ bool LimitedNodeList::packetVersionAndHashMatch(const QByteArray& packet) { versionDebugSuppressMap.insert(senderUUID, checkType); } - + return false; } From e2d3fcc518ee00ba600e5cb5a0072fef28afa6ed Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 14:24:23 -0700 Subject: [PATCH 09/12] changed MovingPercentile to use QList --- libraries/shared/src/MovingPercentile.cpp | 64 ++++++++--------------- libraries/shared/src/MovingPercentile.h | 13 +++-- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/libraries/shared/src/MovingPercentile.cpp b/libraries/shared/src/MovingPercentile.cpp index 07b633e93e..34657110d3 100644 --- a/libraries/shared/src/MovingPercentile.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -13,65 +13,47 @@ MovingPercentile::MovingPercentile(int numSamples, float percentile) : _numSamples(numSamples), _percentile(percentile), - _numExistingSamples(0), - _valueAtPercentile(0.0f), - _indexOfPercentile(0) + _samplesSorted(), + _sampleIds(), + _newSampleId(0), + _indexOfPercentile(0), + _valueAtPercentile(0.0f) { - _samplesSorted = new float[numSamples]; - _sampleAges = new int[numSamples]; -} - -MovingPercentile::~MovingPercentile() { - delete[] _samplesSorted; - delete[] _sampleAges; } void MovingPercentile::updatePercentile(float sample) { - // age all current samples by 1 - for (int i = 0; i < _numExistingSamples; i++) { - _sampleAges[i]++; - } - - // find index at which to insert new sample in _samplesSorted + // insert the new sample into _samplesSorted int newSampleIndex; - if (_numExistingSamples < _numSamples) { - // if samples have not been filled yet, this will be the next empty spot - newSampleIndex = _numExistingSamples; - _numExistingSamples++; + if (_sampleIds.size() < _numSamples) { + // if not all samples have been filled yet, simply append it + newSampleIndex = _samplesSorted.size(); + _samplesSorted.append(sample); + _sampleIds.append(_newSampleId); // update _indexOfPercentile - float index = _percentile * (float)(_numExistingSamples - 1); + float index = _percentile * (float)(_sampleIds.size() - 1); _indexOfPercentile = (int)(index + 0.5f); // round to int } else { - // if samples have been filled, it will be the spot of the oldest sample - newSampleIndex = 0; - while (_sampleAges[newSampleIndex] != _numExistingSamples) { newSampleIndex++; } + // find index of sample with id = _newSampleId and replace it with new sample + newSampleIndex = _sampleIds.indexOf(_newSampleId); + _samplesSorted[newSampleIndex] = sample; } - // insert new sample - _samplesSorted[newSampleIndex] = sample; - _sampleAges[newSampleIndex] = 0; + // increment _newSampleId. cycles from 0 thru N-1 + _newSampleId = (_newSampleId == _numSamples - 1) ? 0 : _newSampleId + 1; - // swap new sample with neighboring elements in _samplesSorted until it's in sorted order + // swap new sample with neighbors in _samplesSorted until it's in sorted order // try swapping up first, then down. element will only be swapped one direction. - while (newSampleIndex < _numExistingSamples-1 && sample > _samplesSorted[newSampleIndex+1]) { - _samplesSorted[newSampleIndex] = _samplesSorted[newSampleIndex + 1]; - _samplesSorted[newSampleIndex+1] = sample; - - _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex+1]; - _sampleAges[newSampleIndex+1] = 0; - + while (newSampleIndex < _sampleIds.size() - 1 && sample > _samplesSorted[newSampleIndex + 1]) { + _samplesSorted.swap(newSampleIndex, newSampleIndex + 1); + _sampleIds.swap(newSampleIndex, newSampleIndex + 1); newSampleIndex++; } while (newSampleIndex > 0 && sample < _samplesSorted[newSampleIndex - 1]) { - _samplesSorted[newSampleIndex] = _samplesSorted[newSampleIndex - 1]; - _samplesSorted[newSampleIndex - 1] = sample; - - _sampleAges[newSampleIndex] = _sampleAges[newSampleIndex - 1]; - _sampleAges[newSampleIndex - 1] = 0; - + _samplesSorted.swap(newSampleIndex, newSampleIndex - 1); + _sampleIds.swap(newSampleIndex, newSampleIndex - 1); newSampleIndex--; } diff --git a/libraries/shared/src/MovingPercentile.h b/libraries/shared/src/MovingPercentile.h index 94a4f36da5..284ed9d890 100644 --- a/libraries/shared/src/MovingPercentile.h +++ b/libraries/shared/src/MovingPercentile.h @@ -11,27 +11,26 @@ #ifndef hifi_MovingPercentile_h #define hifi_MovingPercentile_h +#include + class MovingPercentile { public: MovingPercentile(int numSamples, float percentile = 0.5f); - ~MovingPercentile(); void updatePercentile(float sample); float getValueAtPercentile() const { return _valueAtPercentile; } - private: const int _numSamples; const float _percentile; - float* _samplesSorted; - int* _sampleAges; // _sampleAges[i] is the "age" of the sample at _sampleSorted[i] (higher means older) - int _numExistingSamples; - - float _valueAtPercentile; + QList _samplesSorted; + QList _sampleIds; // incrementally assigned, is cyclic + int _newSampleId; int _indexOfPercentile; + float _valueAtPercentile; }; #endif From 0da6374e19549622c9b8537dd852dedba05f3d0f Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 14:31:58 -0700 Subject: [PATCH 10/12] MovingPercentile .size() consistency --- libraries/shared/src/MovingPercentile.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/shared/src/MovingPercentile.cpp b/libraries/shared/src/MovingPercentile.cpp index 34657110d3..c5c6f4284d 100644 --- a/libraries/shared/src/MovingPercentile.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -25,14 +25,14 @@ void MovingPercentile::updatePercentile(float sample) { // insert the new sample into _samplesSorted int newSampleIndex; - if (_sampleIds.size() < _numSamples) { + if (_samplesSorted.size() < _numSamples) { // if not all samples have been filled yet, simply append it newSampleIndex = _samplesSorted.size(); _samplesSorted.append(sample); _sampleIds.append(_newSampleId); // update _indexOfPercentile - float index = _percentile * (float)(_sampleIds.size() - 1); + float index = _percentile * (float)(_samplesSorted.size() - 1); _indexOfPercentile = (int)(index + 0.5f); // round to int } else { @@ -46,7 +46,7 @@ void MovingPercentile::updatePercentile(float sample) { // swap new sample with neighbors in _samplesSorted until it's in sorted order // try swapping up first, then down. element will only be swapped one direction. - while (newSampleIndex < _sampleIds.size() - 1 && sample > _samplesSorted[newSampleIndex + 1]) { + while (newSampleIndex < _samplesSorted.size() - 1 && sample > _samplesSorted[newSampleIndex + 1]) { _samplesSorted.swap(newSampleIndex, newSampleIndex + 1); _sampleIds.swap(newSampleIndex, newSampleIndex + 1); newSampleIndex++; From 5251057ed62d95a308f3f80b0cc865d0aa210e0d Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 15:13:16 -0700 Subject: [PATCH 11/12] MovingPercentile moved else to same line as } --- libraries/shared/src/MovingPercentile.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/shared/src/MovingPercentile.cpp b/libraries/shared/src/MovingPercentile.cpp index c5c6f4284d..ec007b5c22 100644 --- a/libraries/shared/src/MovingPercentile.cpp +++ b/libraries/shared/src/MovingPercentile.cpp @@ -34,8 +34,7 @@ void MovingPercentile::updatePercentile(float sample) { // update _indexOfPercentile float index = _percentile * (float)(_samplesSorted.size() - 1); _indexOfPercentile = (int)(index + 0.5f); // round to int - } - else { + } else { // find index of sample with id = _newSampleId and replace it with new sample newSampleIndex = _sampleIds.indexOf(_newSampleId); _samplesSorted[newSampleIndex] = sample; From 124bd88b01d3264df7d13f96eef6cfa85539d280 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 4 Jun 2014 17:23:44 -0700 Subject: [PATCH 12/12] added tests for MovingPercentile --- tests/shared/CMakeLists.txt | 38 +++++ tests/shared/src/MovingPercentileTests.cpp | 169 +++++++++++++++++++++ tests/shared/src/MovingPercentileTests.h | 22 +++ tests/shared/src/main.cpp | 16 ++ 4 files changed, 245 insertions(+) create mode 100644 tests/shared/CMakeLists.txt create mode 100644 tests/shared/src/MovingPercentileTests.cpp create mode 100644 tests/shared/src/MovingPercentileTests.h create mode 100644 tests/shared/src/main.cpp diff --git a/tests/shared/CMakeLists.txt b/tests/shared/CMakeLists.txt new file mode 100644 index 0000000000..b9513e3f26 --- /dev/null +++ b/tests/shared/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME shared-tests) + +set(ROOT_DIR ../..) +set(MACRO_DIR ${ROOT_DIR}/cmake/macros) + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +#find_package(Qt5Network REQUIRED) +#find_package(Qt5Script REQUIRED) +#find_package(Qt5Widgets REQUIRED) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +include(${MACRO_DIR}/AutoMTC.cmake) +auto_mtc(${TARGET_NAME} ${ROOT_DIR}) + +#qt5_use_modules(${TARGET_NAME} Network Script Widgets) + +#include glm +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} ${ROOT_DIR}) + +# link in the shared libraries +include(${MACRO_DIR}/LinkHifiLibrary.cmake) +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) + +IF (WIN32) + #target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) + diff --git a/tests/shared/src/MovingPercentileTests.cpp b/tests/shared/src/MovingPercentileTests.cpp new file mode 100644 index 0000000000..26870717ca --- /dev/null +++ b/tests/shared/src/MovingPercentileTests.cpp @@ -0,0 +1,169 @@ +// +// MovingPercentileTests.cpp +// tests/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MovingPercentileTests.h" + +#include "SharedUtil.h" +#include "MovingPercentile.h" + +#include + +float MovingPercentileTests::random() { + return rand() / (float)RAND_MAX; +} + +void MovingPercentileTests::runAllTests() { + + QVector valuesForN; + + valuesForN.append(1); + valuesForN.append(2); + valuesForN.append(3); + valuesForN.append(4); + valuesForN.append(5); + valuesForN.append(10); + valuesForN.append(100); + + + QQueue lastNSamples; + + for (int i=0; i N) { + lastNSamples.pop_front(); + } + + movingMin.updatePercentile(sample); + + float experimentMin = movingMin.getValueAtPercentile(); + + float actualMin = lastNSamples[0]; + for (int j = 0; j < lastNSamples.size(); j++) { + if (lastNSamples.at(j) < actualMin) { + actualMin = lastNSamples.at(j); + } + } + + if (experimentMin != actualMin) { + qDebug() << "\t\t FAIL at sample" << s; + fail = true; + break; + } + } + if (!fail) { + qDebug() << "\t\t PASS"; + } + } + + + { + bool fail = false; + + qDebug() << "\t testing running max..."; + + lastNSamples.clear(); + MovingPercentile movingMax(N, 1.0f); + + for (int s = 0; s < 10000; s++) { + + float sample = random(); + + lastNSamples.push_back(sample); + if (lastNSamples.size() > N) { + lastNSamples.pop_front(); + } + + movingMax.updatePercentile(sample); + + float experimentMax = movingMax.getValueAtPercentile(); + + float actualMax = lastNSamples[0]; + for (int j = 0; j < lastNSamples.size(); j++) { + if (lastNSamples.at(j) > actualMax) { + actualMax = lastNSamples.at(j); + } + } + + if (experimentMax != actualMax) { + qDebug() << "\t\t FAIL at sample" << s; + fail = true; + break; + } + } + if (!fail) { + qDebug() << "\t\t PASS"; + } + } + + + { + bool fail = false; + + qDebug() << "\t testing running median..."; + + lastNSamples.clear(); + MovingPercentile movingMedian(N, 0.5f); + + for (int s = 0; s < 10000; s++) { + + float sample = random(); + + lastNSamples.push_back(sample); + if (lastNSamples.size() > N) { + lastNSamples.pop_front(); + } + + movingMedian.updatePercentile(sample); + + float experimentMedian = movingMedian.getValueAtPercentile(); + + int samplesLessThan = 0; + int samplesMoreThan = 0; + + for (int j=0; j experimentMedian) { + samplesMoreThan++; + } + } + + + if (!(samplesLessThan <= N/2 && samplesMoreThan <= N-1/2)) { + qDebug() << "\t\t FAIL at sample" << s; + fail = true; + break; + } + } + if (!fail) { + qDebug() << "\t\t PASS"; + } + } + } +} + diff --git a/tests/shared/src/MovingPercentileTests.h b/tests/shared/src/MovingPercentileTests.h new file mode 100644 index 0000000000..34460880fb --- /dev/null +++ b/tests/shared/src/MovingPercentileTests.h @@ -0,0 +1,22 @@ +// +// MovingPercentileTests.h +// tests/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MovingPercentileTests_h +#define hifi_MovingPercentileTests_h + +namespace MovingPercentileTests { + + float random(); + + void runAllTests(); +} + +#endif // hifi_MovingPercentileTests_h diff --git a/tests/shared/src/main.cpp b/tests/shared/src/main.cpp new file mode 100644 index 0000000000..3ae1b7b34d --- /dev/null +++ b/tests/shared/src/main.cpp @@ -0,0 +1,16 @@ +// +// main.cpp +// tests/physics/src +// +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MovingPercentileTests.h" + +int main(int argc, char** argv) { + MovingPercentileTests::runAllTests(); + return 0; +}