From bfbaf265dd58f88c1d52ecb66069a8b036f4c3d1 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 19 Dec 2013 13:14:14 -0800 Subject: [PATCH] first cut at added clockSkew support to nodes and PING/PING_REPLY --- interface/src/Application.cpp | 10 ++-- interface/src/ui/VoxelStatsDialog.cpp | 25 ++++++--- libraries/shared/src/Node.h | 4 ++ libraries/shared/src/NodeList.cpp | 77 +++++++++++++++++++++----- libraries/shared/src/NodeList.h | 2 + libraries/shared/src/PacketHeaders.cpp | 3 + libraries/shared/src/SharedUtil.h | 4 +- 7 files changed, 96 insertions(+), 29 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 24cdabe772..7f976798a1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1251,12 +1251,10 @@ void Application::sendPingPackets() { const char nodesToPing[] = {NODE_TYPE_VOXEL_SERVER, NODE_TYPE_PARTICLE_SERVER, NODE_TYPE_AUDIO_MIXER, NODE_TYPE_AVATAR_MIXER}; - uint64_t currentTime = usecTimestampNow(); - unsigned char pingPacket[numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_PING) + sizeof(currentTime)]; - int numHeaderBytes = populateTypeAndVersion(pingPacket, PACKET_TYPE_PING); - - memcpy(pingPacket + numHeaderBytes, ¤tTime, sizeof(currentTime)); - getInstance()->controlledBroadcastToNodes(pingPacket, sizeof(pingPacket), + unsigned char pingPacket[MAX_PACKET_SIZE]; + int length = NodeList::getInstance()->fillPingPacket(pingPacket); + + getInstance()->controlledBroadcastToNodes(pingPacket, length, nodesToPing, sizeof(nodesToPing)); } diff --git a/interface/src/ui/VoxelStatsDialog.cpp b/interface/src/ui/VoxelStatsDialog.cpp index 61226cd99a..db2836ab43 100644 --- a/interface/src/ui/VoxelStatsDialog.cpp +++ b/interface/src/ui/VoxelStatsDialog.cpp @@ -34,16 +34,16 @@ VoxelStatsDialog::VoxelStatsDialog(QWidget* parent, NodeToVoxelSceneStats* model _labels[i] = NULL; } - this->setWindowTitle("Voxel Statistics"); + this->setWindowTitle("Octree Server Statistics"); // Create layouter _form = new QFormLayout(); this->QDialog::setLayout(_form); // Setup stat items - _serverVoxels = AddStatItem("Voxels on Servers"); - _localVoxels = AddStatItem("Local Voxels"); - _localVoxelsMemory = AddStatItem("Voxels Memory"); + _serverVoxels = AddStatItem("Elements on Servers"); + _localVoxels = AddStatItem("Local Elements"); + _localVoxelsMemory = AddStatItem("Elements Memory"); _voxelsRendered = AddStatItem("Voxels Rendered"); _sendingMode = AddStatItem("Sending Mode"); @@ -136,7 +136,7 @@ void VoxelStatsDialog::paintEvent(QPaintEvent* event) { label = _labels[_localVoxelsMemory]; statsValue.str(""); statsValue << - "Nodes RAM: " << OctreeElement::getTotalMemoryUsage() / 1000000.f << "MB " + "Elements RAM: " << OctreeElement::getTotalMemoryUsage() / 1000000.f << "MB " "Geometry RAM: " << voxels->getVoxelMemoryUsageRAM() / 1000000.f << "MB " << "VBO: " << voxels->getVoxelMemoryUsageVBO() / 1000000.f << "MB "; if (voxels->hasVoxelMemoryUsageGPU()) { @@ -344,15 +344,26 @@ void VoxelStatsDialog::showOctreeServersOfType(int& serverCount, NODE_TYPE serve QString incomingWastedBytesString = locale.toString((uint)stats.getIncomingWastedBytes()); QString incomingOutOfOrderString = locale.toString((uint)stats.getIncomingOutOfOrder()); QString incomingLikelyLostString = locale.toString((uint)stats.getIncomingLikelyLost()); - QString incomingFlightTimeString = locale.toString(stats.getIncomingFlightTimeAverage()); + + float clockSkewInMS = (float)node->getClockSkewUsec() / (float)USECS_PER_MSEC; + float adjustedFlightTime = stats.getIncomingFlightTimeAverage() + clockSkewInMS; + QString incomingFlightTimeString = locale.toString((int)adjustedFlightTime); + QString incomingPingTimeString = locale.toString(node->getPingMs()); + QString incomingClockSkewString = locale.toString((int)clockSkewInMS); serverDetails << "
" << "Incoming Packets: " << incomingPacketsString.toLocal8Bit().constData() << " Out of Order: " << incomingOutOfOrderString.toLocal8Bit().constData() << " Likely Lost: " << incomingLikelyLostString.toLocal8Bit().constData(); - serverDetails << "
" << + serverDetails << "
" << " Average Flight Time: " << incomingFlightTimeString.toLocal8Bit().constData() << " msecs"; + + serverDetails << "
" << + " Average Ping Time: " << incomingPingTimeString.toLocal8Bit().constData() << " msecs"; + + serverDetails << "
" << + " Average Clock Skew: " << incomingClockSkewString.toLocal8Bit().constData() << " msecs"; serverDetails << "
" << "Incoming" << " Bytes: " << incomingBytesString.toLocal8Bit().constData() << diff --git a/libraries/shared/src/Node.h b/libraries/shared/src/Node.h index 4614a6f0a8..96f16a31d4 100644 --- a/libraries/shared/src/Node.h +++ b/libraries/shared/src/Node.h @@ -68,6 +68,9 @@ public: int getPingMs() const { return _pingMs; } void setPingMs(int pingMs) { _pingMs = pingMs; } + + int getClockSkewUsec() const { return _clockSkewUsec; } + void setClockSkewUsec(int clockSkew) { _clockSkewUsec = clockSkew; } void lock() { pthread_mutex_lock(&_mutex); } @@ -93,6 +96,7 @@ private: NodeData* _linkedData; bool _isAlive; int _pingMs; + int _clockSkewUsec; pthread_mutex_t _mutex; }; diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index d8661d9350..c7e7a2c3f0 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -112,9 +112,33 @@ void NodeList::timePingReply(const HifiSockAddr& nodeAddress, unsigned char *pac if (node->getPublicSocket() == nodeAddress || node->getLocalSocket() == nodeAddress) { - int pingTime = usecTimestampNow() - *(uint64_t*)(packetData + numBytesForPacketHeader(packetData)); + unsigned char* dataAt = packetData + numBytesForPacketHeader(packetData); + uint64_t ourOriginalTime = *(uint64_t*)(dataAt); + dataAt += sizeof(ourOriginalTime); + uint64_t othersReplyTime = *(uint64_t*)(dataAt); + uint64_t now = usecTimestampNow(); + int pingTime = now - ourOriginalTime; + int oneWayFlightTime = pingTime/2; // half of the ping is our one way flight + + // The other node's expected time should be our original time plus the one way flight time + // anything other than that is clock skew + uint64_t othersExprectedReply = ourOriginalTime + oneWayFlightTime; + int clockSkew = othersReplyTime - othersExprectedReply; node->setPingMs(pingTime / 1000); + node->setClockSkewUsec(clockSkew); + + const bool wantDebug = false; + if (wantDebug) { + qDebug() << "PING_REPLY from node " << *node << "\n" << + " now: " << now << "\n" << + " ourTime: " << ourOriginalTime << "\n" << + " pingTime: " << pingTime << "\n" << + " oneWayFlightTime: " << oneWayFlightTime << "\n" << + " othersReplyTime: " << othersReplyTime << "\n" << + " othersExprectedReply: " << othersExprectedReply << "\n" << + " clockSkew: " << clockSkew << "\n"; + } break; } } @@ -131,9 +155,11 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, unsigned char break; } case PACKET_TYPE_PING: { - // send it right back - populateTypeAndVersion(packetData, PACKET_TYPE_PING_REPLY); - _nodeSocket.writeDatagram((char*) packetData, dataBytes, senderSockAddr.getAddress(), senderSockAddr.getPort()); + // send back a reply + unsigned char replyPacket[MAX_PACKET_SIZE]; + int replyPacketLength = fillPingReplyPacket(packetData, replyPacket); + _nodeSocket.writeDatagram((char*)replyPacket, replyPacketLength, + senderSockAddr.getAddress(), senderSockAddr.getPort()); break; } case PACKET_TYPE_PING_REPLY: { @@ -616,21 +642,42 @@ void NodeList::sendAssignment(Assignment& assignment) { assignmentServerSocket->getPort()); } +int NodeList::fillPingPacket(unsigned char* buffer) { + int numHeaderBytes = populateTypeAndVersion(buffer, PACKET_TYPE_PING); + uint64_t currentTime = usecTimestampNow(); + memcpy(buffer + numHeaderBytes, ¤tTime, sizeof(currentTime)); + return numHeaderBytes + sizeof(currentTime); +} + +int NodeList::fillPingReplyPacket(unsigned char* pingBuffer, unsigned char* replyBuffer) { + int numHeaderBytesOriginal = numBytesForPacketHeader(pingBuffer); + uint64_t timeFromOriginalPing = *(uint64_t*)(pingBuffer + numHeaderBytesOriginal); + + int numHeaderBytesReply = populateTypeAndVersion(replyBuffer, PACKET_TYPE_PING_REPLY); + int length = numHeaderBytesReply; + uint64_t ourReplyTime = usecTimestampNow(); + + unsigned char* dataAt = replyBuffer + numHeaderBytesReply; + memcpy(dataAt, &timeFromOriginalPing, sizeof(timeFromOriginalPing)); + dataAt += sizeof(timeFromOriginalPing); + length += sizeof(timeFromOriginalPing); + + memcpy(dataAt, &ourReplyTime, sizeof(ourReplyTime)); + dataAt += sizeof(ourReplyTime); + length += sizeof(ourReplyTime); + + return length; +} + + void NodeList::pingPublicAndLocalSocketsForInactiveNode(Node* node) { - - uint64_t currentTime = 0; - - // setup a ping packet to send to this node - unsigned char pingPacket[numBytesForPacketHeader((uchar*) &PACKET_TYPE_PING) + sizeof(currentTime)]; - int numHeaderBytes = populateTypeAndVersion(pingPacket, PACKET_TYPE_PING); - - currentTime = usecTimestampNow(); - memcpy(pingPacket + numHeaderBytes, ¤tTime, sizeof(currentTime)); + unsigned char pingPacket[MAX_PACKET_SIZE]; + int pingPacketLength = fillPingPacket(pingPacket); // send the ping packet to the local and public sockets for this node - _nodeSocket.writeDatagram((char*) pingPacket, sizeof(pingPacket), + _nodeSocket.writeDatagram((char*) pingPacket, pingPacketLength, node->getLocalSocket().getAddress(), node->getLocalSocket().getPort()); - _nodeSocket.writeDatagram((char*) pingPacket, sizeof(pingPacket), + _nodeSocket.writeDatagram((char*) pingPacket, pingPacketLength, node->getPublicSocket().getAddress(), node->getPublicSocket().getPort()); } diff --git a/libraries/shared/src/NodeList.h b/libraries/shared/src/NodeList.h index 9ee4c8519d..47cdbac99d 100644 --- a/libraries/shared/src/NodeList.h +++ b/libraries/shared/src/NodeList.h @@ -103,6 +103,8 @@ public: void setAssignmentServerSocket(const HifiSockAddr& serverSocket) { _assignmentServerSocket = serverSocket; } void sendAssignment(Assignment& assignment); + int fillPingPacket(unsigned char* buffer); + int fillPingReplyPacket(unsigned char* pingBuffer, unsigned char* replyBuffer); void pingPublicAndLocalSocketsForInactiveNode(Node* node); void sendKillNode(const char* nodeTypes, int numNodeTypes); diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index 05e4b4e21d..07cdbf76fa 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -56,6 +56,9 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { case PACKET_TYPE_PARTICLE_DATA: return 5; + case PACKET_TYPE_PING_REPLY: + return 1; + default: return 0; } diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index 893922ec28..df04475eba 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -52,7 +52,9 @@ static const float METER = 1.0f; static const float DECIMETER = 0.1f; static const float CENTIMETER = 0.01f; static const float MILLIIMETER = 0.001f; -static const uint64_t USECS_PER_SECOND = 1000 * 1000; +static const uint64_t USECS_PER_MSEC = 1000; +static const uint64_t MSECS_PER_SECOND = 1000; +static const uint64_t USECS_PER_SECOND = USECS_PER_MSEC * MSECS_PER_SECOND; uint64_t usecTimestamp(const timeval *time); uint64_t usecTimestampNow();