From b5fe6120aa996a9828bdc22108d199d4a9851bde Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 17:41:47 -0700 Subject: [PATCH 01/13] base randomization of ice-server from ice.highfidelity.com --- domain-server/src/DomainServer.cpp | 97 +++++++++++++++++++++++++++++- domain-server/src/DomainServer.h | 15 ++++- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 16928f3dee..40f0550568 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -12,6 +12,7 @@ #include "DomainServer.h" #include +#include #include #include @@ -42,7 +43,7 @@ int const DomainServer::EXIT_CODE_REBOOT = 234923; -const QString ICE_SERVER_DEFAULT_HOSTNAME = "ice.highfidelity.io"; +const QString ICE_SERVER_DEFAULT_HOSTNAME = "ice.highfidelity.com"; DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), @@ -59,8 +60,7 @@ DomainServer::DomainServer(int argc, char* argv[]) : _webAuthenticationStateSet(), _cookieSessionHash(), _automaticNetworkingSetting(), - _settingsManager(), - _iceServerSocket(ICE_SERVER_DEFAULT_HOSTNAME, ICE_SERVER_DEFAULT_PORT) + _settingsManager() { qInstallMessageHandler(LogHandler::verboseMessageHandler); @@ -482,6 +482,18 @@ void DomainServer::setupAutomaticNetworking() { void DomainServer::setupICEHeartbeatForFullNetworking() { auto limitedNodeList = DependencyManager::get(); + // lookup the available ice-server hosts now + updateICEServerAddresses(); + + const int ICE_ADDRESS_UPDATE_MSECS = 30 * 1000; + + // hookup a timer to keep those updated every ICE_ADDRESS_UPDATE_MSECS in case of a failure requiring a switchover + if (_iceAddressLookupTimer) { + _iceAddressLookupTimer = new QTimer { this }; + connect(_iceAddressLookupTimer, &QTimer::timeout, this, &DomainServer::updateICEServerAddresses); + _iceAddressLookupTimer->start(ICE_ADDRESS_UPDATE_MSECS); + } + // call our sendHeartbeatToIceServer immediately anytime a local or public socket changes connect(limitedNodeList.data(), &LimitedNodeList::localSockAddrChanged, this, &DomainServer::sendHeartbeatToIceServer); @@ -512,6 +524,12 @@ void DomainServer::setupICEHeartbeatForFullNetworking() { } } +void DomainServer::updateICEServerAddresses() { + if (_iceAddressLookupID == -1) { + _iceAddressLookupID = QHostInfo::lookupHost(ICE_SERVER_DEFAULT_HOSTNAME, this, SLOT(handleICEHostInfo(QHostInfo))); + } +} + void DomainServer::parseAssignmentConfigs(QSet& excludedTypes) { // check for configs from the command line, these take precedence const QString ASSIGNMENT_CONFIG_REGEX_STRING = "config-([\\d]+)"; @@ -1135,6 +1153,10 @@ void DomainServer::sendHeartbeatToIceServer() { // send the heartbeat packet to the ice server now limitedNodeList->sendUnreliablePacket(*_iceServerHeartbeatPacket, _iceServerSocket); + } else { + qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server."; + qDebug() << "Waiting for" << ICE_SERVER_DEFAULT_HOSTNAME << "host lookup response"; + } } @@ -2033,3 +2055,72 @@ void DomainServer::handleKeypairChange() { sendHeartbeatToIceServer(); } } + +void DomainServer::handleICEHostInfo(const QHostInfo& hostInfo) { + // clear the ICE address lookup ID so that it can fire again + _iceAddressLookupID = -1; + + if (hostInfo.error() != QHostInfo::NoError) { + qWarning() << "IP address lookup failed for" << ICE_SERVER_DEFAULT_HOSTNAME << ":" << hostInfo.errorString(); + + // if we don't have an ICE server to use yet, trigger a retry + if (_iceServerSocket.isNull()) { + const int ICE_ADDRESS_LOOKUP_RETRY_MS = 1000; + + QTimer::singleShot(ICE_ADDRESS_LOOKUP_RETRY_MS, this, SLOT(updateICEServerAddresses())); + } + + } else { + int countBefore = _iceServerAddresses.count(); + + _iceServerAddresses = hostInfo.addresses(); + + if (countBefore == 0) { + qInfo() << "Found" << _iceServerAddresses.count() << "ice-server IP addresses for" << ICE_SERVER_DEFAULT_HOSTNAME; + } + + if (_iceServerSocket.isNull()) { + // we don't have a candidate ice-server yet, pick now + randomizeICEServerAddress(); + } + } +} + +void DomainServer::randomizeICEServerAddress() { + // create a list by removing the already failed ice-server addresses + auto candidateICEAddresses = _iceServerAddresses; + + auto it = candidateICEAddresses.begin(); + + while (it != candidateICEAddresses.end()) { + if (_failedIceServerAddresses.contains(*it)) { + // we already tried this address and it failed, remove it from list of candidates + it = candidateICEAddresses.erase(it); + } else { + // keep this candidate, it hasn't failed yet + ++it; + } + } + + if (candidateICEAddresses.empty()) { + // we ended up with an empty list since everything we've tried has failed + // so clear the set of failed addresses and start going through them again + _failedIceServerAddresses.clear(); + candidateICEAddresses = _iceServerAddresses; + } + + // of the list of available addresses that we haven't tried, pick a random one + int maxIndex = candidateICEAddresses.size(); + + static std::random_device randomDevice; + static std::mt19937 generator; + std::uniform_int_distribution<> distribution(0, maxIndex); + + auto indexToTry = distribution(generator); + + _iceServerSocket = HifiSockAddr { candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT }; + qInfo() << "Set candidate ice-server socket to" << _iceServerSocket; + + // immediately fire an ICE heartbeat once we've picked a candidate ice-server + sendHeartbeatToIceServer(); +} diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 3a83e8696b..6decbc29d0 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -81,6 +81,12 @@ private slots: void queuedQuit(QString quitMessage, int exitCode); void handleKeypairChange(); + + void updateICEServerAddresses(); + void handleICEHostInfo(const QHostInfo& hostInfo); + +signals: + void iceServerChanged(); private: void setupNodeListAndAssignments(const QUuid& sessionUUID = QUuid::createUuid()); @@ -95,6 +101,8 @@ private: void setupICEHeartbeatForFullNetworking(); void sendHeartbeatToDataServer(const QString& networkAddress); + void randomizeICEServerAddress(); + unsigned int countConnectedUsers(); void sendDomainListToNode(const SharedNodePointer& node, const HifiSockAddr& senderSockAddr); @@ -157,7 +165,12 @@ private: std::unique_ptr _iceServerHeartbeatPacket; QTimer* _iceHeartbeatTimer { nullptr }; // this looks like it dangles when created but it's parented to the DomainServer - + + QList _iceServerAddresses; + QSet _failedIceServerAddresses; + QTimer* _iceAddressLookupTimer { nullptr }; // this looks like a dangling pointer but is parented to the DomainServer + int _iceAddressLookupID { -1 }; + friend class DomainGatekeeper; }; From f975f480b486ab498626abae3e42a7df12ba4074 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 17:46:47 -0700 Subject: [PATCH 02/13] use the random device for random number generation --- domain-server/src/DomainServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 40f0550568..327afdf292 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2113,7 +2113,7 @@ void DomainServer::randomizeICEServerAddress() { int maxIndex = candidateICEAddresses.size(); static std::random_device randomDevice; - static std::mt19937 generator; + static std::mt19937 generator(randomDevice()); std::uniform_int_distribution<> distribution(0, maxIndex); auto indexToTry = distribution(generator); From 8832a9d9ed6111050c99474fa02a11c86b7cb3a9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 17:59:07 -0700 Subject: [PATCH 03/13] add ICEServerHeartbeatACK packet type --- libraries/networking/src/udt/PacketHeaders.cpp | 4 ++-- libraries/networking/src/udt/PacketHeaders.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index df1a6f1bec..e4aab94090 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -34,8 +34,8 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::DomainServerAddedNode << PacketType::DomainServerConnectionToken << PacketType::DomainSettingsRequest << PacketType::DomainSettings << PacketType::ICEServerPeerInformation << PacketType::ICEServerQuery << PacketType::ICEServerHeartbeat - << PacketType::ICEPing << PacketType::ICEPingReply << PacketType::ICEServerHeartbeatDenied - << PacketType::AssignmentClientStatus << PacketType::StopNode + << PacketType::ICEServerHeartbeatACK << PacketType::ICEPing << PacketType::ICEPingReply + << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode; const QSet RELIABLE_PACKETS = QSet(); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index a2bd0d4d2d..b98a87e439 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -93,7 +93,8 @@ public: MessagesUnsubscribe, ICEServerHeartbeatDenied, AssetMappingOperation, - AssetMappingOperationReply + AssetMappingOperationReply, + ICEServerHeartbeatACK }; }; From 6ef9fbfcc09b2b98446f3d78c171bb0a54e62e2a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 18:01:20 -0700 Subject: [PATCH 04/13] send an ACK packet from ice server for verified hearbeat --- ice-server/src/IceServer.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index b66ccaa057..c3e57b1d36 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -82,6 +82,11 @@ void IceServer::processPacket(std::unique_ptr packet) { if (peer) { // so that we can send packets to the heartbeating peer when we need, we need to activate a socket now peer->activateMatchingOrNewSymmetricSocket(nlPacket->getSenderSockAddr()); + + // we have an active and verified heartbeating peer + // send them an ACK packet so they know that they are being heard and ready for ICE + static auto ackPacket = NLPacket::create(PacketType::ICEServerHeartbeatACK); + _serverSocket.writePacket(*ackPacket, nlPacket->getSenderSockAddr()); } else { // we couldn't verify this peer - respond back to them so they know they may need to perform keypair re-generation static auto deniedPacket = NLPacket::create(PacketType::ICEServerHeartbeatDenied); From f1209dc82d5eb8a2791893c7edcd5b85dfbed14c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 18:04:32 -0700 Subject: [PATCH 05/13] fix index randomization for single candidate ice-server --- domain-server/src/DomainServer.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 327afdf292..4dd7fefd2a 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2110,13 +2110,16 @@ void DomainServer::randomizeICEServerAddress() { } // of the list of available addresses that we haven't tried, pick a random one - int maxIndex = candidateICEAddresses.size(); + int maxIndex = candidateICEAddresses.size() - 1; + int indexToTry = 0; - static std::random_device randomDevice; - static std::mt19937 generator(randomDevice()); - std::uniform_int_distribution<> distribution(0, maxIndex); + if (maxIndex > 0) { + static std::random_device randomDevice; + static std::mt19937 generator(randomDevice()); + std::uniform_int_distribution<> distribution(0, maxIndex); - auto indexToTry = distribution(generator); + indexToTry = distribution(generator); + } _iceServerSocket = HifiSockAddr { candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT }; qInfo() << "Set candidate ice-server socket to" << _iceServerSocket; From f69f59fa90b07bf30e84d957f120b910571e5112 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 18:19:02 -0700 Subject: [PATCH 06/13] handle ice fail for no reply heartbeat in DS --- domain-server/src/DomainServer.cpp | 34 ++++++++++++++++++++++++++++++ domain-server/src/DomainServer.h | 2 ++ 2 files changed, 36 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 4dd7fefd2a..1ba2691dbc 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -369,7 +369,9 @@ void DomainServer::setupNodeListAndAssignments(const QUuid& sessionUUID) { packetReceiver.registerListener(PacketType::ICEPing, &_gatekeeper, "processICEPingPacket"); packetReceiver.registerListener(PacketType::ICEPingReply, &_gatekeeper, "processICEPingReplyPacket"); packetReceiver.registerListener(PacketType::ICEServerPeerInformation, &_gatekeeper, "processICEPeerInformationPacket"); + packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket"); + packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK"); // add whatever static assignments that have been parsed to the queue addStaticAssignmentsToQueue(); @@ -1102,6 +1104,25 @@ void DomainServer::sendHeartbeatToIceServer() { return; } + const int FAILOVER_NO_REPLY_ICE_HEARTBEATS { 3 }; + + // increase the count of no reply ICE heartbeats and check the current value + ++_noReplyICEHeartbeats; + + if (_noReplyICEHeartbeats > FAILOVER_NO_REPLY_ICE_HEARTBEATS) { + qWarning() << "There have been" << _noReplyICEHeartbeats - 1 << "heartbeats sent with no reply from the ice-server"; + qWarning() << "Clearing the current ice-server socket and selecting a new candidate ice-server"; + + // if we've failed to hear back for three heartbeats, we clear the current ice-server socket and attempt + // to randomize a new one + _iceServerSocket.clear(); + + // reset the number of no reply ICE hearbeats + _noReplyICEHeartbeats = 0; + + randomizeICEServerAddress(); + } + // NOTE: I'd love to specify the correct size for the packet here, but it's a little trickey with // QDataStream and the possibility of IPv6 address for the sockets. if (!_iceServerHeartbeatPacket) { @@ -1153,6 +1174,7 @@ void DomainServer::sendHeartbeatToIceServer() { // send the heartbeat packet to the ice server now limitedNodeList->sendUnreliablePacket(*_iceServerHeartbeatPacket, _iceServerSocket); + } else { qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server."; qDebug() << "Waiting for" << ICE_SERVER_DEFAULT_HOSTNAME << "host lookup response"; @@ -2043,6 +2065,14 @@ void DomainServer::processICEServerHeartbeatDenialPacket(QSharedPointer message) { + // we don't do anything with this ACK other than use it to tell us to keep talking to the same ice-server + _noReplyICEHeartbeats = 0; } void DomainServer::handleKeypairChange() { @@ -2105,6 +2135,10 @@ void DomainServer::randomizeICEServerAddress() { if (candidateICEAddresses.empty()) { // we ended up with an empty list since everything we've tried has failed // so clear the set of failed addresses and start going through them again + + qWarning() << "All current ice-server addresses have failed - re-attempting all current addresses for" + << ICE_SERVER_DEFAULT_HOSTNAME; + _failedIceServerAddresses.clear(); candidateICEAddresses = _iceServerAddresses; } diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 6decbc29d0..7fa83a7395 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -62,6 +62,7 @@ public slots: void processPathQueryPacket(QSharedPointer packet); void processNodeDisconnectRequestPacket(QSharedPointer message); void processICEServerHeartbeatDenialPacket(QSharedPointer message); + void processICEServerHeartbeatACK(QSharedPointer message); private slots: void aboutToQuit(); @@ -170,6 +171,7 @@ private: QSet _failedIceServerAddresses; QTimer* _iceAddressLookupTimer { nullptr }; // this looks like a dangling pointer but is parented to the DomainServer int _iceAddressLookupID { -1 }; + int _noReplyICEHeartbeats { 0 }; friend class DomainGatekeeper; }; From ab414f65eb8951f83cdae87a632528308fd69f05 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 18:23:51 -0700 Subject: [PATCH 07/13] output success on first connection to new ice-server --- domain-server/src/DomainServer.cpp | 8 ++++++++ domain-server/src/DomainServer.h | 1 + 2 files changed, 9 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 1ba2691dbc..3b2f13c668 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -1120,6 +1120,9 @@ void DomainServer::sendHeartbeatToIceServer() { // reset the number of no reply ICE hearbeats _noReplyICEHeartbeats = 0; + // reset the connection flag for ICE server + _connectedToICEServer = false; + randomizeICEServerAddress(); } @@ -2073,6 +2076,11 @@ void DomainServer::processICEServerHeartbeatDenialPacket(QSharedPointer message) { // we don't do anything with this ACK other than use it to tell us to keep talking to the same ice-server _noReplyICEHeartbeats = 0; + + if (!_connectedToICEServer) { + _connectedToICEServer = true; + qInfo() << "Connected to ice-server at" << _iceServerSocket; + } } void DomainServer::handleKeypairChange() { diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 7fa83a7395..1165c76559 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -172,6 +172,7 @@ private: QTimer* _iceAddressLookupTimer { nullptr }; // this looks like a dangling pointer but is parented to the DomainServer int _iceAddressLookupID { -1 }; int _noReplyICEHeartbeats { 0 }; + bool _connectedToICEServer { false }; friend class DomainGatekeeper; }; From 55002b8d5ec30d0d509c58a8c25d76005eabb77c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 09:27:43 -0700 Subject: [PATCH 08/13] add ice server address to hearbeat if full networking is on --- domain-server/src/DomainServer.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 3b2f13c668..41089ff44f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -1046,6 +1046,7 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { // setup the domain object to send to the data server const QString PUBLIC_NETWORK_ADDRESS_KEY = "network_address"; const QString AUTOMATIC_NETWORKING_KEY = "automatic_networking"; + const QString ICE_SERVER_ADDRESS = "ice_server_address"; QJsonObject domainObject; if (!networkAddress.isEmpty()) { @@ -1054,6 +1055,11 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { domainObject[AUTOMATIC_NETWORKING_KEY] = _automaticNetworkingSetting; + // if we're using full automatic networking and we have a current ice-server socket, use that now + if (_automaticNetworkingSetting == FULL_AUTOMATIC_NETWORKING_VALUE && !_iceServerSocket.isNull()) { + domainObject[ICE_SERVER_ADDRESS] = _iceServerSocket.getAddress().toString(); + } + // add a flag to indicate if this domain uses restricted access - for now that will exclude it from listings const QString RESTRICTED_ACCESS_FLAG = "restricted"; @@ -1079,7 +1085,7 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { QString domainUpdateJSON = QString("{\"domain\": %1 }").arg(QString(QJsonDocument(domainObject).toJson())); AccountManager::getInstance().sendRequest(DOMAIN_UPDATE.arg(uuidStringWithoutCurlyBraces(domainID)), - AccountManagerAuth::Required, + AccountManagerAuth::Optional, QNetworkAccessManager::PutOperation, JSONCallbackParameters(), domainUpdateJSON.toUtf8()); From 3f0ffc0aff1823455eaaeb872feeb219945f140f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 10:33:17 -0700 Subject: [PATCH 09/13] update ice-server address with API, add failed and clear denials --- domain-server/src/DomainServer.cpp | 70 +++++++++++++++++++++++++----- domain-server/src/DomainServer.h | 6 +++ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 41089ff44f..5df9492f87 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -241,7 +241,7 @@ void DomainServer::optionallyGetTemporaryName(const QStringList& arguments) { // so fire off that request now auto& accountManager = AccountManager::getInstance(); - // ask our auth endpoint for our balance + // get callbacks for temporary domain result JSONCallbackParameters callbackParameters; callbackParameters.jsonCallbackReceiver = this; callbackParameters.jsonCallbackMethod = "handleTempDomainSuccess"; @@ -434,7 +434,9 @@ void DomainServer::setupAutomaticNetworking() { setupICEHeartbeatForFullNetworking(); } - if (!resetAccountManagerAccessToken()) { + _hasAccessToken = resetAccountManagerAccessToken(); + + if (!_hasAccessToken) { qDebug() << "Will not send heartbeat to Metaverse API without an access token."; qDebug() << "If this is not a temporary domain add an access token to your config file or via the web interface."; @@ -1038,15 +1040,16 @@ void DomainServer::performIPAddressUpdate(const HifiSockAddr& newPublicSockAddr) sendHeartbeatToDataServer(newPublicSockAddr.getAddress().toString()); } + void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { const QString DOMAIN_UPDATE = "/api/v1/domains/%1"; + auto nodeList = DependencyManager::get(); const QUuid& domainID = nodeList->getSessionUUID(); // setup the domain object to send to the data server const QString PUBLIC_NETWORK_ADDRESS_KEY = "network_address"; const QString AUTOMATIC_NETWORKING_KEY = "automatic_networking"; - const QString ICE_SERVER_ADDRESS = "ice_server_address"; QJsonObject domainObject; if (!networkAddress.isEmpty()) { @@ -1055,11 +1058,6 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { domainObject[AUTOMATIC_NETWORKING_KEY] = _automaticNetworkingSetting; - // if we're using full automatic networking and we have a current ice-server socket, use that now - if (_automaticNetworkingSetting == FULL_AUTOMATIC_NETWORKING_VALUE && !_iceServerSocket.isNull()) { - domainObject[ICE_SERVER_ADDRESS] = _iceServerSocket.getAddress().toString(); - } - // add a flag to indicate if this domain uses restricted access - for now that will exclude it from listings const QString RESTRICTED_ACCESS_FLAG = "restricted"; @@ -1085,12 +1083,52 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) { QString domainUpdateJSON = QString("{\"domain\": %1 }").arg(QString(QJsonDocument(domainObject).toJson())); AccountManager::getInstance().sendRequest(DOMAIN_UPDATE.arg(uuidStringWithoutCurlyBraces(domainID)), - AccountManagerAuth::Optional, + AccountManagerAuth::Required, QNetworkAccessManager::PutOperation, JSONCallbackParameters(), domainUpdateJSON.toUtf8()); } +void DomainServer::sendICEServerAddressToMetaverseAPI() { + if (!_iceServerSocket.isNull()) { + auto nodeList = DependencyManager::get(); + const QUuid& domainID = nodeList->getSessionUUID(); + + const QString ICE_SERVER_ADDRESS = "ice_server_address"; + + QJsonObject domainObject; + + // we're using full automatic networking and we have a current ice-server socket, use that now + domainObject[ICE_SERVER_ADDRESS] = _iceServerSocket.getAddress().toString(); + + QString domainUpdateJSON = QString("{\"domain\": %1 }").arg(QString(QJsonDocument(domainObject).toJson())); + + // make sure we hear about failure so we can retry + JSONCallbackParameters callbackParameters; + callbackParameters.errorCallbackReceiver = this; + callbackParameters.errorCallbackMethod = "handleFailedICEServerAddressUpdate"; + + qDebug() << "Updating ice-server address in High Fidelity Metaverse API to" << _iceServerSocket.getAddress().toString(); + + static const QString DOMAIN_ICE_ADDRESS_UPDATE = "/api/v1/domains/%1/ice_server_address"; + + AccountManager::getInstance().sendRequest(DOMAIN_ICE_ADDRESS_UPDATE.arg(uuidStringWithoutCurlyBraces(domainID)), + AccountManagerAuth::Optional, + QNetworkAccessManager::PutOperation, + callbackParameters, + domainUpdateJSON.toUtf8()); + } +} + +void DomainServer::handleFailedICEServerAddressUpdate(QNetworkReply& requestReply) { + const int ICE_SERVER_UPDATE_RETRY_MS = 2 * 1000; + + qWarning() << "Failed to update ice-server address with High Fidelity Metaverse - error was" << requestReply.errorString(); + qWarning() << "\tRe-attempting in" << ICE_SERVER_UPDATE_RETRY_MS / 1000 << "seconds"; + + QTimer::singleShot(ICE_SERVER_UPDATE_RETRY_MS, this, SLOT(sendICEServerAddressToMetaverseAPI())); +} + void DomainServer::sendHeartbeatToIceServer() { if (!_iceServerSocket.getAddress().isNull()) { @@ -1119,6 +1157,9 @@ void DomainServer::sendHeartbeatToIceServer() { qWarning() << "There have been" << _noReplyICEHeartbeats - 1 << "heartbeats sent with no reply from the ice-server"; qWarning() << "Clearing the current ice-server socket and selecting a new candidate ice-server"; + // add the current address to our list of failed addresses + _failedIceServerAddresses << _iceServerSocket.getAddress(); + // if we've failed to hear back for three heartbeats, we clear the current ice-server socket and attempt // to randomize a new one _iceServerSocket.clear(); @@ -2062,8 +2103,7 @@ void DomainServer::processNodeDisconnectRequestPacket(QSharedPointer message) { static const int NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN = 3; - static int numHeartbeatDenials = 0; - if (++numHeartbeatDenials > NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN) { + if (++_numHeartbeatDenials > NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN) { qDebug() << "Received" << NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN << "heartbeat denials from ice-server" << "- re-generating keypair now"; @@ -2072,7 +2112,7 @@ void DomainServer::processICEServerHeartbeatDenialPacket(QSharedPointergetSessionUUID()); // reset our number of heartbeat denials - numHeartbeatDenials = 0; + _numHeartbeatDenials = 0; } // even though we can't get into this ice-server it is responding to us, so we reset our number of no-reply heartbeats @@ -2172,6 +2212,12 @@ void DomainServer::randomizeICEServerAddress() { _iceServerSocket = HifiSockAddr { candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT }; qInfo() << "Set candidate ice-server socket to" << _iceServerSocket; + // clear our number of hearbeat denials, this should be re-set on ice-server change + _numHeartbeatDenials = 0; + // immediately fire an ICE heartbeat once we've picked a candidate ice-server sendHeartbeatToIceServer(); + + // immediately send an update to the metaverse API when our ice-server changes + sendICEServerAddressToMetaverseAPI(); } diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 1165c76559..1ece2e30dd 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -86,6 +86,9 @@ private slots: void updateICEServerAddresses(); void handleICEHostInfo(const QHostInfo& hostInfo); + void sendICEServerAddressToMetaverseAPI(); + void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply); + signals: void iceServerChanged(); @@ -172,8 +175,11 @@ private: QTimer* _iceAddressLookupTimer { nullptr }; // this looks like a dangling pointer but is parented to the DomainServer int _iceAddressLookupID { -1 }; int _noReplyICEHeartbeats { 0 }; + int _numHeartbeatDenials { 0 }; bool _connectedToICEServer { false }; + bool _hasAccessToken { false }; + friend class DomainGatekeeper; }; From bc0b3b38b3f763c1831340b6c73015a5aa5b6555 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 12:25:47 -0700 Subject: [PATCH 10/13] remove narrowing conversion for ICE_SERVER_DEFAULT_PORT --- libraries/networking/src/NetworkPeer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/NetworkPeer.h b/libraries/networking/src/NetworkPeer.h index 0011f3da76..8298a2dad4 100644 --- a/libraries/networking/src/NetworkPeer.h +++ b/libraries/networking/src/NetworkPeer.h @@ -21,7 +21,7 @@ #include "HifiSockAddr.h" const QString ICE_SERVER_HOSTNAME = "localhost"; -const int ICE_SERVER_DEFAULT_PORT = 7337; +const quint16 ICE_SERVER_DEFAULT_PORT = 7337; const int ICE_HEARBEAT_INTERVAL_MSECS = 2 * 1000; const int MAX_ICE_CONNECTION_ATTEMPTS = 5; From 757c7dcf29ba58154efaea546155a0bea36e2272 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 13:40:13 -0700 Subject: [PATCH 11/13] removing HTTP health check from ice-server --- ice-server/src/IceServer.cpp | 28 +--------------------------- ice-server/src/IceServer.h | 7 +------ 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index c3e57b1d36..537742ad8b 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -33,9 +33,7 @@ IceServer::IceServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _id(QUuid::createUuid()), _serverSocket(), - _activePeers(), - _httpManager(QHostAddress::AnyIPv4, ICE_SERVER_MONITORING_PORT, QString("%1/web/").arg(QCoreApplication::applicationDirPath()), this), - _lastInactiveCheckTimestamp(QDateTime::currentMSecsSinceEpoch()) + _activePeers() { // start the ice-server socket qDebug() << "ice-server socket is listening on" << ICE_SERVER_DEFAULT_PORT; @@ -287,8 +285,6 @@ void IceServer::sendPeerInformationPacket(const NetworkPeer& peer, const HifiSoc void IceServer::clearInactivePeers() { NetworkPeerHash::iterator peerItem = _activePeers.begin(); - _lastInactiveCheckTimestamp = QDateTime::currentMSecsSinceEpoch(); - while (peerItem != _activePeers.end()) { SharedNetworkPeer peer = peerItem.value(); @@ -306,25 +302,3 @@ void IceServer::clearInactivePeers() { } } } - -bool IceServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) { - // We need an HTTP handler in order to monitor the health of the ice server - // The correct functioning of the ICE server will be determined by its HTTP availability, - - if (connection->requestOperation() == QNetworkAccessManager::GetOperation) { - if (url.path() == "/status") { - // figure out if we respond with 0 (we're good) or 1 (we think we're in trouble) - - const quint64 MAX_PACKET_GAP_MS_FOR_STUCK_SOCKET = 10 * 1000; - - auto sinceLastInactiveCheck = QDateTime::currentMSecsSinceEpoch() - _lastInactiveCheckTimestamp; - int statusNumber = (sinceLastInactiveCheck > MAX_PACKET_GAP_MS_FOR_STUCK_SOCKET) ? 1 : 0; - - connection->respond(HTTPConnection::StatusCode200, QByteArray::number(statusNumber)); - - return true; - } - } - - return false; -} diff --git a/ice-server/src/IceServer.h b/ice-server/src/IceServer.h index 7d1d05324c..a204c59e28 100644 --- a/ice-server/src/IceServer.h +++ b/ice-server/src/IceServer.h @@ -28,11 +28,10 @@ class QNetworkReply; -class IceServer : public QCoreApplication, public HTTPRequestHandler { +class IceServer : public QCoreApplication { Q_OBJECT public: IceServer(int argc, char* argv[]); - bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false); private slots: void clearInactivePeers(); void publicKeyReplyFinished(QNetworkReply* reply); @@ -52,13 +51,9 @@ private: using NetworkPeerHash = QHash; NetworkPeerHash _activePeers; - HTTPManager _httpManager; - using RSAUniquePtr = std::unique_ptr>; using DomainPublicKeyHash = std::unordered_map; DomainPublicKeyHash _domainPublicKeys; - - quint64 _lastInactiveCheckTimestamp; }; #endif // hifi_IceServer_h From 7385894b668a116d2fc3286d0c22d3294ff05225 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 13:57:09 -0700 Subject: [PATCH 12/13] keep a set of pending public key requests --- ice-server/src/IceServer.cpp | 61 +++++++++++++++++++++--------------- ice-server/src/IceServer.h | 2 ++ 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index 537742ad8b..df8afdb82f 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -167,39 +167,42 @@ SharedNetworkPeer IceServer::addOrUpdateHeartbeatingPeer(NLPacket& packet) { } bool IceServer::isVerifiedHeartbeat(const QUuid& domainID, const QByteArray& plaintext, const QByteArray& signature) { - // check if we have a public key for this domain ID - if we do not then fire off the request for it - auto it = _domainPublicKeys.find(domainID); - if (it != _domainPublicKeys.end()) { + // make sure we're not already waiting for a public key for this domain-server + if (!_pendingPublicKeyRequests.contains(domainID)) { + // check if we have a public key for this domain ID - if we do not then fire off the request for it + auto it = _domainPublicKeys.find(domainID); + if (it != _domainPublicKeys.end()) { - // attempt to verify the signature for this heartbeat - const auto rsaPublicKey = it->second.get(); + // attempt to verify the signature for this heartbeat + const auto rsaPublicKey = it->second.get(); - if (rsaPublicKey) { - auto hashedPlaintext = QCryptographicHash::hash(plaintext, QCryptographicHash::Sha256); - int verificationResult = RSA_verify(NID_sha256, - reinterpret_cast(hashedPlaintext.constData()), - hashedPlaintext.size(), - reinterpret_cast(signature.constData()), - signature.size(), - rsaPublicKey); + if (rsaPublicKey) { + auto hashedPlaintext = QCryptographicHash::hash(plaintext, QCryptographicHash::Sha256); + int verificationResult = RSA_verify(NID_sha256, + reinterpret_cast(hashedPlaintext.constData()), + hashedPlaintext.size(), + reinterpret_cast(signature.constData()), + signature.size(), + rsaPublicKey); + + if (verificationResult == 1) { + // this is the only success case - we return true here to indicate that the heartbeat is verified + return true; + } else { + qDebug() << "Failed to verify heartbeat for" << domainID << "- re-requesting public key from API."; + } - if (verificationResult == 1) { - // this is the only success case - we return true here to indicate that the heartbeat is verified - return true; } else { - qDebug() << "Failed to verify heartbeat for" << domainID << "- re-requesting public key from API."; + // we can't let this user in since we couldn't convert their public key to an RSA key we could use + qWarning() << "Public key for" << domainID << "is not a usable RSA* public key."; + qWarning() << "Re-requesting public key from API"; } - - } else { - // we can't let this user in since we couldn't convert their public key to an RSA key we could use - qWarning() << "Public key for" << domainID << "is not a usable RSA* public key."; - qWarning() << "Re-requesting public key from API"; } - } - // we could not verify this heartbeat (missing public key, could not load public key, bad actor) - // ask the metaverse API for the right public key and return false to indicate that this is not verified - requestDomainPublicKey(domainID); + // we could not verify this heartbeat (missing public key, could not load public key, bad actor) + // ask the metaverse API for the right public key and return false to indicate that this is not verified + requestDomainPublicKey(domainID); + } return false; } @@ -217,6 +220,9 @@ void IceServer::requestDomainPublicKey(const QUuid& domainID) { qDebug() << "Requesting public key for domain with ID" << domainID; + // add this to the set of pending public key requests + _pendingPublicKeyRequests.insert(domainID); + networkAccessManager.get(publicKeyRequest); } @@ -269,6 +275,9 @@ void IceServer::publicKeyReplyFinished(QNetworkReply* reply) { qWarning() << "Error retreiving public key for domain with ID" << domainID << "-" << reply->errorString(); } + // remove this domain ID from the list of pending public key requests + _pendingPublicKeyRequests.remove(domainID); + reply->deleteLater(); } diff --git a/ice-server/src/IceServer.h b/ice-server/src/IceServer.h index a204c59e28..2aa9a875a7 100644 --- a/ice-server/src/IceServer.h +++ b/ice-server/src/IceServer.h @@ -54,6 +54,8 @@ private: using RSAUniquePtr = std::unique_ptr>; using DomainPublicKeyHash = std::unordered_map; DomainPublicKeyHash _domainPublicKeys; + + QSet _pendingPublicKeyRequests; }; #endif // hifi_IceServer_h From 9bca2309e098a849c58eb3f4cf876127c45426c2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 14 Apr 2016 13:58:02 -0700 Subject: [PATCH 13/13] remove the monitoring endpoint debug --- ice-server/src/IceServer.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index df8afdb82f..84c48ac5f9 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -27,8 +27,6 @@ const int CLEAR_INACTIVE_PEERS_INTERVAL_MSECS = 1 * 1000; const int PEER_SILENCE_THRESHOLD_MSECS = 5 * 1000; -const quint16 ICE_SERVER_MONITORING_PORT = 40110; - IceServer::IceServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _id(QUuid::createUuid()), @@ -37,7 +35,6 @@ IceServer::IceServer(int argc, char* argv[]) : { // start the ice-server socket qDebug() << "ice-server socket is listening on" << ICE_SERVER_DEFAULT_PORT; - qDebug() << "monitoring http endpoint is listening on " << ICE_SERVER_MONITORING_PORT; _serverSocket.bind(QHostAddress::AnyIPv4, ICE_SERVER_DEFAULT_PORT); // set processPacket as the verified packet callback for the udt::Socket