Merge pull request #7665 from birarda/ice-server-redundancy

allow domain-server to cycle through ice-servers when failing
This commit is contained in:
Seth Alves 2016-04-14 14:36:29 -07:00
commit f114888c1c
7 changed files with 267 additions and 74 deletions

View file

@ -12,6 +12,7 @@
#include "DomainServer.h"
#include <memory>
#include <random>
#include <QDir>
#include <QJsonDocument>
@ -42,7 +43,7 @@
int const DomainServer::EXIT_CODE_REBOOT = 234923;
const QString ICE_SERVER_DEFAULT_HOSTNAME = "ice.highfidelity.io";
const QString ICE_SERVER_DEFAULT_HOSTNAME = "ice.highfidelity.com";
DomainServer::DomainServer(int argc, char* argv[]) :
QCoreApplication(argc, argv),
@ -59,8 +60,7 @@ DomainServer::DomainServer(int argc, char* argv[]) :
_webAuthenticationStateSet(),
_cookieSessionHash(),
_automaticNetworkingSetting(),
_settingsManager(),
_iceServerSocket(ICE_SERVER_DEFAULT_HOSTNAME, ICE_SERVER_DEFAULT_PORT)
_settingsManager()
{
qInstallMessageHandler(LogHandler::verboseMessageHandler);
@ -241,7 +241,7 @@ void DomainServer::optionallyGetTemporaryName(const QStringList& arguments) {
// so fire off that request now
auto& accountManager = AccountManager::getInstance();
// ask our auth endpoint for our balance
// get callbacks for temporary domain result
JSONCallbackParameters callbackParameters;
callbackParameters.jsonCallbackReceiver = this;
callbackParameters.jsonCallbackMethod = "handleTempDomainSuccess";
@ -369,7 +369,9 @@ void DomainServer::setupNodeListAndAssignments(const QUuid& sessionUUID) {
packetReceiver.registerListener(PacketType::ICEPing, &_gatekeeper, "processICEPingPacket");
packetReceiver.registerListener(PacketType::ICEPingReply, &_gatekeeper, "processICEPingReplyPacket");
packetReceiver.registerListener(PacketType::ICEServerPeerInformation, &_gatekeeper, "processICEPeerInformationPacket");
packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
// add whatever static assignments that have been parsed to the queue
addStaticAssignmentsToQueue();
@ -432,7 +434,9 @@ void DomainServer::setupAutomaticNetworking() {
setupICEHeartbeatForFullNetworking();
}
if (!resetAccountManagerAccessToken()) {
_hasAccessToken = resetAccountManagerAccessToken();
if (!_hasAccessToken) {
qDebug() << "Will not send heartbeat to Metaverse API without an access token.";
qDebug() << "If this is not a temporary domain add an access token to your config file or via the web interface.";
@ -482,6 +486,18 @@ void DomainServer::setupAutomaticNetworking() {
void DomainServer::setupICEHeartbeatForFullNetworking() {
auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
// lookup the available ice-server hosts now
updateICEServerAddresses();
const int ICE_ADDRESS_UPDATE_MSECS = 30 * 1000;
// hookup a timer to keep those updated every ICE_ADDRESS_UPDATE_MSECS in case of a failure requiring a switchover
if (_iceAddressLookupTimer) {
_iceAddressLookupTimer = new QTimer { this };
connect(_iceAddressLookupTimer, &QTimer::timeout, this, &DomainServer::updateICEServerAddresses);
_iceAddressLookupTimer->start(ICE_ADDRESS_UPDATE_MSECS);
}
// call our sendHeartbeatToIceServer immediately anytime a local or public socket changes
connect(limitedNodeList.data(), &LimitedNodeList::localSockAddrChanged,
this, &DomainServer::sendHeartbeatToIceServer);
@ -512,6 +528,12 @@ void DomainServer::setupICEHeartbeatForFullNetworking() {
}
}
void DomainServer::updateICEServerAddresses() {
if (_iceAddressLookupID == -1) {
_iceAddressLookupID = QHostInfo::lookupHost(ICE_SERVER_DEFAULT_HOSTNAME, this, SLOT(handleICEHostInfo(QHostInfo)));
}
}
void DomainServer::parseAssignmentConfigs(QSet<Assignment::Type>& excludedTypes) {
// check for configs from the command line, these take precedence
const QString ASSIGNMENT_CONFIG_REGEX_STRING = "config-([\\d]+)";
@ -1018,8 +1040,10 @@ void DomainServer::performIPAddressUpdate(const HifiSockAddr& newPublicSockAddr)
sendHeartbeatToDataServer(newPublicSockAddr.getAddress().toString());
}
void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) {
const QString DOMAIN_UPDATE = "/api/v1/domains/%1";
auto nodeList = DependencyManager::get<LimitedNodeList>();
const QUuid& domainID = nodeList->getSessionUUID();
@ -1065,6 +1089,46 @@ void DomainServer::sendHeartbeatToDataServer(const QString& networkAddress) {
domainUpdateJSON.toUtf8());
}
void DomainServer::sendICEServerAddressToMetaverseAPI() {
if (!_iceServerSocket.isNull()) {
auto nodeList = DependencyManager::get<LimitedNodeList>();
const QUuid& domainID = nodeList->getSessionUUID();
const QString ICE_SERVER_ADDRESS = "ice_server_address";
QJsonObject domainObject;
// we're using full automatic networking and we have a current ice-server socket, use that now
domainObject[ICE_SERVER_ADDRESS] = _iceServerSocket.getAddress().toString();
QString domainUpdateJSON = QString("{\"domain\": %1 }").arg(QString(QJsonDocument(domainObject).toJson()));
// make sure we hear about failure so we can retry
JSONCallbackParameters callbackParameters;
callbackParameters.errorCallbackReceiver = this;
callbackParameters.errorCallbackMethod = "handleFailedICEServerAddressUpdate";
qDebug() << "Updating ice-server address in High Fidelity Metaverse API to" << _iceServerSocket.getAddress().toString();
static const QString DOMAIN_ICE_ADDRESS_UPDATE = "/api/v1/domains/%1/ice_server_address";
AccountManager::getInstance().sendRequest(DOMAIN_ICE_ADDRESS_UPDATE.arg(uuidStringWithoutCurlyBraces(domainID)),
AccountManagerAuth::Optional,
QNetworkAccessManager::PutOperation,
callbackParameters,
domainUpdateJSON.toUtf8());
}
}
void DomainServer::handleFailedICEServerAddressUpdate(QNetworkReply& requestReply) {
const int ICE_SERVER_UPDATE_RETRY_MS = 2 * 1000;
qWarning() << "Failed to update ice-server address with High Fidelity Metaverse - error was" << requestReply.errorString();
qWarning() << "\tRe-attempting in" << ICE_SERVER_UPDATE_RETRY_MS / 1000 << "seconds";
QTimer::singleShot(ICE_SERVER_UPDATE_RETRY_MS, this, SLOT(sendICEServerAddressToMetaverseAPI()));
}
void DomainServer::sendHeartbeatToIceServer() {
if (!_iceServerSocket.getAddress().isNull()) {
@ -1084,6 +1148,31 @@ void DomainServer::sendHeartbeatToIceServer() {
return;
}
const int FAILOVER_NO_REPLY_ICE_HEARTBEATS { 3 };
// increase the count of no reply ICE heartbeats and check the current value
++_noReplyICEHeartbeats;
if (_noReplyICEHeartbeats > FAILOVER_NO_REPLY_ICE_HEARTBEATS) {
qWarning() << "There have been" << _noReplyICEHeartbeats - 1 << "heartbeats sent with no reply from the ice-server";
qWarning() << "Clearing the current ice-server socket and selecting a new candidate ice-server";
// add the current address to our list of failed addresses
_failedIceServerAddresses << _iceServerSocket.getAddress();
// if we've failed to hear back for three heartbeats, we clear the current ice-server socket and attempt
// to randomize a new one
_iceServerSocket.clear();
// reset the number of no reply ICE hearbeats
_noReplyICEHeartbeats = 0;
// reset the connection flag for ICE server
_connectedToICEServer = false;
randomizeICEServerAddress();
}
// NOTE: I'd love to specify the correct size for the packet here, but it's a little trickey with
// QDataStream and the possibility of IPv6 address for the sockets.
if (!_iceServerHeartbeatPacket) {
@ -1135,6 +1224,11 @@ void DomainServer::sendHeartbeatToIceServer() {
// send the heartbeat packet to the ice server now
limitedNodeList->sendUnreliablePacket(*_iceServerHeartbeatPacket, _iceServerSocket);
} else {
qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
qDebug() << "Waiting for" << ICE_SERVER_DEFAULT_HOSTNAME << "host lookup response";
}
}
@ -2009,8 +2103,7 @@ void DomainServer::processNodeDisconnectRequestPacket(QSharedPointer<ReceivedMes
void DomainServer::processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message) {
static const int NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN = 3;
static int numHeartbeatDenials = 0;
if (++numHeartbeatDenials > NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN) {
if (++_numHeartbeatDenials > NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN) {
qDebug() << "Received" << NUM_HEARTBEAT_DENIALS_FOR_KEYPAIR_REGEN << "heartbeat denials from ice-server"
<< "- re-generating keypair now";
@ -2019,7 +2112,20 @@ void DomainServer::processICEServerHeartbeatDenialPacket(QSharedPointer<Received
AccountManager::getInstance().generateNewDomainKeypair(limitedNodeList->getSessionUUID());
// reset our number of heartbeat denials
numHeartbeatDenials = 0;
_numHeartbeatDenials = 0;
}
// even though we can't get into this ice-server it is responding to us, so we reset our number of no-reply heartbeats
_noReplyICEHeartbeats = 0;
}
void DomainServer::processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message) {
// we don't do anything with this ACK other than use it to tell us to keep talking to the same ice-server
_noReplyICEHeartbeats = 0;
if (!_connectedToICEServer) {
_connectedToICEServer = true;
qInfo() << "Connected to ice-server at" << _iceServerSocket;
}
}
@ -2033,3 +2139,85 @@ void DomainServer::handleKeypairChange() {
sendHeartbeatToIceServer();
}
}
void DomainServer::handleICEHostInfo(const QHostInfo& hostInfo) {
// clear the ICE address lookup ID so that it can fire again
_iceAddressLookupID = -1;
if (hostInfo.error() != QHostInfo::NoError) {
qWarning() << "IP address lookup failed for" << ICE_SERVER_DEFAULT_HOSTNAME << ":" << hostInfo.errorString();
// if we don't have an ICE server to use yet, trigger a retry
if (_iceServerSocket.isNull()) {
const int ICE_ADDRESS_LOOKUP_RETRY_MS = 1000;
QTimer::singleShot(ICE_ADDRESS_LOOKUP_RETRY_MS, this, SLOT(updateICEServerAddresses()));
}
} else {
int countBefore = _iceServerAddresses.count();
_iceServerAddresses = hostInfo.addresses();
if (countBefore == 0) {
qInfo() << "Found" << _iceServerAddresses.count() << "ice-server IP addresses for" << ICE_SERVER_DEFAULT_HOSTNAME;
}
if (_iceServerSocket.isNull()) {
// we don't have a candidate ice-server yet, pick now
randomizeICEServerAddress();
}
}
}
void DomainServer::randomizeICEServerAddress() {
// create a list by removing the already failed ice-server addresses
auto candidateICEAddresses = _iceServerAddresses;
auto it = candidateICEAddresses.begin();
while (it != candidateICEAddresses.end()) {
if (_failedIceServerAddresses.contains(*it)) {
// we already tried this address and it failed, remove it from list of candidates
it = candidateICEAddresses.erase(it);
} else {
// keep this candidate, it hasn't failed yet
++it;
}
}
if (candidateICEAddresses.empty()) {
// we ended up with an empty list since everything we've tried has failed
// so clear the set of failed addresses and start going through them again
qWarning() << "All current ice-server addresses have failed - re-attempting all current addresses for"
<< ICE_SERVER_DEFAULT_HOSTNAME;
_failedIceServerAddresses.clear();
candidateICEAddresses = _iceServerAddresses;
}
// of the list of available addresses that we haven't tried, pick a random one
int maxIndex = candidateICEAddresses.size() - 1;
int indexToTry = 0;
if (maxIndex > 0) {
static std::random_device randomDevice;
static std::mt19937 generator(randomDevice());
std::uniform_int_distribution<> distribution(0, maxIndex);
indexToTry = distribution(generator);
}
_iceServerSocket = HifiSockAddr { candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT };
qInfo() << "Set candidate ice-server socket to" << _iceServerSocket;
// clear our number of hearbeat denials, this should be re-set on ice-server change
_numHeartbeatDenials = 0;
// immediately fire an ICE heartbeat once we've picked a candidate ice-server
sendHeartbeatToIceServer();
// immediately send an update to the metaverse API when our ice-server changes
sendICEServerAddressToMetaverseAPI();
}

View file

@ -62,6 +62,7 @@ public slots:
void processPathQueryPacket(QSharedPointer<ReceivedMessage> packet);
void processNodeDisconnectRequestPacket(QSharedPointer<ReceivedMessage> message);
void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
private slots:
void aboutToQuit();
@ -81,6 +82,15 @@ private slots:
void queuedQuit(QString quitMessage, int exitCode);
void handleKeypairChange();
void updateICEServerAddresses();
void handleICEHostInfo(const QHostInfo& hostInfo);
void sendICEServerAddressToMetaverseAPI();
void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
signals:
void iceServerChanged();
private:
void setupNodeListAndAssignments(const QUuid& sessionUUID = QUuid::createUuid());
@ -95,6 +105,8 @@ private:
void setupICEHeartbeatForFullNetworking();
void sendHeartbeatToDataServer(const QString& networkAddress);
void randomizeICEServerAddress();
unsigned int countConnectedUsers();
void sendDomainListToNode(const SharedNodePointer& node, const HifiSockAddr& senderSockAddr);
@ -157,7 +169,17 @@ private:
std::unique_ptr<NLPacket> _iceServerHeartbeatPacket;
QTimer* _iceHeartbeatTimer { nullptr }; // this looks like it dangles when created but it's parented to the DomainServer
QList<QHostAddress> _iceServerAddresses;
QSet<QHostAddress> _failedIceServerAddresses;
QTimer* _iceAddressLookupTimer { nullptr }; // this looks like a dangling pointer but is parented to the DomainServer
int _iceAddressLookupID { -1 };
int _noReplyICEHeartbeats { 0 };
int _numHeartbeatDenials { 0 };
bool _connectedToICEServer { false };
bool _hasAccessToken { false };
friend class DomainGatekeeper;
};

View file

@ -27,19 +27,14 @@
const int CLEAR_INACTIVE_PEERS_INTERVAL_MSECS = 1 * 1000;
const int PEER_SILENCE_THRESHOLD_MSECS = 5 * 1000;
const quint16 ICE_SERVER_MONITORING_PORT = 40110;
IceServer::IceServer(int argc, char* argv[]) :
QCoreApplication(argc, argv),
_id(QUuid::createUuid()),
_serverSocket(),
_activePeers(),
_httpManager(QHostAddress::AnyIPv4, ICE_SERVER_MONITORING_PORT, QString("%1/web/").arg(QCoreApplication::applicationDirPath()), this),
_lastInactiveCheckTimestamp(QDateTime::currentMSecsSinceEpoch())
_activePeers()
{
// start the ice-server socket
qDebug() << "ice-server socket is listening on" << ICE_SERVER_DEFAULT_PORT;
qDebug() << "monitoring http endpoint is listening on " << ICE_SERVER_MONITORING_PORT;
_serverSocket.bind(QHostAddress::AnyIPv4, ICE_SERVER_DEFAULT_PORT);
// set processPacket as the verified packet callback for the udt::Socket
@ -82,6 +77,11 @@ void IceServer::processPacket(std::unique_ptr<udt::Packet> packet) {
if (peer) {
// so that we can send packets to the heartbeating peer when we need, we need to activate a socket now
peer->activateMatchingOrNewSymmetricSocket(nlPacket->getSenderSockAddr());
// we have an active and verified heartbeating peer
// send them an ACK packet so they know that they are being heard and ready for ICE
static auto ackPacket = NLPacket::create(PacketType::ICEServerHeartbeatACK);
_serverSocket.writePacket(*ackPacket, nlPacket->getSenderSockAddr());
} else {
// we couldn't verify this peer - respond back to them so they know they may need to perform keypair re-generation
static auto deniedPacket = NLPacket::create(PacketType::ICEServerHeartbeatDenied);
@ -164,39 +164,42 @@ SharedNetworkPeer IceServer::addOrUpdateHeartbeatingPeer(NLPacket& packet) {
}
bool IceServer::isVerifiedHeartbeat(const QUuid& domainID, const QByteArray& plaintext, const QByteArray& signature) {
// check if we have a public key for this domain ID - if we do not then fire off the request for it
auto it = _domainPublicKeys.find(domainID);
if (it != _domainPublicKeys.end()) {
// make sure we're not already waiting for a public key for this domain-server
if (!_pendingPublicKeyRequests.contains(domainID)) {
// check if we have a public key for this domain ID - if we do not then fire off the request for it
auto it = _domainPublicKeys.find(domainID);
if (it != _domainPublicKeys.end()) {
// attempt to verify the signature for this heartbeat
const auto rsaPublicKey = it->second.get();
// attempt to verify the signature for this heartbeat
const auto rsaPublicKey = it->second.get();
if (rsaPublicKey) {
auto hashedPlaintext = QCryptographicHash::hash(plaintext, QCryptographicHash::Sha256);
int verificationResult = RSA_verify(NID_sha256,
reinterpret_cast<const unsigned char*>(hashedPlaintext.constData()),
hashedPlaintext.size(),
reinterpret_cast<const unsigned char*>(signature.constData()),
signature.size(),
rsaPublicKey);
if (rsaPublicKey) {
auto hashedPlaintext = QCryptographicHash::hash(plaintext, QCryptographicHash::Sha256);
int verificationResult = RSA_verify(NID_sha256,
reinterpret_cast<const unsigned char*>(hashedPlaintext.constData()),
hashedPlaintext.size(),
reinterpret_cast<const unsigned char*>(signature.constData()),
signature.size(),
rsaPublicKey);
if (verificationResult == 1) {
// this is the only success case - we return true here to indicate that the heartbeat is verified
return true;
} else {
qDebug() << "Failed to verify heartbeat for" << domainID << "- re-requesting public key from API.";
}
if (verificationResult == 1) {
// this is the only success case - we return true here to indicate that the heartbeat is verified
return true;
} else {
qDebug() << "Failed to verify heartbeat for" << domainID << "- re-requesting public key from API.";
// we can't let this user in since we couldn't convert their public key to an RSA key we could use
qWarning() << "Public key for" << domainID << "is not a usable RSA* public key.";
qWarning() << "Re-requesting public key from API";
}
} else {
// we can't let this user in since we couldn't convert their public key to an RSA key we could use
qWarning() << "Public key for" << domainID << "is not a usable RSA* public key.";
qWarning() << "Re-requesting public key from API";
}
}
// we could not verify this heartbeat (missing public key, could not load public key, bad actor)
// ask the metaverse API for the right public key and return false to indicate that this is not verified
requestDomainPublicKey(domainID);
// we could not verify this heartbeat (missing public key, could not load public key, bad actor)
// ask the metaverse API for the right public key and return false to indicate that this is not verified
requestDomainPublicKey(domainID);
}
return false;
}
@ -214,6 +217,9 @@ void IceServer::requestDomainPublicKey(const QUuid& domainID) {
qDebug() << "Requesting public key for domain with ID" << domainID;
// add this to the set of pending public key requests
_pendingPublicKeyRequests.insert(domainID);
networkAccessManager.get(publicKeyRequest);
}
@ -266,6 +272,9 @@ void IceServer::publicKeyReplyFinished(QNetworkReply* reply) {
qWarning() << "Error retreiving public key for domain with ID" << domainID << "-" << reply->errorString();
}
// remove this domain ID from the list of pending public key requests
_pendingPublicKeyRequests.remove(domainID);
reply->deleteLater();
}
@ -282,8 +291,6 @@ void IceServer::sendPeerInformationPacket(const NetworkPeer& peer, const HifiSoc
void IceServer::clearInactivePeers() {
NetworkPeerHash::iterator peerItem = _activePeers.begin();
_lastInactiveCheckTimestamp = QDateTime::currentMSecsSinceEpoch();
while (peerItem != _activePeers.end()) {
SharedNetworkPeer peer = peerItem.value();
@ -301,25 +308,3 @@ void IceServer::clearInactivePeers() {
}
}
}
bool IceServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
// We need an HTTP handler in order to monitor the health of the ice server
// The correct functioning of the ICE server will be determined by its HTTP availability,
if (connection->requestOperation() == QNetworkAccessManager::GetOperation) {
if (url.path() == "/status") {
// figure out if we respond with 0 (we're good) or 1 (we think we're in trouble)
const quint64 MAX_PACKET_GAP_MS_FOR_STUCK_SOCKET = 10 * 1000;
auto sinceLastInactiveCheck = QDateTime::currentMSecsSinceEpoch() - _lastInactiveCheckTimestamp;
int statusNumber = (sinceLastInactiveCheck > MAX_PACKET_GAP_MS_FOR_STUCK_SOCKET) ? 1 : 0;
connection->respond(HTTPConnection::StatusCode200, QByteArray::number(statusNumber));
return true;
}
}
return false;
}

View file

@ -28,11 +28,10 @@
class QNetworkReply;
class IceServer : public QCoreApplication, public HTTPRequestHandler {
class IceServer : public QCoreApplication {
Q_OBJECT
public:
IceServer(int argc, char* argv[]);
bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false);
private slots:
void clearInactivePeers();
void publicKeyReplyFinished(QNetworkReply* reply);
@ -52,13 +51,11 @@ private:
using NetworkPeerHash = QHash<QUuid, SharedNetworkPeer>;
NetworkPeerHash _activePeers;
HTTPManager _httpManager;
using RSAUniquePtr = std::unique_ptr<RSA, std::function<void(RSA*)>>;
using DomainPublicKeyHash = std::unordered_map<QUuid, RSAUniquePtr>;
DomainPublicKeyHash _domainPublicKeys;
quint64 _lastInactiveCheckTimestamp;
QSet<QUuid> _pendingPublicKeyRequests;
};
#endif // hifi_IceServer_h

View file

@ -21,7 +21,7 @@
#include "HifiSockAddr.h"
const QString ICE_SERVER_HOSTNAME = "localhost";
const int ICE_SERVER_DEFAULT_PORT = 7337;
const quint16 ICE_SERVER_DEFAULT_PORT = 7337;
const int ICE_HEARBEAT_INTERVAL_MSECS = 2 * 1000;
const int MAX_ICE_CONNECTION_ATTEMPTS = 5;

View file

@ -34,8 +34,8 @@ const QSet<PacketType> NON_SOURCED_PACKETS = QSet<PacketType>()
<< PacketType::DomainServerAddedNode << PacketType::DomainServerConnectionToken
<< PacketType::DomainSettingsRequest << PacketType::DomainSettings
<< PacketType::ICEServerPeerInformation << PacketType::ICEServerQuery << PacketType::ICEServerHeartbeat
<< PacketType::ICEPing << PacketType::ICEPingReply << PacketType::ICEServerHeartbeatDenied
<< PacketType::AssignmentClientStatus << PacketType::StopNode
<< PacketType::ICEServerHeartbeatACK << PacketType::ICEPing << PacketType::ICEPingReply
<< PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode
<< PacketType::DomainServerRemovedNode;
const QSet<PacketType> RELIABLE_PACKETS = QSet<PacketType>();

View file

@ -93,7 +93,8 @@ public:
MessagesUnsubscribe,
ICEServerHeartbeatDenied,
AssetMappingOperation,
AssetMappingOperationReply
AssetMappingOperationReply,
ICEServerHeartbeatACK
};
};