From 6c3762846808c891d77413b172ba93d52a075603 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 31 May 2021 12:29:48 +1200 Subject: [PATCH 01/11] Add WebRTC data channel --- domain-server/src/DomainServer.cpp | 17 +- domain-server/src/DomainServer.h | 7 +- libraries/networking/CMakeLists.txt | 3 +- libraries/networking/src/NodeType.h | 6 +- .../src/webrtc/WebRTCDataChannels.cpp | 475 ++++++++++++++++++ .../src/webrtc/WebRTCDataChannels.h | 278 ++++++++++ .../src/webrtc/WebRTCSignalingServer.cpp | 4 +- .../src/webrtc/WebRTCSignalingServer.h | 12 +- libraries/shared/src/shared/WebRTC.h | 4 +- 9 files changed, 791 insertions(+), 15 deletions(-) create mode 100644 libraries/networking/src/webrtc/WebRTCDataChannels.cpp create mode 100644 libraries/networking/src/webrtc/WebRTCDataChannels.h diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 02fd810b0e..c7a5700de2 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -165,8 +165,9 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection, DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), -#ifdef WEBRTC_DATA_CHANNEL +#ifdef WEBRTC_DATA_CHANNELS _webrtcSignalingServer(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT, this), + _webrtcDataChannels(NodeType::DomainServer, this), #endif _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) @@ -251,6 +252,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : updateDownstreamNodes(); updateUpstreamNodes(); + setUpWebRTC(); + if (_type != NonMetaverse) { // if we have a metaverse domain, we'll use an access token for API calls resetAccountManagerAccessToken(); @@ -3136,6 +3139,18 @@ void DomainServer::updateUpstreamNodes() { updateReplicationNodes(Upstream); } +void DomainServer::setUpWebRTC() { + + // Inbound WebRTC signaling messages received from a client. + connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, + &_webrtcDataChannels, &WebRTCDataChannels::onSignalingMessage); + + // Outbound WebRTC signaling messages being sent to a client. + connect(&_webrtcDataChannels, &WebRTCDataChannels::signalingMessage, + &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); + +} + void DomainServer::initializeExporter() { static const QString ENABLE_EXPORTER = "monitoring.enable_prometheus_exporter"; static const QString EXPORTER_PORT = "monitoring.prometheus_exporter_port"; diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 29142505a8..23e1299374 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "AssetsBackupHandler.h" @@ -143,6 +144,9 @@ private slots: void updateReplicatedNodes(); void updateDownstreamNodes(); void updateUpstreamNodes(); + + void setUpWebRTC(); + void initializeExporter(); void initializeMetadataExporter(); @@ -314,8 +318,9 @@ private: QThread _assetClientThread; -#ifdef WEBRTC_DATA_CHANNEL +#ifdef WEBRTC_DATA_CHANNELS WebRTCSignalingServer _webrtcSignalingServer; + WebRTCDataChannels _webrtcDataChannels; #endif }; diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 50382cda99..1835c7a6cd 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -11,7 +11,8 @@ endif () if (WIN32) # we need ws2_32.lib on windows, but it's static so we don't bubble it up - target_link_libraries(${TARGET_NAME} ws2_32.lib) + # Libraries needed for WebRTC: security.log winmm.lib + target_link_libraries(${TARGET_NAME} ws2_32.lib security.lib winmm.lib) elseif(APPLE) # IOKit is needed for getting machine fingerprint find_library(FRAMEWORK_IOKIT IOKit) diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 2b2cc4e011..8539ce8fb3 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -4,6 +4,7 @@ // // Created by Stephen Birarda on 05/29/15. // Copyright 2015 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -14,6 +15,10 @@ #pragma once +/// @file +/// @brief NodeType + +/// @brief An 8-bit value identifying the type of a node - domain server, audio mixer, etc. typedef quint8 NodeType_t; namespace NodeType { @@ -37,7 +42,6 @@ namespace NodeType { NodeType_t upstreamType(NodeType_t primaryType); NodeType_t downstreamType(NodeType_t primaryType); - NodeType_t fromString(QString type); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp new file mode 100644 index 0000000000..321b844eb0 --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -0,0 +1,475 @@ +// +// WebRTCDataChannels.cpp +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#include "WebRTCDataChannels.h" + +#if defined(WEBRTC_DATA_CHANNELS) + +#include +#include + +#include "../NetworkLogging.h" + + +// References: +// - https://webrtc.github.io/webrtc-org/native-code/native-apis/ +// - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h + +const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; + +#define WEBRTC_DEBUG + + +void WDCSetSessionDescriptionObserver::OnSuccess() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCSetSessionDescriptionObserver::OnSuccess()"; +#endif +} + +void WDCSetSessionDescriptionObserver::OnFailure(RTCError error) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCSetSessionDescriptionObserver::OnFailure() :" << error.message(); +#endif +} + + +WDCCreateSessionDescriptionObserver::WDCCreateSessionDescriptionObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCCreateSessionDescriptionObserver::OnSuccess(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCCreateSessionDescriptionObserver::OnSuccess()"; +#endif + _parent->sendAnswer(description); + _parent->setLocalDescription(description); +} + +void WDCCreateSessionDescriptionObserver::OnFailure(RTCError error) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCCreateSessionDescriptionObserver::OnFailure() :" << error.message(); +#endif +} + + +WDCPeerConnectionObserver::WDCPeerConnectionObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCPeerConnectionObserver::OnSignalingChange(PeerConnectionInterface::SignalingState newState) { +#ifdef WEBRTC_DEBUG + QStringList states{ + "Stable", + "HaveLocalOffer", + "HaveLocalPrAnswer", + "HaveRemoteOffer", + "HaveRemotePrAnswer", + "Closed" + }; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnSignalingChange()" << newState << states[newState]; +#endif +} + +void WDCPeerConnectionObserver::OnRenegotiationNeeded() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnRenegotiationNeeded()"; +#endif +} + +void WDCPeerConnectionObserver::OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceGatheringChange()" << newState; +#endif +} + +void WDCPeerConnectionObserver::OnIceCandidate(const IceCandidateInterface* candidate) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceCandidate()"; +#endif + _parent->sendIceCandidate(candidate); +} + +void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptr dataChannel) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnDataChannel()"; +#endif + _parent->onDataChannelOpened(dataChannel); +} + +void WDCPeerConnectionObserver::OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) { +} + + +WDCDataChannelObserver::WDCDataChannelObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCDataChannelObserver::OnStateChange() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCDataChannelObserver::OnStateChange()"; +#endif + _parent->onDataChannelStateChanged(); +} + +void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCDataChannelObserver::OnMessage()"; +#endif + _parent->onDataChannelMessageReceived(buffer); +} + + +WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : + _webSocketID(webSocketID), + _parent(parent) +{ +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; +#endif + + // Create observers. + _setSessionDescriptionObserver = new rtc::RefCountedObject(); + _createSessionDescriptionObserver = new rtc::RefCountedObject(this); + _dataChannelObserver = std::make_shared(this); + _peerConnectionObserver = std::make_shared(this); + + // Create new peer connection. + _peerConnection = _parent->createPeerConnection(_peerConnectionObserver); +}; + +void WDCConnection::setRemoteDescription(QJsonObject& description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::setRemoteDescription() :" << description; +#endif + + SdpParseError sdpParseError; + auto sessionDescription = CreateSessionDescription( + description.value("type").toString().toStdString(), + description.value("sdp").toString().toStdString(), + &sdpParseError); + if (!sessionDescription) { + qCWarning(networking_webrtc) << "Error creating WebRTC remote description:" + << QString::fromStdString(sdpParseError.description); + return; + } + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "3. Set remote description:" << sessionDescription; +#endif + _peerConnection->SetRemoteDescription(_setSessionDescriptionObserver, sessionDescription); +} + +void WDCConnection::createAnswer() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::createAnswer()"; + qCDebug(networking_webrtc) << "4.a Create answer"; +#endif + _peerConnection->CreateAnswer(_createSessionDescriptionObserver, PeerConnectionInterface::RTCOfferAnswerOptions()); +} + +void WDCConnection::sendAnswer(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendAnswer()"; + qCDebug(networking_webrtc) << "4.b Send answer to the remote peer"; +#endif + + QJsonObject jsonDescription; + std::string descriptionString; + description->ToString(&descriptionString); + jsonDescription.insert("sdp", QString::fromStdString(descriptionString)); + jsonDescription.insert("type", "answer"); + + QJsonObject jsonWebRTCPayload; + jsonWebRTCPayload.insert("description", jsonDescription); + + QJsonObject jsonObject; + jsonObject.insert("from", QString(_parent->getNodeType())); + jsonObject.insert("to", _webSocketID); + jsonObject.insert("data", jsonWebRTCPayload); + + _parent->sendSignalingMessage(jsonObject); +} + +void WDCConnection::setLocalDescription(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::setLocalDescription()"; + qCDebug(networking_webrtc) << "5. Set local description"; +#endif + _peerConnection->SetLocalDescription(_setSessionDescriptionObserver, description); +} + +void WDCConnection::addIceCandidate(QJsonObject& data) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::addIceCandidate()"; +#endif + + SdpParseError sdpParseError; + auto iceCandidate = CreateIceCandidate( + data.value("sdpMid").toString().toStdString(), + data.value("sdpMLineIndex").toInt(), + data.value("candidate").toString().toStdString(), + &sdpParseError); + if (!iceCandidate) { + qCWarning(networking_webrtc) << "Error adding WebRTC ICE candidate:" + << QString::fromStdString(sdpParseError.description); + return; + } + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "6. Add ICE candidate"; +#endif + _peerConnection->AddIceCandidate(iceCandidate); +} + +void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendIceCandidate()"; +#endif + + std::string candidateString; + candidate->ToString(&candidateString); + QJsonObject jsonCandidate; + jsonCandidate.insert("candidate", QString::fromStdString(candidateString)); + jsonCandidate.insert("sdpMid", QString::fromStdString(candidate->sdp_mid())); + jsonCandidate.insert("sdpMLineIndex", candidate->sdp_mline_index()); + + QJsonObject jsonWebRTCData; + jsonWebRTCData.insert("candidate", jsonCandidate); + + QJsonObject jsonObject; + jsonObject.insert("from", QString(_parent->getNodeType())); + jsonObject.insert("to", _webSocketID); + jsonObject.insert("data", jsonWebRTCData); + QJsonDocument jsonDocument = QJsonDocument(jsonObject); + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "7. Send ICE candidate to the remote peer"; +#endif + _parent->sendSignalingMessage(jsonObject); +} + +void WDCConnection::onDataChannelOpened(rtc::scoped_refptr dataChannel) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelOpened() :" + << dataChannel->id() + << QString::fromStdString(dataChannel->label()) + << QString::fromStdString(dataChannel->protocol()) + << dataChannel->negotiated() + << dataChannel->maxRetransmitTime() + << dataChannel->maxRetransmits() + << dataChannel->maxPacketLifeTime().value_or(-1) + << dataChannel->maxRetransmitsOpt().value_or(-1); +#endif + + _dataChannel = dataChannel; + _dataChannelID = dataChannel->id(); + _dataChannel->RegisterObserver(_dataChannelObserver.get()); + + _parent->onDataChannelOpened(this, _dataChannelID); +} + +void WDCConnection::onDataChannelStateChanged() { + auto state = _dataChannel->state(); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::dataChannelStateChanged() :" << (int)state + << DataChannelInterface::DataStateString(state); +#endif + if (state == DataChannelInterface::kClosed) { + _parent->onDataChannelClosed(this, _dataChannelID); + } +} + +void WDCConnection::onDataChannelMessageReceived(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelMessageReceived()"; +#endif + + auto byteArray = QByteArray(buffer.data.data(), (int)buffer.data.size()); + + // Echo message back to sender. + if (byteArray.startsWith("echo:")) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Echo message back"; +#endif + _parent->sendDataMessage(_dataChannelID, byteArray); // Use parent method to exercise the code stack. + return; + } + + _parent->emitDataMessage(_dataChannelID, byteArray); +} + +bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; +#endif + const int MAX_WEBRTC_BUFFER_SIZE = 16 * 1024 * 1024; // 16MB + if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { + // Don't send, otherwise the data channel will be closed. + return false; + } + return _dataChannel->Send(buffer); +} + + +WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : + _nodeType(nodeType), + _parent(parent) +{ +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; +#endif + + // Create a peer connection factory. +#ifdef WEBRTC_DEBUG + // Numbers are per WebRTC's peer_connection_interface.h. + qCDebug(networking_webrtc) << "1. Create a new PeerConnectionFactoryInterface"; +#endif + _rtcNetworkThread = rtc::Thread::CreateWithSocketServer(); + _rtcNetworkThread->Start(); + _rtcWorkerThread = rtc::Thread::Create(); + _rtcWorkerThread->Start(); + _rtcSignalingThread = rtc::Thread::Create(); + _rtcSignalingThread->Start(); + PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = _rtcNetworkThread.get(); + dependencies.worker_thread = _rtcWorkerThread.get(); + dependencies.signaling_thread = _rtcSignalingThread.get(); + _peerConnectionFactory = CreateModularPeerConnectionFactory(std::move(dependencies)); + if (!_peerConnectionFactory) { + qCWarning(networking_webrtc) << "Failed to create WebRTC peer connection factory"; + } +} + +WebRTCDataChannels::~WebRTCDataChannels() { + QHashIterator i(_connectionsByDataChannel); + while (i.hasNext()) { + i.next(); + delete i.value(); + } + _connectionsByWebSocket.clear(); + _connectionsByDataChannel.clear(); + + _peerConnectionFactory = nullptr; + _rtcSignalingThread->Stop(); + _rtcSignalingThread = nullptr; + _rtcWorkerThread->Stop(); + _rtcWorkerThread = nullptr; + _rtcNetworkThread->Stop(); + _rtcNetworkThread = nullptr; +} + +void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, int dataChannelID) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; +#endif + _connectionsByDataChannel.insert(dataChannelID, connection); +} + +void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, int dataChannelID) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; +#endif + + // Delete WDCConnection. + _connectionsByWebSocket.remove(connection->getWebSocketID()); + _connectionsByDataChannel.remove(dataChannelID); + delete connection; +} + +void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; +#endif + + // Validate message. + const int MAX_DEBUG_DETAIL_LENGTH = 64; + auto data = message.value("data").isObject() ? message.value("data").toObject() : QJsonObject(); + int from = message.value("from").isDouble() ? (quint16)(message.value("from").toInt()) : 0; + if (from <= 0 || from > MAXUINT16 || !data.contains("description") && !data.contains("candidate")) { + qCWarning(networking_webrtc) << "Unexpected signaling message:" + << QJsonDocument(message).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); + return; + } + + // Find or create a connection. + WDCConnection* connection; + if (_connectionsByWebSocket.contains(from)) { + connection = _connectionsByWebSocket.value(from); + } else { + connection = new WDCConnection(from, this); + _connectionsByWebSocket.insert(from, connection); + } + + // Set the remote description and reply with an answer. + if (data.contains("description")) { + auto description = data.value("description").toObject(); + if (description.value("type").toString() == "offer") { + connection->setRemoteDescription(description); + connection->createAnswer(); + } else { + qCWarning(networking_webrtc) << "Unexpected signaling description:" + << QJsonDocument(description).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); + } + } + + // Add a remote ICE candidate. + if (data.contains("candidate")) { + connection->addIceCandidate(data); + } + +} + +void WebRTCDataChannels::sendSignalingMessage(const QJsonObject& message) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::sendSignalingMessage() :" << QJsonDocument(message).toJson(QJsonDocument::Compact); +#endif + emit signalingMessage(message); +} + +void WebRTCDataChannels::emitDataMessage(int dataChannelID, const QByteArray& byteArray) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID; +#endif + emit dataMessage(dataChannelID, byteArray); +} + +bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& byteArray) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::sendDataMessage() :" << dataChannelID; +#endif + + // Find connection. + if (!_connectionsByDataChannel.contains(dataChannelID)) { + qCWarning(networking_webrtc) << "Could not find data channel to send message on!"; + return false; + } + + auto connection = _connectionsByDataChannel.value(dataChannelID); + DataBuffer buffer(byteArray.toStdString(), true); + return connection->sendDataMessage(buffer); +} + +rtc::scoped_refptr WebRTCDataChannels::createPeerConnection( + const std::shared_ptr peerConnectionObserver) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::createPeerConnection()"; +#endif + + PeerConnectionInterface::RTCConfiguration configuration; + PeerConnectionInterface::IceServer iceServer; + iceServer.uri = ICE_SERVER_URI; + configuration.servers.push_back(iceServer); + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; +#endif + return _peerConnectionFactory->CreatePeerConnection(configuration, nullptr, nullptr, peerConnectionObserver.get()); +} + + +#endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h new file mode 100644 index 0000000000..9ea75bc5ff --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -0,0 +1,278 @@ +// +// WebRTCDataChannels.h +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#ifndef vircadia_WebRTCDataChannels_h +#define vircadia_WebRTCDataChannels_h + +#include + +#if defined(WEBRTC_DATA_CHANNELS) + + +#include +#include + +#undef emit // Avoid conflict between Qt signals/slots and the WebRTC library's. +#include +#define emit + +#include "../NodeType.h" + +using namespace webrtc; + +class WebRTCDataChannels; +class WDCConnection; + + +// A WebRTC data channel session description observer. +class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { +public: + // The call to SetLocalDescription or SetRemoteDescription succeeded. + void OnSuccess() override; + + // The call to SetLocalDescription or SetRemoteDescription failed. + void OnFailure(RTCError error) override; +}; + + +// A WebRTC data channel create session description observer. +class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { +public: + WDCCreateSessionDescriptionObserver(WDCConnection* parent); + + // The call to CreateAnswer succeeded. + void OnSuccess(SessionDescriptionInterface* desc) override; + + // The call to CreateAnswer failed. + void OnFailure(RTCError error) override; + +private: + WDCConnection* _parent; +}; + + +// A WebRTC data channel peer connection observer. +class WDCPeerConnectionObserver : public PeerConnectionObserver { +public: + WDCPeerConnectionObserver(WDCConnection* parent); + + // Triggered when the SignalingState changed. + void OnSignalingChange(PeerConnectionInterface::SignalingState newState) override; + + // Triggered when renegotiation is needed. For example, an ICE restart has begun. + void OnRenegotiationNeeded() override; + + // Called any time the IceGatheringState changes. + void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) override; + + // A new ICE candidate has been gathered. + void OnIceCandidate(const IceCandidateInterface* candidate) override; + + // Triggered when a remote peer opens a data channel. + void OnDataChannel(rtc::scoped_refptr dataChannel) override; + + // Called any time the PeerConnectionState changes. + void OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) override; + +private: + WDCConnection* _parent; +}; + + +// A WebRTC data channel observer. +class WDCDataChannelObserver : public DataChannelObserver { +public: + WDCDataChannelObserver(WDCConnection* parent); + + // The data channel state changed. + void OnStateChange() override; + + // A data buffer was successfully received. + void OnMessage(const DataBuffer& buffer) override; + +private: + WDCConnection* _parent; +}; + + +/// @brief A WebRTC data channel connection. +/// @details Opens and manages a WebRTC data channel connection. +class WDCConnection { + +public: + /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. + /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. + /// @param parent The parent WebRTCDataChannels object. + WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent); + + /// @brief Gets the WebSocket ID. + /// @return The ID of the WebSocket. + quint16 getWebSocketID() { return _webSocketID; } + + /// @brief Gets the WebRTC data channel ID. + /// @return The WebRTC data channel ID. `-1` if not open yet. + int getDataChannelID() { return _dataChannelID; } + + + /// @brief Sets the remote session description received from the remote client via the signaling channel. + /// @param description The remote session description. + void setRemoteDescription(QJsonObject& description); + + /// @brief Creates an answer to an offer received from the remote client via the signaling channel. + void createAnswer(); + + /// @brief Sends an answer to the remote client via the signaling channel. + /// @param description The answer. + void sendAnswer(SessionDescriptionInterface* description); + + /// @brief Sets the local session description on the WebRTC data channel being connected. + /// @param description The local session description. + void setLocalDescription(SessionDescriptionInterface* description); + + /// @brief Adds an ICE candidate received from the remote client via the signaling channel. + /// @param data The ICE candidate. + void addIceCandidate(QJsonObject& data); + + /// @brief Sends an ICE candidate to the remote vlient via the signaling channel. + /// @param candidate The ICE candidate. + void sendIceCandidate(const IceCandidateInterface* candidate); + + /// @brief Handles the WebRTC data channel being opened. + /// @param dataChannel The WebRTC data channel. + void onDataChannelOpened(rtc::scoped_refptr dataChannel); + + /// @brief Handles a change in the state of the WebRTC data channel. + void onDataChannelStateChanged(); + + + /// @brief Handles a message being received on the WebRTC data channel. + /// @param buffer The message received. + void onDataChannelMessageReceived(const DataBuffer& buffer); + + /// @brief Sends a message on the WebRTC data channel. + /// @param buffer The message to send. + /// @return `true` if the message was sent, otherwise `false`. + bool sendDataMessage(const DataBuffer& buffer); + +private: + WebRTCDataChannels* _parent; + quint16 _webSocketID { 0 }; + int _dataChannelID { -1 }; + + rtc::scoped_refptr _setSessionDescriptionObserver { nullptr }; + rtc::scoped_refptr _createSessionDescriptionObserver { nullptr }; + + std::shared_ptr _dataChannelObserver { nullptr }; + rtc::scoped_refptr _dataChannel { nullptr }; + + std::shared_ptr _peerConnectionObserver { nullptr }; + rtc::scoped_refptr _peerConnection { nullptr }; +}; + + +/// @brief Manages WebRTC data channels on the domain server or an assignment clients that Interface clients can connect to. +/// +/// @details Presents multiple individual WebRTC data channels as a single one-to-many WebRTCDataChannels object. Interface +/// clients may use WebRTC data channels for Vircadia protocol network communications instead of UDP. +/// A WebRTCSignalingServer is used in the process of setting up a WebRTC data channel between an Interface client and the +/// domain server or assignment client. +/// The Interface client initiates the connection - including initiating the data channel - and the domain server or assignment +/// client responds. +/// +/// Additionally, for debugging purposes, instead of containing a Vircadia protocol payload, a WebRTC message may be an echo +/// request. This is bounced back to the client. +/// +class WebRTCDataChannels : public QObject { + Q_OBJECT + +public: + + /// @brief Constructs a new WebRTCDataChannels object. + /// @paramm nodeType The type of node that the WebRTCDataChannels object is being used in. + /// @param parent The parent Qt object. + WebRTCDataChannels(NodeType_t nodeType, QObject* parent); + + /// @brief Destroys a WebRTCDataChannels object. + ~WebRTCDataChannels(); + + /// @brief Returns the type of node that the WebRTCDataChannels object is being used in. + /// @return The type of node. + NodeType_t getNodeType() { + return _nodeType; + } + + /// @brief Handles a WebRTC data channel opening. + /// @param connection The WebRTC data channel connection. + /// @param dataChannelID The WebRTC data channel ID. + void onDataChannelOpened(WDCConnection* connection, int dataChannelID); + + /// @brief Handles a WebRTC data channel closing. + /// @param connection The WebRTC data channel connection. + /// @param dataChannelID The WebRTC data channel ID. + void onDataChannelClosed(WDCConnection* connection, int dataChannelID); + + /// @brief Emits a signalingMessage received for the Interface client. + /// @param message The WebRTC signaling message to send. + void sendSignalingMessage(const QJsonObject& message); + + /// @brief Emits a dataMessage received from the Interface client. + /// @param dataChannelID The WebRTC data channel the message was received on. + /// @param byteArray The data message received. + void emitDataMessage(int dataChannelID, const QByteArray& byteArray); + + /// @brief Sends a data message to an Interface client. + /// @param dataChannelID The WebRTC channel ID of the Interface client. + /// @param message The data message to send. + /// @return `true` if the data message was sent, otherwise `false`. + bool sendDataMessage(int dataChannelID, const QByteArray& message); + + /// @brief Creates a new WebRTC peer connection for connecting to an Interface client. + /// @param peerConnectionObserver An observer to monitor the WebRTC peer connection. + /// @return The new WebRTC peer connection. + rtc::scoped_refptr createPeerConnection( + const std::shared_ptr peerConnectionObserver); + +public slots: + + /// @brief Handles a WebRTC signaling message received from the Interface client. + /// @param message The WebRTC signaling message. + void onSignalingMessage(const QJsonObject& message); + +signals: + + /// @brief A WebRTC signaling message to be sent to the Interface client. + /// @description This message is for the WebRTCSignalingServer to send. + /// @param message The WebRTC signaling message to send. + void signalingMessage(const QJsonObject& message); + + /// @brief A WebRTC data message received from the Interface client. + /// @description This message is for handling at a higher level in the Vircadia protocol. + /// @param dataChannelID The WebRTC data channel ID. + /// @param byteArray The Vircadia protocol message. + void dataMessage(int dataChannelID, const QByteArray& byteArray); + +private: + + QObject* _parent; + + NodeType_t _nodeType; + + std::unique_ptr _rtcNetworkThread { nullptr }; + std::unique_ptr _rtcWorkerThread { nullptr }; + std::unique_ptr _rtcSignalingThread { nullptr }; + + rtc::scoped_refptr _peerConnectionFactory { nullptr }; + + QHash _connectionsByWebSocket; + QHash _connectionsByDataChannel; +}; + + +#endif // WEBRTC_DATA_CHANNELS + +#endif // vircadia_WebRTCDataChannels_h diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index 66b1d6616c..8c661bbf8c 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -8,7 +8,7 @@ #include "WebRTCSignalingServer.h" -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include @@ -93,4 +93,4 @@ void WebRTCSignalingServer::newWebSocketConnection() { _webSockets.insert(webSocket->peerPort(), webSocket); } -#endif // WEBRTC_DATA_CHANNEL +#endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index 41d79dbd57..9f4214d3c7 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -2,18 +2,16 @@ // WebRTCSignalingServer.h // libraries/networking/src/webrtc // -// Provides a signaling channel for setting up WebRTC connections between the Web app and the domain servers and mixers. -// // Created by David Rowe on 16 May 2021. // Copyright 2021 Vircadia contributors. // -#ifndef vircadia_SignalingServer_h -#define vircadia_SignalingServer_h +#ifndef vircadia_WebRTCSignalingServer_h +#define vircadia_WebRTCSignalingServer_h #include -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include @@ -99,6 +97,6 @@ private: }; -#endif // WEBRTC_DATA_CHANNEL +#endif // WEBRTC_DATA_CHANNELS -#endif // vircadia_SignalingServer_h +#endif // vircadia_WebRTCSignalingServer_h diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index f2b64d7a2a..888877eadb 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -17,7 +17,7 @@ #endif // WEBRTC_AUDIO: WebRTC audio features, e.g., echo canceling. -// WEBRTC_DATA_CHANNEL: WebRTC client-server connections in parallel with UDP. +// WEBRTC_DATA_CHANNELS: WebRTC client-server connections in parallel with UDP. #if defined(Q_OS_MAC) # define WEBRTC_AUDIO 1 @@ -25,7 +25,7 @@ # define WEBRTC_LEGACY 1 #elif defined(Q_OS_WIN) # define WEBRTC_AUDIO 1 -# define WEBRTC_DATA_CHANNEL 1 +# define WEBRTC_DATA_CHANNELS 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 From e6c49cf407f395bdbb4ede7dcfee7b4706e3ddc3 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:32:28 +1200 Subject: [PATCH 02/11] Fix data channel ID --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 16 +++++++++++----- .../networking/src/webrtc/WebRTCDataChannels.h | 12 +++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 321b844eb0..92cba6fd96 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -129,7 +129,7 @@ WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : _parent(parent) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; + qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID; #endif // Create observers. @@ -267,7 +267,7 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr #endif _dataChannel = dataChannel; - _dataChannelID = dataChannel->id(); + _dataChannelID = _parent->getNewDataChannelID(); // Not dataChannel->id() because it's only unique per peer connection. _dataChannel->RegisterObserver(_dataChannelObserver.get()); _parent->onDataChannelOpened(this, _dataChannelID); @@ -346,7 +346,7 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : } WebRTCDataChannels::~WebRTCDataChannels() { - QHashIterator i(_connectionsByDataChannel); + QHashIterator i(_connectionsByDataChannel); while (i.hasNext()) { i.next(); delete i.value(); @@ -363,14 +363,20 @@ WebRTCDataChannels::~WebRTCDataChannels() { _rtcNetworkThread = nullptr; } -void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, int dataChannelID) { +quint16 WebRTCDataChannels::getNewDataChannelID() { + static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; + _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); + return _lastDataChannelID; +} + +void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; #endif _connectionsByDataChannel.insert(dataChannelID, connection); } -void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, int dataChannelID) { +void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; #endif diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 9ea75bc5ff..ca14e9ae81 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -206,15 +206,19 @@ public: return _nodeType; } + /// @brief Get a new data channel ID to uniquely identify a WDCConnection. + /// @return A new data channel ID. + quint16 getNewDataChannelID(); + /// @brief Handles a WebRTC data channel opening. /// @param connection The WebRTC data channel connection. /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelOpened(WDCConnection* connection, int dataChannelID); + void onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID); /// @brief Handles a WebRTC data channel closing. /// @param connection The WebRTC data channel connection. /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelClosed(WDCConnection* connection, int dataChannelID); + void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); /// @brief Emits a signalingMessage received for the Interface client. /// @param message The WebRTC signaling message to send. @@ -268,8 +272,10 @@ private: rtc::scoped_refptr _peerConnectionFactory { nullptr }; + quint16 _lastDataChannelID { 0 }; + QHash _connectionsByWebSocket; - QHash _connectionsByDataChannel; + QHash _connectionsByDataChannel; }; From 63ec9332901c1a0025e0080396fbe329672bc03e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:34:17 +1200 Subject: [PATCH 03/11] Code tidying --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 92cba6fd96..497ecf9a55 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -280,6 +280,11 @@ void WDCConnection::onDataChannelStateChanged() { << DataChannelInterface::DataStateString(state); #endif if (state == DataChannelInterface::kClosed) { + _dataChannel->Close(); + _dataChannel = nullptr; + // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. + //_peerConnection->Close(); + //_peerConnection = nullptr; _parent->onDataChannelClosed(this, _dataChannelID); } } @@ -346,6 +351,9 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : } WebRTCDataChannels::~WebRTCDataChannels() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::~WebRTCDataChannels()"; +#endif QHashIterator i(_connectionsByDataChannel); while (i.hasNext()) { i.next(); @@ -384,7 +392,8 @@ void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 // Delete WDCConnection. _connectionsByWebSocket.remove(connection->getWebSocketID()); _connectionsByDataChannel.remove(dataChannelID); - delete connection; + // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. + //delete connection; } void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { @@ -474,7 +483,9 @@ rtc::scoped_refptr WebRTCDataChannels::createPeerConnec #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; #endif - return _peerConnectionFactory->CreatePeerConnection(configuration, nullptr, nullptr, peerConnectionObserver.get()); + PeerConnectionDependencies dependencies(peerConnectionObserver.get()); + auto result = _peerConnectionFactory->CreatePeerConnection(configuration, std::move(dependencies)); + return result; } From c28f4749ed8ca3fe996afdba7be53864e22e6938 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:34:38 +1200 Subject: [PATCH 04/11] Doxygen tidying --- .../src/webrtc/WebRTCDataChannels.h | 57 ++++++++++++------- .../src/webrtc/WebRTCSignalingServer.h | 12 ++-- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index ca14e9ae81..d9bb213a1d 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -29,26 +29,33 @@ class WebRTCDataChannels; class WDCConnection; -// A WebRTC data channel session description observer. +/// @addtogroup Networking +/// @{ + +/// @brief A WebRTC session description observer. class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { public: - // The call to SetLocalDescription or SetRemoteDescription succeeded. + + /// @brief The call to SetLocalDescription or SetRemoteDescription succeeded. void OnSuccess() override; - // The call to SetLocalDescription or SetRemoteDescription failed. + /// @brief The call to SetLocalDescription or SetRemoteDescription failed. + /// @param error Error information. void OnFailure(RTCError error) override; }; -// A WebRTC data channel create session description observer. +/// @brief A WebRTC create session description observer. class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: WDCCreateSessionDescriptionObserver(WDCConnection* parent); - // The call to CreateAnswer succeeded. + /// @brief The call to CreateAnswer succeeded. + /// @param The session description. void OnSuccess(SessionDescriptionInterface* desc) override; - // The call to CreateAnswer failed. + //@ @brief The call to CreateAnswer failed. + /// @param error Error information. void OnFailure(RTCError error) override; private: @@ -56,27 +63,32 @@ private: }; -// A WebRTC data channel peer connection observer. +/// @brief A WebRTC peer connection observer. class WDCPeerConnectionObserver : public PeerConnectionObserver { public: WDCPeerConnectionObserver(WDCConnection* parent); - // Triggered when the SignalingState changed. + /// @brief Called when the SignalingState changes. + /// @param newState The new signaling state. void OnSignalingChange(PeerConnectionInterface::SignalingState newState) override; - // Triggered when renegotiation is needed. For example, an ICE restart has begun. + /// @brief Called when renegotiation is needed. For example, an ICE restart has begun. void OnRenegotiationNeeded() override; - // Called any time the IceGatheringState changes. + /// @brief Called when the ICE gather state changes. + /// @param newState The new ICE gathering state. void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) override; - // A new ICE candidate has been gathered. + /// @brief Called when a new ICE candidate has been gathered. + /// @param candidate The new ICE candidate. void OnIceCandidate(const IceCandidateInterface* candidate) override; - // Triggered when a remote peer opens a data channel. + /// @brief Called when a remote peer opens a data channel. + /// @param dataChannel The data channel. void OnDataChannel(rtc::scoped_refptr dataChannel) override; - // Called any time the PeerConnectionState changes. + /// @brief Called when the peer connection state changes. + /// @param newState The new peer connection state. void OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) override; private: @@ -84,15 +96,16 @@ private: }; -// A WebRTC data channel observer. +/// @brief A WebRTC data channel observer. class WDCDataChannelObserver : public DataChannelObserver { public: WDCDataChannelObserver(WDCConnection* parent); - // The data channel state changed. + /// @brief The data channel state changed. void OnStateChange() override; - // A data buffer was successfully received. + /// @brief A data channel message was received. + /// @param The message received. void OnMessage(const DataBuffer& buffer) override; private: @@ -175,7 +188,7 @@ private: }; -/// @brief Manages WebRTC data channels on the domain server or an assignment clients that Interface clients can connect to. +/// @brief Manages WebRTC data channels on the domain server or an assignment client that Interface clients can connect to. /// /// @details Presents multiple individual WebRTC data channels as a single one-to-many WebRTCDataChannels object. Interface /// clients may use WebRTC data channels for Vircadia protocol network communications instead of UDP. @@ -193,7 +206,7 @@ class WebRTCDataChannels : public QObject { public: /// @brief Constructs a new WebRTCDataChannels object. - /// @paramm nodeType The type of node that the WebRTCDataChannels object is being used in. + /// @param nodeType The type of node that the WebRTCDataChannels object is being used in. /// @param parent The parent Qt object. WebRTCDataChannels(NodeType_t nodeType, QObject* parent); @@ -220,7 +233,7 @@ public: /// @param dataChannelID The WebRTC data channel ID. void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); - /// @brief Emits a signalingMessage received for the Interface client. + /// @brief Emits a signalingMessage to be sent to the Interface client. /// @param message The WebRTC signaling message to send. void sendSignalingMessage(const QJsonObject& message); @@ -250,12 +263,12 @@ public slots: signals: /// @brief A WebRTC signaling message to be sent to the Interface client. - /// @description This message is for the WebRTCSignalingServer to send. + /// @details This message is for the WebRTCSignalingServer to send. /// @param message The WebRTC signaling message to send. void signalingMessage(const QJsonObject& message); /// @brief A WebRTC data message received from the Interface client. - /// @description This message is for handling at a higher level in the Vircadia protocol. + /// @details This message is for handling at a higher level in the Vircadia protocol. /// @param dataChannelID The WebRTC data channel ID. /// @param byteArray The Vircadia protocol message. void dataMessage(int dataChannelID, const QByteArray& byteArray); @@ -279,6 +292,8 @@ private: }; +/// @} + #endif // WEBRTC_DATA_CHANNELS #endif // vircadia_WebRTCDataChannels_h diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index 9f4214d3c7..f2e8594b91 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -19,8 +19,11 @@ #include "../HifiSockAddr.h" -/// @brief WebRTC signaling server that Interface clients can use to initiate WebRTC connections to the domain server and -/// assignment clients. +/// @addtogroup Networking +/// @{ + +/// @brief Provides a WebRTC signaling server that Interface clients can use to initiate WebRTC connections to the domain server +/// and its assignment clients. /// /// @details The signaling server is expected to be hosted in the domain server. It provides a WebSocket for Interface clients /// to use in the WebRTC signaling handshake process to establish WebRTC data channel connections to each of the domain server @@ -48,14 +51,14 @@ /// | `to` | WebSocket port number | /// | `from` | NodeType | /// | [`data`] | WebRTC payload | -/// | [`echo`] | Echo request | +/// | [`echo`] | Echo response | /// class WebRTCSignalingServer : public QObject { Q_OBJECT public: - /// @brief Constructs a new WebRTCSignalingServer. + /// @brief Constructs a new WebRTCSignalingServer object. /// @param address The IP address to use for the WebSocket. /// @param port The port to use for the WebSocket. /// @param parent Qt parent object. @@ -96,6 +99,7 @@ private: QTimer* _isWebSocketServerListeningTimer; }; +/// @} #endif // WEBRTC_DATA_CHANNELS From 645dc265ab5a87792ba055a9770f369c0c59c45d Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 15:41:25 +1200 Subject: [PATCH 05/11] Fix non-WebRTC domain server builds --- domain-server/src/DomainServer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index c7a5700de2..4f80e82681 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -3140,6 +3140,7 @@ void DomainServer::updateUpstreamNodes() { } void DomainServer::setUpWebRTC() { +#ifdef WEBRTC_DATA_CHANNELS // Inbound WebRTC signaling messages received from a client. connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, @@ -3149,6 +3150,7 @@ void DomainServer::setUpWebRTC() { connect(&_webrtcDataChannels, &WebRTCDataChannels::signalingMessage, &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); +#endif } void DomainServer::initializeExporter() { From 28c408de9839dbad34b987607bbc7c6c20e53e0f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 5 Jun 2021 18:23:37 +1200 Subject: [PATCH 06/11] Typo --- domain-server/src/DomainServer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 5d2ac762ac..eccf67d5b6 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -28,7 +28,7 @@ #include #include #include -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include #endif From 6b188d888ec5df42a13c6c527fdf845c02e23678 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 28 Jun 2021 11:15:18 +1200 Subject: [PATCH 07/11] Update to latest Vircadia-Web --- vircadia-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vircadia-web b/vircadia-web index ac1f13a39c..ab6c8b1a54 160000 --- a/vircadia-web +++ b/vircadia-web @@ -1 +1 @@ -Subproject commit ac1f13a39c702ee54bf2cda8bc35e5d34f7f0756 +Subproject commit ab6c8b1a54aec359b1894f70722f69cfec7f04f1 From dacda8405c170d1ecd8ea333267deabd1f2e1d8a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 3 Jul 2021 08:43:31 +1200 Subject: [PATCH 08/11] Typo --- libraries/networking/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 1835c7a6cd..7a11329a14 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -11,7 +11,7 @@ endif () if (WIN32) # we need ws2_32.lib on windows, but it's static so we don't bubble it up - # Libraries needed for WebRTC: security.log winmm.lib + # Libraries needed for WebRTC: security.lib winmm.lib target_link_libraries(${TARGET_NAME} ws2_32.lib security.lib winmm.lib) elseif(APPLE) # IOKit is needed for getting machine fingerprint From f6a8ae285db16d48de9cb35e23c49820bf1b5fb3 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 15:50:52 +1200 Subject: [PATCH 09/11] Fix WebRTC peer connection not being closed properly --- .../src/webrtc/WebRTCDataChannels.cpp | 94 +++++++++++++++---- .../src/webrtc/WebRTCDataChannels.h | 31 +++++- 2 files changed, 100 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 497ecf9a55..48ec1b7b9b 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -22,7 +22,7 @@ const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; -#define WEBRTC_DEBUG +// #define WEBRTC_DEBUG void WDCSetSessionDescriptionObserver::OnSuccess() { @@ -102,6 +102,10 @@ void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptronPeerConnectionStateChanged(newState); } @@ -253,6 +257,20 @@ void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { _parent->sendSignalingMessage(jsonObject); } +void WDCConnection::onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state) { +#ifdef WEBRTC_DEBUG + const char* STATES[] = { + "New", + "Connecting", + "Connected", + "Disconnected", + "Failed", + "Closed" + }; + qCDebug(networking_webrtc) << "WDCConnection::onPeerConnectionStateChanged() :" << (int)state << STATES[(int)state]; +#endif +} + void WDCConnection::onDataChannelOpened(rtc::scoped_refptr dataChannel) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::onDataChannelOpened() :" @@ -276,16 +294,19 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr void WDCConnection::onDataChannelStateChanged() { auto state = _dataChannel->state(); #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCConnection::dataChannelStateChanged() :" << (int)state + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelStateChanged() :" << (int)state << DataChannelInterface::DataStateString(state); #endif if (state == DataChannelInterface::kClosed) { - _dataChannel->Close(); + // Close data channel. + _dataChannel->UnregisterObserver(); + _dataChannelObserver = nullptr; _dataChannel = nullptr; - // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. - //_peerConnection->Close(); - //_peerConnection = nullptr; - _parent->onDataChannelClosed(this, _dataChannelID); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of data channel"; +#endif + // Close peer connection. + _parent->closePeerConnection(this); } } @@ -320,6 +341,18 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { return _dataChannel->Send(buffer); } +void WDCConnection::closePeerConnection() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::closePeerConnection()"; +#endif + _peerConnection->Close(); + _peerConnection = nullptr; + _peerConnectionObserver = nullptr; +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of peer connection"; +#endif +} + WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : _nodeType(nodeType), @@ -348,6 +381,9 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : if (!_peerConnectionFactory) { qCWarning(networking_webrtc) << "Failed to create WebRTC peer connection factory"; } + + // Set up mechanism for closing peer connections. + connect(this, &WebRTCDataChannels::closePeerConnectionSoon, this, &WebRTCDataChannels::closePeerConnectionNow); } WebRTCDataChannels::~WebRTCDataChannels() { @@ -384,18 +420,6 @@ void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 _connectionsByDataChannel.insert(dataChannelID, connection); } -void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID) { -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; -#endif - - // Delete WDCConnection. - _connectionsByWebSocket.remove(connection->getWebSocketID()); - _connectionsByDataChannel.remove(dataChannelID); - // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. - //delete connection; -} - void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; @@ -481,12 +505,42 @@ rtc::scoped_refptr WebRTCDataChannels::createPeerConnec configuration.servers.push_back(iceServer); #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; + qCDebug(networking_webrtc) << "2. Create a new peer connection"; #endif PeerConnectionDependencies dependencies(peerConnectionObserver.get()); auto result = _peerConnectionFactory->CreatePeerConnection(configuration, std::move(dependencies)); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Created peer connection"; +#endif return result; } +void WebRTCDataChannels::closePeerConnection(WDCConnection* connection) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::closePeerConnection()"; +#endif + // Use Qt's signals/slots mechanism to close the peer connection on its own call stack, separate from the DataChannel + // callback that initiated the peer connection. + // https://bugs.chromium.org/p/webrtc/issues/detail?id=3721 + emit closePeerConnectionSoon(connection); +} + + +void WebRTCDataChannels::closePeerConnectionNow(WDCConnection* connection) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::closePeerConnectionNow()"; +#endif + // Close the peer connection. + connection->closePeerConnection(); + + // Delete the WDCConnection. + _connectionsByWebSocket.remove(connection->getWebSocketID()); + _connectionsByDataChannel.remove(connection->getDataChannelID()); + delete connection; +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of connection"; +#endif +} + #endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index d9bb213a1d..0b8caf80b5 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -48,6 +48,7 @@ public: /// @brief A WebRTC create session description observer. class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: + WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. @@ -118,6 +119,7 @@ private: class WDCConnection { public: + /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. /// @param parent The parent WebRTCDataChannels object. @@ -155,6 +157,10 @@ public: /// @param candidate The ICE candidate. void sendIceCandidate(const IceCandidateInterface* candidate); + /// @brief Monitors the peer connection state. + /// @param state The new peer connection state. + void onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state); + /// @brief Handles the WebRTC data channel being opened. /// @param dataChannel The WebRTC data channel. void onDataChannelOpened(rtc::scoped_refptr dataChannel); @@ -171,6 +177,9 @@ public: /// @param buffer The message to send. /// @return `true` if the message was sent, otherwise `false`. bool sendDataMessage(const DataBuffer& buffer); + + /// @brief Closes the WebRTC peer connection. + void closePeerConnection(); private: WebRTCDataChannels* _parent; @@ -228,11 +237,6 @@ public: /// @param dataChannelID The WebRTC data channel ID. void onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID); - /// @brief Handles a WebRTC data channel closing. - /// @param connection The WebRTC data channel connection. - /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); - /// @brief Emits a signalingMessage to be sent to the Interface client. /// @param message The WebRTC signaling message to send. void sendSignalingMessage(const QJsonObject& message); @@ -254,12 +258,24 @@ public: rtc::scoped_refptr createPeerConnection( const std::shared_ptr peerConnectionObserver); + /// @brief Initiates closing the peer connection for a WebRTC data channel. + /// @details Emits a {@link WebRTCDataChannels.closePeerConnectionSoon} signal which is connected to + /// {@link WebRTCDataChannels.closePeerConnectionNow} in order to close the peer connection on a new call stack. This is + /// necessary to work around a WebRTC library limitation. + /// @param connection The WebRTC data channel connection. + void closePeerConnection(WDCConnection* connection); + public slots: /// @brief Handles a WebRTC signaling message received from the Interface client. /// @param message The WebRTC signaling message. void onSignalingMessage(const QJsonObject& message); + /// @brief Closes the peer connection for a WebRTC data channel. + /// @details Used by {@link WebRTCDataChannels.closePeerConnection}. + /// @param connection The WebRTC data channel connection. + void closePeerConnectionNow(WDCConnection* connection); + signals: /// @brief A WebRTC signaling message to be sent to the Interface client. @@ -273,6 +289,11 @@ signals: /// @param byteArray The Vircadia protocol message. void dataMessage(int dataChannelID, const QByteArray& byteArray); + /// @brief Signals that the peer connection for a WebRTC data channel should be closed. + /// @details Used by {@link WebRTCDataChannels.closePeerConnection}. + /// @param connection The WebRTC data channel connection. + void closePeerConnectionSoon(WDCConnection* connection); + private: QObject* _parent; From e682336cc205ea4e0fc9b957efd32e3c34539e83 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 15:57:53 +1200 Subject: [PATCH 10/11] Add missing Doxygen --- libraries/networking/src/webrtc/WebRTCDataChannels.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 0b8caf80b5..3edf9f9b9a 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -49,6 +49,8 @@ public: class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: + /// @brief Constructs a session description observer. + /// @param parent The parent connection object. WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. @@ -67,6 +69,9 @@ private: /// @brief A WebRTC peer connection observer. class WDCPeerConnectionObserver : public PeerConnectionObserver { public: + + /// @brief Constructs a peer connection observer. + /// @param parent The parent connection object. WDCPeerConnectionObserver(WDCConnection* parent); /// @brief Called when the SignalingState changes. @@ -100,6 +105,9 @@ private: /// @brief A WebRTC data channel observer. class WDCDataChannelObserver : public DataChannelObserver { public: + + /// @brief Constructs a data channel observer. + /// @param parent The parent connection object. WDCDataChannelObserver(WDCConnection* parent); /// @brief The data channel state changed. From 7ecd9b6b8e83456715e90407667c91f35991e39e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 16:38:47 +1200 Subject: [PATCH 11/11] Miscellaneous tidying --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 12 +++++++----- libraries/networking/src/webrtc/WebRTCDataChannels.h | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 48ec1b7b9b..69c954ad4b 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -21,6 +21,7 @@ // - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; +const int MAX_WEBRTC_BUFFER_SIZE = 16777216; // 16MB // #define WEBRTC_DEBUG @@ -128,9 +129,9 @@ void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { } -WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : - _webSocketID(webSocketID), - _parent(parent) +WDCConnection::WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID) : + _parent(parent), + _webSocketID(webSocketID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID; @@ -333,10 +334,11 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; #endif - const int MAX_WEBRTC_BUFFER_SIZE = 16 * 1024 * 1024; // 16MB if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { // Don't send, otherwise the data channel will be closed. return false; + } else { + qCDebug(networking_webrtc) << "WebRTC send buffer overflow"; } return _dataChannel->Send(buffer); } @@ -440,7 +442,7 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { if (_connectionsByWebSocket.contains(from)) { connection = _connectionsByWebSocket.value(from); } else { - connection = new WDCConnection(from, this); + connection = new WDCConnection(this, from); _connectionsByWebSocket.insert(from, connection); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 3edf9f9b9a..75325781ed 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -129,9 +129,9 @@ class WDCConnection { public: /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. - /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. /// @param parent The parent WebRTCDataChannels object. - WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent); + /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. + WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID); /// @brief Gets the WebSocket ID. /// @return The ID of the WebSocket.