From 45bb3237231a92962faa90cd40cb0a4cdd6a8e9a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 00:19:04 -0700 Subject: [PATCH] experimental support for mirrored audio packets to other mixers --- assignment-client/src/audio/AudioMixer.cpp | 19 +++++++++++ assignment-client/src/audio/AudioMixer.h | 1 + .../src/audio/AudioMixerClientData.cpp | 32 +++++++++++++++++++ .../src/audio/AudioMixerClientData.h | 2 ++ libraries/networking/src/LimitedNodeList.h | 10 ++++++ libraries/networking/src/NLPacket.h | 1 + libraries/networking/src/Node.h | 5 +++ .../networking/src/udt/PacketHeaders.cpp | 4 ++- libraries/networking/src/udt/PacketHeaders.h | 7 +++- 9 files changed, 79 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bb03a6ec93..87f51ea73c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -92,6 +92,14 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::NodeMuteRequest, this, "handleNodeMuteRequestPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); + packetReceiver.registerListenerForTypes({ + PacketType::MirroredMicrophoneAudioNoEcho, + PacketType::MirroredMicrophoneAudioWithEcho, + PacketType::MirroredInjectAudio, + PacketType::MirroredSilentAudioFrame }, + this, "queueMirroredAudioPacket" + ); + connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -103,6 +111,17 @@ void AudioMixer::queueAudioPacket(QSharedPointer message, Share getOrCreateClientData(node.data())->queuePacket(message, node); } +void AudioMixer::queueMirroredAudioPacket(QSharedPointer message) { + // make sure we have a mirrored node for the original sender of the packet + auto nodeList = DependencyManager::get(); + + auto node = nodeList->addOrUpdateNode(message->getSourceID(), NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr()); + node->setIsMirror(true); + + getOrCreateClientData(node.data())->queuePacket(message, node); +} + void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 18e754016e..b7eebe6aa1 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -63,6 +63,7 @@ private slots: void handleKillAvatarPacket(QSharedPointer packet, SharedNodePointer sendingNode); void queueAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void queueMirroredAudioPacket(QSharedPointer packet); void removeHRTFsForFinishedInjector(const QUuid& streamID); void start(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d5e06504a6..135047f3a5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -71,6 +71,9 @@ void AudioMixerClientData::processPackets() { case PacketType::SilentAudioFrame: { QMutexLocker lock(&getMutex()); parseData(*packet); + + potentiallyMirrorPacket(*packet); + break; } case PacketType::NegotiateAudioFormat: @@ -97,6 +100,35 @@ void AudioMixerClientData::processPackets() { assert(_packetQueue.empty()); } +void AudioMixerClientData::potentiallyMirrorPacket(ReceivedMessage& message) { + auto nodeList = DependencyManager::get(); + if (!nodeList->getMirrorSocket().isNull()) { + PacketType mirroredType; + + if (message.getType() == PacketType::MicrophoneAudioNoEcho) { + mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { + mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::InjectAudio) { + mirroredType = PacketType::MirroredInjectAudio; + } else if (message.getType() == PacketType::SilentAudioFrame) { + mirroredType = PacketType::MirroredSilentAudioFrame; + } else { + return; + } + + // construct an NLPacket to send to the mirror that has the contents of the received packet + std::unique_ptr messageData { new char[message.getSize()] }; + memcpy(messageData.get(), message.getMessage().data(), message.getSize()); + auto packet = NLPacket::fromReceivedPacket(std::move(messageData), message.getSize(), + message.getSenderSockAddr()); + + packet->setType(mirroredType); + + nodeList->sendPacket(std::move(packet), nodeList->getMirrorSocket()); + } +} + void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { quint8 numberOfCodecs; message.readPrimitive(&numberOfCodecs); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 8d76cda2f1..4207362ed3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -124,6 +124,8 @@ private: QReadWriteLock _streamsLock; AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID + void potentiallyMirrorPacket(ReceivedMessage& packet); + using IgnoreZone = AABox; class IgnoreZoneMemo { public: diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 554386f786..16d7f20f99 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -278,6 +278,12 @@ public: void sendFakedHandshakeRequestToNode(SharedNodePointer node); #endif + void setMirrorSocket(const HifiSockAddr& mirrorSocket) { _mirrorSocket = mirrorSocket; } + const HifiSockAddr& getMirrorSocket() { return _mirrorSocket; } + + void setMasterSocket(const HifiSockAddr& masterSocket) { _masterSocket = masterSocket; } + const HifiSockAddr& getMasterSocket() { return _masterSocket; } + public slots: void reset(); void eraseAllNodes(); @@ -386,6 +392,10 @@ protected: } } + + HifiSockAddr _mirrorSocket; + HifiSockAddr _masterSocket; + private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 33de262dfb..f9056bbfaa 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -42,6 +42,7 @@ public: static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); + static std::unique_ptr fromBase(std::unique_ptr packet); // Provided for convenience, try to limit use diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 1092fcc7fa..dd05961678 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -80,6 +80,9 @@ public: bool isIgnoreRadiusEnabled() const { return _ignoreRadiusEnabled; } + bool isMirror() const { return _isMirror; } + void setIsMirror(bool isMirror) { _isMirror = isMirror; } + private: // privatize copy and assignment operator to disallow Node copying Node(const Node &otherNode); @@ -98,6 +101,8 @@ private: mutable QReadWriteLock _ignoredNodeIDSetLock; std::atomic_bool _ignoreRadiusEnabled; + + bool _isMirror { false }; }; Q_DECLARE_METATYPE(Node*) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 9d970fa318..e29bfd0d78 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -39,7 +39,9 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerPeerInformation << PacketType::ICEServerQuery << PacketType::ICEServerHeartbeat << PacketType::ICEServerHeartbeatACK << PacketType::ICEPing << PacketType::ICEPingReply << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode - << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement; + << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement + << PacketType::MirroredMicrophoneAudioNoEcho << PacketType::MirroredMicrophoneAudioWithEcho + << PacketType::MirroredInjectAudio << PacketType::MirroredSilentAudioFrame; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 2cc3a2c42e..f2a13dc449 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -115,7 +115,12 @@ public: AdjustAvatarSorting, OctreeFileReplacement, CollisionEventChanges, - LAST_PACKET_TYPE = CollisionEventChanges + MirroredMicrophoneAudioNoEcho, + MirroredMicrophoneAudioWithEcho, + MirroredInjectAudio, + MirroredSilentAudioFrame, + LAST_PACKET_TYPE = MirroredSilentAudioFrame, + }; };