From da5a89f5827f9940e40fb8ce21515b2722d3bafb Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 13 Feb 2017 21:38:29 -0500 Subject: [PATCH] move codec negotiation to client data --- assignment-client/src/audio/AudioMixer.cpp | 88 +++++++------------ assignment-client/src/audio/AudioMixer.h | 10 +-- .../src/audio/AudioMixerClientData.cpp | 27 ++++-- .../src/audio/AudioMixerClientData.h | 10 ++- .../plugins/src/plugins/PluginManager.cpp | 2 - 5 files changed, 67 insertions(+), 70 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index dc6722cf09..f9a28d91d7 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -47,12 +47,16 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading"; int AudioMixer::_numStaticJitterFrames{ -1 }; float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD }; float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE }; +QString AudioMixer::_codecPreferenceOrder{}; QHash AudioMixer::_audioZones; QVector AudioMixer::_zoneSettings; QVector AudioMixer::_zoneReverbSettings; AudioMixer::AudioMixer(ReceivedMessage& message) : ThreadedAssignment(message) { + // initialize the plugins + PluginManager::getInstance()->getCodecPlugins(); + auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); @@ -61,9 +65,9 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::MicrophoneAudioWithEcho, PacketType::InjectAudio, PacketType::AudioStreamStats, - PacketType::SilentAudioFrame }, + PacketType::SilentAudioFrame, + PacketType::NegotiateAudioFormat }, this, "queueAudioPacket"); - packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); @@ -75,12 +79,12 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } -void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer sendingNode) { +void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer node) { if (message->getType() == PacketType::SilentAudioFrame) { _numSilentPackets++; } - getOrCreateClientData(sendingNode.data())->queuePacket(message); + getOrCreateClientData(node.data())->queuePacket(message, node); } void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { @@ -119,69 +123,41 @@ InputPluginList getInputPlugins() { return result; } -void saveInputPluginSettings(const InputPluginList& plugins) { -} +// must be here to satisfy a reference in PluginManager::saveSettings() +void saveInputPluginSettings(const InputPluginList& plugins) {} - -void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { - QStringList availableCodecs; - auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); - if (codecPlugins.size() > 0) { - for (auto& plugin : codecPlugins) { - auto codecName = plugin->getName(); - qDebug() << "Codec available:" << codecName; - availableCodecs.append(codecName); - } - } else { - qDebug() << "No Codecs available..."; - } - - CodecPluginPointer selectedCodec; +const std::pair AudioMixer::negotiateCodec(std::vector codecs) { QString selectedCodecName; + CodecPluginPointer selectedCodec; + // get the available codecs (on the mixer) + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + std::map availableCodecs; + std::for_each(codecPlugins.cbegin(), codecPlugins.cend(), + [&availableCodecs](const CodecPluginPointer& codec) { + availableCodecs[codec->getName()] = codec; + }); + + // get the codec preferences (on the mixer) QStringList codecPreferenceList = _codecPreferenceOrder.split(","); - // read the codecs requested by the client + // read the codecs requested (by the client) const int MAX_PREFERENCE = 99999; - int preferredCodecIndex = MAX_PREFERENCE; - QString preferredCodec; - quint8 numberOfCodecs = 0; - message->readPrimitive(&numberOfCodecs); - qDebug() << "numberOfCodecs:" << numberOfCodecs; - QStringList codecList; - for (quint16 i = 0; i < numberOfCodecs; i++) { - QString requestedCodec = message->readString(); - int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec); - bool codecAvailable = availableCodecs.contains(requestedCodec); - qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable; + int minPreference = MAX_PREFERENCE; + for (auto& codec : codecs) { + bool codecAvailable = (availableCodecs.count(codec) > 0); + if (codecAvailable) { - codecList.append(requestedCodec); - if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) { - qDebug() << "This codec is preferred..."; - selectedCodecName = requestedCodec; - preferredCodecIndex = preferenceOfThisCodec; - } - } - } - qDebug() << "all requested and available codecs:" << codecList; - - // choose first codec - if (!selectedCodecName.isEmpty()) { - if (codecPlugins.size() > 0) { - for (auto& plugin : codecPlugins) { - if (selectedCodecName == plugin->getName()) { - qDebug() << "Selecting codec:" << selectedCodecName; - selectedCodec = plugin; - break; - } + int preference = codecPreferenceList.indexOf(codec); + if (preference >= 0 && preference < minPreference) { + minPreference = preference; + // choose the preferred, available codec + selectedCodecName = codec; } } } - auto clientData = getOrCreateClientData(sendingNode.data()); - clientData->setupCodec(selectedCodec, selectedCodecName); - qDebug() << "selectedCodecName:" << selectedCodecName; - clientData->sendSelectAudioFormat(sendingNode, selectedCodecName); + return std::make_pair(selectedCodecName, availableCodecs[selectedCodecName]); } void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 6cb7648e50..08f3bf287f 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -49,6 +49,7 @@ public: static const QHash& getAudioZones() { return _audioZones; } static const QVector& getZoneSettings() { return _zoneSettings; } static const QVector& getReverbSettings() { return _zoneReverbSettings; } + static const std::pair negotiateCodec(std::vector codecs); public slots: void run() override; @@ -56,9 +57,7 @@ public slots: private slots: // packet handlers - void queueAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); - void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); void handleNodeKilled(SharedNodePointer killedNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleNodeIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); @@ -67,8 +66,9 @@ private slots: void handleNodeMuteRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handlePerAvatarGainSetDataPacket(QSharedPointer packet, SharedNodePointer sendingNode); - void start(); + void queueAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void removeHRTFsForFinishedInjector(const QUuid& streamID); + void start(); private: // mixing helpers @@ -92,8 +92,6 @@ private: int _numStatFrames { 0 }; AudioMixerStats _stats; - QString _codecPreferenceOrder; - AudioMixerSlavePool _slavePool; class Timer { @@ -128,9 +126,11 @@ private: static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering static float _noiseMutingThreshold; static float _attenuationPerDoublingInDistance; + static QString _codecPreferenceOrder; static QHash _audioZones; static QVector _zoneSettings; static QVector _zoneReverbSettings; + }; #endif // hifi_AudioMixer_h diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index c6b7182a57..b2b01a571b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -47,23 +47,27 @@ AudioMixerClientData::~AudioMixerClientData() { } } -void AudioMixerClientData::queuePacket(QSharedPointer packet) { - _queuedPackets.push(packet); +void AudioMixerClientData::queuePacket(QSharedPointer message, SharedNodePointer node) { + _queuedPackets.push({ message, node }); } void AudioMixerClientData::processPackets() { while (!_queuedPackets.empty()) { - QSharedPointer& packet = _queuedPackets.back(); + auto& packet = _queuedPackets.back(); - switch (packet->getType()) { + switch (packet.message->getType()) { case PacketType::MicrophoneAudioNoEcho: case PacketType::MicrophoneAudioWithEcho: case PacketType::InjectAudio: case PacketType::AudioStreamStats: case PacketType::SilentAudioFrame: { QMutexLocker lock(&getMutex()); - parseData(*packet); + parseData(*packet.message); + break; } + case PacketType::NegotiateAudioFormat: + negotiateAudioFormat(*packet.message, packet.node); + break; default: Q_UNREACHABLE(); } @@ -74,6 +78,19 @@ void AudioMixerClientData::processPackets() { assert(_queuedPackets.empty()); } +void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { + quint8 numberOfCodecs; + message.readPrimitive(&numberOfCodecs); + std::vector codecs; + for (auto i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message.readString(); + } + const std::pair codec = AudioMixer::negotiateCodec(codecs); + + setupCodec(codec.second, codec.first); + sendSelectAudioFormat(node, codec.first); +} + AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { QReadLocker readLocker { &_streamsLock }; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index ff2afe7769..cdc2bc352e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -36,7 +36,7 @@ public: using SharedStreamPointer = std::shared_ptr; using AudioStreamMap = std::unordered_map; - void queuePacket(QSharedPointer packet); + void queuePacket(QSharedPointer packet, SharedNodePointer node); void processPackets(); // locks the mutex to make a copy @@ -61,7 +61,9 @@ public: void removeAgentAvatarAudioStream(); + // packet processors int parseData(ReceivedMessage& message) override; + void negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node); // attempt to pop a frame from each audio stream, and return the number of streams from this client int checkBuffersBeforeFrameSend(); @@ -110,7 +112,11 @@ public slots: void sendSelectAudioFormat(SharedNodePointer node, const QString& selectedCodecName); private: - std::queue> _queuedPackets; + struct Packet { + QSharedPointer message; + SharedNodePointer node; + }; + std::queue _queuedPackets; QReadWriteLock _streamsLock; AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index 75b0f1e3bc..8471dbc7e8 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -123,8 +123,6 @@ const CodecPluginList& PluginManager::getCodecPlugins() { static CodecPluginList codecPlugins; static std::once_flag once; std::call_once(once, [&] { - //codecPlugins = ::getCodecPlugins(); - // Now grab the dynamic plugins for (auto loader : getLoadedPlugins()) { CodecProvider* codecProvider = qobject_cast(loader->instance());