move codec negotiation to client data

assignment-client/src/audio/AudioMixer.cpp

@@ -47,12 +47,16 @@ static const QString AUDIO_THREADING_GROUP_KEY = "audio_threading";
 int AudioMixer::_numStaticJitterFrames{ -1 };
 float AudioMixer::_noiseMutingThreshold{ DEFAULT_NOISE_MUTING_THRESHOLD };
 float AudioMixer::_attenuationPerDoublingInDistance{ DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE };
+QString AudioMixer::_codecPreferenceOrder{};
 QHash<QString, AABox> AudioMixer::_audioZones;
 QVector<AudioMixer::ZoneSettings> AudioMixer::_zoneSettings;
 QVector<AudioMixer::ReverbSettings> AudioMixer::_zoneReverbSettings;
 
 AudioMixer::AudioMixer(ReceivedMessage& message) :
     ThreadedAssignment(message) {
+    // initialize the plugins
+    PluginManager::getInstance()->getCodecPlugins();
+
     auto nodeList = DependencyManager::get<NodeList>();
     auto& packetReceiver = nodeList->getPacketReceiver();
 
@@ -61,9 +65,9 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
             PacketType::MicrophoneAudioWithEcho,
             PacketType::InjectAudio,
             PacketType::AudioStreamStats,
-            PacketType::SilentAudioFrame },
+            PacketType::SilentAudioFrame,
+            PacketType::NegotiateAudioFormat },
             this, "queueAudioPacket");
-    packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat");
     packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
     packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket");
     packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket");
@@ -75,12 +79,12 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
     connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled);
 }
 
-void AudioMixer::queueAudioPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
+void AudioMixer::queueAudioPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {
     if (message->getType() == PacketType::SilentAudioFrame) {
         _numSilentPackets++;
     }
 
-    getOrCreateClientData(sendingNode.data())->queuePacket(message);
+    getOrCreateClientData(node.data())->queuePacket(message, node);
 }
 
 void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
@@ -119,69 +123,41 @@ InputPluginList getInputPlugins() {
     return result;
 }
 
-void saveInputPluginSettings(const InputPluginList& plugins) {
+// must be here to satisfy a reference in PluginManager::saveSettings()
-}
+void saveInputPluginSettings(const InputPluginList& plugins) {}
 
-
+const std::pair<QString, CodecPluginPointer> AudioMixer::negotiateCodec(std::vector<QString> codecs) {
-void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
-    QStringList availableCodecs;
-    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
-    if (codecPlugins.size() > 0) {
-        for (auto& plugin : codecPlugins) {
-            auto codecName = plugin->getName();
-            qDebug() << "Codec available:" << codecName;
-            availableCodecs.append(codecName);
-        }
-    } else {
-        qDebug() << "No Codecs available...";
-    }
-
-    CodecPluginPointer selectedCodec;
     QString selectedCodecName;
+    CodecPluginPointer selectedCodec;
 
+    // get the available codecs (on the mixer)
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    std::map<QString, CodecPluginPointer> availableCodecs;
+    std::for_each(codecPlugins.cbegin(), codecPlugins.cend(),
+            [&availableCodecs](const CodecPluginPointer& codec) {
+                availableCodecs[codec->getName()] = codec;
+            });
+
+    // get the codec preferences (on the mixer)
     QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
 
-    // read the codecs requested by the client
+    // read the codecs requested (by the client)
     const int MAX_PREFERENCE = 99999;
-    int preferredCodecIndex = MAX_PREFERENCE;
+    int minPreference = MAX_PREFERENCE;
-    QString preferredCodec;
+    for (auto& codec : codecs) {
-    quint8 numberOfCodecs = 0;
+        bool codecAvailable = (availableCodecs.count(codec) > 0);
-    message->readPrimitive(&numberOfCodecs);
+
-    qDebug() << "numberOfCodecs:" << numberOfCodecs;
-    QStringList codecList;
-    for (quint16 i = 0; i < numberOfCodecs; i++) {
-        QString requestedCodec = message->readString();
-        int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec);
-        bool codecAvailable = availableCodecs.contains(requestedCodec);
-        qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable;
         if (codecAvailable) {
-            codecList.append(requestedCodec);
+            int preference = codecPreferenceList.indexOf(codec);
-            if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) {
+            if (preference >= 0 && preference < minPreference) {
-                qDebug() << "This codec is preferred...";
+                minPreference = preference;
-                selectedCodecName  = requestedCodec;
+                // choose the preferred, available codec
-                preferredCodecIndex = preferenceOfThisCodec;
+                selectedCodecName  = codec;
-            }
-        }
-    }
-    qDebug() << "all requested and available codecs:" << codecList;
-
-    // choose first codec
-    if (!selectedCodecName.isEmpty()) {
-        if (codecPlugins.size() > 0) {
-            for (auto& plugin : codecPlugins) {
-                if (selectedCodecName == plugin->getName()) {
-                    qDebug() << "Selecting codec:" << selectedCodecName;
-                    selectedCodec = plugin;
-                    break;
-                }
             }
         }
     }
 
-    auto clientData = getOrCreateClientData(sendingNode.data());
+    return std::make_pair(selectedCodecName, availableCodecs[selectedCodecName]);
-    clientData->setupCodec(selectedCodec, selectedCodecName);
-    qDebug() << "selectedCodecName:" << selectedCodecName;
-    clientData->sendSelectAudioFormat(sendingNode, selectedCodecName);
 }
 
 void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {

assignment-client/src/audio/AudioMixer.h

@@ -49,6 +49,7 @@ public:
     static const QHash<QString, AABox>& getAudioZones() { return _audioZones; }
     static const QVector<ZoneSettings>& getZoneSettings() { return _zoneSettings; }
     static const QVector<ReverbSettings>& getReverbSettings() { return _zoneReverbSettings; }
+    static const std::pair<QString, CodecPluginPointer> negotiateCodec(std::vector<QString> codecs);
 
 public slots:
     void run() override;
@@ -56,9 +57,7 @@ public slots:
 
 private slots:
     // packet handlers
-    void queueAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
-    void handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode);
     void handleNodeKilled(SharedNodePointer killedNode);
     void handleRequestsDomainListDataPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleNodeIgnoreRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
@@ -67,8 +66,9 @@ private slots:
     void handleNodeMuteRequestPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void handlePerAvatarGainSetDataPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
 
-    void start();
+    void queueAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void removeHRTFsForFinishedInjector(const QUuid& streamID);
+    void start();
 
 private:
     // mixing helpers
@@ -92,8 +92,6 @@ private:
     int _numStatFrames { 0 };
     AudioMixerStats _stats;
 
-    QString _codecPreferenceOrder;
-
     AudioMixerSlavePool _slavePool;
 
     class Timer {
@@ -128,9 +126,11 @@ private:
     static int _numStaticJitterFrames; // -1 denotes dynamic jitter buffering
     static float _noiseMutingThreshold;
     static float _attenuationPerDoublingInDistance;
+    static QString _codecPreferenceOrder;
     static QHash<QString, AABox> _audioZones;
     static QVector<ZoneSettings> _zoneSettings;
     static QVector<ReverbSettings> _zoneReverbSettings;
+
 };
 
 #endif // hifi_AudioMixer_h

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -47,23 +47,27 @@ AudioMixerClientData::~AudioMixerClientData() {
     }
 }
 
-void AudioMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> packet) {
+void AudioMixerClientData::queuePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer node) {
-    _queuedPackets.push(packet);
+    _queuedPackets.push({ message, node });
 }
 
 void AudioMixerClientData::processPackets() {
     while (!_queuedPackets.empty()) {
-        QSharedPointer<ReceivedMessage>& packet = _queuedPackets.back();
+        auto& packet = _queuedPackets.back();
 
-        switch (packet->getType()) {
+        switch (packet.message->getType()) {
             case PacketType::MicrophoneAudioNoEcho:
             case PacketType::MicrophoneAudioWithEcho:
             case PacketType::InjectAudio:
             case PacketType::AudioStreamStats:
             case PacketType::SilentAudioFrame: {
                 QMutexLocker lock(&getMutex());
-                parseData(*packet);
+                parseData(*packet.message);
+                break;
             }
+            case PacketType::NegotiateAudioFormat:
+                negotiateAudioFormat(*packet.message, packet.node);
+                break;
             default:
                 Q_UNREACHABLE();
         }
@@ -74,6 +78,19 @@ void AudioMixerClientData::processPackets() {
     assert(_queuedPackets.empty());
 }
 
+void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) {
+    quint8 numberOfCodecs;
+    message.readPrimitive(&numberOfCodecs);
+    std::vector<QString> codecs;
+    for (auto i = 0; i < numberOfCodecs; i++) {
+        QString requestedCodec = message.readString();
+    }
+    const std::pair<QString, CodecPluginPointer> codec = AudioMixer::negotiateCodec(codecs);
+
+    setupCodec(codec.second, codec.first);
+    sendSelectAudioFormat(node, codec.first);
+}
+
 AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
     QReadLocker readLocker { &_streamsLock };
 

assignment-client/src/audio/AudioMixerClientData.h

@@ -36,7 +36,7 @@ public:
     using SharedStreamPointer = std::shared_ptr<PositionalAudioStream>;
     using AudioStreamMap = std::unordered_map<QUuid, SharedStreamPointer>;
 
-    void queuePacket(QSharedPointer<ReceivedMessage> packet);
+    void queuePacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer node);
     void processPackets();
 
     // locks the mutex to make a copy
@@ -61,7 +61,9 @@ public:
 
     void removeAgentAvatarAudioStream();
 
+    // packet processors
     int parseData(ReceivedMessage& message) override;
+    void negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node);
 
     // attempt to pop a frame from each audio stream, and return the number of streams from this client
     int checkBuffersBeforeFrameSend();
@@ -110,7 +112,11 @@ public slots:
     void sendSelectAudioFormat(SharedNodePointer node, const QString& selectedCodecName);
 
 private:
-    std::queue<QSharedPointer<ReceivedMessage>> _queuedPackets;
+    struct Packet {
+        QSharedPointer<ReceivedMessage> message;
+        SharedNodePointer node;
+    };
+    std::queue<Packet> _queuedPackets;
 
     QReadWriteLock _streamsLock;
     AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID

libraries/plugins/src/plugins/PluginManager.cpp

@@ -123,8 +123,6 @@ const CodecPluginList& PluginManager::getCodecPlugins() {
     static CodecPluginList codecPlugins;
     static std::once_flag once;
     std::call_once(once, [&] {
-        //codecPlugins = ::getCodecPlugins();
-
         // Now grab the dynamic plugins
         for (auto loader : getLoadedPlugins()) {
             CodecProvider* codecProvider = qobject_cast<CodecProvider*>(loader->instance());