handle codec prioritization, handle multiple codec choices

assignment-client/src/audio/AudioMixer.cpp

@@ -466,34 +466,61 @@ void saveInputPluginSettings(const InputPluginList& plugins) {
 void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
     qDebug() << __FUNCTION__;
 
-    // read the codecs requested by the client
+    QStringList availableCodecs;
-    quint8 numberOfCodecs = 0;
-    message->readPrimitive(&numberOfCodecs);
-    QStringList codecList;
-    for (quint16 i = 0; i < numberOfCodecs; i++) {
-        QString requestedCodec = message->readString();
-        qDebug() << "requestedCodec:" << requestedCodec;
-        codecList.append(requestedCodec);
-    }
-    qDebug() << "all requested codecs:" << codecList;
-
-    CodecPluginPointer selectedCoded;
-    QString selectedCodecName;
     auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
     if (codecPlugins.size() > 0) {
         for (auto& plugin : codecPlugins) {
-            qDebug() << "Codec available:" << plugin->getName();
+            auto codecName = plugin->getName();
-
+            qDebug() << "Codec available:" << codecName;
-            // choose first codec
+            availableCodecs.append(codecName);
-            if (!selectedCoded) {
-                selectedCoded = plugin;
-                selectedCodecName = plugin->getName();
-            }
         }
     } else {
         qDebug() << "No Codecs available...";
     }
 
+    CodecPluginPointer selectedCoded;
+    QString selectedCodecName;
+
+    QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
+
+    // read the codecs requested by the client
+    const int MAX_PREFERENCE = 99999;
+    int preferredCodecIndex = MAX_PREFERENCE;
+    QString preferredCodec;
+    quint8 numberOfCodecs = 0;
+    message->readPrimitive(&numberOfCodecs);
+    qDebug() << "numberOfCodecs:" << numberOfCodecs;
+    QStringList codecList;
+    for (quint16 i = 0; i < numberOfCodecs; i++) {
+        QString requestedCodec = message->readString();
+        int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec);
+        bool codecAvailable = availableCodecs.contains(requestedCodec);
+        qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable;
+        if (codecAvailable) {
+            codecList.append(requestedCodec);
+            if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) {
+                qDebug() << "This codec is preferred...";
+                selectedCodecName  = requestedCodec;
+                preferredCodecIndex = preferenceOfThisCodec;
+            }
+        }
+    }
+    qDebug() << "all requested and available codecs:" << codecList;
+
+    // choose first codec
+    if (!selectedCodecName.isEmpty()) {
+        if (codecPlugins.size() > 0) {
+            for (auto& plugin : codecPlugins) {
+                if (selectedCodecName == plugin->getName()) {
+                    qDebug() << "Selecting codec:" << selectedCodecName;
+                    selectedCoded = plugin;
+                    break;
+                }
+            }
+        }
+    }
+
+
     auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
 
     // FIXME - why would we not have client data at this point??
@@ -882,6 +909,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) {
     if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) {
         QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject();
 
+        const QString CODEC_PREFERENCE_ORDER = "codec_preference_order";
+        if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) {
+            _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString();
+            qDebug() << "Codec preference order changed to" << _codecPreferenceOrder;
+        }
+
         const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance";
         if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) {
             bool ok = false;

assignment-client/src/audio/AudioMixer.h

@@ -92,6 +92,8 @@ private:
     int _manualEchoMixes { 0 };
     int _totalMixes { 0 };
 
+    QString _codecPreferenceOrder;
+
     float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
     int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
 

domain-server/resources/describe-settings.json

@@ -718,6 +718,14 @@
               "placeholder": "(in percent)"
             }
           ]
+        },
+        {
+          "name": "codec_preference_order",
+          "label": "Audio Codec Preference Order",
+          "help": "List of codec names in order of preferred usage",
+          "placeholder": "hifiAC, zlib, pcm",
+          "default": "hifiAC,zlib,pcm",
+          "advanced": true
         }
       ]
     },

libraries/audio-client/src/AudioClient.cpp

@@ -507,19 +507,14 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
 }
 
 void AudioClient::negotiateAudioFormat() {
-    qDebug() << __FUNCTION__;
-
     auto nodeList = DependencyManager::get<NodeList>();
-
     auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
-
     auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
-
     quint8 numberOfCodecs = (quint8)codecPlugins.size();
     negotiateFormatPacket->writePrimitive(numberOfCodecs);
     for (auto& plugin : codecPlugins) {
-        qDebug() << "Codec available:" << plugin->getName();
+        auto codecName = plugin->getName();
-        negotiateFormatPacket->writeString(plugin->getName());
+        negotiateFormatPacket->writeString(codecName);
     }
 
     // grab our audio mixer from the NodeList, if it exists

libraries/audio/src/InboundAudioStream.cpp

@@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
             if (message.getType() == PacketType::SilentAudioFrame) {
                 writeDroppableSilentSamples(networkSamples);
             } else {
-                parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples);
+                parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()));
             }
             break;
         }
@@ -177,25 +177,14 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
     }
 }
 
-int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
+int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
-
-    // codec decode goes here
     QByteArray decodedBuffer;
     if (_codec) {
         _codec->decode(packetAfterStreamProperties, decodedBuffer);
     } else {
         decodedBuffer = packetAfterStreamProperties;
     }
-
     auto actualSize = decodedBuffer.size();
-
-    /*
-    auto expectedSize = numAudioSamples * sizeof(int16_t);
-    if (expectedSize != actualSize) {
-        qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize;
-    }
-    */
-
     return _ringBuffer.writeData(decodedBuffer.data(), actualSize);
 }
 

libraries/audio/src/InboundAudioStream.h

@@ -207,7 +207,7 @@ protected:
 
     /// parses the audio data in the network packet.
     /// default implementation assumes packet contains raw audio samples after stream properties
-    virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+    virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
     /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer
     virtual int writeDroppableSilentSamples(int silentSamples);

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -42,9 +42,7 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
     return deviceSamplesWritten;
 }
 
-int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
+int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
-
-    // TODO - codec decode goes here
     QByteArray decodedBuffer;
     if (_codec) {
         _codec->decode(packetAfterStreamProperties, decodedBuffer);
@@ -52,9 +50,6 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray&
         decodedBuffer = packetAfterStreamProperties;
     }
 
-    qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size();
-
-
     emit addedStereoSamples(decodedBuffer);
 
     QByteArray outputBuffer;

libraries/audio/src/MixedProcessedAudioStream.h

@@ -35,7 +35,7 @@ public:
 protected:
     int writeDroppableSilentSamples(int silentSamples);
     int writeLastFrameRepeatedWithFade(int samples);
-    int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+    int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
 private:
     int networkToDeviceSamples(int networkSamples);

libraries/networking/src/udt/BasePacket.cpp

@@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) {
 
 qint64 BasePacket::writeString(const QString& string) {
     QByteArray data = string.toUtf8();
-    writePrimitive(static_cast<uint32_t>(data.length()));
+    uint32_t length = data.length();
-    return writeData(data.constData(), data.length());
+    writePrimitive(length);
+    auto result = writeData(data.constData(), data.length());
+    seek(pos() + length);
+    return length + sizeof(uint32_t);
 }
 
 QString BasePacket::readString() {

plugins/pcmCodec/src/PCMCodecManager.cpp

@@ -1,5 +1,5 @@
 //
-//  PCMCodecManager.cpp
+//  PCMCodec.cpp
 //  plugins/pcmCodec/src
 //
 //  Created by Brad Hefta-Gaub on 6/9/2016
@@ -15,35 +15,64 @@
 
 #include "PCMCodecManager.h"
 
-const QString PCMCodecManager::NAME = "zlib";
+const QString PCMCodec::NAME = "pcm";
 
-void PCMCodecManager::init() {
+void PCMCodec::init() {
 }
 
-void PCMCodecManager::deinit() {
+void PCMCodec::deinit() {
 }
 
-bool PCMCodecManager::activate() {
+bool PCMCodec::activate() {
     CodecPlugin::activate();
     return true;
 }
 
-void PCMCodecManager::deactivate() {
+void PCMCodec::deactivate() {
     CodecPlugin::deactivate();
 }
 
 
-bool PCMCodecManager::isSupported() const {
+bool PCMCodec::isSupported() const {
     return true;
 }
 
 
-void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
-    //decodedBuffer = encodedBuffer;
+    decodedBuffer = encodedBuffer;
+}
+
+void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+    encodedBuffer = decodedBuffer;
+}
+
+
+const QString zLibCodec::NAME = "zlib";
+
+void zLibCodec::init() {
+}
+
+void zLibCodec::deinit() {
+}
+
+bool zLibCodec::activate() {
+    CodecPlugin::activate();
+    return true;
+}
+
+void zLibCodec::deactivate() {
+    CodecPlugin::deactivate();
+}
+
+
+bool zLibCodec::isSupported() const {
+    return true;
+}
+
+void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
     decodedBuffer = qUncompress(encodedBuffer);
 }
 
-void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
-    //encodedBuffer = decodedBuffer;
     encodedBuffer = qCompress(decodedBuffer);
 }

plugins/pcmCodec/src/PCMCodecManager.h

@@ -15,7 +15,7 @@
 
 #include <plugins/CodecPlugin.h>
 
-class PCMCodecManager : public CodecPlugin {
+class PCMCodec : public CodecPlugin {
     Q_OBJECT
     
 public:
@@ -38,4 +38,27 @@ private:
     static const QString NAME;
 };
 
+class zLibCodec : public CodecPlugin {
+    Q_OBJECT
+
+public:
+    // Plugin functions
+    bool isSupported() const override;
+    const QString& getName() const override { return NAME; }
+
+    void init() override;
+    void deinit() override;
+
+    /// Called when a plugin is being activated for use.  May be called multiple times.
+    bool activate() override;
+    /// Called when a plugin is no longer being used.  May be called multiple times.
+    void deactivate() override;
+
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+
+private:
+    static const QString NAME;
+};
+
 #endif // hifi__PCMCodecManager_h

plugins/pcmCodec/src/PCMCodecProvider.cpp

@@ -29,10 +29,17 @@ public:
     virtual CodecPluginList getCodecPlugins() override {
         static std::once_flag once;
         std::call_once(once, [&] {
-            CodecPluginPointer plugin(new PCMCodecManager());
+
-            if (plugin->isSupported()) {
+            CodecPluginPointer pcmCodec(new PCMCodec());
-                _codecPlugins.push_back(plugin);
+            if (pcmCodec->isSupported()) {
+                _codecPlugins.push_back(pcmCodec);
             }
+
+            CodecPluginPointer zlibCodec(new zLibCodec());
+            if (zlibCodec->isSupported()) {
+                _codecPlugins.push_back(zlibCodec);
+            }
+
         });
         return _codecPlugins;
     }