rework plugins to allow different decoder/encoder instances per streams

assignment-client/src/audio/AudioMixer.cpp

@@ -533,12 +533,16 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
 
     clientData->_codec = selectedCoded;
     clientData->_selectedCodecName = selectedCodecName;
+    clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+    clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+
     qDebug() << "selectedCodecName:" << selectedCodecName;
 
     auto avatarAudioStream = clientData->getAvatarAudioStream();
     if (avatarAudioStream) {
         avatarAudioStream->_codec = selectedCoded;
         avatarAudioStream->_selectedCodecName = selectedCodecName;
+        avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
     }
 
     auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
@@ -777,7 +781,7 @@ void AudioMixer::broadcastMixes() {
                         QByteArray decocedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
                         QByteArray encodedBuffer;
                         if (nodeData->_codec) {
-                            nodeData->_codec->encode(decocedBuffer, encodedBuffer);
+                            nodeData->_encoder->encode(decocedBuffer, encodedBuffer);
                         } else {
                             encodedBuffer = decocedBuffer;
                         }

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) :
     _frameToSendStats = distribution(numberGenerator);
 }
 
+AudioMixerClientData::~AudioMixerClientData() {
+    if (_codec) {
+        _codec->releaseDecoder(_decoder);
+        _codec->releaseEncoder(_encoder);
+    }
+}
+
+
 AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
     QReadLocker readLocker { &_streamsLock };
     
@@ -104,6 +112,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
                 auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings());
                 avatarAudioStream->_codec = _codec;
                 avatarAudioStream->_selectedCodecName = _selectedCodecName;
+                if (_codec) {
+                    avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+                }
                 qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName;
 
                 auto emplaced = _audioStreams.emplace(

assignment-client/src/audio/AudioMixerClientData.h

@@ -29,6 +29,7 @@ class AudioMixerClientData : public NodeData {
     Q_OBJECT
 public:
     AudioMixerClientData(const QUuid& nodeID);
+    ~AudioMixerClientData();
 
     using SharedStreamPointer = std::shared_ptr<PositionalAudioStream>;
     using AudioStreamMap = std::unordered_map<QUuid, SharedStreamPointer>;
@@ -70,6 +71,8 @@ public:
     // FIXME -- maybe make these private
     CodecPluginPointer _codec;
     QString _selectedCodecName;
+    Encoder* _encoder { nullptr }; // for outbound mixed stream
+    Decoder* _decoder { nullptr }; // for mic stream
 
 
 signals:

libraries/audio-client/src/AudioClient.cpp

@@ -536,6 +536,8 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> mess
     for (auto& plugin : codecPlugins) {
         if (_selectedCodecName == plugin->getName()) {
             _receivedAudioStream._codec = _codec = plugin;
+            _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+            _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
             qDebug() << "Selected Codec Plugin:" << _codec.get();
             break;
         }
@@ -815,8 +817,8 @@ void AudioClient::handleAudioInput() {
         // TODO - codec encode goes here
         QByteArray decocedBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
         QByteArray encodedBuffer;
-        if (_codec) {
+        if (_encoder) {
-            _codec->encode(decocedBuffer, encodedBuffer);
+            _encoder->encode(decocedBuffer, encodedBuffer);
         } else {
             encodedBuffer = decocedBuffer;
         }
@@ -833,8 +835,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
 
     // TODO - codec encode goes here
     QByteArray encodedBuffer;
-    if (_codec) {
+    if (_encoder) {
-        _codec->encode(audio, encodedBuffer);
+        _encoder->encode(audio, encodedBuffer);
     } else {
         encodedBuffer = audio;
     }

libraries/audio-client/src/AudioClient.h

@@ -299,6 +299,7 @@ private:
 
     CodecPluginPointer _codec;
     QString _selectedCodecName;
+    Encoder* _encoder { nullptr }; // for outbound mic stream
 };
 
 

libraries/audio/src/InboundAudioStream.cpp

@@ -179,8 +179,8 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
 
 int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
-    if (_codec) {
+    if (_decoder) {
-        _codec->decode(packetAfterStreamProperties, decodedBuffer);
+        _decoder->decode(packetAfterStreamProperties, decodedBuffer);
     } else {
         decodedBuffer = packetAfterStreamProperties;
     }

libraries/audio/src/InboundAudioStream.h

@@ -179,6 +179,7 @@ public:
     // FIXME -- maybe make these private
     CodecPluginPointer _codec;
     QString _selectedCodecName;
+    Decoder* _decoder { nullptr };
 
 public slots:
     /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -44,8 +44,8 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
 
 int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
     QByteArray decodedBuffer;
-    if (_codec) {
+    if (_decoder) {
-        _codec->decode(packetAfterStreamProperties, decodedBuffer);
+        _decoder->decode(packetAfterStreamProperties, decodedBuffer);
     } else {
         decodedBuffer = packetAfterStreamProperties;
     }

libraries/plugins/src/plugins/CodecPlugin.h

@@ -12,9 +12,23 @@
 
 #include "Plugin.h"
 
-class CodecPlugin : public Plugin {
+class Encoder {
 public:
-    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
     virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
 };
 
+class Decoder {
+public:
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+
+    // numFrames - number of samples (mono) or sample-pairs (stereo)
+    virtual void trackLostFrames(int numFrames) = 0;
+};
+
+class CodecPlugin : public Plugin {
+public:
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
+    virtual void releaseEncoder(Encoder* encoder) = 0;
+    virtual void releaseDecoder(Decoder* decoder) = 0;
+};

plugins/pcmCodec/src/PCMCodecManager.cpp

@@ -37,15 +37,54 @@ bool PCMCodec::isSupported() const {
     return true;
 }
 
+class PCMEncoder : public Encoder {
+public:
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+        encodedBuffer = decodedBuffer;
+    }
+};
 
-void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+class PCMDecoder : public Decoder {
-    decodedBuffer = encodedBuffer;
+public:
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+        decodedBuffer = encodedBuffer;
+    }
+
+    virtual void trackLostFrames(int numFrames)  override { }
+};
+
+Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) {
+    return new PCMEncoder();
 }
 
-void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) {
-    encodedBuffer = decodedBuffer;
+    return new PCMDecoder();
 }
 
+void PCMCodec::releaseEncoder(Encoder* encoder) {
+    delete encoder;
+}
+
+void PCMCodec::releaseDecoder(Decoder* decoder) {
+    delete decoder;
+}
+
+
+class zLibEncoder : public Encoder {
+public:
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+        encodedBuffer = qCompress(decodedBuffer);
+    }
+};
+
+class zLibDecoder : public Decoder {
+public:
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+        decodedBuffer = qUncompress(encodedBuffer);
+    }
+
+    virtual void trackLostFrames(int numFrames)  override { }
+};
 
 const QString zLibCodec::NAME = "zlib";
 
@@ -69,10 +108,19 @@ bool zLibCodec::isSupported() const {
     return true;
 }
 
-void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) {
-    decodedBuffer = qUncompress(encodedBuffer);
+    return new zLibEncoder();
 }
 
-void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) {
-    encodedBuffer = qCompress(decodedBuffer);
+    return new zLibDecoder();
 }
+
+void zLibCodec::releaseEncoder(Encoder* encoder) {
+    delete encoder;
+}
+
+void zLibCodec::releaseDecoder(Decoder* decoder) {
+    delete decoder;
+}
+

plugins/pcmCodec/src/PCMCodecManager.h

@@ -12,9 +12,31 @@
 #ifndef hifi__PCMCodecManager_h
 #define hifi__PCMCodecManager_h
 
-
 #include <plugins/CodecPlugin.h>
 
+/*
+class Encoder {
+public:
+virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
+};
+
+class Decoder {
+public:
+virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+
+// numFrames - number of samples (mono) or sample-pairs (stereo)
+virtual void trackLostFrames(int numFrames) = 0;
+};
+
+class CodecPlugin : public Plugin {
+public:
+virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
+virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
+virtual void releaseEncoder(Encoder* encoder) = 0;
+virtual void releaseDecoder(Decoder* decoder) = 0;
+};
+*/
+
 class PCMCodec : public CodecPlugin {
     Q_OBJECT
     
@@ -31,8 +53,10 @@ public:
     /// Called when a plugin is no longer being used.  May be called multiple times.
     void deactivate() override;
 
-    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
-    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+    virtual void releaseEncoder(Encoder* encoder) override;
+    virtual void releaseDecoder(Decoder* decoder) override;
 
 private:
     static const QString NAME;
@@ -54,8 +78,11 @@ public:
     /// Called when a plugin is no longer being used.  May be called multiple times.
     void deactivate() override;
 
-    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
-    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+    virtual void releaseEncoder(Encoder* encoder) override;
+    virtual void releaseDecoder(Decoder* decoder) override;
+
 
 private:
     static const QString NAME;