From ba6bb245956b4657b4d234afd342f40d50296f6c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sun, 10 Jul 2016 16:49:03 -0700 Subject: [PATCH] rework plugins to allow different decoder/encoder instances per streams --- assignment-client/src/audio/AudioMixer.cpp | 6 +- .../src/audio/AudioMixerClientData.cpp | 11 ++++ .../src/audio/AudioMixerClientData.h | 3 + libraries/audio-client/src/AudioClient.cpp | 10 +-- libraries/audio-client/src/AudioClient.h | 1 + libraries/audio/src/InboundAudioStream.cpp | 4 +- libraries/audio/src/InboundAudioStream.h | 1 + .../audio/src/MixedProcessedAudioStream.cpp | 4 +- libraries/plugins/src/plugins/CodecPlugin.h | 18 +++++- plugins/pcmCodec/src/PCMCodecManager.cpp | 64 ++++++++++++++++--- plugins/pcmCodec/src/PCMCodecManager.h | 37 +++++++++-- 11 files changed, 135 insertions(+), 24 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f49b674dce..825f970f2e 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -533,12 +533,16 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess clientData->_codec = selectedCoded; clientData->_selectedCodecName = selectedCodecName; + clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + qDebug() << "selectedCodecName:" << selectedCodecName; auto avatarAudioStream = clientData->getAvatarAudioStream(); if (avatarAudioStream) { avatarAudioStream->_codec = selectedCoded; avatarAudioStream->_selectedCodecName = selectedCodecName; + avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); } auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); @@ -777,7 +781,7 @@ void AudioMixer::broadcastMixes() { QByteArray decocedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); QByteArray encodedBuffer; if (nodeData->_codec) { - nodeData->_codec->encode(decocedBuffer, encodedBuffer); + nodeData->_encoder->encode(decocedBuffer, encodedBuffer); } else { encodedBuffer = decocedBuffer; } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 080a833c0e..3e9b72d60e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) : _frameToSendStats = distribution(numberGenerator); } +AudioMixerClientData::~AudioMixerClientData() { + if (_codec) { + _codec->releaseDecoder(_decoder); + _codec->releaseEncoder(_encoder); + } +} + + AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { QReadLocker readLocker { &_streamsLock }; @@ -104,6 +112,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); avatarAudioStream->_codec = _codec; avatarAudioStream->_selectedCodecName = _selectedCodecName; + if (_codec) { + avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + } qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName; auto emplaced = _audioStreams.emplace( diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 0b3b352e66..ee6f22ff3b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -29,6 +29,7 @@ class AudioMixerClientData : public NodeData { Q_OBJECT public: AudioMixerClientData(const QUuid& nodeID); + ~AudioMixerClientData(); using SharedStreamPointer = std::shared_ptr; using AudioStreamMap = std::unordered_map; @@ -70,6 +71,8 @@ public: // FIXME -- maybe make these private CodecPluginPointer _codec; QString _selectedCodecName; + Encoder* _encoder { nullptr }; // for outbound mixed stream + Decoder* _decoder { nullptr }; // for mic stream signals: diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 97ef4b2981..db5376a380 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -536,6 +536,8 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer mess for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { _receivedAudioStream._codec = _codec = plugin; + _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } @@ -815,8 +817,8 @@ void AudioClient::handleAudioInput() { // TODO - codec encode goes here QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); QByteArray encodedBuffer; - if (_codec) { - _codec->encode(decocedBuffer, encodedBuffer); + if (_encoder) { + _encoder->encode(decocedBuffer, encodedBuffer); } else { encodedBuffer = decocedBuffer; } @@ -833,8 +835,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { // TODO - codec encode goes here QByteArray encodedBuffer; - if (_codec) { - _codec->encode(audio, encodedBuffer); + if (_encoder) { + _encoder->encode(audio, encodedBuffer); } else { encodedBuffer = audio; } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 66555d1e2d..bd0afe453d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -299,6 +299,7 @@ private: CodecPluginPointer _codec; QString _selectedCodecName; + Encoder* _encoder { nullptr }; // for outbound mic stream }; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 7c6a34ae55..f47c6cce46 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -179,8 +179,8 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; - if (_codec) { - _codec->decode(packetAfterStreamProperties, decodedBuffer); + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f9ca088fab..57bff80fff 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -179,6 +179,7 @@ public: // FIXME -- maybe make these private CodecPluginPointer _codec; QString _selectedCodecName; + Decoder* _decoder { nullptr }; public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 6939ee540a..728deae0b1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -44,8 +44,8 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; - if (_codec) { - _codec->decode(packetAfterStreamProperties, decodedBuffer); + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 6944d91bed..280853e37e 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -12,9 +12,23 @@ #include "Plugin.h" -class CodecPlugin : public Plugin { +class Encoder { public: - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; }; +class Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + + // numFrames - number of samples (mono) or sample-pairs (stereo) + virtual void trackLostFrames(int numFrames) = 0; +}; + +class CodecPlugin : public Plugin { +public: + virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; + virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; + virtual void releaseEncoder(Encoder* encoder) = 0; + virtual void releaseDecoder(Decoder* decoder) = 0; +}; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index f787c6682d..1cad1ea4e6 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -37,15 +37,54 @@ bool PCMCodec::isSupported() const { return true; } +class PCMEncoder : public Encoder { +public: + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } +}; -void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - decodedBuffer = encodedBuffer; +class PCMDecoder : public Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void trackLostFrames(int numFrames) override { } +}; + +Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { + return new PCMEncoder(); } -void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - encodedBuffer = decodedBuffer; +Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { + return new PCMDecoder(); } +void PCMCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void PCMCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} + + +class zLibEncoder : public Encoder { +public: + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = qCompress(decodedBuffer); + } +}; + +class zLibDecoder : public Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = qUncompress(encodedBuffer); + } + + virtual void trackLostFrames(int numFrames) override { } +}; const QString zLibCodec::NAME = "zlib"; @@ -69,10 +108,19 @@ bool zLibCodec::isSupported() const { return true; } -void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - decodedBuffer = qUncompress(encodedBuffer); +Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { + return new zLibEncoder(); } -void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - encodedBuffer = qCompress(decodedBuffer); +Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { + return new zLibDecoder(); } + +void zLibCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void zLibCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} + diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 7816660c5d..6eb7f44c15 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -12,9 +12,31 @@ #ifndef hifi__PCMCodecManager_h #define hifi__PCMCodecManager_h - #include +/* +class Encoder { +public: +virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; +}; + +class Decoder { +public: +virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + +// numFrames - number of samples (mono) or sample-pairs (stereo) +virtual void trackLostFrames(int numFrames) = 0; +}; + +class CodecPlugin : public Plugin { +public: +virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; +virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; +virtual void releaseEncoder(Encoder* encoder) = 0; +virtual void releaseDecoder(Decoder* decoder) = 0; +}; +*/ + class PCMCodec : public CodecPlugin { Q_OBJECT @@ -31,8 +53,10 @@ public: /// Called when a plugin is no longer being used. May be called multiple times. void deactivate() override; - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; private: static const QString NAME; @@ -54,8 +78,11 @@ public: /// Called when a plugin is no longer being used. May be called multiple times. void deactivate() override; - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + private: static const QString NAME;