From a71baf5601ea269804ac30ba92eb8a148f89833e Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 8 Jul 2016 21:18:03 -0700 Subject: [PATCH] handle codec prioritization, handle multiple codec choices --- assignment-client/src/audio/AudioMixer.cpp | 73 ++++++++++++++----- assignment-client/src/audio/AudioMixer.h | 2 + .../resources/describe-settings.json | 8 ++ libraries/audio-client/src/AudioClient.cpp | 9 +-- libraries/audio/src/InboundAudioStream.cpp | 15 +--- libraries/audio/src/InboundAudioStream.h | 2 +- .../audio/src/MixedProcessedAudioStream.cpp | 7 +- .../audio/src/MixedProcessedAudioStream.h | 2 +- libraries/networking/src/udt/BasePacket.cpp | 7 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 51 ++++++++++--- plugins/pcmCodec/src/PCMCodecManager.h | 25 ++++++- plugins/pcmCodec/src/PCMCodecProvider.cpp | 13 +++- 12 files changed, 149 insertions(+), 65 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b419939912..f49b674dce 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -466,34 +466,61 @@ void saveInputPluginSettings(const InputPluginList& plugins) { void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { qDebug() << __FUNCTION__; - // read the codecs requested by the client - quint8 numberOfCodecs = 0; - message->readPrimitive(&numberOfCodecs); - QStringList codecList; - for (quint16 i = 0; i < numberOfCodecs; i++) { - QString requestedCodec = message->readString(); - qDebug() << "requestedCodec:" << requestedCodec; - codecList.append(requestedCodec); - } - qDebug() << "all requested codecs:" << codecList; - - CodecPluginPointer selectedCoded; - QString selectedCodecName; + QStringList availableCodecs; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - - // choose first codec - if (!selectedCoded) { - selectedCoded = plugin; - selectedCodecName = plugin->getName(); - } + auto codecName = plugin->getName(); + qDebug() << "Codec available:" << codecName; + availableCodecs.append(codecName); } } else { qDebug() << "No Codecs available..."; } + CodecPluginPointer selectedCoded; + QString selectedCodecName; + + QStringList codecPreferenceList = _codecPreferenceOrder.split(","); + + // read the codecs requested by the client + const int MAX_PREFERENCE = 99999; + int preferredCodecIndex = MAX_PREFERENCE; + QString preferredCodec; + quint8 numberOfCodecs = 0; + message->readPrimitive(&numberOfCodecs); + qDebug() << "numberOfCodecs:" << numberOfCodecs; + QStringList codecList; + for (quint16 i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message->readString(); + int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec); + bool codecAvailable = availableCodecs.contains(requestedCodec); + qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable; + if (codecAvailable) { + codecList.append(requestedCodec); + if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) { + qDebug() << "This codec is preferred..."; + selectedCodecName = requestedCodec; + preferredCodecIndex = preferenceOfThisCodec; + } + } + } + qDebug() << "all requested and available codecs:" << codecList; + + // choose first codec + if (!selectedCodecName.isEmpty()) { + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + if (selectedCodecName == plugin->getName()) { + qDebug() << "Selecting codec:" << selectedCodecName; + selectedCoded = plugin; + break; + } + } + } + } + + auto clientData = dynamic_cast(sendingNode->getLinkedData()); // FIXME - why would we not have client data at this point?? @@ -882,6 +909,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) { QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject(); + const QString CODEC_PREFERENCE_ORDER = "codec_preference_order"; + if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) { + _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString(); + qDebug() << "Codec preference order changed to" << _codecPreferenceOrder; + } + const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance"; if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) { bool ok = false; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index c90a918a5b..4b2a27120d 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -92,6 +92,8 @@ private: int _manualEchoMixes { 0 }; int _totalMixes { 0 }; + QString _codecPreferenceOrder; + float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 7375a0f650..948c6ddc18 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -718,6 +718,14 @@ "placeholder": "(in percent)" } ] + }, + { + "name": "codec_preference_order", + "label": "Audio Codec Preference Order", + "help": "List of codec names in order of preferred usage", + "placeholder": "hifiAC, zlib, pcm", + "default": "hifiAC,zlib,pcm", + "advanced": true } ] }, diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a076b1b290..97ef4b2981 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -507,19 +507,14 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me } void AudioClient::negotiateAudioFormat() { - qDebug() << __FUNCTION__; - auto nodeList = DependencyManager::get(); - auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); - auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); - quint8 numberOfCodecs = (quint8)codecPlugins.size(); negotiateFormatPacket->writePrimitive(numberOfCodecs); for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - negotiateFormatPacket->writeString(plugin->getName()); + auto codecName = plugin->getName(); + negotiateFormatPacket->writeString(codecName); } // grab our audio mixer from the NodeList, if it exists diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 5d4dbd9b94..7c6a34ae55 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { if (message.getType() == PacketType::SilentAudioFrame) { writeDroppableSilentSamples(networkSamples); } else { - parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples); + parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead())); } break; } @@ -177,25 +177,14 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } -int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - - // codec decode goes here +int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_codec) { _codec->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } - auto actualSize = decodedBuffer.size(); - - /* - auto expectedSize = numAudioSamples * sizeof(int16_t); - if (expectedSize != actualSize) { - qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize; - } - */ - return _ringBuffer.writeData(decodedBuffer.data(), actualSize); } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index ddc0dc1dc3..f9ca088fab 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -207,7 +207,7 @@ protected: /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentSamples(int silentSamples); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 220b2bd9ee..6939ee540a 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -42,9 +42,7 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { return deviceSamplesWritten; } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { - - // TODO - codec decode goes here +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_codec) { _codec->decode(packetAfterStreamProperties, decodedBuffer); @@ -52,9 +50,6 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& decodedBuffer = packetAfterStreamProperties; } - qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size(); - - emit addedStereoSamples(decodedBuffer); QByteArray outputBuffer; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 5ea0157421..2f9a691278 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -35,7 +35,7 @@ public: protected: int writeDroppableSilentSamples(int silentSamples); int writeLastFrameRepeatedWithFade(int samples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); private: int networkToDeviceSamples(int networkSamples); diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index dba241f221..0456e095ae 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) { qint64 BasePacket::writeString(const QString& string) { QByteArray data = string.toUtf8(); - writePrimitive(static_cast(data.length())); - return writeData(data.constData(), data.length()); + uint32_t length = data.length(); + writePrimitive(length); + auto result = writeData(data.constData(), data.length()); + seek(pos() + length); + return length + sizeof(uint32_t); } QString BasePacket::readString() { diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 9d55cafedf..f787c6682d 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -1,5 +1,5 @@ // -// PCMCodecManager.cpp +// PCMCodec.cpp // plugins/pcmCodec/src // // Created by Brad Hefta-Gaub on 6/9/2016 @@ -15,35 +15,64 @@ #include "PCMCodecManager.h" -const QString PCMCodecManager::NAME = "zlib"; +const QString PCMCodec::NAME = "pcm"; -void PCMCodecManager::init() { +void PCMCodec::init() { } -void PCMCodecManager::deinit() { +void PCMCodec::deinit() { } -bool PCMCodecManager::activate() { +bool PCMCodec::activate() { CodecPlugin::activate(); return true; } -void PCMCodecManager::deactivate() { +void PCMCodec::deactivate() { CodecPlugin::deactivate(); } -bool PCMCodecManager::isSupported() const { +bool PCMCodec::isSupported() const { return true; } -void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - //decodedBuffer = encodedBuffer; +void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { + decodedBuffer = encodedBuffer; +} + +void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + encodedBuffer = decodedBuffer; +} + + +const QString zLibCodec::NAME = "zlib"; + +void zLibCodec::init() { +} + +void zLibCodec::deinit() { +} + +bool zLibCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void zLibCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool zLibCodec::isSupported() const { + return true; +} + +void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { decodedBuffer = qUncompress(encodedBuffer); } -void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - //encodedBuffer = decodedBuffer; +void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { encodedBuffer = qCompress(decodedBuffer); } diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 3b7ca36c02..7816660c5d 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -15,7 +15,7 @@ #include -class PCMCodecManager : public CodecPlugin { +class PCMCodec : public CodecPlugin { Q_OBJECT public: @@ -38,4 +38,27 @@ private: static const QString NAME; }; +class zLibCodec : public CodecPlugin { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + +private: + static const QString NAME; +}; + #endif // hifi__PCMCodecManager_h diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp index 732ed2d57d..351b1adf3f 100644 --- a/plugins/pcmCodec/src/PCMCodecProvider.cpp +++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp @@ -29,10 +29,17 @@ public: virtual CodecPluginList getCodecPlugins() override { static std::once_flag once; std::call_once(once, [&] { - CodecPluginPointer plugin(new PCMCodecManager()); - if (plugin->isSupported()) { - _codecPlugins.push_back(plugin); + + CodecPluginPointer pcmCodec(new PCMCodec()); + if (pcmCodec->isSupported()) { + _codecPlugins.push_back(pcmCodec); } + + CodecPluginPointer zlibCodec(new zLibCodec()); + if (zlibCodec->isSupported()) { + _codecPlugins.push_back(zlibCodec); + } + }); return _codecPlugins; }