From be37921845d760e9052f68ec04891bf6d2dee237 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 9 Jun 2016 15:49:07 -0700 Subject: [PATCH 01/22] first cut at codec plugins --- interface/src/Application.cpp | 7 +++ libraries/audio-client/src/AudioClient.cpp | 20 ++++++--- libraries/plugins/src/plugins/CodecPlugin.h | 19 ++++++++ libraries/plugins/src/plugins/Forward.h | 4 ++ .../plugins/src/plugins/PluginManager.cpp | 30 +++++++++++++ libraries/plugins/src/plugins/PluginManager.h | 1 + libraries/plugins/src/plugins/RuntimePlugin.h | 9 ++++ plugins/pcmCodec/CMakeLists.txt | 11 +++++ plugins/pcmCodec/src/PCMCodecManager.cpp | 45 +++++++++++++++++++ plugins/pcmCodec/src/PCMCodecManager.h | 40 +++++++++++++++++ plugins/pcmCodec/src/PCMCodecProvider.cpp | 44 ++++++++++++++++++ plugins/pcmCodec/src/plugin.json | 1 + 12 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 libraries/plugins/src/plugins/CodecPlugin.h create mode 100644 plugins/pcmCodec/CMakeLists.txt create mode 100644 plugins/pcmCodec/src/PCMCodecManager.cpp create mode 100644 plugins/pcmCodec/src/PCMCodecManager.h create mode 100644 plugins/pcmCodec/src/PCMCodecProvider.cpp create mode 100644 plugins/pcmCodec/src/plugin.json diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 37c3b361bf..855cf2f313 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include @@ -1236,7 +1237,13 @@ QString Application::getUserAgent() { userAgent += " " + formatPluginName(ip->getName()); } } + // for codecs, we include all of them, even if not active + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& cp : codecPlugins) { + userAgent += " " + formatPluginName(cp->getName()); + } + qDebug() << __FUNCTION__ << ":" << userAgent; return userAgent; } diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0c7a79e2a3..2569f27aaa 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -813,19 +813,25 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } -void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { - const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t); - const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) - / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()); +void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) { + + // TODO - codec decode goes here + QByteArray decodedBuffer = networkBuffer; + + const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t); + const int numDeviceOutputSamples = _outputFrameSize; + + Q_ASSERT(_outputFrameSize == numDecodecSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) + / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount())); outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t)); - const int16_t* receivedSamples = reinterpret_cast(inputBuffer.data()); + const int16_t* decodedSamples = reinterpret_cast(decodedBuffer.data()); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); // copy the packet from the RB to the output - possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples, - numNetworkOutputSamples, numDeviceOutputSamples, + possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples, + numDecodecSamples, numDeviceOutputSamples, _desiredOutputFormat, _outputFormat); // apply stereo reverb at the listener, to the received audio diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h new file mode 100644 index 0000000000..d9e1b947fe --- /dev/null +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -0,0 +1,19 @@ +// +// CodecPlugin.h +// plugins/src/plugins +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#pragma once + +#include "Plugin.h" + +class CodecPlugin : public Plugin { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; +}; + diff --git a/libraries/plugins/src/plugins/Forward.h b/libraries/plugins/src/plugins/Forward.h index 036b42f7d7..723c4f321e 100644 --- a/libraries/plugins/src/plugins/Forward.h +++ b/libraries/plugins/src/plugins/Forward.h @@ -13,10 +13,12 @@ enum class PluginType { DISPLAY_PLUGIN, INPUT_PLUGIN, + CODEC_PLUGIN, }; class DisplayPlugin; class InputPlugin; +class CodecPlugin; class Plugin; class PluginContainer; class PluginManager; @@ -25,4 +27,6 @@ using DisplayPluginPointer = std::shared_ptr; using DisplayPluginList = std::vector; using InputPluginPointer = std::shared_ptr; using InputPluginList = std::vector; +using CodecPluginPointer = std::shared_ptr; +using CodecPluginList = std::vector; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index 29658eeb6b..fe34d2dbee 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -18,6 +18,7 @@ #include #include "RuntimePlugin.h" +#include "CodecPlugin.h" #include "DisplayPlugin.h" #include "InputPlugin.h" @@ -117,6 +118,7 @@ PluginManager::PluginManager() { // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class extern DisplayPluginList getDisplayPlugins(); extern InputPluginList getInputPlugins(); +extern CodecPluginList getCodecPlugins(); extern void saveInputPluginSettings(const InputPluginList& plugins); static DisplayPluginList displayPlugins; @@ -202,6 +204,34 @@ const InputPluginList& PluginManager::getInputPlugins() { return inputPlugins; } +const CodecPluginList& PluginManager::getCodecPlugins() { + static CodecPluginList codecPlugins; + static std::once_flag once; + std::call_once(once, [&] { + //codecPlugins = ::getCodecPlugins(); + + // Now grab the dynamic plugins + for (auto loader : getLoadedPlugins()) { + CodecProvider* codecProvider = qobject_cast(loader->instance()); + if (codecProvider) { + for (auto codecPlugin : codecProvider->getCodecPlugins()) { + if (codecPlugin->isSupported()) { + codecPlugins.push_back(codecPlugin); + } + } + } + } + + auto& container = PluginContainer::getInstance(); + for (auto plugin : codecPlugins) { + plugin->setContainer(&container); + plugin->init(); + } + }); + return codecPlugins; +} + + void PluginManager::setPreferredDisplayPlugins(const QStringList& displays) { preferredDisplayPlugins = displays; } diff --git a/libraries/plugins/src/plugins/PluginManager.h b/libraries/plugins/src/plugins/PluginManager.h index 7903bdd724..30d52298da 100644 --- a/libraries/plugins/src/plugins/PluginManager.h +++ b/libraries/plugins/src/plugins/PluginManager.h @@ -18,6 +18,7 @@ public: const DisplayPluginList& getDisplayPlugins(); const InputPluginList& getInputPlugins(); + const CodecPluginList& getCodecPlugins(); DisplayPluginList getPreferredDisplayPlugins(); void setPreferredDisplayPlugins(const QStringList& displays); diff --git a/libraries/plugins/src/plugins/RuntimePlugin.h b/libraries/plugins/src/plugins/RuntimePlugin.h index d7bf31ea28..9bf15f344d 100644 --- a/libraries/plugins/src/plugins/RuntimePlugin.h +++ b/libraries/plugins/src/plugins/RuntimePlugin.h @@ -34,3 +34,12 @@ public: #define InputProvider_iid "com.highfidelity.plugins.input" Q_DECLARE_INTERFACE(InputProvider, InputProvider_iid) +class CodecProvider { +public: + virtual ~CodecProvider() {} + virtual CodecPluginList getCodecPlugins() = 0; +}; + +#define CodecProvider_iid "com.highfidelity.plugins.codec" +Q_DECLARE_INTERFACE(CodecProvider, CodecProvider_iid) + diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt new file mode 100644 index 0000000000..162fe557cc --- /dev/null +++ b/plugins/pcmCodec/CMakeLists.txt @@ -0,0 +1,11 @@ +# +# Created by Brad Hefta-Gaub on 6/9/2016 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME pcmCodec) +setup_hifi_plugin() +link_hifi_libraries(shared plugins) diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp new file mode 100644 index 0000000000..264c139ac6 --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -0,0 +1,45 @@ +// +// PCMCodecManager.cpp +// plugins/pcmCodec/src +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include + +#include "PCMCodecManager.h" + +const QString PCMCodecManager::NAME = "PCM Codec"; + +void PCMCodecManager::init() { +} + +void PCMCodecManager::deinit() { +} + +bool PCMCodecManager::activate() { + CodecPlugin::activate(); + return true; +} + +void PCMCodecManager::deactivate() { + CodecPlugin::deactivate(); +} + + +bool PCMCodecManager::isSupported() const { + return true; +} + + +void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { + // this codec doesn't actually do anything.... + decodedBuffer = encodedBuffer; +} + diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h new file mode 100644 index 0000000000..a8b3e49a2c --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -0,0 +1,40 @@ +// +// PCMCodecManager.h +// plugins/pcmCodec/src +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi__PCMCodecManager_h +#define hifi__PCMCodecManager_h + + +#include + +class PCMCodecManager : public CodecPlugin { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + +private: + static const QString NAME; +}; + +#endif // hifi__PCMCodecManager_h diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp new file mode 100644 index 0000000000..732ed2d57d --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp @@ -0,0 +1,44 @@ +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "PCMCodecManager.h" + +class PCMCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + PCMCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~PCMCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + CodecPluginPointer plugin(new PCMCodecManager()); + if (plugin->isSupported()) { + _codecPlugins.push_back(plugin); + } + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "PCMCodecProvider.moc" diff --git a/plugins/pcmCodec/src/plugin.json b/plugins/pcmCodec/src/plugin.json new file mode 100644 index 0000000000..2d86251845 --- /dev/null +++ b/plugins/pcmCodec/src/plugin.json @@ -0,0 +1 @@ +{"name":"PCM Codec"} From d7bffc3eabec8bb0f5f26a3eadf44aa7dfdc3a31 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Jun 2016 17:14:41 -0700 Subject: [PATCH 02/22] first cut at negotiating codecs --- assignment-client/src/audio/AudioMixer.cpp | 26 +++++++++++ assignment-client/src/audio/AudioMixer.h | 1 + interface/src/Application.cpp | 4 +- libraries/audio-client/CMakeLists.txt | 2 +- libraries/audio-client/src/AudioClient.cpp | 46 +++++++++++++++++++ libraries/audio-client/src/AudioClient.h | 5 ++ libraries/networking/src/udt/PacketHeaders.h | 4 +- libraries/plugins/src/plugins/CodecPlugin.h | 1 + .../plugins/src/plugins/PluginManager.cpp | 5 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 4 ++ plugins/pcmCodec/src/PCMCodecManager.h | 1 + 11 files changed, 94 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 03bb32cd53..95578e3998 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -90,6 +90,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::AudioStreamStats }, this, "handleNodeAudioPacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -446,6 +447,31 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } +void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { + qDebug() << __FUNCTION__; + + // read the codecs requested by the client + quint8 numberOfCodecs = 0; + message->readPrimitive(&numberOfCodecs); + QStringList codecList; + for (quint16 i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message->readString(); + qDebug() << "requestedCodec:" << requestedCodec; + codecList.append(requestedCodec); + } + qDebug() << "all requested codecs:" << codecList; + + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); + + // write them to our packet + QString selectedCodec = codecList.front(); + qDebug() << "selectedCodec:" << selectedCodec; + replyPacket->writeString(selectedCodec); + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *sendingNode); +} + void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { // enumerate the connected listeners to remove HRTF objects for the disconnected node auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 24b4b39704..c90a918a5b 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -45,6 +45,7 @@ private slots: void broadcastMixes(); void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); void handleNodeKilled(SharedNodePointer killedNode); void removeHRTFsForFinishedInjector(const QUuid& streamID); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 855cf2f313..3c69f716ca 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1243,7 +1243,6 @@ QString Application::getUserAgent() { userAgent += " " + formatPluginName(cp->getName()); } - qDebug() << __FUNCTION__ << ":" << userAgent; return userAgent; } @@ -4440,6 +4439,9 @@ void Application::nodeActivated(SharedNodePointer node) { } } + if (node->getType() == NodeType::AudioMixer) { + DependencyManager::get()->negotiateAudioFormat(); + } } void Application::nodeKilled(SharedNodePointer node) { diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 2c0fc0a9cd..9c298ce664 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME audio-client) setup_hifi_library(Network Multimedia) -link_hifi_libraries(audio) +link_hifi_libraries(audio plugins) # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2569f27aaa..18db23dbf7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include #include @@ -134,6 +136,7 @@ AudioClient::AudioClient() : packetReceiver.registerListener(PacketType::MixedAudio, this, "handleAudioDataPacket"); packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat"); } AudioClient::~AudioClient() { @@ -474,6 +477,32 @@ void AudioClient::stop() { } } +void AudioClient::negotiateAudioFormat() { + qDebug() << __FUNCTION__; + + auto nodeList = DependencyManager::get(); + + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + + quint8 numberOfCodecs = (quint8)codecPlugins.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + negotiateFormatPacket->writeString(plugin->getName()); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + + void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer message) { char bitset; @@ -528,6 +557,16 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me emit muteEnvironmentRequested(position, radius); } +void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { + qDebug() << __FUNCTION__; + + // write them to our packet + QString selectedCodec = message->readString(); + + qDebug() << "selectedCodec:" << selectedCodec; +} + + QString AudioClient::getDefaultDeviceName(QAudio::Mode mode) { QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode); return deviceInfo.deviceName(); @@ -1227,6 +1266,13 @@ void AudioClient::loadSettings() { windowSecondsForDesiredCalcOnTooManyStarves.get()); _receivedAudioStream.setWindowSecondsForDesiredReduction(windowSecondsForDesiredReduction.get()); _receivedAudioStream.setRepetitionWithFade(repetitionWithFade.get()); + + qDebug() << "---- Initializing Audio Client ----"; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + } + } void AudioClient::saveSettings() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 3a14c878f6..e05612f859 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -94,6 +94,8 @@ public: int _unfulfilledReads; }; + void negotiateAudioFormat(); + const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } @@ -139,6 +141,7 @@ public slots: void handleAudioDataPacket(QSharedPointer message); void handleNoisyMutePacket(QSharedPointer message); void handleMuteEnvironmentPacket(QSharedPointer message); + void handleSelectedAudioFormat(QSharedPointer message); void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); } void handleAudioInput(); @@ -292,6 +295,8 @@ private: void checkDevices(); bool _hasReceivedFirstPacket = false; + + //CodecPluginPointer _codec { nullptr }; }; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index ae54450fee..b5d1be077f 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -95,7 +95,9 @@ public: AssetMappingOperation, AssetMappingOperationReply, ICEServerHeartbeatACK, - LAST_PACKET_TYPE = ICEServerHeartbeatACK + NegotiateAudioFormat, + SelectedAudioFormat, + LAST_PACKET_TYPE = SelectedAudioFormat }; }; diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index d9e1b947fe..6944d91bed 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -15,5 +15,6 @@ class CodecPlugin : public Plugin { public: virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; }; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index fe34d2dbee..0b4afe1be0 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -222,10 +222,11 @@ const CodecPluginList& PluginManager::getCodecPlugins() { } } - auto& container = PluginContainer::getInstance(); for (auto plugin : codecPlugins) { - plugin->setContainer(&container); + plugin->setContainer(_container); plugin->init(); + + qDebug() << "init codec:" << plugin->getName(); } }); return codecPlugins; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 264c139ac6..2401e99576 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -43,3 +43,7 @@ void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decode decodedBuffer = encodedBuffer; } +void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + // this codec doesn't actually do anything.... + encodedBuffer = decodedBuffer; +} diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index a8b3e49a2c..3b7ca36c02 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -32,6 +32,7 @@ public: void deactivate() override; virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; private: static const QString NAME; From 7a4b11ee9790e902d55e12292fe9b420c72478f3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 27 Jun 2016 13:06:19 -0700 Subject: [PATCH 03/22] more work on codecs --- assignment-client/CMakeLists.txt | 2 +- assignment-client/src/audio/AudioMixer.cpp | 25 ++++++ libraries/audio-client/src/AudioClient.cpp | 94 +++++++++++++++------- libraries/audio-client/src/AudioClient.h | 5 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 2 + 5 files changed, 95 insertions(+), 33 deletions(-) diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index 1b5840c3c8..d0fd2c1176 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets) link_hifi_libraries( audio avatars octree gpu model fbx entities networking animation recording shared script-engine embedded-webserver - controllers physics + controllers physics plugins ) if (WIN32) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 95578e3998..27b0a092d0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include #include @@ -447,6 +449,20 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } +DisplayPluginList getDisplayPlugins() { + DisplayPluginList result; + return result; +} + +InputPluginList getInputPlugins() { + InputPluginList result; + return result; +} + +void saveInputPluginSettings(const InputPluginList& plugins) { +} + + void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { qDebug() << __FUNCTION__; @@ -461,6 +477,15 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess } qDebug() << "all requested codecs:" << codecList; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + } + } else { + qDebug() << "No Codecs available..."; + } + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); // write them to our packet diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 18db23dbf7..7ef71087ce 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -477,32 +477,6 @@ void AudioClient::stop() { } } -void AudioClient::negotiateAudioFormat() { - qDebug() << __FUNCTION__; - - auto nodeList = DependencyManager::get(); - - auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); - - auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); - - quint8 numberOfCodecs = (quint8)codecPlugins.size(); - negotiateFormatPacket->writePrimitive(numberOfCodecs); - for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - negotiateFormatPacket->writeString(plugin->getName()); - } - - // grab our audio mixer from the NodeList, if it exists - SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); - - if (audioMixer) { - // send off this mute packet - nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); - } -} - - void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer message) { char bitset; @@ -557,13 +531,47 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me emit muteEnvironmentRequested(position, radius); } +void AudioClient::negotiateAudioFormat() { + qDebug() << __FUNCTION__; + + auto nodeList = DependencyManager::get(); + + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + + quint8 numberOfCodecs = (quint8)codecPlugins.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + negotiateFormatPacket->writeString(plugin->getName()); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { qDebug() << __FUNCTION__; // write them to our packet - QString selectedCodec = message->readString(); + _selectedCodecName = message->readString(); + + qDebug() << "Selected Codec:" << _selectedCodecName; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + if (_selectedCodecName == plugin->getName()) { + _codec = plugin; + qDebug() << "Selected Codec Plugin:" << _codec.get(); + break; + } + } - qDebug() << "selectedCodec:" << selectedCodec; } @@ -839,7 +847,17 @@ void AudioClient::handleAudioInput() { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); // FIXME find a way to properly handle both playback audio and user audio concurrently - emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); + + // TODO - codec encode goes here + QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + QByteArray encodedBuffer; + if (_codec) { + _codec->encode(decocedBuffer, encodedBuffer); + } else { + encodedBuffer = decocedBuffer; + } + + emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); _stats.sentPacket(); } } @@ -848,14 +866,28 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { Transform audioTransform; audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); + + // TODO - codec decode goes here + QByteArray encodedBuffer; + if (_codec) { + _codec->encode(audio, encodedBuffer); + } else { + encodedBuffer = audio; + } + // FIXME check a flag to see if we should echo audio? - emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); + emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) { // TODO - codec decode goes here - QByteArray decodedBuffer = networkBuffer; + QByteArray decodedBuffer; + if (_codec) { + _codec->decode(networkBuffer, decodedBuffer); + } else { + decodedBuffer = networkBuffer; + } const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = _outputFrameSize; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e05612f859..45b4a631b2 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -38,6 +38,8 @@ #include #include +#include + #include "AudioIOStats.h" #include "AudioNoiseGate.h" #include "AudioSRC.h" @@ -296,7 +298,8 @@ private: bool _hasReceivedFirstPacket = false; - //CodecPluginPointer _codec { nullptr }; + CodecPluginPointer _codec; + QString _selectedCodecName; }; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 2401e99576..eb2f4761b4 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -39,11 +39,13 @@ bool PCMCodecManager::isSupported() const { void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { + qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size(); // this codec doesn't actually do anything.... decodedBuffer = encodedBuffer; } void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size(); // this codec doesn't actually do anything.... encodedBuffer = decodedBuffer; } From 38f0cd218bc15e2c1d1ba80dc0d47d317d2c3a68 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 7 Jul 2016 10:31:12 -0700 Subject: [PATCH 04/22] make client server plugins properly copy into plugins directory --- .../macros/SetupHifiClientServerPlugin.cmake | 54 +++++++++++++++++++ plugins/pcmCodec/CMakeLists.txt | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 cmake/macros/SetupHifiClientServerPlugin.cmake diff --git a/cmake/macros/SetupHifiClientServerPlugin.cmake b/cmake/macros/SetupHifiClientServerPlugin.cmake new file mode 100644 index 0000000000..5ace348e87 --- /dev/null +++ b/cmake/macros/SetupHifiClientServerPlugin.cmake @@ -0,0 +1,54 @@ +# +# Created by Brad Hefta-Gaub on 2016/07/07 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# +macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN) + set(${TARGET_NAME}_SHARED 1) + setup_hifi_library(${ARGV}) + add_dependencies(interface ${TARGET_NAME}) + set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins") + + if (APPLE) + set(PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns") + else() + set(PLUGIN_PATH "plugins") + endif() + + if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${PLUGIN_PATH}/") + else() + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${PLUGIN_PATH}/") + endif() + + # create the destination for the client plugin binaries + add_custom_command( + TARGET ${TARGET_NAME} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E make_directory + ${CLIENT_PLUGIN_FULL_PATH} + ) + # copy the client plugin binaries + add_custom_command(TARGET ${DIR} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "$" + ${CLIENT_PLUGIN_FULL_PATH} + ) + + # create the destination for the server plugin binaries + add_custom_command( + TARGET ${TARGET_NAME} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E make_directory + ${SERVER_PLUGIN_FULL_PATH} + ) + # copy the server plugin binaries + add_custom_command(TARGET ${DIR} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "$" + ${SERVER_PLUGIN_FULL_PATH} + ) + +endmacro() diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt index 162fe557cc..5dca1f0e14 100644 --- a/plugins/pcmCodec/CMakeLists.txt +++ b/plugins/pcmCodec/CMakeLists.txt @@ -7,5 +7,5 @@ # set(TARGET_NAME pcmCodec) -setup_hifi_plugin() +setup_hifi_client_server_plugin() link_hifi_libraries(shared plugins) From 3c6447326e1e8678d0a05dfc99cc9b858bab6c1a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 7 Jul 2016 21:24:24 -0700 Subject: [PATCH 05/22] more codec plumbing --- assignment-client/src/audio/AudioMixer.cpp | 36 ++++++++++++++++--- .../src/audio/AudioMixerClientData.cpp | 7 +++- .../src/audio/AudioMixerClientData.h | 7 ++++ libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio/CMakeLists.txt | 2 +- libraries/audio/src/InboundAudioStream.cpp | 11 +++++- libraries/audio/src/InboundAudioStream.h | 6 ++++ plugins/pcmCodec/src/PCMCodecManager.cpp | 8 +++-- 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 27b0a092d0..9d9194aef6 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -477,21 +477,39 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess } qDebug() << "all requested codecs:" << codecList; + CodecPluginPointer selectedCoded; + QString selectedCodecName; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { qDebug() << "Codec available:" << plugin->getName(); + + // choose first codec + if (!selectedCoded) { + selectedCoded = plugin; + selectedCodecName = plugin->getName(); + } } } else { qDebug() << "No Codecs available..."; } + auto clientData = dynamic_cast(sendingNode->getLinkedData()); + + clientData->_codec = selectedCoded; + clientData->_selectedCodecName = selectedCodecName; + qDebug() << "selectedCodecName:" << selectedCodecName; + + auto avatarAudioStream = clientData->getAvatarAudioStream(); + if (avatarAudioStream) { + avatarAudioStream->_codec = selectedCoded; + avatarAudioStream->_selectedCodecName = selectedCodecName; + } + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); // write them to our packet - QString selectedCodec = codecList.front(); - qDebug() << "selectedCodec:" << selectedCodec; - replyPacket->writeString(selectedCodec); + replyPacket->writeString(selectedCodecName); auto nodeList = DependencyManager::get(); nodeList->sendPacket(std::move(replyPacket), *sendingNode); @@ -720,9 +738,17 @@ void AudioMixer::broadcastMixes() { quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); + // TODO - codec encode goes here + QByteArray decocedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + QByteArray encodedBuffer; + if (nodeData->_codec) { + nodeData->_codec->encode(decocedBuffer, encodedBuffer); + } else { + encodedBuffer = decocedBuffer; + } + // pack mixed audio samples - mixPacket->write(reinterpret_cast(_clampedSamples), - AudioConstants::NETWORK_FRAME_BYTES_STEREO); + mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16); mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 20003ba10d..080a833c0e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -101,9 +101,14 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isStereo = channelFlag == 1; + auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); + avatarAudioStream->_codec = _codec; + avatarAudioStream->_selectedCodecName = _selectedCodecName; + qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName; + auto emplaced = _audioStreams.emplace( QUuid(), - std::unique_ptr { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) } + std::unique_ptr { avatarAudioStream } ); micStreamIt = emplaced.first; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 17274a1519..0b3b352e66 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -19,6 +19,8 @@ #include #include +#include + #include "PositionalAudioStream.h" #include "AvatarAudioStream.h" @@ -65,6 +67,11 @@ public: AudioLimiter audioLimiter; + // FIXME -- maybe make these private + CodecPluginPointer _codec; + QString _selectedCodecName; + + signals: void injectorStreamFinished(const QUuid& streamIdentifier); diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 147030c831..9dc096eefd 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -837,7 +837,7 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); - // TODO - codec decode goes here + // TODO - codec encode goes here QByteArray encodedBuffer; if (_codec) { _codec->encode(audio, encodedBuffer); diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt index c49c9547a5..1e9360b9a2 100644 --- a/libraries/audio/CMakeLists.txt +++ b/libraries/audio/CMakeLists.txt @@ -1,3 +1,3 @@ set(TARGET_NAME audio) setup_hifi_library(Network) -link_hifi_libraries(networking shared) +link_hifi_libraries(networking shared plugins) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 8b32ada296..c19eb0c161 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -178,7 +178,16 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); + + // codec decode goes here + QByteArray decodedBuffer; + if (_codec) { + _codec->decode(packetAfterStreamProperties, decodedBuffer); + } else { + decodedBuffer = packetAfterStreamProperties; + } + + return _ringBuffer.writeData(decodedBuffer.data(), numAudioSamples * sizeof(int16_t)); // FIXME? } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 3f641f1ba4..ddc0dc1dc3 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -18,6 +18,8 @@ #include #include +#include + #include "AudioRingBuffer.h" #include "MovingMinMaxAvg.h" #include "SequenceNumberStats.h" @@ -174,6 +176,10 @@ public: void setReverb(float reverbTime, float wetLevel); void clearReverb() { _hasReverb = false; } + // FIXME -- maybe make these private + CodecPluginPointer _codec; + QString _selectedCodecName; + public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index eb2f4761b4..d204fb1100 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -39,13 +39,17 @@ bool PCMCodecManager::isSupported() const { void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size(); // this codec doesn't actually do anything.... decodedBuffer = encodedBuffer; + + //decodedBuffer = qUncompress(encodedBuffer); + //qDebug() << __FUNCTION__ << "from:" << encodedBuffer.size() << " to:" << decodedBuffer.size(); } void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size(); // this codec doesn't actually do anything.... encodedBuffer = decodedBuffer; + + //encodedBuffer = qCompress(decodedBuffer); + //qDebug() << __FUNCTION__ << "from:" << decodedBuffer.size() << " to:" << encodedBuffer.size(); } From 126e5c29264566657ef727e6858a992b0ecec759 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 8 Jul 2016 14:34:44 -0700 Subject: [PATCH 06/22] codec pipeline working, zlib example --- assignment-client/src/audio/AudioMixer.cpp | 8 ++++++++ libraries/audio-client/src/AudioClient.cpp | 16 +++------------- libraries/audio/src/InboundAudioStream.cpp | 11 ++++++++++- .../audio/src/MixedProcessedAudioStream.cpp | 15 +++++++++++++-- plugins/pcmCodec/src/PCMCodecManager.cpp | 16 +++++----------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 9d9194aef6..b419939912 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -496,6 +496,14 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess auto clientData = dynamic_cast(sendingNode->getLinkedData()); + // FIXME - why would we not have client data at this point?? + if (!clientData) { + qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__; + sendingNode->setLinkedData(std::unique_ptr { new AudioMixerClientData(sendingNode->getUUID()) }); + clientData = dynamic_cast(sendingNode->getLinkedData()); + connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); + } + clientData->_codec = selectedCoded; clientData->_selectedCodecName = selectedCodecName; qDebug() << "selectedCodecName:" << selectedCodecName; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 9dc096eefd..a076b1b290 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -534,14 +534,13 @@ void AudioClient::negotiateAudioFormat() { void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { qDebug() << __FUNCTION__; - // write them to our packet - _selectedCodecName = message->readString(); + _receivedAudioStream._selectedCodecName = _selectedCodecName = message->readString(); qDebug() << "Selected Codec:" << _selectedCodecName; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { - _codec = plugin; + _receivedAudioStream._codec = _codec = plugin; qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } @@ -849,16 +848,7 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } -void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) { - - // TODO - codec decode goes here - QByteArray decodedBuffer; - if (_codec) { - _codec->decode(networkBuffer, decodedBuffer); - } else { - decodedBuffer = networkBuffer; - } - +void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = _outputFrameSize; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index c19eb0c161..5d4dbd9b94 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -187,7 +187,16 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet decodedBuffer = packetAfterStreamProperties; } - return _ringBuffer.writeData(decodedBuffer.data(), numAudioSamples * sizeof(int16_t)); // FIXME? + auto actualSize = decodedBuffer.size(); + + /* + auto expectedSize = numAudioSamples * sizeof(int16_t); + if (expectedSize != actualSize) { + qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize; + } + */ + + return _ringBuffer.writeData(decodedBuffer.data(), actualSize); } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index d236ac7aad..220b2bd9ee 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -44,10 +44,21 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { - emit addedStereoSamples(packetAfterStreamProperties); + // TODO - codec decode goes here + QByteArray decodedBuffer; + if (_codec) { + _codec->decode(packetAfterStreamProperties, decodedBuffer); + } else { + decodedBuffer = packetAfterStreamProperties; + } + + qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size(); + + + emit addedStereoSamples(decodedBuffer); QByteArray outputBuffer; - emit processSamples(packetAfterStreamProperties, outputBuffer); + emit processSamples(decodedBuffer, outputBuffer); _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index d204fb1100..9d55cafedf 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -15,7 +15,7 @@ #include "PCMCodecManager.h" -const QString PCMCodecManager::NAME = "PCM Codec"; +const QString PCMCodecManager::NAME = "zlib"; void PCMCodecManager::init() { } @@ -39,17 +39,11 @@ bool PCMCodecManager::isSupported() const { void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - // this codec doesn't actually do anything.... - decodedBuffer = encodedBuffer; - - //decodedBuffer = qUncompress(encodedBuffer); - //qDebug() << __FUNCTION__ << "from:" << encodedBuffer.size() << " to:" << decodedBuffer.size(); + //decodedBuffer = encodedBuffer; + decodedBuffer = qUncompress(encodedBuffer); } void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - // this codec doesn't actually do anything.... - encodedBuffer = decodedBuffer; - - //encodedBuffer = qCompress(decodedBuffer); - //qDebug() << __FUNCTION__ << "from:" << decodedBuffer.size() << " to:" << encodedBuffer.size(); + //encodedBuffer = decodedBuffer; + encodedBuffer = qCompress(decodedBuffer); } From 489d2e4ca50ecc3ecc3ece28304c352b4352a472 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 8 Jul 2016 15:33:11 -0700 Subject: [PATCH 07/22] Update max request limit and number of processing threads --- interface/src/Application.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 82cbaf93c2..5d5bf45dcf 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -168,6 +168,9 @@ static QTimer locationUpdateTimer; static QTimer identityPacketTimer; static QTimer pingTimer; +static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16; +static const int PROCESSING_THREAD_POOL_SIZE = 6; + static const QString SNAPSHOT_EXTENSION = ".jpg"; static const QString SVO_EXTENSION = ".svo"; static const QString SVO_JSON_EXTENSION = ".svo.json"; @@ -517,7 +520,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : // (main thread, present thread, random OS load) // More threads == faster concurrent loads, but also more concurrent // load on the GPU until we can serialize GPU transfers (off the main thread) - QThreadPool::globalInstance()->setMaxThreadCount(2); + QThreadPool::globalInstance()->setMaxThreadCount(PROCESSING_THREAD_POOL_SIZE); thread()->setPriority(QThread::HighPriority); thread()->setObjectName("Main Thread"); @@ -728,7 +731,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : connect(&identityPacketTimer, &QTimer::timeout, getMyAvatar(), &MyAvatar::sendIdentityPacket); identityPacketTimer.start(AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS); - ResourceCache::setRequestLimit(3); + ResourceCache::setRequestLimit(MAX_CONCURRENT_RESOURCE_DOWNLOADS); _glWidget = new GLCanvas(); getApplicationCompositor().setRenderingWidget(_glWidget); From a71baf5601ea269804ac30ba92eb8a148f89833e Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Fri, 8 Jul 2016 21:18:03 -0700 Subject: [PATCH 08/22] handle codec prioritization, handle multiple codec choices --- assignment-client/src/audio/AudioMixer.cpp | 73 ++++++++++++++----- assignment-client/src/audio/AudioMixer.h | 2 + .../resources/describe-settings.json | 8 ++ libraries/audio-client/src/AudioClient.cpp | 9 +-- libraries/audio/src/InboundAudioStream.cpp | 15 +--- libraries/audio/src/InboundAudioStream.h | 2 +- .../audio/src/MixedProcessedAudioStream.cpp | 7 +- .../audio/src/MixedProcessedAudioStream.h | 2 +- libraries/networking/src/udt/BasePacket.cpp | 7 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 51 ++++++++++--- plugins/pcmCodec/src/PCMCodecManager.h | 25 ++++++- plugins/pcmCodec/src/PCMCodecProvider.cpp | 13 +++- 12 files changed, 149 insertions(+), 65 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b419939912..f49b674dce 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -466,34 +466,61 @@ void saveInputPluginSettings(const InputPluginList& plugins) { void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { qDebug() << __FUNCTION__; - // read the codecs requested by the client - quint8 numberOfCodecs = 0; - message->readPrimitive(&numberOfCodecs); - QStringList codecList; - for (quint16 i = 0; i < numberOfCodecs; i++) { - QString requestedCodec = message->readString(); - qDebug() << "requestedCodec:" << requestedCodec; - codecList.append(requestedCodec); - } - qDebug() << "all requested codecs:" << codecList; - - CodecPluginPointer selectedCoded; - QString selectedCodecName; + QStringList availableCodecs; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - - // choose first codec - if (!selectedCoded) { - selectedCoded = plugin; - selectedCodecName = plugin->getName(); - } + auto codecName = plugin->getName(); + qDebug() << "Codec available:" << codecName; + availableCodecs.append(codecName); } } else { qDebug() << "No Codecs available..."; } + CodecPluginPointer selectedCoded; + QString selectedCodecName; + + QStringList codecPreferenceList = _codecPreferenceOrder.split(","); + + // read the codecs requested by the client + const int MAX_PREFERENCE = 99999; + int preferredCodecIndex = MAX_PREFERENCE; + QString preferredCodec; + quint8 numberOfCodecs = 0; + message->readPrimitive(&numberOfCodecs); + qDebug() << "numberOfCodecs:" << numberOfCodecs; + QStringList codecList; + for (quint16 i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message->readString(); + int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec); + bool codecAvailable = availableCodecs.contains(requestedCodec); + qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable; + if (codecAvailable) { + codecList.append(requestedCodec); + if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) { + qDebug() << "This codec is preferred..."; + selectedCodecName = requestedCodec; + preferredCodecIndex = preferenceOfThisCodec; + } + } + } + qDebug() << "all requested and available codecs:" << codecList; + + // choose first codec + if (!selectedCodecName.isEmpty()) { + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + if (selectedCodecName == plugin->getName()) { + qDebug() << "Selecting codec:" << selectedCodecName; + selectedCoded = plugin; + break; + } + } + } + } + + auto clientData = dynamic_cast(sendingNode->getLinkedData()); // FIXME - why would we not have client data at this point?? @@ -882,6 +909,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) { QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject(); + const QString CODEC_PREFERENCE_ORDER = "codec_preference_order"; + if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) { + _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString(); + qDebug() << "Codec preference order changed to" << _codecPreferenceOrder; + } + const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance"; if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) { bool ok = false; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index c90a918a5b..4b2a27120d 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -92,6 +92,8 @@ private: int _manualEchoMixes { 0 }; int _totalMixes { 0 }; + QString _codecPreferenceOrder; + float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 7375a0f650..948c6ddc18 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -718,6 +718,14 @@ "placeholder": "(in percent)" } ] + }, + { + "name": "codec_preference_order", + "label": "Audio Codec Preference Order", + "help": "List of codec names in order of preferred usage", + "placeholder": "hifiAC, zlib, pcm", + "default": "hifiAC,zlib,pcm", + "advanced": true } ] }, diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a076b1b290..97ef4b2981 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -507,19 +507,14 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me } void AudioClient::negotiateAudioFormat() { - qDebug() << __FUNCTION__; - auto nodeList = DependencyManager::get(); - auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); - auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); - quint8 numberOfCodecs = (quint8)codecPlugins.size(); negotiateFormatPacket->writePrimitive(numberOfCodecs); for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - negotiateFormatPacket->writeString(plugin->getName()); + auto codecName = plugin->getName(); + negotiateFormatPacket->writeString(codecName); } // grab our audio mixer from the NodeList, if it exists diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 5d4dbd9b94..7c6a34ae55 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { if (message.getType() == PacketType::SilentAudioFrame) { writeDroppableSilentSamples(networkSamples); } else { - parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples); + parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead())); } break; } @@ -177,25 +177,14 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } } -int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - - // codec decode goes here +int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_codec) { _codec->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } - auto actualSize = decodedBuffer.size(); - - /* - auto expectedSize = numAudioSamples * sizeof(int16_t); - if (expectedSize != actualSize) { - qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize; - } - */ - return _ringBuffer.writeData(decodedBuffer.data(), actualSize); } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index ddc0dc1dc3..f9ca088fab 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -207,7 +207,7 @@ protected: /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentSamples(int silentSamples); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 220b2bd9ee..6939ee540a 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -42,9 +42,7 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { return deviceSamplesWritten; } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { - - // TODO - codec decode goes here +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; if (_codec) { _codec->decode(packetAfterStreamProperties, decodedBuffer); @@ -52,9 +50,6 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& decodedBuffer = packetAfterStreamProperties; } - qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size(); - - emit addedStereoSamples(decodedBuffer); QByteArray outputBuffer; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 5ea0157421..2f9a691278 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -35,7 +35,7 @@ public: protected: int writeDroppableSilentSamples(int silentSamples); int writeLastFrameRepeatedWithFade(int samples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); private: int networkToDeviceSamples(int networkSamples); diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index dba241f221..0456e095ae 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) { qint64 BasePacket::writeString(const QString& string) { QByteArray data = string.toUtf8(); - writePrimitive(static_cast(data.length())); - return writeData(data.constData(), data.length()); + uint32_t length = data.length(); + writePrimitive(length); + auto result = writeData(data.constData(), data.length()); + seek(pos() + length); + return length + sizeof(uint32_t); } QString BasePacket::readString() { diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 9d55cafedf..f787c6682d 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -1,5 +1,5 @@ // -// PCMCodecManager.cpp +// PCMCodec.cpp // plugins/pcmCodec/src // // Created by Brad Hefta-Gaub on 6/9/2016 @@ -15,35 +15,64 @@ #include "PCMCodecManager.h" -const QString PCMCodecManager::NAME = "zlib"; +const QString PCMCodec::NAME = "pcm"; -void PCMCodecManager::init() { +void PCMCodec::init() { } -void PCMCodecManager::deinit() { +void PCMCodec::deinit() { } -bool PCMCodecManager::activate() { +bool PCMCodec::activate() { CodecPlugin::activate(); return true; } -void PCMCodecManager::deactivate() { +void PCMCodec::deactivate() { CodecPlugin::deactivate(); } -bool PCMCodecManager::isSupported() const { +bool PCMCodec::isSupported() const { return true; } -void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - //decodedBuffer = encodedBuffer; +void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { + decodedBuffer = encodedBuffer; +} + +void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + encodedBuffer = decodedBuffer; +} + + +const QString zLibCodec::NAME = "zlib"; + +void zLibCodec::init() { +} + +void zLibCodec::deinit() { +} + +bool zLibCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void zLibCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool zLibCodec::isSupported() const { + return true; +} + +void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { decodedBuffer = qUncompress(encodedBuffer); } -void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - //encodedBuffer = decodedBuffer; +void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { encodedBuffer = qCompress(decodedBuffer); } diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 3b7ca36c02..7816660c5d 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -15,7 +15,7 @@ #include -class PCMCodecManager : public CodecPlugin { +class PCMCodec : public CodecPlugin { Q_OBJECT public: @@ -38,4 +38,27 @@ private: static const QString NAME; }; +class zLibCodec : public CodecPlugin { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + +private: + static const QString NAME; +}; + #endif // hifi__PCMCodecManager_h diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp index 732ed2d57d..351b1adf3f 100644 --- a/plugins/pcmCodec/src/PCMCodecProvider.cpp +++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp @@ -29,10 +29,17 @@ public: virtual CodecPluginList getCodecPlugins() override { static std::once_flag once; std::call_once(once, [&] { - CodecPluginPointer plugin(new PCMCodecManager()); - if (plugin->isSupported()) { - _codecPlugins.push_back(plugin); + + CodecPluginPointer pcmCodec(new PCMCodec()); + if (pcmCodec->isSupported()) { + _codecPlugins.push_back(pcmCodec); } + + CodecPluginPointer zlibCodec(new zLibCodec()); + if (zlibCodec->isSupported()) { + _codecPlugins.push_back(zlibCodec); + } + }); return _codecPlugins; } From ba6bb245956b4657b4d234afd342f40d50296f6c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sun, 10 Jul 2016 16:49:03 -0700 Subject: [PATCH 09/22] rework plugins to allow different decoder/encoder instances per streams --- assignment-client/src/audio/AudioMixer.cpp | 6 +- .../src/audio/AudioMixerClientData.cpp | 11 ++++ .../src/audio/AudioMixerClientData.h | 3 + libraries/audio-client/src/AudioClient.cpp | 10 +-- libraries/audio-client/src/AudioClient.h | 1 + libraries/audio/src/InboundAudioStream.cpp | 4 +- libraries/audio/src/InboundAudioStream.h | 1 + .../audio/src/MixedProcessedAudioStream.cpp | 4 +- libraries/plugins/src/plugins/CodecPlugin.h | 18 +++++- plugins/pcmCodec/src/PCMCodecManager.cpp | 64 ++++++++++++++++--- plugins/pcmCodec/src/PCMCodecManager.h | 37 +++++++++-- 11 files changed, 135 insertions(+), 24 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f49b674dce..825f970f2e 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -533,12 +533,16 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess clientData->_codec = selectedCoded; clientData->_selectedCodecName = selectedCodecName; + clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + qDebug() << "selectedCodecName:" << selectedCodecName; auto avatarAudioStream = clientData->getAvatarAudioStream(); if (avatarAudioStream) { avatarAudioStream->_codec = selectedCoded; avatarAudioStream->_selectedCodecName = selectedCodecName; + avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); } auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); @@ -777,7 +781,7 @@ void AudioMixer::broadcastMixes() { QByteArray decocedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); QByteArray encodedBuffer; if (nodeData->_codec) { - nodeData->_codec->encode(decocedBuffer, encodedBuffer); + nodeData->_encoder->encode(decocedBuffer, encodedBuffer); } else { encodedBuffer = decocedBuffer; } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 080a833c0e..3e9b72d60e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) : _frameToSendStats = distribution(numberGenerator); } +AudioMixerClientData::~AudioMixerClientData() { + if (_codec) { + _codec->releaseDecoder(_decoder); + _codec->releaseEncoder(_encoder); + } +} + + AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { QReadLocker readLocker { &_streamsLock }; @@ -104,6 +112,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); avatarAudioStream->_codec = _codec; avatarAudioStream->_selectedCodecName = _selectedCodecName; + if (_codec) { + avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + } qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName; auto emplaced = _audioStreams.emplace( diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 0b3b352e66..ee6f22ff3b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -29,6 +29,7 @@ class AudioMixerClientData : public NodeData { Q_OBJECT public: AudioMixerClientData(const QUuid& nodeID); + ~AudioMixerClientData(); using SharedStreamPointer = std::shared_ptr; using AudioStreamMap = std::unordered_map; @@ -70,6 +71,8 @@ public: // FIXME -- maybe make these private CodecPluginPointer _codec; QString _selectedCodecName; + Encoder* _encoder { nullptr }; // for outbound mixed stream + Decoder* _decoder { nullptr }; // for mic stream signals: diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 97ef4b2981..db5376a380 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -536,6 +536,8 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer mess for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { _receivedAudioStream._codec = _codec = plugin; + _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } @@ -815,8 +817,8 @@ void AudioClient::handleAudioInput() { // TODO - codec encode goes here QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); QByteArray encodedBuffer; - if (_codec) { - _codec->encode(decocedBuffer, encodedBuffer); + if (_encoder) { + _encoder->encode(decocedBuffer, encodedBuffer); } else { encodedBuffer = decocedBuffer; } @@ -833,8 +835,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { // TODO - codec encode goes here QByteArray encodedBuffer; - if (_codec) { - _codec->encode(audio, encodedBuffer); + if (_encoder) { + _encoder->encode(audio, encodedBuffer); } else { encodedBuffer = audio; } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 66555d1e2d..bd0afe453d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -299,6 +299,7 @@ private: CodecPluginPointer _codec; QString _selectedCodecName; + Encoder* _encoder { nullptr }; // for outbound mic stream }; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 7c6a34ae55..f47c6cce46 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -179,8 +179,8 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; - if (_codec) { - _codec->decode(packetAfterStreamProperties, decodedBuffer); + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f9ca088fab..57bff80fff 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -179,6 +179,7 @@ public: // FIXME -- maybe make these private CodecPluginPointer _codec; QString _selectedCodecName; + Decoder* _decoder { nullptr }; public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 6939ee540a..728deae0b1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -44,8 +44,8 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; - if (_codec) { - _codec->decode(packetAfterStreamProperties, decodedBuffer); + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); } else { decodedBuffer = packetAfterStreamProperties; } diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 6944d91bed..280853e37e 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -12,9 +12,23 @@ #include "Plugin.h" -class CodecPlugin : public Plugin { +class Encoder { public: - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; }; +class Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + + // numFrames - number of samples (mono) or sample-pairs (stereo) + virtual void trackLostFrames(int numFrames) = 0; +}; + +class CodecPlugin : public Plugin { +public: + virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; + virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; + virtual void releaseEncoder(Encoder* encoder) = 0; + virtual void releaseDecoder(Decoder* decoder) = 0; +}; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index f787c6682d..1cad1ea4e6 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -37,15 +37,54 @@ bool PCMCodec::isSupported() const { return true; } +class PCMEncoder : public Encoder { +public: + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } +}; -void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - decodedBuffer = encodedBuffer; +class PCMDecoder : public Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void trackLostFrames(int numFrames) override { } +}; + +Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { + return new PCMEncoder(); } -void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - encodedBuffer = decodedBuffer; +Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { + return new PCMDecoder(); } +void PCMCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void PCMCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} + + +class zLibEncoder : public Encoder { +public: + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = qCompress(decodedBuffer); + } +}; + +class zLibDecoder : public Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = qUncompress(encodedBuffer); + } + + virtual void trackLostFrames(int numFrames) override { } +}; const QString zLibCodec::NAME = "zlib"; @@ -69,10 +108,19 @@ bool zLibCodec::isSupported() const { return true; } -void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - decodedBuffer = qUncompress(encodedBuffer); +Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { + return new zLibEncoder(); } -void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - encodedBuffer = qCompress(decodedBuffer); +Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { + return new zLibDecoder(); } + +void zLibCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void zLibCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} + diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 7816660c5d..6eb7f44c15 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -12,9 +12,31 @@ #ifndef hifi__PCMCodecManager_h #define hifi__PCMCodecManager_h - #include +/* +class Encoder { +public: +virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; +}; + +class Decoder { +public: +virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + +// numFrames - number of samples (mono) or sample-pairs (stereo) +virtual void trackLostFrames(int numFrames) = 0; +}; + +class CodecPlugin : public Plugin { +public: +virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; +virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; +virtual void releaseEncoder(Encoder* encoder) = 0; +virtual void releaseDecoder(Decoder* decoder) = 0; +}; +*/ + class PCMCodec : public CodecPlugin { Q_OBJECT @@ -31,8 +53,10 @@ public: /// Called when a plugin is no longer being used. May be called multiple times. void deactivate() override; - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; private: static const QString NAME; @@ -54,8 +78,11 @@ public: /// Called when a plugin is no longer being used. May be called multiple times. void deactivate() override; - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + private: static const QString NAME; From ed9715ae5f25a828dca312740c9da33690653b96 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sun, 10 Jul 2016 17:40:58 -0700 Subject: [PATCH 10/22] some cleanup, proper memory allocation/deallocation --- assignment-client/src/audio/AudioMixer.cpp | 22 ++++---------- .../src/audio/AudioMixerClientData.cpp | 30 +++++++++++++++---- .../src/audio/AudioMixerClientData.h | 20 +++++++++---- libraries/audio-client/src/AudioClient.cpp | 15 ++++++++-- libraries/audio/src/InboundAudioStream.cpp | 18 +++++++++++ libraries/audio/src/InboundAudioStream.h | 11 ++++--- 6 files changed, 81 insertions(+), 35 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 825f970f2e..2f1a5de309 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -478,7 +478,7 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess qDebug() << "No Codecs available..."; } - CodecPluginPointer selectedCoded; + CodecPluginPointer selectedCodec; QString selectedCodecName; QStringList codecPreferenceList = _codecPreferenceOrder.split(","); @@ -513,7 +513,7 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess for (auto& plugin : codecPlugins) { if (selectedCodecName == plugin->getName()) { qDebug() << "Selecting codec:" << selectedCodecName; - selectedCoded = plugin; + selectedCodec = plugin; break; } } @@ -531,18 +531,13 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); } - clientData->_codec = selectedCoded; - clientData->_selectedCodecName = selectedCodecName; - clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + clientData->setupCodec(selectedCodec, selectedCodecName); qDebug() << "selectedCodecName:" << selectedCodecName; auto avatarAudioStream = clientData->getAvatarAudioStream(); if (avatarAudioStream) { - avatarAudioStream->_codec = selectedCoded; - avatarAudioStream->_selectedCodecName = selectedCodecName; - avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + avatarAudioStream->setupCodec(selectedCodec, selectedCodecName, AudioConstants::MONO); } auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); @@ -777,14 +772,9 @@ void AudioMixer::broadcastMixes() { quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); - // TODO - codec encode goes here - QByteArray decocedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); QByteArray encodedBuffer; - if (nodeData->_codec) { - nodeData->_encoder->encode(decocedBuffer, encodedBuffer); - } else { - encodedBuffer = decocedBuffer; - } + nodeData->encode(decodedBuffer, encodedBuffer); // pack mixed audio samples mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 3e9b72d60e..c733683e5a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -110,12 +110,8 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isStereo = channelFlag == 1; auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); - avatarAudioStream->_codec = _codec; - avatarAudioStream->_selectedCodecName = _selectedCodecName; - if (_codec) { - avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); - } - qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName; + avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO); + qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName; auto emplaced = _audioStreams.emplace( QUuid(), @@ -340,3 +336,25 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() { return result; } + +void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) { + cleanupCodec(); // cleanup any previously allocated coders first + _codec = codec; + _selectedCodecName = codecName; + _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); +} + +void AudioMixerClientData::cleanupCodec() { + // release any old codec encoder/decoder first... + if (_codec) { + if (_decoder) { + _codec->releaseDecoder(_decoder); + _decoder = nullptr; + } + if (_encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } + } +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index ee6f22ff3b..da2bf8997c 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -68,12 +68,15 @@ public: AudioLimiter audioLimiter; - // FIXME -- maybe make these private - CodecPluginPointer _codec; - QString _selectedCodecName; - Encoder* _encoder { nullptr }; // for outbound mixed stream - Decoder* _decoder { nullptr }; // for mic stream - + void setupCodec(CodecPluginPointer codec, const QString& codecName); + void cleanupCodec(); + void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + if (_encoder) { + _encoder->encode(decodedBuffer, encodedBuffer); + } else { + encodedBuffer = decodedBuffer; + } + } signals: void injectorStreamFinished(const QUuid& streamIdentifier); @@ -91,6 +94,11 @@ private: AudioStreamStats _downstreamAudioStreamStats; int _frameToSendStats { 0 }; + + CodecPluginPointer _codec; + QString _selectedCodecName; + Encoder* _encoder{ nullptr }; // for outbound mixed stream + Decoder* _decoder{ nullptr }; // for mic stream }; #endif // hifi_AudioMixerClientData_h diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index db5376a380..28fd95f599 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -140,6 +140,10 @@ AudioClient::AudioClient() : AudioClient::~AudioClient() { stop(); + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } } void AudioClient::reset() { @@ -529,14 +533,19 @@ void AudioClient::negotiateAudioFormat() { void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { qDebug() << __FUNCTION__; - _receivedAudioStream._selectedCodecName = _selectedCodecName = message->readString(); + _selectedCodecName = message->readString(); qDebug() << "Selected Codec:" << _selectedCodecName; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { - _receivedAudioStream._codec = _codec = plugin; - _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + // release any old codec encoder/decoder first... + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } + _codec = plugin; + _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index f47c6cce46..144005bc11 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -504,3 +504,21 @@ float calculateRepeatedFrameFadeFactor(int indexOfRepeat) { return 0.0f; } +void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels) { + cleanupCodec(); // cleanup any previously allocated coders first + _codec = codec; + _selectedCodecName = codecName; + if (_codec) { + _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels); + } +} + +void InboundAudioStream::cleanupCodec() { + // release any old codec encoder/decoder first... + if (_codec) { + if (_decoder) { + _codec->releaseDecoder(_decoder); + _decoder = nullptr; + } + } +} diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 57bff80fff..5da63f96c2 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -105,6 +105,7 @@ public: public: InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings); + ~InboundAudioStream() { cleanupCodec(); } void reset(); virtual void resetStats(); @@ -176,10 +177,8 @@ public: void setReverb(float reverbTime, float wetLevel); void clearReverb() { _hasReverb = false; } - // FIXME -- maybe make these private - CodecPluginPointer _codec; - QString _selectedCodecName; - Decoder* _decoder { nullptr }; + void setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels); + void cleanupCodec(); public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers @@ -274,6 +273,10 @@ protected: bool _hasReverb; float _reverbTime; float _wetLevel; + + CodecPluginPointer _codec; + QString _selectedCodecName; + Decoder* _decoder{ nullptr }; }; float calculateRepeatedFrameFadeFactor(int indexOfRepeat); From 7d608ba5925001be0c2a16f9b27425a20527adbb Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Sun, 10 Jul 2016 21:01:20 -0700 Subject: [PATCH 11/22] groundwork for injector support --- assignment-client/src/audio/AudioMixer.cpp | 5 ---- .../src/audio/AudioMixerClientData.cpp | 26 ++++++++++++++++++- libraries/audio/src/AudioInjector.cpp | 9 ++++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2f1a5de309..720f9d52de 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -535,11 +535,6 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess qDebug() << "selectedCodecName:" << selectedCodecName; - auto avatarAudioStream = clientData->getAvatarAudioStream(); - if (avatarAudioStream) { - avatarAudioStream->setupCodec(selectedCodec, selectedCodecName, AudioConstants::MONO); - } - auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); // write them to our packet diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index c733683e5a..182f443225 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -142,9 +142,16 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { if (streamIt == _audioStreams.end()) { // we don't have this injected stream yet, so add it + auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()); + +#if INJECTORS_SUPPORT_CODECS + injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + qDebug() << "creating new injectorStream... codec:" << _selectedCodecName; +#endif + auto emplaced = _audioStreams.emplace( streamIdentifier, - std::unique_ptr { new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()) } + std::unique_ptr { injectorStream } ); streamIt = emplaced.first; @@ -343,6 +350,23 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c _selectedCodecName = codecName; _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + + auto avatarAudioStream = getAvatarAudioStream(); + if (avatarAudioStream) { + avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO); + } + +#if INJECTORS_SUPPORT_CODECS + // fixup codecs for any active injectors... + auto it = _audioStreams.begin(); + while (it != _audioStreams.end()) { + SharedStreamPointer stream = it->second; + if (stream->getType() == PositionalAudioStream::Injector) { + stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); + } + ++it; + } +#endif } void AudioMixerClientData::cleanupCodec() { diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 878a4c627c..fee33dcb92 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -289,16 +289,23 @@ int64_t AudioInjector::injectNextFrame() { _currentPacket->seek(audioDataOffset); + // This code is copying bytes from the _audioData directly into the packet, handling looping appropriately. + // Might be a reasonable place to do the encode step here. + QByteArray decodedAudio; while (totalBytesLeftToCopy > 0) { int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset); - _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); + decodedAudio.append(_audioData.data() + _currentSendOffset, bytesToCopy); _currentSendOffset += bytesToCopy; totalBytesLeftToCopy -= bytesToCopy; if (_options.loop && _currentSendOffset >= _audioData.size()) { _currentSendOffset = 0; } } + // FIXME -- good place to call codec encode here. We need to figure out how to tell the AudioInjector which + // codec to use... possible through AbstractAudioInterface. + QByteArray encodedAudio = decodedAudio; + _currentPacket->write(encodedAudio.data(), encodedAudio.size()); // set the correct size used for this packet _currentPacket->setPayloadSize(_currentPacket->pos()); From f5b693cebb0574486a61a26157d5e77caf96da3a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 12:22:27 -0700 Subject: [PATCH 12/22] added support for external hificodec and hifi codec plugin --- cmake/externals/hifiAudioCodec/CMakeLists.txt | 50 +++++++++++++ plugins/hifiCodec/CMakeLists.txt | 21 ++++++ plugins/hifiCodec/src/HiFiCodec.cpp | 72 +++++++++++++++++++ plugins/hifiCodec/src/HiFiCodec.h | 42 +++++++++++ plugins/hifiCodec/src/HiFiCodecProvider.cpp | 46 ++++++++++++ plugins/hifiCodec/src/plugin.json | 1 + 6 files changed, 232 insertions(+) create mode 100644 cmake/externals/hifiAudioCodec/CMakeLists.txt create mode 100644 plugins/hifiCodec/CMakeLists.txt create mode 100644 plugins/hifiCodec/src/HiFiCodec.cpp create mode 100644 plugins/hifiCodec/src/HiFiCodec.h create mode 100644 plugins/hifiCodec/src/HiFiCodecProvider.cpp create mode 100644 plugins/hifiCodec/src/plugin.json diff --git a/cmake/externals/hifiAudioCodec/CMakeLists.txt b/cmake/externals/hifiAudioCodec/CMakeLists.txt new file mode 100644 index 0000000000..5c563774b7 --- /dev/null +++ b/cmake/externals/hifiAudioCodec/CMakeLists.txt @@ -0,0 +1,50 @@ +include(ExternalProject) +include(SelectLibraryConfigurations) + +set(EXTERNAL_NAME HiFiAudioCodec) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) + +ExternalProject_Add( + ${EXTERNAL_NAME} + URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK.zip + URL_MD5 4add25b7cc5dfdb3cbfc1156b95cfff7 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD 1 +) + +# Hide this external target (for ide users) +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) + +set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNAL) + +if (WIN32) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL) + + # FIXME need to account for different architectures + #if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win64/audio.lib CACHE TYPE INTERNAL) + # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win64) + #else() + # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win32/audio.lib CACHE TYPE INTERNAL) + # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win32) + #endif() + +elseif(APPLE) + + # FIXME need to account for different architectures + #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/osx32/audio.dylib CACHE TYPE INTERNAL) + #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/osx32) + +elseif(NOT ANDROID) + + # FIXME need to account for different architectures + #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL) + #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/linux64) + +endif() + diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt new file mode 100644 index 0000000000..50ba6de54e --- /dev/null +++ b/plugins/hifiCodec/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# Created by Brad Hefta-Gaub on 7/10/2016 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + + + +if (WIN32) + set(TARGET_NAME hifiCodec) + setup_hifi_client_server_plugin() + + link_hifi_libraries(shared plugins) + + add_dependency_external_projects(HiFiAudioCodec) + target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${HIFIAUDIOCODEC_LIBRARIES}) +endif() + diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp new file mode 100644 index 0000000000..03a0b04bb8 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -0,0 +1,72 @@ +// +// HiFiCodec.cpp +// plugins/hifiCodec/src +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include // should be from the external... + +#include + +#include "HiFiCodec.h" + +const QString HiFiCodec::NAME = "hifiAC"; + +void HiFiCodec::init() { +} + +void HiFiCodec::deinit() { +} + +bool HiFiCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void HiFiCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool HiFiCodec::isSupported() const { + return true; +} + +class HiFiEncoder : public Encoder { +public: + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } +}; + +class HiFiDecoder : public Decoder { +public: + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void trackLostFrames(int numFrames) override { } +}; + +Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) { + return new HiFiEncoder(); +} + +Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) { + return new HiFiDecoder(); +} + +void HiFiCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void HiFiCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} diff --git a/plugins/hifiCodec/src/HiFiCodec.h b/plugins/hifiCodec/src/HiFiCodec.h new file mode 100644 index 0000000000..eeba8d56d8 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodec.h @@ -0,0 +1,42 @@ +// +// HiFiCodec.h +// plugins/hifiCodec/src +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_HiFiCodec_h +#define hifi_HiFiCodec_h + +#include + +class HiFiCodec : public CodecPlugin { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + +private: + static const QString NAME; +}; + +#endif // hifi_HiFiCodec_h diff --git a/plugins/hifiCodec/src/HiFiCodecProvider.cpp b/plugins/hifiCodec/src/HiFiCodecProvider.cpp new file mode 100644 index 0000000000..b9698cc821 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodecProvider.cpp @@ -0,0 +1,46 @@ +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "HiFiCodec.h" + +class HiFiCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + HiFiCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~HiFiCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + + CodecPluginPointer hiFiCodec(new HiFiCodec()); + if (hiFiCodec->isSupported()) { + _codecPlugins.push_back(hiFiCodec); + } + + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "HiFiCodecProvider.moc" diff --git a/plugins/hifiCodec/src/plugin.json b/plugins/hifiCodec/src/plugin.json new file mode 100644 index 0000000000..df26a67ea8 --- /dev/null +++ b/plugins/hifiCodec/src/plugin.json @@ -0,0 +1 @@ +{"name":"HiFi 4:1 Audio Codec"} From 540ab9f07ca60357f57cd97bfe8f6562c46775fc Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 13:00:58 -0700 Subject: [PATCH 13/22] wire up audio codec --- plugins/hifiCodec/CMakeLists.txt | 2 +- plugins/hifiCodec/src/HiFiCodec.cpp | 42 ++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt index 50ba6de54e..da964bfdfd 100644 --- a/plugins/hifiCodec/CMakeLists.txt +++ b/plugins/hifiCodec/CMakeLists.txt @@ -12,7 +12,7 @@ if (WIN32) set(TARGET_NAME hifiCodec) setup_hifi_client_server_plugin() - link_hifi_libraries(shared plugins) + link_hifi_libraries(audio shared plugins) add_dependency_external_projects(HiFiAudioCodec) target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS}) diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 03a0b04bb8..3bb17a0275 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -11,7 +11,8 @@ #include -#include // should be from the external... +#include +#include #include @@ -39,28 +40,49 @@ bool HiFiCodec::isSupported() const { return true; } -class HiFiEncoder : public Encoder { +class HiFiEncoder : public Encoder, public AudioEncoder { public: - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; + HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) { + qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels; + _encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4; // codec reduces by 1/4th } + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer.resize(_encodedSize); + AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } +private: + int _encodedSize; }; -class HiFiDecoder : public Decoder { +class HiFiDecoder : public Decoder, public AudioDecoder { public: - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; + HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) { + qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels; + _decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels; } - virtual void trackLostFrames(int numFrames) override { } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer.resize(_decodedSize); + AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); + } + + virtual void trackLostFrames(int numFrames) override { + QByteArray encodedBuffer; + QByteArray decodedBuffer; + decodedBuffer.resize(_decodedSize); + AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + } +private: + int _decodedSize; }; Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) { - return new HiFiEncoder(); + return new HiFiEncoder(sampleRate, numChannels); } Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) { - return new HiFiDecoder(); + return new HiFiDecoder(sampleRate, numChannels); } void HiFiCodec::releaseEncoder(Encoder* encoder) { From 743576134f687322fc44911ebe861d0a2c73eb6d Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 13:38:08 -0700 Subject: [PATCH 14/22] fix warnings --- libraries/networking/src/udt/BasePacket.cpp | 2 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 0456e095ae..18552ca966 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -154,7 +154,7 @@ qint64 BasePacket::writeString(const QString& string) { QByteArray data = string.toUtf8(); uint32_t length = data.length(); writePrimitive(length); - auto result = writeData(data.constData(), data.length()); + writeData(data.constData(), data.length()); seek(pos() + length); return length + sizeof(uint32_t); } diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 1cad1ea4e6..4a2380717d 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -62,11 +62,11 @@ Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { } void PCMCodec::releaseEncoder(Encoder* encoder) { - delete encoder; + delete static_cast(encoder); } void PCMCodec::releaseDecoder(Decoder* decoder) { - delete decoder; + delete static_cast(decoder); } @@ -117,10 +117,10 @@ Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { } void zLibCodec::releaseEncoder(Encoder* encoder) { - delete encoder; + delete static_cast(encoder); } void zLibCodec::releaseDecoder(Decoder* decoder) { - delete decoder; + delete static_cast(decoder); } From 125aa6b337c303a24f7183709e27cc0ea07d4272 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 14:21:32 -0700 Subject: [PATCH 15/22] make zlib and pcm codecs not actually allocate encoder/decoders --- libraries/plugins/src/plugins/CodecPlugin.h | 2 + plugins/pcmCodec/src/PCMCodecManager.cpp | 48 ++++----------------- plugins/pcmCodec/src/PCMCodecManager.h | 45 +++++++++---------- 3 files changed, 30 insertions(+), 65 deletions(-) diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 280853e37e..404f05e860 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -14,11 +14,13 @@ class Encoder { public: + virtual ~Encoder() { } virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; }; class Decoder { public: + virtual ~Decoder() { } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; // numFrames - number of samples (mono) or sample-pairs (stereo) diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 4a2380717d..315d0622ab 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -37,55 +37,24 @@ bool PCMCodec::isSupported() const { return true; } -class PCMEncoder : public Encoder { -public: - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; - } -}; -class PCMDecoder : public Decoder { -public: - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; - } - - virtual void trackLostFrames(int numFrames) override { } -}; Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { - return new PCMEncoder(); + return this; } Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { - return new PCMDecoder(); + return this; } void PCMCodec::releaseEncoder(Encoder* encoder) { - delete static_cast(encoder); + // do nothing } void PCMCodec::releaseDecoder(Decoder* decoder) { - delete static_cast(decoder); + // do nothing } - -class zLibEncoder : public Encoder { -public: - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = qCompress(decodedBuffer); - } -}; - -class zLibDecoder : public Decoder { -public: - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = qUncompress(encodedBuffer); - } - - virtual void trackLostFrames(int numFrames) override { } -}; - const QString zLibCodec::NAME = "zlib"; void zLibCodec::init() { @@ -103,24 +72,23 @@ void zLibCodec::deactivate() { CodecPlugin::deactivate(); } - bool zLibCodec::isSupported() const { return true; } Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { - return new zLibEncoder(); + return this; } Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { - return new zLibDecoder(); + return this; } void zLibCodec::releaseEncoder(Encoder* encoder) { - delete static_cast(encoder); + // do nothing... it wasn't allocated } void zLibCodec::releaseDecoder(Decoder* decoder) { - delete static_cast(decoder); + // do nothing... it wasn't allocated } diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 6eb7f44c15..55d7c866f1 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -14,30 +14,7 @@ #include -/* -class Encoder { -public: -virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; -}; - -class Decoder { -public: -virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - -// numFrames - number of samples (mono) or sample-pairs (stereo) -virtual void trackLostFrames(int numFrames) = 0; -}; - -class CodecPlugin : public Plugin { -public: -virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; -virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; -virtual void releaseEncoder(Encoder* encoder) = 0; -virtual void releaseDecoder(Decoder* decoder) = 0; -}; -*/ - -class PCMCodec : public CodecPlugin { +class PCMCodec : public CodecPlugin, public Encoder, public Decoder { Q_OBJECT public: @@ -58,11 +35,20 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void trackLostFrames(int numFrames) override { } + private: static const QString NAME; }; -class zLibCodec : public CodecPlugin { +class zLibCodec : public CodecPlugin, public Encoder, public Decoder { Q_OBJECT public: @@ -83,6 +69,15 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = qCompress(decodedBuffer); + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = qUncompress(encodedBuffer); + } + + virtual void trackLostFrames(int numFrames) override { } private: static const QString NAME; From ceffa219b3bec454f175eed596aef5b897b5ada6 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 15:18:02 -0700 Subject: [PATCH 16/22] support for lost packets --- libraries/audio/src/InboundAudioStream.cpp | 6 +++++- plugins/hifiCodec/src/HiFiCodec.cpp | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 144005bc11..c9b9363b1b 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -172,7 +172,7 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& return sizeof(quint16); } else { // mixed audio packets do not have any info between the seq num and the audio data. - numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t); + numAudioSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; return 0; } } @@ -189,6 +189,10 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { + if (_decoder) { + _decoder->trackLostFrames(silentSamples); + } + // calculate how many silent frames we should drop. int samplesPerFrame = _ringBuffer.getNumFrameSamples(); int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 3bb17a0275..4e9336ff90 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -43,7 +43,6 @@ bool HiFiCodec::isSupported() const { class HiFiEncoder : public Encoder, public AudioEncoder { public: HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) { - qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels; _encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4; // codec reduces by 1/4th } @@ -58,7 +57,6 @@ private: class HiFiDecoder : public Decoder, public AudioDecoder { public: HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) { - qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels; _decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels; } @@ -71,6 +69,7 @@ public: QByteArray encodedBuffer; QByteArray decodedBuffer; decodedBuffer.resize(_decodedSize); + // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); } private: From 3ddac737182b2a54bbcadc2b3f85dee5a88d5a3f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 18:24:22 -0700 Subject: [PATCH 17/22] add mac lib to the external project --- cmake/externals/hifiAudioCodec/CMakeLists.txt | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/cmake/externals/hifiAudioCodec/CMakeLists.txt b/cmake/externals/hifiAudioCodec/CMakeLists.txt index 5c563774b7..3a8c714d79 100644 --- a/cmake/externals/hifiAudioCodec/CMakeLists.txt +++ b/cmake/externals/hifiAudioCodec/CMakeLists.txt @@ -7,8 +7,8 @@ string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) ExternalProject_Add( ${EXTERNAL_NAME} - URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK.zip - URL_MD5 4add25b7cc5dfdb3cbfc1156b95cfff7 + URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK-1.zip + URL_MD5 23ec3fe51eaa155ea159a4971856fc13 CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" @@ -24,27 +24,10 @@ set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNA if (WIN32) set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL) - - # FIXME need to account for different architectures - #if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") - # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win64/audio.lib CACHE TYPE INTERNAL) - # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win64) - #else() - # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win32/audio.lib CACHE TYPE INTERNAL) - # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win32) - #endif() - elseif(APPLE) - - # FIXME need to account for different architectures - #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/osx32/audio.dylib CACHE TYPE INTERNAL) - #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/osx32) - + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/libaudio.a CACHE TYPE INTERNAL) elseif(NOT ANDROID) - # FIXME need to account for different architectures #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL) - #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/linux64) - endif() From 6f0967f3fc444ebbe1e7a6b013a0c9182f6209be Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 11 Jul 2016 18:28:04 -0700 Subject: [PATCH 18/22] add mac support to the codec plugin --- plugins/hifiCodec/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt index da964bfdfd..3939529c3e 100644 --- a/plugins/hifiCodec/CMakeLists.txt +++ b/plugins/hifiCodec/CMakeLists.txt @@ -6,9 +6,7 @@ # See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html # - - -if (WIN32) +if (WIN32 OR APPLE) set(TARGET_NAME hifiCodec) setup_hifi_client_server_plugin() From 71b6210c4ea812b2cc0d802ba1096bbd417383a7 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 12 Jul 2016 10:18:37 -0700 Subject: [PATCH 19/22] removed some log spam --- assignment-client/src/audio/AudioMixer.cpp | 2 -- libraries/audio-client/src/AudioClient.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 720f9d52de..51c4e25410 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -464,8 +464,6 @@ void saveInputPluginSettings(const InputPluginList& plugins) { void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { - qDebug() << __FUNCTION__; - QStringList availableCodecs; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 28fd95f599..0cd01bb579 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -531,8 +531,6 @@ void AudioClient::negotiateAudioFormat() { } void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { - qDebug() << __FUNCTION__; - _selectedCodecName = message->readString(); qDebug() << "Selected Codec:" << _selectedCodecName; From 2b719e12f72cc8892778ff174babec5531634bd5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 12 Jul 2016 10:54:45 -0700 Subject: [PATCH 20/22] attempt to make mac server plugins work --- cmake/macros/SetupHifiClientServerPlugin.cmake | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cmake/macros/SetupHifiClientServerPlugin.cmake b/cmake/macros/SetupHifiClientServerPlugin.cmake index 5ace348e87..dfe3113fbc 100644 --- a/cmake/macros/SetupHifiClientServerPlugin.cmake +++ b/cmake/macros/SetupHifiClientServerPlugin.cmake @@ -12,17 +12,22 @@ macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN) set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins") if (APPLE) - set(PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns") + set(CLIENT_PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns") + set(SERVER_PLUGIN_PATH "Components.app/Contents/PlugIns") else() - set(PLUGIN_PATH "plugins") + set(CLIENT_PLUGIN_PATH "plugins") + set(SERVER_PLUGIN_PATH "plugins") endif() if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles") - set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${PLUGIN_PATH}/") - set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${PLUGIN_PATH}/") + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${SERVER_PLUGIN_PATH}/") + elseif (APPLE) + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${SERVER_PLUGIN_PATH}/") else() - set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${PLUGIN_PATH}/") - set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${PLUGIN_PATH}/") + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${SERVER_PLUGIN_PATH}/") endif() # create the destination for the client plugin binaries From 6d7c8d5759562e74aa71200185df616ffe500e2c Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 12 Jul 2016 16:11:43 -0700 Subject: [PATCH 21/22] fix crash in audio-mixer if no plugins installed --- assignment-client/src/audio/AudioMixerClientData.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 182f443225..5c2ce8bf57 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -348,8 +348,10 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c cleanupCodec(); // cleanup any previously allocated coders first _codec = codec; _selectedCodecName = codecName; - _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + if (codec) { + _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + } auto avatarAudioStream = getAvatarAudioStream(); if (avatarAudioStream) { From fcd59ea94f122c8644bf5a95cc7246cbf5ff382d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 12 Jul 2016 17:44:23 -0700 Subject: [PATCH 22/22] Update target number of processing threads to be based on system --- interface/src/Application.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5d5bf45dcf..d97dee6fd1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -169,7 +169,12 @@ static QTimer identityPacketTimer; static QTimer pingTimer; static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16; -static const int PROCESSING_THREAD_POOL_SIZE = 6; + +// For processing on QThreadPool, target 2 less than the ideal number of threads, leaving +// 2 logical cores available for time sensitive tasks. +static const int MIN_PROCESSING_THREAD_POOL_SIZE = 2; +static const int PROCESSING_THREAD_POOL_SIZE = std::max(MIN_PROCESSING_THREAD_POOL_SIZE, + QThread::idealThreadCount() - 2); static const QString SNAPSHOT_EXTENSION = ".jpg"; static const QString SVO_EXTENSION = ".svo"; @@ -514,12 +519,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : PluginContainer* pluginContainer = dynamic_cast(this); // set the container for any plugins that care PluginManager::getInstance()->setContainer(pluginContainer); - // FIXME this may be excessively conservative. On the other hand - // maybe I'm used to having an 8-core machine - // Perhaps find the ideal thread count and subtract 2 or 3 - // (main thread, present thread, random OS load) - // More threads == faster concurrent loads, but also more concurrent - // load on the GPU until we can serialize GPU transfers (off the main thread) QThreadPool::globalInstance()->setMaxThreadCount(PROCESSING_THREAD_POOL_SIZE); thread()->setPriority(QThread::HighPriority); thread()->setObjectName("Main Thread");