From 7a4b11ee9790e902d55e12292fe9b420c72478f3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 27 Jun 2016 13:06:19 -0700 Subject: [PATCH] more work on codecs --- assignment-client/CMakeLists.txt | 2 +- assignment-client/src/audio/AudioMixer.cpp | 25 ++++++ libraries/audio-client/src/AudioClient.cpp | 94 +++++++++++++++------- libraries/audio-client/src/AudioClient.h | 5 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 2 + 5 files changed, 95 insertions(+), 33 deletions(-) diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index 1b5840c3c8..d0fd2c1176 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets) link_hifi_libraries( audio avatars octree gpu model fbx entities networking animation recording shared script-engine embedded-webserver - controllers physics + controllers physics plugins ) if (WIN32) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 95578e3998..27b0a092d0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include #include @@ -447,6 +449,20 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } +DisplayPluginList getDisplayPlugins() { + DisplayPluginList result; + return result; +} + +InputPluginList getInputPlugins() { + InputPluginList result; + return result; +} + +void saveInputPluginSettings(const InputPluginList& plugins) { +} + + void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { qDebug() << __FUNCTION__; @@ -461,6 +477,15 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess } qDebug() << "all requested codecs:" << codecList; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + } + } else { + qDebug() << "No Codecs available..."; + } + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); // write them to our packet diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 18db23dbf7..7ef71087ce 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -477,32 +477,6 @@ void AudioClient::stop() { } } -void AudioClient::negotiateAudioFormat() { - qDebug() << __FUNCTION__; - - auto nodeList = DependencyManager::get(); - - auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); - - auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); - - quint8 numberOfCodecs = (quint8)codecPlugins.size(); - negotiateFormatPacket->writePrimitive(numberOfCodecs); - for (auto& plugin : codecPlugins) { - qDebug() << "Codec available:" << plugin->getName(); - negotiateFormatPacket->writeString(plugin->getName()); - } - - // grab our audio mixer from the NodeList, if it exists - SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); - - if (audioMixer) { - // send off this mute packet - nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); - } -} - - void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer message) { char bitset; @@ -557,13 +531,47 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me emit muteEnvironmentRequested(position, radius); } +void AudioClient::negotiateAudioFormat() { + qDebug() << __FUNCTION__; + + auto nodeList = DependencyManager::get(); + + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + + quint8 numberOfCodecs = (quint8)codecPlugins.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + negotiateFormatPacket->writeString(plugin->getName()); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { qDebug() << __FUNCTION__; // write them to our packet - QString selectedCodec = message->readString(); + _selectedCodecName = message->readString(); + + qDebug() << "Selected Codec:" << _selectedCodecName; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + if (_selectedCodecName == plugin->getName()) { + _codec = plugin; + qDebug() << "Selected Codec Plugin:" << _codec.get(); + break; + } + } - qDebug() << "selectedCodec:" << selectedCodec; } @@ -839,7 +847,17 @@ void AudioClient::handleAudioInput() { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); // FIXME find a way to properly handle both playback audio and user audio concurrently - emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); + + // TODO - codec encode goes here + QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + QByteArray encodedBuffer; + if (_codec) { + _codec->encode(decocedBuffer, encodedBuffer); + } else { + encodedBuffer = decocedBuffer; + } + + emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); _stats.sentPacket(); } } @@ -848,14 +866,28 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { Transform audioTransform; audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); + + // TODO - codec decode goes here + QByteArray encodedBuffer; + if (_codec) { + _codec->encode(audio, encodedBuffer); + } else { + encodedBuffer = audio; + } + // FIXME check a flag to see if we should echo audio? - emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); + emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) { // TODO - codec decode goes here - QByteArray decodedBuffer = networkBuffer; + QByteArray decodedBuffer; + if (_codec) { + _codec->decode(networkBuffer, decodedBuffer); + } else { + decodedBuffer = networkBuffer; + } const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = _outputFrameSize; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e05612f859..45b4a631b2 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -38,6 +38,8 @@ #include #include +#include + #include "AudioIOStats.h" #include "AudioNoiseGate.h" #include "AudioSRC.h" @@ -296,7 +298,8 @@ private: bool _hasReceivedFirstPacket = false; - //CodecPluginPointer _codec { nullptr }; + CodecPluginPointer _codec; + QString _selectedCodecName; }; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 2401e99576..eb2f4761b4 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -39,11 +39,13 @@ bool PCMCodecManager::isSupported() const { void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { + qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size(); // this codec doesn't actually do anything.... decodedBuffer = encodedBuffer; } void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size(); // this codec doesn't actually do anything.... encodedBuffer = decodedBuffer; }