From d7bffc3eabec8bb0f5f26a3eadf44aa7dfdc3a31 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 23 Jun 2016 17:14:41 -0700 Subject: [PATCH] first cut at negotiating codecs --- assignment-client/src/audio/AudioMixer.cpp | 26 +++++++++++ assignment-client/src/audio/AudioMixer.h | 1 + interface/src/Application.cpp | 4 +- libraries/audio-client/CMakeLists.txt | 2 +- libraries/audio-client/src/AudioClient.cpp | 46 +++++++++++++++++++ libraries/audio-client/src/AudioClient.h | 5 ++ libraries/networking/src/udt/PacketHeaders.h | 4 +- libraries/plugins/src/plugins/CodecPlugin.h | 1 + .../plugins/src/plugins/PluginManager.cpp | 5 +- plugins/pcmCodec/src/PCMCodecManager.cpp | 4 ++ plugins/pcmCodec/src/PCMCodecManager.h | 1 + 11 files changed, 94 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 03bb32cd53..95578e3998 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -90,6 +90,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::AudioStreamStats }, this, "handleNodeAudioPacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -446,6 +447,31 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } +void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { + qDebug() << __FUNCTION__; + + // read the codecs requested by the client + quint8 numberOfCodecs = 0; + message->readPrimitive(&numberOfCodecs); + QStringList codecList; + for (quint16 i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message->readString(); + qDebug() << "requestedCodec:" << requestedCodec; + codecList.append(requestedCodec); + } + qDebug() << "all requested codecs:" << codecList; + + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); + + // write them to our packet + QString selectedCodec = codecList.front(); + qDebug() << "selectedCodec:" << selectedCodec; + replyPacket->writeString(selectedCodec); + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *sendingNode); +} + void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { // enumerate the connected listeners to remove HRTF objects for the disconnected node auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 24b4b39704..c90a918a5b 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -45,6 +45,7 @@ private slots: void broadcastMixes(); void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); void handleNodeKilled(SharedNodePointer killedNode); void removeHRTFsForFinishedInjector(const QUuid& streamID); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 855cf2f313..3c69f716ca 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1243,7 +1243,6 @@ QString Application::getUserAgent() { userAgent += " " + formatPluginName(cp->getName()); } - qDebug() << __FUNCTION__ << ":" << userAgent; return userAgent; } @@ -4440,6 +4439,9 @@ void Application::nodeActivated(SharedNodePointer node) { } } + if (node->getType() == NodeType::AudioMixer) { + DependencyManager::get()->negotiateAudioFormat(); + } } void Application::nodeKilled(SharedNodePointer node) { diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 2c0fc0a9cd..9c298ce664 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME audio-client) setup_hifi_library(Network Multimedia) -link_hifi_libraries(audio) +link_hifi_libraries(audio plugins) # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 2569f27aaa..18db23dbf7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include #include @@ -134,6 +136,7 @@ AudioClient::AudioClient() : packetReceiver.registerListener(PacketType::MixedAudio, this, "handleAudioDataPacket"); packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat"); } AudioClient::~AudioClient() { @@ -474,6 +477,32 @@ void AudioClient::stop() { } } +void AudioClient::negotiateAudioFormat() { + qDebug() << __FUNCTION__; + + auto nodeList = DependencyManager::get(); + + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + + quint8 numberOfCodecs = (quint8)codecPlugins.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + negotiateFormatPacket->writeString(plugin->getName()); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + + void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer message) { char bitset; @@ -528,6 +557,16 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me emit muteEnvironmentRequested(position, radius); } +void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { + qDebug() << __FUNCTION__; + + // write them to our packet + QString selectedCodec = message->readString(); + + qDebug() << "selectedCodec:" << selectedCodec; +} + + QString AudioClient::getDefaultDeviceName(QAudio::Mode mode) { QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode); return deviceInfo.deviceName(); @@ -1227,6 +1266,13 @@ void AudioClient::loadSettings() { windowSecondsForDesiredCalcOnTooManyStarves.get()); _receivedAudioStream.setWindowSecondsForDesiredReduction(windowSecondsForDesiredReduction.get()); _receivedAudioStream.setRepetitionWithFade(repetitionWithFade.get()); + + qDebug() << "---- Initializing Audio Client ----"; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + } + } void AudioClient::saveSettings() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 3a14c878f6..e05612f859 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -94,6 +94,8 @@ public: int _unfulfilledReads; }; + void negotiateAudioFormat(); + const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } @@ -139,6 +141,7 @@ public slots: void handleAudioDataPacket(QSharedPointer message); void handleNoisyMutePacket(QSharedPointer message); void handleMuteEnvironmentPacket(QSharedPointer message); + void handleSelectedAudioFormat(QSharedPointer message); void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); } void handleAudioInput(); @@ -292,6 +295,8 @@ private: void checkDevices(); bool _hasReceivedFirstPacket = false; + + //CodecPluginPointer _codec { nullptr }; }; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index ae54450fee..b5d1be077f 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -95,7 +95,9 @@ public: AssetMappingOperation, AssetMappingOperationReply, ICEServerHeartbeatACK, - LAST_PACKET_TYPE = ICEServerHeartbeatACK + NegotiateAudioFormat, + SelectedAudioFormat, + LAST_PACKET_TYPE = SelectedAudioFormat }; }; diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index d9e1b947fe..6944d91bed 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -15,5 +15,6 @@ class CodecPlugin : public Plugin { public: virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; }; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index fe34d2dbee..0b4afe1be0 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -222,10 +222,11 @@ const CodecPluginList& PluginManager::getCodecPlugins() { } } - auto& container = PluginContainer::getInstance(); for (auto plugin : codecPlugins) { - plugin->setContainer(&container); + plugin->setContainer(_container); plugin->init(); + + qDebug() << "init codec:" << plugin->getName(); } }); return codecPlugins; diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 264c139ac6..2401e99576 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -43,3 +43,7 @@ void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decode decodedBuffer = encodedBuffer; } +void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + // this codec doesn't actually do anything.... + encodedBuffer = decodedBuffer; +} diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index a8b3e49a2c..3b7ca36c02 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -32,6 +32,7 @@ public: void deactivate() override; virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; private: static const QString NAME;