more work on codecs

assignment-client/CMakeLists.txt

@@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets)
 link_hifi_libraries(
   audio avatars octree gpu model fbx entities
   networking animation recording shared script-engine embedded-webserver
-  controllers physics
+  controllers physics plugins
 )
 
 if (WIN32)

assignment-client/src/audio/AudioMixer.cpp

@@ -47,6 +47,8 @@
 #include <NodeList.h>
 #include <Node.h>
 #include <OctreeConstants.h>
+#include <plugins/PluginManager.h>
+#include <plugins/CodecPlugin.h>
 #include <udt/PacketHeaders.h>
 #include <SharedUtil.h>
 #include <StDev.h>
@@ -447,6 +449,20 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> mes
     }
 }
 
+DisplayPluginList getDisplayPlugins() {
+    DisplayPluginList result;
+    return result;
+}
+
+InputPluginList getInputPlugins() {
+    InputPluginList result;
+    return result;
+}
+
+void saveInputPluginSettings(const InputPluginList& plugins) {
+}
+
+
 void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
     qDebug() << __FUNCTION__;
 
@@ -461,6 +477,15 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
     }
     qDebug() << "all requested codecs:" << codecList;
 
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    if (codecPlugins.size() > 0) {
+        for (auto& plugin : codecPlugins) {
+            qDebug() << "Codec available:" << plugin->getName();
+        }
+    } else {
+        qDebug() << "No Codecs available...";
+    }
+
     auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
 
     // write them to our packet

libraries/audio-client/src/AudioClient.cpp

@@ -477,32 +477,6 @@ void AudioClient::stop() {
     }
 }
 
-void AudioClient::negotiateAudioFormat() {
-    qDebug() << __FUNCTION__;
-
-    auto nodeList = DependencyManager::get<NodeList>();
-
-    auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
-
-    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
-
-    quint8 numberOfCodecs = (quint8)codecPlugins.size();
-    negotiateFormatPacket->writePrimitive(numberOfCodecs);
-    for (auto& plugin : codecPlugins) {
-        qDebug() << "Codec available:" << plugin->getName();
-        negotiateFormatPacket->writeString(plugin->getName());
-    }
-
-    // grab our audio mixer from the NodeList, if it exists
-    SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
-
-    if (audioMixer) {
-        // send off this mute packet
-        nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
-    }
-}
-
-
 void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer<ReceivedMessage> message) {
 
     char bitset;
@@ -557,13 +531,47 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
     emit muteEnvironmentRequested(position, radius);
 }
 
+void AudioClient::negotiateAudioFormat() {
+    qDebug() << __FUNCTION__;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
+
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+
+    quint8 numberOfCodecs = (quint8)codecPlugins.size();
+    negotiateFormatPacket->writePrimitive(numberOfCodecs);
+    for (auto& plugin : codecPlugins) {
+        qDebug() << "Codec available:" << plugin->getName();
+        negotiateFormatPacket->writeString(plugin->getName());
+    }
+
+    // grab our audio mixer from the NodeList, if it exists
+    SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+
+    if (audioMixer) {
+        // send off this mute packet
+        nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
+    }
+}
+
 void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
     qDebug() << __FUNCTION__;
 
     // write them to our packet
-    QString selectedCodec = message->readString();
+    _selectedCodecName = message->readString();
+
+    qDebug() << "Selected Codec:" << _selectedCodecName;
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    for (auto& plugin : codecPlugins) {
+        if (_selectedCodecName == plugin->getName()) {
+            _codec = plugin;
+            qDebug() << "Selected Codec Plugin:" << _codec.get();
+            break;
+        }
+    }
 
-    qDebug() << "selectedCodec:" << selectedCodec;
 }
    
 
@@ -839,7 +847,17 @@ void AudioClient::handleAudioInput() {
         audioTransform.setTranslation(_positionGetter());
         audioTransform.setRotation(_orientationGetter());
         // FIXME find a way to properly handle both playback audio and user audio concurrently
-        emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
+
+        // TODO - codec encode goes here
+        QByteArray decocedBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
+        QByteArray encodedBuffer;
+        if (_codec) {
+            _codec->encode(decocedBuffer, encodedBuffer);
+        } else {
+            encodedBuffer = decocedBuffer;
+        }
+
+        emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
         _stats.sentPacket();
     }
 }
@@ -848,14 +866,28 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
     Transform audioTransform;
     audioTransform.setTranslation(_positionGetter());
     audioTransform.setRotation(_orientationGetter());
+
+    // TODO - codec decode goes here
+    QByteArray encodedBuffer;
+    if (_codec) {
+        _codec->encode(audio, encodedBuffer);
+    } else {
+        encodedBuffer = audio;
+    }
+
     // FIXME check a flag to see if we should echo audio?
-    emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
+    emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
 }
 
 void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) {
 
     // TODO - codec decode goes here
-    QByteArray decodedBuffer = networkBuffer;
+    QByteArray decodedBuffer;
+    if (_codec) {
+        _codec->decode(networkBuffer, decodedBuffer);
+    } else {
+        decodedBuffer = networkBuffer;
+    }
 
     const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
     const int numDeviceOutputSamples = _outputFrameSize;

libraries/audio-client/src/AudioClient.h

@@ -38,6 +38,8 @@
 #include <Sound.h>
 #include <StDev.h>
 
+#include <plugins/CodecPlugin.h>
+
 #include "AudioIOStats.h"
 #include "AudioNoiseGate.h"
 #include "AudioSRC.h"
@@ -296,7 +298,8 @@ private:
 
     bool _hasReceivedFirstPacket = false;
 
-    //CodecPluginPointer _codec { nullptr };
+    CodecPluginPointer _codec;
+    QString _selectedCodecName;
 };
 
 

plugins/pcmCodec/src/PCMCodecManager.cpp

@@ -39,11 +39,13 @@ bool PCMCodecManager::isSupported() const {
 
 
 void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+    qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size();
     // this codec doesn't actually do anything....
     decodedBuffer = encodedBuffer;
 }
 
 void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+    qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size();
     // this codec doesn't actually do anything....
     encodedBuffer = decodedBuffer;
 }