From 3c6447326e1e8678d0a05dfc99cc9b858bab6c1a Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Thu, 7 Jul 2016 21:24:24 -0700 Subject: [PATCH] more codec plumbing --- assignment-client/src/audio/AudioMixer.cpp | 36 ++++++++++++++++--- .../src/audio/AudioMixerClientData.cpp | 7 +++- .../src/audio/AudioMixerClientData.h | 7 ++++ libraries/audio-client/src/AudioClient.cpp | 2 +- libraries/audio/CMakeLists.txt | 2 +- libraries/audio/src/InboundAudioStream.cpp | 11 +++++- libraries/audio/src/InboundAudioStream.h | 6 ++++ plugins/pcmCodec/src/PCMCodecManager.cpp | 8 +++-- 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 27b0a092d0..9d9194aef6 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -477,21 +477,39 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer mess } qDebug() << "all requested codecs:" << codecList; + CodecPluginPointer selectedCoded; + QString selectedCodecName; auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); if (codecPlugins.size() > 0) { for (auto& plugin : codecPlugins) { qDebug() << "Codec available:" << plugin->getName(); + + // choose first codec + if (!selectedCoded) { + selectedCoded = plugin; + selectedCodecName = plugin->getName(); + } } } else { qDebug() << "No Codecs available..."; } + auto clientData = dynamic_cast(sendingNode->getLinkedData()); + + clientData->_codec = selectedCoded; + clientData->_selectedCodecName = selectedCodecName; + qDebug() << "selectedCodecName:" << selectedCodecName; + + auto avatarAudioStream = clientData->getAvatarAudioStream(); + if (avatarAudioStream) { + avatarAudioStream->_codec = selectedCoded; + avatarAudioStream->_selectedCodecName = selectedCodecName; + } + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); // write them to our packet - QString selectedCodec = codecList.front(); - qDebug() << "selectedCodec:" << selectedCodec; - replyPacket->writeString(selectedCodec); + replyPacket->writeString(selectedCodecName); auto nodeList = DependencyManager::get(); nodeList->sendPacket(std::move(replyPacket), *sendingNode); @@ -720,9 +738,17 @@ void AudioMixer::broadcastMixes() { quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); + // TODO - codec encode goes here + QByteArray decocedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + QByteArray encodedBuffer; + if (nodeData->_codec) { + nodeData->_codec->encode(decocedBuffer, encodedBuffer); + } else { + encodedBuffer = decocedBuffer; + } + // pack mixed audio samples - mixPacket->write(reinterpret_cast(_clampedSamples), - AudioConstants::NETWORK_FRAME_BYTES_STEREO); + mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16); mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 20003ba10d..080a833c0e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -101,9 +101,14 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isStereo = channelFlag == 1; + auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); + avatarAudioStream->_codec = _codec; + avatarAudioStream->_selectedCodecName = _selectedCodecName; + qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName; + auto emplaced = _audioStreams.emplace( QUuid(), - std::unique_ptr { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) } + std::unique_ptr { avatarAudioStream } ); micStreamIt = emplaced.first; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 17274a1519..0b3b352e66 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -19,6 +19,8 @@ #include #include +#include + #include "PositionalAudioStream.h" #include "AvatarAudioStream.h" @@ -65,6 +67,11 @@ public: AudioLimiter audioLimiter; + // FIXME -- maybe make these private + CodecPluginPointer _codec; + QString _selectedCodecName; + + signals: void injectorStreamFinished(const QUuid& streamIdentifier); diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 147030c831..9dc096eefd 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -837,7 +837,7 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); - // TODO - codec decode goes here + // TODO - codec encode goes here QByteArray encodedBuffer; if (_codec) { _codec->encode(audio, encodedBuffer); diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt index c49c9547a5..1e9360b9a2 100644 --- a/libraries/audio/CMakeLists.txt +++ b/libraries/audio/CMakeLists.txt @@ -1,3 +1,3 @@ set(TARGET_NAME audio) setup_hifi_library(Network) -link_hifi_libraries(networking shared) +link_hifi_libraries(networking shared plugins) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 8b32ada296..c19eb0c161 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -178,7 +178,16 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& } int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); + + // codec decode goes here + QByteArray decodedBuffer; + if (_codec) { + _codec->decode(packetAfterStreamProperties, decodedBuffer); + } else { + decodedBuffer = packetAfterStreamProperties; + } + + return _ringBuffer.writeData(decodedBuffer.data(), numAudioSamples * sizeof(int16_t)); // FIXME? } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 3f641f1ba4..ddc0dc1dc3 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -18,6 +18,8 @@ #include #include +#include + #include "AudioRingBuffer.h" #include "MovingMinMaxAvg.h" #include "SequenceNumberStats.h" @@ -174,6 +176,10 @@ public: void setReverb(float reverbTime, float wetLevel); void clearReverb() { _hasReverb = false; } + // FIXME -- maybe make these private + CodecPluginPointer _codec; + QString _selectedCodecName; + public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index eb2f4761b4..d204fb1100 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -39,13 +39,17 @@ bool PCMCodecManager::isSupported() const { void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) { - qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size(); // this codec doesn't actually do anything.... decodedBuffer = encodedBuffer; + + //decodedBuffer = qUncompress(encodedBuffer); + //qDebug() << __FUNCTION__ << "from:" << encodedBuffer.size() << " to:" << decodedBuffer.size(); } void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size(); // this codec doesn't actually do anything.... encodedBuffer = decodedBuffer; + + //encodedBuffer = qCompress(decodedBuffer); + //qDebug() << __FUNCTION__ << "from:" << decodedBuffer.size() << " to:" << encodedBuffer.size(); }