From be37921845d760e9052f68ec04891bf6d2dee237 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Thu, 9 Jun 2016 15:49:07 -0700
Subject: [PATCH 01/22] first cut at codec plugins
---
interface/src/Application.cpp | 7 +++
libraries/audio-client/src/AudioClient.cpp | 20 ++++++---
libraries/plugins/src/plugins/CodecPlugin.h | 19 ++++++++
libraries/plugins/src/plugins/Forward.h | 4 ++
.../plugins/src/plugins/PluginManager.cpp | 30 +++++++++++++
libraries/plugins/src/plugins/PluginManager.h | 1 +
libraries/plugins/src/plugins/RuntimePlugin.h | 9 ++++
plugins/pcmCodec/CMakeLists.txt | 11 +++++
plugins/pcmCodec/src/PCMCodecManager.cpp | 45 +++++++++++++++++++
plugins/pcmCodec/src/PCMCodecManager.h | 40 +++++++++++++++++
plugins/pcmCodec/src/PCMCodecProvider.cpp | 44 ++++++++++++++++++
plugins/pcmCodec/src/plugin.json | 1 +
12 files changed, 224 insertions(+), 7 deletions(-)
create mode 100644 libraries/plugins/src/plugins/CodecPlugin.h
create mode 100644 plugins/pcmCodec/CMakeLists.txt
create mode 100644 plugins/pcmCodec/src/PCMCodecManager.cpp
create mode 100644 plugins/pcmCodec/src/PCMCodecManager.h
create mode 100644 plugins/pcmCodec/src/PCMCodecProvider.cpp
create mode 100644 plugins/pcmCodec/src/plugin.json
diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 37c3b361bf..855cf2f313 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -85,6 +85,7 @@
#include <PhysicsEngine.h>
#include <PhysicsHelpers.h>
#include <plugins/PluginManager.h>
+#include <plugins/CodecPlugin.h>
#include <RenderableWebEntityItem.h>
#include <RenderShadowTask.h>
#include <RenderDeferredTask.h>
@@ -1236,7 +1237,13 @@ QString Application::getUserAgent() {
userAgent += " " + formatPluginName(ip->getName());
}
}
+ // for codecs, we include all of them, even if not active
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+ for (auto& cp : codecPlugins) {
+ userAgent += " " + formatPluginName(cp->getName());
+ }
+ qDebug() << __FUNCTION__ << ":" << userAgent;
return userAgent;
}
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 0c7a79e2a3..2569f27aaa 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -813,19 +813,25 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
}
-void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
- const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t);
- const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
- / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount());
+void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) {
+
+ // TODO - codec decode goes here
+ QByteArray decodedBuffer = networkBuffer;
+
+ const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
+ const int numDeviceOutputSamples = _outputFrameSize;
+
+ Q_ASSERT(_outputFrameSize == numDecodecSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
+ / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()));
outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
- const int16_t* receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
+ const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
// copy the packet from the RB to the output
- possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples,
- numNetworkOutputSamples, numDeviceOutputSamples,
+ possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples,
+ numDecodecSamples, numDeviceOutputSamples,
_desiredOutputFormat, _outputFormat);
// apply stereo reverb at the listener, to the received audio
diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h
new file mode 100644
index 0000000000..d9e1b947fe
--- /dev/null
+++ b/libraries/plugins/src/plugins/CodecPlugin.h
@@ -0,0 +1,19 @@
+//
+// CodecPlugin.h
+// plugins/src/plugins
+//
+// Created by Brad Hefta-Gaub on 6/9/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+
+#include "Plugin.h"
+
+class CodecPlugin : public Plugin {
+public:
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+};
+
diff --git a/libraries/plugins/src/plugins/Forward.h b/libraries/plugins/src/plugins/Forward.h
index 036b42f7d7..723c4f321e 100644
--- a/libraries/plugins/src/plugins/Forward.h
+++ b/libraries/plugins/src/plugins/Forward.h
@@ -13,10 +13,12 @@
enum class PluginType {
DISPLAY_PLUGIN,
INPUT_PLUGIN,
+ CODEC_PLUGIN,
};
class DisplayPlugin;
class InputPlugin;
+class CodecPlugin;
class Plugin;
class PluginContainer;
class PluginManager;
@@ -25,4 +27,6 @@ using DisplayPluginPointer = std::shared_ptr<DisplayPlugin>;
using DisplayPluginList = std::vector<DisplayPluginPointer>;
using InputPluginPointer = std::shared_ptr<InputPlugin>;
using InputPluginList = std::vector<InputPluginPointer>;
+using CodecPluginPointer = std::shared_ptr<CodecPlugin>;
+using CodecPluginList = std::vector<CodecPluginPointer>;
diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp
index 29658eeb6b..fe34d2dbee 100644
--- a/libraries/plugins/src/plugins/PluginManager.cpp
+++ b/libraries/plugins/src/plugins/PluginManager.cpp
@@ -18,6 +18,7 @@
#include <UserActivityLogger.h>
#include "RuntimePlugin.h"
+#include "CodecPlugin.h"
#include "DisplayPlugin.h"
#include "InputPlugin.h"
@@ -117,6 +118,7 @@ PluginManager::PluginManager() {
// TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
extern DisplayPluginList getDisplayPlugins();
extern InputPluginList getInputPlugins();
+extern CodecPluginList getCodecPlugins();
extern void saveInputPluginSettings(const InputPluginList& plugins);
static DisplayPluginList displayPlugins;
@@ -202,6 +204,34 @@ const InputPluginList& PluginManager::getInputPlugins() {
return inputPlugins;
}
+const CodecPluginList& PluginManager::getCodecPlugins() {
+ static CodecPluginList codecPlugins;
+ static std::once_flag once;
+ std::call_once(once, [&] {
+ //codecPlugins = ::getCodecPlugins();
+
+ // Now grab the dynamic plugins
+ for (auto loader : getLoadedPlugins()) {
+ CodecProvider* codecProvider = qobject_cast<CodecProvider*>(loader->instance());
+ if (codecProvider) {
+ for (auto codecPlugin : codecProvider->getCodecPlugins()) {
+ if (codecPlugin->isSupported()) {
+ codecPlugins.push_back(codecPlugin);
+ }
+ }
+ }
+ }
+
+ auto& container = PluginContainer::getInstance();
+ for (auto plugin : codecPlugins) {
+ plugin->setContainer(&container);
+ plugin->init();
+ }
+ });
+ return codecPlugins;
+}
+
+
void PluginManager::setPreferredDisplayPlugins(const QStringList& displays) {
preferredDisplayPlugins = displays;
}
diff --git a/libraries/plugins/src/plugins/PluginManager.h b/libraries/plugins/src/plugins/PluginManager.h
index 7903bdd724..30d52298da 100644
--- a/libraries/plugins/src/plugins/PluginManager.h
+++ b/libraries/plugins/src/plugins/PluginManager.h
@@ -18,6 +18,7 @@ public:
const DisplayPluginList& getDisplayPlugins();
const InputPluginList& getInputPlugins();
+ const CodecPluginList& getCodecPlugins();
DisplayPluginList getPreferredDisplayPlugins();
void setPreferredDisplayPlugins(const QStringList& displays);
diff --git a/libraries/plugins/src/plugins/RuntimePlugin.h b/libraries/plugins/src/plugins/RuntimePlugin.h
index d7bf31ea28..9bf15f344d 100644
--- a/libraries/plugins/src/plugins/RuntimePlugin.h
+++ b/libraries/plugins/src/plugins/RuntimePlugin.h
@@ -34,3 +34,12 @@ public:
#define InputProvider_iid "com.highfidelity.plugins.input"
Q_DECLARE_INTERFACE(InputProvider, InputProvider_iid)
+class CodecProvider {
+public:
+ virtual ~CodecProvider() {}
+ virtual CodecPluginList getCodecPlugins() = 0;
+};
+
+#define CodecProvider_iid "com.highfidelity.plugins.codec"
+Q_DECLARE_INTERFACE(CodecProvider, CodecProvider_iid)
+
diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt
new file mode 100644
index 0000000000..162fe557cc
--- /dev/null
+++ b/plugins/pcmCodec/CMakeLists.txt
@@ -0,0 +1,11 @@
+#
+# Created by Brad Hefta-Gaub on 6/9/2016
+# Copyright 2016 High Fidelity, Inc.
+#
+# Distributed under the Apache License, Version 2.0.
+# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+
+set(TARGET_NAME pcmCodec)
+setup_hifi_plugin()
+link_hifi_libraries(shared plugins)
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
new file mode 100644
index 0000000000..264c139ac6
--- /dev/null
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -0,0 +1,45 @@
+//
+// PCMCodecManager.cpp
+// plugins/pcmCodec/src
+//
+// Created by Brad Hefta-Gaub on 6/9/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <qapplication.h>
+
+#include <PerfStat.h>
+
+#include "PCMCodecManager.h"
+
+const QString PCMCodecManager::NAME = "PCM Codec";
+
+void PCMCodecManager::init() {
+}
+
+void PCMCodecManager::deinit() {
+}
+
+bool PCMCodecManager::activate() {
+ CodecPlugin::activate();
+ return true;
+}
+
+void PCMCodecManager::deactivate() {
+ CodecPlugin::deactivate();
+}
+
+
+bool PCMCodecManager::isSupported() const {
+ return true;
+}
+
+
+void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+ // this codec doesn't actually do anything....
+ decodedBuffer = encodedBuffer;
+}
+
diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h
new file mode 100644
index 0000000000..a8b3e49a2c
--- /dev/null
+++ b/plugins/pcmCodec/src/PCMCodecManager.h
@@ -0,0 +1,40 @@
+//
+// PCMCodecManager.h
+// plugins/pcmCodec/src
+//
+// Created by Brad Hefta-Gaub on 6/9/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi__PCMCodecManager_h
+#define hifi__PCMCodecManager_h
+
+
+#include <plugins/CodecPlugin.h>
+
+class PCMCodecManager : public CodecPlugin {
+ Q_OBJECT
+
+public:
+ // Plugin functions
+ bool isSupported() const override;
+ const QString& getName() const override { return NAME; }
+
+ void init() override;
+ void deinit() override;
+
+ /// Called when a plugin is being activated for use. May be called multiple times.
+ bool activate() override;
+ /// Called when a plugin is no longer being used. May be called multiple times.
+ void deactivate() override;
+
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+
+private:
+ static const QString NAME;
+};
+
+#endif // hifi__PCMCodecManager_h
diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp
new file mode 100644
index 0000000000..732ed2d57d
--- /dev/null
+++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp
@@ -0,0 +1,44 @@
+//
+// Created by Brad Hefta-Gaub on 6/9/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <mutex>
+
+#include <QtCore/QObject>
+#include <QtCore/QtPlugin>
+#include <QtCore/QStringList>
+
+#include <plugins/RuntimePlugin.h>
+#include <plugins/CodecPlugin.h>
+
+#include "PCMCodecManager.h"
+
+class PCMCodecProvider : public QObject, public CodecProvider {
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json")
+ Q_INTERFACES(CodecProvider)
+
+public:
+ PCMCodecProvider(QObject* parent = nullptr) : QObject(parent) {}
+ virtual ~PCMCodecProvider() {}
+
+ virtual CodecPluginList getCodecPlugins() override {
+ static std::once_flag once;
+ std::call_once(once, [&] {
+ CodecPluginPointer plugin(new PCMCodecManager());
+ if (plugin->isSupported()) {
+ _codecPlugins.push_back(plugin);
+ }
+ });
+ return _codecPlugins;
+ }
+
+private:
+ CodecPluginList _codecPlugins;
+};
+
+#include "PCMCodecProvider.moc"
diff --git a/plugins/pcmCodec/src/plugin.json b/plugins/pcmCodec/src/plugin.json
new file mode 100644
index 0000000000..2d86251845
--- /dev/null
+++ b/plugins/pcmCodec/src/plugin.json
@@ -0,0 +1 @@
+{"name":"PCM Codec"}
From d7bffc3eabec8bb0f5f26a3eadf44aa7dfdc3a31 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Thu, 23 Jun 2016 17:14:41 -0700
Subject: [PATCH 02/22] first cut at negotiating codecs
---
assignment-client/src/audio/AudioMixer.cpp | 26 +++++++++++
assignment-client/src/audio/AudioMixer.h | 1 +
interface/src/Application.cpp | 4 +-
libraries/audio-client/CMakeLists.txt | 2 +-
libraries/audio-client/src/AudioClient.cpp | 46 +++++++++++++++++++
libraries/audio-client/src/AudioClient.h | 5 ++
libraries/networking/src/udt/PacketHeaders.h | 4 +-
libraries/plugins/src/plugins/CodecPlugin.h | 1 +
.../plugins/src/plugins/PluginManager.cpp | 5 +-
plugins/pcmCodec/src/PCMCodecManager.cpp | 4 ++
plugins/pcmCodec/src/PCMCodecManager.h | 1 +
11 files changed, 94 insertions(+), 5 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 03bb32cd53..95578e3998 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -90,6 +90,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
PacketType::AudioStreamStats },
this, "handleNodeAudioPacket");
packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
+ packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat");
connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled);
}
@@ -446,6 +447,31 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> mes
}
}
+void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
+ qDebug() << __FUNCTION__;
+
+ // read the codecs requested by the client
+ quint8 numberOfCodecs = 0;
+ message->readPrimitive(&numberOfCodecs);
+ QStringList codecList;
+ for (quint16 i = 0; i < numberOfCodecs; i++) {
+ QString requestedCodec = message->readString();
+ qDebug() << "requestedCodec:" << requestedCodec;
+ codecList.append(requestedCodec);
+ }
+ qDebug() << "all requested codecs:" << codecList;
+
+ auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
+
+ // write them to our packet
+ QString selectedCodec = codecList.front();
+ qDebug() << "selectedCodec:" << selectedCodec;
+ replyPacket->writeString(selectedCodec);
+
+ auto nodeList = DependencyManager::get<NodeList>();
+ nodeList->sendPacket(std::move(replyPacket), *sendingNode);
+}
+
void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {
// enumerate the connected listeners to remove HRTF objects for the disconnected node
auto nodeList = DependencyManager::get<NodeList>();
diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h
index 24b4b39704..c90a918a5b 100644
--- a/assignment-client/src/audio/AudioMixer.h
+++ b/assignment-client/src/audio/AudioMixer.h
@@ -45,6 +45,7 @@ private slots:
void broadcastMixes();
void handleNodeAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
+ void handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode);
void handleNodeKilled(SharedNodePointer killedNode);
void removeHRTFsForFinishedInjector(const QUuid& streamID);
diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 855cf2f313..3c69f716ca 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -1243,7 +1243,6 @@ QString Application::getUserAgent() {
userAgent += " " + formatPluginName(cp->getName());
}
- qDebug() << __FUNCTION__ << ":" << userAgent;
return userAgent;
}
@@ -4440,6 +4439,9 @@ void Application::nodeActivated(SharedNodePointer node) {
}
}
+ if (node->getType() == NodeType::AudioMixer) {
+ DependencyManager::get<AudioClient>()->negotiateAudioFormat();
+ }
}
void Application::nodeKilled(SharedNodePointer node) {
diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt
index 2c0fc0a9cd..9c298ce664 100644
--- a/libraries/audio-client/CMakeLists.txt
+++ b/libraries/audio-client/CMakeLists.txt
@@ -1,6 +1,6 @@
set(TARGET_NAME audio-client)
setup_hifi_library(Network Multimedia)
-link_hifi_libraries(audio)
+link_hifi_libraries(audio plugins)
# append audio includes to our list of includes to bubble
target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src")
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 2569f27aaa..18db23dbf7 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -34,6 +34,8 @@
#include <QtMultimedia/QAudioOutput>
#include <NodeList.h>
+#include <plugins/CodecPlugin.h>
+#include <plugins/PluginManager.h>
#include <udt/PacketHeaders.h>
#include <PositionalAudioStream.h>
#include <SettingHandle.h>
@@ -134,6 +136,7 @@ AudioClient::AudioClient() :
packetReceiver.registerListener(PacketType::MixedAudio, this, "handleAudioDataPacket");
packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket");
packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
+ packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat");
}
AudioClient::~AudioClient() {
@@ -474,6 +477,32 @@ void AudioClient::stop() {
}
}
+void AudioClient::negotiateAudioFormat() {
+ qDebug() << __FUNCTION__;
+
+ auto nodeList = DependencyManager::get<NodeList>();
+
+ auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
+
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+
+ quint8 numberOfCodecs = (quint8)codecPlugins.size();
+ negotiateFormatPacket->writePrimitive(numberOfCodecs);
+ for (auto& plugin : codecPlugins) {
+ qDebug() << "Codec available:" << plugin->getName();
+ negotiateFormatPacket->writeString(plugin->getName());
+ }
+
+ // grab our audio mixer from the NodeList, if it exists
+ SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+
+ if (audioMixer) {
+ // send off this mute packet
+ nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
+ }
+}
+
+
void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer<ReceivedMessage> message) {
char bitset;
@@ -528,6 +557,16 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
emit muteEnvironmentRequested(position, radius);
}
+void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
+ qDebug() << __FUNCTION__;
+
+ // write them to our packet
+ QString selectedCodec = message->readString();
+
+ qDebug() << "selectedCodec:" << selectedCodec;
+}
+
+
QString AudioClient::getDefaultDeviceName(QAudio::Mode mode) {
QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode);
return deviceInfo.deviceName();
@@ -1227,6 +1266,13 @@ void AudioClient::loadSettings() {
windowSecondsForDesiredCalcOnTooManyStarves.get());
_receivedAudioStream.setWindowSecondsForDesiredReduction(windowSecondsForDesiredReduction.get());
_receivedAudioStream.setRepetitionWithFade(repetitionWithFade.get());
+
+ qDebug() << "---- Initializing Audio Client ----";
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+ for (auto& plugin : codecPlugins) {
+ qDebug() << "Codec available:" << plugin->getName();
+ }
+
}
void AudioClient::saveSettings() {
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index 3a14c878f6..e05612f859 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -94,6 +94,8 @@ public:
int _unfulfilledReads;
};
+ void negotiateAudioFormat();
+
const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }
@@ -139,6 +141,7 @@ public slots:
void handleAudioDataPacket(QSharedPointer<ReceivedMessage> message);
void handleNoisyMutePacket(QSharedPointer<ReceivedMessage> message);
void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message);
+ void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); }
void handleAudioInput();
@@ -292,6 +295,8 @@ private:
void checkDevices();
bool _hasReceivedFirstPacket = false;
+
+ //CodecPluginPointer _codec { nullptr };
};
diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h
index ae54450fee..b5d1be077f 100644
--- a/libraries/networking/src/udt/PacketHeaders.h
+++ b/libraries/networking/src/udt/PacketHeaders.h
@@ -95,7 +95,9 @@ public:
AssetMappingOperation,
AssetMappingOperationReply,
ICEServerHeartbeatACK,
- LAST_PACKET_TYPE = ICEServerHeartbeatACK
+ NegotiateAudioFormat,
+ SelectedAudioFormat,
+ LAST_PACKET_TYPE = SelectedAudioFormat
};
};
diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h
index d9e1b947fe..6944d91bed 100644
--- a/libraries/plugins/src/plugins/CodecPlugin.h
+++ b/libraries/plugins/src/plugins/CodecPlugin.h
@@ -15,5 +15,6 @@
class CodecPlugin : public Plugin {
public:
virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
};
diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp
index fe34d2dbee..0b4afe1be0 100644
--- a/libraries/plugins/src/plugins/PluginManager.cpp
+++ b/libraries/plugins/src/plugins/PluginManager.cpp
@@ -222,10 +222,11 @@ const CodecPluginList& PluginManager::getCodecPlugins() {
}
}
- auto& container = PluginContainer::getInstance();
for (auto plugin : codecPlugins) {
- plugin->setContainer(&container);
+ plugin->setContainer(_container);
plugin->init();
+
+ qDebug() << "init codec:" << plugin->getName();
}
});
return codecPlugins;
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index 264c139ac6..2401e99576 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -43,3 +43,7 @@ void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decode
decodedBuffer = encodedBuffer;
}
+void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+ // this codec doesn't actually do anything....
+ encodedBuffer = decodedBuffer;
+}
diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h
index a8b3e49a2c..3b7ca36c02 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.h
+++ b/plugins/pcmCodec/src/PCMCodecManager.h
@@ -32,6 +32,7 @@ public:
void deactivate() override;
virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
private:
static const QString NAME;
From 7a4b11ee9790e902d55e12292fe9b420c72478f3 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 27 Jun 2016 13:06:19 -0700
Subject: [PATCH 03/22] more work on codecs
---
assignment-client/CMakeLists.txt | 2 +-
assignment-client/src/audio/AudioMixer.cpp | 25 ++++++
libraries/audio-client/src/AudioClient.cpp | 94 +++++++++++++++-------
libraries/audio-client/src/AudioClient.h | 5 +-
plugins/pcmCodec/src/PCMCodecManager.cpp | 2 +
5 files changed, 95 insertions(+), 33 deletions(-)
diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt
index 1b5840c3c8..d0fd2c1176 100644
--- a/assignment-client/CMakeLists.txt
+++ b/assignment-client/CMakeLists.txt
@@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets)
link_hifi_libraries(
audio avatars octree gpu model fbx entities
networking animation recording shared script-engine embedded-webserver
- controllers physics
+ controllers physics plugins
)
if (WIN32)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 95578e3998..27b0a092d0 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -47,6 +47,8 @@
#include <NodeList.h>
#include <Node.h>
#include <OctreeConstants.h>
+#include <plugins/PluginManager.h>
+#include <plugins/CodecPlugin.h>
#include <udt/PacketHeaders.h>
#include <SharedUtil.h>
#include <StDev.h>
@@ -447,6 +449,20 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> mes
}
}
+DisplayPluginList getDisplayPlugins() {
+ DisplayPluginList result;
+ return result;
+}
+
+InputPluginList getInputPlugins() {
+ InputPluginList result;
+ return result;
+}
+
+void saveInputPluginSettings(const InputPluginList& plugins) {
+}
+
+
void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
qDebug() << __FUNCTION__;
@@ -461,6 +477,15 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
}
qDebug() << "all requested codecs:" << codecList;
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+ if (codecPlugins.size() > 0) {
+ for (auto& plugin : codecPlugins) {
+ qDebug() << "Codec available:" << plugin->getName();
+ }
+ } else {
+ qDebug() << "No Codecs available...";
+ }
+
auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
// write them to our packet
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 18db23dbf7..7ef71087ce 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -477,32 +477,6 @@ void AudioClient::stop() {
}
}
-void AudioClient::negotiateAudioFormat() {
- qDebug() << __FUNCTION__;
-
- auto nodeList = DependencyManager::get<NodeList>();
-
- auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
-
- auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
-
- quint8 numberOfCodecs = (quint8)codecPlugins.size();
- negotiateFormatPacket->writePrimitive(numberOfCodecs);
- for (auto& plugin : codecPlugins) {
- qDebug() << "Codec available:" << plugin->getName();
- negotiateFormatPacket->writeString(plugin->getName());
- }
-
- // grab our audio mixer from the NodeList, if it exists
- SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
-
- if (audioMixer) {
- // send off this mute packet
- nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
- }
-}
-
-
void AudioClient::handleAudioEnvironmentDataPacket(QSharedPointer<ReceivedMessage> message) {
char bitset;
@@ -557,13 +531,47 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
emit muteEnvironmentRequested(position, radius);
}
+void AudioClient::negotiateAudioFormat() {
+ qDebug() << __FUNCTION__;
+
+ auto nodeList = DependencyManager::get<NodeList>();
+
+ auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
+
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+
+ quint8 numberOfCodecs = (quint8)codecPlugins.size();
+ negotiateFormatPacket->writePrimitive(numberOfCodecs);
+ for (auto& plugin : codecPlugins) {
+ qDebug() << "Codec available:" << plugin->getName();
+ negotiateFormatPacket->writeString(plugin->getName());
+ }
+
+ // grab our audio mixer from the NodeList, if it exists
+ SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+
+ if (audioMixer) {
+ // send off this mute packet
+ nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
+ }
+}
+
void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
qDebug() << __FUNCTION__;
// write them to our packet
- QString selectedCodec = message->readString();
+ _selectedCodecName = message->readString();
+
+ qDebug() << "Selected Codec:" << _selectedCodecName;
+ auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+ for (auto& plugin : codecPlugins) {
+ if (_selectedCodecName == plugin->getName()) {
+ _codec = plugin;
+ qDebug() << "Selected Codec Plugin:" << _codec.get();
+ break;
+ }
+ }
- qDebug() << "selectedCodec:" << selectedCodec;
}
@@ -839,7 +847,17 @@ void AudioClient::handleAudioInput() {
audioTransform.setTranslation(_positionGetter());
audioTransform.setRotation(_orientationGetter());
// FIXME find a way to properly handle both playback audio and user audio concurrently
- emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
+
+ // TODO - codec encode goes here
+ QByteArray decocedBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
+ QByteArray encodedBuffer;
+ if (_codec) {
+ _codec->encode(decocedBuffer, encodedBuffer);
+ } else {
+ encodedBuffer = decocedBuffer;
+ }
+
+ emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
_stats.sentPacket();
}
}
@@ -848,14 +866,28 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
Transform audioTransform;
audioTransform.setTranslation(_positionGetter());
audioTransform.setRotation(_orientationGetter());
+
+ // TODO - codec decode goes here
+ QByteArray encodedBuffer;
+ if (_codec) {
+ _codec->encode(audio, encodedBuffer);
+ } else {
+ encodedBuffer = audio;
+ }
+
// FIXME check a flag to see if we should echo audio?
- emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
+ emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
}
void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) {
// TODO - codec decode goes here
- QByteArray decodedBuffer = networkBuffer;
+ QByteArray decodedBuffer;
+ if (_codec) {
+ _codec->decode(networkBuffer, decodedBuffer);
+ } else {
+ decodedBuffer = networkBuffer;
+ }
const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
const int numDeviceOutputSamples = _outputFrameSize;
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index e05612f859..45b4a631b2 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -38,6 +38,8 @@
#include <Sound.h>
#include <StDev.h>
+#include <plugins/CodecPlugin.h>
+
#include "AudioIOStats.h"
#include "AudioNoiseGate.h"
#include "AudioSRC.h"
@@ -296,7 +298,8 @@ private:
bool _hasReceivedFirstPacket = false;
- //CodecPluginPointer _codec { nullptr };
+ CodecPluginPointer _codec;
+ QString _selectedCodecName;
};
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index 2401e99576..eb2f4761b4 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -39,11 +39,13 @@ bool PCMCodecManager::isSupported() const {
void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+ qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size();
// this codec doesn't actually do anything....
decodedBuffer = encodedBuffer;
}
void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+ qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size();
// this codec doesn't actually do anything....
encodedBuffer = decodedBuffer;
}
From 38f0cd218bc15e2c1d1ba80dc0d47d317d2c3a68 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Thu, 7 Jul 2016 10:31:12 -0700
Subject: [PATCH 04/22] make client server plugins properly copy into plugins
directory
---
.../macros/SetupHifiClientServerPlugin.cmake | 54 +++++++++++++++++++
plugins/pcmCodec/CMakeLists.txt | 2 +-
2 files changed, 55 insertions(+), 1 deletion(-)
create mode 100644 cmake/macros/SetupHifiClientServerPlugin.cmake
diff --git a/cmake/macros/SetupHifiClientServerPlugin.cmake b/cmake/macros/SetupHifiClientServerPlugin.cmake
new file mode 100644
index 0000000000..5ace348e87
--- /dev/null
+++ b/cmake/macros/SetupHifiClientServerPlugin.cmake
@@ -0,0 +1,54 @@
+#
+# Created by Brad Hefta-Gaub on 2016/07/07
+# Copyright 2016 High Fidelity, Inc.
+#
+# Distributed under the Apache License, Version 2.0.
+# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN)
+ set(${TARGET_NAME}_SHARED 1)
+ setup_hifi_library(${ARGV})
+ add_dependencies(interface ${TARGET_NAME})
+ set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins")
+
+ if (APPLE)
+ set(PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns")
+ else()
+ set(PLUGIN_PATH "plugins")
+ endif()
+
+ if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles")
+ set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${PLUGIN_PATH}/")
+ set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${PLUGIN_PATH}/")
+ else()
+ set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${PLUGIN_PATH}/")
+ set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${PLUGIN_PATH}/")
+ endif()
+
+ # create the destination for the client plugin binaries
+ add_custom_command(
+ TARGET ${TARGET_NAME} POST_BUILD
+ COMMAND "${CMAKE_COMMAND}" -E make_directory
+ ${CLIENT_PLUGIN_FULL_PATH}
+ )
+ # copy the client plugin binaries
+ add_custom_command(TARGET ${DIR} POST_BUILD
+ COMMAND "${CMAKE_COMMAND}" -E copy
+ "$<TARGET_FILE:${TARGET_NAME}>"
+ ${CLIENT_PLUGIN_FULL_PATH}
+ )
+
+ # create the destination for the server plugin binaries
+ add_custom_command(
+ TARGET ${TARGET_NAME} POST_BUILD
+ COMMAND "${CMAKE_COMMAND}" -E make_directory
+ ${SERVER_PLUGIN_FULL_PATH}
+ )
+ # copy the server plugin binaries
+ add_custom_command(TARGET ${DIR} POST_BUILD
+ COMMAND "${CMAKE_COMMAND}" -E copy
+ "$<TARGET_FILE:${TARGET_NAME}>"
+ ${SERVER_PLUGIN_FULL_PATH}
+ )
+
+endmacro()
diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt
index 162fe557cc..5dca1f0e14 100644
--- a/plugins/pcmCodec/CMakeLists.txt
+++ b/plugins/pcmCodec/CMakeLists.txt
@@ -7,5 +7,5 @@
#
set(TARGET_NAME pcmCodec)
-setup_hifi_plugin()
+setup_hifi_client_server_plugin()
link_hifi_libraries(shared plugins)
From 3c6447326e1e8678d0a05dfc99cc9b858bab6c1a Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Thu, 7 Jul 2016 21:24:24 -0700
Subject: [PATCH 05/22] more codec plumbing
---
assignment-client/src/audio/AudioMixer.cpp | 36 ++++++++++++++++---
.../src/audio/AudioMixerClientData.cpp | 7 +++-
.../src/audio/AudioMixerClientData.h | 7 ++++
libraries/audio-client/src/AudioClient.cpp | 2 +-
libraries/audio/CMakeLists.txt | 2 +-
libraries/audio/src/InboundAudioStream.cpp | 11 +++++-
libraries/audio/src/InboundAudioStream.h | 6 ++++
plugins/pcmCodec/src/PCMCodecManager.cpp | 8 +++--
8 files changed, 68 insertions(+), 11 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 27b0a092d0..9d9194aef6 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -477,21 +477,39 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
}
qDebug() << "all requested codecs:" << codecList;
+ CodecPluginPointer selectedCoded;
+ QString selectedCodecName;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
if (codecPlugins.size() > 0) {
for (auto& plugin : codecPlugins) {
qDebug() << "Codec available:" << plugin->getName();
+
+ // choose first codec
+ if (!selectedCoded) {
+ selectedCoded = plugin;
+ selectedCodecName = plugin->getName();
+ }
}
} else {
qDebug() << "No Codecs available...";
}
+ auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
+
+ clientData->_codec = selectedCoded;
+ clientData->_selectedCodecName = selectedCodecName;
+ qDebug() << "selectedCodecName:" << selectedCodecName;
+
+ auto avatarAudioStream = clientData->getAvatarAudioStream();
+ if (avatarAudioStream) {
+ avatarAudioStream->_codec = selectedCoded;
+ avatarAudioStream->_selectedCodecName = selectedCodecName;
+ }
+
auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
// write them to our packet
- QString selectedCodec = codecList.front();
- qDebug() << "selectedCodec:" << selectedCodec;
- replyPacket->writeString(selectedCodec);
+ replyPacket->writeString(selectedCodecName);
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *sendingNode);
@@ -720,9 +738,17 @@ void AudioMixer::broadcastMixes() {
quint16 sequence = nodeData->getOutgoingSequenceNumber();
mixPacket->writePrimitive(sequence);
+ // TODO - codec encode goes here
+ QByteArray decocedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+ QByteArray encodedBuffer;
+ if (nodeData->_codec) {
+ nodeData->_codec->encode(decocedBuffer, encodedBuffer);
+ } else {
+ encodedBuffer = decocedBuffer;
+ }
+
// pack mixed audio samples
- mixPacket->write(reinterpret_cast<char*>(_clampedSamples),
- AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+ mixPacket->write(encodedBuffer.constData(), encodedBuffer.size());
} else {
int silentPacketBytes = sizeof(quint16) + sizeof(quint16);
mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes);
diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp
index 20003ba10d..080a833c0e 100644
--- a/assignment-client/src/audio/AudioMixerClientData.cpp
+++ b/assignment-client/src/audio/AudioMixerClientData.cpp
@@ -101,9 +101,14 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
bool isStereo = channelFlag == 1;
+ auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings());
+ avatarAudioStream->_codec = _codec;
+ avatarAudioStream->_selectedCodecName = _selectedCodecName;
+ qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName;
+
auto emplaced = _audioStreams.emplace(
QUuid(),
- std::unique_ptr<PositionalAudioStream> { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) }
+ std::unique_ptr<PositionalAudioStream> { avatarAudioStream }
);
micStreamIt = emplaced.first;
diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h
index 17274a1519..0b3b352e66 100644
--- a/assignment-client/src/audio/AudioMixerClientData.h
+++ b/assignment-client/src/audio/AudioMixerClientData.h
@@ -19,6 +19,8 @@
#include <AudioLimiter.h>
#include <UUIDHasher.h>
+#include <plugins/CodecPlugin.h>
+
#include "PositionalAudioStream.h"
#include "AvatarAudioStream.h"
@@ -65,6 +67,11 @@ public:
AudioLimiter audioLimiter;
+ // FIXME -- maybe make these private
+ CodecPluginPointer _codec;
+ QString _selectedCodecName;
+
+
signals:
void injectorStreamFinished(const QUuid& streamIdentifier);
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 147030c831..9dc096eefd 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -837,7 +837,7 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
audioTransform.setTranslation(_positionGetter());
audioTransform.setRotation(_orientationGetter());
- // TODO - codec decode goes here
+ // TODO - codec encode goes here
QByteArray encodedBuffer;
if (_codec) {
_codec->encode(audio, encodedBuffer);
diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt
index c49c9547a5..1e9360b9a2 100644
--- a/libraries/audio/CMakeLists.txt
+++ b/libraries/audio/CMakeLists.txt
@@ -1,3 +1,3 @@
set(TARGET_NAME audio)
setup_hifi_library(Network)
-link_hifi_libraries(networking shared)
+link_hifi_libraries(networking shared plugins)
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index 8b32ada296..c19eb0c161 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -178,7 +178,16 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
}
int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
- return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
+
+ // codec decode goes here
+ QByteArray decodedBuffer;
+ if (_codec) {
+ _codec->decode(packetAfterStreamProperties, decodedBuffer);
+ } else {
+ decodedBuffer = packetAfterStreamProperties;
+ }
+
+ return _ringBuffer.writeData(decodedBuffer.data(), numAudioSamples * sizeof(int16_t)); // FIXME?
}
int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) {
diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h
index 3f641f1ba4..ddc0dc1dc3 100644
--- a/libraries/audio/src/InboundAudioStream.h
+++ b/libraries/audio/src/InboundAudioStream.h
@@ -18,6 +18,8 @@
#include <ReceivedMessage.h>
#include <StDev.h>
+#include <plugins/CodecPlugin.h>
+
#include "AudioRingBuffer.h"
#include "MovingMinMaxAvg.h"
#include "SequenceNumberStats.h"
@@ -174,6 +176,10 @@ public:
void setReverb(float reverbTime, float wetLevel);
void clearReverb() { _hasReverb = false; }
+ // FIXME -- maybe make these private
+ CodecPluginPointer _codec;
+ QString _selectedCodecName;
+
public slots:
/// This function should be called every second for all the stats to function properly. If dynamic jitter buffers
/// is enabled, those stats are used to calculate _desiredJitterBufferFrames.
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index eb2f4761b4..d204fb1100 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -39,13 +39,17 @@ bool PCMCodecManager::isSupported() const {
void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
- qDebug() << __FUNCTION__ << "encodedBuffer:" << encodedBuffer.size();
// this codec doesn't actually do anything....
decodedBuffer = encodedBuffer;
+
+ //decodedBuffer = qUncompress(encodedBuffer);
+ //qDebug() << __FUNCTION__ << "from:" << encodedBuffer.size() << " to:" << decodedBuffer.size();
}
void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
- qDebug() << __FUNCTION__ << "decodedBuffer:" << decodedBuffer.size();
// this codec doesn't actually do anything....
encodedBuffer = decodedBuffer;
+
+ //encodedBuffer = qCompress(decodedBuffer);
+ //qDebug() << __FUNCTION__ << "from:" << decodedBuffer.size() << " to:" << encodedBuffer.size();
}
From 126e5c29264566657ef727e6858a992b0ecec759 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Fri, 8 Jul 2016 14:34:44 -0700
Subject: [PATCH 06/22] codec pipeline working, zlib example
---
assignment-client/src/audio/AudioMixer.cpp | 8 ++++++++
libraries/audio-client/src/AudioClient.cpp | 16 +++-------------
libraries/audio/src/InboundAudioStream.cpp | 11 ++++++++++-
.../audio/src/MixedProcessedAudioStream.cpp | 15 +++++++++++++--
plugins/pcmCodec/src/PCMCodecManager.cpp | 16 +++++-----------
5 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 9d9194aef6..b419939912 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -496,6 +496,14 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
+ // FIXME - why would we not have client data at this point??
+ if (!clientData) {
+ qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__;
+ sendingNode->setLinkedData(std::unique_ptr<NodeData> { new AudioMixerClientData(sendingNode->getUUID()) });
+ clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
+ connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);
+ }
+
clientData->_codec = selectedCoded;
clientData->_selectedCodecName = selectedCodecName;
qDebug() << "selectedCodecName:" << selectedCodecName;
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 9dc096eefd..a076b1b290 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -534,14 +534,13 @@ void AudioClient::negotiateAudioFormat() {
void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
qDebug() << __FUNCTION__;
- // write them to our packet
- _selectedCodecName = message->readString();
+ _receivedAudioStream._selectedCodecName = _selectedCodecName = message->readString();
qDebug() << "Selected Codec:" << _selectedCodecName;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
for (auto& plugin : codecPlugins) {
if (_selectedCodecName == plugin->getName()) {
- _codec = plugin;
+ _receivedAudioStream._codec = _codec = plugin;
qDebug() << "Selected Codec Plugin:" << _codec.get();
break;
}
@@ -849,16 +848,7 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
}
-void AudioClient::processReceivedSamples(const QByteArray& networkBuffer, QByteArray& outputBuffer) {
-
- // TODO - codec decode goes here
- QByteArray decodedBuffer;
- if (_codec) {
- _codec->decode(networkBuffer, decodedBuffer);
- } else {
- decodedBuffer = networkBuffer;
- }
-
+void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
const int numDeviceOutputSamples = _outputFrameSize;
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index c19eb0c161..5d4dbd9b94 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -187,7 +187,16 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet
decodedBuffer = packetAfterStreamProperties;
}
- return _ringBuffer.writeData(decodedBuffer.data(), numAudioSamples * sizeof(int16_t)); // FIXME?
+ auto actualSize = decodedBuffer.size();
+
+ /*
+ auto expectedSize = numAudioSamples * sizeof(int16_t);
+ if (expectedSize != actualSize) {
+ qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize;
+ }
+ */
+
+ return _ringBuffer.writeData(decodedBuffer.data(), actualSize);
}
int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) {
diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp
index d236ac7aad..220b2bd9ee 100644
--- a/libraries/audio/src/MixedProcessedAudioStream.cpp
+++ b/libraries/audio/src/MixedProcessedAudioStream.cpp
@@ -44,10 +44,21 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
- emit addedStereoSamples(packetAfterStreamProperties);
+ // TODO - codec decode goes here
+ QByteArray decodedBuffer;
+ if (_codec) {
+ _codec->decode(packetAfterStreamProperties, decodedBuffer);
+ } else {
+ decodedBuffer = packetAfterStreamProperties;
+ }
+
+ qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size();
+
+
+ emit addedStereoSamples(decodedBuffer);
QByteArray outputBuffer;
- emit processSamples(packetAfterStreamProperties, outputBuffer);
+ emit processSamples(decodedBuffer, outputBuffer);
_ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index d204fb1100..9d55cafedf 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -15,7 +15,7 @@
#include "PCMCodecManager.h"
-const QString PCMCodecManager::NAME = "PCM Codec";
+const QString PCMCodecManager::NAME = "zlib";
void PCMCodecManager::init() {
}
@@ -39,17 +39,11 @@ bool PCMCodecManager::isSupported() const {
void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
- // this codec doesn't actually do anything....
- decodedBuffer = encodedBuffer;
-
- //decodedBuffer = qUncompress(encodedBuffer);
- //qDebug() << __FUNCTION__ << "from:" << encodedBuffer.size() << " to:" << decodedBuffer.size();
+ //decodedBuffer = encodedBuffer;
+ decodedBuffer = qUncompress(encodedBuffer);
}
void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
- // this codec doesn't actually do anything....
- encodedBuffer = decodedBuffer;
-
- //encodedBuffer = qCompress(decodedBuffer);
- //qDebug() << __FUNCTION__ << "from:" << decodedBuffer.size() << " to:" << encodedBuffer.size();
+ //encodedBuffer = decodedBuffer;
+ encodedBuffer = qCompress(decodedBuffer);
}
From 489d2e4ca50ecc3ecc3ece28304c352b4352a472 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Fri, 8 Jul 2016 15:33:11 -0700
Subject: [PATCH 07/22] Update max request limit and number of processing
threads
---
interface/src/Application.cpp | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 82cbaf93c2..5d5bf45dcf 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -168,6 +168,9 @@ static QTimer locationUpdateTimer;
static QTimer identityPacketTimer;
static QTimer pingTimer;
+static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16;
+static const int PROCESSING_THREAD_POOL_SIZE = 6;
+
static const QString SNAPSHOT_EXTENSION = ".jpg";
static const QString SVO_EXTENSION = ".svo";
static const QString SVO_JSON_EXTENSION = ".svo.json";
@@ -517,7 +520,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
// (main thread, present thread, random OS load)
// More threads == faster concurrent loads, but also more concurrent
// load on the GPU until we can serialize GPU transfers (off the main thread)
- QThreadPool::globalInstance()->setMaxThreadCount(2);
+ QThreadPool::globalInstance()->setMaxThreadCount(PROCESSING_THREAD_POOL_SIZE);
thread()->setPriority(QThread::HighPriority);
thread()->setObjectName("Main Thread");
@@ -728,7 +731,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
connect(&identityPacketTimer, &QTimer::timeout, getMyAvatar(), &MyAvatar::sendIdentityPacket);
identityPacketTimer.start(AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS);
- ResourceCache::setRequestLimit(3);
+ ResourceCache::setRequestLimit(MAX_CONCURRENT_RESOURCE_DOWNLOADS);
_glWidget = new GLCanvas();
getApplicationCompositor().setRenderingWidget(_glWidget);
From a71baf5601ea269804ac30ba92eb8a148f89833e Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Fri, 8 Jul 2016 21:18:03 -0700
Subject: [PATCH 08/22] handle codec prioritization, handle multiple codec
choices
---
assignment-client/src/audio/AudioMixer.cpp | 73 ++++++++++++++-----
assignment-client/src/audio/AudioMixer.h | 2 +
.../resources/describe-settings.json | 8 ++
libraries/audio-client/src/AudioClient.cpp | 9 +--
libraries/audio/src/InboundAudioStream.cpp | 15 +---
libraries/audio/src/InboundAudioStream.h | 2 +-
.../audio/src/MixedProcessedAudioStream.cpp | 7 +-
.../audio/src/MixedProcessedAudioStream.h | 2 +-
libraries/networking/src/udt/BasePacket.cpp | 7 +-
plugins/pcmCodec/src/PCMCodecManager.cpp | 51 ++++++++++---
plugins/pcmCodec/src/PCMCodecManager.h | 25 ++++++-
plugins/pcmCodec/src/PCMCodecProvider.cpp | 13 +++-
12 files changed, 149 insertions(+), 65 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index b419939912..f49b674dce 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -466,34 +466,61 @@ void saveInputPluginSettings(const InputPluginList& plugins) {
void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
qDebug() << __FUNCTION__;
- // read the codecs requested by the client
- quint8 numberOfCodecs = 0;
- message->readPrimitive(&numberOfCodecs);
- QStringList codecList;
- for (quint16 i = 0; i < numberOfCodecs; i++) {
- QString requestedCodec = message->readString();
- qDebug() << "requestedCodec:" << requestedCodec;
- codecList.append(requestedCodec);
- }
- qDebug() << "all requested codecs:" << codecList;
-
- CodecPluginPointer selectedCoded;
- QString selectedCodecName;
+ QStringList availableCodecs;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
if (codecPlugins.size() > 0) {
for (auto& plugin : codecPlugins) {
- qDebug() << "Codec available:" << plugin->getName();
-
- // choose first codec
- if (!selectedCoded) {
- selectedCoded = plugin;
- selectedCodecName = plugin->getName();
- }
+ auto codecName = plugin->getName();
+ qDebug() << "Codec available:" << codecName;
+ availableCodecs.append(codecName);
}
} else {
qDebug() << "No Codecs available...";
}
+ CodecPluginPointer selectedCoded;
+ QString selectedCodecName;
+
+ QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
+
+ // read the codecs requested by the client
+ const int MAX_PREFERENCE = 99999;
+ int preferredCodecIndex = MAX_PREFERENCE;
+ QString preferredCodec;
+ quint8 numberOfCodecs = 0;
+ message->readPrimitive(&numberOfCodecs);
+ qDebug() << "numberOfCodecs:" << numberOfCodecs;
+ QStringList codecList;
+ for (quint16 i = 0; i < numberOfCodecs; i++) {
+ QString requestedCodec = message->readString();
+ int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec);
+ bool codecAvailable = availableCodecs.contains(requestedCodec);
+ qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable;
+ if (codecAvailable) {
+ codecList.append(requestedCodec);
+ if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) {
+ qDebug() << "This codec is preferred...";
+ selectedCodecName = requestedCodec;
+ preferredCodecIndex = preferenceOfThisCodec;
+ }
+ }
+ }
+ qDebug() << "all requested and available codecs:" << codecList;
+
+ // choose first codec
+ if (!selectedCodecName.isEmpty()) {
+ if (codecPlugins.size() > 0) {
+ for (auto& plugin : codecPlugins) {
+ if (selectedCodecName == plugin->getName()) {
+ qDebug() << "Selecting codec:" << selectedCodecName;
+ selectedCoded = plugin;
+ break;
+ }
+ }
+ }
+ }
+
+
auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
// FIXME - why would we not have client data at this point??
@@ -882,6 +909,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) {
if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) {
QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject();
+ const QString CODEC_PREFERENCE_ORDER = "codec_preference_order";
+ if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) {
+ _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString();
+ qDebug() << "Codec preference order changed to" << _codecPreferenceOrder;
+ }
+
const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance";
if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) {
bool ok = false;
diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h
index c90a918a5b..4b2a27120d 100644
--- a/assignment-client/src/audio/AudioMixer.h
+++ b/assignment-client/src/audio/AudioMixer.h
@@ -92,6 +92,8 @@ private:
int _manualEchoMixes { 0 };
int _totalMixes { 0 };
+ QString _codecPreferenceOrder;
+
float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json
index 7375a0f650..948c6ddc18 100644
--- a/domain-server/resources/describe-settings.json
+++ b/domain-server/resources/describe-settings.json
@@ -718,6 +718,14 @@
"placeholder": "(in percent)"
}
]
+ },
+ {
+ "name": "codec_preference_order",
+ "label": "Audio Codec Preference Order",
+ "help": "List of codec names in order of preferred usage",
+ "placeholder": "hifiAC, zlib, pcm",
+ "default": "hifiAC,zlib,pcm",
+ "advanced": true
}
]
},
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index a076b1b290..97ef4b2981 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -507,19 +507,14 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
}
void AudioClient::negotiateAudioFormat() {
- qDebug() << __FUNCTION__;
-
auto nodeList = DependencyManager::get<NodeList>();
-
auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
-
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
-
quint8 numberOfCodecs = (quint8)codecPlugins.size();
negotiateFormatPacket->writePrimitive(numberOfCodecs);
for (auto& plugin : codecPlugins) {
- qDebug() << "Codec available:" << plugin->getName();
- negotiateFormatPacket->writeString(plugin->getName());
+ auto codecName = plugin->getName();
+ negotiateFormatPacket->writeString(codecName);
}
// grab our audio mixer from the NodeList, if it exists
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index 5d4dbd9b94..7c6a34ae55 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
if (message.getType() == PacketType::SilentAudioFrame) {
writeDroppableSilentSamples(networkSamples);
} else {
- parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples);
+ parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()));
}
break;
}
@@ -177,25 +177,14 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
}
}
-int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
-
- // codec decode goes here
+int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
if (_codec) {
_codec->decode(packetAfterStreamProperties, decodedBuffer);
} else {
decodedBuffer = packetAfterStreamProperties;
}
-
auto actualSize = decodedBuffer.size();
-
- /*
- auto expectedSize = numAudioSamples * sizeof(int16_t);
- if (expectedSize != actualSize) {
- qDebug() << "DECODED SIZE NOT EXPECTED!!!! ----- buffer size:" << actualSize << "expected:" << expectedSize;
- }
- */
-
return _ringBuffer.writeData(decodedBuffer.data(), actualSize);
}
diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h
index ddc0dc1dc3..f9ca088fab 100644
--- a/libraries/audio/src/InboundAudioStream.h
+++ b/libraries/audio/src/InboundAudioStream.h
@@ -207,7 +207,7 @@ protected:
/// parses the audio data in the network packet.
/// default implementation assumes packet contains raw audio samples after stream properties
- virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+ virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
/// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer
virtual int writeDroppableSilentSamples(int silentSamples);
diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp
index 220b2bd9ee..6939ee540a 100644
--- a/libraries/audio/src/MixedProcessedAudioStream.cpp
+++ b/libraries/audio/src/MixedProcessedAudioStream.cpp
@@ -42,9 +42,7 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
return deviceSamplesWritten;
}
-int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
-
- // TODO - codec decode goes here
+int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
if (_codec) {
_codec->decode(packetAfterStreamProperties, decodedBuffer);
@@ -52,9 +50,6 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray&
decodedBuffer = packetAfterStreamProperties;
}
- qDebug() << __FUNCTION__ << "packetAfterStreamProperties:" << packetAfterStreamProperties.size() << "networkSamples:" << networkSamples << "decodedBuffer:" << decodedBuffer.size();
-
-
emit addedStereoSamples(decodedBuffer);
QByteArray outputBuffer;
diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h
index 5ea0157421..2f9a691278 100644
--- a/libraries/audio/src/MixedProcessedAudioStream.h
+++ b/libraries/audio/src/MixedProcessedAudioStream.h
@@ -35,7 +35,7 @@ public:
protected:
int writeDroppableSilentSamples(int silentSamples);
int writeLastFrameRepeatedWithFade(int samples);
- int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+ int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
private:
int networkToDeviceSamples(int networkSamples);
diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp
index dba241f221..0456e095ae 100644
--- a/libraries/networking/src/udt/BasePacket.cpp
+++ b/libraries/networking/src/udt/BasePacket.cpp
@@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) {
qint64 BasePacket::writeString(const QString& string) {
QByteArray data = string.toUtf8();
- writePrimitive(static_cast<uint32_t>(data.length()));
- return writeData(data.constData(), data.length());
+ uint32_t length = data.length();
+ writePrimitive(length);
+ auto result = writeData(data.constData(), data.length());
+ seek(pos() + length);
+ return length + sizeof(uint32_t);
}
QString BasePacket::readString() {
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index 9d55cafedf..f787c6682d 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -1,5 +1,5 @@
//
-// PCMCodecManager.cpp
+// PCMCodec.cpp
// plugins/pcmCodec/src
//
// Created by Brad Hefta-Gaub on 6/9/2016
@@ -15,35 +15,64 @@
#include "PCMCodecManager.h"
-const QString PCMCodecManager::NAME = "zlib";
+const QString PCMCodec::NAME = "pcm";
-void PCMCodecManager::init() {
+void PCMCodec::init() {
}
-void PCMCodecManager::deinit() {
+void PCMCodec::deinit() {
}
-bool PCMCodecManager::activate() {
+bool PCMCodec::activate() {
CodecPlugin::activate();
return true;
}
-void PCMCodecManager::deactivate() {
+void PCMCodec::deactivate() {
CodecPlugin::deactivate();
}
-bool PCMCodecManager::isSupported() const {
+bool PCMCodec::isSupported() const {
return true;
}
-void PCMCodecManager::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
- //decodedBuffer = encodedBuffer;
+void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
+ decodedBuffer = encodedBuffer;
+}
+
+void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+ encodedBuffer = decodedBuffer;
+}
+
+
+const QString zLibCodec::NAME = "zlib";
+
+void zLibCodec::init() {
+}
+
+void zLibCodec::deinit() {
+}
+
+bool zLibCodec::activate() {
+ CodecPlugin::activate();
+ return true;
+}
+
+void zLibCodec::deactivate() {
+ CodecPlugin::deactivate();
+}
+
+
+bool zLibCodec::isSupported() const {
+ return true;
+}
+
+void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
decodedBuffer = qUncompress(encodedBuffer);
}
-void PCMCodecManager::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
- //encodedBuffer = decodedBuffer;
+void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
encodedBuffer = qCompress(decodedBuffer);
}
diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h
index 3b7ca36c02..7816660c5d 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.h
+++ b/plugins/pcmCodec/src/PCMCodecManager.h
@@ -15,7 +15,7 @@
#include <plugins/CodecPlugin.h>
-class PCMCodecManager : public CodecPlugin {
+class PCMCodec : public CodecPlugin {
Q_OBJECT
public:
@@ -38,4 +38,27 @@ private:
static const QString NAME;
};
+class zLibCodec : public CodecPlugin {
+ Q_OBJECT
+
+public:
+ // Plugin functions
+ bool isSupported() const override;
+ const QString& getName() const override { return NAME; }
+
+ void init() override;
+ void deinit() override;
+
+ /// Called when a plugin is being activated for use. May be called multiple times.
+ bool activate() override;
+ /// Called when a plugin is no longer being used. May be called multiple times.
+ void deactivate() override;
+
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+
+private:
+ static const QString NAME;
+};
+
#endif // hifi__PCMCodecManager_h
diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp
index 732ed2d57d..351b1adf3f 100644
--- a/plugins/pcmCodec/src/PCMCodecProvider.cpp
+++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp
@@ -29,10 +29,17 @@ public:
virtual CodecPluginList getCodecPlugins() override {
static std::once_flag once;
std::call_once(once, [&] {
- CodecPluginPointer plugin(new PCMCodecManager());
- if (plugin->isSupported()) {
- _codecPlugins.push_back(plugin);
+
+ CodecPluginPointer pcmCodec(new PCMCodec());
+ if (pcmCodec->isSupported()) {
+ _codecPlugins.push_back(pcmCodec);
}
+
+ CodecPluginPointer zlibCodec(new zLibCodec());
+ if (zlibCodec->isSupported()) {
+ _codecPlugins.push_back(zlibCodec);
+ }
+
});
return _codecPlugins;
}
From ba6bb245956b4657b4d234afd342f40d50296f6c Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Sun, 10 Jul 2016 16:49:03 -0700
Subject: [PATCH 09/22] rework plugins to allow different decoder/encoder
instances per streams
---
assignment-client/src/audio/AudioMixer.cpp | 6 +-
.../src/audio/AudioMixerClientData.cpp | 11 ++++
.../src/audio/AudioMixerClientData.h | 3 +
libraries/audio-client/src/AudioClient.cpp | 10 +--
libraries/audio-client/src/AudioClient.h | 1 +
libraries/audio/src/InboundAudioStream.cpp | 4 +-
libraries/audio/src/InboundAudioStream.h | 1 +
.../audio/src/MixedProcessedAudioStream.cpp | 4 +-
libraries/plugins/src/plugins/CodecPlugin.h | 18 +++++-
plugins/pcmCodec/src/PCMCodecManager.cpp | 64 ++++++++++++++++---
plugins/pcmCodec/src/PCMCodecManager.h | 37 +++++++++--
11 files changed, 135 insertions(+), 24 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index f49b674dce..825f970f2e 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -533,12 +533,16 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
clientData->_codec = selectedCoded;
clientData->_selectedCodecName = selectedCodecName;
+ clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+ clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+
qDebug() << "selectedCodecName:" << selectedCodecName;
auto avatarAudioStream = clientData->getAvatarAudioStream();
if (avatarAudioStream) {
avatarAudioStream->_codec = selectedCoded;
avatarAudioStream->_selectedCodecName = selectedCodecName;
+ avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
}
auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
@@ -777,7 +781,7 @@ void AudioMixer::broadcastMixes() {
QByteArray decocedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
QByteArray encodedBuffer;
if (nodeData->_codec) {
- nodeData->_codec->encode(decocedBuffer, encodedBuffer);
+ nodeData->_encoder->encode(decocedBuffer, encodedBuffer);
} else {
encodedBuffer = decocedBuffer;
}
diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp
index 080a833c0e..3e9b72d60e 100644
--- a/assignment-client/src/audio/AudioMixerClientData.cpp
+++ b/assignment-client/src/audio/AudioMixerClientData.cpp
@@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) :
_frameToSendStats = distribution(numberGenerator);
}
+AudioMixerClientData::~AudioMixerClientData() {
+ if (_codec) {
+ _codec->releaseDecoder(_decoder);
+ _codec->releaseEncoder(_encoder);
+ }
+}
+
+
AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
QReadLocker readLocker { &_streamsLock };
@@ -104,6 +112,9 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings());
avatarAudioStream->_codec = _codec;
avatarAudioStream->_selectedCodecName = _selectedCodecName;
+ if (_codec) {
+ avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+ }
qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName;
auto emplaced = _audioStreams.emplace(
diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h
index 0b3b352e66..ee6f22ff3b 100644
--- a/assignment-client/src/audio/AudioMixerClientData.h
+++ b/assignment-client/src/audio/AudioMixerClientData.h
@@ -29,6 +29,7 @@ class AudioMixerClientData : public NodeData {
Q_OBJECT
public:
AudioMixerClientData(const QUuid& nodeID);
+ ~AudioMixerClientData();
using SharedStreamPointer = std::shared_ptr<PositionalAudioStream>;
using AudioStreamMap = std::unordered_map<QUuid, SharedStreamPointer>;
@@ -70,6 +71,8 @@ public:
// FIXME -- maybe make these private
CodecPluginPointer _codec;
QString _selectedCodecName;
+ Encoder* _encoder { nullptr }; // for outbound mixed stream
+ Decoder* _decoder { nullptr }; // for mic stream
signals:
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 97ef4b2981..db5376a380 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -536,6 +536,8 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> mess
for (auto& plugin : codecPlugins) {
if (_selectedCodecName == plugin->getName()) {
_receivedAudioStream._codec = _codec = plugin;
+ _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+ _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
qDebug() << "Selected Codec Plugin:" << _codec.get();
break;
}
@@ -815,8 +817,8 @@ void AudioClient::handleAudioInput() {
// TODO - codec encode goes here
QByteArray decocedBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
QByteArray encodedBuffer;
- if (_codec) {
- _codec->encode(decocedBuffer, encodedBuffer);
+ if (_encoder) {
+ _encoder->encode(decocedBuffer, encodedBuffer);
} else {
encodedBuffer = decocedBuffer;
}
@@ -833,8 +835,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
// TODO - codec encode goes here
QByteArray encodedBuffer;
- if (_codec) {
- _codec->encode(audio, encodedBuffer);
+ if (_encoder) {
+ _encoder->encode(audio, encodedBuffer);
} else {
encodedBuffer = audio;
}
diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h
index 66555d1e2d..bd0afe453d 100644
--- a/libraries/audio-client/src/AudioClient.h
+++ b/libraries/audio-client/src/AudioClient.h
@@ -299,6 +299,7 @@ private:
CodecPluginPointer _codec;
QString _selectedCodecName;
+ Encoder* _encoder { nullptr }; // for outbound mic stream
};
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index 7c6a34ae55..f47c6cce46 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -179,8 +179,8 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
- if (_codec) {
- _codec->decode(packetAfterStreamProperties, decodedBuffer);
+ if (_decoder) {
+ _decoder->decode(packetAfterStreamProperties, decodedBuffer);
} else {
decodedBuffer = packetAfterStreamProperties;
}
diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h
index f9ca088fab..57bff80fff 100644
--- a/libraries/audio/src/InboundAudioStream.h
+++ b/libraries/audio/src/InboundAudioStream.h
@@ -179,6 +179,7 @@ public:
// FIXME -- maybe make these private
CodecPluginPointer _codec;
QString _selectedCodecName;
+ Decoder* _decoder { nullptr };
public slots:
/// This function should be called every second for all the stats to function properly. If dynamic jitter buffers
diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp
index 6939ee540a..728deae0b1 100644
--- a/libraries/audio/src/MixedProcessedAudioStream.cpp
+++ b/libraries/audio/src/MixedProcessedAudioStream.cpp
@@ -44,8 +44,8 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
QByteArray decodedBuffer;
- if (_codec) {
- _codec->decode(packetAfterStreamProperties, decodedBuffer);
+ if (_decoder) {
+ _decoder->decode(packetAfterStreamProperties, decodedBuffer);
} else {
decodedBuffer = packetAfterStreamProperties;
}
diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h
index 6944d91bed..280853e37e 100644
--- a/libraries/plugins/src/plugins/CodecPlugin.h
+++ b/libraries/plugins/src/plugins/CodecPlugin.h
@@ -12,9 +12,23 @@
#include "Plugin.h"
-class CodecPlugin : public Plugin {
+class Encoder {
public:
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
};
+class Decoder {
+public:
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+
+ // numFrames - number of samples (mono) or sample-pairs (stereo)
+ virtual void trackLostFrames(int numFrames) = 0;
+};
+
+class CodecPlugin : public Plugin {
+public:
+ virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
+ virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
+ virtual void releaseEncoder(Encoder* encoder) = 0;
+ virtual void releaseDecoder(Decoder* decoder) = 0;
+};
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index f787c6682d..1cad1ea4e6 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -37,15 +37,54 @@ bool PCMCodec::isSupported() const {
return true;
}
+class PCMEncoder : public Encoder {
+public:
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer = decodedBuffer;
+ }
+};
-void PCMCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
- decodedBuffer = encodedBuffer;
+class PCMDecoder : public Decoder {
+public:
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer = encodedBuffer;
+ }
+
+ virtual void trackLostFrames(int numFrames) override { }
+};
+
+Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) {
+ return new PCMEncoder();
}
-void PCMCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
- encodedBuffer = decodedBuffer;
+Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) {
+ return new PCMDecoder();
}
+void PCMCodec::releaseEncoder(Encoder* encoder) {
+ delete encoder;
+}
+
+void PCMCodec::releaseDecoder(Decoder* decoder) {
+ delete decoder;
+}
+
+
+class zLibEncoder : public Encoder {
+public:
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer = qCompress(decodedBuffer);
+ }
+};
+
+class zLibDecoder : public Decoder {
+public:
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer = qUncompress(encodedBuffer);
+ }
+
+ virtual void trackLostFrames(int numFrames) override { }
+};
const QString zLibCodec::NAME = "zlib";
@@ -69,10 +108,19 @@ bool zLibCodec::isSupported() const {
return true;
}
-void zLibCodec::decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) {
- decodedBuffer = qUncompress(encodedBuffer);
+Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) {
+ return new zLibEncoder();
}
-void zLibCodec::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
- encodedBuffer = qCompress(decodedBuffer);
+Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) {
+ return new zLibDecoder();
}
+
+void zLibCodec::releaseEncoder(Encoder* encoder) {
+ delete encoder;
+}
+
+void zLibCodec::releaseDecoder(Decoder* decoder) {
+ delete decoder;
+}
+
diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h
index 7816660c5d..6eb7f44c15 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.h
+++ b/plugins/pcmCodec/src/PCMCodecManager.h
@@ -12,9 +12,31 @@
#ifndef hifi__PCMCodecManager_h
#define hifi__PCMCodecManager_h
-
#include <plugins/CodecPlugin.h>
+/*
+class Encoder {
+public:
+virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
+};
+
+class Decoder {
+public:
+virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+
+// numFrames - number of samples (mono) or sample-pairs (stereo)
+virtual void trackLostFrames(int numFrames) = 0;
+};
+
+class CodecPlugin : public Plugin {
+public:
+virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
+virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
+virtual void releaseEncoder(Encoder* encoder) = 0;
+virtual void releaseDecoder(Decoder* decoder) = 0;
+};
+*/
+
class PCMCodec : public CodecPlugin {
Q_OBJECT
@@ -31,8 +53,10 @@ public:
/// Called when a plugin is no longer being used. May be called multiple times.
void deactivate() override;
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
- virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+ virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+ virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+ virtual void releaseEncoder(Encoder* encoder) override;
+ virtual void releaseDecoder(Decoder* decoder) override;
private:
static const QString NAME;
@@ -54,8 +78,11 @@ public:
/// Called when a plugin is no longer being used. May be called multiple times.
void deactivate() override;
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override;
- virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override;
+ virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+ virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+ virtual void releaseEncoder(Encoder* encoder) override;
+ virtual void releaseDecoder(Decoder* decoder) override;
+
private:
static const QString NAME;
From ed9715ae5f25a828dca312740c9da33690653b96 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Sun, 10 Jul 2016 17:40:58 -0700
Subject: [PATCH 10/22] some cleanup, proper memory allocation/deallocation
---
assignment-client/src/audio/AudioMixer.cpp | 22 ++++----------
.../src/audio/AudioMixerClientData.cpp | 30 +++++++++++++++----
.../src/audio/AudioMixerClientData.h | 20 +++++++++----
libraries/audio-client/src/AudioClient.cpp | 15 ++++++++--
libraries/audio/src/InboundAudioStream.cpp | 18 +++++++++++
libraries/audio/src/InboundAudioStream.h | 11 ++++---
6 files changed, 81 insertions(+), 35 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 825f970f2e..2f1a5de309 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -478,7 +478,7 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
qDebug() << "No Codecs available...";
}
- CodecPluginPointer selectedCoded;
+ CodecPluginPointer selectedCodec;
QString selectedCodecName;
QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
@@ -513,7 +513,7 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
for (auto& plugin : codecPlugins) {
if (selectedCodecName == plugin->getName()) {
qDebug() << "Selecting codec:" << selectedCodecName;
- selectedCoded = plugin;
+ selectedCodec = plugin;
break;
}
}
@@ -531,18 +531,13 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);
}
- clientData->_codec = selectedCoded;
- clientData->_selectedCodecName = selectedCodecName;
- clientData->_encoder = selectedCoded->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
- clientData->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+ clientData->setupCodec(selectedCodec, selectedCodecName);
qDebug() << "selectedCodecName:" << selectedCodecName;
auto avatarAudioStream = clientData->getAvatarAudioStream();
if (avatarAudioStream) {
- avatarAudioStream->_codec = selectedCoded;
- avatarAudioStream->_selectedCodecName = selectedCodecName;
- avatarAudioStream->_decoder = selectedCoded->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+ avatarAudioStream->setupCodec(selectedCodec, selectedCodecName, AudioConstants::MONO);
}
auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
@@ -777,14 +772,9 @@ void AudioMixer::broadcastMixes() {
quint16 sequence = nodeData->getOutgoingSequenceNumber();
mixPacket->writePrimitive(sequence);
- // TODO - codec encode goes here
- QByteArray decocedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+ QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
QByteArray encodedBuffer;
- if (nodeData->_codec) {
- nodeData->_encoder->encode(decocedBuffer, encodedBuffer);
- } else {
- encodedBuffer = decocedBuffer;
- }
+ nodeData->encode(decodedBuffer, encodedBuffer);
// pack mixed audio samples
mixPacket->write(encodedBuffer.constData(), encodedBuffer.size());
diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp
index 3e9b72d60e..c733683e5a 100644
--- a/assignment-client/src/audio/AudioMixerClientData.cpp
+++ b/assignment-client/src/audio/AudioMixerClientData.cpp
@@ -110,12 +110,8 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
bool isStereo = channelFlag == 1;
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings());
- avatarAudioStream->_codec = _codec;
- avatarAudioStream->_selectedCodecName = _selectedCodecName;
- if (_codec) {
- avatarAudioStream->_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
- }
- qDebug() << "creating new AvatarAudioStream... codec:" << avatarAudioStream->_selectedCodecName;
+ avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO);
+ qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName;
auto emplaced = _audioStreams.emplace(
QUuid(),
@@ -340,3 +336,25 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() {
return result;
}
+
+void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) {
+ cleanupCodec(); // cleanup any previously allocated coders first
+ _codec = codec;
+ _selectedCodecName = codecName;
+ _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+ _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+}
+
+void AudioMixerClientData::cleanupCodec() {
+ // release any old codec encoder/decoder first...
+ if (_codec) {
+ if (_decoder) {
+ _codec->releaseDecoder(_decoder);
+ _decoder = nullptr;
+ }
+ if (_encoder) {
+ _codec->releaseEncoder(_encoder);
+ _encoder = nullptr;
+ }
+ }
+}
diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h
index ee6f22ff3b..da2bf8997c 100644
--- a/assignment-client/src/audio/AudioMixerClientData.h
+++ b/assignment-client/src/audio/AudioMixerClientData.h
@@ -68,12 +68,15 @@ public:
AudioLimiter audioLimiter;
- // FIXME -- maybe make these private
- CodecPluginPointer _codec;
- QString _selectedCodecName;
- Encoder* _encoder { nullptr }; // for outbound mixed stream
- Decoder* _decoder { nullptr }; // for mic stream
-
+ void setupCodec(CodecPluginPointer codec, const QString& codecName);
+ void cleanupCodec();
+ void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+ if (_encoder) {
+ _encoder->encode(decodedBuffer, encodedBuffer);
+ } else {
+ encodedBuffer = decodedBuffer;
+ }
+ }
signals:
void injectorStreamFinished(const QUuid& streamIdentifier);
@@ -91,6 +94,11 @@ private:
AudioStreamStats _downstreamAudioStreamStats;
int _frameToSendStats { 0 };
+
+ CodecPluginPointer _codec;
+ QString _selectedCodecName;
+ Encoder* _encoder{ nullptr }; // for outbound mixed stream
+ Decoder* _decoder{ nullptr }; // for mic stream
};
#endif // hifi_AudioMixerClientData_h
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index db5376a380..28fd95f599 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -140,6 +140,10 @@ AudioClient::AudioClient() :
AudioClient::~AudioClient() {
stop();
+ if (_codec && _encoder) {
+ _codec->releaseEncoder(_encoder);
+ _encoder = nullptr;
+ }
}
void AudioClient::reset() {
@@ -529,14 +533,19 @@ void AudioClient::negotiateAudioFormat() {
void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
qDebug() << __FUNCTION__;
- _receivedAudioStream._selectedCodecName = _selectedCodecName = message->readString();
+ _selectedCodecName = message->readString();
qDebug() << "Selected Codec:" << _selectedCodecName;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
for (auto& plugin : codecPlugins) {
if (_selectedCodecName == plugin->getName()) {
- _receivedAudioStream._codec = _codec = plugin;
- _receivedAudioStream._decoder = plugin->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+ // release any old codec encoder/decoder first...
+ if (_codec && _encoder) {
+ _codec->releaseEncoder(_encoder);
+ _encoder = nullptr;
+ }
+ _codec = plugin;
+ _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO);
_encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
qDebug() << "Selected Codec Plugin:" << _codec.get();
break;
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index f47c6cce46..144005bc11 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -504,3 +504,21 @@ float calculateRepeatedFrameFadeFactor(int indexOfRepeat) {
return 0.0f;
}
+void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels) {
+ cleanupCodec(); // cleanup any previously allocated coders first
+ _codec = codec;
+ _selectedCodecName = codecName;
+ if (_codec) {
+ _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels);
+ }
+}
+
+void InboundAudioStream::cleanupCodec() {
+ // release any old codec encoder/decoder first...
+ if (_codec) {
+ if (_decoder) {
+ _codec->releaseDecoder(_decoder);
+ _decoder = nullptr;
+ }
+ }
+}
diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h
index 57bff80fff..5da63f96c2 100644
--- a/libraries/audio/src/InboundAudioStream.h
+++ b/libraries/audio/src/InboundAudioStream.h
@@ -105,6 +105,7 @@ public:
public:
InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings);
+ ~InboundAudioStream() { cleanupCodec(); }
void reset();
virtual void resetStats();
@@ -176,10 +177,8 @@ public:
void setReverb(float reverbTime, float wetLevel);
void clearReverb() { _hasReverb = false; }
- // FIXME -- maybe make these private
- CodecPluginPointer _codec;
- QString _selectedCodecName;
- Decoder* _decoder { nullptr };
+ void setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels);
+ void cleanupCodec();
public slots:
/// This function should be called every second for all the stats to function properly. If dynamic jitter buffers
@@ -274,6 +273,10 @@ protected:
bool _hasReverb;
float _reverbTime;
float _wetLevel;
+
+ CodecPluginPointer _codec;
+ QString _selectedCodecName;
+ Decoder* _decoder{ nullptr };
};
float calculateRepeatedFrameFadeFactor(int indexOfRepeat);
From 7d608ba5925001be0c2a16f9b27425a20527adbb Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Sun, 10 Jul 2016 21:01:20 -0700
Subject: [PATCH 11/22] groundwork for injector support
---
assignment-client/src/audio/AudioMixer.cpp | 5 ----
.../src/audio/AudioMixerClientData.cpp | 26 ++++++++++++++++++-
libraries/audio/src/AudioInjector.cpp | 9 ++++++-
3 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 2f1a5de309..720f9d52de 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -535,11 +535,6 @@ void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> mess
qDebug() << "selectedCodecName:" << selectedCodecName;
- auto avatarAudioStream = clientData->getAvatarAudioStream();
- if (avatarAudioStream) {
- avatarAudioStream->setupCodec(selectedCodec, selectedCodecName, AudioConstants::MONO);
- }
-
auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
// write them to our packet
diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp
index c733683e5a..182f443225 100644
--- a/assignment-client/src/audio/AudioMixerClientData.cpp
+++ b/assignment-client/src/audio/AudioMixerClientData.cpp
@@ -142,9 +142,16 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
if (streamIt == _audioStreams.end()) {
// we don't have this injected stream yet, so add it
+ auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings());
+
+#if INJECTORS_SUPPORT_CODECS
+ injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
+ qDebug() << "creating new injectorStream... codec:" << _selectedCodecName;
+#endif
+
auto emplaced = _audioStreams.emplace(
streamIdentifier,
- std::unique_ptr<InjectedAudioStream> { new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()) }
+ std::unique_ptr<InjectedAudioStream> { injectorStream }
);
streamIt = emplaced.first;
@@ -343,6 +350,23 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c
_selectedCodecName = codecName;
_encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
_decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+
+ auto avatarAudioStream = getAvatarAudioStream();
+ if (avatarAudioStream) {
+ avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO);
+ }
+
+#if INJECTORS_SUPPORT_CODECS
+ // fixup codecs for any active injectors...
+ auto it = _audioStreams.begin();
+ while (it != _audioStreams.end()) {
+ SharedStreamPointer stream = it->second;
+ if (stream->getType() == PositionalAudioStream::Injector) {
+ stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
+ }
+ ++it;
+ }
+#endif
}
void AudioMixerClientData::cleanupCodec() {
diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp
index 878a4c627c..fee33dcb92 100644
--- a/libraries/audio/src/AudioInjector.cpp
+++ b/libraries/audio/src/AudioInjector.cpp
@@ -289,16 +289,23 @@ int64_t AudioInjector::injectNextFrame() {
_currentPacket->seek(audioDataOffset);
+ // This code is copying bytes from the _audioData directly into the packet, handling looping appropriately.
+ // Might be a reasonable place to do the encode step here.
+ QByteArray decodedAudio;
while (totalBytesLeftToCopy > 0) {
int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset);
- _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy);
+ decodedAudio.append(_audioData.data() + _currentSendOffset, bytesToCopy);
_currentSendOffset += bytesToCopy;
totalBytesLeftToCopy -= bytesToCopy;
if (_options.loop && _currentSendOffset >= _audioData.size()) {
_currentSendOffset = 0;
}
}
+ // FIXME -- good place to call codec encode here. We need to figure out how to tell the AudioInjector which
+ // codec to use... possible through AbstractAudioInterface.
+ QByteArray encodedAudio = decodedAudio;
+ _currentPacket->write(encodedAudio.data(), encodedAudio.size());
// set the correct size used for this packet
_currentPacket->setPayloadSize(_currentPacket->pos());
From f5b693cebb0574486a61a26157d5e77caf96da3a Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 12:22:27 -0700
Subject: [PATCH 12/22] added support for external hificodec and hifi codec
plugin
---
cmake/externals/hifiAudioCodec/CMakeLists.txt | 50 +++++++++++++
plugins/hifiCodec/CMakeLists.txt | 21 ++++++
plugins/hifiCodec/src/HiFiCodec.cpp | 72 +++++++++++++++++++
plugins/hifiCodec/src/HiFiCodec.h | 42 +++++++++++
plugins/hifiCodec/src/HiFiCodecProvider.cpp | 46 ++++++++++++
plugins/hifiCodec/src/plugin.json | 1 +
6 files changed, 232 insertions(+)
create mode 100644 cmake/externals/hifiAudioCodec/CMakeLists.txt
create mode 100644 plugins/hifiCodec/CMakeLists.txt
create mode 100644 plugins/hifiCodec/src/HiFiCodec.cpp
create mode 100644 plugins/hifiCodec/src/HiFiCodec.h
create mode 100644 plugins/hifiCodec/src/HiFiCodecProvider.cpp
create mode 100644 plugins/hifiCodec/src/plugin.json
diff --git a/cmake/externals/hifiAudioCodec/CMakeLists.txt b/cmake/externals/hifiAudioCodec/CMakeLists.txt
new file mode 100644
index 0000000000..5c563774b7
--- /dev/null
+++ b/cmake/externals/hifiAudioCodec/CMakeLists.txt
@@ -0,0 +1,50 @@
+include(ExternalProject)
+include(SelectLibraryConfigurations)
+
+set(EXTERNAL_NAME HiFiAudioCodec)
+
+string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER)
+
+ExternalProject_Add(
+ ${EXTERNAL_NAME}
+ URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK.zip
+ URL_MD5 4add25b7cc5dfdb3cbfc1156b95cfff7
+ CONFIGURE_COMMAND ""
+ BUILD_COMMAND ""
+ INSTALL_COMMAND ""
+ LOG_DOWNLOAD 1
+)
+
+# Hide this external target (for ide users)
+set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals")
+
+ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR)
+
+set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNAL)
+
+if (WIN32)
+ set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL)
+
+ # FIXME need to account for different architectures
+ #if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+ # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win64/audio.lib CACHE TYPE INTERNAL)
+ # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win64)
+ #else()
+ # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win32/audio.lib CACHE TYPE INTERNAL)
+ # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win32)
+ #endif()
+
+elseif(APPLE)
+
+ # FIXME need to account for different architectures
+ #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/osx32/audio.dylib CACHE TYPE INTERNAL)
+ #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/osx32)
+
+elseif(NOT ANDROID)
+
+ # FIXME need to account for different architectures
+ #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL)
+ #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/linux64)
+
+endif()
+
diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt
new file mode 100644
index 0000000000..50ba6de54e
--- /dev/null
+++ b/plugins/hifiCodec/CMakeLists.txt
@@ -0,0 +1,21 @@
+#
+# Created by Brad Hefta-Gaub on 7/10/2016
+# Copyright 2016 High Fidelity, Inc.
+#
+# Distributed under the Apache License, Version 2.0.
+# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+
+
+
+if (WIN32)
+ set(TARGET_NAME hifiCodec)
+ setup_hifi_client_server_plugin()
+
+ link_hifi_libraries(shared plugins)
+
+ add_dependency_external_projects(HiFiAudioCodec)
+ target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS})
+ target_link_libraries(${TARGET_NAME} ${HIFIAUDIOCODEC_LIBRARIES})
+endif()
+
diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp
new file mode 100644
index 0000000000..03a0b04bb8
--- /dev/null
+++ b/plugins/hifiCodec/src/HiFiCodec.cpp
@@ -0,0 +1,72 @@
+//
+// HiFiCodec.cpp
+// plugins/hifiCodec/src
+//
+// Created by Brad Hefta-Gaub on 7/10/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <qapplication.h>
+
+#include <AudioCodec.h> // should be from the external...
+
+#include <PerfStat.h>
+
+#include "HiFiCodec.h"
+
+const QString HiFiCodec::NAME = "hifiAC";
+
+void HiFiCodec::init() {
+}
+
+void HiFiCodec::deinit() {
+}
+
+bool HiFiCodec::activate() {
+ CodecPlugin::activate();
+ return true;
+}
+
+void HiFiCodec::deactivate() {
+ CodecPlugin::deactivate();
+}
+
+
+bool HiFiCodec::isSupported() const {
+ return true;
+}
+
+class HiFiEncoder : public Encoder {
+public:
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer = decodedBuffer;
+ }
+};
+
+class HiFiDecoder : public Decoder {
+public:
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer = encodedBuffer;
+ }
+
+ virtual void trackLostFrames(int numFrames) override { }
+};
+
+Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) {
+ return new HiFiEncoder();
+}
+
+Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) {
+ return new HiFiDecoder();
+}
+
+void HiFiCodec::releaseEncoder(Encoder* encoder) {
+ delete encoder;
+}
+
+void HiFiCodec::releaseDecoder(Decoder* decoder) {
+ delete decoder;
+}
diff --git a/plugins/hifiCodec/src/HiFiCodec.h b/plugins/hifiCodec/src/HiFiCodec.h
new file mode 100644
index 0000000000..eeba8d56d8
--- /dev/null
+++ b/plugins/hifiCodec/src/HiFiCodec.h
@@ -0,0 +1,42 @@
+//
+// HiFiCodec.h
+// plugins/hifiCodec/src
+//
+// Created by Brad Hefta-Gaub on 7/10/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_HiFiCodec_h
+#define hifi_HiFiCodec_h
+
+#include <plugins/CodecPlugin.h>
+
+class HiFiCodec : public CodecPlugin {
+ Q_OBJECT
+
+public:
+ // Plugin functions
+ bool isSupported() const override;
+ const QString& getName() const override { return NAME; }
+
+ void init() override;
+ void deinit() override;
+
+ /// Called when a plugin is being activated for use. May be called multiple times.
+ bool activate() override;
+ /// Called when a plugin is no longer being used. May be called multiple times.
+ void deactivate() override;
+
+ virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+ virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+ virtual void releaseEncoder(Encoder* encoder) override;
+ virtual void releaseDecoder(Decoder* decoder) override;
+
+private:
+ static const QString NAME;
+};
+
+#endif // hifi_HiFiCodec_h
diff --git a/plugins/hifiCodec/src/HiFiCodecProvider.cpp b/plugins/hifiCodec/src/HiFiCodecProvider.cpp
new file mode 100644
index 0000000000..b9698cc821
--- /dev/null
+++ b/plugins/hifiCodec/src/HiFiCodecProvider.cpp
@@ -0,0 +1,46 @@
+//
+// Created by Brad Hefta-Gaub on 7/10/2016
+// Copyright 2016 High Fidelity, Inc.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <mutex>
+
+#include <QtCore/QObject>
+#include <QtCore/QtPlugin>
+#include <QtCore/QStringList>
+
+#include <plugins/RuntimePlugin.h>
+#include <plugins/CodecPlugin.h>
+
+#include "HiFiCodec.h"
+
+class HiFiCodecProvider : public QObject, public CodecProvider {
+ Q_OBJECT
+ Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json")
+ Q_INTERFACES(CodecProvider)
+
+public:
+ HiFiCodecProvider(QObject* parent = nullptr) : QObject(parent) {}
+ virtual ~HiFiCodecProvider() {}
+
+ virtual CodecPluginList getCodecPlugins() override {
+ static std::once_flag once;
+ std::call_once(once, [&] {
+
+ CodecPluginPointer hiFiCodec(new HiFiCodec());
+ if (hiFiCodec->isSupported()) {
+ _codecPlugins.push_back(hiFiCodec);
+ }
+
+ });
+ return _codecPlugins;
+ }
+
+private:
+ CodecPluginList _codecPlugins;
+};
+
+#include "HiFiCodecProvider.moc"
diff --git a/plugins/hifiCodec/src/plugin.json b/plugins/hifiCodec/src/plugin.json
new file mode 100644
index 0000000000..df26a67ea8
--- /dev/null
+++ b/plugins/hifiCodec/src/plugin.json
@@ -0,0 +1 @@
+{"name":"HiFi 4:1 Audio Codec"}
From 540ab9f07ca60357f57cd97bfe8f6562c46775fc Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 13:00:58 -0700
Subject: [PATCH 13/22] wire up audio codec
---
plugins/hifiCodec/CMakeLists.txt | 2 +-
plugins/hifiCodec/src/HiFiCodec.cpp | 42 ++++++++++++++++++++++-------
2 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt
index 50ba6de54e..da964bfdfd 100644
--- a/plugins/hifiCodec/CMakeLists.txt
+++ b/plugins/hifiCodec/CMakeLists.txt
@@ -12,7 +12,7 @@ if (WIN32)
set(TARGET_NAME hifiCodec)
setup_hifi_client_server_plugin()
- link_hifi_libraries(shared plugins)
+ link_hifi_libraries(audio shared plugins)
add_dependency_external_projects(HiFiAudioCodec)
target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS})
diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp
index 03a0b04bb8..3bb17a0275 100644
--- a/plugins/hifiCodec/src/HiFiCodec.cpp
+++ b/plugins/hifiCodec/src/HiFiCodec.cpp
@@ -11,7 +11,8 @@
#include <qapplication.h>
-#include <AudioCodec.h> // should be from the external...
+#include <AudioCodec.h>
+#include <AudioConstants.h>
#include <PerfStat.h>
@@ -39,28 +40,49 @@ bool HiFiCodec::isSupported() const {
return true;
}
-class HiFiEncoder : public Encoder {
+class HiFiEncoder : public Encoder, public AudioEncoder {
public:
- virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
- encodedBuffer = decodedBuffer;
+ HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) {
+ qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels;
+ _encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4; // codec reduces by 1/4th
}
+
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer.resize(_encodedSize);
+ AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+ }
+private:
+ int _encodedSize;
};
-class HiFiDecoder : public Decoder {
+class HiFiDecoder : public Decoder, public AudioDecoder {
public:
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
- decodedBuffer = encodedBuffer;
+ HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) {
+ qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels;
+ _decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels;
}
- virtual void trackLostFrames(int numFrames) override { }
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer.resize(_decodedSize);
+ AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true);
+ }
+
+ virtual void trackLostFrames(int numFrames) override {
+ QByteArray encodedBuffer;
+ QByteArray decodedBuffer;
+ decodedBuffer.resize(_decodedSize);
+ AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
+ }
+private:
+ int _decodedSize;
};
Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) {
- return new HiFiEncoder();
+ return new HiFiEncoder(sampleRate, numChannels);
}
Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) {
- return new HiFiDecoder();
+ return new HiFiDecoder(sampleRate, numChannels);
}
void HiFiCodec::releaseEncoder(Encoder* encoder) {
From 743576134f687322fc44911ebe861d0a2c73eb6d Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 13:38:08 -0700
Subject: [PATCH 14/22] fix warnings
---
libraries/networking/src/udt/BasePacket.cpp | 2 +-
plugins/pcmCodec/src/PCMCodecManager.cpp | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp
index 0456e095ae..18552ca966 100644
--- a/libraries/networking/src/udt/BasePacket.cpp
+++ b/libraries/networking/src/udt/BasePacket.cpp
@@ -154,7 +154,7 @@ qint64 BasePacket::writeString(const QString& string) {
QByteArray data = string.toUtf8();
uint32_t length = data.length();
writePrimitive(length);
- auto result = writeData(data.constData(), data.length());
+ writeData(data.constData(), data.length());
seek(pos() + length);
return length + sizeof(uint32_t);
}
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index 1cad1ea4e6..4a2380717d 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -62,11 +62,11 @@ Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) {
}
void PCMCodec::releaseEncoder(Encoder* encoder) {
- delete encoder;
+ delete static_cast<PCMEncoder*>(encoder);
}
void PCMCodec::releaseDecoder(Decoder* decoder) {
- delete decoder;
+ delete static_cast<PCMDecoder*>(decoder);
}
@@ -117,10 +117,10 @@ Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) {
}
void zLibCodec::releaseEncoder(Encoder* encoder) {
- delete encoder;
+ delete static_cast<zLibEncoder*>(encoder);
}
void zLibCodec::releaseDecoder(Decoder* decoder) {
- delete decoder;
+ delete static_cast<zLibDecoder*>(decoder);
}
From 125aa6b337c303a24f7183709e27cc0ea07d4272 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 14:21:32 -0700
Subject: [PATCH 15/22] make zlib and pcm codecs not actually allocate
encoder/decoders
---
libraries/plugins/src/plugins/CodecPlugin.h | 2 +
plugins/pcmCodec/src/PCMCodecManager.cpp | 48 ++++-----------------
plugins/pcmCodec/src/PCMCodecManager.h | 45 +++++++++----------
3 files changed, 30 insertions(+), 65 deletions(-)
diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h
index 280853e37e..404f05e860 100644
--- a/libraries/plugins/src/plugins/CodecPlugin.h
+++ b/libraries/plugins/src/plugins/CodecPlugin.h
@@ -14,11 +14,13 @@
class Encoder {
public:
+ virtual ~Encoder() { }
virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
};
class Decoder {
public:
+ virtual ~Decoder() { }
virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
// numFrames - number of samples (mono) or sample-pairs (stereo)
diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp
index 4a2380717d..315d0622ab 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.cpp
+++ b/plugins/pcmCodec/src/PCMCodecManager.cpp
@@ -37,55 +37,24 @@ bool PCMCodec::isSupported() const {
return true;
}
-class PCMEncoder : public Encoder {
-public:
- virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
- encodedBuffer = decodedBuffer;
- }
-};
-class PCMDecoder : public Decoder {
-public:
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
- decodedBuffer = encodedBuffer;
- }
-
- virtual void trackLostFrames(int numFrames) override { }
-};
Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) {
- return new PCMEncoder();
+ return this;
}
Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) {
- return new PCMDecoder();
+ return this;
}
void PCMCodec::releaseEncoder(Encoder* encoder) {
- delete static_cast<PCMEncoder*>(encoder);
+ // do nothing
}
void PCMCodec::releaseDecoder(Decoder* decoder) {
- delete static_cast<PCMDecoder*>(decoder);
+ // do nothing
}
-
-class zLibEncoder : public Encoder {
-public:
- virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
- encodedBuffer = qCompress(decodedBuffer);
- }
-};
-
-class zLibDecoder : public Decoder {
-public:
- virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
- decodedBuffer = qUncompress(encodedBuffer);
- }
-
- virtual void trackLostFrames(int numFrames) override { }
-};
-
const QString zLibCodec::NAME = "zlib";
void zLibCodec::init() {
@@ -103,24 +72,23 @@ void zLibCodec::deactivate() {
CodecPlugin::deactivate();
}
-
bool zLibCodec::isSupported() const {
return true;
}
Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) {
- return new zLibEncoder();
+ return this;
}
Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) {
- return new zLibDecoder();
+ return this;
}
void zLibCodec::releaseEncoder(Encoder* encoder) {
- delete static_cast<zLibEncoder*>(encoder);
+ // do nothing... it wasn't allocated
}
void zLibCodec::releaseDecoder(Decoder* decoder) {
- delete static_cast<zLibDecoder*>(decoder);
+ // do nothing... it wasn't allocated
}
diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h
index 6eb7f44c15..55d7c866f1 100644
--- a/plugins/pcmCodec/src/PCMCodecManager.h
+++ b/plugins/pcmCodec/src/PCMCodecManager.h
@@ -14,30 +14,7 @@
#include <plugins/CodecPlugin.h>
-/*
-class Encoder {
-public:
-virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
-};
-
-class Decoder {
-public:
-virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
-
-// numFrames - number of samples (mono) or sample-pairs (stereo)
-virtual void trackLostFrames(int numFrames) = 0;
-};
-
-class CodecPlugin : public Plugin {
-public:
-virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
-virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
-virtual void releaseEncoder(Encoder* encoder) = 0;
-virtual void releaseDecoder(Decoder* decoder) = 0;
-};
-*/
-
-class PCMCodec : public CodecPlugin {
+class PCMCodec : public CodecPlugin, public Encoder, public Decoder {
Q_OBJECT
public:
@@ -58,11 +35,20 @@ public:
virtual void releaseEncoder(Encoder* encoder) override;
virtual void releaseDecoder(Decoder* decoder) override;
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer = decodedBuffer;
+ }
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer = encodedBuffer;
+ }
+
+ virtual void trackLostFrames(int numFrames) override { }
+
private:
static const QString NAME;
};
-class zLibCodec : public CodecPlugin {
+class zLibCodec : public CodecPlugin, public Encoder, public Decoder {
Q_OBJECT
public:
@@ -83,6 +69,15 @@ public:
virtual void releaseEncoder(Encoder* encoder) override;
virtual void releaseDecoder(Decoder* decoder) override;
+ virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+ encodedBuffer = qCompress(decodedBuffer);
+ }
+
+ virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+ decodedBuffer = qUncompress(encodedBuffer);
+ }
+
+ virtual void trackLostFrames(int numFrames) override { }
private:
static const QString NAME;
From ceffa219b3bec454f175eed596aef5b897b5ada6 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 15:18:02 -0700
Subject: [PATCH 16/22] support for lost packets
---
libraries/audio/src/InboundAudioStream.cpp | 6 +++++-
plugins/hifiCodec/src/HiFiCodec.cpp | 3 +--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp
index 144005bc11..c9b9363b1b 100644
--- a/libraries/audio/src/InboundAudioStream.cpp
+++ b/libraries/audio/src/InboundAudioStream.cpp
@@ -172,7 +172,7 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
return sizeof(quint16);
} else {
// mixed audio packets do not have any info between the seq num and the audio data.
- numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t);
+ numAudioSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
return 0;
}
}
@@ -189,6 +189,10 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet
}
int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) {
+ if (_decoder) {
+ _decoder->trackLostFrames(silentSamples);
+ }
+
// calculate how many silent frames we should drop.
int samplesPerFrame = _ringBuffer.getNumFrameSamples();
int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp
index 3bb17a0275..4e9336ff90 100644
--- a/plugins/hifiCodec/src/HiFiCodec.cpp
+++ b/plugins/hifiCodec/src/HiFiCodec.cpp
@@ -43,7 +43,6 @@ bool HiFiCodec::isSupported() const {
class HiFiEncoder : public Encoder, public AudioEncoder {
public:
HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) {
- qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels;
_encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4; // codec reduces by 1/4th
}
@@ -58,7 +57,6 @@ private:
class HiFiDecoder : public Decoder, public AudioDecoder {
public:
HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) {
- qDebug() << __FUNCTION__ << "sampleRate:" << sampleRate << "numChannels:" << numChannels;
_decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels;
}
@@ -71,6 +69,7 @@ public:
QByteArray encodedBuffer;
QByteArray decodedBuffer;
decodedBuffer.resize(_decodedSize);
+ // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean
AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
}
private:
From 3ddac737182b2a54bbcadc2b3f85dee5a88d5a3f Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 18:24:22 -0700
Subject: [PATCH 17/22] add mac lib to the external project
---
cmake/externals/hifiAudioCodec/CMakeLists.txt | 23 +++----------------
1 file changed, 3 insertions(+), 20 deletions(-)
diff --git a/cmake/externals/hifiAudioCodec/CMakeLists.txt b/cmake/externals/hifiAudioCodec/CMakeLists.txt
index 5c563774b7..3a8c714d79 100644
--- a/cmake/externals/hifiAudioCodec/CMakeLists.txt
+++ b/cmake/externals/hifiAudioCodec/CMakeLists.txt
@@ -7,8 +7,8 @@ string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER)
ExternalProject_Add(
${EXTERNAL_NAME}
- URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK.zip
- URL_MD5 4add25b7cc5dfdb3cbfc1156b95cfff7
+ URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK-1.zip
+ URL_MD5 23ec3fe51eaa155ea159a4971856fc13
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
@@ -24,27 +24,10 @@ set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNA
if (WIN32)
set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL)
-
- # FIXME need to account for different architectures
- #if ("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
- # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win64/audio.lib CACHE TYPE INTERNAL)
- # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win64)
- #else()
- # set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/win32/audio.lib CACHE TYPE INTERNAL)
- # add_paths_to_fixup_libs(${SOURCE_DIR}/bin/win32)
- #endif()
-
elseif(APPLE)
-
- # FIXME need to account for different architectures
- #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/osx32/audio.dylib CACHE TYPE INTERNAL)
- #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/osx32)
-
+ set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/libaudio.a CACHE TYPE INTERNAL)
elseif(NOT ANDROID)
-
# FIXME need to account for different architectures
#set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL)
- #add_paths_to_fixup_libs(${SOURCE_DIR}/bin/linux64)
-
endif()
From 6f0967f3fc444ebbe1e7a6b013a0c9182f6209be Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Mon, 11 Jul 2016 18:28:04 -0700
Subject: [PATCH 18/22] add mac support to the codec plugin
---
plugins/hifiCodec/CMakeLists.txt | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt
index da964bfdfd..3939529c3e 100644
--- a/plugins/hifiCodec/CMakeLists.txt
+++ b/plugins/hifiCodec/CMakeLists.txt
@@ -6,9 +6,7 @@
# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
#
-
-
-if (WIN32)
+if (WIN32 OR APPLE)
set(TARGET_NAME hifiCodec)
setup_hifi_client_server_plugin()
From 71b6210c4ea812b2cc0d802ba1096bbd417383a7 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Tue, 12 Jul 2016 10:18:37 -0700
Subject: [PATCH 19/22] removed some log spam
---
assignment-client/src/audio/AudioMixer.cpp | 2 --
libraries/audio-client/src/AudioClient.cpp | 2 --
2 files changed, 4 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp
index 720f9d52de..51c4e25410 100644
--- a/assignment-client/src/audio/AudioMixer.cpp
+++ b/assignment-client/src/audio/AudioMixer.cpp
@@ -464,8 +464,6 @@ void saveInputPluginSettings(const InputPluginList& plugins) {
void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
- qDebug() << __FUNCTION__;
-
QStringList availableCodecs;
auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
if (codecPlugins.size() > 0) {
diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp
index 28fd95f599..0cd01bb579 100644
--- a/libraries/audio-client/src/AudioClient.cpp
+++ b/libraries/audio-client/src/AudioClient.cpp
@@ -531,8 +531,6 @@ void AudioClient::negotiateAudioFormat() {
}
void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
- qDebug() << __FUNCTION__;
-
_selectedCodecName = message->readString();
qDebug() << "Selected Codec:" << _selectedCodecName;
From 2b719e12f72cc8892778ff174babec5531634bd5 Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Tue, 12 Jul 2016 10:54:45 -0700
Subject: [PATCH 20/22] attempt to make mac server plugins work
---
cmake/macros/SetupHifiClientServerPlugin.cmake | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/cmake/macros/SetupHifiClientServerPlugin.cmake b/cmake/macros/SetupHifiClientServerPlugin.cmake
index 5ace348e87..dfe3113fbc 100644
--- a/cmake/macros/SetupHifiClientServerPlugin.cmake
+++ b/cmake/macros/SetupHifiClientServerPlugin.cmake
@@ -12,17 +12,22 @@ macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN)
set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins")
if (APPLE)
- set(PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns")
+ set(CLIENT_PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns")
+ set(SERVER_PLUGIN_PATH "Components.app/Contents/PlugIns")
else()
- set(PLUGIN_PATH "plugins")
+ set(CLIENT_PLUGIN_PATH "plugins")
+ set(SERVER_PLUGIN_PATH "plugins")
endif()
if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles")
- set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${PLUGIN_PATH}/")
- set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${PLUGIN_PATH}/")
+ set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${CLIENT_PLUGIN_PATH}/")
+ set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${SERVER_PLUGIN_PATH}/")
+ elseif (APPLE)
+ set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${CLIENT_PLUGIN_PATH}/")
+ set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${SERVER_PLUGIN_PATH}/")
else()
- set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${PLUGIN_PATH}/")
- set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${PLUGIN_PATH}/")
+ set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${CLIENT_PLUGIN_PATH}/")
+ set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${SERVER_PLUGIN_PATH}/")
endif()
# create the destination for the client plugin binaries
From 6d7c8d5759562e74aa71200185df616ffe500e2c Mon Sep 17 00:00:00 2001
From: Brad Hefta-Gaub <brad@highfidelity.io>
Date: Tue, 12 Jul 2016 16:11:43 -0700
Subject: [PATCH 21/22] fix crash in audio-mixer if no plugins installed
---
assignment-client/src/audio/AudioMixerClientData.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp
index 182f443225..5c2ce8bf57 100644
--- a/assignment-client/src/audio/AudioMixerClientData.cpp
+++ b/assignment-client/src/audio/AudioMixerClientData.cpp
@@ -348,8 +348,10 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c
cleanupCodec(); // cleanup any previously allocated coders first
_codec = codec;
_selectedCodecName = codecName;
- _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
- _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+ if (codec) {
+ _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+ _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+ }
auto avatarAudioStream = getAvatarAudioStream();
if (avatarAudioStream) {
From fcd59ea94f122c8644bf5a95cc7246cbf5ff382d Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Tue, 12 Jul 2016 17:44:23 -0700
Subject: [PATCH 22/22] Update target number of processing threads to be based
on system
---
interface/src/Application.cpp | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 5d5bf45dcf..d97dee6fd1 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -169,7 +169,12 @@ static QTimer identityPacketTimer;
static QTimer pingTimer;
static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16;
-static const int PROCESSING_THREAD_POOL_SIZE = 6;
+
+// For processing on QThreadPool, target 2 less than the ideal number of threads, leaving
+// 2 logical cores available for time sensitive tasks.
+static const int MIN_PROCESSING_THREAD_POOL_SIZE = 2;
+static const int PROCESSING_THREAD_POOL_SIZE = std::max(MIN_PROCESSING_THREAD_POOL_SIZE,
+ QThread::idealThreadCount() - 2);
static const QString SNAPSHOT_EXTENSION = ".jpg";
static const QString SVO_EXTENSION = ".svo";
@@ -514,12 +519,6 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
PluginContainer* pluginContainer = dynamic_cast<PluginContainer*>(this); // set the container for any plugins that care
PluginManager::getInstance()->setContainer(pluginContainer);
- // FIXME this may be excessively conservative. On the other hand
- // maybe I'm used to having an 8-core machine
- // Perhaps find the ideal thread count and subtract 2 or 3
- // (main thread, present thread, random OS load)
- // More threads == faster concurrent loads, but also more concurrent
- // load on the GPU until we can serialize GPU transfers (off the main thread)
QThreadPool::globalInstance()->setMaxThreadCount(PROCESSING_THREAD_POOL_SIZE);
thread()->setPriority(QThread::HighPriority);
thread()->setObjectName("Main Thread");