diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index 1b5840c3c8..d0fd2c1176 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets) link_hifi_libraries( audio avatars octree gpu model fbx entities networking animation recording shared script-engine embedded-webserver - controllers physics + controllers physics plugins ) if (WIN32) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 03bb32cd53..51c4e25410 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include #include @@ -90,6 +92,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::AudioStreamStats }, this, "handleNodeAudioPacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -446,6 +449,99 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer mes } } +DisplayPluginList getDisplayPlugins() { + DisplayPluginList result; + return result; +} + +InputPluginList getInputPlugins() { + InputPluginList result; + return result; +} + +void saveInputPluginSettings(const InputPluginList& plugins) { +} + + +void AudioMixer::handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode) { + QStringList availableCodecs; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + auto codecName = plugin->getName(); + qDebug() << "Codec available:" << codecName; + availableCodecs.append(codecName); + } + } else { + qDebug() << "No Codecs available..."; + } + + CodecPluginPointer selectedCodec; + QString selectedCodecName; + + QStringList codecPreferenceList = _codecPreferenceOrder.split(","); + + // read the codecs requested by the client + const int MAX_PREFERENCE = 99999; + int preferredCodecIndex = MAX_PREFERENCE; + QString preferredCodec; + quint8 numberOfCodecs = 0; + message->readPrimitive(&numberOfCodecs); + qDebug() << "numberOfCodecs:" << numberOfCodecs; + QStringList codecList; + for (quint16 i = 0; i < numberOfCodecs; i++) { + QString requestedCodec = message->readString(); + int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec); + bool codecAvailable = availableCodecs.contains(requestedCodec); + qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable; + if (codecAvailable) { + codecList.append(requestedCodec); + if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) { + qDebug() << "This codec is preferred..."; + selectedCodecName = requestedCodec; + preferredCodecIndex = preferenceOfThisCodec; + } + } + } + qDebug() << "all requested and available codecs:" << codecList; + + // choose first codec + if (!selectedCodecName.isEmpty()) { + if (codecPlugins.size() > 0) { + for (auto& plugin : codecPlugins) { + if (selectedCodecName == plugin->getName()) { + qDebug() << "Selecting codec:" << selectedCodecName; + selectedCodec = plugin; + break; + } + } + } + } + + + auto clientData = dynamic_cast(sendingNode->getLinkedData()); + + // FIXME - why would we not have client data at this point?? + if (!clientData) { + qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__; + sendingNode->setLinkedData(std::unique_ptr { new AudioMixerClientData(sendingNode->getUUID()) }); + clientData = dynamic_cast(sendingNode->getLinkedData()); + connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); + } + + clientData->setupCodec(selectedCodec, selectedCodecName); + + qDebug() << "selectedCodecName:" << selectedCodecName; + + auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat); + + // write them to our packet + replyPacket->writeString(selectedCodecName); + + auto nodeList = DependencyManager::get(); + nodeList->sendPacket(std::move(replyPacket), *sendingNode); +} + void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { // enumerate the connected listeners to remove HRTF objects for the disconnected node auto nodeList = DependencyManager::get(); @@ -669,9 +765,12 @@ void AudioMixer::broadcastMixes() { quint16 sequence = nodeData->getOutgoingSequenceNumber(); mixPacket->writePrimitive(sequence); + QByteArray decodedBuffer(reinterpret_cast(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO); + QByteArray encodedBuffer; + nodeData->encode(decodedBuffer, encodedBuffer); + // pack mixed audio samples - mixPacket->write(reinterpret_cast(_clampedSamples), - AudioConstants::NETWORK_FRAME_BYTES_STEREO); + mixPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } else { int silentPacketBytes = sizeof(quint16) + sizeof(quint16); mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes); @@ -797,6 +896,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) { QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject(); + const QString CODEC_PREFERENCE_ORDER = "codec_preference_order"; + if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) { + _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString(); + qDebug() << "Codec preference order changed to" << _codecPreferenceOrder; + } + const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance"; if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) { bool ok = false; diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 24b4b39704..4b2a27120d 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -45,6 +45,7 @@ private slots: void broadcastMixes(); void handleNodeAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); void handleNodeKilled(SharedNodePointer killedNode); void removeHRTFsForFinishedInjector(const QUuid& streamID); @@ -91,6 +92,8 @@ private: int _manualEchoMixes { 0 }; int _totalMixes { 0 }; + QString _codecPreferenceOrder; + float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 20003ba10d..5c2ce8bf57 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) : _frameToSendStats = distribution(numberGenerator); } +AudioMixerClientData::~AudioMixerClientData() { + if (_codec) { + _codec->releaseDecoder(_decoder); + _codec->releaseEncoder(_encoder); + } +} + + AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { QReadLocker readLocker { &_streamsLock }; @@ -101,9 +109,13 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isStereo = channelFlag == 1; + auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()); + avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO); + qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName; + auto emplaced = _audioStreams.emplace( QUuid(), - std::unique_ptr { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) } + std::unique_ptr { avatarAudioStream } ); micStreamIt = emplaced.first; @@ -130,9 +142,16 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { if (streamIt == _audioStreams.end()) { // we don't have this injected stream yet, so add it + auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()); + +#if INJECTORS_SUPPORT_CODECS + injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + qDebug() << "creating new injectorStream... codec:" << _selectedCodecName; +#endif + auto emplaced = _audioStreams.emplace( streamIdentifier, - std::unique_ptr { new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()) } + std::unique_ptr { injectorStream } ); streamIt = emplaced.first; @@ -324,3 +343,44 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() { return result; } + +void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) { + cleanupCodec(); // cleanup any previously allocated coders first + _codec = codec; + _selectedCodecName = codecName; + if (codec) { + _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + } + + auto avatarAudioStream = getAvatarAudioStream(); + if (avatarAudioStream) { + avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO); + } + +#if INJECTORS_SUPPORT_CODECS + // fixup codecs for any active injectors... + auto it = _audioStreams.begin(); + while (it != _audioStreams.end()) { + SharedStreamPointer stream = it->second; + if (stream->getType() == PositionalAudioStream::Injector) { + stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); + } + ++it; + } +#endif +} + +void AudioMixerClientData::cleanupCodec() { + // release any old codec encoder/decoder first... + if (_codec) { + if (_decoder) { + _codec->releaseDecoder(_decoder); + _decoder = nullptr; + } + if (_encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } + } +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 17274a1519..da2bf8997c 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -19,6 +19,8 @@ #include #include +#include + #include "PositionalAudioStream.h" #include "AvatarAudioStream.h" @@ -27,6 +29,7 @@ class AudioMixerClientData : public NodeData { Q_OBJECT public: AudioMixerClientData(const QUuid& nodeID); + ~AudioMixerClientData(); using SharedStreamPointer = std::shared_ptr; using AudioStreamMap = std::unordered_map; @@ -65,6 +68,16 @@ public: AudioLimiter audioLimiter; + void setupCodec(CodecPluginPointer codec, const QString& codecName); + void cleanupCodec(); + void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + if (_encoder) { + _encoder->encode(decodedBuffer, encodedBuffer); + } else { + encodedBuffer = decodedBuffer; + } + } + signals: void injectorStreamFinished(const QUuid& streamIdentifier); @@ -81,6 +94,11 @@ private: AudioStreamStats _downstreamAudioStreamStats; int _frameToSendStats { 0 }; + + CodecPluginPointer _codec; + QString _selectedCodecName; + Encoder* _encoder{ nullptr }; // for outbound mixed stream + Decoder* _decoder{ nullptr }; // for mic stream }; #endif // hifi_AudioMixerClientData_h diff --git a/cmake/externals/hifiAudioCodec/CMakeLists.txt b/cmake/externals/hifiAudioCodec/CMakeLists.txt new file mode 100644 index 0000000000..3a8c714d79 --- /dev/null +++ b/cmake/externals/hifiAudioCodec/CMakeLists.txt @@ -0,0 +1,33 @@ +include(ExternalProject) +include(SelectLibraryConfigurations) + +set(EXTERNAL_NAME HiFiAudioCodec) + +string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER) + +ExternalProject_Add( + ${EXTERNAL_NAME} + URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK-1.zip + URL_MD5 23ec3fe51eaa155ea159a4971856fc13 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD 1 +) + +# Hide this external target (for ide users) +set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals") + +ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR) + +set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNAL) + +if (WIN32) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL) +elseif(APPLE) + set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/libaudio.a CACHE TYPE INTERNAL) +elseif(NOT ANDROID) + # FIXME need to account for different architectures + #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL) +endif() + diff --git a/cmake/macros/SetupHifiClientServerPlugin.cmake b/cmake/macros/SetupHifiClientServerPlugin.cmake new file mode 100644 index 0000000000..dfe3113fbc --- /dev/null +++ b/cmake/macros/SetupHifiClientServerPlugin.cmake @@ -0,0 +1,59 @@ +# +# Created by Brad Hefta-Gaub on 2016/07/07 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# +macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN) + set(${TARGET_NAME}_SHARED 1) + setup_hifi_library(${ARGV}) + add_dependencies(interface ${TARGET_NAME}) + set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins") + + if (APPLE) + set(CLIENT_PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns") + set(SERVER_PLUGIN_PATH "Components.app/Contents/PlugIns") + else() + set(CLIENT_PLUGIN_PATH "plugins") + set(SERVER_PLUGIN_PATH "plugins") + endif() + + if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${SERVER_PLUGIN_PATH}/") + elseif (APPLE) + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${SERVER_PLUGIN_PATH}/") + else() + set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$/${CLIENT_PLUGIN_PATH}/") + set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$/${SERVER_PLUGIN_PATH}/") + endif() + + # create the destination for the client plugin binaries + add_custom_command( + TARGET ${TARGET_NAME} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E make_directory + ${CLIENT_PLUGIN_FULL_PATH} + ) + # copy the client plugin binaries + add_custom_command(TARGET ${DIR} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "$" + ${CLIENT_PLUGIN_FULL_PATH} + ) + + # create the destination for the server plugin binaries + add_custom_command( + TARGET ${TARGET_NAME} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E make_directory + ${SERVER_PLUGIN_FULL_PATH} + ) + # copy the server plugin binaries + add_custom_command(TARGET ${DIR} POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "$" + ${SERVER_PLUGIN_FULL_PATH} + ) + +endmacro() diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 7375a0f650..948c6ddc18 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -718,6 +718,14 @@ "placeholder": "(in percent)" } ] + }, + { + "name": "codec_preference_order", + "label": "Audio Codec Preference Order", + "help": "List of codec names in order of preferred usage", + "placeholder": "hifiAC, zlib, pcm", + "default": "hifiAC,zlib,pcm", + "advanced": true } ] }, diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 604099054b..6f06ef08db 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include @@ -1245,6 +1246,11 @@ QString Application::getUserAgent() { userAgent += " " + formatPluginName(ip->getName()); } } + // for codecs, we include all of them, even if not active + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& cp : codecPlugins) { + userAgent += " " + formatPluginName(cp->getName()); + } return userAgent; } @@ -4482,6 +4488,9 @@ void Application::nodeActivated(SharedNodePointer node) { } } + if (node->getType() == NodeType::AudioMixer) { + DependencyManager::get()->negotiateAudioFormat(); + } } void Application::nodeKilled(SharedNodePointer node) { diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index 2c0fc0a9cd..9c298ce664 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET_NAME audio-client) setup_hifi_library(Network Multimedia) -link_hifi_libraries(audio) +link_hifi_libraries(audio plugins) # append audio includes to our list of includes to bubble target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 7628c09748..0cd01bb579 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include #include @@ -133,10 +135,15 @@ AudioClient::AudioClient() : packetReceiver.registerListener(PacketType::MixedAudio, this, "handleAudioDataPacket"); packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); + packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat"); } AudioClient::~AudioClient() { stop(); + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } } void AudioClient::reset() { @@ -503,6 +510,49 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer me emit muteEnvironmentRequested(position, radius); } +void AudioClient::negotiateAudioFormat() { + auto nodeList = DependencyManager::get(); + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + quint8 numberOfCodecs = (quint8)codecPlugins.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (auto& plugin : codecPlugins) { + auto codecName = plugin->getName(); + negotiateFormatPacket->writeString(codecName); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + +void AudioClient::handleSelectedAudioFormat(QSharedPointer message) { + _selectedCodecName = message->readString(); + + qDebug() << "Selected Codec:" << _selectedCodecName; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + if (_selectedCodecName == plugin->getName()) { + // release any old codec encoder/decoder first... + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } + _codec = plugin; + _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); + _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + qDebug() << "Selected Codec Plugin:" << _codec.get(); + break; + } + } + +} + + QString AudioClient::getDefaultDeviceName(QAudio::Mode mode) { QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode); return deviceInfo.deviceName(); @@ -770,7 +820,17 @@ void AudioClient::handleAudioInput() { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); // FIXME find a way to properly handle both playback audio and user audio concurrently - emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); + + // TODO - codec encode goes here + QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + QByteArray encodedBuffer; + if (_encoder) { + _encoder->encode(decocedBuffer, encodedBuffer); + } else { + encodedBuffer = decocedBuffer; + } + + emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType); _stats.sentPacket(); } } @@ -779,23 +839,34 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { Transform audioTransform; audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); + + // TODO - codec encode goes here + QByteArray encodedBuffer; + if (_encoder) { + _encoder->encode(audio, encodedBuffer); + } else { + encodedBuffer = audio; + } + // FIXME check a flag to see if we should echo audio? - emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); + emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } -void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { - const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t); - const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) - / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()); +void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { + const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t); + const int numDeviceOutputSamples = _outputFrameSize; + + Q_ASSERT(_outputFrameSize == numDecodecSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) + / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount())); outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t)); - const int16_t* receivedSamples = reinterpret_cast(inputBuffer.data()); + const int16_t* decodedSamples = reinterpret_cast(decodedBuffer.data()); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); // copy the packet from the RB to the output - possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples, - numNetworkOutputSamples, numDeviceOutputSamples, + possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples, + numDecodecSamples, numDeviceOutputSamples, _desiredOutputFormat, _outputFormat); // apply stereo reverb at the listener, to the received audio @@ -1185,6 +1256,13 @@ void AudioClient::loadSettings() { windowSecondsForDesiredCalcOnTooManyStarves.get()); _receivedAudioStream.setWindowSecondsForDesiredReduction(windowSecondsForDesiredReduction.get()); _receivedAudioStream.setRepetitionWithFade(repetitionWithFade.get()); + + qDebug() << "---- Initializing Audio Client ----"; + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (auto& plugin : codecPlugins) { + qDebug() << "Codec available:" << plugin->getName(); + } + } void AudioClient::saveSettings() { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index dc46db5657..bd0afe453d 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -38,6 +38,8 @@ #include #include +#include + #include "AudioIOStats.h" #include "AudioNoiseGate.h" #include "AudioSRC.h" @@ -94,6 +96,8 @@ public: int _unfulfilledReads; }; + void negotiateAudioFormat(); + const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } @@ -139,6 +143,7 @@ public slots: void handleAudioDataPacket(QSharedPointer message); void handleNoisyMutePacket(QSharedPointer message); void handleMuteEnvironmentPacket(QSharedPointer message); + void handleSelectedAudioFormat(QSharedPointer message); void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); } void handleAudioInput(); @@ -291,6 +296,10 @@ private: void checkDevices(); bool _hasReceivedFirstPacket = false; + + CodecPluginPointer _codec; + QString _selectedCodecName; + Encoder* _encoder { nullptr }; // for outbound mic stream }; diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt index c49c9547a5..1e9360b9a2 100644 --- a/libraries/audio/CMakeLists.txt +++ b/libraries/audio/CMakeLists.txt @@ -1,3 +1,3 @@ set(TARGET_NAME audio) setup_hifi_library(Network) -link_hifi_libraries(networking shared) +link_hifi_libraries(networking shared plugins) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 878a4c627c..fee33dcb92 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -289,16 +289,23 @@ int64_t AudioInjector::injectNextFrame() { _currentPacket->seek(audioDataOffset); + // This code is copying bytes from the _audioData directly into the packet, handling looping appropriately. + // Might be a reasonable place to do the encode step here. + QByteArray decodedAudio; while (totalBytesLeftToCopy > 0) { int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset); - _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy); + decodedAudio.append(_audioData.data() + _currentSendOffset, bytesToCopy); _currentSendOffset += bytesToCopy; totalBytesLeftToCopy -= bytesToCopy; if (_options.loop && _currentSendOffset >= _audioData.size()) { _currentSendOffset = 0; } } + // FIXME -- good place to call codec encode here. We need to figure out how to tell the AudioInjector which + // codec to use... possible through AbstractAudioInterface. + QByteArray encodedAudio = decodedAudio; + _currentPacket->write(encodedAudio.data(), encodedAudio.size()); // set the correct size used for this packet _currentPacket->setPayloadSize(_currentPacket->pos()); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 8b32ada296..c9b9363b1b 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { if (message.getType() == PacketType::SilentAudioFrame) { writeDroppableSilentSamples(networkSamples); } else { - parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples); + parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead())); } break; } @@ -172,16 +172,27 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& return sizeof(quint16); } else { // mixed audio packets do not have any info between the seq num and the audio data. - numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t); + numAudioSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; return 0; } } -int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); +int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { + QByteArray decodedBuffer; + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); + } else { + decodedBuffer = packetAfterStreamProperties; + } + auto actualSize = decodedBuffer.size(); + return _ringBuffer.writeData(decodedBuffer.data(), actualSize); } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { + if (_decoder) { + _decoder->trackLostFrames(silentSamples); + } + // calculate how many silent frames we should drop. int samplesPerFrame = _ringBuffer.getNumFrameSamples(); int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; @@ -497,3 +508,21 @@ float calculateRepeatedFrameFadeFactor(int indexOfRepeat) { return 0.0f; } +void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels) { + cleanupCodec(); // cleanup any previously allocated coders first + _codec = codec; + _selectedCodecName = codecName; + if (_codec) { + _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels); + } +} + +void InboundAudioStream::cleanupCodec() { + // release any old codec encoder/decoder first... + if (_codec) { + if (_decoder) { + _codec->releaseDecoder(_decoder); + _decoder = nullptr; + } + } +} diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 3f641f1ba4..5da63f96c2 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -18,6 +18,8 @@ #include #include +#include + #include "AudioRingBuffer.h" #include "MovingMinMaxAvg.h" #include "SequenceNumberStats.h" @@ -103,6 +105,7 @@ public: public: InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings); + ~InboundAudioStream() { cleanupCodec(); } void reset(); virtual void resetStats(); @@ -174,6 +177,9 @@ public: void setReverb(float reverbTime, float wetLevel); void clearReverb() { _hasReverb = false; } + void setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels); + void cleanupCodec(); + public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. @@ -201,7 +207,7 @@ protected: /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentSamples(int silentSamples); @@ -267,6 +273,10 @@ protected: bool _hasReverb; float _reverbTime; float _wetLevel; + + CodecPluginPointer _codec; + QString _selectedCodecName; + Decoder* _decoder{ nullptr }; }; float calculateRepeatedFrameFadeFactor(int indexOfRepeat); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index d236ac7aad..728deae0b1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -42,12 +42,18 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { return deviceSamplesWritten; } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { + QByteArray decodedBuffer; + if (_decoder) { + _decoder->decode(packetAfterStreamProperties, decodedBuffer); + } else { + decodedBuffer = packetAfterStreamProperties; + } - emit addedStereoSamples(packetAfterStreamProperties); + emit addedStereoSamples(decodedBuffer); QByteArray outputBuffer; - emit processSamples(packetAfterStreamProperties, outputBuffer); + emit processSamples(decodedBuffer, outputBuffer); _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 5ea0157421..2f9a691278 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -35,7 +35,7 @@ public: protected: int writeDroppableSilentSamples(int silentSamples); int writeLastFrameRepeatedWithFade(int samples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); private: int networkToDeviceSamples(int networkSamples); diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index dba241f221..18552ca966 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) { qint64 BasePacket::writeString(const QString& string) { QByteArray data = string.toUtf8(); - writePrimitive(static_cast(data.length())); - return writeData(data.constData(), data.length()); + uint32_t length = data.length(); + writePrimitive(length); + writeData(data.constData(), data.length()); + seek(pos() + length); + return length + sizeof(uint32_t); } QString BasePacket::readString() { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 9140cf8738..296ebb8e68 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -95,7 +95,9 @@ public: AssetMappingOperation, AssetMappingOperationReply, ICEServerHeartbeatACK, - LAST_PACKET_TYPE = ICEServerHeartbeatACK + NegotiateAudioFormat, + SelectedAudioFormat, + LAST_PACKET_TYPE = SelectedAudioFormat }; }; diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h new file mode 100644 index 0000000000..404f05e860 --- /dev/null +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -0,0 +1,36 @@ +// +// CodecPlugin.h +// plugins/src/plugins +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// +#pragma once + +#include "Plugin.h" + +class Encoder { +public: + virtual ~Encoder() { } + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; +}; + +class Decoder { +public: + virtual ~Decoder() { } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + + // numFrames - number of samples (mono) or sample-pairs (stereo) + virtual void trackLostFrames(int numFrames) = 0; +}; + +class CodecPlugin : public Plugin { +public: + virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; + virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; + virtual void releaseEncoder(Encoder* encoder) = 0; + virtual void releaseDecoder(Decoder* decoder) = 0; +}; diff --git a/libraries/plugins/src/plugins/Forward.h b/libraries/plugins/src/plugins/Forward.h index 036b42f7d7..723c4f321e 100644 --- a/libraries/plugins/src/plugins/Forward.h +++ b/libraries/plugins/src/plugins/Forward.h @@ -13,10 +13,12 @@ enum class PluginType { DISPLAY_PLUGIN, INPUT_PLUGIN, + CODEC_PLUGIN, }; class DisplayPlugin; class InputPlugin; +class CodecPlugin; class Plugin; class PluginContainer; class PluginManager; @@ -25,4 +27,6 @@ using DisplayPluginPointer = std::shared_ptr; using DisplayPluginList = std::vector; using InputPluginPointer = std::shared_ptr; using InputPluginList = std::vector; +using CodecPluginPointer = std::shared_ptr; +using CodecPluginList = std::vector; diff --git a/libraries/plugins/src/plugins/PluginManager.cpp b/libraries/plugins/src/plugins/PluginManager.cpp index 29658eeb6b..0b4afe1be0 100644 --- a/libraries/plugins/src/plugins/PluginManager.cpp +++ b/libraries/plugins/src/plugins/PluginManager.cpp @@ -18,6 +18,7 @@ #include #include "RuntimePlugin.h" +#include "CodecPlugin.h" #include "DisplayPlugin.h" #include "InputPlugin.h" @@ -117,6 +118,7 @@ PluginManager::PluginManager() { // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class extern DisplayPluginList getDisplayPlugins(); extern InputPluginList getInputPlugins(); +extern CodecPluginList getCodecPlugins(); extern void saveInputPluginSettings(const InputPluginList& plugins); static DisplayPluginList displayPlugins; @@ -202,6 +204,35 @@ const InputPluginList& PluginManager::getInputPlugins() { return inputPlugins; } +const CodecPluginList& PluginManager::getCodecPlugins() { + static CodecPluginList codecPlugins; + static std::once_flag once; + std::call_once(once, [&] { + //codecPlugins = ::getCodecPlugins(); + + // Now grab the dynamic plugins + for (auto loader : getLoadedPlugins()) { + CodecProvider* codecProvider = qobject_cast(loader->instance()); + if (codecProvider) { + for (auto codecPlugin : codecProvider->getCodecPlugins()) { + if (codecPlugin->isSupported()) { + codecPlugins.push_back(codecPlugin); + } + } + } + } + + for (auto plugin : codecPlugins) { + plugin->setContainer(_container); + plugin->init(); + + qDebug() << "init codec:" << plugin->getName(); + } + }); + return codecPlugins; +} + + void PluginManager::setPreferredDisplayPlugins(const QStringList& displays) { preferredDisplayPlugins = displays; } diff --git a/libraries/plugins/src/plugins/PluginManager.h b/libraries/plugins/src/plugins/PluginManager.h index 7903bdd724..30d52298da 100644 --- a/libraries/plugins/src/plugins/PluginManager.h +++ b/libraries/plugins/src/plugins/PluginManager.h @@ -18,6 +18,7 @@ public: const DisplayPluginList& getDisplayPlugins(); const InputPluginList& getInputPlugins(); + const CodecPluginList& getCodecPlugins(); DisplayPluginList getPreferredDisplayPlugins(); void setPreferredDisplayPlugins(const QStringList& displays); diff --git a/libraries/plugins/src/plugins/RuntimePlugin.h b/libraries/plugins/src/plugins/RuntimePlugin.h index d7bf31ea28..9bf15f344d 100644 --- a/libraries/plugins/src/plugins/RuntimePlugin.h +++ b/libraries/plugins/src/plugins/RuntimePlugin.h @@ -34,3 +34,12 @@ public: #define InputProvider_iid "com.highfidelity.plugins.input" Q_DECLARE_INTERFACE(InputProvider, InputProvider_iid) +class CodecProvider { +public: + virtual ~CodecProvider() {} + virtual CodecPluginList getCodecPlugins() = 0; +}; + +#define CodecProvider_iid "com.highfidelity.plugins.codec" +Q_DECLARE_INTERFACE(CodecProvider, CodecProvider_iid) + diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt new file mode 100644 index 0000000000..3939529c3e --- /dev/null +++ b/plugins/hifiCodec/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Created by Brad Hefta-Gaub on 7/10/2016 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +if (WIN32 OR APPLE) + set(TARGET_NAME hifiCodec) + setup_hifi_client_server_plugin() + + link_hifi_libraries(audio shared plugins) + + add_dependency_external_projects(HiFiAudioCodec) + target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${HIFIAUDIOCODEC_LIBRARIES}) +endif() + diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp new file mode 100644 index 0000000000..4e9336ff90 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -0,0 +1,93 @@ +// +// HiFiCodec.cpp +// plugins/hifiCodec/src +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include + +#include + +#include "HiFiCodec.h" + +const QString HiFiCodec::NAME = "hifiAC"; + +void HiFiCodec::init() { +} + +void HiFiCodec::deinit() { +} + +bool HiFiCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void HiFiCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool HiFiCodec::isSupported() const { + return true; +} + +class HiFiEncoder : public Encoder, public AudioEncoder { +public: + HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) { + _encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4; // codec reduces by 1/4th + } + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer.resize(_encodedSize); + AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } +private: + int _encodedSize; +}; + +class HiFiDecoder : public Decoder, public AudioDecoder { +public: + HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) { + _decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels; + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer.resize(_decodedSize); + AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); + } + + virtual void trackLostFrames(int numFrames) override { + QByteArray encodedBuffer; + QByteArray decodedBuffer; + decodedBuffer.resize(_decodedSize); + // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean + AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); + } +private: + int _decodedSize; +}; + +Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) { + return new HiFiEncoder(sampleRate, numChannels); +} + +Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) { + return new HiFiDecoder(sampleRate, numChannels); +} + +void HiFiCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void HiFiCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} diff --git a/plugins/hifiCodec/src/HiFiCodec.h b/plugins/hifiCodec/src/HiFiCodec.h new file mode 100644 index 0000000000..eeba8d56d8 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodec.h @@ -0,0 +1,42 @@ +// +// HiFiCodec.h +// plugins/hifiCodec/src +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_HiFiCodec_h +#define hifi_HiFiCodec_h + +#include + +class HiFiCodec : public CodecPlugin { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + +private: + static const QString NAME; +}; + +#endif // hifi_HiFiCodec_h diff --git a/plugins/hifiCodec/src/HiFiCodecProvider.cpp b/plugins/hifiCodec/src/HiFiCodecProvider.cpp new file mode 100644 index 0000000000..b9698cc821 --- /dev/null +++ b/plugins/hifiCodec/src/HiFiCodecProvider.cpp @@ -0,0 +1,46 @@ +// +// Created by Brad Hefta-Gaub on 7/10/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "HiFiCodec.h" + +class HiFiCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + HiFiCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~HiFiCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + + CodecPluginPointer hiFiCodec(new HiFiCodec()); + if (hiFiCodec->isSupported()) { + _codecPlugins.push_back(hiFiCodec); + } + + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "HiFiCodecProvider.moc" diff --git a/plugins/hifiCodec/src/plugin.json b/plugins/hifiCodec/src/plugin.json new file mode 100644 index 0000000000..df26a67ea8 --- /dev/null +++ b/plugins/hifiCodec/src/plugin.json @@ -0,0 +1 @@ +{"name":"HiFi 4:1 Audio Codec"} diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt new file mode 100644 index 0000000000..5dca1f0e14 --- /dev/null +++ b/plugins/pcmCodec/CMakeLists.txt @@ -0,0 +1,11 @@ +# +# Created by Brad Hefta-Gaub on 6/9/2016 +# Copyright 2016 High Fidelity, Inc. +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME pcmCodec) +setup_hifi_client_server_plugin() +link_hifi_libraries(shared plugins) diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp new file mode 100644 index 0000000000..315d0622ab --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -0,0 +1,94 @@ +// +// PCMCodec.cpp +// plugins/pcmCodec/src +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include + +#include "PCMCodecManager.h" + +const QString PCMCodec::NAME = "pcm"; + +void PCMCodec::init() { +} + +void PCMCodec::deinit() { +} + +bool PCMCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void PCMCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool PCMCodec::isSupported() const { + return true; +} + + + +Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void PCMCodec::releaseEncoder(Encoder* encoder) { + // do nothing +} + +void PCMCodec::releaseDecoder(Decoder* decoder) { + // do nothing +} + +const QString zLibCodec::NAME = "zlib"; + +void zLibCodec::init() { +} + +void zLibCodec::deinit() { +} + +bool zLibCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void zLibCodec::deactivate() { + CodecPlugin::deactivate(); +} + +bool zLibCodec::isSupported() const { + return true; +} + +Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void zLibCodec::releaseEncoder(Encoder* encoder) { + // do nothing... it wasn't allocated +} + +void zLibCodec::releaseDecoder(Decoder* decoder) { + // do nothing... it wasn't allocated +} + diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h new file mode 100644 index 0000000000..55d7c866f1 --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -0,0 +1,86 @@ +// +// PCMCodecManager.h +// plugins/pcmCodec/src +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi__PCMCodecManager_h +#define hifi__PCMCodecManager_h + +#include + +class PCMCodec : public CodecPlugin, public Encoder, public Decoder { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void trackLostFrames(int numFrames) override { } + +private: + static const QString NAME; +}; + +class zLibCodec : public CodecPlugin, public Encoder, public Decoder { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString& getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = qCompress(decodedBuffer); + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = qUncompress(encodedBuffer); + } + + virtual void trackLostFrames(int numFrames) override { } + +private: + static const QString NAME; +}; + +#endif // hifi__PCMCodecManager_h diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp new file mode 100644 index 0000000000..351b1adf3f --- /dev/null +++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp @@ -0,0 +1,51 @@ +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "PCMCodecManager.h" + +class PCMCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + PCMCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~PCMCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + + CodecPluginPointer pcmCodec(new PCMCodec()); + if (pcmCodec->isSupported()) { + _codecPlugins.push_back(pcmCodec); + } + + CodecPluginPointer zlibCodec(new zLibCodec()); + if (zlibCodec->isSupported()) { + _codecPlugins.push_back(zlibCodec); + } + + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "PCMCodecProvider.moc" diff --git a/plugins/pcmCodec/src/plugin.json b/plugins/pcmCodec/src/plugin.json new file mode 100644 index 0000000000..2d86251845 --- /dev/null +++ b/plugins/pcmCodec/src/plugin.json @@ -0,0 +1 @@ +{"name":"PCM Codec"}