Merge branch 'master' of github.com:highfidelity/hifi into groups

assignment-client/CMakeLists.txt

@@ -6,7 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick Widgets WebSockets)
 link_hifi_libraries(
   audio avatars octree gpu model fbx entities
   networking animation recording shared script-engine embedded-webserver
-  controllers physics
+  controllers physics plugins
 )
 
 if (WIN32)

assignment-client/src/audio/AudioMixer.cpp

@@ -47,6 +47,8 @@
 #include <NodeList.h>
 #include <Node.h>
 #include <OctreeConstants.h>
+#include <plugins/PluginManager.h>
+#include <plugins/CodecPlugin.h>
 #include <udt/PacketHeaders.h>
 #include <SharedUtil.h>
 #include <StDev.h>
@@ -90,6 +92,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) :
                                               PacketType::AudioStreamStats },
                                             this, "handleNodeAudioPacket");
     packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
+    packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat");
 
     connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled);
 }
@@ -446,6 +449,99 @@ void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> mes
     }
 }
 
+DisplayPluginList getDisplayPlugins() {
+    DisplayPluginList result;
+    return result;
+}
+
+InputPluginList getInputPlugins() {
+    InputPluginList result;
+    return result;
+}
+
+void saveInputPluginSettings(const InputPluginList& plugins) {
+}
+
+
+void AudioMixer::handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
+    QStringList availableCodecs;
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    if (codecPlugins.size() > 0) {
+        for (auto& plugin : codecPlugins) {
+            auto codecName = plugin->getName();
+            qDebug() << "Codec available:" << codecName;
+            availableCodecs.append(codecName);
+        }
+    } else {
+        qDebug() << "No Codecs available...";
+    }
+
+    CodecPluginPointer selectedCodec;
+    QString selectedCodecName;
+
+    QStringList codecPreferenceList = _codecPreferenceOrder.split(",");
+
+    // read the codecs requested by the client
+    const int MAX_PREFERENCE = 99999;
+    int preferredCodecIndex = MAX_PREFERENCE;
+    QString preferredCodec;
+    quint8 numberOfCodecs = 0;
+    message->readPrimitive(&numberOfCodecs);
+    qDebug() << "numberOfCodecs:" << numberOfCodecs;
+    QStringList codecList;
+    for (quint16 i = 0; i < numberOfCodecs; i++) {
+        QString requestedCodec = message->readString();
+        int preferenceOfThisCodec = codecPreferenceList.indexOf(requestedCodec);
+        bool codecAvailable = availableCodecs.contains(requestedCodec);
+        qDebug() << "requestedCodec:" << requestedCodec << "preference:" << preferenceOfThisCodec << "available:" << codecAvailable;
+        if (codecAvailable) {
+            codecList.append(requestedCodec);
+            if (preferenceOfThisCodec >= 0 && preferenceOfThisCodec < preferredCodecIndex) {
+                qDebug() << "This codec is preferred...";
+                selectedCodecName  = requestedCodec;
+                preferredCodecIndex = preferenceOfThisCodec;
+            }
+        }
+    }
+    qDebug() << "all requested and available codecs:" << codecList;
+
+    // choose first codec
+    if (!selectedCodecName.isEmpty()) {
+        if (codecPlugins.size() > 0) {
+            for (auto& plugin : codecPlugins) {
+                if (selectedCodecName == plugin->getName()) {
+                    qDebug() << "Selecting codec:" << selectedCodecName;
+                    selectedCodec = plugin;
+                    break;
+                }
+            }
+        }
+    }
+
+
+    auto clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
+
+    // FIXME - why would we not have client data at this point??
+    if (!clientData) {
+        qDebug() << "UNEXPECTED -- didn't have node linked data in " << __FUNCTION__;
+        sendingNode->setLinkedData(std::unique_ptr<NodeData> { new AudioMixerClientData(sendingNode->getUUID()) });
+        clientData = dynamic_cast<AudioMixerClientData*>(sendingNode->getLinkedData());
+        connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector);
+    }
+
+    clientData->setupCodec(selectedCodec, selectedCodecName);
+
+    qDebug() << "selectedCodecName:" << selectedCodecName;
+
+    auto replyPacket = NLPacket::create(PacketType::SelectedAudioFormat);
+
+    // write them to our packet
+    replyPacket->writeString(selectedCodecName);
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    nodeList->sendPacket(std::move(replyPacket), *sendingNode);
+}
+
 void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) {
     // enumerate the connected listeners to remove HRTF objects for the disconnected node
     auto nodeList = DependencyManager::get<NodeList>();
@@ -669,9 +765,12 @@ void AudioMixer::broadcastMixes() {
                         quint16 sequence = nodeData->getOutgoingSequenceNumber();
                         mixPacket->writePrimitive(sequence);
 
+                        QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+                        QByteArray encodedBuffer;
+                        nodeData->encode(decodedBuffer, encodedBuffer);
+
                         // pack mixed audio samples
-                        mixPacket->write(reinterpret_cast<char*>(_clampedSamples),
+                        mixPacket->write(encodedBuffer.constData(), encodedBuffer.size());
-                                         AudioConstants::NETWORK_FRAME_BYTES_STEREO);
                     } else {
                         int silentPacketBytes = sizeof(quint16) + sizeof(quint16);
                         mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes);
@@ -797,6 +896,12 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) {
     if (settingsObject.contains(AUDIO_ENV_GROUP_KEY)) {
         QJsonObject audioEnvGroupObject = settingsObject[AUDIO_ENV_GROUP_KEY].toObject();
 
+        const QString CODEC_PREFERENCE_ORDER = "codec_preference_order";
+        if (audioEnvGroupObject[CODEC_PREFERENCE_ORDER].isString()) {
+            _codecPreferenceOrder = audioEnvGroupObject[CODEC_PREFERENCE_ORDER].toString();
+            qDebug() << "Codec preference order changed to" << _codecPreferenceOrder;
+        }
+
         const QString ATTENATION_PER_DOULING_IN_DISTANCE = "attenuation_per_doubling_in_distance";
         if (audioEnvGroupObject[ATTENATION_PER_DOULING_IN_DISTANCE].isString()) {
             bool ok = false;

assignment-client/src/audio/AudioMixer.h

@@ -45,6 +45,7 @@ private slots:
     void broadcastMixes();
     void handleNodeAudioPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
     void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> packet, SharedNodePointer sendingNode);
+    void handleNegotiateAudioFormat(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode);
     void handleNodeKilled(SharedNodePointer killedNode);
 
     void removeHRTFsForFinishedInjector(const QUuid& streamID);
@@ -91,6 +92,8 @@ private:
     int _manualEchoMixes { 0 };
     int _totalMixes { 0 };
 
+    QString _codecPreferenceOrder;
+
     float _mixedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
     int16_t _clampedSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];
 

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -39,6 +39,14 @@ AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) :
     _frameToSendStats = distribution(numberGenerator);
 }
 
+AudioMixerClientData::~AudioMixerClientData() {
+    if (_codec) {
+        _codec->releaseDecoder(_decoder);
+        _codec->releaseEncoder(_encoder);
+    }
+}
+
+
 AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
     QReadLocker readLocker { &_streamsLock };
     
@@ -101,9 +109,13 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
 
                 bool isStereo = channelFlag == 1;
 
+                auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings());
+                avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO);
+                qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName;
+
                 auto emplaced = _audioStreams.emplace(
                     QUuid(),
-                    std::unique_ptr<PositionalAudioStream> { new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings()) }
+                    std::unique_ptr<PositionalAudioStream> { avatarAudioStream }
                 );
 
                 micStreamIt = emplaced.first;
@@ -130,9 +142,16 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
 
             if (streamIt == _audioStreams.end()) {
                 // we don't have this injected stream yet, so add it
+                auto injectorStream = new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings());
+
+#if INJECTORS_SUPPORT_CODECS
+                injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
+                qDebug() << "creating new injectorStream... codec:" << _selectedCodecName;
+#endif
+
                 auto emplaced = _audioStreams.emplace(
                     streamIdentifier,
-                    std::unique_ptr<InjectedAudioStream> { new InjectedAudioStream(streamIdentifier, isStereo, AudioMixer::getStreamSettings()) }
+                    std::unique_ptr<InjectedAudioStream> { injectorStream }
                 );
 
                 streamIt = emplaced.first;
@@ -324,3 +343,44 @@ QJsonObject AudioMixerClientData::getAudioStreamStats() {
 
     return result;
 }
+
+void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) {
+    cleanupCodec(); // cleanup any previously allocated coders first
+    _codec = codec;
+    _selectedCodecName = codecName;
+    if (codec) {
+        _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO);
+        _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+    }
+
+    auto avatarAudioStream = getAvatarAudioStream();
+    if (avatarAudioStream) {
+        avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO);
+    }
+
+#if INJECTORS_SUPPORT_CODECS
+    // fixup codecs for any active injectors...
+    auto it = _audioStreams.begin();
+    while (it != _audioStreams.end()) {
+        SharedStreamPointer stream = it->second;
+        if (stream->getType() == PositionalAudioStream::Injector) {
+            stream->setupCodec(codec, codecName, stream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
+        }
+        ++it;
+    }
+#endif
+}
+
+void AudioMixerClientData::cleanupCodec() {
+    // release any old codec encoder/decoder first...
+    if (_codec) {
+        if (_decoder) {
+            _codec->releaseDecoder(_decoder);
+            _decoder = nullptr;
+        }
+        if (_encoder) {
+            _codec->releaseEncoder(_encoder);
+            _encoder = nullptr;
+        }
+    }
+}

assignment-client/src/audio/AudioMixerClientData.h

@@ -19,6 +19,8 @@
 #include <AudioLimiter.h>
 #include <UUIDHasher.h>
 
+#include <plugins/CodecPlugin.h>
+
 #include "PositionalAudioStream.h"
 #include "AvatarAudioStream.h"
 
@@ -27,6 +29,7 @@ class AudioMixerClientData : public NodeData {
     Q_OBJECT
 public:
     AudioMixerClientData(const QUuid& nodeID);
+    ~AudioMixerClientData();
 
     using SharedStreamPointer = std::shared_ptr<PositionalAudioStream>;
     using AudioStreamMap = std::unordered_map<QUuid, SharedStreamPointer>;
@@ -65,6 +68,16 @@ public:
 
     AudioLimiter audioLimiter;
 
+    void setupCodec(CodecPluginPointer codec, const QString& codecName);
+    void cleanupCodec();
+    void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) {
+        if (_encoder) {
+            _encoder->encode(decodedBuffer, encodedBuffer);
+        } else {
+            encodedBuffer = decodedBuffer;
+        }
+    }
+
 signals:
     void injectorStreamFinished(const QUuid& streamIdentifier);
 
@@ -81,6 +94,11 @@ private:
     AudioStreamStats _downstreamAudioStreamStats;
 
     int _frameToSendStats { 0 };
+
+    CodecPluginPointer _codec;
+    QString _selectedCodecName;
+    Encoder* _encoder{ nullptr }; // for outbound mixed stream
+    Decoder* _decoder{ nullptr }; // for mic stream
 };
 
 #endif // hifi_AudioMixerClientData_h

cmake/externals/hifiAudioCodec/CMakeLists.txt

@@ -0,0 +1,33 @@
+include(ExternalProject)
+include(SelectLibraryConfigurations)
+
+set(EXTERNAL_NAME HiFiAudioCodec)
+
+string(TOUPPER ${EXTERNAL_NAME} EXTERNAL_NAME_UPPER)
+
+ExternalProject_Add(
+  ${EXTERNAL_NAME}
+  URL https://s3.amazonaws.com/hifi-public/dependencies/codecSDK-1.zip
+  URL_MD5 23ec3fe51eaa155ea159a4971856fc13
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND ""
+  LOG_DOWNLOAD 1
+)
+
+# Hide this external target (for ide users)
+set_target_properties(${EXTERNAL_NAME} PROPERTIES FOLDER "hidden/externals")
+
+ExternalProject_Get_Property(${EXTERNAL_NAME} SOURCE_DIR)
+
+set(${EXTERNAL_NAME_UPPER}_INCLUDE_DIRS ${SOURCE_DIR}/include CACHE TYPE INTERNAL)
+
+if (WIN32)
+    set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/audio.lib CACHE TYPE INTERNAL)
+elseif(APPLE)
+  set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/Release/libaudio.a CACHE TYPE INTERNAL)
+elseif(NOT ANDROID)
+  # FIXME need to account for different architectures 
+  #set(${EXTERNAL_NAME_UPPER}_LIBRARIES ${SOURCE_DIR}/lib/linux64/audio.so CACHE TYPE INTERNAL)
+endif()
+

cmake/macros/SetupHifiClientServerPlugin.cmake

@@ -0,0 +1,59 @@
+#
+#  Created by Brad Hefta-Gaub on 2016/07/07
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+macro(SETUP_HIFI_CLIENT_SERVER_PLUGIN)
+    set(${TARGET_NAME}_SHARED 1)
+    setup_hifi_library(${ARGV})
+    add_dependencies(interface ${TARGET_NAME})
+    set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Plugins")
+
+    if (APPLE)
+        set(CLIENT_PLUGIN_PATH "${INTERFACE_BUNDLE_NAME}.app/Contents/PlugIns")
+        set(SERVER_PLUGIN_PATH "Components.app/Contents/PlugIns")
+    else()
+        set(CLIENT_PLUGIN_PATH "plugins")
+        set(SERVER_PLUGIN_PATH "plugins")
+    endif()
+
+    if (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_GENERATOR STREQUAL "Unix Makefiles")
+        set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/${CLIENT_PLUGIN_PATH}/")
+        set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/${SERVER_PLUGIN_PATH}/")
+    elseif (APPLE)
+        set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${CLIENT_PLUGIN_PATH}/")
+        set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${SERVER_PLUGIN_PATH}/")
+    else()
+        set(CLIENT_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/interface/$<CONFIGURATION>/${CLIENT_PLUGIN_PATH}/")
+        set(SERVER_PLUGIN_FULL_PATH "${CMAKE_BINARY_DIR}/assignment-client/$<CONFIGURATION>/${SERVER_PLUGIN_PATH}/")
+    endif()
+
+    # create the destination for the client plugin binaries
+    add_custom_command(
+        TARGET ${TARGET_NAME} POST_BUILD
+        COMMAND "${CMAKE_COMMAND}" -E make_directory
+        ${CLIENT_PLUGIN_FULL_PATH}
+    )
+    # copy the client plugin binaries
+    add_custom_command(TARGET ${DIR} POST_BUILD
+        COMMAND "${CMAKE_COMMAND}" -E copy
+        "$<TARGET_FILE:${TARGET_NAME}>"
+        ${CLIENT_PLUGIN_FULL_PATH}
+    )
+
+    # create the destination for the server plugin binaries
+    add_custom_command(
+        TARGET ${TARGET_NAME} POST_BUILD
+        COMMAND "${CMAKE_COMMAND}" -E make_directory
+        ${SERVER_PLUGIN_FULL_PATH}
+    )
+    # copy the server plugin binaries
+    add_custom_command(TARGET ${DIR} POST_BUILD
+        COMMAND "${CMAKE_COMMAND}" -E copy
+        "$<TARGET_FILE:${TARGET_NAME}>"
+        ${SERVER_PLUGIN_FULL_PATH}
+    )
+
+endmacro()

domain-server/resources/describe-settings.json

@@ -868,6 +868,14 @@
               "placeholder": "(in percent)"
             }
           ]
+        },
+        {
+          "name": "codec_preference_order",
+          "label": "Audio Codec Preference Order",
+          "help": "List of codec names in order of preferred usage",
+          "placeholder": "hifiAC, zlib, pcm",
+          "default": "hifiAC,zlib,pcm",
+          "advanced": true
         }
       ]
     },

interface/src/Application.cpp

@@ -85,6 +85,7 @@
 #include <PhysicsEngine.h>
 #include <PhysicsHelpers.h>
 #include <plugins/PluginManager.h>
+#include <plugins/CodecPlugin.h>
 #include <RenderableWebEntityItem.h>
 #include <RenderShadowTask.h>
 #include <RenderDeferredTask.h>
@@ -168,6 +169,14 @@ static QTimer locationUpdateTimer;
 static QTimer identityPacketTimer;
 static QTimer pingTimer;
 
+static const int MAX_CONCURRENT_RESOURCE_DOWNLOADS = 16;
+
+// For processing on QThreadPool, target 2 less than the ideal number of threads, leaving
+// 2 logical cores available for time sensitive tasks.
+static const int MIN_PROCESSING_THREAD_POOL_SIZE = 2;
+static const int PROCESSING_THREAD_POOL_SIZE = std::max(MIN_PROCESSING_THREAD_POOL_SIZE,
+                                                        QThread::idealThreadCount() - 2);
+
 static const QString SNAPSHOT_EXTENSION  = ".jpg";
 static const QString SVO_EXTENSION  = ".svo";
 static const QString SVO_JSON_EXTENSION = ".svo.json";
@@ -515,13 +524,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
     PluginContainer* pluginContainer = dynamic_cast<PluginContainer*>(this); // set the container for any plugins that care
     PluginManager::getInstance()->setContainer(pluginContainer);
 
-    // FIXME this may be excessively conservative.  On the other hand
+    QThreadPool::globalInstance()->setMaxThreadCount(PROCESSING_THREAD_POOL_SIZE);
-    // maybe I'm used to having an 8-core machine
-    // Perhaps find the ideal thread count  and subtract 2 or 3
-    // (main thread, present thread, random OS load)
-    // More threads == faster concurrent loads, but also more concurrent
-    // load on the GPU until we can serialize GPU transfers (off the main thread)
-    QThreadPool::globalInstance()->setMaxThreadCount(2);
     thread()->setPriority(QThread::HighPriority);
     thread()->setObjectName("Main Thread");
 
@@ -732,7 +735,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) :
     connect(&identityPacketTimer, &QTimer::timeout, getMyAvatar(), &MyAvatar::sendIdentityPacket);
     identityPacketTimer.start(AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS);
 
-    ResourceCache::setRequestLimit(3);
+    ResourceCache::setRequestLimit(MAX_CONCURRENT_RESOURCE_DOWNLOADS);
 
     _glWidget = new GLCanvas();
     getApplicationCompositor().setRenderingWidget(_glWidget);
@@ -1245,6 +1248,11 @@ QString Application::getUserAgent() {
             userAgent += " " + formatPluginName(ip->getName());
         }
     }
+    // for codecs, we include all of them, even if not active
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    for (auto& cp : codecPlugins) {
+        userAgent += " " + formatPluginName(cp->getName());
+    }
 
     return userAgent;
 }
@@ -4482,6 +4490,9 @@ void Application::nodeActivated(SharedNodePointer node) {
         }
     }
 
+    if (node->getType() == NodeType::AudioMixer) {
+        DependencyManager::get<AudioClient>()->negotiateAudioFormat();
+    }
 }
 
 void Application::nodeKilled(SharedNodePointer node) {

libraries/audio-client/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(TARGET_NAME audio-client)
 setup_hifi_library(Network Multimedia)
-link_hifi_libraries(audio)
+link_hifi_libraries(audio plugins)
 
 # append audio includes to our list of includes to bubble
 target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src")

libraries/audio-client/src/AudioClient.cpp

@@ -34,6 +34,8 @@
 #include <QtMultimedia/QAudioOutput>
 
 #include <NodeList.h>
+#include <plugins/CodecPlugin.h>
+#include <plugins/PluginManager.h>
 #include <udt/PacketHeaders.h>
 #include <PositionalAudioStream.h>
 #include <SettingHandle.h>
@@ -133,10 +135,15 @@ AudioClient::AudioClient() :
     packetReceiver.registerListener(PacketType::MixedAudio, this, "handleAudioDataPacket");
     packetReceiver.registerListener(PacketType::NoisyMute, this, "handleNoisyMutePacket");
     packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket");
+    packetReceiver.registerListener(PacketType::SelectedAudioFormat, this, "handleSelectedAudioFormat");
 }
 
 AudioClient::~AudioClient() {
     stop();
+    if (_codec && _encoder) {
+        _codec->releaseEncoder(_encoder);
+        _encoder = nullptr;
+    }
 }
 
 void AudioClient::reset() {
@@ -503,6 +510,49 @@ void AudioClient::handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> me
     emit muteEnvironmentRequested(position, radius);
 }
 
+void AudioClient::negotiateAudioFormat() {
+    auto nodeList = DependencyManager::get<NodeList>();
+    auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat);
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    quint8 numberOfCodecs = (quint8)codecPlugins.size();
+    negotiateFormatPacket->writePrimitive(numberOfCodecs);
+    for (auto& plugin : codecPlugins) {
+        auto codecName = plugin->getName();
+        negotiateFormatPacket->writeString(codecName);
+    }
+
+    // grab our audio mixer from the NodeList, if it exists
+    SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+
+    if (audioMixer) {
+        // send off this mute packet
+        nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer);
+    }
+}
+
+void AudioClient::handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message) {
+    _selectedCodecName = message->readString();
+
+    qDebug() << "Selected Codec:" << _selectedCodecName;
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    for (auto& plugin : codecPlugins) {
+        if (_selectedCodecName == plugin->getName()) {
+            // release any old codec encoder/decoder first...
+            if (_codec && _encoder) {
+                _codec->releaseEncoder(_encoder);
+                _encoder = nullptr;
+            }
+            _codec = plugin;
+            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); 
+            _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
+            qDebug() << "Selected Codec Plugin:" << _codec.get();
+            break;
+        }
+    }
+
+}
+   
+
 QString AudioClient::getDefaultDeviceName(QAudio::Mode mode) {
     QAudioDeviceInfo deviceInfo = defaultAudioDeviceForMode(mode);
     return deviceInfo.deviceName();
@@ -770,7 +820,17 @@ void AudioClient::handleAudioInput() {
         audioTransform.setTranslation(_positionGetter());
         audioTransform.setRotation(_orientationGetter());
         // FIXME find a way to properly handle both playback audio and user audio concurrently
-        emitAudioPacket(networkAudioSamples, numNetworkBytes, _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
+
+        // TODO - codec encode goes here
+        QByteArray decocedBuffer(reinterpret_cast<char*>(networkAudioSamples), numNetworkBytes);
+        QByteArray encodedBuffer;
+        if (_encoder) {
+            _encoder->encode(decocedBuffer, encodedBuffer);
+        } else {
+            encodedBuffer = decocedBuffer;
+        }
+
+        emitAudioPacket(encodedBuffer.constData(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, packetType);
         _stats.sentPacket();
     }
 }
@@ -779,23 +839,34 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
     Transform audioTransform;
     audioTransform.setTranslation(_positionGetter());
     audioTransform.setRotation(_orientationGetter());
+
+    // TODO - codec encode goes here
+    QByteArray encodedBuffer;
+    if (_encoder) {
+        _encoder->encode(audio, encodedBuffer);
+    } else {
+        encodedBuffer = audio;
+    }
+
     // FIXME check a flag to see if we should echo audio?
-    emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
+    emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho);
 }
 
-void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
+void AudioClient::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) {
-    const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t);
+    const int numDecodecSamples = decodedBuffer.size() / sizeof(int16_t);
-    const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
+    const int numDeviceOutputSamples = _outputFrameSize;
-        / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount());
+
+    Q_ASSERT(_outputFrameSize == numDecodecSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
+        / (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()));
 
     outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
 
-    const int16_t* receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
+    const int16_t* decodedSamples = reinterpret_cast<const int16_t*>(decodedBuffer.data());
     int16_t* outputSamples = reinterpret_cast<int16_t*>(outputBuffer.data());
 
     // copy the packet from the RB to the output
-    possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples,
+    possibleResampling(_networkToOutputResampler, decodedSamples, outputSamples,
-                       numNetworkOutputSamples, numDeviceOutputSamples,
+                       numDecodecSamples, numDeviceOutputSamples,
                        _desiredOutputFormat, _outputFormat);
 
     // apply stereo reverb at the listener, to the received audio
@@ -1185,6 +1256,13 @@ void AudioClient::loadSettings() {
                                                                         windowSecondsForDesiredCalcOnTooManyStarves.get());
     _receivedAudioStream.setWindowSecondsForDesiredReduction(windowSecondsForDesiredReduction.get());
     _receivedAudioStream.setRepetitionWithFade(repetitionWithFade.get());
+
+    qDebug() << "---- Initializing Audio Client ----";
+    auto codecPlugins = PluginManager::getInstance()->getCodecPlugins();
+    for (auto& plugin : codecPlugins) {
+        qDebug() << "Codec available:" << plugin->getName();
+    }
+
 }
 
 void AudioClient::saveSettings() {

libraries/audio-client/src/AudioClient.h

@@ -38,6 +38,8 @@
 #include <Sound.h>
 #include <StDev.h>
 
+#include <plugins/CodecPlugin.h>
+
 #include "AudioIOStats.h"
 #include "AudioNoiseGate.h"
 #include "AudioSRC.h"
@@ -94,6 +96,8 @@ public:
         int _unfulfilledReads;
     };
 
+    void negotiateAudioFormat();
+
     const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
     MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }
 
@@ -139,6 +143,7 @@ public slots:
     void handleAudioDataPacket(QSharedPointer<ReceivedMessage> message);
     void handleNoisyMutePacket(QSharedPointer<ReceivedMessage> message);
     void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message);
+    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
 
     void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); }
     void handleAudioInput();
@@ -291,6 +296,10 @@ private:
     void checkDevices();
 
     bool _hasReceivedFirstPacket = false;
+
+    CodecPluginPointer _codec;
+    QString _selectedCodecName;
+    Encoder* _encoder { nullptr }; // for outbound mic stream
 };
 
 

libraries/audio/CMakeLists.txt

@@ -1,3 +1,3 @@
 set(TARGET_NAME audio)
 setup_hifi_library(Network)
-link_hifi_libraries(networking shared)
+link_hifi_libraries(networking shared plugins)

libraries/audio/src/AudioInjector.cpp

@@ -289,16 +289,23 @@ int64_t AudioInjector::injectNextFrame() {
 
     _currentPacket->seek(audioDataOffset);
 
+    // This code is copying bytes from the _audioData directly into the packet, handling looping appropriately.
+    // Might be a reasonable place to do the encode step here.
+    QByteArray decodedAudio;
     while (totalBytesLeftToCopy > 0) {
         int bytesToCopy = std::min(totalBytesLeftToCopy, _audioData.size() - _currentSendOffset);
 
-        _currentPacket->write(_audioData.data() + _currentSendOffset, bytesToCopy);
+        decodedAudio.append(_audioData.data() + _currentSendOffset, bytesToCopy);
         _currentSendOffset += bytesToCopy;
         totalBytesLeftToCopy -= bytesToCopy;
         if (_options.loop && _currentSendOffset >= _audioData.size()) {
             _currentSendOffset = 0;
         }
     }
+    // FIXME -- good place to call codec encode here. We need to figure out how to tell the AudioInjector which 
+    // codec to use... possible through AbstractAudioInterface.
+    QByteArray encodedAudio = decodedAudio;
+    _currentPacket->write(encodedAudio.data(), encodedAudio.size());
 
     // set the correct size used for this packet
     _currentPacket->setPayloadSize(_currentPacket->pos());

libraries/audio/src/InboundAudioStream.cpp

@@ -131,7 +131,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) {
             if (message.getType() == PacketType::SilentAudioFrame) {
                 writeDroppableSilentSamples(networkSamples);
             } else {
-                parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkSamples);
+                parseAudioData(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()));
             }
             break;
         }
@@ -172,16 +172,27 @@ int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray&
         return sizeof(quint16);
     } else {
         // mixed audio packets do not have any info between the seq num and the audio data.
-        numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t);
+        numAudioSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
         return 0;
     }
 }
 
-int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
+int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
-    return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
+    QByteArray decodedBuffer;
+    if (_decoder) {
+        _decoder->decode(packetAfterStreamProperties, decodedBuffer);
+    } else {
+        decodedBuffer = packetAfterStreamProperties;
+    }
+    auto actualSize = decodedBuffer.size();
+    return _ringBuffer.writeData(decodedBuffer.data(), actualSize);
 }
 
 int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) {
+    if (_decoder) {
+        _decoder->trackLostFrames(silentSamples);
+    }
+
     // calculate how many silent frames we should drop.
     int samplesPerFrame = _ringBuffer.getNumFrameSamples();
     int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
@@ -497,3 +508,21 @@ float calculateRepeatedFrameFadeFactor(int indexOfRepeat) {
     return 0.0f;
 }
 
+void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels) {
+    cleanupCodec(); // cleanup any previously allocated coders first
+    _codec = codec;
+    _selectedCodecName = codecName;
+    if (_codec) {
+        _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, numChannels);
+    }
+}
+
+void InboundAudioStream::cleanupCodec() {
+    // release any old codec encoder/decoder first...
+    if (_codec) {
+        if (_decoder) {
+            _codec->releaseDecoder(_decoder);
+            _decoder = nullptr;
+        }
+    }
+}

libraries/audio/src/InboundAudioStream.h

@@ -18,6 +18,8 @@
 #include <ReceivedMessage.h>
 #include <StDev.h>
 
+#include <plugins/CodecPlugin.h>
+
 #include "AudioRingBuffer.h"
 #include "MovingMinMaxAvg.h"
 #include "SequenceNumberStats.h"
@@ -103,6 +105,7 @@ public:
 
 public:
     InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings);
+    ~InboundAudioStream() { cleanupCodec(); }
 
     void reset();
     virtual void resetStats();
@@ -174,6 +177,9 @@ public:
     void setReverb(float reverbTime, float wetLevel);
     void clearReverb() { _hasReverb = false; }
 
+    void setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels);
+    void cleanupCodec();
+
 public slots:
     /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers
     /// is enabled, those stats are used to calculate _desiredJitterBufferFrames.
@@ -201,7 +207,7 @@ protected:
 
     /// parses the audio data in the network packet.
     /// default implementation assumes packet contains raw audio samples after stream properties
-    virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+    virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
     /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer
     virtual int writeDroppableSilentSamples(int silentSamples);
@@ -267,6 +273,10 @@ protected:
     bool _hasReverb;
     float _reverbTime;
     float _wetLevel;
+
+    CodecPluginPointer _codec;
+    QString _selectedCodecName;
+    Decoder* _decoder{ nullptr };
 };
 
 float calculateRepeatedFrameFadeFactor(int indexOfRepeat);

libraries/audio/src/MixedProcessedAudioStream.cpp

@@ -42,12 +42,18 @@ int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
     return deviceSamplesWritten;
 }
 
-int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
+int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) {
+    QByteArray decodedBuffer;
+    if (_decoder) {
+        _decoder->decode(packetAfterStreamProperties, decodedBuffer);
+    } else {
+        decodedBuffer = packetAfterStreamProperties;
+    }
 
-    emit addedStereoSamples(packetAfterStreamProperties);
+    emit addedStereoSamples(decodedBuffer);
 
     QByteArray outputBuffer;
-    emit processSamples(packetAfterStreamProperties, outputBuffer);
+    emit processSamples(decodedBuffer, outputBuffer);
 
     _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
     

libraries/audio/src/MixedProcessedAudioStream.h

@@ -35,7 +35,7 @@ public:
 protected:
     int writeDroppableSilentSamples(int silentSamples);
     int writeLastFrameRepeatedWithFade(int samples);
-    int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
+    int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties);
 
 private:
     int networkToDeviceSamples(int networkSamples);

libraries/networking/src/udt/BasePacket.cpp

@@ -152,8 +152,11 @@ QByteArray BasePacket::readWithoutCopy(qint64 maxSize) {
 
 qint64 BasePacket::writeString(const QString& string) {
     QByteArray data = string.toUtf8();
-    writePrimitive(static_cast<uint32_t>(data.length()));
+    uint32_t length = data.length();
-    return writeData(data.constData(), data.length());
+    writePrimitive(length);
+    writeData(data.constData(), data.length());
+    seek(pos() + length);
+    return length + sizeof(uint32_t);
 }
 
 QString BasePacket::readString() {

libraries/networking/src/udt/PacketHeaders.h

@@ -95,7 +95,9 @@ public:
         AssetMappingOperation,
         AssetMappingOperationReply,
         ICEServerHeartbeatACK,
-        LAST_PACKET_TYPE = ICEServerHeartbeatACK
+        NegotiateAudioFormat,
+        SelectedAudioFormat,
+        LAST_PACKET_TYPE = SelectedAudioFormat
     };
 };
 

libraries/plugins/src/plugins/CodecPlugin.h

@@ -0,0 +1,36 @@
+//
+//  CodecPlugin.h
+//  plugins/src/plugins
+//
+//  Created by Brad Hefta-Gaub on 6/9/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#pragma once
+
+#include "Plugin.h"
+
+class Encoder {
+public:
+    virtual ~Encoder() { }
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0;
+};
+
+class Decoder {
+public:
+    virtual ~Decoder() { }
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0;
+
+    // numFrames - number of samples (mono) or sample-pairs (stereo)
+    virtual void trackLostFrames(int numFrames) = 0;
+};
+
+class CodecPlugin : public Plugin {
+public:
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0;
+    virtual void releaseEncoder(Encoder* encoder) = 0;
+    virtual void releaseDecoder(Decoder* decoder) = 0;
+};

libraries/plugins/src/plugins/Forward.h

@@ -13,10 +13,12 @@
 enum class PluginType {
     DISPLAY_PLUGIN,
     INPUT_PLUGIN,
+    CODEC_PLUGIN,
 };
 
 class DisplayPlugin;
 class InputPlugin;
+class CodecPlugin;
 class Plugin;
 class PluginContainer;
 class PluginManager;
@@ -25,4 +27,6 @@ using DisplayPluginPointer = std::shared_ptr<DisplayPlugin>;
 using DisplayPluginList = std::vector<DisplayPluginPointer>;
 using InputPluginPointer = std::shared_ptr<InputPlugin>;
 using InputPluginList = std::vector<InputPluginPointer>;
+using CodecPluginPointer = std::shared_ptr<CodecPlugin>;
+using CodecPluginList = std::vector<CodecPluginPointer>;
 

libraries/plugins/src/plugins/PluginManager.cpp

@@ -18,6 +18,7 @@
 #include <UserActivityLogger.h>
 
 #include "RuntimePlugin.h"
+#include "CodecPlugin.h"
 #include "DisplayPlugin.h"
 #include "InputPlugin.h"
 
@@ -117,6 +118,7 @@ PluginManager::PluginManager() {
 // TODO migrate to a DLL model where plugins are discovered and loaded at runtime by the PluginManager class
 extern DisplayPluginList getDisplayPlugins();
 extern InputPluginList getInputPlugins();
+extern CodecPluginList getCodecPlugins();
 extern void saveInputPluginSettings(const InputPluginList& plugins);
 static DisplayPluginList displayPlugins;
 
@@ -202,6 +204,35 @@ const InputPluginList& PluginManager::getInputPlugins() {
     return inputPlugins;
 }
 
+const CodecPluginList& PluginManager::getCodecPlugins() {
+    static CodecPluginList codecPlugins;
+    static std::once_flag once;
+    std::call_once(once, [&] {
+        //codecPlugins = ::getCodecPlugins();
+
+        // Now grab the dynamic plugins
+        for (auto loader : getLoadedPlugins()) {
+            CodecProvider* codecProvider = qobject_cast<CodecProvider*>(loader->instance());
+            if (codecProvider) {
+                for (auto codecPlugin : codecProvider->getCodecPlugins()) {
+                    if (codecPlugin->isSupported()) {
+                        codecPlugins.push_back(codecPlugin);
+                    }
+                }
+            }
+        }
+
+        for (auto plugin : codecPlugins) {
+            plugin->setContainer(_container);
+            plugin->init();
+
+            qDebug() << "init codec:" << plugin->getName();
+        }
+    });
+    return codecPlugins;
+}
+
+
 void PluginManager::setPreferredDisplayPlugins(const QStringList& displays) {
     preferredDisplayPlugins = displays;
 }

libraries/plugins/src/plugins/PluginManager.h

@@ -18,6 +18,7 @@ public:
 
     const DisplayPluginList& getDisplayPlugins();
     const InputPluginList& getInputPlugins();
+    const CodecPluginList& getCodecPlugins();
 
     DisplayPluginList getPreferredDisplayPlugins();
     void setPreferredDisplayPlugins(const QStringList& displays);

libraries/plugins/src/plugins/RuntimePlugin.h

@@ -34,3 +34,12 @@ public:
 #define InputProvider_iid "com.highfidelity.plugins.input"
 Q_DECLARE_INTERFACE(InputProvider, InputProvider_iid)
 
+class CodecProvider {
+public:
+    virtual ~CodecProvider() {}
+    virtual CodecPluginList getCodecPlugins() = 0;
+};
+
+#define CodecProvider_iid "com.highfidelity.plugins.codec"
+Q_DECLARE_INTERFACE(CodecProvider, CodecProvider_iid)
+

plugins/hifiCodec/CMakeLists.txt

@@ -0,0 +1,19 @@
+#
+#  Created by Brad Hefta-Gaub on 7/10/2016
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+
+if (WIN32 OR APPLE)
+    set(TARGET_NAME hifiCodec)
+    setup_hifi_client_server_plugin()
+
+    link_hifi_libraries(audio shared plugins)
+
+    add_dependency_external_projects(HiFiAudioCodec)
+    target_include_directories(${TARGET_NAME} PRIVATE ${HIFIAUDIOCODEC_INCLUDE_DIRS})
+    target_link_libraries(${TARGET_NAME} ${HIFIAUDIOCODEC_LIBRARIES})
+endif()
+

plugins/hifiCodec/src/HiFiCodec.cpp

@@ -0,0 +1,93 @@
+//
+//  HiFiCodec.cpp
+//  plugins/hifiCodec/src
+//
+//  Created by Brad Hefta-Gaub on 7/10/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <qapplication.h>
+
+#include <AudioCodec.h>
+#include <AudioConstants.h>
+
+#include <PerfStat.h>
+
+#include "HiFiCodec.h"
+
+const QString HiFiCodec::NAME = "hifiAC";
+
+void HiFiCodec::init() {
+}
+
+void HiFiCodec::deinit() {
+}
+
+bool HiFiCodec::activate() {
+    CodecPlugin::activate();
+    return true;
+}
+
+void HiFiCodec::deactivate() {
+    CodecPlugin::deactivate();
+}
+
+
+bool HiFiCodec::isSupported() const {
+    return true;
+}
+
+class HiFiEncoder : public Encoder, public AudioEncoder {
+public:
+    HiFiEncoder(int sampleRate, int numChannels) : AudioEncoder(sampleRate, numChannels) { 
+        _encodedSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels) / 4;  // codec reduces by 1/4th
+    }
+
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+        encodedBuffer.resize(_encodedSize);
+        AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
+    }
+private:
+    int _encodedSize;
+};
+
+class HiFiDecoder : public Decoder, public AudioDecoder {
+public:
+    HiFiDecoder(int sampleRate, int numChannels) : AudioDecoder(sampleRate, numChannels) { 
+        _decodedSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(int16_t) * numChannels;
+    }
+
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+        decodedBuffer.resize(_decodedSize);
+        AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true);
+    }
+
+    virtual void trackLostFrames(int numFrames)  override { 
+        QByteArray encodedBuffer;
+        QByteArray decodedBuffer;
+        decodedBuffer.resize(_decodedSize);
+        // NOTE: we don't actually use the results of this decode, we just do it to keep the state of the codec clean
+        AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false);
+    }
+private:
+    int _decodedSize;
+};
+
+Encoder* HiFiCodec::createEncoder(int sampleRate, int numChannels) {
+    return new HiFiEncoder(sampleRate, numChannels);
+}
+
+Decoder* HiFiCodec::createDecoder(int sampleRate, int numChannels) {
+    return new HiFiDecoder(sampleRate, numChannels);
+}
+
+void HiFiCodec::releaseEncoder(Encoder* encoder) {
+    delete encoder;
+}
+
+void HiFiCodec::releaseDecoder(Decoder* decoder) {
+    delete decoder;
+}

plugins/hifiCodec/src/HiFiCodec.h

@@ -0,0 +1,42 @@
+//
+//  HiFiCodec.h
+//  plugins/hifiCodec/src
+//
+//  Created by Brad Hefta-Gaub on 7/10/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_HiFiCodec_h
+#define hifi_HiFiCodec_h
+
+#include <plugins/CodecPlugin.h>
+
+class HiFiCodec : public CodecPlugin {
+    Q_OBJECT
+    
+public:
+    // Plugin functions
+    bool isSupported() const override;
+    const QString& getName() const override { return NAME; }
+
+    void init() override;
+    void deinit() override;
+
+    /// Called when a plugin is being activated for use.  May be called multiple times.
+    bool activate() override;
+    /// Called when a plugin is no longer being used.  May be called multiple times.
+    void deactivate() override;
+
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+    virtual void releaseEncoder(Encoder* encoder) override;
+    virtual void releaseDecoder(Decoder* decoder) override;
+
+private:
+    static const QString NAME;
+};
+
+#endif // hifi_HiFiCodec_h

plugins/hifiCodec/src/HiFiCodecProvider.cpp

@@ -0,0 +1,46 @@
+//
+//  Created by Brad Hefta-Gaub on 7/10/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <mutex>
+
+#include <QtCore/QObject>
+#include <QtCore/QtPlugin>
+#include <QtCore/QStringList>
+
+#include <plugins/RuntimePlugin.h>
+#include <plugins/CodecPlugin.h>
+
+#include "HiFiCodec.h"
+
+class HiFiCodecProvider : public QObject, public CodecProvider {
+    Q_OBJECT
+    Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json")
+    Q_INTERFACES(CodecProvider)
+
+public:
+    HiFiCodecProvider(QObject* parent = nullptr) : QObject(parent) {}
+    virtual ~HiFiCodecProvider() {}
+
+    virtual CodecPluginList getCodecPlugins() override {
+        static std::once_flag once;
+        std::call_once(once, [&] {
+
+            CodecPluginPointer hiFiCodec(new HiFiCodec());
+            if (hiFiCodec->isSupported()) {
+                _codecPlugins.push_back(hiFiCodec);
+            }
+
+        });
+        return _codecPlugins;
+    }
+
+private:
+    CodecPluginList _codecPlugins;
+};
+
+#include "HiFiCodecProvider.moc"

plugins/hifiCodec/src/plugin.json

@@ -0,0 +1 @@
+{"name":"HiFi 4:1 Audio Codec"}

plugins/pcmCodec/CMakeLists.txt

@@ -0,0 +1,11 @@
+#
+#  Created by Brad Hefta-Gaub on 6/9/2016
+#  Copyright 2016 High Fidelity, Inc.
+#
+#  Distributed under the Apache License, Version 2.0.
+#  See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html
+#
+
+set(TARGET_NAME pcmCodec)
+setup_hifi_client_server_plugin()
+link_hifi_libraries(shared plugins)

plugins/pcmCodec/src/PCMCodecManager.cpp

@@ -0,0 +1,94 @@
+//
+//  PCMCodec.cpp
+//  plugins/pcmCodec/src
+//
+//  Created by Brad Hefta-Gaub on 6/9/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <qapplication.h>
+
+#include <PerfStat.h>
+
+#include "PCMCodecManager.h"
+
+const QString PCMCodec::NAME = "pcm";
+
+void PCMCodec::init() {
+}
+
+void PCMCodec::deinit() {
+}
+
+bool PCMCodec::activate() {
+    CodecPlugin::activate();
+    return true;
+}
+
+void PCMCodec::deactivate() {
+    CodecPlugin::deactivate();
+}
+
+
+bool PCMCodec::isSupported() const {
+    return true;
+}
+
+
+
+Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) {
+    return this;
+}
+
+Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) {
+    return this;
+}
+
+void PCMCodec::releaseEncoder(Encoder* encoder) {
+    // do nothing
+}
+
+void PCMCodec::releaseDecoder(Decoder* decoder) {
+    // do nothing
+}
+
+const QString zLibCodec::NAME = "zlib";
+
+void zLibCodec::init() {
+}
+
+void zLibCodec::deinit() {
+}
+
+bool zLibCodec::activate() {
+    CodecPlugin::activate();
+    return true;
+}
+
+void zLibCodec::deactivate() {
+    CodecPlugin::deactivate();
+}
+
+bool zLibCodec::isSupported() const {
+    return true;
+}
+
+Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) {
+    return this;
+}
+
+Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) {
+    return this;
+}
+
+void zLibCodec::releaseEncoder(Encoder* encoder) {
+    // do nothing... it wasn't allocated
+}
+
+void zLibCodec::releaseDecoder(Decoder* decoder) {
+    // do nothing... it wasn't allocated
+}
+

plugins/pcmCodec/src/PCMCodecManager.h

@@ -0,0 +1,86 @@
+//
+//  PCMCodecManager.h
+//  plugins/pcmCodec/src
+//
+//  Created by Brad Hefta-Gaub on 6/9/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi__PCMCodecManager_h
+#define hifi__PCMCodecManager_h
+
+#include <plugins/CodecPlugin.h>
+
+class PCMCodec : public CodecPlugin, public Encoder, public Decoder {
+    Q_OBJECT
+    
+public:
+    // Plugin functions
+    bool isSupported() const override;
+    const QString& getName() const override { return NAME; }
+
+    void init() override;
+    void deinit() override;
+
+    /// Called when a plugin is being activated for use.  May be called multiple times.
+    bool activate() override;
+    /// Called when a plugin is no longer being used.  May be called multiple times.
+    void deactivate() override;
+
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+    virtual void releaseEncoder(Encoder* encoder) override;
+    virtual void releaseDecoder(Decoder* decoder) override;
+
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+        encodedBuffer = decodedBuffer;
+    }
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+        decodedBuffer = encodedBuffer;
+    }
+
+    virtual void trackLostFrames(int numFrames)  override { }
+
+private:
+    static const QString NAME;
+};
+
+class zLibCodec : public CodecPlugin, public Encoder, public Decoder {
+    Q_OBJECT
+
+public:
+    // Plugin functions
+    bool isSupported() const override;
+    const QString& getName() const override { return NAME; }
+
+    void init() override;
+    void deinit() override;
+
+    /// Called when a plugin is being activated for use.  May be called multiple times.
+    bool activate() override;
+    /// Called when a plugin is no longer being used.  May be called multiple times.
+    void deactivate() override;
+
+    virtual Encoder* createEncoder(int sampleRate, int numChannels) override;
+    virtual Decoder* createDecoder(int sampleRate, int numChannels) override;
+    virtual void releaseEncoder(Encoder* encoder) override;
+    virtual void releaseDecoder(Decoder* decoder) override;
+
+    virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override {
+        encodedBuffer = qCompress(decodedBuffer);
+    }
+
+    virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override {
+        decodedBuffer = qUncompress(encodedBuffer);
+    }
+
+    virtual void trackLostFrames(int numFrames)  override { }
+
+private:
+    static const QString NAME;
+};
+
+#endif // hifi__PCMCodecManager_h

plugins/pcmCodec/src/PCMCodecProvider.cpp

@@ -0,0 +1,51 @@
+//
+//  Created by Brad Hefta-Gaub on 6/9/2016
+//  Copyright 2016 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <mutex>
+
+#include <QtCore/QObject>
+#include <QtCore/QtPlugin>
+#include <QtCore/QStringList>
+
+#include <plugins/RuntimePlugin.h>
+#include <plugins/CodecPlugin.h>
+
+#include "PCMCodecManager.h"
+
+class PCMCodecProvider : public QObject, public CodecProvider {
+    Q_OBJECT
+    Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json")
+    Q_INTERFACES(CodecProvider)
+
+public:
+    PCMCodecProvider(QObject* parent = nullptr) : QObject(parent) {}
+    virtual ~PCMCodecProvider() {}
+
+    virtual CodecPluginList getCodecPlugins() override {
+        static std::once_flag once;
+        std::call_once(once, [&] {
+
+            CodecPluginPointer pcmCodec(new PCMCodec());
+            if (pcmCodec->isSupported()) {
+                _codecPlugins.push_back(pcmCodec);
+            }
+
+            CodecPluginPointer zlibCodec(new zLibCodec());
+            if (zlibCodec->isSupported()) {
+                _codecPlugins.push_back(zlibCodec);
+            }
+
+        });
+        return _codecPlugins;
+    }
+
+private:
+    CodecPluginList _codecPlugins;
+};
+
+#include "PCMCodecProvider.moc"

plugins/pcmCodec/src/plugin.json

@@ -0,0 +1 @@
+{"name":"PCM Codec"}