Merge remote-tracking branch 'upstream/master' into slaps

assignment-client/src/audio/AudioMixer.cpp

@@ -10,7 +10,6 @@
 #include <fcntl.h>
 #include <fstream>
 #include <iostream>
-#include <limits>
 #include <math.h>
 #include <signal.h>
 #include <stdio.h>
@@ -54,10 +53,7 @@
 const short JITTER_BUFFER_MSECS = 12;
 const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0);
 
-const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000);
+const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000);
-
-const int MAX_SAMPLE_VALUE = std::numeric_limits<int16_t>::max();
-const int MIN_SAMPLE_VALUE = std::numeric_limits<int16_t>::min();
 
 const char AUDIO_MIXER_LOGGING_TARGET_NAME[] = "audio-mixer";
 
@@ -160,35 +156,30 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
         }
     }
     
-    int16_t* sourceBuffer = bufferToAdd->getNextOutput();
+    // if the bearing relative angle to source is > 0 then the delayed channel is the right one
+    int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0;
+    int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0;
     
-    int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f)
+    for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 2) {
-        ? _clientSamples
+        if ((s / 2) < numSamplesDelay) {
-        : _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
-    int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f)
-        ? _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL
-        : _clientSamples;
-    
-    int16_t* delaySamplePointer = bufferToAdd->getNextOutput() == bufferToAdd->getBuffer()
-        ? bufferToAdd->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay
-        : bufferToAdd->getNextOutput() - numSamplesDelay;
-    
-    for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) {
-        if (s < numSamplesDelay) {
             // pull the earlier sample for the delayed channel
-            int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio;
+            int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio;
             
-            delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+            _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample,
+                                                                    MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
         }
         
         // pull the current sample for the good channel
-        int16_t currentSample = sourceBuffer[s] * attenuationCoefficient;
+        int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient;
-        goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+        _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample,
+                                                           MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
         
-        if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) {
+        if ((s / 2) + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) {
             // place the curernt sample at the right spot in the delayed channel
-            int sumSample = delayedChannel[s + numSamplesDelay] + (currentSample * weakChannelAmplitudeRatio);
+            int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset]
-            delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+                                               + (currentSample * weakChannelAmplitudeRatio)),
+                                               MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+            _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample;
         }
     }
 }
@@ -282,7 +273,7 @@ void AudioMixer::run() {
     gettimeofday(&startTime, NULL);
     
     int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO);
-    unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader];
+    unsigned char clientPacket[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader];
     populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO);
     
     while (!_isFinished) {

assignment-client/src/audio/AudioMixer.h

@@ -35,7 +35,7 @@ private:
     void prepareMixForListeningNode(Node* node);
     
     
-    int16_t _clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2];
+    int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
 };
 
 #endif /* defined(__hifi__AudioMixer__) */

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -90,17 +90,15 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() {
         // this was a used buffer, push the output pointer forwards
         PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i];
         
-        if (audioBuffer->willBeAddedToMix()) {            
+        if (audioBuffer->willBeAddedToMix()) {
-            audioBuffer->setNextOutput(audioBuffer->getNextOutput() + BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
+            audioBuffer->shiftReadPosition(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
-            
-            if (audioBuffer->getNextOutput() >= audioBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) {
-                audioBuffer->setNextOutput(audioBuffer->getBuffer());
-            }
             
             audioBuffer->setWillBeAddedToMix(false);
-        } else if (audioBuffer->hasStarted() && audioBuffer->isStarved()) {
+        } else if (audioBuffer->isStarved()) {
-            delete audioBuffer;
+            // this was previously the kill for injected audio from a client
-            _ringBuffers.erase(_ringBuffers.begin() + i);
+            // fix when that is added back
+            // delete audioBuffer;
+            // _ringBuffers.erase(_ringBuffers.begin() + i);
         }
     }
 }

interface/src/Application.cpp

@@ -72,7 +72,7 @@ const int IDLE_SIMULATE_MSECS = 16;              //  How often should call simul
                                                  //  in the idle loop?  (60 FPS is default)
 static QTimer* idleTimer = NULL;
 
-const int STARTUP_JITTER_SAMPLES = PACKET_LENGTH_SAMPLES_PER_CHANNEL / 2;
+const int STARTUP_JITTER_SAMPLES = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / 2;
                                                  //  Startup optimistically with small jitter buffer that 
                                                  //  will start playback on the second received audio packet.
 

interface/src/Audio.cpp

@@ -13,6 +13,7 @@
 #include <CoreAudio/AudioHardware.h>
 #endif
 
+#include <QtCore/QBuffer>
 #include <QtMultimedia/QAudioInput>
 #include <QtMultimedia/QAudioOutput>
 #include <QSvgRenderer>
@@ -33,7 +34,7 @@
 static const float JITTER_BUFFER_LENGTH_MSECS = 12;
 static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0);
 
-static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0;
+static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0;
 
 // Mute icon configration
 static const int ICON_SIZE = 24;
@@ -43,12 +44,16 @@ static const int BOTTOM_PADDING = 110;
 Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) :
     QObject(parent),
     _audioInput(NULL),
-    _inputDevice(NULL),
+    _desiredInputFormat(),
+    _inputFormat(),
+    _numInputCallbackBytes(0),
     _audioOutput(NULL),
+    _desiredOutputFormat(),
+    _outputFormat(),
     _outputDevice(NULL),
-    _isBufferSendCallback(false),
+    _numOutputCallbackBytes(0),
-    _nextOutputSamples(NULL),
+    _inputRingBuffer(0),
-    _ringBuffer(true),
+    _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL),
     _scope(scope),
     _averagedLatency(0.0),
     _measuredJitter(0),
@@ -65,7 +70,8 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p
     _numFramesDisplayStarve(0),
     _muted(false)
 {
-    
+    // clear the array of locally injected samples
+    memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
 }
 
 void Audio::init(QGLWidget *parent) {
@@ -124,242 +130,254 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {
     return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice();
 }
 
-const int QT_SAMPLE_RATE = 44100;
+bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
-const int SAMPLE_RATE_RATIO = QT_SAMPLE_RATE / SAMPLE_RATE;
+                                  const QAudioFormat& desiredAudioFormat,
+                                  QAudioFormat& adjustedAudioFormat) {
+    if (!audioDevice.isFormatSupported(desiredAudioFormat)) {
+        qDebug() << "The desired format for audio I/O is" << desiredAudioFormat << "\n";
+        qDebug() << "The desired audio format is not supported by this device.\n";
+        
+        if (desiredAudioFormat.channelCount() == 1) {
+            adjustedAudioFormat = desiredAudioFormat;
+            adjustedAudioFormat.setChannelCount(2);
+            
+            if (audioDevice.isFormatSupported(adjustedAudioFormat)) {
+                return true;
+            } else {
+                adjustedAudioFormat.setChannelCount(1);
+            }
+        }
+        
+        if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) {
+            // use 48, which is a sample downsample, upsample
+            adjustedAudioFormat = desiredAudioFormat;
+            adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2);
+            
+            // return the nearest in case it needs 2 channels
+            adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat);
+            return true;
+        }
+        
+        return false;
+    } else {
+        // set the adjustedAudioFormat to the desiredAudioFormat, since it will work
+        adjustedAudioFormat = desiredAudioFormat;
+        return true;
+    }
+}
+
+void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples,
+                      unsigned int numSourceSamples, unsigned int numDestinationSamples,
+                      const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) {
+    if (sourceAudioFormat == destinationAudioFormat) {
+        memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t));
+    } else {
+        int destinationChannels = (destinationAudioFormat.channelCount() >= 2) ? 2 : destinationAudioFormat.channelCount();
+        float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate())
+            * (sourceAudioFormat.channelCount() / (float) destinationChannels);
+        
+        // take into account the number of channels in source and destination
+        // accomodate for the case where have an output with > 2 channels
+        // this is the case with our HDMI capture
+        
+        if (sourceToDestinationFactor >= 2) {
+            // we need to downsample from 48 to 24
+            // for now this only supports a mono output - this would be the case for audio input
+            
+            for (int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) {
+                if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) {
+                    destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =
+                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 2)
+                        + (sourceSamples[i] / 2);
+                } else {
+                    destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] =
+                        (sourceSamples[i - sourceAudioFormat.channelCount()] / 4)
+                        + (sourceSamples[i] / 2)
+                        + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4);
+                }
+            }
+            
+        } else {
+            // upsample from 24 to 48
+            // for now this only supports a stereo to stereo conversion - this is our case for network audio to output
+            int sourceIndex = 0;
+            int destinationToSourceFactor =  (1 / sourceToDestinationFactor);
+            
+            for (int i = 0; i < numDestinationSamples; i += destinationAudioFormat.channelCount() * destinationToSourceFactor) {
+                sourceIndex = (i / destinationToSourceFactor);
+                
+                // fill the L/R channels and make the rest silent
+                for (int j = i; j < i + (destinationToSourceFactor * destinationAudioFormat.channelCount()); j++) {
+                    if (j % destinationAudioFormat.channelCount() == 0) {
+                        // left channel
+                        destinationSamples[j] = sourceSamples[sourceIndex];
+                    } else if (j % destinationAudioFormat.channelCount() == 1) {
+                         // right channel
+                        destinationSamples[j] = sourceSamples[sourceIndex + 1];
+                    } else {
+                        // channels above 2, fill with silence
+                        destinationSamples[j] = 0;
+                    }
+                }
+            }
+        }
+    }
+}
+
 const int CALLBACK_ACCELERATOR_RATIO = 2;
-const int CALLBACK_IO_BUFFER_SIZE = BUFFER_LENGTH_BYTES_STEREO * SAMPLE_RATE_RATIO / CALLBACK_ACCELERATOR_RATIO;
 
 void Audio::start() {
     
-    QAudioFormat audioFormat;
     // set up the desired audio format
-    audioFormat.setSampleRate(QT_SAMPLE_RATE);
+    _desiredInputFormat.setSampleRate(SAMPLE_RATE);
-    audioFormat.setSampleSize(16);
+    _desiredInputFormat.setSampleSize(16);
-    audioFormat.setCodec("audio/pcm");
+    _desiredInputFormat.setCodec("audio/pcm");
-    audioFormat.setSampleType(QAudioFormat::SignedInt);
+    _desiredInputFormat.setSampleType(QAudioFormat::SignedInt);
-    audioFormat.setByteOrder(QAudioFormat::LittleEndian);
+    _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian);
-    audioFormat.setChannelCount(2);
+    _desiredInputFormat.setChannelCount(1);
     
-    qDebug() << "The format for audio I/O is" << audioFormat << "\n";
+    _desiredOutputFormat = _desiredInputFormat;
+    _desiredOutputFormat.setChannelCount(2);
     
-    QAudioDeviceInfo inputAudioDevice = defaultAudioDeviceForMode(QAudio::AudioInput);
+    QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput);
     
-    qDebug() << "Audio input device is" << inputAudioDevice.deviceName() << "\n";
+    qDebug() << "The audio input device is" << inputDeviceInfo.deviceName() << "\n";
-    if (!inputAudioDevice.isFormatSupported(audioFormat)) {
+    
-        qDebug() << "The desired audio input format is not supported by this device. Not starting audio input.\n";
+    if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {
+        qDebug() << "The format to be used for audio input is" << _inputFormat << "\n";
+        
+        _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);
+        _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount()
+            * (_inputFormat.sampleRate() / SAMPLE_RATE)
+            / CALLBACK_ACCELERATOR_RATIO;
+        _audioInput->setBufferSize(_numInputCallbackBytes);
+        
+        QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);
+        
+        qDebug() << "The audio output device is" << outputDeviceInfo.deviceName() << "\n";
+
+        if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {
+            qDebug() << "The format to be used for audio output is" << _outputFormat << "\n";
+            
+            _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t));
+            _inputDevice = _audioInput->start();
+            connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput()));
+            
+            _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
+            _outputDevice = _audioOutput->start();
+            
+            gettimeofday(&_lastReceiveTime, NULL);
+        }
+        
         return;
     }
     
-    _audioInput = new QAudioInput(inputAudioDevice, audioFormat, this);
+    qDebug() << "Unable to set up audio I/O because of a problem with input or output formats.\n";
-    _audioInput->setBufferSize(CALLBACK_IO_BUFFER_SIZE);
-    _inputDevice = _audioInput->start();
-    
-    connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput()));
-    
-    QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);
-    
-    qDebug() << outputDeviceInfo.supportedSampleRates() << "\n";
-    
-    qDebug() << "Audio output device is" << outputDeviceInfo.deviceName() << "\n";
-    
-    if (!outputDeviceInfo.isFormatSupported(audioFormat)) {
-        qDebug() << "The desired audio output format is not supported by this device.\n";
-        return;
-    }
-    
-    _audioOutput = new QAudioOutput(outputDeviceInfo, audioFormat, this);
-    _audioOutput->setBufferSize(CALLBACK_IO_BUFFER_SIZE);
-    _outputDevice = _audioOutput->start();
-    
-    gettimeofday(&_lastReceiveTime, NULL);
 }
 
 void Audio::handleAudioInput() {
-    static int16_t stereoInputBuffer[CALLBACK_IO_BUFFER_SIZE * 2];
     static char monoAudioDataPacket[MAX_PACKET_SIZE];
-    static int bufferSizeSamples = _audioInput->bufferSize() / sizeof(int16_t);
     
     static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO);
     static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) +  NUM_BYTES_RFC4122_UUID;
+    
     static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes);
     
-    QByteArray inputByteArray = _inputDevice->read(CALLBACK_IO_BUFFER_SIZE);
+    static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO
+        / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL;
     
-    if (_isBufferSendCallback) {
+    static int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio;
-        // copy samples from the inputByteArray to the stereoInputBuffer
-        memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size());
-        
-        //  Measure the loudness of the signal from the microphone and store in audio object
-        float loudness = 0;
-        for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; i += 2) {
-            loudness += abs(stereoInputBuffer[i]);
-        }
-        
-        loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO;
-        _lastInputLoudness = loudness;
-        
-    } else {
-        // this is the first half of a full buffer of data
-        // zero out the monoAudioSamples array
-        memset(monoAudioSamples, 0, BUFFER_LENGTH_BYTES_PER_CHANNEL);
-        
-        // take samples we have in this callback and store them in the first half of the static buffer
-        // to send off in the next callback
-        memcpy((char*) stereoInputBuffer, inputByteArray.data(), inputByteArray.size());
-    }
     
-    // add input data just written to the scope
+    QByteArray inputByteArray = _inputDevice->readAll();
-    QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection,
-                              Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true));
     
-    QByteArray stereoOutputBuffer;
+    _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
     
-    if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) {
+    while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) {
-        //  if local loopback enabled, copy input to output
+        
-        if (_isBufferSendCallback) {
+        int16_t inputAudioSamples[inputSamplesRequired];
-            stereoOutputBuffer.append((char*) (stereoInputBuffer + bufferSizeSamples), CALLBACK_IO_BUFFER_SIZE);
+        _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired);
+        
+        // zero out the monoAudioSamples array and the locally injected audio
+        memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
+        
+        // zero out the locally injected audio in preparation for audio procedural sounds
+        memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
+        
+        if (!_muted) {
+            // we aren't muted, downsample the input audio
+            linearResampling((int16_t*) inputAudioSamples,
+                             monoAudioSamples,
+                             inputSamplesRequired,
+                             NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
+                             _inputFormat, _desiredInputFormat);
+            
+            float loudness = 0;
+            
+            for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {
+                loudness += fabsf(monoAudioSamples[i]);
+            }
+            
+            _lastInputLoudness = loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
+            
+            // add input data just written to the scope
+            QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection,
+                                      Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples,
+                                                                   NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)),
+                                      Q_ARG(bool, false), Q_ARG(bool, true));
+            
+            if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) {
+                // if this person wants local loopback add that to the locally injected audio
+                memcpy(_localInjectedSamples, monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
+            }
         } else {
-            stereoOutputBuffer.append((char*) stereoInputBuffer, CALLBACK_IO_BUFFER_SIZE);
+            // our input loudness is 0, since we're muted
+            _lastInputLoudness = 0;
         }
-    } else {
+        
-        // zero out the stereoOutputBuffer
+        // add procedural effects to the appropriate input samples
-        stereoOutputBuffer = QByteArray(CALLBACK_IO_BUFFER_SIZE, 0);
+        addProceduralSounds(monoAudioSamples,
-    }
+                            NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
-    
+        
-    // add procedural effects to the appropriate input samples
-    addProceduralSounds(monoAudioSamples + (_isBufferSendCallback
-                                            ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0),
-                        (int16_t*) stereoOutputBuffer.data(),
-                        BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO);
-    
-    if (_isBufferSendCallback) {
         NodeList* nodeList = NodeList::getInstance();
         Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER);
         
-        if (audioMixer) {
+        if (audioMixer && nodeList->getNodeActiveSocketOrPing(audioMixer)) {
-            if (audioMixer->getActiveSocket()) {
+            MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar();
-                MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar();
+            
-                
+            glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition();
-                glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition();
+            glm::quat headOrientation = interfaceAvatar->getHead().getOrientation();
-                glm::quat headOrientation = interfaceAvatar->getHead().getOrientation();
+            
-                
+            // we need the amount of bytes in the buffer + 1 for type
-                // we need the amount of bytes in the buffer + 1 for type
+            // + 12 for 3 floats for position + float for bearing + 1 attenuation byte
-                // + 12 for 3 floats for position + float for bearing + 1 attenuation byte
+            
-                
+            PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)
-                PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)
+                ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO;
-                    ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO;
+            
-                
+            char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket,
-                char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket,
+                                                                                  packetType);
-                                                                                      packetType);
+            
-                
+            // pack Source Data
-                // pack Source Data
+            QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122();
-                QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122();
+            memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size());
-                memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size());
+            currentPacketPtr += rfcUUID.size();
-                currentPacketPtr += rfcUUID.size();
+            
-                
+            // memcpy the three float positions
-                // memcpy the three float positions
+            memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));
-                memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));
+            currentPacketPtr += (sizeof(headPosition));
-                currentPacketPtr += (sizeof(headPosition));
+            
-                
+            // memcpy our orientation
-                // memcpy our orientation
+            memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));
-                memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));
+            currentPacketPtr += sizeof(headOrientation);
-                currentPacketPtr += sizeof(headOrientation);
+            
-                
+            nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket,
-                if (!_muted) {
+                                                    NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes,
-                    // we aren't muted, average each set of four samples together to set up the mono input buffers
+                                                    audioMixer->getActiveSocket()->getAddress(),
-                    for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO; i += 4) {
+                                                    audioMixer->getActiveSocket()->getPort());
-                        
+            
-                        int16_t averagedSample = 0;
+            Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO)
-                        if (i + 2 == BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO) {
+                .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes);
-                            averagedSample = (stereoInputBuffer[i - 2] / 2) + (stereoInputBuffer[i] / 2);
-                        } else {
-                            averagedSample = (stereoInputBuffer[i - 2] / 4) + (stereoInputBuffer[i] / 2)
-                            + (stereoInputBuffer[i + 2] / 4);
-                        }
-                        
-                        // add the averaged sample to our array of audio samples
-                        monoAudioSamples[(i - 2) / 4] += averagedSample;
-                    }
-                }
-                
-                nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes,
-                                                        audioMixer->getActiveSocket()->getAddress(),
-                                                        audioMixer->getActiveSocket()->getPort());
-                
-                Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO)
-                    .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes);
-            } else {
-                nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer);
-            }
         }
     }
-    
-    // if there is anything in the ring buffer, decide what to do
-    
-    if (!_nextOutputSamples) {
-        if (_ringBuffer.getEndOfLastWrite()) {
-            if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() <
-                (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) {
-                //  If not enough audio has arrived to start playback, keep waiting
-            } else if (!_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() == 0) {
-                //  If we have started and now have run out of audio to send to the audio device,
-                //  this means we've starved and should restart.
-                _ringBuffer.setIsStarved(true);
-                
-                // show a starve in the GUI for 10 frames
-                _numFramesDisplayStarve = 10;
-                
-            } else {
-                //  We are either already playing back, or we have enough audio to start playing back.
-                if (_ringBuffer.isStarved()) {
-                    _ringBuffer.setIsStarved(false);
-                    _ringBuffer.setHasStarted(true);
-                }
-                
-                _nextOutputSamples = _ringBuffer.getNextOutput();
-            }
-        }
-    }
-    
-    if (_nextOutputSamples) {
-        
-        int16_t* stereoOutputBufferSamples = (int16_t*) stereoOutputBuffer.data();
-        
-        // play whatever we have in the audio buffer
-        for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO; s++) {
-            int16_t leftSample = _nextOutputSamples[s];
-            int16_t rightSample = _nextOutputSamples[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL];
-            
-            stereoOutputBufferSamples[(s * 4)] += leftSample;
-            stereoOutputBufferSamples[(s * 4) + 2] += leftSample;
-            
-            stereoOutputBufferSamples[(s * 4) + 1] += rightSample;
-            stereoOutputBufferSamples[(s * 4) + 3] += rightSample;
-        }
-        
-        if (_isBufferSendCallback) {
-            _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES);
-            
-            if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) {
-                _ringBuffer.setNextOutput(_ringBuffer.getBuffer());
-            }
-            
-            _nextOutputSamples = NULL;
-        } else {
-            _nextOutputSamples += PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO;
-        }
-    }
-    
-    _outputDevice->write(stereoOutputBuffer);
-    
-    
-    // add output (@speakers) data just written to the scope
-    QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection,
-                              Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false));
-    
-    _isBufferSendCallback = !_isBufferSendCallback;
-    
-    gettimeofday(&_lastCallbackTime, NULL);
 }
 
 void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) {
@@ -381,7 +399,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) {
         _measuredJitter = _stdev.getStDev();
         _stdev.reset();
         //  Set jitter buffer to be a multiple of the measured standard deviation
-        const int MAX_JITTER_BUFFER_SAMPLES = RING_BUFFER_LENGTH_SAMPLES / 2;
+        const int MAX_JITTER_BUFFER_SAMPLES = _ringBuffer.getSampleCapacity() / 2;
         const float NUM_STANDARD_DEVIATIONS = 3.f;
         if (Menu::getInstance()->getAudioJitterBufferSamples() == 0) {
             float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter) / 1000.f * SAMPLE_RATE;
@@ -389,22 +407,69 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) {
         }
     }
     
-    if (_ringBuffer.diffLastWriteNextOutput() + PACKET_LENGTH_SAMPLES >
-        PACKET_LENGTH_SAMPLES + (ceilf((float) (_jitterBufferSamples * 2) / PACKET_LENGTH_SAMPLES) * PACKET_LENGTH_SAMPLES)) {
-        // this packet would give us more than the required amount for play out
-        // discard the first packet in the buffer
-        
-        _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES);
-        
-        if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) {
-            _ringBuffer.setNextOutput(_ringBuffer.getBuffer());
-        }
-    }
-    
     _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size());
     
-    Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(PACKET_LENGTH_BYTES
+    static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate())
-                                                                                                    + sizeof(PACKET_TYPE));
+        * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount());
+    
+    static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio;
+    
+    QByteArray outputBuffer;
+    outputBuffer.resize(numRequiredOutputSamples * sizeof(int16_t));
+    
+    // if there is anything in the ring buffer, decide what to do
+    if (_ringBuffer.samplesAvailable() > 0) {
+        if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO
+                                                         + (_jitterBufferSamples * 2))) {
+            // starved and we don't have enough to start, keep waiting
+            qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n";
+        } else {
+            //  We are either already playing back, or we have enough audio to start playing back.
+            _ringBuffer.setIsStarved(false);
+            
+            // copy the samples we'll resample from the ring buffer - this also
+            // pushes the read pointer of the ring buffer forwards
+            int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
+            _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO);
+            
+            // add to the output samples whatever is in the _localAudioOutput byte array
+            // that lets this user hear sound effects and loopback (if enabled)
+            
+            for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {
+                ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i],
+                                                      MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+                ringBufferSamples[(i * 2) + 1] += glm::clamp(ringBufferSamples[(i * 2) + 1] + _localInjectedSamples[i],
+                                                             MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+            }
+            
+            // copy the packet from the RB to the output
+            linearResampling(ringBufferSamples,
+                             (int16_t*) outputBuffer.data(),
+                             NETWORK_BUFFER_LENGTH_SAMPLES_STEREO,
+                             numRequiredOutputSamples,
+                             _desiredOutputFormat, _outputFormat);
+            
+            if (_outputDevice) {
+                
+                _outputDevice->write(outputBuffer);
+                
+                // add output (@speakers) data just written to the scope
+                QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection,
+                                          Q_ARG(QByteArray, QByteArray((char*) ringBufferSamples,
+                                                                       NETWORK_BUFFER_LENGTH_BYTES_STEREO)),
+                                          Q_ARG(bool, true), Q_ARG(bool, false));
+            }
+        }
+        
+    } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) {
+        // we don't have any audio data left in the output buffer, and the ring buffer from
+        // the network has nothing in it either - we just starved
+        qDebug() << "Audio output just starved.\n";
+        _ringBuffer.setIsStarved(true);
+        _numFramesDisplayStarve = 10;
+    }
+    
+    Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size());
     
     _lastReceiveTime = currentReceiveTime;
 }
@@ -435,7 +500,7 @@ void Audio::render(int screenWidth, int screenHeight) {
         glVertex2f(currentX, topY);
         glVertex2f(currentX, bottomY);
         
-        for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) {
+        for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES; i++) {
             glVertex2f(currentX, halfY);
             glVertex2f(currentX + frameWidth, halfY);
             currentX += frameWidth;
@@ -445,17 +510,15 @@ void Audio::render(int screenWidth, int screenHeight) {
         }
         glEnd();
         
-        //  Show a bar with the amount of audio remaining in ring buffer beyond current playback
+        // show a bar with the amount of audio remaining in ring buffer and output device
-        float remainingBuffer = 0;
+        // beyond the current playback
-        timeval currentTime;
-        gettimeofday(&currentTime, NULL);
-        float timeLeftInCurrentBuffer = 0;
-        if (_lastCallbackTime.tv_usec > 0) {
-            timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, &currentTime);
-        }
         
-        if (_ringBuffer.getEndOfLastWrite() != NULL)
+        int bytesLeftInAudioOutput = _audioOutput->bufferSize() - _audioOutput->bytesFree();
-            remainingBuffer = _ringBuffer.diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS;
+        float secondsLeftForAudioOutput = (bytesLeftInAudioOutput / sizeof(int16_t))
+            / ((float) _outputFormat.sampleRate() * _outputFormat.channelCount());
+        float secondsLeftForRingBuffer = _ringBuffer.samplesAvailable()
+            / ((float) _desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount());
+        float msLeftForAudioOutput = (secondsLeftForAudioOutput + secondsLeftForRingBuffer) * 1000;
         
         if (_numFramesDisplayStarve == 0) {
             glColor3f(0, 1, 0);
@@ -464,19 +527,19 @@ void Audio::render(int screenWidth, int screenHeight) {
             _numFramesDisplayStarve--;
         }
         
+        if (_averagedLatency == 0.0) {
+            _averagedLatency = msLeftForAudioOutput;
+        } else {
+            _averagedLatency = 0.99f * _averagedLatency + 0.01f * (msLeftForAudioOutput);
+        }
+        
         glBegin(GL_QUADS);
         glVertex2f(startX, topY + 2);
-        glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2);
+        glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2);
-        glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2);
+        glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2);
         glVertex2f(startX, bottomY - 2);
         glEnd();
         
-        if (_averagedLatency == 0.0) {
-            _averagedLatency = remainingBuffer + timeLeftInCurrentBuffer;
-        } else {
-            _averagedLatency = 0.99f * _averagedLatency + 0.01f * (remainingBuffer + timeLeftInCurrentBuffer);
-        }
-        
         //  Show a yellow bar with the averaged msecs latency you are hearing (from time of packet receipt)
         glColor3f(1,1,0);
         glBegin(GL_QUADS);
@@ -493,7 +556,8 @@ void Audio::render(int screenWidth, int screenHeight) {
         //  Show a red bar with the 'start' point of one frame plus the jitter buffer
         
         glColor3f(1, 0, 0);
-        int jitterBufferPels = (1.f + (float)getJitterBufferSamples() / (float) PACKET_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth;
+        int jitterBufferPels = (1.f + (float)getJitterBufferSamples()
+                                / (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth;
         sprintf(out, "%.0f\n", getJitterBufferSamples() / SAMPLE_RATE * 1000.f);
         drawtext(startX + jitterBufferPels - 5, topY - 9, 0.10, 0, 1, 0, out, 1, 0, 0);
         sprintf(out, "j %.1f\n", _measuredJitter);
@@ -515,7 +579,7 @@ void Audio::render(int screenWidth, int screenHeight) {
 }
 
 //  Take a pointer to the acquired microphone input samples and add procedural sounds
-void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples) {
+void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) {
     const float MAX_AUDIBLE_VELOCITY = 6.0;
     const float MIN_AUDIBLE_VELOCITY = 0.1;
     const int VOLUME_BASELINE = 400;
@@ -551,11 +615,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp
             
             int16_t collisionSample = (int16_t) sample;
             
-            monoInput[i] += collisionSample;
+            monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
-            
+            _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample,
-            for (int j = (i * 4); j < (i * 4) + 4; j++) {
+                                                  MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
-                stereoUpsampledOutput[j] += collisionSample;
-            }
             
             _collisionSoundMagnitude *= _collisionSoundDuration;
         }
@@ -577,11 +639,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp
             
             int16_t collisionSample = (int16_t) sample;
             
-            monoInput[i] += collisionSample;
+            monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
-            
+            _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample,
-            for (int j = (i * 4); j < (i * 4) + 4; j++) {
+                                                  MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
-                stereoUpsampledOutput[j] += collisionSample;
-            }
             
             _drumSoundVolume *= (1.f - _drumSoundDecay);
         }

interface/src/Audio.h

@@ -15,6 +15,7 @@
 #include "InterfaceConfig.h"
 
 #include <QtCore/QObject>
+#include <QtMultimedia/QAudioFormat>
 
 #include <AbstractAudioInterface.h>
 #include <AudioRingBuffer.h>
@@ -26,11 +27,6 @@
 
 static const int NUM_AUDIO_CHANNELS = 2;
 
-static const int PACKET_LENGTH_BYTES = 1024;
-static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2;
-static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t);
-static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2;
-
 class QAudioInput;
 class QAudioOutput;
 class QIODevice;
@@ -70,16 +66,22 @@ public slots:
     void reset();
     
 private:
+    QByteArray firstInputFrame;
     QAudioInput* _audioInput;
+    QAudioFormat _desiredInputFormat;
+    QAudioFormat _inputFormat;
     QIODevice* _inputDevice;
+    int16_t _localInjectedSamples[NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL];
+    int _numInputCallbackBytes;
     QAudioOutput* _audioOutput;
+    QAudioFormat _desiredOutputFormat;
+    QAudioFormat _outputFormat;
     QIODevice* _outputDevice;
-    bool _isBufferSendCallback;
+    int _numOutputCallbackBytes;
-    int16_t* _nextOutputSamples;
+    AudioRingBuffer _inputRingBuffer;
     AudioRingBuffer _ringBuffer;
     Oscilloscope* _scope;
     StDev _stdev;
-    timeval _lastCallbackTime;
     timeval _lastReceiveTime;
     float _averagedLatency;
     float _measuredJitter;
@@ -114,7 +116,7 @@ private:
     inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight);
     
     // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
-    void addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples);
+    void addProceduralSounds(int16_t* monoInput, int numSamples);
     
     void renderToolIcon(int screenHeight);
 };

interface/src/Oscilloscope.cpp

@@ -68,24 +68,18 @@ Oscilloscope::~Oscilloscope() {
     delete[] _samples;
 }
 
-void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInput) {
+void Oscilloscope::addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput) {
 
     if (! enabled || inputPaused) {
         return;
     }
     
-    unsigned int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * 2);
+    int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * (isStereo ? 2 : 1));
-    int16_t samples[numSamplesPerChannel];
+    int16_t* samples = (int16_t*) audioByteArray.data();
-    const int16_t* stereoSamples = (int16_t*) audioByteArray.constData();
     
-    for (int channel = 0; channel < (isInput ? 1 : 2); channel++) {
+    for (int channel = 0; channel < (isStereo ? 2 : 1); channel++) {
         // add samples for each of the channels
-        
+
-        // enumerate the interleaved stereoSamples array and pull out the samples for this channel
-        for (int i = 0; i < audioByteArray.size() / sizeof(int16_t); i += 2) {
-            samples[i / 2] = stereoSamples[i + channel];
-        }
-        
         // determine start/end offset of this channel's region
         unsigned baseOffs = MAX_SAMPLES_PER_CHANNEL * (channel + !isInput);
         unsigned endOffs = baseOffs + MAX_SAMPLES_PER_CHANNEL;
@@ -103,10 +97,21 @@ void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInp
             numSamplesPerChannel -= n2;
         }
         
-        // copy data
+        if (!isStereo) {
-        memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t));
+            // copy data
-        if (n2 > 0) {
+            memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t));
-            memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t));
+            if (n2 > 0) {
+                memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t));
+            }
+        } else {
+            // we have interleaved samples we need to separate into two channels
+            for (int i = 0; i < numSamplesPerChannel + n2; i++) {
+                if (i < numSamplesPerChannel - n2) {
+                    _samples[writePos] = samples[(i * 2) + channel];
+                } else {
+                    _samples[baseOffs] = samples[(i * 2) + channel];
+                }
+            }
         }
         
         // set new write position for this channel 

interface/src/Oscilloscope.h

@@ -59,7 +59,7 @@ public:
     // just uses every nTh sample.
     void setDownsampleRatio(unsigned n) { assert(n > 0); _downsampleRatio = n; }
 public slots:
-    void addStereoSamples(const QByteArray& audioByteArray, bool isInput);
+    void addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput);
 private:
     // don't copy/assign
     Oscilloscope(Oscilloscope const&); // = delete;

libraries/audio/src/AudioRingBuffer.cpp

@@ -9,19 +9,27 @@
 #include <cstring>
 #include <math.h>
 
+#include <QtCore/QDebug>
+
 #include "PacketHeaders.h"
 
 #include "AudioRingBuffer.h"
 
-AudioRingBuffer::AudioRingBuffer(bool isStereo) :
+AudioRingBuffer::AudioRingBuffer(int numFrameSamples) :
     NodeData(NULL),
-    _endOfLastWrite(NULL),
+    _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES),
     _isStarved(true),
-    _hasStarted(false),
+    _hasStarted(false)
-    _isStereo(isStereo)
 {
-    _buffer = new int16_t[RING_BUFFER_LENGTH_SAMPLES];
+    if (numFrameSamples) {
-    _nextOutput = _buffer;
+        _buffer = new int16_t[_sampleCapacity];
+        _nextOutput = _buffer;
+        _endOfLastWrite = _buffer;
+    } else {
+        _buffer = NULL;
+        _nextOutput = NULL;
+        _endOfLastWrite = NULL;
+    }
 };
 
 AudioRingBuffer::~AudioRingBuffer() {
@@ -32,53 +40,130 @@ void AudioRingBuffer::reset() {
     _endOfLastWrite = _buffer;
     _nextOutput = _buffer;
     _isStarved = true;
-    _hasStarted = false;
+}
+
+void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) {
+    delete[] _buffer;
+    _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES;
+    _buffer = new int16_t[_sampleCapacity];
+    _nextOutput = _buffer;
+    _endOfLastWrite = _buffer;
 }
 
 int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) {
     int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer);
-    return parseAudioSamples(sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader);
+    return writeData((char*) sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader);
 }
 
-int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes) {
+qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) {
+    return readData((char*) destination, maxSamples * sizeof(int16_t));
+}
+
+qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) {
+    
+    // only copy up to the number of samples we have available
+    int numReadSamples = std::min((unsigned) (maxSize / sizeof(int16_t)), samplesAvailable());
+    
+    if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) {
+        // we're going to need to do two reads to get this data, it wraps around the edge
+        
+        // read to the end of the buffer
+        int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput;
+        memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t));
+        
+        // read the rest from the beginning of the buffer
+        memcpy(data + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t));
+    } else {
+        // read the data
+        memcpy(data, _nextOutput, numReadSamples * sizeof(int16_t));
+    }
+    
+    // push the position of _nextOutput by the number of samples read
+    _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples);
+    
+    return numReadSamples * sizeof(int16_t);
+}
+
+qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) {
+    return writeData((const char*) source, maxSamples * sizeof(int16_t));
+}
+
+qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) {
     // make sure we have enough bytes left for this to be the right amount of audio
     // otherwise we should not copy that data, and leave the buffer pointers where they are
-    int samplesToCopy = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (_isStereo ? 2 : 1);
     
-    if (numBytes == samplesToCopy * sizeof(int16_t)) {
+    int samplesToCopy = std::min(maxSize / sizeof(int16_t), (quint64) _sampleCapacity);
-        
+    
-        if (!_endOfLastWrite) {
+    std::less<int16_t*> less;
-            _endOfLastWrite = _buffer;
+    std::less_equal<int16_t*> lessEqual;
-        } else if (diffLastWriteNextOutput() > RING_BUFFER_LENGTH_SAMPLES - samplesToCopy) {
+    
-            _endOfLastWrite = _buffer;
+    if (_hasStarted
-            _nextOutput = _buffer;
+        && (less(_endOfLastWrite, _nextOutput)
-            _isStarved = true;
+            && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) {
-        }
+        // this read will cross the next output, so call us starved and reset the buffer
-        
+        qDebug() << "Filled the ring buffer. Resetting.\n";
-        memcpy(_endOfLastWrite, sourceBuffer, numBytes);
+        _endOfLastWrite = _buffer;
-        
+        _nextOutput = _buffer;
-        _endOfLastWrite += samplesToCopy;
+        _isStarved = true;
-        
+    }
-        if (_endOfLastWrite >= _buffer + RING_BUFFER_LENGTH_SAMPLES) {
+    
-            _endOfLastWrite = _buffer;
+    _hasStarted = true;
-        }
+    
-        
+    if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) {
-        return numBytes;
+        memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t));
     } else {
-        return 0;
+        int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite;
-    }    
+        memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t));
+        memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t));
+    }
+    
+    _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy);
+    
+    return samplesToCopy * sizeof(int16_t);
 }
 
-int AudioRingBuffer::diffLastWriteNextOutput() const {
+int16_t& AudioRingBuffer::operator[](const int index) {
+    // make sure this is a valid index
+    assert(index > -_sampleCapacity && index < _sampleCapacity);
+    
+    return *shiftedPositionAccomodatingWrap(_nextOutput, index);
+}
+
+void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) {
+    _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples);
+}
+
+unsigned int AudioRingBuffer::samplesAvailable() const {
     if (!_endOfLastWrite) {
         return 0;
     } else {
         int sampleDifference = _endOfLastWrite - _nextOutput;
         
         if (sampleDifference < 0) {
-            sampleDifference += RING_BUFFER_LENGTH_SAMPLES;
+            sampleDifference += _sampleCapacity;
         }
         
         return sampleDifference;
     }
 }
+
+bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const {
+    if (!_isStarved) {
+        return true;
+    } else {
+        return samplesAvailable() >= numRequiredSamples;
+    }
+}
+
+int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const {
+    
+    if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) {
+        // this shift will wrap the position around to the beginning of the ring
+        return position + numSamplesShift - _sampleCapacity;
+    } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) {
+        // this shift will go around to the end of the ring
+        return position + numSamplesShift - _sampleCapacity;
+    } else {
+        return position + numSamplesShift;
+    }
+}

libraries/audio/src/AudioRingBuffer.h

@@ -9,61 +9,69 @@
 #ifndef __interface__AudioRingBuffer__
 #define __interface__AudioRingBuffer__
 
+#include <limits>
 #include <stdint.h>
-#include <map>
 
 #include <glm/glm.hpp>
 
+#include <QtCore/QIODevice>
+
 #include "NodeData.h"
 
-const int SAMPLE_RATE = 22050;
+const int SAMPLE_RATE = 24000;
 
-const int BUFFER_LENGTH_BYTES_STEREO = 1024;
+const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024;
-const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512;
+const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STEREO / sizeof(int16_t);
-const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t);
+const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512;
+const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t);
 
-const short RING_BUFFER_LENGTH_FRAMES = 20;
+const short RING_BUFFER_LENGTH_FRAMES = 10;
-const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
+
+const int MAX_SAMPLE_VALUE = std::numeric_limits<int16_t>::max();
+const int MIN_SAMPLE_VALUE = std::numeric_limits<int16_t>::min();
 
 class AudioRingBuffer : public NodeData {
+    Q_OBJECT
 public:
-    AudioRingBuffer(bool isStereo);
+    AudioRingBuffer(int numFrameSamples);
     ~AudioRingBuffer();
 
     void reset();
-
+    void resizeForFrameSize(qint64 numFrameSamples);
+    
+    int getSampleCapacity() const { return _sampleCapacity; }
+    
     int parseData(unsigned char* sourceBuffer, int numBytes);
-    int parseAudioSamples(unsigned char* sourceBuffer, int numBytes);
 
-    int16_t* getNextOutput() const { return _nextOutput; }
+    qint64 readSamples(int16_t* destination, qint64 maxSamples);
-    void setNextOutput(int16_t* nextOutput) { _nextOutput = nextOutput; }
+    qint64 writeSamples(const int16_t* source, qint64 maxSamples);
     
-    int16_t* getEndOfLastWrite() const { return _endOfLastWrite; }
+    qint64 readData(char* data, qint64 maxSize);
-    void setEndOfLastWrite(int16_t* endOfLastWrite) { _endOfLastWrite = endOfLastWrite; }
+    qint64 writeData(const char* data, qint64 maxSize);
     
-    int16_t* getBuffer() const { return _buffer; }
+    int16_t& operator[](const int index);
+    
+    void shiftReadPosition(unsigned int numSamples);
+    
+    unsigned int samplesAvailable() const;
+    
+    bool isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const;
     
     bool isStarved() const { return _isStarved; }
     void setIsStarved(bool isStarved) { _isStarved = isStarved; }
-    
-    bool hasStarted() const { return _hasStarted; }
-    void setHasStarted(bool hasStarted) { _hasStarted = hasStarted; }
-    
-    int diffLastWriteNextOutput() const;
-    
-    bool isStereo() const { return _isStereo; }
-    
 protected:
     // disallow copying of AudioRingBuffer objects
     AudioRingBuffer(const AudioRingBuffer&);
     AudioRingBuffer& operator= (const AudioRingBuffer&);
     
+    int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const;
+    
+    int _sampleCapacity;
     int16_t* _nextOutput;
     int16_t* _endOfLastWrite;
     int16_t* _buffer;
     bool _isStarved;
     bool _hasStarted;
-    bool _isStereo;
 };
 
 #endif /* defined(__interface__AudioRingBuffer__) */

libraries/audio/src/InjectedAudioRingBuffer.cpp

@@ -42,7 +42,7 @@ int InjectedAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes
     unsigned int attenuationByte = *(currentBuffer++);
     _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME;
     
-    currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer));
+    currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer));
     
     return currentBuffer - sourceBuffer;
 }

libraries/audio/src/PositionalAudioRingBuffer.cpp

@@ -15,7 +15,7 @@
 #include "PositionalAudioRingBuffer.h"
 
 PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type) :
-    AudioRingBuffer(false),
+    AudioRingBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL),
     _type(type),
     _position(0.0f, 0.0f, 0.0f),
     _orientation(0.0f, 0.0f, 0.0f, 0.0f),
@@ -31,7 +31,7 @@ int PositionalAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numByt
     unsigned char* currentBuffer = sourceBuffer + numBytesForPacketHeader(sourceBuffer);
     currentBuffer += NUM_BYTES_RFC4122_UUID; // the source UUID
     currentBuffer += parsePositionalData(currentBuffer, numBytes - (currentBuffer - sourceBuffer));
-    currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer));
+    currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer));
     
     return currentBuffer - sourceBuffer;
 }
@@ -47,8 +47,7 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer,
     
     // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
     if (std::isnan(_orientation.x)) {
-        _endOfLastWrite = _nextOutput = _buffer;
+        reset();
-        _isStarved = true;
         return 0;
     }
     
@@ -56,20 +55,17 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer,
 }
 
 bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) {
-    if (_endOfLastWrite) {
+    if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) {
-        if (_isStarved && diffLastWriteNextOutput() <= BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples) {
+        qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n";
-            printf("Buffer held back\n");
+        return false;
-            return false;
+    } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) {
-        } else if (diffLastWriteNextOutput() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) {
+        qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n";
-            printf("Buffer starved.\n");
+        _isStarved = true;
-            _isStarved = true;
+        return false;
-            return false;
+    } else {
-        } else {
+        // good buffer, add this to the mix
-            // good buffer, add this to the mix
+        _isStarved = false;
-            _isStarved = false;
+        return true;
-            _hasStarted = true;
-            return true;
-        }
     }
     
     return false;