From 098010e0991284bb191b086f7573fd4fee319e41 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 6 Dec 2013 13:22:07 -0800 Subject: [PATCH 01/24] don't crash if there is no output --- interface/src/Audio.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c3565a4eb0..993f655a99 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -142,15 +142,15 @@ void Audio::start() { qDebug() << "The format for audio I/O is" << audioFormat << "\n"; - QAudioDeviceInfo inputAudioDevice = defaultAudioDeviceForMode(QAudio::AudioInput); + QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); - qDebug() << "Audio input device is" << inputAudioDevice.deviceName() << "\n"; - if (!inputAudioDevice.isFormatSupported(audioFormat)) { + qDebug() << "Audio input device is" << inputDeviceInfo.deviceName() << "\n"; + if (!inputDeviceInfo.isFormatSupported(audioFormat)) { qDebug() << "The desired audio input format is not supported by this device. Not starting audio input.\n"; return; } - _audioInput = new QAudioInput(inputAudioDevice, audioFormat, this); + _audioInput = new QAudioInput(inputDeviceInfo, audioFormat, this); _audioInput->setBufferSize(CALLBACK_IO_BUFFER_SIZE); _inputDevice = _audioInput->start(); @@ -158,8 +158,6 @@ void Audio::start() { QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); - qDebug() << outputDeviceInfo.supportedSampleRates() << "\n"; - qDebug() << "Audio output device is" << outputDeviceInfo.deviceName() << "\n"; if (!outputDeviceInfo.isFormatSupported(audioFormat)) { @@ -350,8 +348,9 @@ void Audio::handleAudioInput() { } } - _outputDevice->write(stereoOutputBuffer); - + if (_outputDevice) { + _outputDevice->write(stereoOutputBuffer); + } // add output (@speakers) data just written to the scope QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, From bcdc066a98b09cf26037c6f3dc97c08b4ac585bd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 6 Dec 2013 13:26:40 -0800 Subject: [PATCH 02/24] don't set _lastInputLoudness if input is muted --- interface/src/Audio.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 993f655a99..51559c1a34 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -185,16 +185,7 @@ void Audio::handleAudioInput() { if (_isBufferSendCallback) { // copy samples from the inputByteArray to the stereoInputBuffer - memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); - - // Measure the loudness of the signal from the microphone and store in audio object - float loudness = 0; - for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; i += 2) { - loudness += abs(stereoInputBuffer[i]); - } - - loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; - _lastInputLoudness = loudness; + memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); } else { // this is the first half of a full buffer of data @@ -264,6 +255,8 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); if (!_muted) { + float loudness = 0; + // we aren't muted, average each set of four samples together to set up the mono input buffers for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO; i += 4) { @@ -275,9 +268,16 @@ void Audio::handleAudioInput() { + (stereoInputBuffer[i + 2] / 4); } + loudness += abs(averagedSample); + // add the averaged sample to our array of audio samples monoAudioSamples[(i - 2) / 4] += averagedSample; } + + loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + _lastInputLoudness = loudness; + } else { + _lastInputLoudness = 0; } nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, From 67d9bc0b14f31e924ffb884f319ff05441c8b46a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 6 Dec 2013 14:29:55 -0800 Subject: [PATCH 03/24] add the resample library to interface --- .../libresample/include/libresample.h | 44 +++ .../external/libresample/src/configtemplate.h | 7 + .../external/libresample/src/filterkit.c | 215 +++++++++++ .../external/libresample/src/filterkit.h | 28 ++ interface/external/libresample/src/resample.c | 347 ++++++++++++++++++ .../external/libresample/src/resample_defs.h | 86 +++++ .../external/libresample/src/resamplesubs.c | 123 +++++++ 7 files changed, 850 insertions(+) create mode 100644 interface/external/libresample/include/libresample.h create mode 100644 interface/external/libresample/src/configtemplate.h create mode 100644 interface/external/libresample/src/filterkit.c create mode 100644 interface/external/libresample/src/filterkit.h create mode 100644 interface/external/libresample/src/resample.c create mode 100644 interface/external/libresample/src/resample_defs.h create mode 100644 interface/external/libresample/src/resamplesubs.c diff --git a/interface/external/libresample/include/libresample.h b/interface/external/libresample/include/libresample.h new file mode 100644 index 0000000000..ca08e044f0 --- /dev/null +++ b/interface/external/libresample/include/libresample.h @@ -0,0 +1,44 @@ +/********************************************************************** + + resample.h + + Real-time library interface by Dominic Mazzoni + + Based on resample-1.7: + http://www-ccrma.stanford.edu/~jos/resample/ + + License: LGPL - see the file LICENSE.txt for more information + +**********************************************************************/ + +#ifndef LIBRESAMPLE_INCLUDED +#define LIBRESAMPLE_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +void *resample_open(int highQuality, + double minFactor, + double maxFactor); + +void *resample_dup(const void *handle); + +int resample_get_filter_width(const void *handle); + +int resample_process(void *handle, + double factor, + float *inBuffer, + int inBufferLen, + int lastFlag, + int *inBufferUsed, + float *outBuffer, + int outBufferLen); + +void resample_close(void *handle); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* LIBRESAMPLE_INCLUDED */ diff --git a/interface/external/libresample/src/configtemplate.h b/interface/external/libresample/src/configtemplate.h new file mode 100644 index 0000000000..94ae1cea90 --- /dev/null +++ b/interface/external/libresample/src/configtemplate.h @@ -0,0 +1,7 @@ +/* Run configure to generate config.h automatically on any + system supported by GNU autoconf. For all other systems, + use this file as a template to create config.h +*/ + +#undef HAVE_INTTYPES_H + diff --git a/interface/external/libresample/src/filterkit.c b/interface/external/libresample/src/filterkit.c new file mode 100644 index 0000000000..bc92285f2c --- /dev/null +++ b/interface/external/libresample/src/filterkit.c @@ -0,0 +1,215 @@ +/********************************************************************** + + resamplesubs.c + + Real-time library interface by Dominic Mazzoni + + Based on resample-1.7: + http://www-ccrma.stanford.edu/~jos/resample/ + + License: LGPL - see the file LICENSE.txt for more information + + This file provides Kaiser-windowed low-pass filter support, + including a function to create the filter coefficients, and + two functions to apply the filter at a particular point. + +**********************************************************************/ + +/* Definitions */ +#include "resample_defs.h" + +#include "filterkit.h" + +#include +#include +#include +#include + +/* LpFilter() + * + * reference: "Digital Filters, 2nd edition" + * R.W. Hamming, pp. 178-179 + * + * Izero() computes the 0th order modified bessel function of the first kind. + * (Needed to compute Kaiser window). + * + * LpFilter() computes the coeffs of a Kaiser-windowed low pass filter with + * the following characteristics: + * + * c[] = array in which to store computed coeffs + * frq = roll-off frequency of filter + * N = Half the window length in number of coeffs + * Beta = parameter of Kaiser window + * Num = number of coeffs before 1/frq + * + * Beta trades the rejection of the lowpass filter against the transition + * width from passband to stopband. Larger Beta means a slower + * transition and greater stopband rejection. See Rabiner and Gold + * (Theory and Application of DSP) under Kaiser windows for more about + * Beta. The following table from Rabiner and Gold gives some feel + * for the effect of Beta: + * + * All ripples in dB, width of transition band = D*N where N = window length + * + * BETA D PB RIP SB RIP + * 2.120 1.50 +-0.27 -30 + * 3.384 2.23 0.0864 -40 + * 4.538 2.93 0.0274 -50 + * 5.658 3.62 0.00868 -60 + * 6.764 4.32 0.00275 -70 + * 7.865 5.0 0.000868 -80 + * 8.960 5.7 0.000275 -90 + * 10.056 6.4 0.000087 -100 + */ + +#define IzeroEPSILON 1E-21 /* Max error acceptable in Izero */ + +static double Izero(double x) +{ + double sum, u, halfx, temp; + int n; + + sum = u = n = 1; + halfx = x/2.0; + do { + temp = halfx/(double)n; + n += 1; + temp *= temp; + u *= temp; + sum += u; + } while (u >= IzeroEPSILON*sum); + return(sum); +} + +void lrsLpFilter(double c[], int N, double frq, double Beta, int Num) +{ + double IBeta, temp, temp1, inm1; + int i; + + /* Calculate ideal lowpass filter impulse response coefficients: */ + c[0] = 2.0*frq; + for (i=1; i +#include +#include +#include + +typedef struct { + float *Imp; + float *ImpD; + float LpScl; + UWORD Nmult; + UWORD Nwing; + double minFactor; + double maxFactor; + UWORD XSize; + float *X; + UWORD Xp; /* Current "now"-sample pointer for input */ + UWORD Xread; /* Position to put new samples */ + UWORD Xoff; + UWORD YSize; + float *Y; + UWORD Yp; + double Time; +} rsdata; + +void *resample_dup(const void * handle) +{ + const rsdata *cpy = (const rsdata *)handle; + rsdata *hp = (rsdata *)malloc(sizeof(rsdata)); + + hp->minFactor = cpy->minFactor; + hp->maxFactor = cpy->maxFactor; + hp->Nmult = cpy->Nmult; + hp->LpScl = cpy->LpScl; + hp->Nwing = cpy->Nwing; + + hp->Imp = (float *)malloc(hp->Nwing * sizeof(float)); + memcpy(hp->Imp, cpy->Imp, hp->Nwing * sizeof(float)); + hp->ImpD = (float *)malloc(hp->Nwing * sizeof(float)); + memcpy(hp->ImpD, cpy->ImpD, hp->Nwing * sizeof(float)); + + hp->Xoff = cpy->Xoff; + hp->XSize = cpy->XSize; + hp->X = (float *)malloc((hp->XSize + hp->Xoff) * sizeof(float)); + memcpy(hp->X, cpy->X, (hp->XSize + hp->Xoff) * sizeof(float)); + hp->Xp = cpy->Xp; + hp->Xread = cpy->Xread; + hp->YSize = cpy->YSize; + hp->Y = (float *)malloc(hp->YSize * sizeof(float)); + memcpy(hp->Y, cpy->Y, hp->YSize * sizeof(float)); + hp->Yp = cpy->Yp; + hp->Time = cpy->Time; + + return (void *)hp; +} + +void *resample_open(int highQuality, double minFactor, double maxFactor) +{ + double *Imp64; + double Rolloff, Beta; + rsdata *hp; + UWORD Xoff_min, Xoff_max; + int i; + + /* Just exit if we get invalid factors */ + if (minFactor <= 0.0 || maxFactor <= 0.0 || maxFactor < minFactor) { + #if DEBUG + fprintf(stderr, + "libresample: " + "minFactor and maxFactor must be positive real numbers,\n" + "and maxFactor should be larger than minFactor.\n"); + #endif + return 0; + } + + hp = (rsdata *)malloc(sizeof(rsdata)); + + hp->minFactor = minFactor; + hp->maxFactor = maxFactor; + + if (highQuality) + hp->Nmult = 35; + else + hp->Nmult = 11; + + hp->LpScl = 1.0; + hp->Nwing = Npc*(hp->Nmult-1)/2; /* # of filter coeffs in right wing */ + + Rolloff = 0.90; + Beta = 6; + + Imp64 = (double *)malloc(hp->Nwing * sizeof(double)); + + lrsLpFilter(Imp64, hp->Nwing, 0.5*Rolloff, Beta, Npc); + + hp->Imp = (float *)malloc(hp->Nwing * sizeof(float)); + hp->ImpD = (float *)malloc(hp->Nwing * sizeof(float)); + for(i=0; iNwing; i++) + hp->Imp[i] = Imp64[i]; + + /* Storing deltas in ImpD makes linear interpolation + of the filter coefficients faster */ + for (i=0; iNwing-1; i++) + hp->ImpD[i] = hp->Imp[i+1] - hp->Imp[i]; + + /* Last coeff. not interpolated */ + hp->ImpD[hp->Nwing-1] = - hp->Imp[hp->Nwing-1]; + + free(Imp64); + + /* Calc reach of LP filter wing (plus some creeping room) */ + Xoff_min = ((hp->Nmult+1)/2.0) * MAX(1.0, 1.0/minFactor) + 10; + Xoff_max = ((hp->Nmult+1)/2.0) * MAX(1.0, 1.0/maxFactor) + 10; + hp->Xoff = MAX(Xoff_min, Xoff_max); + + /* Make the inBuffer size at least 4096, but larger if necessary + in order to store the minimum reach of the LP filter and then some. + Then allocate the buffer an extra Xoff larger so that + we can zero-pad up to Xoff zeros at the end when we reach the + end of the input samples. */ + hp->XSize = MAX(2*hp->Xoff+10, 4096); + hp->X = (float *)malloc((hp->XSize + hp->Xoff) * sizeof(float)); + hp->Xp = hp->Xoff; + hp->Xread = hp->Xoff; + + /* Need Xoff zeros at begining of X buffer */ + for(i=0; iXoff; i++) + hp->X[i]=0; + + /* Make the outBuffer long enough to hold the entire processed + output of one inBuffer */ + hp->YSize = (int)(((double)hp->XSize)*maxFactor+2.0); + hp->Y = (float *)malloc(hp->YSize * sizeof(float)); + hp->Yp = 0; + + hp->Time = (double)hp->Xoff; /* Current-time pointer for converter */ + + return (void *)hp; +} + +int resample_get_filter_width(const void *handle) +{ + const rsdata *hp = (const rsdata *)handle; + return hp->Xoff; +} + +int resample_process(void *handle, + double factor, + float *inBuffer, + int inBufferLen, + int lastFlag, + int *inBufferUsed, /* output param */ + float *outBuffer, + int outBufferLen) +{ + rsdata *hp = (rsdata *)handle; + float *Imp = hp->Imp; + float *ImpD = hp->ImpD; + float LpScl = hp->LpScl; + UWORD Nwing = hp->Nwing; + BOOL interpFilt = FALSE; /* TRUE means interpolate filter coeffs */ + int outSampleCount; + UWORD Nout, Ncreep, Nreuse; + int Nx; + int i, len; + + #if DEBUG + fprintf(stderr, "resample_process: in=%d, out=%d lastFlag=%d\n", + inBufferLen, outBufferLen, lastFlag); + #endif + + /* Initialize inBufferUsed and outSampleCount to 0 */ + *inBufferUsed = 0; + outSampleCount = 0; + + if (factor < hp->minFactor || factor > hp->maxFactor) { + #if DEBUG + fprintf(stderr, + "libresample: factor %f is not between " + "minFactor=%f and maxFactor=%f", + factor, hp->minFactor, hp->maxFactor); + #endif + return -1; + } + + /* Start by copying any samples still in the Y buffer to the output + buffer */ + if (hp->Yp && (outBufferLen-outSampleCount)>0) { + len = MIN(outBufferLen-outSampleCount, hp->Yp); + for(i=0; iY[i]; + outSampleCount += len; + for(i=0; iYp-len; i++) + hp->Y[i] = hp->Y[i+len]; + hp->Yp -= len; + } + + /* If there are still output samples left, return now - we need + the full output buffer available to us... */ + if (hp->Yp) + return outSampleCount; + + /* Account for increased filter gain when using factors less than 1 */ + if (factor < 1) + LpScl = LpScl*factor; + + for(;;) { + + /* This is the maximum number of samples we can process + per loop iteration */ + + #ifdef DEBUG + printf("XSize: %d Xoff: %d Xread: %d Xp: %d lastFlag: %d\n", + hp->XSize, hp->Xoff, hp->Xread, hp->Xp, lastFlag); + #endif + + /* Copy as many samples as we can from the input buffer into X */ + len = hp->XSize - hp->Xread; + + if (len >= (inBufferLen - (*inBufferUsed))) + len = (inBufferLen - (*inBufferUsed)); + + for(i=0; iX[hp->Xread + i] = inBuffer[(*inBufferUsed) + i]; + + *inBufferUsed += len; + hp->Xread += len; + + if (lastFlag && (*inBufferUsed == inBufferLen)) { + /* If these are the last samples, zero-pad the + end of the input buffer and make sure we process + all the way to the end */ + Nx = hp->Xread - hp->Xoff; + for(i=0; iXoff; i++) + hp->X[hp->Xread + i] = 0; + } + else + Nx = hp->Xread - 2 * hp->Xoff; + + #ifdef DEBUG + fprintf(stderr, "new len=%d Nx=%d\n", len, Nx); + #endif + + if (Nx <= 0) + break; + + /* Resample stuff in input buffer */ + if (factor >= 1) { /* SrcUp() is faster if we can use it */ + Nout = lrsSrcUp(hp->X, hp->Y, factor, &hp->Time, Nx, + Nwing, LpScl, Imp, ImpD, interpFilt); + } + else { + Nout = lrsSrcUD(hp->X, hp->Y, factor, &hp->Time, Nx, + Nwing, LpScl, Imp, ImpD, interpFilt); + } + + #ifdef DEBUG + printf("Nout: %d\n", Nout); + #endif + + hp->Time -= Nx; /* Move converter Nx samples back in time */ + hp->Xp += Nx; /* Advance by number of samples processed */ + + /* Calc time accumulation in Time */ + Ncreep = (int)(hp->Time) - hp->Xoff; + if (Ncreep) { + hp->Time -= Ncreep; /* Remove time accumulation */ + hp->Xp += Ncreep; /* and add it to read pointer */ + } + + /* Copy part of input signal that must be re-used */ + Nreuse = hp->Xread - (hp->Xp - hp->Xoff); + + for (i=0; iX[i] = hp->X[i + (hp->Xp - hp->Xoff)]; + + #ifdef DEBUG + printf("New Xread=%d\n", Nreuse); + #endif + + hp->Xread = Nreuse; /* Pos in input buff to read new data into */ + hp->Xp = hp->Xoff; + + /* Check to see if output buff overflowed (shouldn't happen!) */ + if (Nout > hp->YSize) { + #ifdef DEBUG + printf("Nout: %d YSize: %d\n", Nout, hp->YSize); + #endif + fprintf(stderr, "libresample: Output array overflow!\n"); + return -1; + } + + hp->Yp = Nout; + + /* Copy as many samples as possible to the output buffer */ + if (hp->Yp && (outBufferLen-outSampleCount)>0) { + len = MIN(outBufferLen-outSampleCount, hp->Yp); + for(i=0; iY[i]; + outSampleCount += len; + for(i=0; iYp-len; i++) + hp->Y[i] = hp->Y[i+len]; + hp->Yp -= len; + } + + /* If there are still output samples left, return now, + since we need the full output buffer available */ + if (hp->Yp) + break; + } + + return outSampleCount; +} + +void resample_close(void *handle) +{ + rsdata *hp = (rsdata *)handle; + free(hp->X); + free(hp->Y); + free(hp->Imp); + free(hp->ImpD); + free(hp); +} + diff --git a/interface/external/libresample/src/resample_defs.h b/interface/external/libresample/src/resample_defs.h new file mode 100644 index 0000000000..576c1bc2e2 --- /dev/null +++ b/interface/external/libresample/src/resample_defs.h @@ -0,0 +1,86 @@ +/********************************************************************** + + resample_defs.h + + Real-time library interface by Dominic Mazzoni + + Based on resample-1.7: + http://www-ccrma.stanford.edu/~jos/resample/ + + License: LGPL - see the file LICENSE.txt for more information + +**********************************************************************/ + +#ifndef __RESAMPLE_DEFS__ +#define __RESAMPLE_DEFS__ + +#if !defined(WIN32) && !defined(__CYGWIN__) +#include "config.h" +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef PI +#define PI (3.14159265358979232846) +#endif + +#ifndef PI2 +#define PI2 (6.28318530717958465692) +#endif + +#define D2R (0.01745329348) /* (2*pi)/360 */ +#define R2D (57.29577951) /* 360/(2*pi) */ + +#ifndef MAX +#define MAX(x,y) ((x)>(y) ?(x):(y)) +#endif +#ifndef MIN +#define MIN(x,y) ((x)<(y) ?(x):(y)) +#endif + +#ifndef ABS +#define ABS(x) ((x)<0 ?(-(x)):(x)) +#endif + +#ifndef SGN +#define SGN(x) ((x)<0 ?(-1):((x)==0?(0):(1))) +#endif + +#if HAVE_INTTYPES_H + #include + typedef char BOOL; + typedef int32_t WORD; + typedef uint32_t UWORD; +#else + typedef char BOOL; + typedef int WORD; + typedef unsigned int UWORD; +#endif + +#ifdef DEBUG +#define INLINE +#else +#define INLINE inline +#endif + +/* Accuracy */ + +#define Npc 4096 + +/* Function prototypes */ + +int lrsSrcUp(float X[], float Y[], double factor, double *Time, + UWORD Nx, UWORD Nwing, float LpScl, + float Imp[], float ImpD[], BOOL Interp); + +int lrsSrcUD(float X[], float Y[], double factor, double *Time, + UWORD Nx, UWORD Nwing, float LpScl, + float Imp[], float ImpD[], BOOL Interp); + +#endif diff --git a/interface/external/libresample/src/resamplesubs.c b/interface/external/libresample/src/resamplesubs.c new file mode 100644 index 0000000000..c3c095dc04 --- /dev/null +++ b/interface/external/libresample/src/resamplesubs.c @@ -0,0 +1,123 @@ +/********************************************************************** + + resamplesubs.c + + Real-time library interface by Dominic Mazzoni + + Based on resample-1.7: + http://www-ccrma.stanford.edu/~jos/resample/ + + License: LGPL - see the file LICENSE.txt for more information + + This file provides the routines that do sample-rate conversion + on small arrays, calling routines from filterkit. + +**********************************************************************/ + +/* Definitions */ +#include "resample_defs.h" + +#include "filterkit.h" + +#include +#include +#include +#include + +/* Sampling rate up-conversion only subroutine; + * Slightly faster than down-conversion; + */ +int lrsSrcUp(float X[], + float Y[], + double factor, + double *TimePtr, + UWORD Nx, + UWORD Nwing, + float LpScl, + float Imp[], + float ImpD[], + BOOL Interp) +{ + float *Xp, *Ystart; + float v; + + double CurrentTime = *TimePtr; + double dt; /* Step through input signal */ + double endTime; /* When Time reaches EndTime, return to user */ + + dt = 1.0/factor; /* Output sampling period */ + + Ystart = Y; + endTime = CurrentTime + Nx; + while (CurrentTime < endTime) + { + double LeftPhase = CurrentTime-floor(CurrentTime); + double RightPhase = 1.0 - LeftPhase; + + Xp = &X[(int)CurrentTime]; /* Ptr to current input sample */ + /* Perform left-wing inner product */ + v = lrsFilterUp(Imp, ImpD, Nwing, Interp, Xp, + LeftPhase, -1); + /* Perform right-wing inner product */ + v += lrsFilterUp(Imp, ImpD, Nwing, Interp, Xp+1, + RightPhase, 1); + + v *= LpScl; /* Normalize for unity filter gain */ + + *Y++ = v; /* Deposit output */ + CurrentTime += dt; /* Move to next sample by time increment */ + } + + *TimePtr = CurrentTime; + return (Y - Ystart); /* Return the number of output samples */ +} + +/* Sampling rate conversion subroutine */ + +int lrsSrcUD(float X[], + float Y[], + double factor, + double *TimePtr, + UWORD Nx, + UWORD Nwing, + float LpScl, + float Imp[], + float ImpD[], + BOOL Interp) +{ + float *Xp, *Ystart; + float v; + + double CurrentTime = (*TimePtr); + double dh; /* Step through filter impulse response */ + double dt; /* Step through input signal */ + double endTime; /* When Time reaches EndTime, return to user */ + + dt = 1.0/factor; /* Output sampling period */ + + dh = MIN(Npc, factor*Npc); /* Filter sampling period */ + + Ystart = Y; + endTime = CurrentTime + Nx; + while (CurrentTime < endTime) + { + double LeftPhase = CurrentTime-floor(CurrentTime); + double RightPhase = 1.0 - LeftPhase; + + Xp = &X[(int)CurrentTime]; /* Ptr to current input sample */ + /* Perform left-wing inner product */ + v = lrsFilterUD(Imp, ImpD, Nwing, Interp, Xp, + LeftPhase, -1, dh); + /* Perform right-wing inner product */ + v += lrsFilterUD(Imp, ImpD, Nwing, Interp, Xp+1, + RightPhase, 1, dh); + + v *= LpScl; /* Normalize for unity filter gain */ + *Y++ = v; /* Deposit output */ + + CurrentTime += dt; /* Move to next sample by time increment */ + } + + *TimePtr = CurrentTime; + return (Y - Ystart); /* Return the number of output samples */ +} From e0cbb13aa6819b20b84b739a82fd5d9f97897336 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 6 Dec 2013 14:38:03 -0800 Subject: [PATCH 04/24] Revert "add the resample library to interface" This reverts commit 67d9bc0b14f31e924ffb884f319ff05441c8b46a. --- .../libresample/include/libresample.h | 44 --- .../external/libresample/src/configtemplate.h | 7 - .../external/libresample/src/filterkit.c | 215 ----------- .../external/libresample/src/filterkit.h | 28 -- interface/external/libresample/src/resample.c | 347 ------------------ .../external/libresample/src/resample_defs.h | 86 ----- .../external/libresample/src/resamplesubs.c | 123 ------- 7 files changed, 850 deletions(-) delete mode 100644 interface/external/libresample/include/libresample.h delete mode 100644 interface/external/libresample/src/configtemplate.h delete mode 100644 interface/external/libresample/src/filterkit.c delete mode 100644 interface/external/libresample/src/filterkit.h delete mode 100644 interface/external/libresample/src/resample.c delete mode 100644 interface/external/libresample/src/resample_defs.h delete mode 100644 interface/external/libresample/src/resamplesubs.c diff --git a/interface/external/libresample/include/libresample.h b/interface/external/libresample/include/libresample.h deleted file mode 100644 index ca08e044f0..0000000000 --- a/interface/external/libresample/include/libresample.h +++ /dev/null @@ -1,44 +0,0 @@ -/********************************************************************** - - resample.h - - Real-time library interface by Dominic Mazzoni - - Based on resample-1.7: - http://www-ccrma.stanford.edu/~jos/resample/ - - License: LGPL - see the file LICENSE.txt for more information - -**********************************************************************/ - -#ifndef LIBRESAMPLE_INCLUDED -#define LIBRESAMPLE_INCLUDED - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -void *resample_open(int highQuality, - double minFactor, - double maxFactor); - -void *resample_dup(const void *handle); - -int resample_get_filter_width(const void *handle); - -int resample_process(void *handle, - double factor, - float *inBuffer, - int inBufferLen, - int lastFlag, - int *inBufferUsed, - float *outBuffer, - int outBufferLen); - -void resample_close(void *handle); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - -#endif /* LIBRESAMPLE_INCLUDED */ diff --git a/interface/external/libresample/src/configtemplate.h b/interface/external/libresample/src/configtemplate.h deleted file mode 100644 index 94ae1cea90..0000000000 --- a/interface/external/libresample/src/configtemplate.h +++ /dev/null @@ -1,7 +0,0 @@ -/* Run configure to generate config.h automatically on any - system supported by GNU autoconf. For all other systems, - use this file as a template to create config.h -*/ - -#undef HAVE_INTTYPES_H - diff --git a/interface/external/libresample/src/filterkit.c b/interface/external/libresample/src/filterkit.c deleted file mode 100644 index bc92285f2c..0000000000 --- a/interface/external/libresample/src/filterkit.c +++ /dev/null @@ -1,215 +0,0 @@ -/********************************************************************** - - resamplesubs.c - - Real-time library interface by Dominic Mazzoni - - Based on resample-1.7: - http://www-ccrma.stanford.edu/~jos/resample/ - - License: LGPL - see the file LICENSE.txt for more information - - This file provides Kaiser-windowed low-pass filter support, - including a function to create the filter coefficients, and - two functions to apply the filter at a particular point. - -**********************************************************************/ - -/* Definitions */ -#include "resample_defs.h" - -#include "filterkit.h" - -#include -#include -#include -#include - -/* LpFilter() - * - * reference: "Digital Filters, 2nd edition" - * R.W. Hamming, pp. 178-179 - * - * Izero() computes the 0th order modified bessel function of the first kind. - * (Needed to compute Kaiser window). - * - * LpFilter() computes the coeffs of a Kaiser-windowed low pass filter with - * the following characteristics: - * - * c[] = array in which to store computed coeffs - * frq = roll-off frequency of filter - * N = Half the window length in number of coeffs - * Beta = parameter of Kaiser window - * Num = number of coeffs before 1/frq - * - * Beta trades the rejection of the lowpass filter against the transition - * width from passband to stopband. Larger Beta means a slower - * transition and greater stopband rejection. See Rabiner and Gold - * (Theory and Application of DSP) under Kaiser windows for more about - * Beta. The following table from Rabiner and Gold gives some feel - * for the effect of Beta: - * - * All ripples in dB, width of transition band = D*N where N = window length - * - * BETA D PB RIP SB RIP - * 2.120 1.50 +-0.27 -30 - * 3.384 2.23 0.0864 -40 - * 4.538 2.93 0.0274 -50 - * 5.658 3.62 0.00868 -60 - * 6.764 4.32 0.00275 -70 - * 7.865 5.0 0.000868 -80 - * 8.960 5.7 0.000275 -90 - * 10.056 6.4 0.000087 -100 - */ - -#define IzeroEPSILON 1E-21 /* Max error acceptable in Izero */ - -static double Izero(double x) -{ - double sum, u, halfx, temp; - int n; - - sum = u = n = 1; - halfx = x/2.0; - do { - temp = halfx/(double)n; - n += 1; - temp *= temp; - u *= temp; - sum += u; - } while (u >= IzeroEPSILON*sum); - return(sum); -} - -void lrsLpFilter(double c[], int N, double frq, double Beta, int Num) -{ - double IBeta, temp, temp1, inm1; - int i; - - /* Calculate ideal lowpass filter impulse response coefficients: */ - c[0] = 2.0*frq; - for (i=1; i -#include -#include -#include - -typedef struct { - float *Imp; - float *ImpD; - float LpScl; - UWORD Nmult; - UWORD Nwing; - double minFactor; - double maxFactor; - UWORD XSize; - float *X; - UWORD Xp; /* Current "now"-sample pointer for input */ - UWORD Xread; /* Position to put new samples */ - UWORD Xoff; - UWORD YSize; - float *Y; - UWORD Yp; - double Time; -} rsdata; - -void *resample_dup(const void * handle) -{ - const rsdata *cpy = (const rsdata *)handle; - rsdata *hp = (rsdata *)malloc(sizeof(rsdata)); - - hp->minFactor = cpy->minFactor; - hp->maxFactor = cpy->maxFactor; - hp->Nmult = cpy->Nmult; - hp->LpScl = cpy->LpScl; - hp->Nwing = cpy->Nwing; - - hp->Imp = (float *)malloc(hp->Nwing * sizeof(float)); - memcpy(hp->Imp, cpy->Imp, hp->Nwing * sizeof(float)); - hp->ImpD = (float *)malloc(hp->Nwing * sizeof(float)); - memcpy(hp->ImpD, cpy->ImpD, hp->Nwing * sizeof(float)); - - hp->Xoff = cpy->Xoff; - hp->XSize = cpy->XSize; - hp->X = (float *)malloc((hp->XSize + hp->Xoff) * sizeof(float)); - memcpy(hp->X, cpy->X, (hp->XSize + hp->Xoff) * sizeof(float)); - hp->Xp = cpy->Xp; - hp->Xread = cpy->Xread; - hp->YSize = cpy->YSize; - hp->Y = (float *)malloc(hp->YSize * sizeof(float)); - memcpy(hp->Y, cpy->Y, hp->YSize * sizeof(float)); - hp->Yp = cpy->Yp; - hp->Time = cpy->Time; - - return (void *)hp; -} - -void *resample_open(int highQuality, double minFactor, double maxFactor) -{ - double *Imp64; - double Rolloff, Beta; - rsdata *hp; - UWORD Xoff_min, Xoff_max; - int i; - - /* Just exit if we get invalid factors */ - if (minFactor <= 0.0 || maxFactor <= 0.0 || maxFactor < minFactor) { - #if DEBUG - fprintf(stderr, - "libresample: " - "minFactor and maxFactor must be positive real numbers,\n" - "and maxFactor should be larger than minFactor.\n"); - #endif - return 0; - } - - hp = (rsdata *)malloc(sizeof(rsdata)); - - hp->minFactor = minFactor; - hp->maxFactor = maxFactor; - - if (highQuality) - hp->Nmult = 35; - else - hp->Nmult = 11; - - hp->LpScl = 1.0; - hp->Nwing = Npc*(hp->Nmult-1)/2; /* # of filter coeffs in right wing */ - - Rolloff = 0.90; - Beta = 6; - - Imp64 = (double *)malloc(hp->Nwing * sizeof(double)); - - lrsLpFilter(Imp64, hp->Nwing, 0.5*Rolloff, Beta, Npc); - - hp->Imp = (float *)malloc(hp->Nwing * sizeof(float)); - hp->ImpD = (float *)malloc(hp->Nwing * sizeof(float)); - for(i=0; iNwing; i++) - hp->Imp[i] = Imp64[i]; - - /* Storing deltas in ImpD makes linear interpolation - of the filter coefficients faster */ - for (i=0; iNwing-1; i++) - hp->ImpD[i] = hp->Imp[i+1] - hp->Imp[i]; - - /* Last coeff. not interpolated */ - hp->ImpD[hp->Nwing-1] = - hp->Imp[hp->Nwing-1]; - - free(Imp64); - - /* Calc reach of LP filter wing (plus some creeping room) */ - Xoff_min = ((hp->Nmult+1)/2.0) * MAX(1.0, 1.0/minFactor) + 10; - Xoff_max = ((hp->Nmult+1)/2.0) * MAX(1.0, 1.0/maxFactor) + 10; - hp->Xoff = MAX(Xoff_min, Xoff_max); - - /* Make the inBuffer size at least 4096, but larger if necessary - in order to store the minimum reach of the LP filter and then some. - Then allocate the buffer an extra Xoff larger so that - we can zero-pad up to Xoff zeros at the end when we reach the - end of the input samples. */ - hp->XSize = MAX(2*hp->Xoff+10, 4096); - hp->X = (float *)malloc((hp->XSize + hp->Xoff) * sizeof(float)); - hp->Xp = hp->Xoff; - hp->Xread = hp->Xoff; - - /* Need Xoff zeros at begining of X buffer */ - for(i=0; iXoff; i++) - hp->X[i]=0; - - /* Make the outBuffer long enough to hold the entire processed - output of one inBuffer */ - hp->YSize = (int)(((double)hp->XSize)*maxFactor+2.0); - hp->Y = (float *)malloc(hp->YSize * sizeof(float)); - hp->Yp = 0; - - hp->Time = (double)hp->Xoff; /* Current-time pointer for converter */ - - return (void *)hp; -} - -int resample_get_filter_width(const void *handle) -{ - const rsdata *hp = (const rsdata *)handle; - return hp->Xoff; -} - -int resample_process(void *handle, - double factor, - float *inBuffer, - int inBufferLen, - int lastFlag, - int *inBufferUsed, /* output param */ - float *outBuffer, - int outBufferLen) -{ - rsdata *hp = (rsdata *)handle; - float *Imp = hp->Imp; - float *ImpD = hp->ImpD; - float LpScl = hp->LpScl; - UWORD Nwing = hp->Nwing; - BOOL interpFilt = FALSE; /* TRUE means interpolate filter coeffs */ - int outSampleCount; - UWORD Nout, Ncreep, Nreuse; - int Nx; - int i, len; - - #if DEBUG - fprintf(stderr, "resample_process: in=%d, out=%d lastFlag=%d\n", - inBufferLen, outBufferLen, lastFlag); - #endif - - /* Initialize inBufferUsed and outSampleCount to 0 */ - *inBufferUsed = 0; - outSampleCount = 0; - - if (factor < hp->minFactor || factor > hp->maxFactor) { - #if DEBUG - fprintf(stderr, - "libresample: factor %f is not between " - "minFactor=%f and maxFactor=%f", - factor, hp->minFactor, hp->maxFactor); - #endif - return -1; - } - - /* Start by copying any samples still in the Y buffer to the output - buffer */ - if (hp->Yp && (outBufferLen-outSampleCount)>0) { - len = MIN(outBufferLen-outSampleCount, hp->Yp); - for(i=0; iY[i]; - outSampleCount += len; - for(i=0; iYp-len; i++) - hp->Y[i] = hp->Y[i+len]; - hp->Yp -= len; - } - - /* If there are still output samples left, return now - we need - the full output buffer available to us... */ - if (hp->Yp) - return outSampleCount; - - /* Account for increased filter gain when using factors less than 1 */ - if (factor < 1) - LpScl = LpScl*factor; - - for(;;) { - - /* This is the maximum number of samples we can process - per loop iteration */ - - #ifdef DEBUG - printf("XSize: %d Xoff: %d Xread: %d Xp: %d lastFlag: %d\n", - hp->XSize, hp->Xoff, hp->Xread, hp->Xp, lastFlag); - #endif - - /* Copy as many samples as we can from the input buffer into X */ - len = hp->XSize - hp->Xread; - - if (len >= (inBufferLen - (*inBufferUsed))) - len = (inBufferLen - (*inBufferUsed)); - - for(i=0; iX[hp->Xread + i] = inBuffer[(*inBufferUsed) + i]; - - *inBufferUsed += len; - hp->Xread += len; - - if (lastFlag && (*inBufferUsed == inBufferLen)) { - /* If these are the last samples, zero-pad the - end of the input buffer and make sure we process - all the way to the end */ - Nx = hp->Xread - hp->Xoff; - for(i=0; iXoff; i++) - hp->X[hp->Xread + i] = 0; - } - else - Nx = hp->Xread - 2 * hp->Xoff; - - #ifdef DEBUG - fprintf(stderr, "new len=%d Nx=%d\n", len, Nx); - #endif - - if (Nx <= 0) - break; - - /* Resample stuff in input buffer */ - if (factor >= 1) { /* SrcUp() is faster if we can use it */ - Nout = lrsSrcUp(hp->X, hp->Y, factor, &hp->Time, Nx, - Nwing, LpScl, Imp, ImpD, interpFilt); - } - else { - Nout = lrsSrcUD(hp->X, hp->Y, factor, &hp->Time, Nx, - Nwing, LpScl, Imp, ImpD, interpFilt); - } - - #ifdef DEBUG - printf("Nout: %d\n", Nout); - #endif - - hp->Time -= Nx; /* Move converter Nx samples back in time */ - hp->Xp += Nx; /* Advance by number of samples processed */ - - /* Calc time accumulation in Time */ - Ncreep = (int)(hp->Time) - hp->Xoff; - if (Ncreep) { - hp->Time -= Ncreep; /* Remove time accumulation */ - hp->Xp += Ncreep; /* and add it to read pointer */ - } - - /* Copy part of input signal that must be re-used */ - Nreuse = hp->Xread - (hp->Xp - hp->Xoff); - - for (i=0; iX[i] = hp->X[i + (hp->Xp - hp->Xoff)]; - - #ifdef DEBUG - printf("New Xread=%d\n", Nreuse); - #endif - - hp->Xread = Nreuse; /* Pos in input buff to read new data into */ - hp->Xp = hp->Xoff; - - /* Check to see if output buff overflowed (shouldn't happen!) */ - if (Nout > hp->YSize) { - #ifdef DEBUG - printf("Nout: %d YSize: %d\n", Nout, hp->YSize); - #endif - fprintf(stderr, "libresample: Output array overflow!\n"); - return -1; - } - - hp->Yp = Nout; - - /* Copy as many samples as possible to the output buffer */ - if (hp->Yp && (outBufferLen-outSampleCount)>0) { - len = MIN(outBufferLen-outSampleCount, hp->Yp); - for(i=0; iY[i]; - outSampleCount += len; - for(i=0; iYp-len; i++) - hp->Y[i] = hp->Y[i+len]; - hp->Yp -= len; - } - - /* If there are still output samples left, return now, - since we need the full output buffer available */ - if (hp->Yp) - break; - } - - return outSampleCount; -} - -void resample_close(void *handle) -{ - rsdata *hp = (rsdata *)handle; - free(hp->X); - free(hp->Y); - free(hp->Imp); - free(hp->ImpD); - free(hp); -} - diff --git a/interface/external/libresample/src/resample_defs.h b/interface/external/libresample/src/resample_defs.h deleted file mode 100644 index 576c1bc2e2..0000000000 --- a/interface/external/libresample/src/resample_defs.h +++ /dev/null @@ -1,86 +0,0 @@ -/********************************************************************** - - resample_defs.h - - Real-time library interface by Dominic Mazzoni - - Based on resample-1.7: - http://www-ccrma.stanford.edu/~jos/resample/ - - License: LGPL - see the file LICENSE.txt for more information - -**********************************************************************/ - -#ifndef __RESAMPLE_DEFS__ -#define __RESAMPLE_DEFS__ - -#if !defined(WIN32) && !defined(__CYGWIN__) -#include "config.h" -#endif - -#ifndef TRUE -#define TRUE 1 -#endif - -#ifndef FALSE -#define FALSE 0 -#endif - -#ifndef PI -#define PI (3.14159265358979232846) -#endif - -#ifndef PI2 -#define PI2 (6.28318530717958465692) -#endif - -#define D2R (0.01745329348) /* (2*pi)/360 */ -#define R2D (57.29577951) /* 360/(2*pi) */ - -#ifndef MAX -#define MAX(x,y) ((x)>(y) ?(x):(y)) -#endif -#ifndef MIN -#define MIN(x,y) ((x)<(y) ?(x):(y)) -#endif - -#ifndef ABS -#define ABS(x) ((x)<0 ?(-(x)):(x)) -#endif - -#ifndef SGN -#define SGN(x) ((x)<0 ?(-1):((x)==0?(0):(1))) -#endif - -#if HAVE_INTTYPES_H - #include - typedef char BOOL; - typedef int32_t WORD; - typedef uint32_t UWORD; -#else - typedef char BOOL; - typedef int WORD; - typedef unsigned int UWORD; -#endif - -#ifdef DEBUG -#define INLINE -#else -#define INLINE inline -#endif - -/* Accuracy */ - -#define Npc 4096 - -/* Function prototypes */ - -int lrsSrcUp(float X[], float Y[], double factor, double *Time, - UWORD Nx, UWORD Nwing, float LpScl, - float Imp[], float ImpD[], BOOL Interp); - -int lrsSrcUD(float X[], float Y[], double factor, double *Time, - UWORD Nx, UWORD Nwing, float LpScl, - float Imp[], float ImpD[], BOOL Interp); - -#endif diff --git a/interface/external/libresample/src/resamplesubs.c b/interface/external/libresample/src/resamplesubs.c deleted file mode 100644 index c3c095dc04..0000000000 --- a/interface/external/libresample/src/resamplesubs.c +++ /dev/null @@ -1,123 +0,0 @@ -/********************************************************************** - - resamplesubs.c - - Real-time library interface by Dominic Mazzoni - - Based on resample-1.7: - http://www-ccrma.stanford.edu/~jos/resample/ - - License: LGPL - see the file LICENSE.txt for more information - - This file provides the routines that do sample-rate conversion - on small arrays, calling routines from filterkit. - -**********************************************************************/ - -/* Definitions */ -#include "resample_defs.h" - -#include "filterkit.h" - -#include -#include -#include -#include - -/* Sampling rate up-conversion only subroutine; - * Slightly faster than down-conversion; - */ -int lrsSrcUp(float X[], - float Y[], - double factor, - double *TimePtr, - UWORD Nx, - UWORD Nwing, - float LpScl, - float Imp[], - float ImpD[], - BOOL Interp) -{ - float *Xp, *Ystart; - float v; - - double CurrentTime = *TimePtr; - double dt; /* Step through input signal */ - double endTime; /* When Time reaches EndTime, return to user */ - - dt = 1.0/factor; /* Output sampling period */ - - Ystart = Y; - endTime = CurrentTime + Nx; - while (CurrentTime < endTime) - { - double LeftPhase = CurrentTime-floor(CurrentTime); - double RightPhase = 1.0 - LeftPhase; - - Xp = &X[(int)CurrentTime]; /* Ptr to current input sample */ - /* Perform left-wing inner product */ - v = lrsFilterUp(Imp, ImpD, Nwing, Interp, Xp, - LeftPhase, -1); - /* Perform right-wing inner product */ - v += lrsFilterUp(Imp, ImpD, Nwing, Interp, Xp+1, - RightPhase, 1); - - v *= LpScl; /* Normalize for unity filter gain */ - - *Y++ = v; /* Deposit output */ - CurrentTime += dt; /* Move to next sample by time increment */ - } - - *TimePtr = CurrentTime; - return (Y - Ystart); /* Return the number of output samples */ -} - -/* Sampling rate conversion subroutine */ - -int lrsSrcUD(float X[], - float Y[], - double factor, - double *TimePtr, - UWORD Nx, - UWORD Nwing, - float LpScl, - float Imp[], - float ImpD[], - BOOL Interp) -{ - float *Xp, *Ystart; - float v; - - double CurrentTime = (*TimePtr); - double dh; /* Step through filter impulse response */ - double dt; /* Step through input signal */ - double endTime; /* When Time reaches EndTime, return to user */ - - dt = 1.0/factor; /* Output sampling period */ - - dh = MIN(Npc, factor*Npc); /* Filter sampling period */ - - Ystart = Y; - endTime = CurrentTime + Nx; - while (CurrentTime < endTime) - { - double LeftPhase = CurrentTime-floor(CurrentTime); - double RightPhase = 1.0 - LeftPhase; - - Xp = &X[(int)CurrentTime]; /* Ptr to current input sample */ - /* Perform left-wing inner product */ - v = lrsFilterUD(Imp, ImpD, Nwing, Interp, Xp, - LeftPhase, -1, dh); - /* Perform right-wing inner product */ - v += lrsFilterUD(Imp, ImpD, Nwing, Interp, Xp+1, - RightPhase, 1, dh); - - v *= LpScl; /* Normalize for unity filter gain */ - *Y++ = v; /* Deposit output */ - - CurrentTime += dt; /* Move to next sample by time increment */ - } - - *TimePtr = CurrentTime; - return (Y - Ystart); /* Return the number of output samples */ -} From af2fa15f55c3aaf8cd7fa82d596bf3ccb28c1665 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Dec 2013 12:02:08 -0800 Subject: [PATCH 05/24] initial work for change to 48KHz --- interface/src/Audio.cpp | 372 ++++++++++++++------------ interface/src/Audio.h | 10 +- libraries/audio/src/AudioRingBuffer.h | 2 +- 3 files changed, 207 insertions(+), 177 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 51559c1a34..419a4ee21b 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -43,10 +43,17 @@ static const int BOTTOM_PADDING = 110; Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent) : QObject(parent), _audioInput(NULL), + _desiredInputFormat(), + _inputFormat(), _inputDevice(NULL), + _inputBuffer(), + _numInputCallbackBytes(0), _audioOutput(NULL), + _desiredOutputFormat(), + _outputFormat(), _outputDevice(NULL), - _isBufferSendCallback(false), + _outputBuffer(), + _numOutputCallbackBytes(0), _nextOutputSamples(NULL), _ringBuffer(true), _scope(scope), @@ -124,180 +131,216 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { return (mode == QAudio::AudioInput) ? QAudioDeviceInfo::defaultInputDevice() : QAudioDeviceInfo::defaultOutputDevice(); } -const int QT_SAMPLE_RATE = 44100; -const int SAMPLE_RATE_RATIO = QT_SAMPLE_RATE / SAMPLE_RATE; +bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, + const QAudioFormat& desiredAudioFormat, + QAudioFormat& adjustedAudioFormat) { + if (!audioDevice.isFormatSupported(desiredAudioFormat)) { + qDebug() << "The desired format for audio I/O is" << desiredAudioFormat << "\n"; + qDebug() << "The desired audio format is not supported by this device.\n"; + + if (desiredAudioFormat.channelCount() == 1) { + adjustedAudioFormat = desiredAudioFormat; + adjustedAudioFormat.setChannelCount(2); + + if (audioDevice.isFormatSupported(adjustedAudioFormat)) { + return true; + } else { + adjustedAudioFormat.setChannelCount(1); + } + } + + if (audioDevice.supportedSampleRates().contains(SAMPLE_RATE * 2)) { + // use 48, which is a sample downsample, upsample + adjustedAudioFormat = desiredAudioFormat; + adjustedAudioFormat.setSampleRate(SAMPLE_RATE * 2); + + // return the nearest in case it needs 2 channels + adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat); + return true; + } + + return false; + } else { + // set the adjustedAudioFormat to the desiredAudioFormat, since it will work + adjustedAudioFormat = desiredAudioFormat; + return true; + } +} + +void nearestNeighborResampling(int16_t* sourceSamples, int16_t* destinationSamples, + int numDestinationSamples, + const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { + if (sourceAudioFormat == destinationAudioFormat) { + memcpy(destinationSamples, sourceSamples, numDestinationSamples * sizeof(int16_t)); + } else { + float sourceMultiplier = sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate(); + + // take into account the number of channels in source and destination + // accomodate for the case where have an output with > 2 channels + // this is the case with our HDMI capture + sourceMultiplier *= sourceAudioFormat.channelCount() + / (float) (destinationAudioFormat.channelCount() > 2 ? 2 : destinationAudioFormat.channelCount()); + + for (int i = 0; i < numDestinationSamples; i++) { + destinationSamples[i] = sourceSamples[static_cast(i * sourceMultiplier)]; + } + } +} + const int CALLBACK_ACCELERATOR_RATIO = 2; -const int CALLBACK_IO_BUFFER_SIZE = BUFFER_LENGTH_BYTES_STEREO * SAMPLE_RATE_RATIO / CALLBACK_ACCELERATOR_RATIO; void Audio::start() { - QAudioFormat audioFormat; // set up the desired audio format - audioFormat.setSampleRate(QT_SAMPLE_RATE); - audioFormat.setSampleSize(16); - audioFormat.setCodec("audio/pcm"); - audioFormat.setSampleType(QAudioFormat::SignedInt); - audioFormat.setByteOrder(QAudioFormat::LittleEndian); - audioFormat.setChannelCount(2); + _desiredInputFormat.setSampleRate(SAMPLE_RATE); + _desiredInputFormat.setSampleSize(16); + _desiredInputFormat.setCodec("audio/pcm"); + _desiredInputFormat.setSampleType(QAudioFormat::SignedInt); + _desiredInputFormat.setByteOrder(QAudioFormat::LittleEndian); + _desiredInputFormat.setChannelCount(1); - qDebug() << "The format for audio I/O is" << audioFormat << "\n"; + _desiredOutputFormat = _desiredInputFormat; + _desiredOutputFormat.setChannelCount(2); QAudioDeviceInfo inputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioInput); - qDebug() << "Audio input device is" << inputDeviceInfo.deviceName() << "\n"; - if (!inputDeviceInfo.isFormatSupported(audioFormat)) { - qDebug() << "The desired audio input format is not supported by this device. Not starting audio input.\n"; + qDebug() << "The audio input device is" << inputDeviceInfo.deviceName() << "\n"; + + if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) { + qDebug() << "The format to be used for audio input is" << _inputFormat << "\n"; + + _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); + _numInputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() + * (_inputFormat.sampleRate() / SAMPLE_RATE) + / CALLBACK_ACCELERATOR_RATIO; + _audioInput->setBufferSize(_numInputCallbackBytes); + + QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); + + qDebug() << "The audio output device is" << outputDeviceInfo.deviceName() << "\n"; + + if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { + qDebug() << "The format to be used for audio output is" << _outputFormat << "\n"; + + _inputDevice = _audioInput->start(); + connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput())); + + _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); + _numOutputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _outputFormat.channelCount() + * (_outputFormat.sampleRate() / SAMPLE_RATE) + / CALLBACK_ACCELERATOR_RATIO; + _audioOutput->setBufferSize(_numOutputCallbackBytes); + _outputDevice = _audioOutput->start(); + + gettimeofday(&_lastReceiveTime, NULL); + } + return; } - _audioInput = new QAudioInput(inputDeviceInfo, audioFormat, this); - _audioInput->setBufferSize(CALLBACK_IO_BUFFER_SIZE); - _inputDevice = _audioInput->start(); - - connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput())); - - QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput); - - qDebug() << "Audio output device is" << outputDeviceInfo.deviceName() << "\n"; - - if (!outputDeviceInfo.isFormatSupported(audioFormat)) { - qDebug() << "The desired audio output format is not supported by this device.\n"; - return; - } - - _audioOutput = new QAudioOutput(outputDeviceInfo, audioFormat, this); - _audioOutput->setBufferSize(CALLBACK_IO_BUFFER_SIZE); - _outputDevice = _audioOutput->start(); - - gettimeofday(&_lastReceiveTime, NULL); + qDebug() << "Unable to set up audio I/O because of a problem with input or output formats.\n"; } void Audio::handleAudioInput() { - static int16_t stereoInputBuffer[CALLBACK_IO_BUFFER_SIZE * 2]; + static char monoAudioDataPacket[MAX_PACKET_SIZE]; - static int bufferSizeSamples = _audioInput->bufferSize() / sizeof(int16_t); static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - QByteArray inputByteArray = _inputDevice->read(CALLBACK_IO_BUFFER_SIZE); + QByteArray inputByteArray = _inputDevice->readAll(); - if (_isBufferSendCallback) { - // copy samples from the inputByteArray to the stereoInputBuffer - memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); - + int numResampledBytes = inputByteArray.size() / (_numInputCallbackBytes / BUFFER_LENGTH_BYTES_PER_CHANNEL); + qDebug() << "NSB:" << numResampledBytes << "\n"; + + // zero out the monoAudioSamples array + memset(monoAudioSamples, 0, numResampledBytes); + + if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { + _outputBuffer.resize(inputByteArray.size()); + // if local loopback enabled, copy input to output + nearestNeighborResampling((int16_t*) inputByteArray.data(), (int16_t*) _outputBuffer.data(), + inputByteArray.size() / sizeof(int16_t), + _inputFormat, _outputFormat); } else { - // this is the first half of a full buffer of data - // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, BUFFER_LENGTH_BYTES_PER_CHANNEL); - - // take samples we have in this callback and store them in the first half of the static buffer - // to send off in the next callback - memcpy((char*) stereoInputBuffer, inputByteArray.data(), inputByteArray.size()); + _outputBuffer.fill(0, inputByteArray.size()); } // add input data just written to the scope - QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); - - QByteArray stereoOutputBuffer; - - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { - // if local loopback enabled, copy input to output - if (_isBufferSendCallback) { - stereoOutputBuffer.append((char*) (stereoInputBuffer + bufferSizeSamples), CALLBACK_IO_BUFFER_SIZE); - } else { - stereoOutputBuffer.append((char*) stereoInputBuffer, CALLBACK_IO_BUFFER_SIZE); - } - } else { - // zero out the stereoOutputBuffer - stereoOutputBuffer = QByteArray(CALLBACK_IO_BUFFER_SIZE, 0); - } + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); // add procedural effects to the appropriate input samples - addProceduralSounds(monoAudioSamples + (_isBufferSendCallback - ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), - (int16_t*) stereoOutputBuffer.data(), - BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); + // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback + // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), + // (int16_t*) stereoOutputBuffer.data(), + // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); - if (_isBufferSendCallback) { - NodeList* nodeList = NodeList::getInstance(); - Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - - if (audioMixer) { - if (audioMixer->getActiveSocket()) { - MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); + NodeList* nodeList = NodeList::getInstance(); + Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); + + if (audioMixer) { + if (audioMixer->getActiveSocket()) { + MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); + + glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); + glm::quat headOrientation = interfaceAvatar->getHead().getOrientation(); + + // we need the amount of bytes in the buffer + 1 for type + // + 12 for 3 floats for position + float for bearing + 1 attenuation byte + + PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + + char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, + packetType); + + // pack Source Data + QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122(); + memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size()); + currentPacketPtr += rfcUUID.size(); + + // memcpy the three float positions + memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); + currentPacketPtr += (sizeof(headPosition)); + + // memcpy our orientation + memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); + currentPacketPtr += sizeof(headOrientation); + + if (!_muted) { + float loudness = 0; + // loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + _lastInputLoudness = loudness; - glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); - glm::quat headOrientation = interfaceAvatar->getHead().getOrientation(); + // we aren't muted - pull our input audio to send off to the mixer + nearestNeighborResampling((int16_t*) inputByteArray.data(), + monoAudioSamples, numResampledBytes / sizeof(int16_t), + _inputFormat, _desiredInputFormat); - // we need the amount of bytes in the buffer + 1 for type - // + 12 for 3 floats for position + float for bearing + 1 attenuation byte - - PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; - - char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, - packetType); - - // pack Source Data - QByteArray rfcUUID = NodeList::getInstance()->getOwnerUUID().toRfc4122(); - memcpy(currentPacketPtr, rfcUUID.constData(), rfcUUID.size()); - currentPacketPtr += rfcUUID.size(); - - // memcpy the three float positions - memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); - currentPacketPtr += (sizeof(headPosition)); - - // memcpy our orientation - memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); - currentPacketPtr += sizeof(headOrientation); - - if (!_muted) { - float loudness = 0; - - // we aren't muted, average each set of four samples together to set up the mono input buffers - for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO; i += 4) { - - int16_t averagedSample = 0; - if (i + 2 == BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2 * SAMPLE_RATE_RATIO) { - averagedSample = (stereoInputBuffer[i - 2] / 2) + (stereoInputBuffer[i] / 2); - } else { - averagedSample = (stereoInputBuffer[i - 2] / 4) + (stereoInputBuffer[i] / 2) - + (stereoInputBuffer[i + 2] / 4); - } - - loudness += abs(averagedSample); - - // add the averaged sample to our array of audio samples - monoAudioSamples[(i - 2) / 4] += averagedSample; - } - - loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - _lastInputLoudness = loudness; - } else { - _lastInputLoudness = 0; - } - - nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, - audioMixer->getActiveSocket()->getAddress(), - audioMixer->getActiveSocket()->getPort()); - - Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); } else { - nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); + _lastInputLoudness = 0; } + + nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, + numResampledBytes + leadingBytes, + audioMixer->getActiveSocket()->getAddress(), + audioMixer->getActiveSocket()->getPort()); + + Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) + .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); + } else { + nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); } } - // if there is anything in the ring buffer, decide what to do - - if (!_nextOutputSamples) { + if (_outputDevice) { + // if there is anything in the ring buffer, decide what to do + if (_ringBuffer.getEndOfLastWrite()) { if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() < - (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { + ((_outputBuffer.size() / sizeof(int16_t)) + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { // If not enough audio has arrived to start playback, keep waiting } else if (!_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() == 0) { // If we have started and now have run out of audio to send to the audio device, @@ -314,50 +357,29 @@ void Audio::handleAudioInput() { _ringBuffer.setHasStarted(true); } - _nextOutputSamples = _ringBuffer.getNextOutput(); + int numOutputBufferSamples = _outputBuffer.size() / sizeof(int16_t); + + // copy the packet from the RB to the output + nearestNeighborResampling(_ringBuffer.getNextOutput(), + (int16_t*) _outputBuffer.data(), + numOutputBufferSamples, + _desiredOutputFormat, _outputFormat); + + _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + numOutputBufferSamples); + + if (_ringBuffer.getNextOutput() >= _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { + _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); + } } } - } - - if (_nextOutputSamples) { - int16_t* stereoOutputBufferSamples = (int16_t*) stereoOutputBuffer.data(); + // add output (@speakers) data just written to the scope + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - // play whatever we have in the audio buffer - for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO; s++) { - int16_t leftSample = _nextOutputSamples[s]; - int16_t rightSample = _nextOutputSamples[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - - stereoOutputBufferSamples[(s * 4)] += leftSample; - stereoOutputBufferSamples[(s * 4) + 2] += leftSample; - - stereoOutputBufferSamples[(s * 4) + 1] += rightSample; - stereoOutputBufferSamples[(s * 4) + 3] += rightSample; - } - - if (_isBufferSendCallback) { - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); - - if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); - } - - _nextOutputSamples = NULL; - } else { - _nextOutputSamples += PACKET_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO; - } + _outputDevice->write(_outputBuffer); } - if (_outputDevice) { - _outputDevice->write(stereoOutputBuffer); - } - - // add output (@speakers) data just written to the scope - QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - - _isBufferSendCallback = !_isBufferSendCallback; - gettimeofday(&_lastCallbackTime, NULL); } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 60f563b221..844c44c6dc 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -15,6 +15,7 @@ #include "InterfaceConfig.h" #include +#include #include #include @@ -70,10 +71,17 @@ public slots: private: QAudioInput* _audioInput; + QAudioFormat _desiredInputFormat; + QAudioFormat _inputFormat; QIODevice* _inputDevice; + QByteArray _inputBuffer; + int _numInputCallbackBytes; QAudioOutput* _audioOutput; + QAudioFormat _desiredOutputFormat; + QAudioFormat _outputFormat; QIODevice* _outputDevice; - bool _isBufferSendCallback; + QByteArray _outputBuffer; + int _numOutputCallbackBytes; int16_t* _nextOutputSamples; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index f31fa780fb..092ef7102e 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -16,7 +16,7 @@ #include "NodeData.h" -const int SAMPLE_RATE = 22050; +const int SAMPLE_RATE = 24000; const int BUFFER_LENGTH_BYTES_STEREO = 1024; const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; From 6fab49b1b71494f220c7a7b72752c8d9bd80580c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Dec 2013 12:02:22 -0800 Subject: [PATCH 06/24] change ring buffer to interleaved for stereo --- assignment-client/src/audio/AudioMixer.cpp | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 81b7d5e614..cfa5855be4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -162,33 +162,34 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf int16_t* sourceBuffer = bufferToAdd->getNextOutput(); - int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) - ? _clientSamples - : _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) - ? _clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL - : _clientSamples; + // if the bearing relative angle to source is > 0 then the delayed channel is the right one + int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; + int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; int16_t* delaySamplePointer = bufferToAdd->getNextOutput() == bufferToAdd->getBuffer() ? bufferToAdd->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay : bufferToAdd->getNextOutput() - numSamplesDelay; - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { + for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2; s += 2) { if (s < numSamplesDelay) { // pull the earlier sample for the delayed channel - int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; + int earlierSample = delaySamplePointer[s / 2] * attenuationCoefficient * weakChannelAmplitudeRatio; - delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } // pull the current sample for the good channel - int16_t currentSample = sourceBuffer[s] * attenuationCoefficient; - goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + int16_t currentSample = sourceBuffer[s / 2] * attenuationCoefficient; + _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { // place the curernt sample at the right spot in the delayed channel - int sumSample = delayedChannel[s + numSamplesDelay] + (currentSample * weakChannelAmplitudeRatio); - delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] + + (currentSample * weakChannelAmplitudeRatio)), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; } } } From e27aa11216ca562c2bfb4063bc9c2b5aa21e3cb8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 11 Dec 2013 12:03:14 -0800 Subject: [PATCH 07/24] remove a debug line --- interface/src/Audio.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 419a4ee21b..edffa0108a 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -252,7 +252,6 @@ void Audio::handleAudioInput() { QByteArray inputByteArray = _inputDevice->readAll(); int numResampledBytes = inputByteArray.size() / (_numInputCallbackBytes / BUFFER_LENGTH_BYTES_PER_CHANNEL); - qDebug() << "NSB:" << numResampledBytes << "\n"; // zero out the monoAudioSamples array memset(monoAudioSamples, 0, numResampledBytes); From b06af3cd6c5f47296b8120707f8bafbf25ed4d28 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 12 Dec 2013 10:15:43 -0800 Subject: [PATCH 08/24] copy all passed samples to AudioRingBuffer --- interface/src/Audio.cpp | 141 +++++++++++++++++------- libraries/audio/src/AudioRingBuffer.cpp | 44 ++++---- 2 files changed, 125 insertions(+), 60 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index edffa0108a..ec85987fbb 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -167,22 +167,65 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice, } } -void nearestNeighborResampling(int16_t* sourceSamples, int16_t* destinationSamples, - int numDestinationSamples, - const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { +void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, + unsigned int numSourceSamples, unsigned int numDestinationSamples, + const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) { if (sourceAudioFormat == destinationAudioFormat) { - memcpy(destinationSamples, sourceSamples, numDestinationSamples * sizeof(int16_t)); + memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { - float sourceMultiplier = sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate(); + float sourceToDestinationFactor = numSourceSamples / numDestinationSamples; // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels // this is the case with our HDMI capture - sourceMultiplier *= sourceAudioFormat.channelCount() - / (float) (destinationAudioFormat.channelCount() > 2 ? 2 : destinationAudioFormat.channelCount()); - for (int i = 0; i < numDestinationSamples; i++) { - destinationSamples[i] = sourceSamples[static_cast(i * sourceMultiplier)]; + if (sourceToDestinationFactor >= 2) { + // we need to downsample from 48 to 24 + // for now this only supports a mono output - this would be the case for audio input + + for (int i = 2; i < numSourceSamples; i += 4) { + + if (i + 2 >= numSourceSamples) { + destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 2) + + (sourceSamples[i] / 2); + } else { + destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 4) + + (sourceSamples[i] / 2) + + (sourceSamples[i + 2] / 4); + } + } + + } else { + int numResultingDestinationSamples = numSourceSamples + * (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()) + * (destinationAudioFormat.channelCount() / sourceAudioFormat.channelCount()); + + // upsample from 24 to 48 + for (int i = 0; i < numResultingDestinationSamples; i += destinationAudioFormat.channelCount()) { + + destinationSamples[i] = sourceSamples[i / 4]; + + if (sourceAudioFormat.channelCount() == 1) { + destinationSamples[i + 1] = sourceSamples[i / 4]; + } else { + destinationSamples[i + 1] = sourceSamples[(i / 4) + 1]; + + if (destinationAudioFormat.channelCount() > 2) { + // fill the rest of the channels with silence + for (int j = 2; j < destinationAudioFormat.channelCount(); j++) { + destinationSamples[i] = 0; + } + } + } + + if (numResultingDestinationSamples < numDestinationSamples + && i + destinationAudioFormat.channelCount() >= numResultingDestinationSamples) { + // make sure we don't leave a gap on the number of destination samples + for (int k = numResultingDestinationSamples; k < numDestinationSamples; k++) { + destinationSamples[k] = destinationSamples[k - destinationAudioFormat.channelCount()]; + } + } + } } } } @@ -249,19 +292,24 @@ void Audio::handleAudioInput() { static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - QByteArray inputByteArray = _inputDevice->readAll(); + static int inputToOutputRatio = _numOutputCallbackBytes / _numInputCallbackBytes; + static int inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / BUFFER_LENGTH_BYTES_PER_CHANNEL; - int numResampledBytes = inputByteArray.size() / (_numInputCallbackBytes / BUFFER_LENGTH_BYTES_PER_CHANNEL); + QByteArray inputByteArray = _inputDevice->readAll(); + + int numResampledNetworkBytes = inputByteArray.size() / inputToNetworkInputRatio; + int numResampledNetworkSamples = numResampledNetworkBytes / sizeof(int16_t); // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, numResampledBytes); + memset(monoAudioSamples, 0, numResampledNetworkBytes); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { _outputBuffer.resize(inputByteArray.size()); // if local loopback enabled, copy input to output - nearestNeighborResampling((int16_t*) inputByteArray.data(), (int16_t*) _outputBuffer.data(), - inputByteArray.size() / sizeof(int16_t), - _inputFormat, _outputFormat); + linearResampling((int16_t*) inputByteArray.data(), (int16_t*) _outputBuffer.data(), + inputByteArray.size() / sizeof(int16_t), + inputByteArray.size() * inputToOutputRatio / sizeof(int16_t), + _inputFormat, _outputFormat); } else { _outputBuffer.fill(0, inputByteArray.size()); } @@ -279,7 +327,7 @@ void Audio::handleAudioInput() { NodeList* nodeList = NodeList::getInstance(); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - if (audioMixer) { + if (false) { if (audioMixer->getActiveSocket()) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); @@ -314,30 +362,45 @@ void Audio::handleAudioInput() { _lastInputLoudness = loudness; // we aren't muted - pull our input audio to send off to the mixer - nearestNeighborResampling((int16_t*) inputByteArray.data(), - monoAudioSamples, numResampledBytes / sizeof(int16_t), - _inputFormat, _desiredInputFormat); + linearResampling((int16_t*) inputByteArray.data(), + monoAudioSamples, + inputByteArray.size() / sizeof(int16_t), + numResampledNetworkSamples, + _inputFormat, _desiredInputFormat); } else { _lastInputLoudness = 0; } nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, - numResampledBytes + leadingBytes, + numResampledNetworkBytes + leadingBytes, audioMixer->getActiveSocket()->getAddress(), audioMixer->getActiveSocket()->getPort()); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); + .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); } else { nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); } } + // we aren't muted - pull our input audio to send off to the mixer + linearResampling((int16_t*) inputByteArray.data(), + monoAudioSamples, + inputByteArray.size() / sizeof(int16_t), + numResampledNetworkSamples, + _inputFormat, _desiredInputFormat); + + linearResampling(monoAudioSamples, + (int16_t*) _outputBuffer.data(), + numResampledNetworkSamples, + inputByteArray.size() * inputToOutputRatio / sizeof(int16_t), + _desiredInputFormat, _outputFormat); + if (_outputDevice) { // if there is anything in the ring buffer, decide what to do - if (_ringBuffer.getEndOfLastWrite()) { + if (false) { if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() < ((_outputBuffer.size() / sizeof(int16_t)) + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { // If not enough audio has arrived to start playback, keep waiting @@ -357,12 +420,16 @@ void Audio::handleAudioInput() { } int numOutputBufferSamples = _outputBuffer.size() / sizeof(int16_t); + if (_ringBuffer.getNextOutput() + numOutputBufferSamples > _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { + qDebug() << _ringBuffer.getNextOutput() - _ringBuffer.getBuffer() << "\n"; + numOutputBufferSamples = (_ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) - _ringBuffer.getNextOutput(); + } // copy the packet from the RB to the output - nearestNeighborResampling(_ringBuffer.getNextOutput(), - (int16_t*) _outputBuffer.data(), - numOutputBufferSamples, - _desiredOutputFormat, _outputFormat); +// nearestNeighborResampling(_ringBuffer.getNextOutput(), +// (int16_t*) _outputBuffer.data(), +// numOutputBufferSamples, +// _desiredOutputFormat, _outputFormat); _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + numOutputBufferSamples); @@ -409,17 +476,17 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { } } - if (_ringBuffer.diffLastWriteNextOutput() + PACKET_LENGTH_SAMPLES > - PACKET_LENGTH_SAMPLES + (ceilf((float) (_jitterBufferSamples * 2) / PACKET_LENGTH_SAMPLES) * PACKET_LENGTH_SAMPLES)) { - // this packet would give us more than the required amount for play out - // discard the first packet in the buffer - - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); - - if (_ringBuffer.getNextOutput() == _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); - } - } +// if (_ringBuffer.diffLastWriteNextOutput() + PACKET_LENGTH_SAMPLES > +// PACKET_LENGTH_SAMPLES + (ceilf((float) (_jitterBufferSamples * 2) / PACKET_LENGTH_SAMPLES) * PACKET_LENGTH_SAMPLES)) { +// // this packet would give us more than the required amount for play out +// // discard the first packet in the buffer +// +// _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); +// +// if (_ringBuffer.getNextOutput() >= _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { +// _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); +// } +// } _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index b18ff01b95..13145ea23c 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -9,6 +9,8 @@ #include #include +#include + #include "PacketHeaders.h" #include "AudioRingBuffer.h" @@ -43,30 +45,26 @@ int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes) { // make sure we have enough bytes left for this to be the right amount of audio // otherwise we should not copy that data, and leave the buffer pointers where they are - int samplesToCopy = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (_isStereo ? 2 : 1); - if (numBytes == samplesToCopy * sizeof(int16_t)) { - - if (!_endOfLastWrite) { - _endOfLastWrite = _buffer; - } else if (diffLastWriteNextOutput() > RING_BUFFER_LENGTH_SAMPLES - samplesToCopy) { - _endOfLastWrite = _buffer; - _nextOutput = _buffer; - _isStarved = true; - } - - memcpy(_endOfLastWrite, sourceBuffer, numBytes); - - _endOfLastWrite += samplesToCopy; - - if (_endOfLastWrite >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { - _endOfLastWrite = _buffer; - } - - return numBytes; - } else { - return 0; - } + int samplesToCopy = numBytes / sizeof(int16_t); + + if (!_endOfLastWrite) { + _endOfLastWrite = _buffer; + } else if (diffLastWriteNextOutput() > RING_BUFFER_LENGTH_SAMPLES - samplesToCopy) { + _endOfLastWrite = _buffer; + _nextOutput = _buffer; + _isStarved = true; + } + + memcpy(_endOfLastWrite, sourceBuffer, numBytes); + + _endOfLastWrite += samplesToCopy; + + if (_endOfLastWrite >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { + _endOfLastWrite = _buffer; + } + + return numBytes; } int AudioRingBuffer::diffLastWriteNextOutput() const { From d5aadf6598ba83c318bbfcf00e82290952dd4277 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 12 Dec 2013 10:48:42 -0800 Subject: [PATCH 09/24] repair linear resampling for same number of channels --- interface/src/Audio.cpp | 67 +++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ec85987fbb..cb14dde453 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -173,7 +173,7 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { - float sourceToDestinationFactor = numSourceSamples / numDestinationSamples; + float sourceToDestinationFactor = numSourceSamples / (float) numDestinationSamples; // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels @@ -200,15 +200,18 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, * (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()) * (destinationAudioFormat.channelCount() / sourceAudioFormat.channelCount()); + int sourceIndex = 0; + // upsample from 24 to 48 for (int i = 0; i < numResultingDestinationSamples; i += destinationAudioFormat.channelCount()) { - destinationSamples[i] = sourceSamples[i / 4]; + sourceIndex = i * sourceToDestinationFactor; + destinationSamples[i] = sourceSamples[sourceIndex]; if (sourceAudioFormat.channelCount() == 1) { - destinationSamples[i + 1] = sourceSamples[i / 4]; + destinationSamples[i + 1] = sourceSamples[sourceIndex]; } else { - destinationSamples[i + 1] = sourceSamples[(i / 4) + 1]; + destinationSamples[i + 1] = sourceSamples[(sourceIndex) + 1]; if (destinationAudioFormat.channelCount() > 2) { // fill the rest of the channels with silence @@ -292,16 +295,16 @@ void Audio::handleAudioInput() { static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - static int inputToOutputRatio = _numOutputCallbackBytes / _numInputCallbackBytes; - static int inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / BUFFER_LENGTH_BYTES_PER_CHANNEL; + static float inputToOutputRatio = _numOutputCallbackBytes / _numInputCallbackBytes; + static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / BUFFER_LENGTH_BYTES_PER_CHANNEL; QByteArray inputByteArray = _inputDevice->readAll(); - int numResampledNetworkBytes = inputByteArray.size() / inputToNetworkInputRatio; - int numResampledNetworkSamples = numResampledNetworkBytes / sizeof(int16_t); + int numResampledNetworkInputBytes = inputByteArray.size() / inputToNetworkInputRatio; + int numResampledNetworkInputSamples = numResampledNetworkInputBytes / sizeof(int16_t); // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, numResampledNetworkBytes); + memset(monoAudioSamples, 0, numResampledNetworkInputBytes); if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { _outputBuffer.resize(inputByteArray.size()); @@ -327,7 +330,7 @@ void Audio::handleAudioInput() { NodeList* nodeList = NodeList::getInstance(); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - if (false) { + if (audioMixer) { if (audioMixer->getActiveSocket()) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); @@ -338,7 +341,7 @@ void Audio::handleAudioInput() { // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, packetType); @@ -365,7 +368,7 @@ void Audio::handleAudioInput() { linearResampling((int16_t*) inputByteArray.data(), monoAudioSamples, inputByteArray.size() / sizeof(int16_t), - numResampledNetworkSamples, + numResampledNetworkInputSamples, _inputFormat, _desiredInputFormat); } else { @@ -373,7 +376,7 @@ void Audio::handleAudioInput() { } nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, - numResampledNetworkBytes + leadingBytes, + numResampledNetworkInputBytes + leadingBytes, audioMixer->getActiveSocket()->getAddress(), audioMixer->getActiveSocket()->getPort()); @@ -384,23 +387,10 @@ void Audio::handleAudioInput() { } } - // we aren't muted - pull our input audio to send off to the mixer - linearResampling((int16_t*) inputByteArray.data(), - monoAudioSamples, - inputByteArray.size() / sizeof(int16_t), - numResampledNetworkSamples, - _inputFormat, _desiredInputFormat); - - linearResampling(monoAudioSamples, - (int16_t*) _outputBuffer.data(), - numResampledNetworkSamples, - inputByteArray.size() * inputToOutputRatio / sizeof(int16_t), - _desiredInputFormat, _outputFormat); - if (_outputDevice) { // if there is anything in the ring buffer, decide what to do - if (false) { + if (_ringBuffer.getEndOfLastWrite()) { if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() < ((_outputBuffer.size() / sizeof(int16_t)) + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { // If not enough audio has arrived to start playback, keep waiting @@ -419,19 +409,24 @@ void Audio::handleAudioInput() { _ringBuffer.setHasStarted(true); } - int numOutputBufferSamples = _outputBuffer.size() / sizeof(int16_t); - if (_ringBuffer.getNextOutput() + numOutputBufferSamples > _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - qDebug() << _ringBuffer.getNextOutput() - _ringBuffer.getBuffer() << "\n"; - numOutputBufferSamples = (_ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) - _ringBuffer.getNextOutput(); + int numRequiredNetworkOutputBytes = numResampledNetworkInputBytes * 2; + int numRequiredNetworkOutputSamples = numRequiredNetworkOutputBytes / sizeof(int16_t); + + int numResampledOutputBytes = inputByteArray.size() * inputToOutputRatio; + + if (_ringBuffer.getNextOutput() + numRequiredNetworkOutputSamples + > _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { + numRequiredNetworkOutputSamples = (_ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) - _ringBuffer.getNextOutput(); } // copy the packet from the RB to the output -// nearestNeighborResampling(_ringBuffer.getNextOutput(), -// (int16_t*) _outputBuffer.data(), -// numOutputBufferSamples, -// _desiredOutputFormat, _outputFormat); + linearResampling(_ringBuffer.getNextOutput(), + (int16_t*) _outputBuffer.data(), + numRequiredNetworkOutputSamples, + numResampledOutputBytes / sizeof(int16_t), + _desiredOutputFormat, _outputFormat); - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + numOutputBufferSamples); + _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + numRequiredNetworkOutputSamples); if (_ringBuffer.getNextOutput() >= _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); From f17ee1af7abccab59ff460a37248e67289ac5532 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 12 Dec 2013 13:37:18 -0800 Subject: [PATCH 10/24] cleanup AudioRingBuffer API --- assignment-client/src/audio/AudioMixer.cpp | 10 +-- .../src/audio/AudioMixerClientData.cpp | 16 ++-- interface/src/Audio.cpp | 62 +++++--------- libraries/audio/src/AudioRingBuffer.cpp | 80 ++++++++++++++++--- libraries/audio/src/AudioRingBuffer.h | 26 +++--- .../audio/src/PositionalAudioRingBuffer.cpp | 9 +-- 6 files changed, 112 insertions(+), 91 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index cfa5855be4..0b89f5a83c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -160,27 +160,21 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf } } - int16_t* sourceBuffer = bufferToAdd->getNextOutput(); - // if the bearing relative angle to source is > 0 then the delayed channel is the right one int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; - int16_t* delaySamplePointer = bufferToAdd->getNextOutput() == bufferToAdd->getBuffer() - ? bufferToAdd->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay - : bufferToAdd->getNextOutput() - numSamplesDelay; - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2; s += 2) { if (s < numSamplesDelay) { // pull the earlier sample for the delayed channel - int earlierSample = delaySamplePointer[s / 2] * attenuationCoefficient * weakChannelAmplitudeRatio; + int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } // pull the current sample for the good channel - int16_t currentSample = sourceBuffer[s / 2] * attenuationCoefficient; + int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 627bf4ec11..b7f0aeac5f 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,17 +90,15 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { // this was a used buffer, push the output pointer forwards PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; - if (audioBuffer->willBeAddedToMix()) { - audioBuffer->setNextOutput(audioBuffer->getNextOutput() + BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - - if (audioBuffer->getNextOutput() >= audioBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - audioBuffer->setNextOutput(audioBuffer->getBuffer()); - } + if (audioBuffer->willBeAddedToMix()) { + audioBuffer->shiftReadPosition(BUFFER_LENGTH_SAMPLES_PER_CHANNEL); audioBuffer->setWillBeAddedToMix(false); - } else if (audioBuffer->hasStarted() && audioBuffer->isStarved()) { - delete audioBuffer; - _ringBuffers.erase(_ringBuffers.begin() + i); + } else if (audioBuffer->isStarved()) { + // this was previously the kill for injected audio from a client + // fix when that is added back + // delete audioBuffer; + // _ringBuffers.erase(_ringBuffers.begin() + i); } } } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index cb14dde453..11f1c92cec 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -388,50 +388,43 @@ void Audio::handleAudioInput() { } if (_outputDevice) { - // if there is anything in the ring buffer, decide what to do - if (_ringBuffer.getEndOfLastWrite()) { - if (_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() < - ((_outputBuffer.size() / sizeof(int16_t)) + _jitterBufferSamples * (_ringBuffer.isStereo() ? 2 : 1))) { - // If not enough audio has arrived to start playback, keep waiting - } else if (!_ringBuffer.isStarved() && _ringBuffer.diffLastWriteNextOutput() == 0) { - // If we have started and now have run out of audio to send to the audio device, - // this means we've starved and should restart. - _ringBuffer.setIsStarved(true); - - // show a starve in the GUI for 10 frames - _numFramesDisplayStarve = 10; - + // if there is anything in the ring buffer, decide what to do + if (_ringBuffer.samplesAvailable() > 0) { + + int numRequiredNetworkOutputBytes = numResampledNetworkInputBytes + * (_desiredOutputFormat.channelCount() / _desiredInputFormat.channelCount()); + int numRequiredNetworkOutputSamples = numRequiredNetworkOutputBytes / sizeof(int16_t); + + if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numRequiredNetworkOutputSamples)) { + // starved and we don't have enough to start, keep waiting + qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; } else { // We are either already playing back, or we have enough audio to start playing back. if (_ringBuffer.isStarved()) { _ringBuffer.setIsStarved(false); - _ringBuffer.setHasStarted(true); } - int numRequiredNetworkOutputBytes = numResampledNetworkInputBytes * 2; - int numRequiredNetworkOutputSamples = numRequiredNetworkOutputBytes / sizeof(int16_t); - int numResampledOutputBytes = inputByteArray.size() * inputToOutputRatio; - if (_ringBuffer.getNextOutput() + numRequiredNetworkOutputSamples - > _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - numRequiredNetworkOutputSamples = (_ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) - _ringBuffer.getNextOutput(); - } + // copy the samples we'll resample from the ring buffer - this also + // pushes the read pointer of the ring buffer forwards + int16_t ringBufferSamples[numRequiredNetworkOutputSamples]; + _ringBuffer.read(ringBufferSamples, numRequiredNetworkOutputSamples); // copy the packet from the RB to the output - linearResampling(_ringBuffer.getNextOutput(), + linearResampling(ringBufferSamples, (int16_t*) _outputBuffer.data(), numRequiredNetworkOutputSamples, numResampledOutputBytes / sizeof(int16_t), _desiredOutputFormat, _outputFormat); - _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + numRequiredNetworkOutputSamples); - - if (_ringBuffer.getNextOutput() >= _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); - } } + } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { + // we don't have any audio data left in the output buffer, and the ring buffer from + // the network has nothing in it either - we just starved + _ringBuffer.setIsStarved(true); + _numFramesDisplayStarve = 10; } // add output (@speakers) data just written to the scope @@ -471,18 +464,6 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { } } -// if (_ringBuffer.diffLastWriteNextOutput() + PACKET_LENGTH_SAMPLES > -// PACKET_LENGTH_SAMPLES + (ceilf((float) (_jitterBufferSamples * 2) / PACKET_LENGTH_SAMPLES) * PACKET_LENGTH_SAMPLES)) { -// // this packet would give us more than the required amount for play out -// // discard the first packet in the buffer -// -// _ringBuffer.setNextOutput(_ringBuffer.getNextOutput() + PACKET_LENGTH_SAMPLES); -// -// if (_ringBuffer.getNextOutput() >= _ringBuffer.getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { -// _ringBuffer.setNextOutput(_ringBuffer.getBuffer()); -// } -// } - _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(PACKET_LENGTH_BYTES @@ -536,8 +517,7 @@ void Audio::render(int screenWidth, int screenHeight) { timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, ¤tTime); } - if (_ringBuffer.getEndOfLastWrite() != NULL) - remainingBuffer = _ringBuffer.diffLastWriteNextOutput() / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; + remainingBuffer = PACKET_LENGTH_SAMPLES / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; if (_numFramesDisplayStarve == 0) { glColor3f(0, 1, 0); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 13145ea23c..e0c4ebb32a 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -18,9 +18,7 @@ AudioRingBuffer::AudioRingBuffer(bool isStereo) : NodeData(NULL), _endOfLastWrite(NULL), - _isStarved(true), - _hasStarted(false), - _isStereo(isStereo) + _isStarved(true) { _buffer = new int16_t[RING_BUFFER_LENGTH_SAMPLES]; _nextOutput = _buffer; @@ -34,7 +32,6 @@ void AudioRingBuffer::reset() { _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; - _hasStarted = false; } int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { @@ -50,24 +47,62 @@ int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes if (!_endOfLastWrite) { _endOfLastWrite = _buffer; - } else if (diffLastWriteNextOutput() > RING_BUFFER_LENGTH_SAMPLES - samplesToCopy) { + } else if (samplesToCopy > RING_BUFFER_LENGTH_SAMPLES - samplesAvailable()) { + // this read will cross the next output, so call us starved and reset the buffer + qDebug() << "Filled the ring buffer. Resetting.\n"; _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; } - memcpy(_endOfLastWrite, sourceBuffer, numBytes); - - _endOfLastWrite += samplesToCopy; - - if (_endOfLastWrite >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { - _endOfLastWrite = _buffer; + if (_endOfLastWrite + samplesToCopy <= _buffer + RING_BUFFER_LENGTH_SAMPLES) { + memcpy(_endOfLastWrite, sourceBuffer, numBytes); + } else { + int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _endOfLastWrite; + memcpy(_endOfLastWrite, sourceBuffer, numSamplesToEnd * sizeof(int16_t)); + memcpy(_buffer, sourceBuffer + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); + return numBytes; } -int AudioRingBuffer::diffLastWriteNextOutput() const { +int16_t& AudioRingBuffer::operator[](const int index) { + // make sure this is a valid index + assert(index > -RING_BUFFER_LENGTH_SAMPLES && index < RING_BUFFER_LENGTH_SAMPLES); + + return *shiftedPositionAccomodatingWrap(_nextOutput, index); +} + +void AudioRingBuffer::read(int16_t* destination, unsigned int maxSamples) { + + // only copy up to the number of samples we have available + int numReadSamples = std::min(maxSamples, samplesAvailable()); + + if (_nextOutput + numReadSamples > _buffer + RING_BUFFER_LENGTH_SAMPLES) { + // we're going to need to do two reads to get this data, it wraps around the edge + + // read to the end of the buffer + int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _nextOutput; + memcpy(destination, _nextOutput, numSamplesToEnd * sizeof(int16_t)); + + // read the rest from the beginning of the buffer + memcpy(destination + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); + } else { + // read the data + memcpy(destination, _nextOutput, numReadSamples * sizeof(int16_t)); + } + + // push the position of _nextOutput by the number of samples read + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); +} + +void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); +} + +unsigned int AudioRingBuffer::samplesAvailable() const { if (!_endOfLastWrite) { return 0; } else { @@ -80,3 +115,24 @@ int AudioRingBuffer::diffLastWriteNextOutput() const { return sampleDifference; } } + +bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const { + if (!_isStarved) { + return true; + } else { + return samplesAvailable() >= numRequiredSamples; + } +} + +int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { + + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { + // this shift will wrap the position around to the beginning of the ring + return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { + // this shift will go around to the end of the ring + return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + } else { + return position + numSamplesShift; + } +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 092ef7102e..aafc3f20f6 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -34,36 +34,30 @@ public: int parseData(unsigned char* sourceBuffer, int numBytes); int parseAudioSamples(unsigned char* sourceBuffer, int numBytes); - - int16_t* getNextOutput() const { return _nextOutput; } - void setNextOutput(int16_t* nextOutput) { _nextOutput = nextOutput; } - int16_t* getEndOfLastWrite() const { return _endOfLastWrite; } - void setEndOfLastWrite(int16_t* endOfLastWrite) { _endOfLastWrite = endOfLastWrite; } + int16_t& operator[](const int index); - int16_t* getBuffer() const { return _buffer; } + void read(int16_t* destination, unsigned int numSamples); + + void shiftReadPosition(unsigned int numSamples); + + unsigned int samplesAvailable() const; + + bool isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const; bool isStarved() const { return _isStarved; } void setIsStarved(bool isStarved) { _isStarved = isStarved; } - - bool hasStarted() const { return _hasStarted; } - void setHasStarted(bool hasStarted) { _hasStarted = hasStarted; } - - int diffLastWriteNextOutput() const; - - bool isStereo() const { return _isStereo; } - protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); + int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; + int16_t* _nextOutput; int16_t* _endOfLastWrite; int16_t* _buffer; bool _isStarved; - bool _hasStarted; - bool _isStereo; }; #endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index ea8d2aca4a..c84444516e 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -57,17 +57,16 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { if (_endOfLastWrite) { - if (_isStarved && diffLastWriteNextOutput() <= BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples) { - printf("Buffer held back\n"); + if (!isNotStarvedOrHasMinimumSamples(BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { + qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; return false; - } else if (diffLastWriteNextOutput() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - printf("Buffer starved.\n"); + } else if (samplesAvailable() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; _isStarved = true; return false; } else { // good buffer, add this to the mix _isStarved = false; - _hasStarted = true; return true; } } From 25b706529856aa62cd13c684575b80180f45e1b5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 12 Dec 2013 17:19:54 -0800 Subject: [PATCH 11/24] changes for broken test --- interface/src/Audio.cpp | 56 ++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 11f1c92cec..5c359ce898 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -173,7 +173,8 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { - float sourceToDestinationFactor = numSourceSamples / (float) numDestinationSamples; + float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) + * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()) ; // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels @@ -187,25 +188,25 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, if (i + 2 >= numSourceSamples) { destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 2) - + (sourceSamples[i] / 2); + + (sourceSamples[i] / 2); } else { destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 4) - + (sourceSamples[i] / 2) - + (sourceSamples[i + 2] / 4); + + (sourceSamples[i] / 2) + + (sourceSamples[i + 2] / 4); } } } else { - int numResultingDestinationSamples = numSourceSamples - * (destinationAudioFormat.sampleRate() / sourceAudioFormat.sampleRate()) - * (destinationAudioFormat.channelCount() / sourceAudioFormat.channelCount()); - int sourceIndex = 0; // upsample from 24 to 48 - for (int i = 0; i < numResultingDestinationSamples; i += destinationAudioFormat.channelCount()) { - + for (int i = 0; i < numDestinationSamples; i += destinationAudioFormat.channelCount()) { sourceIndex = i * sourceToDestinationFactor; + + if (sourceIndex >= numSourceSamples) { + sourceIndex -= destinationAudioFormat.channelCount(); + } + destinationSamples[i] = sourceSamples[sourceIndex]; if (sourceAudioFormat.channelCount() == 1) { @@ -220,14 +221,6 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, } } } - - if (numResultingDestinationSamples < numDestinationSamples - && i + destinationAudioFormat.channelCount() >= numResultingDestinationSamples) { - // make sure we don't leave a gap on the number of destination samples - for (int k = numResultingDestinationSamples; k < numDestinationSamples; k++) { - destinationSamples[k] = destinationSamples[k - destinationAudioFormat.channelCount()]; - } - } } } } @@ -330,7 +323,7 @@ void Audio::handleAudioInput() { NodeList* nodeList = NodeList::getInstance(); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - if (audioMixer) { + if (false) { if (audioMixer->getActiveSocket()) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); @@ -389,12 +382,29 @@ void Audio::handleAudioInput() { if (_outputDevice) { + int numRequiredNetworkOutputSamples = numResampledNetworkInputSamples + * (_desiredOutputFormat.channelCount() / _desiredInputFormat.channelCount()); + + int numResampledOutputBytes = _inputBuffer.size() * inputToOutputRatio; + + // linearResampling((int16_t*) inputByteArray.data(), + // monoAudioSamples, + // inputByteArray.size() / sizeof(int16_t), + // numResampledNetworkInputSamples, + // _inputFormat, _desiredInputFormat); + + // copy the packet from the RB to the output + // linearResampling(monoAudioSamples, + // (int16_t*) _outputBuffer.data(), + // numResampledNetworkInputSamples, + // numResampledOutputBytes / sizeof(int16_t), + // _desiredInputFormat, _outputFormat); + + // if there is anything in the ring buffer, decide what to do - if (_ringBuffer.samplesAvailable() > 0) { + if (false) { - int numRequiredNetworkOutputBytes = numResampledNetworkInputBytes - * (_desiredOutputFormat.channelCount() / _desiredInputFormat.channelCount()); - int numRequiredNetworkOutputSamples = numRequiredNetworkOutputBytes / sizeof(int16_t); + if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numRequiredNetworkOutputSamples)) { // starved and we don't have enough to start, keep waiting From 1f9ca00317019017bc97880838e8da77b23835f3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 11:58:23 -0800 Subject: [PATCH 12/24] drive input from buffer callback and output from network --- assignment-client/src/audio/AudioMixer.cpp | 46 ++-- assignment-client/src/audio/AudioMixer.h | 2 +- .../src/audio/AudioMixerClientData.cpp | 2 +- interface/src/Audio.cpp | 255 +++++++++--------- interface/src/Audio.h | 4 +- libraries/audio/src/AudioRingBuffer.cpp | 127 +++++---- libraries/audio/src/AudioRingBuffer.h | 32 ++- .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 31 +-- 9 files changed, 270 insertions(+), 231 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0b89f5a83c..463746c8f2 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -54,7 +54,7 @@ const short JITTER_BUFFER_MSECS = 12; const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); -const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); +const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); @@ -164,27 +164,29 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2; s += 2) { - if (s < numSamplesDelay) { - // pull the earlier sample for the delayed channel - int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; - - _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - } + for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 2) { +// if (s < numSamplesDelay) { +// // pull the earlier sample for the delayed channel +// int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; +// +// _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// } +// +// // pull the current sample for the good channel +// int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; +// _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// +// if (s + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO) { +// // place the curernt sample at the right spot in the delayed channel +// int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] +// + (currentSample * weakChannelAmplitudeRatio)), +// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); +// _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; +// } - // pull the current sample for the good channel - int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; - _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - - if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - // place the curernt sample at the right spot in the delayed channel - int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] - + (currentSample * weakChannelAmplitudeRatio)), - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; - } + _clientSamples[s] = _clientSamples[s + 1] = (*bufferToAdd)[s / 2] * attenuationCoefficient; } } @@ -277,7 +279,7 @@ void AudioMixer::run() { gettimeofday(&startTime, NULL); int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO); - unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; + unsigned char clientPacket[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO); while (!_isFinished) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 2c07e8747a..7326e1a161 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -35,7 +35,7 @@ private: void prepareMixForListeningNode(Node* node); - int16_t _clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2]; + int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b7f0aeac5f..4827fbc918 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -91,7 +91,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; if (audioBuffer->willBeAddedToMix()) { - audioBuffer->shiftReadPosition(BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + audioBuffer->shiftReadPosition(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); audioBuffer->setWillBeAddedToMix(false); } else if (audioBuffer->isStarved()) { diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 5c359ce898..9c6b66edc5 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -13,6 +13,7 @@ #include #endif +#include #include #include #include @@ -33,7 +34,7 @@ static const float JITTER_BUFFER_LENGTH_MSECS = 12; static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); -static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; +static const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; // Mute icon configration static const int ICON_SIZE = 24; @@ -45,17 +46,15 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _audioInput(NULL), _desiredInputFormat(), _inputFormat(), - _inputDevice(NULL), _inputBuffer(), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), _outputFormat(), _outputDevice(NULL), - _outputBuffer(), _numOutputCallbackBytes(0), - _nextOutputSamples(NULL), - _ringBuffer(true), + _inputRingBuffer(0), + _ringBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), @@ -249,7 +248,7 @@ void Audio::start() { qDebug() << "The format to be used for audio input is" << _inputFormat << "\n"; _audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this); - _numInputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() + _numInputCallbackBytes = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL * _inputFormat.channelCount() * (_inputFormat.sampleRate() / SAMPLE_RATE) / CALLBACK_ACCELERATOR_RATIO; _audioInput->setBufferSize(_numInputCallbackBytes); @@ -261,14 +260,11 @@ void Audio::start() { if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) { qDebug() << "The format to be used for audio output is" << _outputFormat << "\n"; + _inputRingBuffer.resizeForFrameSize(_numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / sizeof(int16_t)); _inputDevice = _audioInput->start(); - connect(_inputDevice, SIGNAL(readyRead()), SLOT(handleAudioInput())); + connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput())); _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); - _numOutputCallbackBytes = BUFFER_LENGTH_BYTES_PER_CHANNEL * _outputFormat.channelCount() - * (_outputFormat.sampleRate() / SAMPLE_RATE) - / CALLBACK_ACCELERATOR_RATIO; - _audioOutput->setBufferSize(_numOutputCallbackBytes); _outputDevice = _audioOutput->start(); gettimeofday(&_lastReceiveTime, NULL); @@ -281,50 +277,66 @@ void Audio::start() { } void Audio::handleAudioInput() { - static char monoAudioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); - static float inputToOutputRatio = _numOutputCallbackBytes / _numInputCallbackBytes; - static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / BUFFER_LENGTH_BYTES_PER_CHANNEL; + static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO + / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + + static int inputSamplesRequired = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio; QByteArray inputByteArray = _inputDevice->readAll(); + + _inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size()); + + while (_inputRingBuffer.samplesAvailable() > inputSamplesRequired) { + + int16_t inputAudioSamples[inputSamplesRequired]; + _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); + + // zero out the monoAudioSamples array + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + + if (!_muted) { + // we aren't muted, downsample the input audio + linearResampling((int16_t*) inputAudioSamples, + monoAudioSamples, + inputSamplesRequired, + NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + _inputFormat, _desiredInputFormat); + + // add input data just written to the scope + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); + } + +// if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { +// // if local loopback enabled, copy input to output +// QByteArray samplesForOutput; +// samplesForOutput.resize(inputSamplesRequired * outputToInputRatio * sizeof(int16_t)); +// +// linearResampling(monoAudioSamples, (int16_t*) samplesForOutput.data(), +// NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, +// inputSamplesRequired, +// _desiredInputFormat, _outputFormat); +// +// _outputDevice->write(samplesForOutput); +// } - int numResampledNetworkInputBytes = inputByteArray.size() / inputToNetworkInputRatio; - int numResampledNetworkInputSamples = numResampledNetworkInputBytes / sizeof(int16_t); - - // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, numResampledNetworkInputBytes); - - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio) && !_muted) { - _outputBuffer.resize(inputByteArray.size()); - // if local loopback enabled, copy input to output - linearResampling((int16_t*) inputByteArray.data(), (int16_t*) _outputBuffer.data(), - inputByteArray.size() / sizeof(int16_t), - inputByteArray.size() * inputToOutputRatio / sizeof(int16_t), - _inputFormat, _outputFormat); - } else { - _outputBuffer.fill(0, inputByteArray.size()); - } - - // add input data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); - - // add procedural effects to the appropriate input samples - // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback - // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), - // (int16_t*) stereoOutputBuffer.data(), - // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); - - NodeList* nodeList = NodeList::getInstance(); - Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); - - if (false) { - if (audioMixer->getActiveSocket()) { + + // add procedural effects to the appropriate input samples + // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback + // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), + // (int16_t*) stereoOutputBuffer.data(), + // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); + + NodeList* nodeList = NodeList::getInstance(); + Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); + + if (audioMixer && nodeList->getNodeActiveSocketOrPing(audioMixer)) { MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition(); @@ -334,7 +346,7 @@ void Audio::handleAudioInput() { // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, packetType); @@ -357,94 +369,19 @@ void Audio::handleAudioInput() { // loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; _lastInputLoudness = loudness; - // we aren't muted - pull our input audio to send off to the mixer - linearResampling((int16_t*) inputByteArray.data(), - monoAudioSamples, - inputByteArray.size() / sizeof(int16_t), - numResampledNetworkInputSamples, - _inputFormat, _desiredInputFormat); - } else { _lastInputLoudness = 0; } nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, - numResampledNetworkInputBytes + leadingBytes, + NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, audioMixer->getActiveSocket()->getAddress(), audioMixer->getActiveSocket()->getPort()); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); - } else { - nodeList->pingPublicAndLocalSocketsForInactiveNode(audioMixer); + .updateValue(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes); } } - - if (_outputDevice) { - - int numRequiredNetworkOutputSamples = numResampledNetworkInputSamples - * (_desiredOutputFormat.channelCount() / _desiredInputFormat.channelCount()); - - int numResampledOutputBytes = _inputBuffer.size() * inputToOutputRatio; - - // linearResampling((int16_t*) inputByteArray.data(), - // monoAudioSamples, - // inputByteArray.size() / sizeof(int16_t), - // numResampledNetworkInputSamples, - // _inputFormat, _desiredInputFormat); - - // copy the packet from the RB to the output - // linearResampling(monoAudioSamples, - // (int16_t*) _outputBuffer.data(), - // numResampledNetworkInputSamples, - // numResampledOutputBytes / sizeof(int16_t), - // _desiredInputFormat, _outputFormat); - - - // if there is anything in the ring buffer, decide what to do - if (false) { - - - - if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numRequiredNetworkOutputSamples)) { - // starved and we don't have enough to start, keep waiting - qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; - } else { - // We are either already playing back, or we have enough audio to start playing back. - if (_ringBuffer.isStarved()) { - _ringBuffer.setIsStarved(false); - } - - int numResampledOutputBytes = inputByteArray.size() * inputToOutputRatio; - - // copy the samples we'll resample from the ring buffer - this also - // pushes the read pointer of the ring buffer forwards - int16_t ringBufferSamples[numRequiredNetworkOutputSamples]; - _ringBuffer.read(ringBufferSamples, numRequiredNetworkOutputSamples); - - // copy the packet from the RB to the output - linearResampling(ringBufferSamples, - (int16_t*) _outputBuffer.data(), - numRequiredNetworkOutputSamples, - numResampledOutputBytes / sizeof(int16_t), - _desiredOutputFormat, _outputFormat); - - } - } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { - // we don't have any audio data left in the output buffer, and the ring buffer from - // the network has nothing in it either - we just starved - _ringBuffer.setIsStarved(true); - _numFramesDisplayStarve = 10; - } - - // add output (@speakers) data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - - _outputDevice->write(_outputBuffer); - } - - gettimeofday(&_lastCallbackTime, NULL); } void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { @@ -466,7 +403,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _measuredJitter = _stdev.getStDev(); _stdev.reset(); // Set jitter buffer to be a multiple of the measured standard deviation - const int MAX_JITTER_BUFFER_SAMPLES = RING_BUFFER_LENGTH_SAMPLES / 2; + const int MAX_JITTER_BUFFER_SAMPLES = _ringBuffer.getSampleCapacity() / 2; const float NUM_STANDARD_DEVIATIONS = 3.f; if (Menu::getInstance()->getAudioJitterBufferSamples() == 0) { float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter) / 1000.f * SAMPLE_RATE; @@ -476,8 +413,70 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); - Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(PACKET_LENGTH_BYTES - + sizeof(PACKET_TYPE)); + static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) + * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); + + static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio; + + int16_t outputBuffer[numRequiredOutputSamples]; + + // linearResampling((int16_t*) inputByteArray.data(), + // monoAudioSamples, + // inputByteArray.size() / sizeof(int16_t), + // numResampledNetworkInputSamples, + // _inputFormat, _desiredInputFormat); + + // copy the packet from the RB to the output + // linearResampling(monoAudioSamples, + // (int16_t*) _outputBuffer.data(), + // numResampledNetworkInputSamples, + // numResampledOutputBytes / sizeof(int16_t), + // _desiredInputFormat, _outputFormat); + + + // if there is anything in the ring buffer, decide what to do + if (_ringBuffer.samplesAvailable() > 0) { + if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + + (_jitterBufferSamples * 2))) { + // starved and we don't have enough to start, keep waiting + qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; + } else { + // We are either already playing back, or we have enough audio to start playing back. + if (_ringBuffer.isStarved()) { + _ringBuffer.setIsStarved(false); + } + + // copy the samples we'll resample from the ring buffer - this also + // pushes the read pointer of the ring buffer forwards + int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; + _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + + // copy the packet from the RB to the output + linearResampling(ringBufferSamples, + outputBuffer, + NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, + numRequiredOutputSamples, + _desiredOutputFormat, _outputFormat); + + if (_outputDevice) { + _outputDevice->write((char*) outputBuffer, numRequiredOutputSamples * sizeof(int16_t)); + } + } + + } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { + // we don't have any audio data left in the output buffer, and the ring buffer from + // the network has nothing in it either - we just starved + _ringBuffer.setIsStarved(true); + _numFramesDisplayStarve = 10; + } + + // add output (@speakers) data just written to the scope + // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, + // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); + + + + Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); _lastReceiveTime = currentReceiveTime; } @@ -508,7 +507,7 @@ void Audio::render(int screenWidth, int screenHeight) { glVertex2f(currentX, topY); glVertex2f(currentX, bottomY); - for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) { + for (int i = 0; i < _ringBuffer.getSampleCapacity() / 2; i++) { glVertex2f(currentX, halfY); glVertex2f(currentX + frameWidth, halfY); currentX += frameWidth; diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 844c44c6dc..8557120ab7 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -70,6 +70,7 @@ public slots: void reset(); private: + QByteArray firstInputFrame; QAudioInput* _audioInput; QAudioFormat _desiredInputFormat; QAudioFormat _inputFormat; @@ -80,9 +81,8 @@ private: QAudioFormat _desiredOutputFormat; QAudioFormat _outputFormat; QIODevice* _outputDevice; - QByteArray _outputBuffer; int _numOutputCallbackBytes; - int16_t* _nextOutputSamples; + AudioRingBuffer _inputRingBuffer; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; StDev _stdev; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index e0c4ebb32a..83c2fe59a0 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -15,13 +15,23 @@ #include "AudioRingBuffer.h" -AudioRingBuffer::AudioRingBuffer(bool isStereo) : +const short RING_BUFFER_LENGTH_FRAMES = 10; + +AudioRingBuffer::AudioRingBuffer(int numFrameSamples) : NodeData(NULL), - _endOfLastWrite(NULL), - _isStarved(true) + _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), + _isStarved(true), + _hasStarted(false) { - _buffer = new int16_t[RING_BUFFER_LENGTH_SAMPLES]; - _nextOutput = _buffer; + if (numFrameSamples) { + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; + } else { + _buffer = NULL; + _nextOutput = NULL; + _endOfLastWrite = NULL; + } }; AudioRingBuffer::~AudioRingBuffer() { @@ -34,20 +44,64 @@ void AudioRingBuffer::reset() { _isStarved = true; } -int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { - int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); - return parseAudioSamples(sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); +void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { + delete[] _buffer; + _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; + _buffer = new int16_t[_sampleCapacity]; + _nextOutput = _buffer; + _endOfLastWrite = _buffer; } -int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes) { +int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { + int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); + return writeData((char*) sourceBuffer + numBytesPacketHeader, numBytes - numBytesPacketHeader); +} + +qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { + return readData((char*) destination, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { + + // only copy up to the number of samples we have available + int numReadSamples = std::min((unsigned) (maxSize / sizeof(int16_t)), samplesAvailable()); + + if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { + // we're going to need to do two reads to get this data, it wraps around the edge + + // read to the end of the buffer + int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; + memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); + + // read the rest from the beginning of the buffer + memcpy(data + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); + } else { + // read the data + memcpy(data, _nextOutput, numReadSamples * sizeof(int16_t)); + } + + // push the position of _nextOutput by the number of samples read + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); + + return numReadSamples * sizeof(int16_t); +} + +qint64 AudioRingBuffer::writeSamples(const int16_t* source, qint64 maxSamples) { + return writeData((const char*) source, maxSamples * sizeof(int16_t)); +} + +qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // make sure we have enough bytes left for this to be the right amount of audio // otherwise we should not copy that data, and leave the buffer pointers where they are - int samplesToCopy = numBytes / sizeof(int16_t); + int samplesToCopy = std::min(maxSize / sizeof(int16_t), (quint64) _sampleCapacity); - if (!_endOfLastWrite) { - _endOfLastWrite = _buffer; - } else if (samplesToCopy > RING_BUFFER_LENGTH_SAMPLES - samplesAvailable()) { + std::less less; + std::less_equal lessEqual; + + if (_hasStarted + && (less(_endOfLastWrite, _nextOutput) + && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting.\n"; _endOfLastWrite = _buffer; @@ -55,49 +109,28 @@ int AudioRingBuffer::parseAudioSamples(unsigned char* sourceBuffer, int numBytes _isStarved = true; } - if (_endOfLastWrite + samplesToCopy <= _buffer + RING_BUFFER_LENGTH_SAMPLES) { - memcpy(_endOfLastWrite, sourceBuffer, numBytes); + _hasStarted = true; + + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { + memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _endOfLastWrite; - memcpy(_endOfLastWrite, sourceBuffer, numSamplesToEnd * sizeof(int16_t)); - memcpy(_buffer, sourceBuffer + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); + memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - return numBytes; + return samplesToCopy * sizeof(int16_t); } int16_t& AudioRingBuffer::operator[](const int index) { // make sure this is a valid index - assert(index > -RING_BUFFER_LENGTH_SAMPLES && index < RING_BUFFER_LENGTH_SAMPLES); + assert(index > -_sampleCapacity && index < _sampleCapacity); return *shiftedPositionAccomodatingWrap(_nextOutput, index); } -void AudioRingBuffer::read(int16_t* destination, unsigned int maxSamples) { - - // only copy up to the number of samples we have available - int numReadSamples = std::min(maxSamples, samplesAvailable()); - - if (_nextOutput + numReadSamples > _buffer + RING_BUFFER_LENGTH_SAMPLES) { - // we're going to need to do two reads to get this data, it wraps around the edge - - // read to the end of the buffer - int numSamplesToEnd = (_buffer + RING_BUFFER_LENGTH_SAMPLES) - _nextOutput; - memcpy(destination, _nextOutput, numSamplesToEnd * sizeof(int16_t)); - - // read the rest from the beginning of the buffer - memcpy(destination + numSamplesToEnd, _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); - } else { - // read the data - memcpy(destination, _nextOutput, numReadSamples * sizeof(int16_t)); - } - - // push the position of _nextOutput by the number of samples read - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); -} - void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); } @@ -109,7 +142,7 @@ unsigned int AudioRingBuffer::samplesAvailable() const { int sampleDifference = _endOfLastWrite - _nextOutput; if (sampleDifference < 0) { - sampleDifference += RING_BUFFER_LENGTH_SAMPLES; + sampleDifference += _sampleCapacity; } return sampleDifference; @@ -126,12 +159,12 @@ bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSa int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + RING_BUFFER_LENGTH_SAMPLES) { + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + return position + numSamplesShift - _sampleCapacity; } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { // this shift will go around to the end of the ring - return position + numSamplesShift - RING_BUFFER_LENGTH_SAMPLES; + return position + numSamplesShift - _sampleCapacity; } else { return position + numSamplesShift; } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index aafc3f20f6..d715e63a97 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -10,35 +10,41 @@ #define __interface__AudioRingBuffer__ #include -#include #include +#include + #include "NodeData.h" const int SAMPLE_RATE = 24000; -const int BUFFER_LENGTH_BYTES_STEREO = 1024; -const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; -const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); - -const short RING_BUFFER_LENGTH_FRAMES = 20; -const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL; +const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; +const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STEREO / sizeof(int16_t); +const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; +const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); class AudioRingBuffer : public NodeData { + Q_OBJECT public: - AudioRingBuffer(bool isStereo); + AudioRingBuffer(int numFrameSamples); ~AudioRingBuffer(); void reset(); - + void resizeForFrameSize(qint64 numFrameSamples); + + int getSampleCapacity() const { return _sampleCapacity; } + int parseData(unsigned char* sourceBuffer, int numBytes); - int parseAudioSamples(unsigned char* sourceBuffer, int numBytes); + + qint64 readSamples(int16_t* destination, qint64 maxSamples); + qint64 writeSamples(const int16_t* source, qint64 maxSamples); + + qint64 readData(char* data, qint64 maxSize); + qint64 writeData(const char* data, qint64 maxSize); int16_t& operator[](const int index); - void read(int16_t* destination, unsigned int numSamples); - void shiftReadPosition(unsigned int numSamples); unsigned int samplesAvailable() const; @@ -54,10 +60,12 @@ protected: int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; + int _sampleCapacity; int16_t* _nextOutput; int16_t* _endOfLastWrite; int16_t* _buffer; bool _isStarved; + bool _hasStarted; }; #endif /* defined(__interface__AudioRingBuffer__) */ diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 9adb525b93..d66a24672a 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -42,7 +42,7 @@ int InjectedAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes unsigned int attenuationByte = *(currentBuffer++); _attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index c84444516e..0b7c26dc7d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -15,7 +15,7 @@ #include "PositionalAudioRingBuffer.h" PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type) : - AudioRingBuffer(false), + AudioRingBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), _type(type), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), @@ -31,7 +31,7 @@ int PositionalAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numByt unsigned char* currentBuffer = sourceBuffer + numBytesForPacketHeader(sourceBuffer); currentBuffer += NUM_BYTES_RFC4122_UUID; // the source UUID currentBuffer += parsePositionalData(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); - currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); + currentBuffer += writeData((char*) currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } @@ -47,8 +47,7 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail if (std::isnan(_orientation.x)) { - _endOfLastWrite = _nextOutput = _buffer; - _isStarved = true; + reset(); return 0; } @@ -56,19 +55,17 @@ int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, } bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { - if (_endOfLastWrite) { - if (!isNotStarvedOrHasMinimumSamples(BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { - qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; - return false; - } else if (samplesAvailable() < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; - _isStarved = true; - return false; - } else { - // good buffer, add this to the mix - _isStarved = false; - return true; - } + if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { + qDebug() << "Starved and do not have minimum samples to start. Buffer held back.\n"; + return false; + } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + qDebug() << "Do not have number of samples needed for interval. Buffer starved.\n"; + _isStarved = true; + return false; + } else { + // good buffer, add this to the mix + _isStarved = false; + return true; } return false; From cdc6f33128dc3825ba79e7ba525243892040beaa Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 12:16:08 -0800 Subject: [PATCH 13/24] fix scope for input --- interface/src/Audio.cpp | 20 ++++--------------- interface/src/Oscilloscope.cpp | 35 +++++++++++++++++++--------------- interface/src/Oscilloscope.h | 2 +- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9c6b66edc5..066dd9c40b 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -309,8 +309,10 @@ void Audio::handleAudioInput() { _inputFormat, _desiredInputFormat); // add input data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, inputByteArray), Q_ARG(bool, true)); + QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, + Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, + NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), + Q_ARG(bool, false), Q_ARG(bool, true)); } // if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { @@ -420,20 +422,6 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { int16_t outputBuffer[numRequiredOutputSamples]; - // linearResampling((int16_t*) inputByteArray.data(), - // monoAudioSamples, - // inputByteArray.size() / sizeof(int16_t), - // numResampledNetworkInputSamples, - // _inputFormat, _desiredInputFormat); - - // copy the packet from the RB to the output - // linearResampling(monoAudioSamples, - // (int16_t*) _outputBuffer.data(), - // numResampledNetworkInputSamples, - // numResampledOutputBytes / sizeof(int16_t), - // _desiredInputFormat, _outputFormat); - - // if there is anything in the ring buffer, decide what to do if (_ringBuffer.samplesAvailable() > 0) { if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO diff --git a/interface/src/Oscilloscope.cpp b/interface/src/Oscilloscope.cpp index 2c65f2a07e..ff8da61130 100644 --- a/interface/src/Oscilloscope.cpp +++ b/interface/src/Oscilloscope.cpp @@ -68,24 +68,18 @@ Oscilloscope::~Oscilloscope() { delete[] _samples; } -void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInput) { +void Oscilloscope::addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput) { if (! enabled || inputPaused) { return; } - unsigned int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * 2); - int16_t samples[numSamplesPerChannel]; - const int16_t* stereoSamples = (int16_t*) audioByteArray.constData(); + int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * (isStereo ? 2 : 1)); + int16_t* samples = (int16_t*) audioByteArray.data(); - for (int channel = 0; channel < (isInput ? 1 : 2); channel++) { + for (int channel = 0; channel < (isStereo ? 1 : 2); channel++) { // add samples for each of the channels - - // enumerate the interleaved stereoSamples array and pull out the samples for this channel - for (int i = 0; i < audioByteArray.size() / sizeof(int16_t); i += 2) { - samples[i / 2] = stereoSamples[i + channel]; - } - + // determine start/end offset of this channel's region unsigned baseOffs = MAX_SAMPLES_PER_CHANNEL * (channel + !isInput); unsigned endOffs = baseOffs + MAX_SAMPLES_PER_CHANNEL; @@ -103,10 +97,21 @@ void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInp numSamplesPerChannel -= n2; } - // copy data - memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t)); - if (n2 > 0) { - memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t)); + if (!isStereo) { + // copy data + memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t)); + if (n2 > 0) { + memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t)); + } + } else { + // we have interleaved samples we need to separate into two channels + for (int i = 0; i < numSamplesPerChannel + n2; i++) { + if (i < numSamplesPerChannel - n2) { + _samples[writePos] = samples[(i * 2) + channel]; + } else { + _samples[baseOffs] = samples[(i * 2) + channel]; + } + } } // set new write position for this channel diff --git a/interface/src/Oscilloscope.h b/interface/src/Oscilloscope.h index f17976d4e4..a245f79f05 100644 --- a/interface/src/Oscilloscope.h +++ b/interface/src/Oscilloscope.h @@ -59,7 +59,7 @@ public: // just uses every nTh sample. void setDownsampleRatio(unsigned n) { assert(n > 0); _downsampleRatio = n; } public slots: - void addStereoSamples(const QByteArray& audioByteArray, bool isInput); + void addSamples(const QByteArray& audioByteArray, bool isStereo, bool isInput); private: // don't copy/assign Oscilloscope(Oscilloscope const&); // = delete; From 2d4a8ca997b75dc3f14dc311bc615ac1761f5fe7 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 12:19:19 -0800 Subject: [PATCH 14/24] fix scope for audio output --- interface/src/Audio.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 066dd9c40b..e5f5df2311 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -416,11 +416,12 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _ringBuffer.parseData((unsigned char*) audioByteArray.data(), audioByteArray.size()); static float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) - * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); + * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); static int numRequiredOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / networkOutputToOutputRatio; - int16_t outputBuffer[numRequiredOutputSamples]; + QByteArray outputBuffer; + outputBuffer.resize(numRequiredOutputSamples * sizeof(int16_t)); // if there is anything in the ring buffer, decide what to do if (_ringBuffer.samplesAvailable() > 0) { @@ -441,13 +442,18 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { // copy the packet from the RB to the output linearResampling(ringBufferSamples, - outputBuffer, + (int16_t*) outputBuffer.data(), NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, numRequiredOutputSamples, _desiredOutputFormat, _outputFormat); if (_outputDevice) { - _outputDevice->write((char*) outputBuffer, numRequiredOutputSamples * sizeof(int16_t)); + _outputDevice->write(outputBuffer); + + // add output (@speakers) data just written to the scope + QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, + Q_ARG(QByteArray, outputBuffer), + Q_ARG(bool, true), Q_ARG(bool, false)); } } @@ -458,12 +464,6 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _numFramesDisplayStarve = 10; } - // add output (@speakers) data just written to the scope - // QMetaObject::invokeMethod(_scope, "addStereoSamples", Qt::QueuedConnection, - // Q_ARG(QByteArray, stereoOutputBuffer), Q_ARG(bool, false)); - - - Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); _lastReceiveTime = currentReceiveTime; From 0970ed55a8b2d7a840f387a587caae0e50aba15c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 12:36:55 -0800 Subject: [PATCH 15/24] re-instate positional audio in the audio-mixer --- assignment-client/src/audio/AudioMixer.cpp | 40 ++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 463746c8f2..b511c255b9 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -165,28 +165,26 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 2) { -// if (s < numSamplesDelay) { -// // pull the earlier sample for the delayed channel -// int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; -// -// _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, -// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); -// } -// -// // pull the current sample for the good channel -// int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; -// _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, -// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); -// -// if (s + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO) { -// // place the curernt sample at the right spot in the delayed channel -// int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] -// + (currentSample * weakChannelAmplitudeRatio)), -// MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); -// _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; -// } + if ((s / 2) < numSamplesDelay) { + // pull the earlier sample for the delayed channel + int earlierSample = (*bufferToAdd)[(s / 2) - numSamplesDelay] * attenuationCoefficient * weakChannelAmplitudeRatio; + + _clientSamples[s + delayedChannelOffset] = glm::clamp(_clientSamples[s + delayedChannelOffset] + earlierSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + } - _clientSamples[s] = _clientSamples[s + 1] = (*bufferToAdd)[s / 2] * attenuationCoefficient; + // pull the current sample for the good channel + int16_t currentSample = (*bufferToAdd)[s / 2] * attenuationCoefficient; + _clientSamples[s + goodChannelOffset] = glm::clamp(_clientSamples[s + goodChannelOffset] + currentSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + + if ((s / 2) + numSamplesDelay < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + // place the curernt sample at the right spot in the delayed channel + int16_t clampedSample = glm::clamp((int) (_clientSamples[s + numSamplesDelay + delayedChannelOffset] + + (currentSample * weakChannelAmplitudeRatio)), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + numSamplesDelay + delayedChannelOffset] = clampedSample; + } } } From 6b644eb1301408b63e84334bfc7827bcf1d9a91d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 15:15:25 -0800 Subject: [PATCH 16/24] repair upsampling and local loopback by correcting for limits --- assignment-client/src/audio/AudioMixer.cpp | 4 -- interface/src/Audio.cpp | 83 +++++++++++----------- interface/src/Audio.h | 1 + libraries/audio/src/AudioRingBuffer.h | 4 ++ 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b511c255b9..700642bf59 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -56,9 +55,6 @@ const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0) const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); -const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); -const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); - const char AUDIO_MIXER_LOGGING_TARGET_NAME[] = "audio-mixer"; void attachNewBufferToNode(Node *newNode) { diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e5f5df2311..bd7da88aef 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -47,6 +47,7 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _desiredInputFormat(), _inputFormat(), _inputBuffer(), + _monoAudioSamples(NULL), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), @@ -172,8 +173,9 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, if (sourceAudioFormat == destinationAudioFormat) { memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t)); } else { + int destinationChannels = (destinationAudioFormat.channelCount() >= 2) ? 2 : destinationAudioFormat.channelCount(); float sourceToDestinationFactor = (sourceAudioFormat.sampleRate() / (float) destinationAudioFormat.sampleRate()) - * (sourceAudioFormat.channelCount() / (float) destinationAudioFormat.channelCount()) ; + * (sourceAudioFormat.channelCount() / (float) destinationChannels); // take into account the number of channels in source and destination // accomodate for the case where have an output with > 2 channels @@ -184,40 +186,36 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, // for now this only supports a mono output - this would be the case for audio input for (int i = 2; i < numSourceSamples; i += 4) { - if (i + 2 >= numSourceSamples) { destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 2) - + (sourceSamples[i] / 2); + + (sourceSamples[i] / 2); } else { destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 4) - + (sourceSamples[i] / 2) - + (sourceSamples[i + 2] / 4); + + (sourceSamples[i] / 2) + + (sourceSamples[i + 2] / 4); } } } else { - int sourceIndex = 0; - // upsample from 24 to 48 - for (int i = 0; i < numDestinationSamples; i += destinationAudioFormat.channelCount()) { - sourceIndex = i * sourceToDestinationFactor; + // for now this only supports a stereo to stereo conversion - this is our case for network audio to output + int sourceIndex = 0; + int destinationToSourceFactor = (1 / sourceToDestinationFactor); + + for (int i = 0; i < numDestinationSamples; i += destinationAudioFormat.channelCount() * destinationToSourceFactor) { + sourceIndex = (i / destinationToSourceFactor); - if (sourceIndex >= numSourceSamples) { - sourceIndex -= destinationAudioFormat.channelCount(); - } - - destinationSamples[i] = sourceSamples[sourceIndex]; - - if (sourceAudioFormat.channelCount() == 1) { - destinationSamples[i + 1] = sourceSamples[sourceIndex]; - } else { - destinationSamples[i + 1] = sourceSamples[(sourceIndex) + 1]; - - if (destinationAudioFormat.channelCount() > 2) { - // fill the rest of the channels with silence - for (int j = 2; j < destinationAudioFormat.channelCount(); j++) { - destinationSamples[i] = 0; - } + // fill the L/R channels and make the rest silent + for (int j = i; j < i + (destinationToSourceFactor * destinationAudioFormat.channelCount()); j++) { + if (j % destinationAudioFormat.channelCount() == 0) { + // left channel + destinationSamples[j] = sourceSamples[sourceIndex]; + } else if (j % destinationAudioFormat.channelCount() == 1) { + // right channel + destinationSamples[j] = sourceSamples[sourceIndex + 1]; + } else { + // channels above 2, fill with silence + destinationSamples[j] = 0; } } } @@ -281,7 +279,8 @@ void Audio::handleAudioInput() { static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; - static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); + + _monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; @@ -298,37 +297,23 @@ void Audio::handleAudioInput() { _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + memset(_monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, - monoAudioSamples, + _monoAudioSamples, inputSamplesRequired, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, _inputFormat, _desiredInputFormat); // add input data just written to the scope QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, + Q_ARG(QByteArray, QByteArray((char*) _monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), Q_ARG(bool, false), Q_ARG(bool, true)); } -// if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { -// // if local loopback enabled, copy input to output -// QByteArray samplesForOutput; -// samplesForOutput.resize(inputSamplesRequired * outputToInputRatio * sizeof(int16_t)); -// -// linearResampling(monoAudioSamples, (int16_t*) samplesForOutput.data(), -// NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, -// inputSamplesRequired, -// _desiredInputFormat, _outputFormat); -// -// _outputDevice->write(samplesForOutput); -// } - - // add procedural effects to the appropriate input samples // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), @@ -440,6 +425,17 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + if (!_muted && Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { + // copy whatever is pointed to at _monoAudioSamples into our ringBufferSamples + // so that local audio is echoed back + + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _monoAudioSamples[i], + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2]; + } + } + // copy the packet from the RB to the output linearResampling(ringBufferSamples, (int16_t*) outputBuffer.data(), @@ -460,6 +456,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { } else if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) { // we don't have any audio data left in the output buffer, and the ring buffer from // the network has nothing in it either - we just starved + qDebug() << "Audio output just starved.\n"; _ringBuffer.setIsStarved(true); _numFramesDisplayStarve = 10; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 8557120ab7..9a051b246b 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -76,6 +76,7 @@ private: QAudioFormat _inputFormat; QIODevice* _inputDevice; QByteArray _inputBuffer; + int16_t* _monoAudioSamples; int _numInputCallbackBytes; QAudioOutput* _audioOutput; QAudioFormat _desiredOutputFormat; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index d715e63a97..3e6917456f 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -9,6 +9,7 @@ #ifndef __interface__AudioRingBuffer__ #define __interface__AudioRingBuffer__ +#include #include #include @@ -24,6 +25,9 @@ const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STE const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); +const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); +const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); + class AudioRingBuffer : public NodeData { Q_OBJECT public: From f7ed8cef8fd9a6ad68b813e9149620d9c30fda41 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 15:31:05 -0800 Subject: [PATCH 17/24] fixes for all current headsets work --- interface/src/Audio.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index bd7da88aef..c1794fc4dd 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -185,14 +185,16 @@ void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples, // we need to downsample from 48 to 24 // for now this only supports a mono output - this would be the case for audio input - for (int i = 2; i < numSourceSamples; i += 4) { - if (i + 2 >= numSourceSamples) { - destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 2) + for (int i = sourceAudioFormat.channelCount(); i < numSourceSamples; i += 2 * sourceAudioFormat.channelCount()) { + if (i + (sourceAudioFormat.channelCount()) >= numSourceSamples) { + destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = + (sourceSamples[i - sourceAudioFormat.channelCount()] / 2) + (sourceSamples[i] / 2); } else { - destinationSamples[(i - 2) / 4] = (sourceSamples[i - 2] / 4) + destinationSamples[(i - sourceAudioFormat.channelCount()) / (int) sourceToDestinationFactor] = + (sourceSamples[i - sourceAudioFormat.channelCount()] / 4) + (sourceSamples[i] / 2) - + (sourceSamples[i + 2] / 4); + + (sourceSamples[i + sourceAudioFormat.channelCount()] / 4); } } @@ -448,7 +450,8 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { // add output (@speakers) data just written to the scope QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, outputBuffer), + Q_ARG(QByteArray, QByteArray((char*) ringBufferSamples, + NETWORK_BUFFER_LENGTH_BYTES_STEREO)), Q_ARG(bool, true), Q_ARG(bool, false)); } } From 89beff42532a19fa0d011c173cde13615766afe5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 15:41:58 -0800 Subject: [PATCH 18/24] fix the scope and last input loudness --- interface/src/Audio.cpp | 13 ++++++++----- interface/src/Oscilloscope.cpp | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c1794fc4dd..c17f5ff1dd 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -353,15 +353,18 @@ void Audio::handleAudioInput() { memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); + float loudness = 0; + if (!_muted) { - float loudness = 0; - // loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - _lastInputLoudness = loudness; + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + loudness += fabsf(_monoAudioSamples[i]); + } - } else { - _lastInputLoudness = 0; + loudness /= NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + _lastInputLoudness = loudness; + nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, audioMixer->getActiveSocket()->getAddress(), diff --git a/interface/src/Oscilloscope.cpp b/interface/src/Oscilloscope.cpp index ff8da61130..5b3cfb9f09 100644 --- a/interface/src/Oscilloscope.cpp +++ b/interface/src/Oscilloscope.cpp @@ -77,7 +77,7 @@ void Oscilloscope::addSamples(const QByteArray& audioByteArray, bool isStereo, b int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * (isStereo ? 2 : 1)); int16_t* samples = (int16_t*) audioByteArray.data(); - for (int channel = 0; channel < (isStereo ? 1 : 2); channel++) { + for (int channel = 0; channel < (isStereo ? 2 : 1); channel++) { // add samples for each of the channels // determine start/end offset of this channel's region From e5d2647e70f40abf6328be7f0bfd8205d0a22161 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:01:03 -0800 Subject: [PATCH 19/24] fix last input loudness for mouth movement with no depth camera --- interface/src/Audio.cpp | 82 ++++++++++++++++++++--------------------- interface/src/Audio.h | 5 +-- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c17f5ff1dd..e741443217 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -46,8 +46,6 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _audioInput(NULL), _desiredInputFormat(), _inputFormat(), - _inputBuffer(), - _monoAudioSamples(NULL), _numInputCallbackBytes(0), _audioOutput(NULL), _desiredOutputFormat(), @@ -55,7 +53,7 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _outputDevice(NULL), _numOutputCallbackBytes(0), _inputRingBuffer(0), - _ringBuffer(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2), + _ringBuffer(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL), _scope(scope), _averagedLatency(0.0), _measuredJitter(0), @@ -72,7 +70,8 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* p _numFramesDisplayStarve(0), _muted(false) { - + // clear the array of locally injected samples + memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); } void Audio::init(QGLWidget *parent) { @@ -282,7 +281,7 @@ void Audio::handleAudioInput() { static int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO); static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + NUM_BYTES_RFC4122_UUID; - _monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); + static int16_t* monoAudioSamples = (int16_t*) (monoAudioDataPacket + leadingBytes); static float inputToNetworkInputRatio = _numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO / NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; @@ -298,29 +297,46 @@ void Audio::handleAudioInput() { int16_t inputAudioSamples[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); - // zero out the monoAudioSamples array - memset(_monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + // zero out the monoAudioSamples array and the locally injected audio + memset(monoAudioSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + + // zero out the locally injected audio in preparation for audio procedural sounds + memset(_localInjectedSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); if (!_muted) { // we aren't muted, downsample the input audio linearResampling((int16_t*) inputAudioSamples, - _monoAudioSamples, + monoAudioSamples, inputSamplesRequired, NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, _inputFormat, _desiredInputFormat); + float loudness = 0; + + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + loudness += fabsf(monoAudioSamples[i]); + } + + _lastInputLoudness = loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + // add input data just written to the scope QMetaObject::invokeMethod(_scope, "addSamples", Qt::QueuedConnection, - Q_ARG(QByteArray, QByteArray((char*) _monoAudioSamples, + Q_ARG(QByteArray, QByteArray((char*) monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL)), Q_ARG(bool, false), Q_ARG(bool, true)); + + if (Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { + // if this person wants local loopback add that to the locally injected audio + memcpy(_localInjectedSamples, monoAudioSamples, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); + } + } else { + // our input loudness is 0, since we're muted + _lastInputLoudness = 0; } // add procedural effects to the appropriate input samples - // addProceduralSounds(monoAudioSamples + (_isBufferSendCallback - // ? BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO : 0), - // (int16_t*) stereoOutputBuffer.data(), - // BUFFER_LENGTH_SAMPLES_PER_CHANNEL / CALLBACK_ACCELERATOR_RATIO); + addProceduralSounds(monoAudioSamples, + NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); NodeList* nodeList = NodeList::getInstance(); Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER); @@ -353,18 +369,6 @@ void Audio::handleAudioInput() { memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); currentPacketPtr += sizeof(headOrientation); - float loudness = 0; - - if (!_muted) { - for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - loudness += fabsf(_monoAudioSamples[i]); - } - - loudness /= NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - } - - _lastInputLoudness = loudness; - nodeList->getNodeSocket().writeDatagram(monoAudioDataPacket, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes, audioMixer->getActiveSocket()->getAddress(), @@ -430,15 +434,13 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { int16_t ringBufferSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; _ringBuffer.readSamples(ringBufferSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); - if (!_muted && Menu::getInstance()->isOptionChecked(MenuOption::EchoLocalAudio)) { - // copy whatever is pointed to at _monoAudioSamples into our ringBufferSamples - // so that local audio is echoed back - - for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _monoAudioSamples[i], - MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2]; - } + // add to the output samples whatever is in the _localAudioOutput byte array + // that lets this user hear sound effects and loopback (if enabled) + + for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { + ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i], + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2]; } // copy the packet from the RB to the output @@ -577,7 +579,7 @@ void Audio::render(int screenWidth, int screenHeight) { } // Take a pointer to the acquired microphone input samples and add procedural sounds -void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples) { +void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; const int VOLUME_BASELINE = 400; @@ -614,10 +616,7 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp int16_t collisionSample = (int16_t) sample; monoInput[i] += collisionSample; - - for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoUpsampledOutput[j] += collisionSample; - } + _localInjectedSamples[i] += collisionSample; _collisionSoundMagnitude *= _collisionSoundDuration; } @@ -640,10 +639,7 @@ void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutp int16_t collisionSample = (int16_t) sample; monoInput[i] += collisionSample; - - for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoUpsampledOutput[j] += collisionSample; - } + _localInjectedSamples[i] += collisionSample; _drumSoundVolume *= (1.f - _drumSoundDecay); } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 9a051b246b..f0c71189dd 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -75,8 +75,7 @@ private: QAudioFormat _desiredInputFormat; QAudioFormat _inputFormat; QIODevice* _inputDevice; - QByteArray _inputBuffer; - int16_t* _monoAudioSamples; + int16_t _localInjectedSamples[NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL]; int _numInputCallbackBytes; QAudioOutput* _audioOutput; QAudioFormat _desiredOutputFormat; @@ -122,7 +121,7 @@ private: inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal - void addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples); + void addProceduralSounds(int16_t* monoInput, int numSamples); void renderToolIcon(int screenHeight); }; From 90ea978278196b49c0c442ae09e21669dac0100e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:03:46 -0800 Subject: [PATCH 20/24] fix some clipping in procedural audio --- interface/src/Audio.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e741443217..6114ce4984 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -351,7 +351,7 @@ void Audio::handleAudioInput() { // + 12 for 3 floats for position + float for bearing + 1 attenuation byte PACKET_TYPE packetType = Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio) - ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; + ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO; char* currentPacketPtr = monoAudioDataPacket + populateTypeAndVersion((unsigned char*) monoAudioDataPacket, packetType); @@ -615,8 +615,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; - monoInput[i] += collisionSample; - _localInjectedSamples[i] += collisionSample; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); _collisionSoundMagnitude *= _collisionSoundDuration; } @@ -638,8 +639,9 @@ void Audio::addProceduralSounds(int16_t* monoInput, int numSamples) { int16_t collisionSample = (int16_t) sample; - monoInput[i] += collisionSample; - _localInjectedSamples[i] += collisionSample; + monoInput[i] = glm::clamp(monoInput[i] + collisionSample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _localInjectedSamples[i] = glm::clamp(_localInjectedSamples[i] + collisionSample, + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); _drumSoundVolume *= (1.f - _drumSoundDecay); } From bb93c64d8d0bfcd8781444a26add7dfe5f1e440c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:21:50 -0800 Subject: [PATCH 21/24] some initial audio stat render fixes --- interface/src/Application.cpp | 2 +- interface/src/Audio.cpp | 18 +++++++++--------- interface/src/Audio.h | 6 ------ libraries/audio/src/AudioRingBuffer.cpp | 2 -- libraries/audio/src/AudioRingBuffer.h | 2 ++ 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index e75c708841..36dccb061a 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -71,7 +71,7 @@ const int IDLE_SIMULATE_MSECS = 16; // How often should call simul // in the idle loop? (60 FPS is default) static QTimer* idleTimer = NULL; -const int STARTUP_JITTER_SAMPLES = PACKET_LENGTH_SAMPLES_PER_CHANNEL / 2; +const int STARTUP_JITTER_SAMPLES = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / 2; // Startup optimistically with small jitter buffer that // will start playback on the second received audio packet. diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 6114ce4984..add99ec4f2 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -451,6 +451,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { _desiredOutputFormat, _outputFormat); if (_outputDevice) { + _outputDevice->write(outputBuffer); // add output (@speakers) data just written to the scope @@ -500,7 +501,7 @@ void Audio::render(int screenWidth, int screenHeight) { glVertex2f(currentX, topY); glVertex2f(currentX, bottomY); - for (int i = 0; i < _ringBuffer.getSampleCapacity() / 2; i++) { + for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES; i++) { glVertex2f(currentX, halfY); glVertex2f(currentX + frameWidth, halfY); currentX += frameWidth; @@ -512,14 +513,12 @@ void Audio::render(int screenWidth, int screenHeight) { // Show a bar with the amount of audio remaining in ring buffer beyond current playback float remainingBuffer = 0; - timeval currentTime; - gettimeofday(¤tTime, NULL); - float timeLeftInCurrentBuffer = 0; - if (_lastCallbackTime.tv_usec > 0) { - timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - diffclock(&_lastCallbackTime, ¤tTime); - } - remainingBuffer = PACKET_LENGTH_SAMPLES / PACKET_LENGTH_SAMPLES * AUDIO_CALLBACK_MSECS; + int bytesLeftInAudioOutput = _audioOutput->bufferSize() - _audioOutput->bytesFree(); + float secondsLeftForAudioOutput = (bytesLeftInAudioOutput / sizeof(int16_t)) / _outputFormat.sampleRate(); + float timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - (secondsLeftForAudioOutput * 1000); + + remainingBuffer = 1 / AUDIO_CALLBACK_MSECS; if (_numFramesDisplayStarve == 0) { glColor3f(0, 1, 0); @@ -557,7 +556,8 @@ void Audio::render(int screenWidth, int screenHeight) { // Show a red bar with the 'start' point of one frame plus the jitter buffer glColor3f(1, 0, 0); - int jitterBufferPels = (1.f + (float)getJitterBufferSamples() / (float) PACKET_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth; + int jitterBufferPels = (1.f + (float)getJitterBufferSamples() + / (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth; sprintf(out, "%.0f\n", getJitterBufferSamples() / SAMPLE_RATE * 1000.f); drawtext(startX + jitterBufferPels - 5, topY - 9, 0.10, 0, 1, 0, out, 1, 0, 0); sprintf(out, "j %.1f\n", _measuredJitter); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index f0c71189dd..eff1be345b 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -26,11 +26,6 @@ static const int NUM_AUDIO_CHANNELS = 2; -static const int PACKET_LENGTH_BYTES = 1024; -static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; -static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); -static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; - class QAudioInput; class QAudioOutput; class QIODevice; @@ -86,7 +81,6 @@ private: AudioRingBuffer _ringBuffer; Oscilloscope* _scope; StDev _stdev; - timeval _lastCallbackTime; timeval _lastReceiveTime; float _averagedLatency; float _measuredJitter; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 83c2fe59a0..5e9abf38b7 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -15,8 +15,6 @@ #include "AudioRingBuffer.h" -const short RING_BUFFER_LENGTH_FRAMES = 10; - AudioRingBuffer::AudioRingBuffer(int numFrameSamples) : NodeData(NULL), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 3e6917456f..addad13146 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -25,6 +25,8 @@ const int NETWORK_BUFFER_LENGTH_SAMPLES_STEREO = NETWORK_BUFFER_LENGTH_BYTES_STE const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); +const short RING_BUFFER_LENGTH_FRAMES = 10; + const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); From 30b22a13ad0878c178eda7dc2d2a4d5e8034ac36 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:30:04 -0800 Subject: [PATCH 22/24] remove an unecessary condition in Audio.cpp --- interface/src/Audio.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index add99ec4f2..d898bd8566 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -425,9 +425,7 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { qDebug() << "Buffer is starved and doesn't have enough samples to start. Held back.\n"; } else { // We are either already playing back, or we have enough audio to start playing back. - if (_ringBuffer.isStarved()) { - _ringBuffer.setIsStarved(false); - } + _ringBuffer.setIsStarved(false); // copy the samples we'll resample from the ring buffer - this also // pushes the read pointer of the ring buffer forwards From efe05adcae892ecb4ce33ba1bd045a9cd4e0f911 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:52:04 -0800 Subject: [PATCH 23/24] fix audio graphs for new audio infrastructure --- interface/src/Audio.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index d898bd8566..95b1a4fbd9 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -509,14 +509,15 @@ void Audio::render(int screenWidth, int screenHeight) { } glEnd(); - // Show a bar with the amount of audio remaining in ring buffer beyond current playback - float remainingBuffer = 0; + // show a bar with the amount of audio remaining in ring buffer and output device + // beyond the current playback int bytesLeftInAudioOutput = _audioOutput->bufferSize() - _audioOutput->bytesFree(); - float secondsLeftForAudioOutput = (bytesLeftInAudioOutput / sizeof(int16_t)) / _outputFormat.sampleRate(); - float timeLeftInCurrentBuffer = AUDIO_CALLBACK_MSECS - (secondsLeftForAudioOutput * 1000); - - remainingBuffer = 1 / AUDIO_CALLBACK_MSECS; + float secondsLeftForAudioOutput = (bytesLeftInAudioOutput / sizeof(int16_t)) + / ((float) _outputFormat.sampleRate() * _outputFormat.channelCount()); + float secondsLeftForRingBuffer = _ringBuffer.samplesAvailable() + / ((float) _desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount()); + float msLeftForAudioOutput = (secondsLeftForAudioOutput + secondsLeftForRingBuffer) * 1000; if (_numFramesDisplayStarve == 0) { glColor3f(0, 1, 0); @@ -525,19 +526,19 @@ void Audio::render(int screenWidth, int screenHeight) { _numFramesDisplayStarve--; } + if (_averagedLatency == 0.0) { + _averagedLatency = msLeftForAudioOutput; + } else { + _averagedLatency = 0.99f * _averagedLatency + 0.01f * (msLeftForAudioOutput); + } + glBegin(GL_QUADS); glVertex2f(startX, topY + 2); - glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2); - glVertex2f(startX + (remainingBuffer + timeLeftInCurrentBuffer) / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, topY + 2); + glVertex2f(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth, bottomY - 2); glVertex2f(startX, bottomY - 2); glEnd(); - if (_averagedLatency == 0.0) { - _averagedLatency = remainingBuffer + timeLeftInCurrentBuffer; - } else { - _averagedLatency = 0.99f * _averagedLatency + 0.01f * (remainingBuffer + timeLeftInCurrentBuffer); - } - // Show a yellow bar with the averaged msecs latency you are hearing (from time of packet receipt) glColor3f(1,1,0); glBegin(GL_QUADS); From a31f15ee79100d957a2b38f1333b802be1348f1d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Dec 2013 16:57:30 -0800 Subject: [PATCH 24/24] don't blow away a channel on local injection loopback --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 95b1a4fbd9..4129973bb7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -438,7 +438,8 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { for (int i = 0; i < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { ringBufferSamples[i * 2] = glm::clamp(ringBufferSamples[i * 2] + _localInjectedSamples[i], MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - ringBufferSamples[(i * 2) + 1] = ringBufferSamples[i * 2]; + ringBufferSamples[(i * 2) + 1] += glm::clamp(ringBufferSamples[(i * 2) + 1] + _localInjectedSamples[i], + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } // copy the packet from the RB to the output