mirror of
https://github.com/lubosz/overte.git
synced 2025-04-23 23:33:48 +02:00
repetition-with-fade implemented; testing interface crash
This commit is contained in:
wangyix
parent
1f011bfe9d
commit
e276d15ed4
10 changed files with 261 additions and 75 deletions
|
|
@ -95,6 +95,26 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f;
|
|||
|
||||
int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd,
|
||||
AvatarAudioStream* listeningNodeStream) {
|
||||
// If repetition with fade is enabled:
|
||||
// If streamToAdd could not provide a frame (it was starved), then we'll mix its previously-mixed frame
|
||||
// This is preferable to not mixing it at all since that's equivalent to inserting silence.
|
||||
// Basically, we'll repeat that last frame until it has a frame to mix. Depending on how many times
|
||||
// we've repeated that frame in a row, we'll gradually fade that repeated frame into silence.
|
||||
// This improves the perceived quality of the audio slightly.
|
||||
|
||||
float repeatedFrameFadeFactor = 1.0f;
|
||||
|
||||
if (!streamToAdd->lastPopSucceeded()) {
|
||||
if (_streamSettings._repetitionWithFade) {
|
||||
repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1);
|
||||
if (repeatedFrameFadeFactor == 0.0f) {
|
||||
return 0;
|
||||
}
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
float bearingRelativeAngleToSource = 0.0f;
|
||||
float attenuationCoefficient = 1.0f;
|
||||
int numSamplesDelay = 0;
|
||||
|
|
@ -216,12 +236,13 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
|
|||
int delayedChannelIndex = 0;
|
||||
|
||||
const int SINGLE_STEREO_OFFSET = 2;
|
||||
float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor;
|
||||
|
||||
for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 4) {
|
||||
|
||||
// setup the int16_t variables for the two sample sets
|
||||
correctStreamSample[0] = streamPopOutput[s / 2] * attenuationCoefficient;
|
||||
correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationCoefficient;
|
||||
correctStreamSample[0] = streamPopOutput[s / 2] * attenuationAndFade;
|
||||
correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationAndFade;
|
||||
|
||||
delayedChannelIndex = s + (numSamplesDelay * 2) + delayedChannelOffset;
|
||||
|
||||
|
|
@ -237,7 +258,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
|
|||
if (numSamplesDelay > 0) {
|
||||
// if there was a sample delay for this stream, we need to pull samples prior to the popped output
|
||||
// to stick at the beginning
|
||||
float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio;
|
||||
float attenuationAndWeakChannelRatioAndFade = attenuationCoefficient * weakChannelAmplitudeRatio * repeatedFrameFadeFactor;
|
||||
AudioRingBuffer::ConstIterator delayStreamPopOutput = streamPopOutput - numSamplesDelay;
|
||||
|
||||
// TODO: delayStreamPopOutput may be inside the last frame written if the ringbuffer is completely full
|
||||
|
|
@ -245,7 +266,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
|
|||
|
||||
for (int i = 0; i < numSamplesDelay; i++) {
|
||||
int parentIndex = i * 2;
|
||||
_clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatio;
|
||||
_clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatioAndFade;
|
||||
++delayStreamPopOutput;
|
||||
}
|
||||
}
|
||||
|
|
@ -256,8 +277,10 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
|
|||
attenuationCoefficient = 1.0f;
|
||||
}
|
||||
|
||||
float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor;
|
||||
|
||||
for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s++) {
|
||||
_clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationCoefficient),
|
||||
_clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationAndFade),
|
||||
MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
|
||||
}
|
||||
}
|
||||
|
|
@ -285,7 +308,6 @@ int AudioMixer::prepareMixForListeningNode(Node* node) {
|
|||
PositionalAudioStream* otherNodeStream = i.value();
|
||||
|
||||
if ((*otherNode != *node || otherNodeStream->shouldLoopbackForNode())
|
||||
&& otherNodeStream->lastPopSucceeded()
|
||||
&& otherNodeStream->getLastPopOutputFrameLoudness() > 0.0f) {
|
||||
//&& otherNodeStream->getLastPopOutputTrailingLoudness() > 0.0f) {
|
||||
|
||||
|
|
@ -627,6 +649,7 @@ void AudioMixer::run() {
|
|||
}
|
||||
|
||||
// send mixed audio packet
|
||||
if (nodeData->getOutgoingSequenceNumber() % 100 < 50)
|
||||
nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node);
|
||||
nodeData->incrementOutgoingMixedAudioSequenceNumber();
|
||||
|
||||
|
|
@ -654,6 +677,4 @@ void AudioMixer::run() {
|
|||
usleep(usecToSleep);
|
||||
}
|
||||
}
|
||||
|
||||
delete[] clientMixBuffer;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
"type": "checkbox",
|
||||
"label": "Dynamic Jitter Buffers",
|
||||
"help": "Dynamically buffer client audio based on perceived jitter in packet receipt timing",
|
||||
"default": true
|
||||
"default": false
|
||||
},
|
||||
"B-static-desired-jitter-buffer-frames": {
|
||||
"label": "Static Desired Jitter Buffer Frames",
|
||||
|
|
@ -49,7 +49,7 @@
|
|||
"type": "checkbox",
|
||||
"label": "Repetition with Fade:",
|
||||
"help": "If enabled, dropped frames and mixing during starves will repeat the last frame, eventually fading to silence",
|
||||
"default": true
|
||||
"default": false
|
||||
},
|
||||
"Z-unattenuated-zone": {
|
||||
"label": "Unattenuated Zone",
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ Audio::Audio(QObject* parent) :
|
|||
_scopeInput(0),
|
||||
_scopeOutputLeft(0),
|
||||
_scopeOutputRight(0),
|
||||
_scopeLastFrame(),
|
||||
_statsEnabled(false),
|
||||
_statsShowInjectedStreams(false),
|
||||
_outgoingAvatarAudioSequenceNumber(0),
|
||||
|
|
@ -113,15 +114,17 @@ Audio::Audio(QObject* parent) :
|
|||
_audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS),
|
||||
_lastSentAudioPacket(0),
|
||||
_packetSentTimeGaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS),
|
||||
_audioOutputIODevice(*this)
|
||||
_audioOutputIODevice(_receivedAudioStream)
|
||||
{
|
||||
// clear the array of locally injected samples
|
||||
memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
|
||||
// Create the noise sample array
|
||||
_noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES];
|
||||
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedAudioStreamSamples, Qt::DirectConnection);
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::dataParsed, this, &Audio::updateScopeBuffers, Qt::DirectConnection);
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedSilence, this, &Audio::addStereoSilenceToScope, Qt::DirectConnection);
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedLastFrameRepeatedWithFade, this, &Audio::addLastFrameRepeatedWithFadeToScope, Qt::DirectConnection);
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedStereoSamples, this, &Audio::addStereoSamplesToScope, Qt::DirectConnection);
|
||||
connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedSamples, Qt::DirectConnection);
|
||||
}
|
||||
|
||||
void Audio::init(QGLWidget *parent) {
|
||||
|
|
@ -657,9 +660,7 @@ void Audio::handleAudioInput() {
|
|||
if (!_isStereoInput && _scopeEnabled && !_scopeEnabledPause) {
|
||||
unsigned int numMonoAudioChannels = 1;
|
||||
unsigned int monoAudioChannel = 0;
|
||||
addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, monoAudioChannel, numMonoAudioChannels);
|
||||
_scopeInputOffset += NETWORK_SAMPLES_PER_FRAME;
|
||||
_scopeInputOffset %= _samplesPerScope;
|
||||
_scopeInputOffset = addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, NETWORK_SAMPLES_PER_FRAME, monoAudioChannel, numMonoAudioChannels);
|
||||
}
|
||||
|
||||
NodeList* nodeList = NodeList::getInstance();
|
||||
|
|
@ -733,7 +734,48 @@ void Audio::handleAudioInput() {
|
|||
}
|
||||
}
|
||||
|
||||
void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
|
||||
const int STEREO_FACTOR = 2;
|
||||
|
||||
void Audio::addStereoSilenceToScope(int silentSamplesPerChannel) {
|
||||
if (!_scopeEnabled || _scopeEnabledPause) {
|
||||
return;
|
||||
}
|
||||
addSilenceToScope(_scopeOutputLeft, _scopeOutputOffset, silentSamplesPerChannel);
|
||||
_scopeOutputOffset = addSilenceToScope(_scopeOutputRight, _scopeOutputOffset, silentSamplesPerChannel);
|
||||
}
|
||||
|
||||
void Audio::addStereoSamplesToScope(const QByteArray& samples) {
|
||||
if (!_scopeEnabled || _scopeEnabledPause) {
|
||||
return;
|
||||
}
|
||||
const int16_t* samplesData = reinterpret_cast<const int16_t*>(samples.data());
|
||||
int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR;
|
||||
|
||||
addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR);
|
||||
_scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR);
|
||||
|
||||
_scopeLastFrame = samples.right(NETWORK_BUFFER_LENGTH_BYTES_STEREO);
|
||||
}
|
||||
|
||||
void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) {
|
||||
printf("addLastFrameRepeatedWithFadeToScope");
|
||||
const int16_t* lastFrameData = reinterpret_cast<const int16_t*>(_scopeLastFrame.data());
|
||||
|
||||
int samplesRemaining = samplesPerChannel;
|
||||
int indexOfRepeat = 0;
|
||||
do {
|
||||
int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME);
|
||||
float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat);
|
||||
printf("%f ", fade);
|
||||
addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade);
|
||||
_scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade);
|
||||
|
||||
samplesRemaining -= samplesToWriteThisIteration;
|
||||
} while (samplesRemaining > 0);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
|
||||
|
||||
const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t);
|
||||
const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
|
||||
|
|
@ -780,7 +822,7 @@ void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QBy
|
|||
_desiredOutputFormat, _outputFormat);
|
||||
}
|
||||
|
||||
void Audio::updateScopeBuffers() {
|
||||
/*void Audio::updateScopeBuffers() {
|
||||
if (_scopeEnabled && !_scopeEnabledPause) {
|
||||
unsigned int numAudioChannels = _desiredOutputFormat.channelCount();
|
||||
const int16_t* samples = _receivedAudioStream.getNetworkSamples();
|
||||
|
|
@ -794,7 +836,7 @@ void Audio::updateScopeBuffers() {
|
|||
samples, audioChannel, numAudioChannels);
|
||||
|
||||
audioChannel = 1;
|
||||
addBufferToScope(
|
||||
_scopeOutputOffset = addBufferToScope(
|
||||
_scopeOutputRight,
|
||||
_scopeOutputOffset,
|
||||
samples, audioChannel, numAudioChannels);
|
||||
|
|
@ -806,7 +848,7 @@ void Audio::updateScopeBuffers() {
|
|||
}
|
||||
|
||||
_receivedAudioStream.clearNetworkSamples();
|
||||
}
|
||||
}*/
|
||||
|
||||
void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) {
|
||||
|
||||
|
|
@ -1259,12 +1301,15 @@ void Audio::freeScope() {
|
|||
}
|
||||
}
|
||||
|
||||
void Audio::addBufferToScope(
|
||||
QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels) {
|
||||
int Audio::addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamplesPerChannel,
|
||||
unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade) {
|
||||
|
||||
// Constant multiplier to map sample value to vertical size of scope
|
||||
float multiplier = (float)MULTIPLIER_SCOPE_HEIGHT / logf(2.0f);
|
||||
|
||||
// Used to scale each sample. (logf(sample) + fadeOffset) is same as logf(sample * fade).
|
||||
float fadeOffset = logf(fade);
|
||||
|
||||
// Temporary variable receives sample value
|
||||
float sample;
|
||||
|
||||
|
|
@ -1275,17 +1320,41 @@ void Audio::addBufferToScope(
|
|||
// Short int pointer to mapped samples in byte array
|
||||
int16_t* destination = (int16_t*) byteArray->data();
|
||||
|
||||
for (unsigned int i = 0; i < NETWORK_SAMPLES_PER_FRAME; i++) {
|
||||
for (int i = 0; i < sourceSamplesPerChannel; i++) {
|
||||
sample = (float)source[i * sourceNumberOfChannels + sourceChannel];
|
||||
if (sample > 0) {
|
||||
value = (int16_t)(multiplier * logf(sample));
|
||||
} else if (sample < 0) {
|
||||
value = (int16_t)(-multiplier * logf(-sample));
|
||||
if (sample > 1) {
|
||||
value = (int16_t)(multiplier * (logf(sample) + fadeOffset));
|
||||
} else if (sample < -1) {
|
||||
value = (int16_t)(-multiplier * (logf(-sample) + fadeOffset));
|
||||
} else {
|
||||
value = 0;
|
||||
}
|
||||
destination[i + frameOffset] = value;
|
||||
destination[frameOffset] = value;
|
||||
frameOffset = (frameOffset == _samplesPerScope - 1) ? 0 : frameOffset + 1;
|
||||
}
|
||||
return frameOffset;
|
||||
}
|
||||
|
||||
int Audio::addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples) {
|
||||
|
||||
QMutexLocker lock(&_guard);
|
||||
// Short int pointer to mapped samples in byte array
|
||||
int16_t* destination = (int16_t*)byteArray->data();
|
||||
|
||||
if (silentSamples >= _samplesPerScope) {
|
||||
memset(destination, 0, byteArray->size());
|
||||
return frameOffset;
|
||||
}
|
||||
|
||||
int samplesToBufferEnd = _samplesPerScope - frameOffset;
|
||||
if (silentSamples > samplesToBufferEnd) {
|
||||
memset(destination + frameOffset, 0, samplesToBufferEnd * sizeof(int16_t));
|
||||
memset(destination, 0, silentSamples - samplesToBufferEnd * sizeof(int16_t));
|
||||
} else {
|
||||
memset(destination + frameOffset, 0, silentSamples * sizeof(int16_t));
|
||||
}
|
||||
|
||||
return (frameOffset + silentSamples) % _samplesPerScope;
|
||||
}
|
||||
|
||||
void Audio::renderStats(const float* color, int width, int height) {
|
||||
|
|
@ -1761,13 +1830,11 @@ float Audio::getInputRingBufferMsecsAvailable() const {
|
|||
}
|
||||
|
||||
qint64 Audio::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
|
||||
MixedProcessedAudioStream& receivedAUdioStream = _parent._receivedAudioStream;
|
||||
|
||||
int samplesRequested = maxSize / sizeof(int16_t);
|
||||
int samplesPopped;
|
||||
int bytesWritten;
|
||||
if ((samplesPopped = receivedAUdioStream.popSamples(samplesRequested, false)) > 0) {
|
||||
AudioRingBuffer::ConstIterator lastPopOutput = receivedAUdioStream.getLastPopOutput();
|
||||
if ((samplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
|
||||
AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
|
||||
lastPopOutput.readSamples((int16_t*)data, samplesPopped);
|
||||
bytesWritten = samplesPopped * sizeof(int16_t);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -48,14 +48,14 @@ public:
|
|||
|
||||
class AudioOutputIODevice : public QIODevice {
|
||||
public:
|
||||
AudioOutputIODevice(Audio& parent) : _parent(parent) {};
|
||||
AudioOutputIODevice(MixedProcessedAudioStream& receivedAudioStream) : _receivedAudioStream(receivedAudioStream) {};
|
||||
|
||||
void start() { open(QIODevice::ReadOnly); }
|
||||
void stop() { close(); }
|
||||
qint64 readData(char * data, qint64 maxSize);
|
||||
qint64 writeData(const char * data, qint64 maxSize) { return 0; }
|
||||
private:
|
||||
Audio& _parent;
|
||||
MixedProcessedAudioStream& _receivedAudioStream;
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -105,8 +105,6 @@ public slots:
|
|||
void addReceivedAudioToStream(const QByteArray& audioByteArray);
|
||||
void parseAudioStreamStatsPacket(const QByteArray& packet);
|
||||
void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples);
|
||||
void processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
|
||||
void updateScopeBuffers();
|
||||
void handleAudioInput();
|
||||
void reset();
|
||||
void resetStats();
|
||||
|
|
@ -123,6 +121,11 @@ public slots:
|
|||
void selectAudioScopeFiveFrames();
|
||||
void selectAudioScopeTwentyFrames();
|
||||
void selectAudioScopeFiftyFrames();
|
||||
|
||||
void addStereoSilenceToScope(int silentSamplesPerChannel);
|
||||
void addLastFrameRepeatedWithFadeToScope(int samplesPerChannel);
|
||||
void addStereoSamplesToScope(const QByteArray& samples);
|
||||
void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
|
||||
|
||||
virtual void handleAudioByteArray(const QByteArray& audioByteArray);
|
||||
|
||||
|
|
@ -244,8 +247,9 @@ private:
|
|||
void reallocateScope(int frames);
|
||||
|
||||
// Audio scope methods for data acquisition
|
||||
void addBufferToScope(
|
||||
QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels);
|
||||
int addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamples,
|
||||
unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade = 1.0f);
|
||||
int addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples);
|
||||
|
||||
// Audio scope methods for rendering
|
||||
void renderBackground(const float* color, int x, int y, int width, int height);
|
||||
|
|
@ -272,6 +276,7 @@ private:
|
|||
QByteArray* _scopeInput;
|
||||
QByteArray* _scopeOutputLeft;
|
||||
QByteArray* _scopeOutputRight;
|
||||
QByteArray _scopeLastFrame;
|
||||
#ifdef _WIN32
|
||||
static const unsigned int STATS_WIDTH = 1500;
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -224,3 +224,45 @@ float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const {
|
|||
float AudioRingBuffer::getNextOutputFrameLoudness() const {
|
||||
return getFrameLoudness(_nextOutput);
|
||||
}
|
||||
|
||||
int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) {
|
||||
int samplesToCopy = std::min(maxSamples, _sampleCapacity);
|
||||
int samplesRoomFor = _sampleCapacity - samplesAvailable();
|
||||
if (samplesToCopy > samplesRoomFor) {
|
||||
// there's not enough room for this write. erase old data to make room for this new data
|
||||
int samplesToDelete = samplesToCopy - samplesRoomFor;
|
||||
_nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete);
|
||||
_overflowCount++;
|
||||
qDebug() << "Overflowed ring buffer! Overwriting old data";
|
||||
}
|
||||
|
||||
int16_t* bufferLast = _buffer + _bufferLength - 1;
|
||||
for (int i = 0; i < samplesToCopy; i++) {
|
||||
*_endOfLastWrite = *source;
|
||||
_endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1;
|
||||
++source;
|
||||
}
|
||||
|
||||
return samplesToCopy;
|
||||
}
|
||||
|
||||
int AudioRingBuffer::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) {
|
||||
int samplesToCopy = std::min(maxSamples, _sampleCapacity);
|
||||
int samplesRoomFor = _sampleCapacity - samplesAvailable();
|
||||
if (samplesToCopy > samplesRoomFor) {
|
||||
// there's not enough room for this write. erase old data to make room for this new data
|
||||
int samplesToDelete = samplesToCopy - samplesRoomFor;
|
||||
_nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete);
|
||||
_overflowCount++;
|
||||
qDebug() << "Overflowed ring buffer! Overwriting old data";
|
||||
}
|
||||
|
||||
int16_t* bufferLast = _buffer + _bufferLength - 1;
|
||||
for (int i = 0; i < samplesToCopy; i++) {
|
||||
*_endOfLastWrite = (int16_t)((float)(*source) * fade);
|
||||
_endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1;
|
||||
++source;
|
||||
}
|
||||
|
||||
return samplesToCopy;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -189,8 +189,12 @@ public:
|
|||
};
|
||||
|
||||
ConstIterator nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); }
|
||||
|
||||
ConstIterator lastFrameWritten() const { return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; }
|
||||
|
||||
float getFrameLoudness(ConstIterator frameStart) const;
|
||||
|
||||
int writeSamples(ConstIterator source, int maxSamples);
|
||||
int writeSamplesWithFade(ConstIterator source, int maxSamples, float fade);
|
||||
};
|
||||
|
||||
#endif // hifi_AudioRingBuffer_h
|
||||
|
|
@ -162,8 +162,6 @@ int InboundAudioStream::parseData(const QByteArray& packet) {
|
|||
|
||||
framesAvailableChanged();
|
||||
|
||||
emit dataParsed();
|
||||
|
||||
return readBytes;
|
||||
}
|
||||
|
||||
|
|
@ -418,9 +416,31 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() {
|
|||
}
|
||||
|
||||
int InboundAudioStream::writeSamplesForDroppedPackets(int networkSamples) {
|
||||
if (_repetitionWithFade) {
|
||||
return writeLastFrameRepeatedWithFade(networkSamples);
|
||||
}
|
||||
return writeDroppableSilentSamples(networkSamples);
|
||||
}
|
||||
|
||||
int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) {
|
||||
AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten();
|
||||
int frameSize = _ringBuffer.getNumFrameSamples();
|
||||
int samplesToWrite = samples;
|
||||
int indexOfRepeat = 0;
|
||||
do {
|
||||
int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize);
|
||||
float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat);
|
||||
if (fade == 1.0f) {
|
||||
samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration);
|
||||
} else {
|
||||
samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade);
|
||||
}
|
||||
indexOfRepeat++;
|
||||
} while (samplesToWrite > 0);
|
||||
|
||||
return samples;
|
||||
}
|
||||
|
||||
float InboundAudioStream::getLastPopOutputFrameLoudness() const {
|
||||
return _ringBuffer.getFrameLoudness(_lastPopOutput);
|
||||
}
|
||||
|
|
@ -448,3 +468,25 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const {
|
|||
|
||||
return streamStats;
|
||||
}
|
||||
|
||||
float calculateRepeatedFrameFadeFactor(int indexOfRepeat) {
|
||||
// fade factor scheme is from this paper:
|
||||
// http://inst.eecs.berkeley.edu/~ee290t/sp04/lectures/packet_loss_recov_paper11.pdf
|
||||
|
||||
const float INITIAL_MSECS_NO_FADE = 20.0f;
|
||||
const float MSECS_FADE_TO_ZERO = 320.0f;
|
||||
|
||||
const float INITIAL_FRAMES_NO_FADE = INITIAL_MSECS_NO_FADE * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS;
|
||||
const float FRAMES_FADE_TO_ZERO = MSECS_FADE_TO_ZERO * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS;
|
||||
|
||||
const float SAMPLE_RANGE = std::numeric_limits<int16_t>::max();
|
||||
|
||||
if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE) {
|
||||
return 1.0f;
|
||||
} else if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE + FRAMES_FADE_TO_ZERO) {
|
||||
return pow(SAMPLE_RANGE, -(indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO);
|
||||
|
||||
//return 1.0f - ((indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO);
|
||||
}
|
||||
return 0.0f;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30;
|
|||
|
||||
// this controls the window size of the time-weighted avg of frames available. Every time the window fills up,
|
||||
// _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset.
|
||||
const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND;
|
||||
const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 10 * USECS_PER_SECOND;
|
||||
|
||||
// default values for members of the Settings struct
|
||||
const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10;
|
||||
|
|
@ -157,9 +157,6 @@ public:
|
|||
|
||||
int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); }
|
||||
|
||||
signals:
|
||||
void dataParsed();
|
||||
|
||||
public slots:
|
||||
/// This function should be called every second for all the stats to function properly. If dynamic jitter buffers
|
||||
/// is enabled, those stats are used to calculate _desiredJitterBufferFrames.
|
||||
|
|
@ -191,6 +188,10 @@ protected:
|
|||
|
||||
/// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer
|
||||
virtual int writeDroppableSilentSamples(int silentSamples);
|
||||
|
||||
/// writes the last written frame repeatedly, gradually fading to silence.
|
||||
/// used for writing samples for dropped packets.
|
||||
virtual int writeLastFrameRepeatedWithFade(int samples);
|
||||
|
||||
protected:
|
||||
|
||||
|
|
@ -246,4 +247,6 @@ protected:
|
|||
bool _repetitionWithFade;
|
||||
};
|
||||
|
||||
float calculateRepeatedFrameFadeFactor(int indexOfRepeat);
|
||||
|
||||
#endif // hifi_InboundAudioStream_h
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@
|
|||
|
||||
#include "MixedProcessedAudioStream.h"
|
||||
|
||||
static const int STEREO_FACTOR = 2;
|
||||
|
||||
MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings)
|
||||
: InboundAudioStream(numFrameSamples, numFramesCapacity, settings),
|
||||
_networkSamplesWritten(0)
|
||||
: InboundAudioStream(numFrameSamples, numFramesCapacity, settings)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -23,30 +24,33 @@ void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCount
|
|||
_ringBuffer.resizeForFrameSize(deviceOutputFrameSize);
|
||||
}
|
||||
|
||||
int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
|
||||
|
||||
memcpy(&_networkSamples[_networkSamplesWritten], packetAfterStreamProperties.data(), packetAfterStreamProperties.size());
|
||||
_networkSamplesWritten += packetAfterStreamProperties.size() / sizeof(int16_t);
|
||||
|
||||
QByteArray outputBuffer;
|
||||
emit processSamples(packetAfterStreamProperties, outputBuffer);
|
||||
|
||||
_ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
|
||||
|
||||
return packetAfterStreamProperties.size();
|
||||
}
|
||||
|
||||
int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) {
|
||||
int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples));
|
||||
|
||||
int networkSilentSamplesWritten = deviceToNetworkSamples(deviceSilentSamplesWritten);
|
||||
memset(&_networkSamples[_networkSamplesWritten], 0, networkSilentSamplesWritten * sizeof(int16_t));
|
||||
_networkSamplesWritten += networkSilentSamplesWritten;
|
||||
emit addedSilence(deviceToNetworkSamples(deviceSilentSamplesWritten) / STEREO_FACTOR);
|
||||
|
||||
return deviceSilentSamplesWritten;
|
||||
}
|
||||
|
||||
static const int STEREO_FACTOR = 2;
|
||||
int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) {
|
||||
int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(networkToDeviceSamples(samples));
|
||||
|
||||
emit addedLastFrameRepeatedWithFade(deviceToNetworkSamples(deviceSamplesWritten) / STEREO_FACTOR);
|
||||
|
||||
return deviceSamplesWritten;
|
||||
}
|
||||
|
||||
int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) {
|
||||
|
||||
emit addedStereoSamples(packetAfterStreamProperties);
|
||||
|
||||
QByteArray outputBuffer;
|
||||
emit processSamples(packetAfterStreamProperties, outputBuffer);
|
||||
|
||||
_ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
|
||||
|
||||
return packetAfterStreamProperties.size();
|
||||
}
|
||||
|
||||
int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) {
|
||||
return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_FACTOR * SAMPLE_RATE);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
#include "InboundAudioStream.h"
|
||||
|
||||
class Audio;
|
||||
|
||||
class MixedProcessedAudioStream : public InboundAudioStream {
|
||||
Q_OBJECT
|
||||
public:
|
||||
|
|
@ -21,30 +23,26 @@ public:
|
|||
|
||||
signals:
|
||||
|
||||
void addedSilence(int silentSamplesPerChannel);
|
||||
void addedLastFrameRepeatedWithFade(int samplesPerChannel);
|
||||
void addedStereoSamples(const QByteArray& samples);
|
||||
|
||||
void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
|
||||
|
||||
public:
|
||||
void outputFormatChanged(int outputFormatChannelCountTimesSampleRate);
|
||||
|
||||
const int16_t* getNetworkSamples() const { return _networkSamples; }
|
||||
int getNetworkSamplesWritten() const { return _networkSamplesWritten; }
|
||||
|
||||
void clearNetworkSamples() { _networkSamplesWritten = 0; }
|
||||
|
||||
protected:
|
||||
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
|
||||
int writeDroppableSilentSamples(int silentSamples);
|
||||
int writeLastFrameRepeatedWithFade(int samples);
|
||||
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples);
|
||||
|
||||
private:
|
||||
int networkToDeviceSamples(int networkSamples);
|
||||
int deviceToNetworkSamples(int deviceSamples);
|
||||
|
||||
private:
|
||||
int _outputFormatChannelsTimesSampleRate;
|
||||
|
||||
// this buffer keeps a copy of the network samples written during parseData() for the sole purpose
|
||||
// of passing it on to the audio scope
|
||||
int16_t _networkSamples[10 * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
|
||||
int _networkSamplesWritten;
|
||||
};
|
||||
|
||||
#endif // hifi_MixedProcessedAudioStream_h
|
||||
|
|
|
|||
Loading…
Reference in a new issue