Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-20 11:45:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

removed heap alloc of receivedSamples; cleaned up code

Browse source
This commit is contained in:
wangyix 2014-08-04 17:14:26 -07:00
parent 4d7d6f1e25
commit 3fe4c29848
8 changed files with 100 additions and 247 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

205
interface/src/Audio.cpp
View file

@ -64,7 +64,7 @@ Audio::Audio(QObject* parent) :
_audioOutput(NULL),
_desiredOutputFormat(),
_outputFormat(),
//_outputDevice(NULL),
_outputFrameSize(0),
_numOutputCallbackBytes(0),
_loopbackAudioOutput(NULL),
_loopbackOutputDevice(NULL),
@ -129,7 +129,7 @@ void Audio::init(QGLWidget *parent) {
_muteTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/mic-mute.svg"));
_boxTextureId = parent->bindTexture(QImage(Application::resourcesPath() + "images/audio-box.svg"));
connect(&_receivedAudioStream, &RawMixedAudioStream::processSamples, this, &Audio::receivedAudioStreamProcessSamples, Qt::DirectConnection);
connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedAudioStreamSamples, Qt::DirectConnection);
}
void Audio::reset() {
@ -314,7 +314,7 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
}
}
void linearResampling(int16_t* sourceSamples, int16_t* destinationSamples,
void linearResampling(const int16_t* sourceSamples, int16_t* destinationSamples,
unsigned int numSourceSamples, unsigned int numDestinationSamples,
const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) {
if (sourceAudioFormat == destinationAudioFormat) {
@ -725,24 +725,20 @@ void Audio::handleAudioInput() {
}
delete[] inputAudioSamples;
}
/*if (_receivedAudioStream.getPacketsReceived() > 0) {
pushAudioToOutput();
}*/
}
void Audio::receivedAudioStreamProcessSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
printf("receivedAudioStreamProcessSamples()\n");
static const int numNetworkOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO;
static const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
// NOTE: we assume inputBuffer contains NETWORK_BUFFER_LENGTH_SAMPLES_STEREO audio samples
const int numNetworkOutputSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO;
const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())
/ (_desiredOutputFormat.sampleRate() * _desiredOutputFormat.channelCount());
outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
int16_t* receivedSamples = new int16_t[numNetworkOutputSamples];
const int16_t* receivedSamples;
if (_processSpatialAudio) {
unsigned int sampleTime = _spatialAudioStart;
QByteArray buffer = inputBuffer.left(numNetworkOutputSamples * sizeof(int16_t));
@ -758,16 +754,18 @@ void Audio::receivedAudioStreamProcessSamples(const QByteArray& inputBuffer, QBy
// copy the samples we'll resample from the spatial audio ring buffer - this also
// pushes the read pointer of the spatial audio ring buffer forwards
_spatialAudioRingBuffer.readSamples(receivedSamples, numNetworkOutputSamples);
_spatialAudioRingBuffer.readSamples(_spatialProcessingBuffer, numNetworkOutputSamples);
// Advance the start point for the next packet of audio to arrive
_spatialAudioStart += numNetworkOutputSamples / _desiredOutputFormat.channelCount();
receivedSamples = _spatialProcessingBuffer;
} else {
// copy the samples we'll resample from the ring buffer - this also
// pushes the read pointer of the ring buffer forwards
//receivedAudioStreamPopOutput.readSamples(receivedSamples, numNetworkOutputSamples);
memcpy(receivedSamples, inputBuffer.data(), numNetworkOutputSamples * sizeof(int16_t));
receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
}
// copy the packet from the RB to the output
@ -777,14 +775,10 @@ void Audio::receivedAudioStreamProcessSamples(const QByteArray& inputBuffer, QBy
numDeviceOutputSamples,
_desiredOutputFormat, _outputFormat);
/*if (_outputDevice) {
_outputDevice->write(outputBuffer);
}*/
printf("\t outputBuffer now size %d\n", outputBuffer.size());
if (_scopeEnabled && !_scopeEnabledPause) {
unsigned int numAudioChannels = _desiredOutputFormat.channelCount();
int16_t* samples = receivedSamples;
const int16_t* samples = receivedSamples;
for (int numSamples = numNetworkOutputSamples / numAudioChannels; numSamples > 0; numSamples -= NETWORK_SAMPLES_PER_FRAME) {
unsigned int audioChannel = 0;
@ -804,15 +798,13 @@ void Audio::receivedAudioStreamProcessSamples(const QByteArray& inputBuffer, QBy
samples += NETWORK_SAMPLES_PER_FRAME * numAudioChannels;
}
}
delete[] receivedSamples;
}
void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) {
if (_audioOutput) {
// Audio output must exist and be correctly set up if we're going to process received audio
processReceivedAudio(audioByteArray);
_receivedAudioStream.parseData(audioByteArray);
}
Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size());
@ -984,119 +976,6 @@ void Audio::toggleStereoInput() {
}
}
void Audio::processReceivedAudio(const QByteArray& audioByteArray) {
// parse audio data
_receivedAudioStream.parseData(audioByteArray);
// This call has been moved to handleAudioInput. handleAudioInput is called at a much more regular interval
// than processReceivedAudio since handleAudioInput does not experience network-related jitter.
// This way, we reduce the jitter of the frames being pushed to the audio output, allowing us to use a reduced
// buffer size for it, which reduces latency.
//pushAudioToOutput();
}
void Audio::pushAudioToOutput() {
/*if (_audioOutput->bytesFree() == _audioOutput->bufferSize()) {
// the audio output has no samples to play. set the downstream audio to starved so that it
// refills to its desired size before pushing frames
_receivedAudioStream.setToStarved();
}
float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float)_outputFormat.sampleRate())
* (_desiredOutputFormat.channelCount() / (float)_outputFormat.channelCount());
int numFramesToPush;
if (Menu::getInstance()->isOptionChecked(MenuOption::DisableQAudioOutputOverflowCheck)) {
numFramesToPush = _receivedAudioStream.getFramesAvailable();
} else {
// make sure to push a whole number of frames to the audio output
int numFramesAudioOutputRoomFor = (int)(_audioOutput->bytesFree() / sizeof(int16_t) * networkOutputToOutputRatio) / _receivedAudioStream.getNumFrameSamples();
numFramesToPush = std::min(_receivedAudioStream.getFramesAvailable(), numFramesAudioOutputRoomFor);
}
// if there is data in the received stream and room in the audio output, decide what to do
if (numFramesToPush > 0 && _receivedAudioStream.popFrames(numFramesToPush, false)) {
int numNetworkOutputSamples = numFramesToPush * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO;
int numDeviceOutputSamples = numNetworkOutputSamples / networkOutputToOutputRatio;
QByteArray outputBuffer;
outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
AudioRingBuffer::ConstIterator receivedAudioStreamPopOutput = _receivedAudioStream.getLastPopOutput();
int16_t* receivedSamples = new int16_t[numNetworkOutputSamples];
if (_processSpatialAudio) {
unsigned int sampleTime = _spatialAudioStart;
QByteArray buffer;
buffer.resize(numNetworkOutputSamples * sizeof(int16_t));
receivedAudioStreamPopOutput.readSamples((int16_t*)buffer.data(), numNetworkOutputSamples);
// Accumulate direct transmission of audio from sender to receiver
if (Menu::getInstance()->isOptionChecked(MenuOption::AudioSpatialProcessingIncludeOriginal)) {
emit preProcessOriginalInboundAudio(sampleTime, buffer, _desiredOutputFormat);
addSpatialAudioToBuffer(sampleTime, buffer, numNetworkOutputSamples);
}
// Send audio off for spatial processing
emit processInboundAudio(sampleTime, buffer, _desiredOutputFormat);
// copy the samples we'll resample from the spatial audio ring buffer - this also
// pushes the read pointer of the spatial audio ring buffer forwards
_spatialAudioRingBuffer.readSamples(receivedSamples, numNetworkOutputSamples);
// Advance the start point for the next packet of audio to arrive
_spatialAudioStart += numNetworkOutputSamples / _desiredOutputFormat.channelCount();
} else {
// copy the samples we'll resample from the ring buffer - this also
// pushes the read pointer of the ring buffer forwards
receivedAudioStreamPopOutput.readSamples(receivedSamples, numNetworkOutputSamples);
}
// copy the packet from the RB to the output
linearResampling(receivedSamples,
(int16_t*)outputBuffer.data(),
numNetworkOutputSamples,
numDeviceOutputSamples,
_desiredOutputFormat, _outputFormat);
if (_outputDevice) {
_outputDevice->write(outputBuffer);
}
if (_scopeEnabled && !_scopeEnabledPause) {
unsigned int numAudioChannels = _desiredOutputFormat.channelCount();
int16_t* samples = receivedSamples;
for (int numSamples = numNetworkOutputSamples / numAudioChannels; numSamples > 0; numSamples -= NETWORK_SAMPLES_PER_FRAME) {
unsigned int audioChannel = 0;
addBufferToScope(
_scopeOutputLeft,
_scopeOutputOffset,
samples, audioChannel, numAudioChannels);
audioChannel = 1;
addBufferToScope(
_scopeOutputRight,
_scopeOutputOffset,
samples, audioChannel, numAudioChannels);
_scopeOutputOffset += NETWORK_SAMPLES_PER_FRAME;
_scopeOutputOffset %= _samplesPerScope;
samples += NETWORK_SAMPLES_PER_FRAME * numAudioChannels;
}
}
delete[] receivedSamples;
}*/
}
void Audio::processProceduralAudio(int16_t* monoInput, int numSamples) {
// zero out the locally injected audio in preparation for audio procedural sounds
@ -1715,8 +1594,8 @@ void Audio::renderLineStrip(const float* color, int x, int y, int n, int offset,
void Audio::outputFormatChanged() {
int deviceOutputFrameSize = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * _outputFormat.channelCount() * _outputFormat.sampleRate() / _desiredOutputFormat.sampleRate();
_receivedAudioStream.resizeFrame(deviceOutputFrameSize);
_outputFrameSize = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * _outputFormat.channelCount() * _outputFormat.sampleRate() / _desiredOutputFormat.sampleRate();
_receivedAudioStream.resizeFrame(_outputFrameSize);
}
bool Audio::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo) {
@ -1798,12 +1677,8 @@ bool Audio::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo)
// setup our general output device for audio-mixer audio
_audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
_audioOutput->setBufferSize(AUDIO_OUTPUT_BUFFER_SIZE_FRAMES * _outputFormat.bytesForDuration(BUFFER_SEND_INTERVAL_USECS));
printf("\n\n");
qDebug() << "Ring Buffer capacity in frames: " << (float)_outputFormat.durationForBytes(_audioOutput->bufferSize()) / (float)BUFFER_SEND_INTERVAL_USECS;
printf("\n\n");
//_outputDevice = _audioOutput->start();
_audioOutputIODevice.start();
_audioOutput->start(&_audioOutputIODevice);
@ -1877,53 +1752,25 @@ float Audio::getInputRingBufferMsecsAvailable() const {
}
qint64 Audio::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
printf("readData() request %d bytes\n", maxSize);
/*
float framesRequested = (float)_parent._outputFormat.durationForBytes(maxSize) / (float)BUFFER_SEND_INTERVAL_USECS;
if (framesRequested > 67.0f) {
maxSize /= 2;
MixedProcessedAudioStream& receivedAUdioStream = _parent._receivedAudioStream;
const QAudioOutput* audioOutput = _parent._audioOutput;
if (audioOutput->bytesFree() == audioOutput->bufferSize()) {
receivedAUdioStream.setToStarved();
}
quint64 now = usecTimestampNow();
printf("%llu\n", now - _lastReadTime);
_lastReadTime = now;
int16_t* buffer;
buffer = (int16_t*)data;
for (int i = 0; i < maxSize / 2; i++) {
*(buffer++) = (int16_t)randIntInRange(0, 10000);
}
return 2 * (maxSize / 2);
*/
int samplesRequested = maxSize / sizeof(int16_t);
printf("requesting %d samples\n", samplesRequested);
int samplesPopped;
int bytesWritten;
if ((samplesPopped = _parent._receivedAudioStream.popSamples(samplesRequested, false, false)) > 0) {
printf("\t pop succeeded: %d samples\n", samplesPopped);
AudioRingBuffer::ConstIterator lastPopOutput = _parent._receivedAudioStream.getLastPopOutput();
if ((samplesPopped = receivedAUdioStream.popSamples(samplesRequested, false, false)) > 0) {
AudioRingBuffer::ConstIterator lastPopOutput = receivedAUdioStream.getLastPopOutput();
lastPopOutput.readSamples((int16_t*)data, samplesPopped);
bytesWritten = samplesPopped * sizeof(int16_t);
} else {
printf("\t pop failed\n");
memset(data, 0, maxSize);
bytesWritten = maxSize;
}
printf("\t wrote %d bytes\n", bytesWritten);
return bytesWritten;
}

17
interface/src/Audio.h
View file

@ -33,7 +33,7 @@
#include <AbstractAudioInterface.h>
#include <StdDev.h>
#include "RawMixedAudioStream.h"
#include "MixedProcessedAudioStream.h"
static const int NUM_AUDIO_CHANNELS = 2;
@ -58,7 +58,6 @@ public:
private:
Audio& _parent;
quint64 _lastReadTime;
};
@ -111,7 +110,7 @@ public slots:
void addReceivedAudioToStream(const QByteArray& audioByteArray);
void parseAudioStreamStatsPacket(const QByteArray& packet);
void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples);
void receivedAudioStreamProcessSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
void processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
void handleAudioInput();
void reset();
void resetStats();
@ -167,14 +166,15 @@ private:
QAudioOutput* _audioOutput;
QAudioFormat _desiredOutputFormat;
QAudioFormat _outputFormat;
//QIODevice* _outputDevice;
int _outputFrameSize;
int16_t _spatialProcessingBuffer[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO];
int _numOutputCallbackBytes;
QAudioOutput* _loopbackAudioOutput;
QIODevice* _loopbackOutputDevice;
QAudioOutput* _proceduralAudioOutput;
QIODevice* _proceduralOutputDevice;
AudioRingBuffer _inputRingBuffer;
RawMixedAudioStream _receivedAudioStream;
MixedProcessedAudioStream _receivedAudioStream;
bool _isStereoInput;
QString _inputAudioDeviceName;
@ -232,12 +232,6 @@ private:
// Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
void addProceduralSounds(int16_t* monoInput, int numSamples);
// Process received audio
void processReceivedAudio(const QByteArray& audioByteArray);
// Pushes frames from the output ringbuffer to the audio output device
void pushAudioToOutput();
bool switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo);
bool switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo);
@ -305,6 +299,7 @@ private:
MovingMinMaxAvg<quint64> _packetSentTimeGaps;
AudioOutputIODevice _audioOutputIODevice;
};

10
libraries/audio/src/MixedAudioStream.cpp
View file

@ -1,3 +1,13 @@
//
// MixedAudioStream.cpp
// libraries/audio/src
//
// Created by Yixin Wang on 8/4/14.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "MixedAudioStream.h"

2
libraries/audio/src/MixedAudioStream.h
View file

@ -2,7 +2,7 @@
// MixedAudioStream.h
// libraries/audio/src
//
// Created by Stephen Birarda on 6/5/13.
// Created by Yixin Wang on 8/4/14.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.

27
libraries/audio/src/MixedProcessedAudioStream.cpp Normal file
View file

@ -0,0 +1,27 @@
//
// MixedProcessedAudioStream.cpp
// libraries/audio/src
//
// Created by Yixin Wang on 8/4/14.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "MixedProcessedAudioStream.h"
MixedProcessedAudioStream ::MixedProcessedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc)
: MixedAudioStream(numFrameSamples, numFramesCapacity, dynamicJitterBuffers, staticDesiredJitterBufferFrames, maxFramesOverDesired, useStDevForJitterCalc)
{
}
int MixedProcessedAudioStream ::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
QByteArray outputBuffer;
emit processSamples(packetAfterStreamProperties, outputBuffer);
_ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
return packetAfterStreamProperties.size();
}

30
libraries/audio/src/MixedProcessedAudioStream.h Normal file
View file

@ -0,0 +1,30 @@
//
// MixedProcessedAudioStream.h
// libraries/audio/src
//
// Created by Yixin Wang on 8/4/14.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_MixedProcessedAudioStream_h
#define hifi_MixedProcessedAudioStream_h
#include "MixedAudioStream.h"
class MixedProcessedAudioStream : public MixedAudioStream {
Q_OBJECT
public:
MixedProcessedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc);
signals:
void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
protected:
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
};
#endif // hifi_MixedProcessedAudioStream_h

24
libraries/audio/src/RawMixedAudioStream.cpp
View file

@ -1,24 +0,0 @@
#include "RawMixedAudioStream.h"
RawMixedAudioStream ::RawMixedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc)
: InboundAudioStream(numFrameSamples, numFramesCapacity, dynamicJitterBuffers, staticDesiredJitterBufferFrames, maxFramesOverDesired, useStDevForJitterCalc)
{
}
int RawMixedAudioStream ::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) {
// mixed audio packets do not have any info between the seq num and the audio data.
numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t);
return 0;
}
int RawMixedAudioStream ::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
QByteArray outputBuffer;
emit processSamples(packetAfterStreamProperties, outputBuffer);
int bytesWritten = _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size());
printf("wrote %d samples to ringbuffer\n", bytesWritten / 2);
return packetAfterStreamProperties.size();
}

32
libraries/audio/src/RawMixedAudioStream.h
View file

@ -1,32 +0,0 @@
//
// RawMixedAudioStream.h
// libraries/audio/src
//
// Created by Stephen Birarda on 6/5/13.
// Copyright 2013 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_RawMixedAudioStream_h
#define hifi_RawMixedAudioStream_h
#include "InboundAudioStream.h"
#include "PacketHeaders.h"
class RawMixedAudioStream : public InboundAudioStream {
Q_OBJECT
public:
RawMixedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc);
signals:
void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
protected:
int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples);
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
};
#endif // hifi_RawMixedAudioStream_h