Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-26 11:56:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

audiomixer code complete; need to test

Browse source
This commit is contained in:
wangyix 2014-07-24 10:48:27 -07:00
parent e2f957d6dc
commit c9b6879ca8
13 changed files with 394 additions and 485 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
assignment-client/src/audio/AudioMixer.cpp
View file

@ -95,6 +95,19 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f;
void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd, void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd,
AvatarAudioRingBuffer* listeningNodeBuffer) { AvatarAudioRingBuffer* listeningNodeBuffer) {
// if the frame to be mixed is silent, don't mix it
if (bufferToAdd->getNextOutputTrailingLoudness() == 0.0f) {
bufferToAdd->popFrames(1);
return;
}
// get pointer to frame to be mixed. If the stream cannot provide a frame (is starved), bail
AudioRingBuffer::ConstIterator nextOutputStart;
if (!bufferToAdd->popFrames(&nextOutputStart, 1)) {
return;
}
float bearingRelativeAngleToSource = 0.0f; float bearingRelativeAngleToSource = 0.0f;
float attenuationCoefficient = 1.0f; float attenuationCoefficient = 1.0f;
int numSamplesDelay = 0; int numSamplesDelay = 0;
@ -203,7 +216,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
} }
} }
const int16_t* nextOutputStart = bufferToAdd->getNextOutput();
if (!bufferToAdd->isStereo() && shouldAttenuate) { if (!bufferToAdd->isStereo() && shouldAttenuate) {
// this is a mono buffer, which means it gets full attenuation and spatialization // this is a mono buffer, which means it gets full attenuation and spatialization
@ -212,8 +225,8 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0; int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0;
int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0; int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0;
const int16_t* bufferStart = bufferToAdd->getBuffer(); //const int16_t* bufferStart = bufferToAdd->getBuffer();
int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity(); //int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity();
int16_t correctBufferSample[2], delayBufferSample[2]; int16_t correctBufferSample[2], delayBufferSample[2];
int delayedChannelIndex = 0; int delayedChannelIndex = 0;
@ -241,14 +254,15 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
// if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput // if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput
// to stick at the beginning // to stick at the beginning
float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio; float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio;
const int16_t* delayNextOutputStart = nextOutputStart - numSamplesDelay; AudioRingBuffer::ConstIterator delayNextOutputStart = nextOutputStart - numSamplesDelay;
if (delayNextOutputStart < bufferStart) { //if (delayNextOutputStart < bufferStart) {
delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay; //delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay;
} //}
for (int i = 0; i < numSamplesDelay; i++) { for (int i = 0; i < numSamplesDelay; i++) {
int parentIndex = i * 2; int parentIndex = i * 2;
_clientSamples[parentIndex + delayedChannelOffset] += delayNextOutputStart[i] * attenuationAndWeakChannelRatio; _clientSamples[parentIndex + delayedChannelOffset] += *delayNextOutputStart * attenuationAndWeakChannelRatio;
++delayNextOutputStart;
} }
} }
} else { } else {
@ -293,13 +307,13 @@ void AudioMixer::prepareMixForListeningNode(Node* node) {
AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData();
// enumerate the ARBs attached to the otherNode and add all that should be added to mix // enumerate the ARBs attached to the otherNode and add all that should be added to mix
for (int i = 0; i < otherNodeClientData->getRingBuffers().size(); i++) {
PositionalAudioRingBuffer* otherNodeBuffer = otherNodeClientData->getRingBuffers()[i];
if ((*otherNode != *node const QHash<QUuid, PositionalAudioRingBuffer*>& otherNodeRingBuffers = otherNodeClientData->getRingBuffers();
|| otherNodeBuffer->shouldLoopbackForNode()) QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = otherNodeRingBuffers.constEnd();
&& otherNodeBuffer->willBeAddedToMix() for (i = otherNodeRingBuffers.begin(); i != end; i++) {
&& otherNodeBuffer->getNextOutputTrailingLoudness() > 0.0f) { PositionalAudioRingBuffer* otherNodeBuffer = i.value();
if (*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) {
addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer);
} }
} }
@ -307,7 +321,6 @@ void AudioMixer::prepareMixForListeningNode(Node* node) {
} }
} }
void AudioMixer::readPendingDatagrams() { void AudioMixer::readPendingDatagrams() {
QByteArray receivedPacket; QByteArray receivedPacket;
HifiSockAddr senderSockAddr; HifiSockAddr senderSockAddr;
@ -500,12 +513,12 @@ void AudioMixer::run() {
while (!_isFinished) { while (!_isFinished) {
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { /*foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData()) { if (node->getLinkedData()) {
((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone, ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone,
_listenerUnattenuatedZone); _listenerUnattenuatedZone);
} }
} }*/
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
@ -599,13 +612,13 @@ void AudioMixer::run() {
++_sumListeners; ++_sumListeners;
} }
} }
/*
// push forward the next output pointers for any audio buffers we used // push forward the next output pointers for any audio buffers we used
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getLinkedData()) { if (node->getLinkedData()) {
((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend(); ((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend();
} }
} }*/
++_numStatFrames; ++_numStatFrames;

187
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -18,34 +18,28 @@
#include "AudioMixer.h" #include "AudioMixer.h"
#include "AudioMixerClientData.h" #include "AudioMixerClientData.h"
#include "MovingMinMaxAvg.h"
const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS /
(TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS / USECS_PER_SECOND);
AudioMixerClientData::AudioMixerClientData() : AudioMixerClientData::AudioMixerClientData() :
_ringBuffers(), _ringBuffers(),
_outgoingMixedAudioSequenceNumber(0), _outgoingMixedAudioSequenceNumber(0)
_incomingAvatarAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH)
{ {
} }
AudioMixerClientData::~AudioMixerClientData() { AudioMixerClientData::~AudioMixerClientData() {
for (int i = 0; i < _ringBuffers.size(); i++) { QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = _ringBuffers.constEnd();
// delete this attached PositionalAudioRingBuffer for (i = _ringBuffers.begin(); i != end; i++) {
delete _ringBuffers[i]; // delete this attached InboundAudioStream
delete i.value();
} }
} }
AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const { AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const {
for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers.contains(QUuid())) {
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) { return (AvatarAudioRingBuffer*)_ringBuffers.value(QUuid());
return (AvatarAudioRingBuffer*) _ringBuffers[i];
}
} }
// no AvatarAudioRingBuffer found - return NULL // no mic stream found - return NULL
return NULL; return NULL;
} }
@ -57,96 +51,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
quint16 sequence = *(reinterpret_cast<const quint16*>(sequenceAt)); quint16 sequence = *(reinterpret_cast<const quint16*>(sequenceAt));
PacketType packetType = packetTypeForPacket(packet); PacketType packetType = packetTypeForPacket(packet);
if (packetType == PacketTypeMicrophoneAudioWithEcho if (packetType == PacketTypeAudioStreamStats) {
|| packetType == PacketTypeMicrophoneAudioNoEcho
|| packetType == PacketTypeSilentAudioFrame) {
SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
// grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
// read the first byte after the header to see if this is a stereo or mono buffer
quint8 channelFlag = packet.at(numBytesForPacketHeader(packet) + sizeof(quint16));
bool isStereo = channelFlag == 1;
if (avatarRingBuffer && avatarRingBuffer->isStereo() != isStereo) {
// there's a mismatch in the buffer channels for the incoming and current buffer
// so delete our current buffer and create a new one
_ringBuffers.removeOne(avatarRingBuffer);
avatarRingBuffer->deleteLater();
avatarRingBuffer = NULL;
}
if (!avatarRingBuffer) {
// we don't have an AvatarAudioRingBuffer yet, so add it
avatarRingBuffer = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers());
_ringBuffers.push_back(avatarRingBuffer);
}
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
// in the ring buffer (if that frame hasn't been mixed yet)
switch (packetArrivalInfo._status) {
case SequenceNumberStats::Early: {
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost);
break;
}
case SequenceNumberStats::OnTime: {
// ask the AvatarAudioRingBuffer instance to parse the data
avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, 0);
break;
}
default: {
break;
}
}
} else if (packetType == PacketTypeInjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID));
if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) {
_incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH));
}
SequenceNumberStats::ArrivalInfo packetArrivalInfo =
_incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
for (int i = 0; i < _ringBuffers.size(); i++) {
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector
&& ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) {
matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i];
}
}
if (!matchingInjectedRingBuffer) {
// we don't have a matching injected audio ring buffer, so add it
matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers());
_ringBuffers.push_back(matchingInjectedRingBuffer);
}
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
// in the ring buffer (if that frame hasn't been mixed yet)
switch (packetArrivalInfo._status) {
case SequenceNumberStats::Early: {
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost);
break;
}
case SequenceNumberStats::OnTime: {
// ask the AvatarAudioRingBuffer instance to parse the data
matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, 0);
break;
}
default: {
break;
}
}
} else if (packetType == PacketTypeAudioStreamStats) {
const char* dataAt = packet.data(); const char* dataAt = packet.data();
@ -155,12 +60,52 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
// read the downstream audio stream stats // read the downstream audio stream stats
memcpy(&_downstreamAudioStreamStats, dataAt, sizeof(AudioStreamStats)); memcpy(&_downstreamAudioStreamStats, dataAt, sizeof(AudioStreamStats));
dataAt += sizeof(AudioStreamStats);
return dataAt - packet.data();
} else {
PositionalAudioRingBuffer* matchingStream = NULL;
if (packetType == PacketTypeMicrophoneAudioWithEcho
|| packetType == PacketTypeMicrophoneAudioNoEcho
|| packetType == PacketTypeSilentAudioFrame) {
QUuid nullUUID = QUuid();
if (!_ringBuffers.contains(nullUUID)) {
// we don't have a mic stream yet, so add it
// read the channel flag to see if our stream is stereo or not
const char* channelFlagAt = packet.constData() + numBytesForPacketHeader(packet) + sizeof(quint16);
quint8 channelFlag = *(reinterpret_cast<const quint8*>(channelFlagAt));
bool isStereo = channelFlag == 1;
_ringBuffers.insert(nullUUID,
matchingStream = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers()));
} else {
matchingStream = _ringBuffers.value(nullUUID);
}
} else if (packetType == PacketTypeInjectAudio) {
// this is injected audio
// grab the stream identifier for this injected audio
int bytesBeforeStreamIdentifier = numBytesForPacketHeader(packet) + sizeof(quint16);
QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(bytesBeforeStreamIdentifier, NUM_BYTES_RFC4122_UUID));
if (!_ringBuffers.contains(streamIdentifier)) {
_ringBuffers.insert(streamIdentifier,
matchingStream = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()));
} else {
matchingStream = _ringBuffers.value(streamIdentifier);
}
} }
return matchingStream->parseData(packet);
}
return 0; return 0;
} }
void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) { /*void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) {
for (int i = 0; i < _ringBuffers.size(); i++) { for (int i = 0; i < _ringBuffers.size(); i++) {
if (_ringBuffers[i]->shouldBeAddedToMix()) { if (_ringBuffers[i]->shouldBeAddedToMix()) {
// this is a ring buffer that is ready to go // this is a ring buffer that is ready to go
@ -205,9 +150,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() {
} }
i++; i++;
} }
} }*/
AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { /*AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const {
AudioStreamStats streamStats; AudioStreamStats streamStats;
@ -239,20 +184,9 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio
streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped(); streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped();
return streamStats; return streamStats;
} }*/
void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) { void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) {
// have all the seq number stats of each audio stream push their current stats into their history,
// which moves that history window 1 second forward (since that's how long since the last stats were pushed into history)
_incomingAvatarAudioSequenceNumberStats.pushStatsToHistory();
QHash<QUuid, SequenceNumberStats>::Iterator i = _incomingInjectedAudioSequenceNumberStatsMap.begin();
QHash<QUuid, SequenceNumberStats>::Iterator end = _incomingInjectedAudioSequenceNumberStatsMap.end();
while (i != end) {
i.value().pushStatsToHistory();
i++;
}
char packet[MAX_PACKET_SIZE]; char packet[MAX_PACKET_SIZE];
NodeList* nodeList = NodeList::getInstance(); NodeList* nodeList = NodeList::getInstance();
@ -271,7 +205,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
// pack and send stream stats packets until all ring buffers' stats are sent // pack and send stream stats packets until all ring buffers' stats are sent
int numStreamStatsRemaining = _ringBuffers.size(); int numStreamStatsRemaining = _ringBuffers.size();
QList<PositionalAudioRingBuffer*>::ConstIterator ringBuffersIterator = _ringBuffers.constBegin(); QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator ringBuffersIterator = _ringBuffers.constBegin();
while (numStreamStatsRemaining > 0) { while (numStreamStatsRemaining > 0) {
char* dataAt = headerEndAt; char* dataAt = headerEndAt;
@ -288,7 +222,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
// pack the calculated number of stream stats // pack the calculated number of stream stats
for (int i = 0; i < numStreamStatsToPack; i++) { for (int i = 0; i < numStreamStatsToPack; i++) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(*ringBuffersIterator); AudioStreamStats streamStats = ringBuffersIterator.value()->getAudioStreamStats();
memcpy(dataAt, &streamStats, sizeof(AudioStreamStats)); memcpy(dataAt, &streamStats, sizeof(AudioStreamStats));
dataAt += sizeof(AudioStreamStats); dataAt += sizeof(AudioStreamStats);
@ -322,7 +256,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
if (avatarRingBuffer) { if (avatarRingBuffer) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); AudioStreamStats streamStats = avatarRingBuffer->getAudioStreamStats();
result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
+ " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames()) + " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames())
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
@ -343,11 +277,12 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
result = "mic unknown"; result = "mic unknown";
} }
for (int i = 0; i < _ringBuffers.size(); i++) { QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = _ringBuffers.end();
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { for (i = _ringBuffers.begin(); i != end; i++) {
AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); if (i.value()->getType() == PositionalAudioRingBuffer::Injector) {
AudioStreamStats streamStats = i.value()->getAudioStreamStats();
result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames) result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
+ " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames()) + " desired_calc:" + QString::number(i.value()->getCalculatedDesiredJitterBufferFrames())
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage) + " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
+ " available:" + QString::number(streamStats._ringBufferFramesAvailable) + " available:" + QString::number(streamStats._ringBufferFramesAvailable)
+ " starves:" + QString::number(streamStats._ringBufferStarveCount) + " starves:" + QString::number(streamStats._ringBufferStarveCount)

20
assignment-client/src/audio/AudioMixerClientData.h
View file

@ -13,29 +13,23 @@
#define hifi_AudioMixerClientData_h #define hifi_AudioMixerClientData_h
#include <AABox.h> #include <AABox.h>
#include <NodeData.h>
#include <PositionalAudioRingBuffer.h>
#include "PositionalAudioRingBuffer.h"
#include "AvatarAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h"
#include "AudioStreamStats.h"
#include "SequenceNumberStats.h"
const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30;
class AudioMixerClientData : public NodeData { class AudioMixerClientData : public NodeData {
public: public:
AudioMixerClientData(); AudioMixerClientData();
~AudioMixerClientData(); ~AudioMixerClientData();
const QList<PositionalAudioRingBuffer*> getRingBuffers() const { return _ringBuffers; } const QHash<QUuid, PositionalAudioRingBuffer*>& getRingBuffers() const { return _ringBuffers; }
AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const;
int parseData(const QByteArray& packet); int parseData(const QByteArray& packet);
void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); //void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL);
void pushBuffersAfterFrameSend(); //void pushBuffersAfterFrameSend();
AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const; //AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const;
QString getAudioStreamStatsString() const; QString getAudioStreamStatsString() const;
void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode); void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode);
@ -44,11 +38,9 @@ public:
quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; }
private: private:
QList<PositionalAudioRingBuffer*> _ringBuffers; QHash<QUuid, PositionalAudioRingBuffer*> _ringBuffers; // mic stream stored under key of null UUID
quint16 _outgoingMixedAudioSequenceNumber; quint16 _outgoingMixedAudioSequenceNumber;
SequenceNumberStats _incomingAvatarAudioSequenceNumberStats;
QHash<QUuid, SequenceNumberStats> _incomingInjectedAudioSequenceNumberStatsMap;
AudioStreamStats _downstreamAudioStreamStats; AudioStreamStats _downstreamAudioStreamStats;
}; };

75
assignment-client/src/audio/AvatarAudioRingBuffer.cpp
View file

@ -14,57 +14,50 @@
#include "AvatarAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h"
AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBuffer) : AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBuffer) :
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) { PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) {
} }
int AvatarAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { int AvatarAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) {
frameReceivedUpdateTimingStats();
_shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); _shouldLoopbackForNode = (type == PacketTypeMicrophoneAudioWithEcho);
// skip the packet header (includes the source UUID) int readBytes = 0;
int readBytes = numBytesForPacketHeader(packet);
// skip the sequence number // read the channel flag
readBytes += sizeof(quint16); quint8 channelFlag = packetAfterSeqNum.at(readBytes);
bool isStereo = channelFlag == 1;
// hop over the channel flag that has already been read in AudioMixerClientData
readBytes += sizeof(quint8); readBytes += sizeof(quint8);
// read the positional data
readBytes += parsePositionalData(packet.mid(readBytes));
if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) { // if isStereo value has changed, restart the ring buffer with new frame size
// this source had no audio to send us, but this counts as a packet if (isStereo != _isStereo) {
// write silence equivalent to the number of silent samples they just sent us _ringBuffer.resizeForFrameSize(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
int16_t numSilentSamples; _isStereo = isStereo;
memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
readBytes += sizeof(int16_t);
// add silent samples for the dropped packets as well.
// ASSUME that each dropped packet had same number of silent samples as this one
numSilentSamples *= (packetsSkipped + 1);
// NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
// CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval
// (every 16.667ms) than Audio.cpp (every 10.667ms)
//numSilentSamples = getSamplesPerFrame();
addDroppableSilentSamples(numSilentSamples);
} else {
int numAudioBytes = packet.size() - readBytes;
int numAudioSamples = numAudioBytes / sizeof(int16_t);
// add silent samples for the dropped packets.
// ASSUME that each dropped packet had same number of samples as this one
if (packetsSkipped > 0) {
addDroppableSilentSamples(packetsSkipped * numAudioSamples);
} }
// there is audio data to read // read the positional data
readBytes += writeData(packet.data() + readBytes, numAudioBytes); readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes));
if (type == PacketTypeSilentAudioFrame) {
int16_t numSilentSamples;
memcpy(&numSilentSamples, packetAfterSeqNum.data() + readBytes, sizeof(int16_t));
readBytes += sizeof(int16_t);
numAudioSamples = numSilentSamples;
} else {
int numAudioBytes = packetAfterSeqNum.size() - readBytes;
numAudioSamples = numAudioBytes / sizeof(int16_t);
}
return readBytes;
}
int AvatarAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
int readBytes = 0;
if (type == PacketTypeSilentAudioFrame) {
writeDroppableSilentSamples(numAudioSamples);
} else {
// there is audio data to read
readBytes += _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
} }
return readBytes; return readBytes;
} }

4
assignment-client/src/audio/AvatarAudioRingBuffer.h
View file

@ -20,11 +20,13 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer {
public: public:
AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false);
int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped);
private: private:
// disallow copying of AvatarAudioRingBuffer objects // disallow copying of AvatarAudioRingBuffer objects
AvatarAudioRingBuffer(const AvatarAudioRingBuffer&); AvatarAudioRingBuffer(const AvatarAudioRingBuffer&);
AvatarAudioRingBuffer& operator= (const AvatarAudioRingBuffer&); AvatarAudioRingBuffer& operator= (const AvatarAudioRingBuffer&);
int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples);
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
}; };
#endif // hifi_AvatarAudioRingBuffer_h #endif // hifi_AvatarAudioRingBuffer_h

15
libraries/audio/src/AudioRingBuffer.cpp
View file

@ -231,3 +231,18 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int
return position + numSamplesShift; return position + numSamplesShift;
} }
} }
float AudioRingBuffer::getNextOutputFrameLoudness() const {
float loudness = 0.0f;
int16_t* sampleAt = _nextOutput;
int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1;
if (samplesAvailable() >= _numFrameSamples) {
for (int i = 0; i < _numFrameSamples; ++i) {
loudness += fabsf(*sampleAt);
sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1;
}
loudness /= _numFrameSamples;
loudness /= MAX_SAMPLE_VALUE;
}
return loudness;
}

82
libraries/audio/src/AudioRingBuffer.h
View file

@ -66,6 +66,8 @@ public:
void shiftReadPosition(unsigned int numSamples); void shiftReadPosition(unsigned int numSamples);
float getNextOutputFrameLoudness() const;
int samplesAvailable() const; int samplesAvailable() const;
int framesAvailable() const { return samplesAvailable() / _numFrameSamples; } int framesAvailable() const { return samplesAvailable() / _numFrameSamples; }
@ -99,6 +101,86 @@ protected:
bool _isStarved; bool _isStarved;
bool _hasStarted; bool _hasStarted;
bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing
public:
class ConstIterator { //public std::iterator < std::forward_iterator_tag, int16_t > {
public:
ConstIterator()
: _capacity(0),
_bufferFirst(NULL),
_bufferLast(NULL),
_at(NULL) {}
ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at)
: _capacity(capacity),
_bufferFirst(bufferFirst),
_bufferLast(bufferFirst + capacity - 1),
_at(at) {}
bool operator==(const ConstIterator& rhs) { return _at == rhs._at; }
bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; }
int16_t operator*() { return *_at; }
ConstIterator& operator=(const ConstIterator& rhs) {
_capacity = rhs._capacity;
_bufferFirst = rhs._bufferFirst;
_bufferLast = rhs._bufferLast;
_at = rhs._at;
return *this;
}
ConstIterator& operator++() {
_at = (_at == _bufferLast) ? _bufferFirst : _at + 1;
return *this;
}
ConstIterator operator++(int) {
ConstIterator tmp(*this);
++(*this);
return tmp;
}
ConstIterator& operator--() {
_at = (_at == _bufferFirst) ? _bufferLast : _at - 1;
return *this;
}
ConstIterator operator--(int) {
ConstIterator tmp(*this);
--(*this);
return tmp;
}
int16_t operator[] (int i) {
return *atShiftedBy(i);
}
ConstIterator operator+(int i) {
return ConstIterator(_bufferFirst, _capacity, atShiftedBy(i));
}
ConstIterator operator-(int i) {
return ConstIterator(_bufferFirst, _capacity, atShiftedBy(-i));
}
private:
int16_t* atShiftedBy(int i) {
i = (_at - _bufferFirst + i) % _capacity;
if (i < 0) {
i += _capacity;
}
return _bufferFirst + i;
}
private:
int _capacity;
int16_t* _bufferFirst;
int16_t* _bufferLast;
int16_t* _at;
};
ConstIterator nextOutput() const { return ConstIterator(_buffer, _sampleCapacity, _nextOutput); }
}; };
#endif // hifi_AudioRingBuffer_h #endif // hifi_AudioRingBuffer_h

89
libraries/audio/src/InboundAudioStream.cpp
View file

@ -13,19 +13,19 @@
#include "PacketHeaders.h" #include "PacketHeaders.h"
InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers) : InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers) :
_ringBuffer(numFrameSamples, false, numFramesCapacity), _ringBuffer(numFrameSamples, false, numFramesCapacity),
_dynamicJitterBuffers(dynamicJitterBuffers), _dynamicJitterBuffers(dynamicJitterBuffers),
_desiredJitterBufferFrames(1), _desiredJitterBufferFrames(1),
_isStarved(true), _isStarved(true),
_hasStarted(false), _hasStarted(false),
_consecutiveNotMixedCount(0), _consecutiveNotMixedCount(0),
_starveCount(0), _starveCount(0),
_silentFramesDropped(0), _silentFramesDropped(0),
_incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS), _incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS),
_lastFrameReceivedTime(0), _lastFrameReceivedTime(0),
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS) _framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS)
{ {
} }
@ -94,19 +94,19 @@ int InboundAudioStream::parseData(const QByteArray& packet) {
return readBytes; return readBytes;
} }
bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) { bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) {
if (_isStarved) { if (_isStarved) {
_consecutiveNotMixedCount++; _consecutiveNotMixedCount++;
return false; return false;
} }
bool framesPopped = false; bool popped = false;
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples(); int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
if (_ringBuffer.samplesAvailable >= numSamplesRequested) { if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
_ringBuffer.readSamples(dest, numSamplesRequested); _ringBuffer.shiftReadPosition(numSamplesRequested);
_hasStarted = true; _hasStarted = true;
framesPopped = true; popped = true;
} else { } else {
if (starveOnFail) { if (starveOnFail) {
setToStarved(); setToStarved();
@ -116,7 +116,58 @@ bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFa
_framesAvailableStats.update(_ringBuffer.framesAvailable()); _framesAvailableStats.update(_ringBuffer.framesAvailable());
return framesPopped; return popped;
}
bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) {
if (_isStarved) {
_consecutiveNotMixedCount++;
return false;
}
bool popped = false;
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
_ringBuffer.readSamples(dest, numSamplesRequested);
_hasStarted = true;
popped = true;
} else {
if (starveOnFail) {
setToStarved();
_consecutiveNotMixedCount++;
}
}
_framesAvailableStats.update(_ringBuffer.framesAvailable());
return popped;
}
bool InboundAudioStream::popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail) {
if (_isStarved) {
_consecutiveNotMixedCount++;
return false;
}
bool popped = false;
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
*nextOutput = _ringBuffer.nextOutput();
_ringBuffer.shiftReadPosition(numSamplesRequested);
_hasStarted = true;
popped = true;
} else {
if (starveOnFail) {
setToStarved();
_consecutiveNotMixedCount++;
}
}
_framesAvailableStats.update(_ringBuffer.framesAvailable());
return popped;
} }
void InboundAudioStream::setToStarved() { void InboundAudioStream::setToStarved() {

5
libraries/audio/src/InboundAudioStream.h
View file

@ -53,13 +53,14 @@ public:
int parseData(const QByteArray& packet); int parseData(const QByteArray& packet);
bool popFrames(int numFrames, bool starveOnFail = true);
bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true); bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true);
bool popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail = true);
void setToStarved(); void setToStarved();
/// this function should be called once per second to ensure the seq num stats history spans ~30 seconds
AudioStreamStats updateSeqHistoryAndGetAudioStreamStats(); AudioStreamStats updateSeqHistoryAndGetAudioStreamStats();
virtual AudioStreamStats getAudioStreamStats() const; virtual AudioStreamStats getAudioStreamStats() const;

44
libraries/audio/src/InjectedAudioRingBuffer.cpp
View file

@ -20,27 +20,21 @@
#include "InjectedAudioRingBuffer.h" #include "InjectedAudioRingBuffer.h"
InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, bool dynamicJitterBuffer) : InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, bool dynamicJitterBuffer) :
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer), PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer),
_streamIdentifier(streamIdentifier), _streamIdentifier(streamIdentifier),
_radius(0.0f), _radius(0.0f),
_attenuationRatio(0) _attenuationRatio(0)
{ {
} }
const uchar MAX_INJECTOR_VOLUME = 255; const uchar MAX_INJECTOR_VOLUME = 255;
int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) { int InjectedAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) {
frameReceivedUpdateTimingStats();
// setup a data stream to read from this packet // setup a data stream to read from this packet
QDataStream packetStream(packet); QDataStream packetStream(packetAfterSeqNum);
packetStream.skipRawData(numBytesForPacketHeader(packet));
// push past the sequence number // skip the stream identifier
packetStream.skipRawData(sizeof(quint16));
// push past the stream identifier
packetStream.skipRawData(NUM_BYTES_RFC4122_UUID); packetStream.skipRawData(NUM_BYTES_RFC4122_UUID);
// pull the loopback flag and set our boolean // pull the loopback flag and set our boolean
@ -49,23 +43,27 @@ int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray&
_shouldLoopbackForNode = (shouldLoopback == 1); _shouldLoopbackForNode = (shouldLoopback == 1);
// use parsePositionalData in parent PostionalAudioRingBuffer class to pull common positional data // use parsePositionalData in parent PostionalAudioRingBuffer class to pull common positional data
packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos()))); packetStream.skipRawData(parsePositionalData(packetAfterSeqNum.mid(packetStream.device()->pos())));
// pull out the radius for this injected source - if it's zero this is a point source // pull out the radius for this injected source - if it's zero this is a point source
packetStream >> _radius; packetStream >> _radius;
quint8 attenuationByte = 0; quint8 attenuationByte = 0;
packetStream >> attenuationByte; packetStream >> attenuationByte;
_attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME; _attenuationRatio = attenuationByte / (float)MAX_INJECTOR_VOLUME;
int numAudioBytes = packet.size() - packetStream.device()->pos(); int numAudioBytes = packetAfterSeqNum.size() - packetStream.device()->pos();
int numAudioSamples = numAudioBytes / sizeof(int16_t); numAudioSamples = numAudioBytes / sizeof(int16_t);
// add silent samples for the dropped packets.
// ASSUME that each dropped packet had same number of samples as this one
addDroppableSilentSamples(numAudioSamples * packetsSkipped);
packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes));
return packetStream.device()->pos(); return packetStream.device()->pos();
} }
int InjectedAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
}
AudioStreamStats InjectedAudioRingBuffer::getAudioStreamStats() const {
AudioStreamStats streamStats = PositionalAudioRingBuffer::getAudioStreamStats();
streamStats._streamIdentifier = _streamIdentifier;
return streamStats;
}

12
libraries/audio/src/InjectedAudioRingBuffer.h
View file

@ -20,17 +20,21 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer {
public: public:
InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false);
int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped);
const QUuid& getStreamIdentifier() const { return _streamIdentifier; }
float getRadius() const { return _radius; } float getRadius() const { return _radius; }
float getAttenuationRatio() const { return _attenuationRatio; } float getAttenuationRatio() const { return _attenuationRatio; }
QUuid getStreamIdentifier() const { return _streamIdentifier; }
private: private:
// disallow copying of InjectedAudioRingBuffer objects // disallow copying of InjectedAudioRingBuffer objects
InjectedAudioRingBuffer(const InjectedAudioRingBuffer&); InjectedAudioRingBuffer(const InjectedAudioRingBuffer&);
InjectedAudioRingBuffer& operator= (const InjectedAudioRingBuffer&); InjectedAudioRingBuffer& operator= (const InjectedAudioRingBuffer&);
QUuid _streamIdentifier; AudioStreamStats getAudioStreamStats() const;
int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples);
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
const QUuid _streamIdentifier;
float _radius; float _radius;
float _attenuationRatio; float _attenuationRatio;
}; };

186
libraries/audio/src/PositionalAudioRingBuffer.cpp
View file

@ -9,6 +9,9 @@
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
// //
#include "PositionalAudioRingBuffer.h"
#include "SharedUtil.h"
#include <cstring> #include <cstring>
#include <glm/detail/func_common.hpp> #include <glm/detail/func_common.hpp>
@ -18,60 +21,21 @@
#include <PacketHeaders.h> #include <PacketHeaders.h>
#include <UUID.h> #include <UUID.h>
#include "PositionalAudioRingBuffer.h"
#include "SharedUtil.h"
PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo, bool dynamicJitterBuffers) : PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo, bool dynamicJitterBuffers) :
InboundAudioStream(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, dynamicJitterBuffers),
false, AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY), _type(type),
_type(type), _position(0.0f, 0.0f, 0.0f),
_position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f),
_orientation(0.0f, 0.0f, 0.0f, 0.0f), _shouldLoopbackForNode(false),
_willBeAddedToMix(false), _isStereo(isStereo),
_shouldLoopbackForNode(false), _nextOutputTrailingLoudness(0.0f),
_shouldOutputStarveDebug(true), _listenerUnattenuatedZone(NULL)
_isStereo(isStereo),
_nextOutputTrailingLoudness(0.0f),
_listenerUnattenuatedZone(NULL),
_lastFrameReceivedTime(0),
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS),
_desiredJitterBufferFrames(1),
_dynamicJitterBuffers(dynamicJitterBuffers),
_consecutiveNotMixedCount(0),
_starveCount(0),
_silentFramesDropped(0)
{ {
} }
int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
QDataStream packetStream(positionalByteArray);
packetStream.readRawData(reinterpret_cast<char*>(&_position), sizeof(_position));
packetStream.readRawData(reinterpret_cast<char*>(&_orientation), sizeof(_orientation));
// if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
if (glm::isnan(_orientation.x)) {
reset();
return 0;
}
return packetStream.device()->pos();
}
void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
// ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer float nextLoudness = _ringBuffer.getNextOutputFrameLoudness();
float nextLoudness = 0;
if (samplesAvailable() >= _numFrameSamples) {
for (int i = 0; i < _numFrameSamples; ++i) {
nextLoudness += fabsf(_nextOutput[i]);
}
nextLoudness /= _numFrameSamples;
nextLoudness /= MAX_SAMPLE_VALUE;
}
const int TRAILING_AVERAGE_FRAMES = 100; const int TRAILING_AVERAGE_FRAMES = 100;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
@ -89,120 +53,24 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
} }
} }
bool PositionalAudioRingBuffer::shouldBeAddedToMix() { int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples; QDataStream packetStream(positionalByteArray);
if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) { packetStream.readRawData(reinterpret_cast<char*>(&_position), sizeof(_position));
// if the buffer was starved, allow it to accrue at least the desired number of packetStream.readRawData(reinterpret_cast<char*>(&_orientation), sizeof(_orientation));
// jitter buffer frames before we start taking frames from it for mixing
if (_shouldOutputStarveDebug) { // if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
_shouldOutputStarveDebug = false; if (glm::isnan(_orientation.x)) {
// NOTE: why would we reset the ring buffer here?
_ringBuffer.reset();
return 0;
} }
_consecutiveNotMixedCount++; return packetStream.device()->pos();
return false;
} else if (samplesAvailable() < _numFrameSamples) {
// if the buffer doesn't have a full frame of samples to take for mixing, it is starved
_isStarved = true;
_starveCount++;
_framesAvailableStats.reset();
// reset our _shouldOutputStarveDebug to true so the next is printed
_shouldOutputStarveDebug = true;
_consecutiveNotMixedCount = 1;
return false;
}
// good buffer, add this to the mix
// if we just finished refilling after a starve, we have a new jitter buffer length.
// reset the frames available stats.
_isStarved = false;
_framesAvailableStats.update(framesAvailable());
// since we've read data from ring buffer at least once - we've started
_hasStarted = true;
return true;
} }
int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { AudioStreamStats PositionalAudioRingBuffer::getAudioStreamStats() const {
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; AudioStreamStats streamStats = InboundAudioStream::getAudioStreamStats();
streamStats._streamType = _type;
int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); return streamStats;
if (calculatedDesiredJitterBufferFrames < 1) {
calculatedDesiredJitterBufferFrames = 1;
}
return calculatedDesiredJitterBufferFrames;
}
void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() {
// update the two time gap stats we're keeping
quint64 now = usecTimestampNow();
if (_lastFrameReceivedTime != 0) {
quint64 gap = now - _lastFrameReceivedTime;
_interframeTimeGapStatsForJitterCalc.update(gap);
_interframeTimeGapStatsForStatsPacket.update(gap);
}
_lastFrameReceivedTime = now;
// recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us
if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) {
if (!_dynamicJitterBuffers) {
_desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence
} else {
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
_desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
if (_desiredJitterBufferFrames < 1) {
_desiredJitterBufferFrames = 1;
}
const int maxDesired = _frameCapacity - 1;
if (_desiredJitterBufferFrames > maxDesired) {
_desiredJitterBufferFrames = maxDesired;
}
}
_interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag();
}
}
void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) {
// This adds some number of frames to the desired jitter buffer frames target we use.
// The larger this value is, the less aggressive we are about reducing the jitter buffer length.
// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long,
// which could lead immediately to a starve.
const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
// calculate how many silent frames we should drop. We only drop silent frames if
// the running avg num frames available has stabilized and it's more than
// our desired number of frames by the margin defined above.
int numSilentFramesToDrop = 0;
if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled()
&& numSilentSamples >= _numFrameSamples) {
_framesAvailableStats.clearNewStatsAvailableFlag();
int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage();
int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) {
// our avg jitter buffer size exceeds its desired value, so ignore some silent
// frames to get that size as close to desired as possible
int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding;
int numSilentFramesReceived = numSilentSamples / _numFrameSamples;
numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
// since we now have a new jitter buffer length, reset the frames available stats.
_framesAvailableStats.reset();
_silentFramesDropped += numSilentFramesToDrop;
}
}
addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples);
} }

77
libraries/audio/src/PositionalAudioRingBuffer.h
View file

@ -13,30 +13,14 @@
#define hifi_PositionalAudioRingBuffer_h #define hifi_PositionalAudioRingBuffer_h
#include <glm/gtx/quaternion.hpp> #include <glm/gtx/quaternion.hpp>
#include <AABox.h> #include <AABox.h>
#include "AudioRingBuffer.h" #include "InboundAudioStream.h"
#include "MovingMinMaxAvg.h"
// the time gaps stats for _desiredJitterBufferFrames calculation
// will recalculate the max for the past 5000 samples every 500 samples
const int TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES = 500;
const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10;
// the time gap stats for constructing AudioStreamStats will
// recalculate min/max/avg every ~1 second for the past ~30 seconds of time gap data
const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30;
// the stats for calculating the average frames available will recalculate every ~1 second
// and will include data for the past ~10 seconds
const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 10;
const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
class PositionalAudioRingBuffer : public AudioRingBuffer { class PositionalAudioRingBuffer : public InboundAudioStream {
Q_OBJECT
public: public:
enum Type { enum Type {
Microphone, Microphone,
@ -45,73 +29,44 @@ public:
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false);
virtual int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) = 0; virtual AudioStreamStats getAudioStreamStats() const;
int parsePositionalData(const QByteArray& positionalByteArray);
int parseListenModeData(const QByteArray& listenModeByteArray);
void updateNextOutputTrailingLoudness(); void updateNextOutputTrailingLoudness();
float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; } float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; }
bool shouldBeAddedToMix();
bool willBeAddedToMix() const { return _willBeAddedToMix; }
void setWillBeAddedToMix(bool willBeAddedToMix) { _willBeAddedToMix = willBeAddedToMix; }
bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; } bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; }
bool isStereo() const { return _isStereo; } bool isStereo() const { return _isStereo; }
PositionalAudioRingBuffer::Type getType() const { return _type; } PositionalAudioRingBuffer::Type getType() const { return _type; }
const glm::vec3& getPosition() const { return _position; } const glm::vec3& getPosition() const { return _position; }
const glm::quat& getOrientation() const { return _orientation; } const glm::quat& getOrientation() const { return _orientation; }
AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; }
void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; }
int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; }
const MovingMinMaxAvg<quint64>& getInterframeTimeGapStatsForStatsPacket() const { return _interframeTimeGapStatsForStatsPacket; }
int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked
int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; }
double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); }
int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; }
int getStarveCount() const { return _starveCount; }
int getSilentFramesDropped() const { return _silentFramesDropped; }
protected: protected:
// disallow copying of PositionalAudioRingBuffer objects // disallow copying of PositionalAudioRingBuffer objects
PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer(const PositionalAudioRingBuffer&);
PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&);
void frameReceivedUpdateTimingStats(); /// parses the info between the seq num and the audio data in the network packet and calculates
void addDroppableSilentSamples(int numSilentSamples); /// how many audio samples this packet contains
virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0;
PositionalAudioRingBuffer::Type _type; /// parses the audio data in the network packet
virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) = 0;
int parsePositionalData(const QByteArray& positionalByteArray);
protected:
Type _type;
glm::vec3 _position; glm::vec3 _position;
glm::quat _orientation; glm::quat _orientation;
bool _willBeAddedToMix;
bool _shouldLoopbackForNode; bool _shouldLoopbackForNode;
bool _shouldOutputStarveDebug;
bool _isStereo; bool _isStereo;
float _nextOutputTrailingLoudness; float _nextOutputTrailingLoudness;
AABox* _listenerUnattenuatedZone; AABox* _listenerUnattenuatedZone;
quint64 _lastFrameReceivedTime;
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForJitterCalc;
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForStatsPacket;
MovingMinMaxAvg<int> _framesAvailableStats;
int _desiredJitterBufferFrames;
bool _dynamicJitterBuffers;
// extra stats
int _consecutiveNotMixedCount;
int _starveCount;
int _silentFramesDropped;
}; };
#endif // hifi_PositionalAudioRingBuffer_h #endif // hifi_PositionalAudioRingBuffer_h