mirror of
https://github.com/overte-org/overte.git
synced 2025-04-25 18:16:08 +02:00
audiomixer code complete; need to test
This commit is contained in:
wangyix
parent
e2f957d6dc
commit
c9b6879ca8
13 changed files with 394 additions and 485 deletions
|
|
@ -95,6 +95,19 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f;
|
|||
|
||||
void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd,
|
||||
AvatarAudioRingBuffer* listeningNodeBuffer) {
|
||||
// if the frame to be mixed is silent, don't mix it
|
||||
if (bufferToAdd->getNextOutputTrailingLoudness() == 0.0f) {
|
||||
bufferToAdd->popFrames(1);
|
||||
return;
|
||||
}
|
||||
|
||||
// get pointer to frame to be mixed. If the stream cannot provide a frame (is starved), bail
|
||||
AudioRingBuffer::ConstIterator nextOutputStart;
|
||||
if (!bufferToAdd->popFrames(&nextOutputStart, 1)) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
float bearingRelativeAngleToSource = 0.0f;
|
||||
float attenuationCoefficient = 1.0f;
|
||||
int numSamplesDelay = 0;
|
||||
|
|
@ -203,7 +216,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
|
|||
}
|
||||
}
|
||||
|
||||
const int16_t* nextOutputStart = bufferToAdd->getNextOutput();
|
||||
|
||||
|
||||
if (!bufferToAdd->isStereo() && shouldAttenuate) {
|
||||
// this is a mono buffer, which means it gets full attenuation and spatialization
|
||||
|
|
@ -212,8 +225,8 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
|
|||
int delayedChannelOffset = (bearingRelativeAngleToSource > 0.0f) ? 1 : 0;
|
||||
int goodChannelOffset = delayedChannelOffset == 0 ? 1 : 0;
|
||||
|
||||
const int16_t* bufferStart = bufferToAdd->getBuffer();
|
||||
int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity();
|
||||
//const int16_t* bufferStart = bufferToAdd->getBuffer();
|
||||
//int ringBufferSampleCapacity = bufferToAdd->getSampleCapacity();
|
||||
|
||||
int16_t correctBufferSample[2], delayBufferSample[2];
|
||||
int delayedChannelIndex = 0;
|
||||
|
|
@ -241,14 +254,15 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf
|
|||
// if there was a sample delay for this buffer, we need to pull samples prior to the nextOutput
|
||||
// to stick at the beginning
|
||||
float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio;
|
||||
const int16_t* delayNextOutputStart = nextOutputStart - numSamplesDelay;
|
||||
if (delayNextOutputStart < bufferStart) {
|
||||
delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay;
|
||||
}
|
||||
AudioRingBuffer::ConstIterator delayNextOutputStart = nextOutputStart - numSamplesDelay;
|
||||
//if (delayNextOutputStart < bufferStart) {
|
||||
//delayNextOutputStart = bufferStart + ringBufferSampleCapacity - numSamplesDelay;
|
||||
//}
|
||||
|
||||
for (int i = 0; i < numSamplesDelay; i++) {
|
||||
int parentIndex = i * 2;
|
||||
_clientSamples[parentIndex + delayedChannelOffset] += delayNextOutputStart[i] * attenuationAndWeakChannelRatio;
|
||||
_clientSamples[parentIndex + delayedChannelOffset] += *delayNextOutputStart * attenuationAndWeakChannelRatio;
|
||||
++delayNextOutputStart;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -293,13 +307,13 @@ void AudioMixer::prepareMixForListeningNode(Node* node) {
|
|||
AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData();
|
||||
|
||||
// enumerate the ARBs attached to the otherNode and add all that should be added to mix
|
||||
for (int i = 0; i < otherNodeClientData->getRingBuffers().size(); i++) {
|
||||
PositionalAudioRingBuffer* otherNodeBuffer = otherNodeClientData->getRingBuffers()[i];
|
||||
|
||||
const QHash<QUuid, PositionalAudioRingBuffer*>& otherNodeRingBuffers = otherNodeClientData->getRingBuffers();
|
||||
QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = otherNodeRingBuffers.constEnd();
|
||||
for (i = otherNodeRingBuffers.begin(); i != end; i++) {
|
||||
PositionalAudioRingBuffer* otherNodeBuffer = i.value();
|
||||
|
||||
if ((*otherNode != *node
|
||||
|| otherNodeBuffer->shouldLoopbackForNode())
|
||||
&& otherNodeBuffer->willBeAddedToMix()
|
||||
&& otherNodeBuffer->getNextOutputTrailingLoudness() > 0.0f) {
|
||||
if (*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) {
|
||||
addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer);
|
||||
}
|
||||
}
|
||||
|
|
@ -307,7 +321,6 @@ void AudioMixer::prepareMixForListeningNode(Node* node) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
void AudioMixer::readPendingDatagrams() {
|
||||
QByteArray receivedPacket;
|
||||
HifiSockAddr senderSockAddr;
|
||||
|
|
@ -500,12 +513,12 @@ void AudioMixer::run() {
|
|||
|
||||
while (!_isFinished) {
|
||||
|
||||
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
|
||||
/*foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
|
||||
if (node->getLinkedData()) {
|
||||
((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone,
|
||||
_listenerUnattenuatedZone);
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f;
|
||||
const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f;
|
||||
|
|
@ -599,13 +612,13 @@ void AudioMixer::run() {
|
|||
++_sumListeners;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
// push forward the next output pointers for any audio buffers we used
|
||||
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
|
||||
if (node->getLinkedData()) {
|
||||
((AudioMixerClientData*) node->getLinkedData())->pushBuffersAfterFrameSend();
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
++_numStatFrames;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,34 +18,28 @@
|
|||
|
||||
#include "AudioMixer.h"
|
||||
#include "AudioMixerClientData.h"
|
||||
#include "MovingMinMaxAvg.h"
|
||||
|
||||
const int INCOMING_SEQ_STATS_HISTORY_LENGTH = INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS /
|
||||
(TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS / USECS_PER_SECOND);
|
||||
|
||||
AudioMixerClientData::AudioMixerClientData() :
|
||||
_ringBuffers(),
|
||||
_outgoingMixedAudioSequenceNumber(0),
|
||||
_incomingAvatarAudioSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH)
|
||||
_outgoingMixedAudioSequenceNumber(0)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
AudioMixerClientData::~AudioMixerClientData() {
|
||||
for (int i = 0; i < _ringBuffers.size(); i++) {
|
||||
// delete this attached PositionalAudioRingBuffer
|
||||
delete _ringBuffers[i];
|
||||
QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = _ringBuffers.constEnd();
|
||||
for (i = _ringBuffers.begin(); i != end; i++) {
|
||||
// delete this attached InboundAudioStream
|
||||
delete i.value();
|
||||
}
|
||||
}
|
||||
|
||||
AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const {
|
||||
for (int i = 0; i < _ringBuffers.size(); i++) {
|
||||
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) {
|
||||
return (AvatarAudioRingBuffer*) _ringBuffers[i];
|
||||
}
|
||||
if (_ringBuffers.contains(QUuid())) {
|
||||
return (AvatarAudioRingBuffer*)_ringBuffers.value(QUuid());
|
||||
}
|
||||
|
||||
// no AvatarAudioRingBuffer found - return NULL
|
||||
// no mic stream found - return NULL
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -57,96 +51,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
|
|||
quint16 sequence = *(reinterpret_cast<const quint16*>(sequenceAt));
|
||||
|
||||
PacketType packetType = packetTypeForPacket(packet);
|
||||
if (packetType == PacketTypeMicrophoneAudioWithEcho
|
||||
|| packetType == PacketTypeMicrophoneAudioNoEcho
|
||||
|| packetType == PacketTypeSilentAudioFrame) {
|
||||
|
||||
SequenceNumberStats::ArrivalInfo packetArrivalInfo = _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence);
|
||||
|
||||
// grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
|
||||
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
|
||||
|
||||
// read the first byte after the header to see if this is a stereo or mono buffer
|
||||
quint8 channelFlag = packet.at(numBytesForPacketHeader(packet) + sizeof(quint16));
|
||||
bool isStereo = channelFlag == 1;
|
||||
|
||||
if (avatarRingBuffer && avatarRingBuffer->isStereo() != isStereo) {
|
||||
// there's a mismatch in the buffer channels for the incoming and current buffer
|
||||
// so delete our current buffer and create a new one
|
||||
_ringBuffers.removeOne(avatarRingBuffer);
|
||||
avatarRingBuffer->deleteLater();
|
||||
avatarRingBuffer = NULL;
|
||||
}
|
||||
|
||||
if (!avatarRingBuffer) {
|
||||
// we don't have an AvatarAudioRingBuffer yet, so add it
|
||||
avatarRingBuffer = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers());
|
||||
_ringBuffers.push_back(avatarRingBuffer);
|
||||
}
|
||||
|
||||
|
||||
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
|
||||
// in the ring buffer (if that frame hasn't been mixed yet)
|
||||
switch (packetArrivalInfo._status) {
|
||||
case SequenceNumberStats::Early: {
|
||||
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
|
||||
avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost);
|
||||
break;
|
||||
}
|
||||
case SequenceNumberStats::OnTime: {
|
||||
// ask the AvatarAudioRingBuffer instance to parse the data
|
||||
avatarRingBuffer->parseDataAndHandleDroppedPackets(packet, 0);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (packetType == PacketTypeInjectAudio) {
|
||||
// this is injected audio
|
||||
|
||||
// grab the stream identifier for this injected audio
|
||||
QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID));
|
||||
|
||||
if (!_incomingInjectedAudioSequenceNumberStatsMap.contains(streamIdentifier)) {
|
||||
_incomingInjectedAudioSequenceNumberStatsMap.insert(streamIdentifier, SequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH));
|
||||
}
|
||||
SequenceNumberStats::ArrivalInfo packetArrivalInfo =
|
||||
_incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
|
||||
|
||||
InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
|
||||
|
||||
for (int i = 0; i < _ringBuffers.size(); i++) {
|
||||
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector
|
||||
&& ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) {
|
||||
matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (!matchingInjectedRingBuffer) {
|
||||
// we don't have a matching injected audio ring buffer, so add it
|
||||
matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers());
|
||||
_ringBuffers.push_back(matchingInjectedRingBuffer);
|
||||
}
|
||||
|
||||
// for now, late packets are simply discarded. In the future, it may be good to insert them into their correct place
|
||||
// in the ring buffer (if that frame hasn't been mixed yet)
|
||||
switch (packetArrivalInfo._status) {
|
||||
case SequenceNumberStats::Early: {
|
||||
int packetsLost = packetArrivalInfo._seqDiffFromExpected;
|
||||
matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, packetsLost);
|
||||
break;
|
||||
}
|
||||
case SequenceNumberStats::OnTime: {
|
||||
// ask the AvatarAudioRingBuffer instance to parse the data
|
||||
matchingInjectedRingBuffer->parseDataAndHandleDroppedPackets(packet, 0);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (packetType == PacketTypeAudioStreamStats) {
|
||||
if (packetType == PacketTypeAudioStreamStats) {
|
||||
|
||||
const char* dataAt = packet.data();
|
||||
|
||||
|
|
@ -155,12 +60,52 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
|
|||
|
||||
// read the downstream audio stream stats
|
||||
memcpy(&_downstreamAudioStreamStats, dataAt, sizeof(AudioStreamStats));
|
||||
}
|
||||
dataAt += sizeof(AudioStreamStats);
|
||||
|
||||
return dataAt - packet.data();
|
||||
|
||||
} else {
|
||||
PositionalAudioRingBuffer* matchingStream = NULL;
|
||||
|
||||
if (packetType == PacketTypeMicrophoneAudioWithEcho
|
||||
|| packetType == PacketTypeMicrophoneAudioNoEcho
|
||||
|| packetType == PacketTypeSilentAudioFrame) {
|
||||
|
||||
QUuid nullUUID = QUuid();
|
||||
if (!_ringBuffers.contains(nullUUID)) {
|
||||
// we don't have a mic stream yet, so add it
|
||||
|
||||
// read the channel flag to see if our stream is stereo or not
|
||||
const char* channelFlagAt = packet.constData() + numBytesForPacketHeader(packet) + sizeof(quint16);
|
||||
quint8 channelFlag = *(reinterpret_cast<const quint8*>(channelFlagAt));
|
||||
bool isStereo = channelFlag == 1;
|
||||
|
||||
_ringBuffers.insert(nullUUID,
|
||||
matchingStream = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers()));
|
||||
} else {
|
||||
matchingStream = _ringBuffers.value(nullUUID);
|
||||
}
|
||||
} else if (packetType == PacketTypeInjectAudio) {
|
||||
// this is injected audio
|
||||
|
||||
// grab the stream identifier for this injected audio
|
||||
int bytesBeforeStreamIdentifier = numBytesForPacketHeader(packet) + sizeof(quint16);
|
||||
QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(bytesBeforeStreamIdentifier, NUM_BYTES_RFC4122_UUID));
|
||||
|
||||
if (!_ringBuffers.contains(streamIdentifier)) {
|
||||
_ringBuffers.insert(streamIdentifier,
|
||||
matchingStream = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()));
|
||||
} else {
|
||||
matchingStream = _ringBuffers.value(streamIdentifier);
|
||||
}
|
||||
}
|
||||
|
||||
return matchingStream->parseData(packet);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) {
|
||||
/*void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) {
|
||||
for (int i = 0; i < _ringBuffers.size(); i++) {
|
||||
if (_ringBuffers[i]->shouldBeAddedToMix()) {
|
||||
// this is a ring buffer that is ready to go
|
||||
|
|
@ -205,9 +150,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() {
|
|||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const {
|
||||
/*AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const {
|
||||
|
||||
AudioStreamStats streamStats;
|
||||
|
||||
|
|
@ -239,20 +184,9 @@ AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const Positio
|
|||
streamStats._ringBufferSilentFramesDropped = ringBuffer->getSilentFramesDropped();
|
||||
|
||||
return streamStats;
|
||||
}
|
||||
}*/
|
||||
|
||||
void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) {
|
||||
|
||||
// have all the seq number stats of each audio stream push their current stats into their history,
|
||||
// which moves that history window 1 second forward (since that's how long since the last stats were pushed into history)
|
||||
_incomingAvatarAudioSequenceNumberStats.pushStatsToHistory();
|
||||
QHash<QUuid, SequenceNumberStats>::Iterator i = _incomingInjectedAudioSequenceNumberStatsMap.begin();
|
||||
QHash<QUuid, SequenceNumberStats>::Iterator end = _incomingInjectedAudioSequenceNumberStatsMap.end();
|
||||
while (i != end) {
|
||||
i.value().pushStatsToHistory();
|
||||
i++;
|
||||
}
|
||||
|
||||
char packet[MAX_PACKET_SIZE];
|
||||
NodeList* nodeList = NodeList::getInstance();
|
||||
|
||||
|
|
@ -271,7 +205,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
|
|||
|
||||
// pack and send stream stats packets until all ring buffers' stats are sent
|
||||
int numStreamStatsRemaining = _ringBuffers.size();
|
||||
QList<PositionalAudioRingBuffer*>::ConstIterator ringBuffersIterator = _ringBuffers.constBegin();
|
||||
QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator ringBuffersIterator = _ringBuffers.constBegin();
|
||||
while (numStreamStatsRemaining > 0) {
|
||||
|
||||
char* dataAt = headerEndAt;
|
||||
|
|
@ -288,7 +222,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
|
|||
|
||||
// pack the calculated number of stream stats
|
||||
for (int i = 0; i < numStreamStatsToPack; i++) {
|
||||
AudioStreamStats streamStats = getAudioStreamStatsOfStream(*ringBuffersIterator);
|
||||
AudioStreamStats streamStats = ringBuffersIterator.value()->getAudioStreamStats();
|
||||
memcpy(dataAt, &streamStats, sizeof(AudioStreamStats));
|
||||
dataAt += sizeof(AudioStreamStats);
|
||||
|
||||
|
|
@ -322,7 +256,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
|
|||
|
||||
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
|
||||
if (avatarRingBuffer) {
|
||||
AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer);
|
||||
AudioStreamStats streamStats = avatarRingBuffer->getAudioStreamStats();
|
||||
result += " UPSTREAM.mic.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
|
||||
+ " desired_calc:" + QString::number(avatarRingBuffer->getCalculatedDesiredJitterBufferFrames())
|
||||
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
|
||||
|
|
@ -343,11 +277,12 @@ QString AudioMixerClientData::getAudioStreamStatsString() const {
|
|||
result = "mic unknown";
|
||||
}
|
||||
|
||||
for (int i = 0; i < _ringBuffers.size(); i++) {
|
||||
if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) {
|
||||
AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]);
|
||||
QHash<QUuid, PositionalAudioRingBuffer*>::ConstIterator i, end = _ringBuffers.end();
|
||||
for (i = _ringBuffers.begin(); i != end; i++) {
|
||||
if (i.value()->getType() == PositionalAudioRingBuffer::Injector) {
|
||||
AudioStreamStats streamStats = i.value()->getAudioStreamStats();
|
||||
result += " UPSTREAM.inj.desired:" + QString::number(streamStats._ringBufferDesiredJitterBufferFrames)
|
||||
+ " desired_calc:" + QString::number(_ringBuffers[i]->getCalculatedDesiredJitterBufferFrames())
|
||||
+ " desired_calc:" + QString::number(i.value()->getCalculatedDesiredJitterBufferFrames())
|
||||
+ " available_avg_10s:" + QString::number(streamStats._ringBufferFramesAvailableAverage)
|
||||
+ " available:" + QString::number(streamStats._ringBufferFramesAvailable)
|
||||
+ " starves:" + QString::number(streamStats._ringBufferStarveCount)
|
||||
|
|
|
|||
|
|
@ -13,29 +13,23 @@
|
|||
#define hifi_AudioMixerClientData_h
|
||||
|
||||
#include <AABox.h>
|
||||
#include <NodeData.h>
|
||||
#include <PositionalAudioRingBuffer.h>
|
||||
|
||||
#include "PositionalAudioRingBuffer.h"
|
||||
#include "AvatarAudioRingBuffer.h"
|
||||
#include "AudioStreamStats.h"
|
||||
#include "SequenceNumberStats.h"
|
||||
|
||||
|
||||
const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30;
|
||||
|
||||
class AudioMixerClientData : public NodeData {
|
||||
public:
|
||||
AudioMixerClientData();
|
||||
~AudioMixerClientData();
|
||||
|
||||
const QList<PositionalAudioRingBuffer*> getRingBuffers() const { return _ringBuffers; }
|
||||
const QHash<QUuid, PositionalAudioRingBuffer*>& getRingBuffers() const { return _ringBuffers; }
|
||||
AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const;
|
||||
|
||||
int parseData(const QByteArray& packet);
|
||||
void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL);
|
||||
void pushBuffersAfterFrameSend();
|
||||
//void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL);
|
||||
//void pushBuffersAfterFrameSend();
|
||||
|
||||
AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const;
|
||||
//AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const;
|
||||
QString getAudioStreamStatsString() const;
|
||||
|
||||
void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode);
|
||||
|
|
@ -44,11 +38,9 @@ public:
|
|||
quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; }
|
||||
|
||||
private:
|
||||
QList<PositionalAudioRingBuffer*> _ringBuffers;
|
||||
QHash<QUuid, PositionalAudioRingBuffer*> _ringBuffers; // mic stream stored under key of null UUID
|
||||
|
||||
quint16 _outgoingMixedAudioSequenceNumber;
|
||||
SequenceNumberStats _incomingAvatarAudioSequenceNumberStats;
|
||||
QHash<QUuid, SequenceNumberStats> _incomingInjectedAudioSequenceNumberStatsMap;
|
||||
|
||||
AudioStreamStats _downstreamAudioStreamStats;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -14,57 +14,50 @@
|
|||
#include "AvatarAudioRingBuffer.h"
|
||||
|
||||
AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBuffer) :
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) {
|
||||
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) {
|
||||
|
||||
}
|
||||
|
||||
int AvatarAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) {
|
||||
frameReceivedUpdateTimingStats();
|
||||
int AvatarAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) {
|
||||
|
||||
_shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho);
|
||||
_shouldLoopbackForNode = (type == PacketTypeMicrophoneAudioWithEcho);
|
||||
|
||||
// skip the packet header (includes the source UUID)
|
||||
int readBytes = numBytesForPacketHeader(packet);
|
||||
int readBytes = 0;
|
||||
|
||||
// skip the sequence number
|
||||
readBytes += sizeof(quint16);
|
||||
|
||||
// hop over the channel flag that has already been read in AudioMixerClientData
|
||||
// read the channel flag
|
||||
quint8 channelFlag = packetAfterSeqNum.at(readBytes);
|
||||
bool isStereo = channelFlag == 1;
|
||||
readBytes += sizeof(quint8);
|
||||
|
||||
// if isStereo value has changed, restart the ring buffer with new frame size
|
||||
if (isStereo != _isStereo) {
|
||||
_ringBuffer.resizeForFrameSize(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
|
||||
_isStereo = isStereo;
|
||||
}
|
||||
|
||||
// read the positional data
|
||||
readBytes += parsePositionalData(packet.mid(readBytes));
|
||||
readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes));
|
||||
|
||||
if (packetTypeForPacket(packet) == PacketTypeSilentAudioFrame) {
|
||||
// this source had no audio to send us, but this counts as a packet
|
||||
// write silence equivalent to the number of silent samples they just sent us
|
||||
if (type == PacketTypeSilentAudioFrame) {
|
||||
int16_t numSilentSamples;
|
||||
|
||||
memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t));
|
||||
memcpy(&numSilentSamples, packetAfterSeqNum.data() + readBytes, sizeof(int16_t));
|
||||
readBytes += sizeof(int16_t);
|
||||
|
||||
// add silent samples for the dropped packets as well.
|
||||
// ASSUME that each dropped packet had same number of silent samples as this one
|
||||
numSilentSamples *= (packetsSkipped + 1);
|
||||
|
||||
// NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples
|
||||
// CAN'T DO THIS because ScriptEngine.cpp sends frames of different size due to having a different sending interval
|
||||
// (every 16.667ms) than Audio.cpp (every 10.667ms)
|
||||
//numSilentSamples = getSamplesPerFrame();
|
||||
|
||||
addDroppableSilentSamples(numSilentSamples);
|
||||
|
||||
numAudioSamples = numSilentSamples;
|
||||
} else {
|
||||
int numAudioBytes = packet.size() - readBytes;
|
||||
int numAudioSamples = numAudioBytes / sizeof(int16_t);
|
||||
|
||||
// add silent samples for the dropped packets.
|
||||
// ASSUME that each dropped packet had same number of samples as this one
|
||||
if (packetsSkipped > 0) {
|
||||
addDroppableSilentSamples(packetsSkipped * numAudioSamples);
|
||||
}
|
||||
|
||||
// there is audio data to read
|
||||
readBytes += writeData(packet.data() + readBytes, numAudioBytes);
|
||||
int numAudioBytes = packetAfterSeqNum.size() - readBytes;
|
||||
numAudioSamples = numAudioBytes / sizeof(int16_t);
|
||||
}
|
||||
return readBytes;
|
||||
}
|
||||
|
||||
int AvatarAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
|
||||
int readBytes = 0;
|
||||
if (type == PacketTypeSilentAudioFrame) {
|
||||
writeDroppableSilentSamples(numAudioSamples);
|
||||
} else {
|
||||
// there is audio data to read
|
||||
readBytes += _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
|
||||
}
|
||||
return readBytes;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@
|
|||
class AvatarAudioRingBuffer : public PositionalAudioRingBuffer {
|
||||
public:
|
||||
AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false);
|
||||
|
||||
int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped);
|
||||
|
||||
private:
|
||||
// disallow copying of AvatarAudioRingBuffer objects
|
||||
AvatarAudioRingBuffer(const AvatarAudioRingBuffer&);
|
||||
AvatarAudioRingBuffer& operator= (const AvatarAudioRingBuffer&);
|
||||
|
||||
int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples);
|
||||
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
|
||||
};
|
||||
|
||||
#endif // hifi_AvatarAudioRingBuffer_h
|
||||
|
|
|
|||
|
|
@ -231,3 +231,18 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int
|
|||
return position + numSamplesShift;
|
||||
}
|
||||
}
|
||||
|
||||
float AudioRingBuffer::getNextOutputFrameLoudness() const {
|
||||
float loudness = 0.0f;
|
||||
int16_t* sampleAt = _nextOutput;
|
||||
int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1;
|
||||
if (samplesAvailable() >= _numFrameSamples) {
|
||||
for (int i = 0; i < _numFrameSamples; ++i) {
|
||||
loudness += fabsf(*sampleAt);
|
||||
sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1;
|
||||
}
|
||||
loudness /= _numFrameSamples;
|
||||
loudness /= MAX_SAMPLE_VALUE;
|
||||
}
|
||||
return loudness;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,6 +65,8 @@ public:
|
|||
const int16_t& operator[] (const int index) const;
|
||||
|
||||
void shiftReadPosition(unsigned int numSamples);
|
||||
|
||||
float getNextOutputFrameLoudness() const;
|
||||
|
||||
int samplesAvailable() const;
|
||||
int framesAvailable() const { return samplesAvailable() / _numFrameSamples; }
|
||||
|
|
@ -99,6 +101,86 @@ protected:
|
|||
bool _isStarved;
|
||||
bool _hasStarted;
|
||||
bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing
|
||||
|
||||
public:
|
||||
class ConstIterator { //public std::iterator < std::forward_iterator_tag, int16_t > {
|
||||
public:
|
||||
ConstIterator()
|
||||
: _capacity(0),
|
||||
_bufferFirst(NULL),
|
||||
_bufferLast(NULL),
|
||||
_at(NULL) {}
|
||||
|
||||
ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at)
|
||||
: _capacity(capacity),
|
||||
_bufferFirst(bufferFirst),
|
||||
_bufferLast(bufferFirst + capacity - 1),
|
||||
_at(at) {}
|
||||
|
||||
bool operator==(const ConstIterator& rhs) { return _at == rhs._at; }
|
||||
bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; }
|
||||
int16_t operator*() { return *_at; }
|
||||
|
||||
ConstIterator& operator=(const ConstIterator& rhs) {
|
||||
_capacity = rhs._capacity;
|
||||
_bufferFirst = rhs._bufferFirst;
|
||||
_bufferLast = rhs._bufferLast;
|
||||
_at = rhs._at;
|
||||
return *this;
|
||||
}
|
||||
|
||||
ConstIterator& operator++() {
|
||||
_at = (_at == _bufferLast) ? _bufferFirst : _at + 1;
|
||||
return *this;
|
||||
}
|
||||
|
||||
ConstIterator operator++(int) {
|
||||
ConstIterator tmp(*this);
|
||||
++(*this);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
ConstIterator& operator--() {
|
||||
_at = (_at == _bufferFirst) ? _bufferLast : _at - 1;
|
||||
return *this;
|
||||
}
|
||||
|
||||
ConstIterator operator--(int) {
|
||||
ConstIterator tmp(*this);
|
||||
--(*this);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
int16_t operator[] (int i) {
|
||||
return *atShiftedBy(i);
|
||||
}
|
||||
|
||||
ConstIterator operator+(int i) {
|
||||
return ConstIterator(_bufferFirst, _capacity, atShiftedBy(i));
|
||||
}
|
||||
|
||||
ConstIterator operator-(int i) {
|
||||
return ConstIterator(_bufferFirst, _capacity, atShiftedBy(-i));
|
||||
}
|
||||
|
||||
private:
|
||||
int16_t* atShiftedBy(int i) {
|
||||
i = (_at - _bufferFirst + i) % _capacity;
|
||||
if (i < 0) {
|
||||
i += _capacity;
|
||||
}
|
||||
return _bufferFirst + i;
|
||||
}
|
||||
|
||||
private:
|
||||
int _capacity;
|
||||
int16_t* _bufferFirst;
|
||||
int16_t* _bufferLast;
|
||||
int16_t* _at;
|
||||
};
|
||||
|
||||
|
||||
ConstIterator nextOutput() const { return ConstIterator(_buffer, _sampleCapacity, _nextOutput); }
|
||||
};
|
||||
|
||||
#endif // hifi_AudioRingBuffer_h
|
||||
|
|
|
|||
|
|
@ -13,19 +13,19 @@
|
|||
#include "PacketHeaders.h"
|
||||
|
||||
InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers) :
|
||||
_ringBuffer(numFrameSamples, false, numFramesCapacity),
|
||||
_dynamicJitterBuffers(dynamicJitterBuffers),
|
||||
_desiredJitterBufferFrames(1),
|
||||
_isStarved(true),
|
||||
_hasStarted(false),
|
||||
_consecutiveNotMixedCount(0),
|
||||
_starveCount(0),
|
||||
_silentFramesDropped(0),
|
||||
_incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS),
|
||||
_lastFrameReceivedTime(0),
|
||||
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
|
||||
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
|
||||
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS)
|
||||
_ringBuffer(numFrameSamples, false, numFramesCapacity),
|
||||
_dynamicJitterBuffers(dynamicJitterBuffers),
|
||||
_desiredJitterBufferFrames(1),
|
||||
_isStarved(true),
|
||||
_hasStarted(false),
|
||||
_consecutiveNotMixedCount(0),
|
||||
_starveCount(0),
|
||||
_silentFramesDropped(0),
|
||||
_incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS),
|
||||
_lastFrameReceivedTime(0),
|
||||
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
|
||||
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
|
||||
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS)
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -94,19 +94,19 @@ int InboundAudioStream::parseData(const QByteArray& packet) {
|
|||
return readBytes;
|
||||
}
|
||||
|
||||
bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) {
|
||||
bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) {
|
||||
if (_isStarved) {
|
||||
_consecutiveNotMixedCount++;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool framesPopped = false;
|
||||
bool popped = false;
|
||||
|
||||
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
|
||||
if (_ringBuffer.samplesAvailable >= numSamplesRequested) {
|
||||
_ringBuffer.readSamples(dest, numSamplesRequested);
|
||||
if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
|
||||
_ringBuffer.shiftReadPosition(numSamplesRequested);
|
||||
_hasStarted = true;
|
||||
framesPopped = true;
|
||||
popped = true;
|
||||
} else {
|
||||
if (starveOnFail) {
|
||||
setToStarved();
|
||||
|
|
@ -116,7 +116,58 @@ bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFa
|
|||
|
||||
_framesAvailableStats.update(_ringBuffer.framesAvailable());
|
||||
|
||||
return framesPopped;
|
||||
return popped;
|
||||
}
|
||||
|
||||
bool InboundAudioStream::popFrames(int16_t* dest, int numFrames, bool starveOnFail) {
|
||||
if (_isStarved) {
|
||||
_consecutiveNotMixedCount++;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool popped = false;
|
||||
|
||||
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
|
||||
if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
|
||||
_ringBuffer.readSamples(dest, numSamplesRequested);
|
||||
_hasStarted = true;
|
||||
popped = true;
|
||||
} else {
|
||||
if (starveOnFail) {
|
||||
setToStarved();
|
||||
_consecutiveNotMixedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
_framesAvailableStats.update(_ringBuffer.framesAvailable());
|
||||
|
||||
return popped;
|
||||
}
|
||||
|
||||
bool InboundAudioStream::popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail) {
|
||||
if (_isStarved) {
|
||||
_consecutiveNotMixedCount++;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool popped = false;
|
||||
|
||||
int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples();
|
||||
if (_ringBuffer.samplesAvailable() >= numSamplesRequested) {
|
||||
*nextOutput = _ringBuffer.nextOutput();
|
||||
_ringBuffer.shiftReadPosition(numSamplesRequested);
|
||||
_hasStarted = true;
|
||||
popped = true;
|
||||
} else {
|
||||
if (starveOnFail) {
|
||||
setToStarved();
|
||||
_consecutiveNotMixedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
_framesAvailableStats.update(_ringBuffer.framesAvailable());
|
||||
|
||||
return popped;
|
||||
}
|
||||
|
||||
void InboundAudioStream::setToStarved() {
|
||||
|
|
|
|||
|
|
@ -52,14 +52,15 @@ public:
|
|||
|
||||
|
||||
int parseData(const QByteArray& packet);
|
||||
|
||||
bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true);
|
||||
|
||||
bool popFrames(int numFrames, bool starveOnFail = true);
|
||||
bool popFrames(int16_t* dest, int numFrames, bool starveOnFail = true);
|
||||
bool popFrames(AudioRingBuffer::ConstIterator* nextOutput, int numFrames, bool starveOnFail = true);
|
||||
|
||||
void setToStarved();
|
||||
|
||||
|
||||
|
||||
/// this function should be called once per second to ensure the seq num stats history spans ~30 seconds
|
||||
AudioStreamStats updateSeqHistoryAndGetAudioStreamStats();
|
||||
virtual AudioStreamStats getAudioStreamStats() const;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,52 +20,50 @@
|
|||
#include "InjectedAudioRingBuffer.h"
|
||||
|
||||
InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, bool dynamicJitterBuffer) :
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer),
|
||||
_streamIdentifier(streamIdentifier),
|
||||
_radius(0.0f),
|
||||
_attenuationRatio(0)
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, false, dynamicJitterBuffer),
|
||||
_streamIdentifier(streamIdentifier),
|
||||
_radius(0.0f),
|
||||
_attenuationRatio(0)
|
||||
{
|
||||
|
||||
|
||||
}
|
||||
|
||||
const uchar MAX_INJECTOR_VOLUME = 255;
|
||||
|
||||
int InjectedAudioRingBuffer::parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) {
|
||||
frameReceivedUpdateTimingStats();
|
||||
|
||||
int InjectedAudioRingBuffer::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) {
|
||||
// setup a data stream to read from this packet
|
||||
QDataStream packetStream(packet);
|
||||
packetStream.skipRawData(numBytesForPacketHeader(packet));
|
||||
|
||||
// push past the sequence number
|
||||
packetStream.skipRawData(sizeof(quint16));
|
||||
QDataStream packetStream(packetAfterSeqNum);
|
||||
|
||||
// push past the stream identifier
|
||||
// skip the stream identifier
|
||||
packetStream.skipRawData(NUM_BYTES_RFC4122_UUID);
|
||||
|
||||
|
||||
// pull the loopback flag and set our boolean
|
||||
uchar shouldLoopback;
|
||||
packetStream >> shouldLoopback;
|
||||
_shouldLoopbackForNode = (shouldLoopback == 1);
|
||||
|
||||
|
||||
// use parsePositionalData in parent PostionalAudioRingBuffer class to pull common positional data
|
||||
packetStream.skipRawData(parsePositionalData(packet.mid(packetStream.device()->pos())));
|
||||
|
||||
packetStream.skipRawData(parsePositionalData(packetAfterSeqNum.mid(packetStream.device()->pos())));
|
||||
|
||||
// pull out the radius for this injected source - if it's zero this is a point source
|
||||
packetStream >> _radius;
|
||||
|
||||
|
||||
quint8 attenuationByte = 0;
|
||||
packetStream >> attenuationByte;
|
||||
_attenuationRatio = attenuationByte / (float) MAX_INJECTOR_VOLUME;
|
||||
|
||||
int numAudioBytes = packet.size() - packetStream.device()->pos();
|
||||
int numAudioSamples = numAudioBytes / sizeof(int16_t);
|
||||
_attenuationRatio = attenuationByte / (float)MAX_INJECTOR_VOLUME;
|
||||
|
||||
// add silent samples for the dropped packets.
|
||||
// ASSUME that each dropped packet had same number of samples as this one
|
||||
addDroppableSilentSamples(numAudioSamples * packetsSkipped);
|
||||
int numAudioBytes = packetAfterSeqNum.size() - packetStream.device()->pos();
|
||||
numAudioSamples = numAudioBytes / sizeof(int16_t);
|
||||
|
||||
packetStream.skipRawData(writeData(packet.data() + packetStream.device()->pos(), numAudioBytes));
|
||||
|
||||
return packetStream.device()->pos();
|
||||
}
|
||||
|
||||
int InjectedAudioRingBuffer::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) {
|
||||
return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t));
|
||||
}
|
||||
|
||||
AudioStreamStats InjectedAudioRingBuffer::getAudioStreamStats() const {
|
||||
AudioStreamStats streamStats = PositionalAudioRingBuffer::getAudioStreamStats();
|
||||
streamStats._streamIdentifier = _streamIdentifier;
|
||||
return streamStats;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,18 +19,22 @@
|
|||
class InjectedAudioRingBuffer : public PositionalAudioRingBuffer {
|
||||
public:
|
||||
InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false);
|
||||
|
||||
int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped);
|
||||
|
||||
const QUuid& getStreamIdentifier() const { return _streamIdentifier; }
|
||||
|
||||
float getRadius() const { return _radius; }
|
||||
float getAttenuationRatio() const { return _attenuationRatio; }
|
||||
|
||||
QUuid getStreamIdentifier() const { return _streamIdentifier; }
|
||||
|
||||
private:
|
||||
// disallow copying of InjectedAudioRingBuffer objects
|
||||
InjectedAudioRingBuffer(const InjectedAudioRingBuffer&);
|
||||
InjectedAudioRingBuffer& operator= (const InjectedAudioRingBuffer&);
|
||||
|
||||
QUuid _streamIdentifier;
|
||||
|
||||
AudioStreamStats getAudioStreamStats() const;
|
||||
int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples);
|
||||
int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples);
|
||||
|
||||
const QUuid _streamIdentifier;
|
||||
float _radius;
|
||||
float _attenuationRatio;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@
|
|||
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
|
||||
//
|
||||
|
||||
#include "PositionalAudioRingBuffer.h"
|
||||
#include "SharedUtil.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include <glm/detail/func_common.hpp>
|
||||
|
|
@ -18,66 +21,27 @@
|
|||
#include <PacketHeaders.h>
|
||||
#include <UUID.h>
|
||||
|
||||
#include "PositionalAudioRingBuffer.h"
|
||||
#include "SharedUtil.h"
|
||||
|
||||
PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo, bool dynamicJitterBuffers) :
|
||||
|
||||
AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
|
||||
false, AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY),
|
||||
_type(type),
|
||||
_position(0.0f, 0.0f, 0.0f),
|
||||
_orientation(0.0f, 0.0f, 0.0f, 0.0f),
|
||||
_willBeAddedToMix(false),
|
||||
_shouldLoopbackForNode(false),
|
||||
_shouldOutputStarveDebug(true),
|
||||
_isStereo(isStereo),
|
||||
_nextOutputTrailingLoudness(0.0f),
|
||||
_listenerUnattenuatedZone(NULL),
|
||||
_lastFrameReceivedTime(0),
|
||||
_interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS),
|
||||
_interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS),
|
||||
_framesAvailableStats(FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES, FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS),
|
||||
_desiredJitterBufferFrames(1),
|
||||
_dynamicJitterBuffers(dynamicJitterBuffers),
|
||||
_consecutiveNotMixedCount(0),
|
||||
_starveCount(0),
|
||||
_silentFramesDropped(0)
|
||||
InboundAudioStream(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
|
||||
AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, dynamicJitterBuffers),
|
||||
_type(type),
|
||||
_position(0.0f, 0.0f, 0.0f),
|
||||
_orientation(0.0f, 0.0f, 0.0f, 0.0f),
|
||||
_shouldLoopbackForNode(false),
|
||||
_isStereo(isStereo),
|
||||
_nextOutputTrailingLoudness(0.0f),
|
||||
_listenerUnattenuatedZone(NULL)
|
||||
{
|
||||
}
|
||||
|
||||
int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
|
||||
QDataStream packetStream(positionalByteArray);
|
||||
|
||||
packetStream.readRawData(reinterpret_cast<char*>(&_position), sizeof(_position));
|
||||
packetStream.readRawData(reinterpret_cast<char*>(&_orientation), sizeof(_orientation));
|
||||
|
||||
// if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
|
||||
if (glm::isnan(_orientation.x)) {
|
||||
reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
return packetStream.device()->pos();
|
||||
}
|
||||
|
||||
void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
|
||||
// ForBoundarySamples means that we expect the number of samples not to roll of the end of the ring buffer
|
||||
float nextLoudness = 0;
|
||||
|
||||
if (samplesAvailable() >= _numFrameSamples) {
|
||||
for (int i = 0; i < _numFrameSamples; ++i) {
|
||||
nextLoudness += fabsf(_nextOutput[i]);
|
||||
}
|
||||
nextLoudness /= _numFrameSamples;
|
||||
nextLoudness /= MAX_SAMPLE_VALUE;
|
||||
}
|
||||
|
||||
float nextLoudness = _ringBuffer.getNextOutputFrameLoudness();
|
||||
|
||||
const int TRAILING_AVERAGE_FRAMES = 100;
|
||||
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
|
||||
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
|
||||
const float LOUDNESS_EPSILON = 0.000001f;
|
||||
|
||||
|
||||
if (nextLoudness >= _nextOutputTrailingLoudness) {
|
||||
_nextOutputTrailingLoudness = nextLoudness;
|
||||
} else {
|
||||
|
|
@ -89,120 +53,24 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() {
|
|||
}
|
||||
}
|
||||
|
||||
bool PositionalAudioRingBuffer::shouldBeAddedToMix() {
|
||||
int desiredJitterBufferSamples = _desiredJitterBufferFrames * _numFrameSamples;
|
||||
|
||||
if (!isNotStarvedOrHasMinimumSamples(_numFrameSamples + desiredJitterBufferSamples)) {
|
||||
// if the buffer was starved, allow it to accrue at least the desired number of
|
||||
// jitter buffer frames before we start taking frames from it for mixing
|
||||
int PositionalAudioRingBuffer::parsePositionalData(const QByteArray& positionalByteArray) {
|
||||
QDataStream packetStream(positionalByteArray);
|
||||
|
||||
if (_shouldOutputStarveDebug) {
|
||||
_shouldOutputStarveDebug = false;
|
||||
}
|
||||
packetStream.readRawData(reinterpret_cast<char*>(&_position), sizeof(_position));
|
||||
packetStream.readRawData(reinterpret_cast<char*>(&_orientation), sizeof(_orientation));
|
||||
|
||||
_consecutiveNotMixedCount++;
|
||||
return false;
|
||||
} else if (samplesAvailable() < _numFrameSamples) {
|
||||
// if the buffer doesn't have a full frame of samples to take for mixing, it is starved
|
||||
_isStarved = true;
|
||||
_starveCount++;
|
||||
|
||||
_framesAvailableStats.reset();
|
||||
|
||||
// reset our _shouldOutputStarveDebug to true so the next is printed
|
||||
_shouldOutputStarveDebug = true;
|
||||
|
||||
_consecutiveNotMixedCount = 1;
|
||||
return false;
|
||||
// if this node sent us a NaN for first float in orientation then don't consider this good audio and bail
|
||||
if (glm::isnan(_orientation.x)) {
|
||||
// NOTE: why would we reset the ring buffer here?
|
||||
_ringBuffer.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// good buffer, add this to the mix
|
||||
|
||||
// if we just finished refilling after a starve, we have a new jitter buffer length.
|
||||
// reset the frames available stats.
|
||||
|
||||
_isStarved = false;
|
||||
|
||||
_framesAvailableStats.update(framesAvailable());
|
||||
|
||||
// since we've read data from ring buffer at least once - we've started
|
||||
_hasStarted = true;
|
||||
|
||||
return true;
|
||||
return packetStream.device()->pos();
|
||||
}
|
||||
|
||||
int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const {
|
||||
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
|
||||
|
||||
int calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
|
||||
if (calculatedDesiredJitterBufferFrames < 1) {
|
||||
calculatedDesiredJitterBufferFrames = 1;
|
||||
}
|
||||
return calculatedDesiredJitterBufferFrames;
|
||||
}
|
||||
|
||||
|
||||
void PositionalAudioRingBuffer::frameReceivedUpdateTimingStats() {
|
||||
// update the two time gap stats we're keeping
|
||||
quint64 now = usecTimestampNow();
|
||||
if (_lastFrameReceivedTime != 0) {
|
||||
quint64 gap = now - _lastFrameReceivedTime;
|
||||
_interframeTimeGapStatsForJitterCalc.update(gap);
|
||||
_interframeTimeGapStatsForStatsPacket.update(gap);
|
||||
}
|
||||
_lastFrameReceivedTime = now;
|
||||
|
||||
// recalculate the _desiredJitterBufferFrames if _interframeTimeGapStatsForJitterCalc has updated stats for us
|
||||
if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) {
|
||||
if (!_dynamicJitterBuffers) {
|
||||
_desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence
|
||||
} else {
|
||||
const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE;
|
||||
|
||||
_desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME);
|
||||
if (_desiredJitterBufferFrames < 1) {
|
||||
_desiredJitterBufferFrames = 1;
|
||||
}
|
||||
const int maxDesired = _frameCapacity - 1;
|
||||
if (_desiredJitterBufferFrames > maxDesired) {
|
||||
_desiredJitterBufferFrames = maxDesired;
|
||||
}
|
||||
}
|
||||
_interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag();
|
||||
}
|
||||
}
|
||||
|
||||
void PositionalAudioRingBuffer::addDroppableSilentSamples(int numSilentSamples) {
|
||||
|
||||
// This adds some number of frames to the desired jitter buffer frames target we use.
|
||||
// The larger this value is, the less aggressive we are about reducing the jitter buffer length.
|
||||
// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long,
|
||||
// which could lead immediately to a starve.
|
||||
const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1;
|
||||
|
||||
// calculate how many silent frames we should drop. We only drop silent frames if
|
||||
// the running avg num frames available has stabilized and it's more than
|
||||
// our desired number of frames by the margin defined above.
|
||||
int numSilentFramesToDrop = 0;
|
||||
if (_framesAvailableStats.getNewStatsAvailableFlag() && _framesAvailableStats.isWindowFilled()
|
||||
&& numSilentSamples >= _numFrameSamples) {
|
||||
_framesAvailableStats.clearNewStatsAvailableFlag();
|
||||
int averageJitterBufferFrames = (int)_framesAvailableStats.getWindowAverage();
|
||||
int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING;
|
||||
|
||||
if (averageJitterBufferFrames > desiredJitterBufferFramesPlusPadding) {
|
||||
// our avg jitter buffer size exceeds its desired value, so ignore some silent
|
||||
// frames to get that size as close to desired as possible
|
||||
int numSilentFramesToDropDesired = averageJitterBufferFrames - desiredJitterBufferFramesPlusPadding;
|
||||
int numSilentFramesReceived = numSilentSamples / _numFrameSamples;
|
||||
numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived);
|
||||
|
||||
// since we now have a new jitter buffer length, reset the frames available stats.
|
||||
_framesAvailableStats.reset();
|
||||
|
||||
_silentFramesDropped += numSilentFramesToDrop;
|
||||
}
|
||||
}
|
||||
|
||||
addSilentFrame(numSilentSamples - numSilentFramesToDrop * _numFrameSamples);
|
||||
AudioStreamStats PositionalAudioRingBuffer::getAudioStreamStats() const {
|
||||
AudioStreamStats streamStats = InboundAudioStream::getAudioStreamStats();
|
||||
streamStats._streamType = _type;
|
||||
return streamStats;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,105 +13,60 @@
|
|||
#define hifi_PositionalAudioRingBuffer_h
|
||||
|
||||
#include <glm/gtx/quaternion.hpp>
|
||||
|
||||
#include <AABox.h>
|
||||
|
||||
#include "AudioRingBuffer.h"
|
||||
#include "MovingMinMaxAvg.h"
|
||||
|
||||
// the time gaps stats for _desiredJitterBufferFrames calculation
|
||||
// will recalculate the max for the past 5000 samples every 500 samples
|
||||
const int TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES = 500;
|
||||
const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10;
|
||||
|
||||
// the time gap stats for constructing AudioStreamStats will
|
||||
// recalculate min/max/avg every ~1 second for the past ~30 seconds of time gap data
|
||||
const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
|
||||
const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30;
|
||||
|
||||
// the stats for calculating the average frames available will recalculate every ~1 second
|
||||
// and will include data for the past ~10 seconds
|
||||
const int FRAMES_AVAILABLE_STATS_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS;
|
||||
const int FRAMES_AVAILABLE_STATS_WINDOW_INTERVALS = 10;
|
||||
#include "InboundAudioStream.h"
|
||||
|
||||
const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
|
||||
|
||||
class PositionalAudioRingBuffer : public AudioRingBuffer {
|
||||
class PositionalAudioRingBuffer : public InboundAudioStream {
|
||||
Q_OBJECT
|
||||
public:
|
||||
enum Type {
|
||||
Microphone,
|
||||
Injector
|
||||
};
|
||||
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false);
|
||||
|
||||
virtual int parseDataAndHandleDroppedPackets(const QByteArray& packet, int packetsSkipped) = 0;
|
||||
|
||||
int parsePositionalData(const QByteArray& positionalByteArray);
|
||||
int parseListenModeData(const QByteArray& listenModeByteArray);
|
||||
|
||||
PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false);
|
||||
|
||||
virtual AudioStreamStats getAudioStreamStats() const;
|
||||
|
||||
void updateNextOutputTrailingLoudness();
|
||||
float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; }
|
||||
|
||||
bool shouldBeAddedToMix();
|
||||
|
||||
bool willBeAddedToMix() const { return _willBeAddedToMix; }
|
||||
void setWillBeAddedToMix(bool willBeAddedToMix) { _willBeAddedToMix = willBeAddedToMix; }
|
||||
|
||||
|
||||
bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; }
|
||||
|
||||
bool isStereo() const { return _isStereo; }
|
||||
|
||||
PositionalAudioRingBuffer::Type getType() const { return _type; }
|
||||
const glm::vec3& getPosition() const { return _position; }
|
||||
const glm::quat& getOrientation() const { return _orientation; }
|
||||
|
||||
AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; }
|
||||
|
||||
void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; }
|
||||
|
||||
int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; }
|
||||
|
||||
const MovingMinMaxAvg<quint64>& getInterframeTimeGapStatsForStatsPacket() const { return _interframeTimeGapStatsForStatsPacket; }
|
||||
|
||||
int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked
|
||||
int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; }
|
||||
double getFramesAvailableAverage() const { return _framesAvailableStats.getWindowAverage(); }
|
||||
|
||||
int getConsecutiveNotMixedCount() const { return _consecutiveNotMixedCount; }
|
||||
int getStarveCount() const { return _starveCount; }
|
||||
int getSilentFramesDropped() const { return _silentFramesDropped; }
|
||||
|
||||
protected:
|
||||
// disallow copying of PositionalAudioRingBuffer objects
|
||||
PositionalAudioRingBuffer(const PositionalAudioRingBuffer&);
|
||||
PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&);
|
||||
|
||||
void frameReceivedUpdateTimingStats();
|
||||
void addDroppableSilentSamples(int numSilentSamples);
|
||||
|
||||
PositionalAudioRingBuffer::Type _type;
|
||||
/// parses the info between the seq num and the audio data in the network packet and calculates
|
||||
/// how many audio samples this packet contains
|
||||
virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0;
|
||||
|
||||
/// parses the audio data in the network packet
|
||||
virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) = 0;
|
||||
|
||||
int parsePositionalData(const QByteArray& positionalByteArray);
|
||||
|
||||
protected:
|
||||
Type _type;
|
||||
glm::vec3 _position;
|
||||
glm::quat _orientation;
|
||||
bool _willBeAddedToMix;
|
||||
|
||||
bool _shouldLoopbackForNode;
|
||||
bool _shouldOutputStarveDebug;
|
||||
bool _isStereo;
|
||||
|
||||
|
||||
float _nextOutputTrailingLoudness;
|
||||
AABox* _listenerUnattenuatedZone;
|
||||
|
||||
quint64 _lastFrameReceivedTime;
|
||||
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForJitterCalc;
|
||||
MovingMinMaxAvg<quint64> _interframeTimeGapStatsForStatsPacket;
|
||||
MovingMinMaxAvg<int> _framesAvailableStats;
|
||||
|
||||
int _desiredJitterBufferFrames;
|
||||
bool _dynamicJitterBuffers;
|
||||
|
||||
// extra stats
|
||||
int _consecutiveNotMixedCount;
|
||||
int _starveCount;
|
||||
int _silentFramesDropped;
|
||||
};
|
||||
|
||||
#endif // hifi_PositionalAudioRingBuffer_h
|
||||
|
|
|
|||
Loading…
Reference in a new issue