From 4825457f4d0895799b4ea5d20e4fbbc7f2cb8d17 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 28 Jul 2014 16:49:53 -0700 Subject: [PATCH 01/34] silent audio packet type generalized --- .../src/audio/AvatarAudioStream.cpp | 25 ++------- .../src/audio/AvatarAudioStream.h | 1 - interface/src/Audio.cpp | 54 +++++++++++-------- libraries/audio/src/InboundAudioStream.cpp | 45 ++++++++++++---- libraries/audio/src/InboundAudioStream.h | 2 +- libraries/audio/src/InjectedAudioStream.cpp | 4 -- libraries/audio/src/InjectedAudioStream.h | 1 - libraries/audio/src/PositionalAudioStream.h | 7 --- libraries/networking/src/PacketHeaders.cpp | 3 +- libraries/script-engine/src/ScriptEngine.cpp | 26 ++++----- 10 files changed, 87 insertions(+), 81 deletions(-) diff --git a/assignment-client/src/audio/AvatarAudioStream.cpp b/assignment-client/src/audio/AvatarAudioStream.cpp index c6a7d31468..509f2889f2 100644 --- a/assignment-client/src/audio/AvatarAudioStream.cpp +++ b/assignment-client/src/audio/AvatarAudioStream.cpp @@ -38,26 +38,9 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray& // read the positional data readBytes += parsePositionalData(packetAfterSeqNum.mid(readBytes)); - if (type == PacketTypeSilentAudioFrame) { - int16_t numSilentSamples; - memcpy(&numSilentSamples, packetAfterSeqNum.data() + readBytes, sizeof(int16_t)); - readBytes += sizeof(int16_t); - - numAudioSamples = numSilentSamples; - } else { - int numAudioBytes = packetAfterSeqNum.size() - readBytes; - numAudioSamples = numAudioBytes / sizeof(int16_t); - } - return readBytes; -} - -int AvatarAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - int readBytes = 0; - if (type == PacketTypeSilentAudioFrame) { - writeDroppableSilentSamples(numAudioSamples); - } else { - // there is audio data to read - readBytes += _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); - } + // calculate how many samples are in this packet + int numAudioBytes = packetAfterSeqNum.size() - readBytes; + numAudioSamples = numAudioBytes / sizeof(int16_t); + return readBytes; } diff --git a/assignment-client/src/audio/AvatarAudioStream.h b/assignment-client/src/audio/AvatarAudioStream.h index de7920c278..e6735d4975 100644 --- a/assignment-client/src/audio/AvatarAudioStream.h +++ b/assignment-client/src/audio/AvatarAudioStream.h @@ -26,7 +26,6 @@ private: AvatarAudioStream& operator= (const AvatarAudioStream&); int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); }; #endif // hifi_AvatarAudioStream_h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e830e5f6d4..6b0ce30de4 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -453,9 +453,12 @@ void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); - static int leadingBytes = numBytesPacketHeader + sizeof(quint16) + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); - static int16_t* networkAudioSamples = (int16_t*) (audioDataPacket + leadingBytes); + // NOTE: we assume PacketTypeMicrophoneAudioWithEcho has same size headers as + // PacketTypeMicrophoneAudioNoEcho. If not, then networkAudioSamples will be pointing to the wrong place for writing + // audio samples with echo. + static int leadingBytes = numBytesPacketHeader + sizeof(quint16) + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); + static int16_t* networkAudioSamples = (int16_t*)(audioDataPacket + leadingBytes); float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(_numInputCallbackBytes); @@ -666,19 +669,13 @@ void Audio::handleAudioInput() { glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); quint8 isStereo = _isStereoInput ? 1 : 0; - - int numAudioBytes = 0; - + + int numPacketBytes = 0; + PacketType packetType; if (_lastInputLoudness == 0) { packetType = PacketTypeSilentAudioFrame; - - // we need to indicate how many silent samples this is to the audio mixer - networkAudioSamples[0] = numNetworkSamples; - numAudioBytes = sizeof(int16_t); } else { - numAudioBytes = numNetworkBytes; - if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { packetType = PacketTypeMicrophoneAudioWithEcho; } else { @@ -687,27 +684,38 @@ void Audio::handleAudioInput() { } char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); - + // pack sequence number memcpy(currentPacketPtr, &_outgoingAvatarAudioSequenceNumber, sizeof(quint16)); currentPacketPtr += sizeof(quint16); - // set the mono/stereo byte - *currentPacketPtr++ = isStereo; + if (packetType == PacketTypeSilentAudioFrame) { + // pack num silent samples + quint16 numSilentSamples = numNetworkSamples; + memcpy(currentPacketPtr, &numSilentSamples, sizeof(quint16)); + currentPacketPtr += sizeof(quint16); + } else { + // set the mono/stereo byte + *currentPacketPtr++ = isStereo; - // memcpy the three float positions - memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); - currentPacketPtr += (sizeof(headPosition)); + // memcpy the three float positions + memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); + currentPacketPtr += (sizeof(headPosition)); - // memcpy our orientation - memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); - currentPacketPtr += sizeof(headOrientation); - - nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); + // memcpy our orientation + memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation)); + currentPacketPtr += sizeof(headOrientation); + + // audio samples have already been packed (written to networkAudioSamples) + currentPacketPtr += numNetworkBytes; + } + + int packetBytes = currentPacketPtr - audioDataPacket; + nodeList->writeDatagram(audioDataPacket, packetBytes, audioMixer); _outgoingAvatarAudioSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) - .updateValue(numAudioBytes + leadingBytes); + .updateValue(packetBytes); } delete[] inputAudioSamples; } diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index bfaa4c6d63..85ad10081c 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -67,34 +67,52 @@ int InboundAudioStream::parseData(const QByteArray& packet) { // parse header int numBytesHeader = numBytesForPacketHeader(packet); - const char* sequenceAt = packet.constData() + numBytesHeader; + const char* dataAt = packet.constData() + numBytesHeader; int readBytes = numBytesHeader; // parse sequence number and track it - quint16 sequence = *(reinterpret_cast(sequenceAt)); + quint16 sequence = *(reinterpret_cast(dataAt)); + dataAt += sizeof(quint16); readBytes += sizeof(quint16); SequenceNumberStats::ArrivalInfo arrivalInfo = frameReceivedUpdateNetworkStats(sequence, senderUUID); - // TODO: handle generalized silent packet here????? - - // parse the info after the seq number and before the audio data.(the stream properties) int numAudioSamples; - readBytes += parseStreamProperties(packetType, packet.mid(readBytes), numAudioSamples); + + if (packetType == PacketTypeSilentAudioFrame) { + // this is a general silent packet; parse the number of silent samples + quint16 numSilentSamples = *(reinterpret_cast(dataAt)); + dataAt += sizeof(quint16); + readBytes += sizeof(quint16); + + numAudioSamples = numSilentSamples; + } else { + // parse the info after the seq number and before the audio data (the stream properties) + readBytes += parseStreamProperties(packetType, packet.mid(readBytes), numAudioSamples); + } // handle this packet based on its arrival status. - // For now, late packets are ignored. It may be good in the future to insert the late audio frame - // into the ring buffer to fill in the missing frame if it hasn't been mixed yet. switch (arrivalInfo._status) { case SequenceNumberStats::Early: { + // Packet is early; write droppable silent samples for each of the skipped packets. + // NOTE: we assume that each dropped packet contains the same number of samples + // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; writeSamplesForDroppedPackets(packetsDropped * numAudioSamples); + // fall through to OnTime case } case SequenceNumberStats::OnTime: { - readBytes += parseAudioData(packetType, packet.mid(readBytes), numAudioSamples); + // Packet is on time; parse its data to the ringbuffer + if (packetType == PacketTypeSilentAudioFrame) { + writeDroppableSilentSamples(numAudioSamples); + } else { + readBytes += parseAudioData(packetType, packet.mid(readBytes), numAudioSamples); + } break; } default: { + // For now, late packets are ignored. It may be good in the future to insert the late audio packet data + // into the ring buffer to fill in the missing frame if it hasn't been mixed yet. break; } } @@ -108,6 +126,10 @@ int InboundAudioStream::parseData(const QByteArray& packet) { return readBytes; } +int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { + return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); +} + bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { int numSamplesRequested = numFrames * _ringBuffer.getNumFrameSamples(); if (_isStarved) { @@ -119,6 +141,8 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { // we have enough samples to pop, so we're good to mix _lastPopOutput = _ringBuffer.nextOutput(); _ringBuffer.shiftReadPosition(numSamplesRequested); + + _framesAvailableStats.update(_ringBuffer.framesAvailable()); _hasStarted = true; _lastPopSucceeded = true; @@ -132,6 +156,7 @@ bool InboundAudioStream::popFrames(int numFrames, bool starveOnFail) { _lastPopSucceeded = false; } } + return _lastPopSucceeded; } @@ -145,6 +170,8 @@ void InboundAudioStream::starved() { _isStarved = true; _consecutiveNotMixedCount = 0; _starveCount++; + + _framesAvailableStats.reset(); } void InboundAudioStream::overrideDesiredJitterBufferFramesTo(int desired) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 958491bca1..c679bdeedd 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -113,7 +113,7 @@ protected: virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0; /// parses the audio data in the network packet - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) = 0; + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); int writeDroppableSilentSamples(int numSilentSamples); diff --git a/libraries/audio/src/InjectedAudioStream.cpp b/libraries/audio/src/InjectedAudioStream.cpp index 4c23fbd823..3afcc50fdb 100644 --- a/libraries/audio/src/InjectedAudioStream.cpp +++ b/libraries/audio/src/InjectedAudioStream.cpp @@ -58,10 +58,6 @@ int InjectedAudioStream::parseStreamProperties(PacketType type, const QByteArray return packetStream.device()->pos(); } -int InjectedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); -} - AudioStreamStats InjectedAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats = PositionalAudioStream::getAudioStreamStats(); streamStats._streamIdentifier = _streamIdentifier; diff --git a/libraries/audio/src/InjectedAudioStream.h b/libraries/audio/src/InjectedAudioStream.h index b92736b0ba..cf750c6440 100644 --- a/libraries/audio/src/InjectedAudioStream.h +++ b/libraries/audio/src/InjectedAudioStream.h @@ -32,7 +32,6 @@ private: AudioStreamStats getAudioStreamStats() const; int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); const QUuid _streamIdentifier; float _radius; diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index 06835b93a8..66db79c0db 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -50,13 +50,6 @@ protected: PositionalAudioStream(const PositionalAudioStream&); PositionalAudioStream& operator= (const PositionalAudioStream&); - /// parses the info between the seq num and the audio data in the network packet and calculates - /// how many audio samples this packet contains - virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0; - - /// parses the audio data in the network packet - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) = 0; - int parsePositionalData(const QByteArray& positionalByteArray); protected: diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index f17715ddfe..a074c45095 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -49,8 +49,9 @@ PacketVersion versionForPacketType(PacketType type) { switch (type) { case PacketTypeMicrophoneAudioNoEcho: case PacketTypeMicrophoneAudioWithEcho: - case PacketTypeSilentAudioFrame: return 2; + case PacketTypeSilentAudioFrame: + return 3; case PacketTypeMixedAudio: return 1; case PacketTypeAvatarData: diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index fab21ea928..c67a499dcb 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -490,14 +490,6 @@ void ScriptEngine::run() { // pack a placeholder value for sequence number for now, will be packed when destination node is known int numPreSequenceNumberBytes = audioPacket.size(); packetStream << (quint16) 0; - - // assume scripted avatar audio is mono and set channel flag to zero - packetStream << (quint8) 0; - - // use the orientation and position of this avatar for the source of this audio - packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); - glm::quat headOrientation = _avatarData->getHeadOrientation(); - packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); if (silentFrame) { if (!_isListeningToAudioStream) { @@ -507,12 +499,20 @@ void ScriptEngine::run() { // write the number of silent samples so the audio-mixer can uphold timing packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); - } else if (nextSoundOutput) { - // write the raw audio data - packetStream.writeRawData(reinterpret_cast(nextSoundOutput), - numAvailableSamples * sizeof(int16_t)); - } + } else if (nextSoundOutput) { + // assume scripted avatar audio is mono and set channel flag to zero + packetStream << (quint8)0; + + // use the orientation and position of this avatar for the source of this audio + packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); + glm::quat headOrientation = _avatarData->getHeadOrientation(); + packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); + + // write the raw audio data + packetStream.writeRawData(reinterpret_cast(nextSoundOutput), numAvailableSamples * sizeof(int16_t)); + } + // write audio packet to AudioMixer nodes NodeList* nodeList = NodeList::getInstance(); foreach(const SharedNodePointer& node, nodeList->getNodeHash()) { From a3b44a6a7342f3d42ed31654fa693971ae345c77 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 29 Jul 2014 10:08:55 -0700 Subject: [PATCH 02/34] downstream silent packets seem to be working --- assignment-client/src/Agent.cpp | 2 +- assignment-client/src/audio/AudioMixer.cpp | 65 ++++++++++++------- assignment-client/src/audio/AudioMixer.h | 4 +- .../src/audio/AudioMixerClientData.cpp | 1 + interface/src/DatagramProcessor.cpp | 1 + libraries/audio/src/AudioRingBuffer.cpp | 27 +++++--- libraries/audio/src/AudioRingBuffer.h | 7 +- libraries/audio/src/InboundAudioStream.cpp | 4 ++ libraries/audio/src/InboundAudioStream.h | 2 + libraries/audio/src/PositionalAudioStream.cpp | 6 -- libraries/audio/src/PositionalAudioStream.h | 2 - 11 files changed, 76 insertions(+), 45 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index d4da989198..9150e24303 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -146,7 +146,7 @@ void Agent::readPendingDatagrams() { _voxelViewer.processDatagram(mutablePacket, sourceNode); } - } else if (datagramPacketType == PacketTypeMixedAudio) { + } else if (datagramPacketType == PacketTypeMixedAudio || datagramPacketType == PacketTypeSilentAudioFrame) { _receivedAudioStream.parseData(receivedPacket); diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d3ec39ace1..9c7f51ef15 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -93,7 +93,7 @@ const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; const float ATTENUATION_AMOUNT_PER_DOUBLING_IN_DISTANCE = 0.18f; const float ATTENUATION_EPSILON_DISTANCE = 0.1f; -void AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, +int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, AvatarAudioStream* listeningNodeStream) { float bearingRelativeAngleToSource = 0.0f; float attenuationCoefficient = 1.0f; @@ -116,7 +116,7 @@ void AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* if (streamToAdd->getNextOutputTrailingLoudness() / distanceBetween <= _minAudibilityThreshold) { // according to mixer performance we have decided this does not get to be mixed in // bail out - return; + return 0; } ++_sumMixes; @@ -261,36 +261,39 @@ void AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } } + + return 1; } -void AudioMixer::prepareMixForListeningNode(Node* node) { +int AudioMixer::prepareMixForListeningNode(Node* node) { AvatarAudioStream* nodeAudioStream = ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioStream(); - + // zero out the client mix for this node memset(_clientSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_STEREO); // loop through all other nodes that have sufficient audio to mix + int streamsMixed = 0; foreach (const SharedNodePointer& otherNode, NodeList::getInstance()->getNodeHash()) { if (otherNode->getLinkedData()) { - AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); // enumerate the ARBs attached to the otherNode and add all that should be added to mix const QHash& otherNodeAudioStreams = otherNodeClientData->getAudioStreams(); QHash::ConstIterator i; - for (i = otherNodeAudioStreams.begin(); i != otherNodeAudioStreams.constEnd(); i++) { + for (i = otherNodeAudioStreams.constBegin(); i != otherNodeAudioStreams.constEnd(); i++) { PositionalAudioStream* otherNodeStream = i.value(); - + if ((*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) && otherNodeStream->lastPopSucceeded() - && otherNodeStream->getNextOutputTrailingLoudness() > 0.0f) { + && otherNodeStream->getLastPopOutputFrameLoudness() > 0.0f) { - addStreamToMixForListeningNodeWithStream(otherNodeStream, nodeAudioStream); + streamsMixed += addStreamToMixForListeningNodeWithStream(otherNodeStream, nodeAudioStream); } } } } + return streamsMixed; } void AudioMixer::readPendingDatagrams() { @@ -474,9 +477,8 @@ void AudioMixer::run() { int nextFrame = 0; QElapsedTimer timer; timer.start(); - - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(quint16) - + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; + + char clientMixBuffer[MAX_PACKET_SIZE]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -555,20 +557,37 @@ void AudioMixer::run() { if (node->getType() == NodeType::Agent && ((AudioMixerClientData*)node->getLinkedData())->getAvatarAudioStream()) { - prepareMixForListeningNode(node.data()); + int streamsMixed = prepareMixForListeningNode(node.data()); - // pack header - int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); - char* dataAt = clientMixBuffer + numBytesPacketHeader; + char* dataAt; + if (streamsMixed > 0) { + // pack header + int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); + dataAt = clientMixBuffer + numBytesPacketHeader; - // pack sequence number - quint16 sequence = nodeData->getOutgoingSequenceNumber(); - memcpy(dataAt, &sequence, sizeof(quint16)); - dataAt += sizeof(quint16); + // pack sequence number + quint16 sequence = nodeData->getOutgoingSequenceNumber(); + memcpy(dataAt, &sequence, sizeof(quint16)); + dataAt += sizeof(quint16); - // pack mixed audio samples - memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; + // pack mixed audio samples + memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; + } else { + // pack header + int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeSilentAudioFrame); + dataAt = clientMixBuffer + numBytesPacketHeader; + + // pack sequence number + quint16 sequence = nodeData->getOutgoingSequenceNumber(); + memcpy(dataAt, &sequence, sizeof(quint16)); + dataAt += sizeof(quint16); + + // pack number of silent audio samples + quint16 numSilentSamples = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; + memcpy(dataAt, &numSilentSamples, sizeof(quint16)); + dataAt += sizeof(quint16); + } // send mixed audio packet nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index bfdb49f393..639b01b78a 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -41,11 +41,11 @@ public slots: private: /// adds one stream to the mix for a listening node - void addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, + int addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, AvatarAudioStream* listeningNodeStream); /// prepares and sends a mix to one Node - void prepareMixForListeningNode(Node* node); + int prepareMixForListeningNode(Node* node); // client samples capacity is larger than what will be sent to optimize mixing // we are MMX adding 4 samples at a time so we need client samples to have an extra 4 diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 17e46f3692..39a789580b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -101,6 +101,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { void AudioMixerClientData::audioStreamsPopFrameForMixing() { QHash::ConstIterator i; for (i = _audioStreams.constBegin(); i != _audioStreams.constEnd(); i++) { + i.value()->updateNextOutputTrailingLoudness(); i.value()->popFrames(1); } } diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index 8fda094f42..9a31306a51 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -48,6 +48,7 @@ void DatagramProcessor::processDatagrams() { // only process this packet if we have a match on the packet version switch (packetTypeForPacket(incomingPacket)) { case PacketTypeMixedAudio: + case PacketTypeSilentAudioFrame: QMetaObject::invokeMethod(&application->_audio, "addReceivedAudioToStream", Qt::QueuedConnection, Q_ARG(QByteArray, incomingPacket)); break; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 8dbc90883b..5042cc7388 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -218,17 +218,24 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int } } -float AudioRingBuffer::getNextOutputFrameLoudness() const { +float AudioRingBuffer::getFrameLoudness(const int16_t* frameStart) const { float loudness = 0.0f; - int16_t* sampleAt = _nextOutput; - int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1; - if (samplesAvailable() >= _numFrameSamples) { - for (int i = 0; i < _numFrameSamples; ++i) { - loudness += fabsf(*sampleAt); - sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1; - } - loudness /= _numFrameSamples; - loudness /= MAX_SAMPLE_VALUE; + const int16_t* sampleAt = frameStart; + const int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1; + for (int i = 0; i < _numFrameSamples; ++i) { + loudness += fabsf(*sampleAt); + sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1; } + loudness /= _numFrameSamples; + loudness /= MAX_SAMPLE_VALUE; + return loudness; } + +float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { + return getFrameLoudness(&(*frameStart)); +} + +float AudioRingBuffer::getNextOutputFrameLoudness() const { + return getFrameLoudness(_nextOutput); +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 824b197c93..c8be4cc420 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -75,6 +75,10 @@ public: int getOverflowCount() const { return _overflowCount; } /// how many times has the ring buffer has overwritten old data int addSilentFrame(int numSilentSamples); + +private: + float getFrameLoudness(const int16_t* frameStart) const; + protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); @@ -110,7 +114,7 @@ public: bool operator==(const ConstIterator& rhs) { return _at == rhs._at; } bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; } - int16_t operator*() { return *_at; } + const int16_t& operator*() { return *_at; } ConstIterator& operator=(const ConstIterator& rhs) { _capacity = rhs._capacity; @@ -179,6 +183,7 @@ public: }; ConstIterator nextOutput() const { return ConstIterator(_buffer, _sampleCapacity, _nextOutput); } + float getFrameLoudness(ConstIterator frameStart) const; }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 85ad10081c..6dd16fb88f 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -174,6 +174,10 @@ void InboundAudioStream::starved() { _framesAvailableStats.reset(); } +float InboundAudioStream::getLastPopOutputFrameLoudness() const { + return _ringBuffer.getFrameLoudness(_lastPopOutput); +} + void InboundAudioStream::overrideDesiredJitterBufferFramesTo(int desired) { _dynamicJitterBuffersOverride = true; _desiredJitterBufferFrames = clampDesiredJitterBufferFramesValue(desired); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index c679bdeedd..a9204a344c 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -80,6 +80,8 @@ public: /// returns the desired number of jitter buffer frames using Freddy's method int getCalculatedJitterBufferFramesUsingMaxGap() const { return _calculatedJitterBufferFramesUsingMaxGap; } + + float getLastPopOutputFrameLoudness() const; int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } int getNumFrameSamples() const { return _ringBuffer.getNumFrameSamples(); } diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index b50e339185..bd21e844af 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -34,12 +34,6 @@ PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, b { } -int PositionalAudioStream::parseData(const QByteArray& packet) { - int bytesRead = InboundAudioStream::parseData(packet); - updateNextOutputTrailingLoudness(); - return bytesRead; -} - void PositionalAudioStream::updateNextOutputTrailingLoudness() { float nextLoudness = _ringBuffer.getNextOutputFrameLoudness(); diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index 66db79c0db..9d56ff3e8d 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -29,8 +29,6 @@ public: PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); - int parseData(const QByteArray& packet); - virtual AudioStreamStats getAudioStreamStats() const; void updateNextOutputTrailingLoudness(); From 357ba921815a9176952eac9745ac4b92bcc4164f Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 7 Aug 2014 12:41:09 -0700 Subject: [PATCH 03/34] working towards more dials for InboundAdioStream --- .../src/audio/AudioMixerClientData.cpp | 4 +- interface/src/Audio.cpp | 4 +- libraries/audio/src/InboundAudioStream.cpp | 63 ++++--- libraries/audio/src/InboundAudioStream.h | 82 ++++++--- libraries/shared/src/MovingEvent.h | 36 ++++ libraries/shared/src/MovingMinMaxAvg.h | 173 ++++++++++-------- .../shared/src/MovingPercentile - Copy.cpp | 61 ++++++ libraries/shared/src/RingBufferHistory.h | 92 +++++++++- tests/shared/src/main.cpp | 1 + 9 files changed, 383 insertions(+), 133 deletions(-) create mode 100644 libraries/shared/src/MovingEvent.h create mode 100644 libraries/shared/src/MovingPercentile - Copy.cpp diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index fb805e11d8..ecb4d29171 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -184,7 +184,9 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // pack the calculated number of stream stats for (int i = 0; i < numStreamStatsToPack; i++) { - AudioStreamStats streamStats = audioStreamsIterator.value()->updateSeqHistoryAndGetAudioStreamStats(); + PositionalAudioStream* stream = audioStreamsIterator.value(); + stream->perSecondCallbackForUpdatingStats(); + AudioStreamStats streamStats = stream->getAudioStreamStats(); memcpy(dataAt, &streamStats, sizeof(AudioStreamStats)); dataAt += sizeof(AudioStreamStats); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 2cba22b630..6ff053e5db 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -846,6 +846,8 @@ void Audio::sendDownstreamAudioStatsPacket() { _audioOutputMsecsUnplayedStats.update(getAudioOutputMsecsUnplayed()); + _receivedAudioStream.perSecondCallbackForUpdatingStats(); + char packet[MAX_PACKET_SIZE]; // pack header @@ -863,7 +865,7 @@ void Audio::sendDownstreamAudioStatsPacket() { dataAt += sizeof(quint16); // pack downstream audio stream stats - AudioStreamStats stats = _receivedAudioStream.updateSeqHistoryAndGetAudioStreamStats(); + AudioStreamStats stats = _receivedAudioStream.getAudioStreamStats(); memcpy(dataAt, &stats, sizeof(AudioStreamStats)); dataAt += sizeof(AudioStreamStats); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 0cd9be4a18..f59070ee83 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -12,30 +12,35 @@ #include "InboundAudioStream.h" #include "PacketHeaders.h" -InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, - bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc) : +const int STARVE_HISTORY_CAPACITY = 50; + +InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings) : _ringBuffer(numFrameSamples, false, numFramesCapacity), _lastPopSucceeded(false), _lastPopOutput(), - _dynamicJitterBuffers(dynamicJitterBuffers), - _staticDesiredJitterBufferFrames(staticDesiredJitterBufferFrames), - _useStDevForJitterCalc(useStDevForJitterCalc), + _dynamicJitterBuffers(settings._dynamicJitterBuffers), + _staticDesiredJitterBufferFrames(settings._staticDesiredJitterBufferFrames), + _useStDevForJitterCalc(settings._useStDevForJitterCalc), _calculatedJitterBufferFramesUsingMaxGap(0), _calculatedJitterBufferFramesUsingStDev(0), - _desiredJitterBufferFrames(dynamicJitterBuffers ? 1 : staticDesiredJitterBufferFrames), - _maxFramesOverDesired(maxFramesOverDesired), + _desiredJitterBufferFrames(settings._dynamicJitterBuffers ? 1 : settings._staticDesiredJitterBufferFrames), + _maxFramesOverDesired(settings._maxFramesOverDesired), _isStarved(true), _hasStarted(false), _consecutiveNotMixedCount(0), _starveCount(0), _silentFramesDropped(0), _oldFramesDropped(0), - _incomingSequenceNumberStats(INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS), + _incomingSequenceNumberStats(STATS_FOR_STATS_PACKET_WINDOW_SECONDS), _lastFrameReceivedTime(0), - _interframeTimeGapStatsForJitterCalc(TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES, TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS), - _interframeTimeGapStatsForStatsPacket(TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES, TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS), + _timeGapStatsForDesiredCalcOnTooManyStarves(0, settings._windowSecondsForDesiredCalcOnTooManyStarves), + _stdevStatsForDesiredCalcOnTooManyStarves(), + _timeGapStatsForDesiredReduction(0, settings._windowSecondsForDesiredReduction), + _starveHistoryWindowSeconds(settings._windowSecondsForDesiredCalcOnTooManyStarves), + _starveHistory(STARVE_HISTORY_CAPACITY), _framesAvailableStat(), - _currentJitterBufferFrames(0) + _currentJitterBufferFrames(0), + _timeGapStatsForStatsPacket(0, STATS_FOR_STATS_PACKET_WINDOW_SECONDS) { } @@ -58,10 +63,13 @@ void InboundAudioStream::resetStats() { _oldFramesDropped = 0; _incomingSequenceNumberStats.reset(); _lastFrameReceivedTime = 0; - _interframeTimeGapStatsForJitterCalc.reset(); - _interframeTimeGapStatsForStatsPacket.reset(); + _timeGapStatsForDesiredCalcOnTooManyStarves.reset(); + _stdevStatsForDesiredCalcOnTooManyStarves = StDev(); + _timeGapStatsForDesiredReduction.reset(); + _starveHistory.clear(); _framesAvailableStat.reset(); _currentJitterBufferFrames = 0; + _timeGapStatsForStatsPacket.reset(); } void InboundAudioStream::clearBuffer() { @@ -70,6 +78,13 @@ void InboundAudioStream::clearBuffer() { _currentJitterBufferFrames = 0; } +void InboundAudioStream::perSecondCallbackForUpdatingStats() { + _incomingSequenceNumberStats.pushStatsToHistory(); + _timeGapStatsForDesiredCalcOnTooManyStarves.currentIntervalComplete(); + _timeGapStatsForDesiredReduction.currentIntervalComplete(); + _timeGapStatsForStatsPacket.currentIntervalComplete(); +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); } @@ -247,14 +262,14 @@ int InboundAudioStream::clampDesiredJitterBufferFramesValue(int desired) const { } void InboundAudioStream::frameReceivedUpdateTimingStats() { - + /* // update our timegap stats and desired jitter buffer frames if necessary // discard the first few packets we receive since they usually have gaps that aren't represensative of normal jitter const int NUM_INITIAL_PACKETS_DISCARD = 3; quint64 now = usecTimestampNow(); if (_incomingSequenceNumberStats.getReceived() > NUM_INITIAL_PACKETS_DISCARD) { quint64 gap = now - _lastFrameReceivedTime; - _interframeTimeGapStatsForStatsPacket.update(gap); + _timeGapStatsForStatsPacket.update(gap); const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; @@ -282,7 +297,7 @@ void InboundAudioStream::frameReceivedUpdateTimingStats() { } } } - _lastFrameReceivedTime = now; + _lastFrameReceivedTime = now;*/ } int InboundAudioStream::writeDroppableSilentSamples(int numSilentSamples) { @@ -318,12 +333,12 @@ int InboundAudioStream::writeSamplesForDroppedPackets(int numSamples) { AudioStreamStats InboundAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats; - streamStats._timeGapMin = _interframeTimeGapStatsForStatsPacket.getMin(); - streamStats._timeGapMax = _interframeTimeGapStatsForStatsPacket.getMax(); - streamStats._timeGapAverage = _interframeTimeGapStatsForStatsPacket.getAverage(); - streamStats._timeGapWindowMin = _interframeTimeGapStatsForStatsPacket.getWindowMin(); - streamStats._timeGapWindowMax = _interframeTimeGapStatsForStatsPacket.getWindowMax(); - streamStats._timeGapWindowAverage = _interframeTimeGapStatsForStatsPacket.getWindowAverage(); + streamStats._timeGapMin = _timeGapStatsForStatsPacket.getMin(); + streamStats._timeGapMax = _timeGapStatsForStatsPacket.getMax(); + streamStats._timeGapAverage = _timeGapStatsForStatsPacket.getAverage(); + streamStats._timeGapWindowMin = _timeGapStatsForStatsPacket.getWindowMin(); + streamStats._timeGapWindowMax = _timeGapStatsForStatsPacket.getWindowMax(); + streamStats._timeGapWindowAverage = _timeGapStatsForStatsPacket.getWindowAverage(); streamStats._framesAvailable = _ringBuffer.framesAvailable(); streamStats._framesAvailableAverage = _framesAvailableStat.getAverage(); @@ -339,7 +354,3 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const { return streamStats; } -AudioStreamStats InboundAudioStream::updateSeqHistoryAndGetAudioStreamStats() { - _incomingSequenceNumberStats.pushStatsToHistory(); - return getAudioStreamStats(); -} diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index b65d5c5de0..88760a35cb 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -22,40 +22,63 @@ #include "TimeWeightedAvg.h" // This adds some number of frames to the desired jitter buffer frames target we use when we're dropping frames. -// The larger this value is, the less aggressive we are about reducing the jitter buffer length. -// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames long when dropping frames, +// The larger this value is, the less frames we drop when attempting to reduce the jitter buffer length. +// Setting this to 0 will try to get the jitter buffer to be exactly _desiredJitterBufferFrames when dropping frames, // which could lead to a starve soon after. const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; -// the time gaps stats for _desiredJitterBufferFrames calculation -// will recalculate the max for the past 5000 samples every 500 samples -const int TIME_GAPS_FOR_JITTER_CALC_INTERVAL_SAMPLES = 500; -const int TIME_GAPS_FOR_JITTER_CALC_WINDOW_INTERVALS = 10; +// this controls the length of the window for stats used in the stats packet (not the stats used in +// _desiredJitterBufferFrames calculation) +const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30; -// the time gap stats for constructing AudioStreamStats will -// recalculate min/max/avg every ~1 second for the past ~30 seconds of time gap data -const int TIME_GAPS_FOR_STATS_PACKET_INTERVAL_SAMPLES = USECS_PER_SECOND / BUFFER_SEND_INTERVAL_USECS; -const int TIME_GAPS_FOR_STATS_PACKET_WINDOW_INTERVALS = 30; // this controls the window size of the time-weighted avg of frames available. Every time the window fills up, // _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset. const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND; -// the internal history buffer of the incoming seq stats will cover 30s to calculate -// packet loss % over last 30s -const int INCOMING_SEQ_STATS_HISTORY_LENGTH_SECONDS = 30; - const int INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10; const int DEFAULT_DESIRED_JITTER_BUFFER_FRAMES = 1; +const int DEFAULT_WINDOW_STARVE_THRESHOLD = 3; +const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES = 50; +const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION = 10; + class InboundAudioStream : public NodeData { Q_OBJECT public: - InboundAudioStream(int numFrameSamples, int numFramesCapacity, - bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, - bool useStDevForJitterCalc = false); + class Settings { + public: + Settings() + : _maxFramesOverDesired(DEFAULT_MAX_FRAMES_OVER_DESIRED), + _dynamicJitterBuffers(true), + _staticDesiredJitterBufferFrames(DEFAULT_DESIRED_JITTER_BUFFER_FRAMES), + _useStDevForJitterCalc(false), + _windowStarveThreshold(DEFAULT_WINDOW_STARVE_THRESHOLD), + _windowSecondsForDesiredCalcOnTooManyStarves(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES), + _windowSecondsForDesiredReduction(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION) + {} + + // max number of frames over desired in the ringbuffer. + int _maxFramesOverDesired; + + // if false, _desiredJitterBufferFrames will always be _staticDesiredJitterBufferFrames. Otherwise, + // either fred or philip's method will be used to calculate _desiredJitterBufferFrames based on packet timegaps. + bool _dynamicJitterBuffers; + + // settings for static jitter buffer mode + int _staticDesiredJitterBufferFrames; + + // settings for dynamic jitter buffer mode + bool _useStDevForJitterCalc; // if true, philip's method is used. otherwise, fred's method is used. + int _windowStarveThreshold; + int _windowSecondsForDesiredCalcOnTooManyStarves; + int _windowSecondsForDesiredReduction; + }; + +public: + InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings); void reset(); void resetStats(); @@ -77,7 +100,8 @@ public: void setStaticDesiredJitterBufferFrames(int staticDesiredJitterBufferFrames); /// this function should be called once per second to ensure the seq num stats history spans ~30 seconds - AudioStreamStats updateSeqHistoryAndGetAudioStreamStats(); + //AudioStreamStats updateSeqHistoryAndGetAudioStreamStats(); + void setMaxFramesOverDesired(int maxFramesOverDesired) { _maxFramesOverDesired = maxFramesOverDesired; } @@ -110,6 +134,12 @@ public: int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); } +public slots: + /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers + /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. + /// If the stats are not used and dynamic jitter buffers is disabled, it's not necessary to call this function. + void perSecondCallbackForUpdatingStats(); + private: void frameReceivedUpdateTimingStats(); int clampDesiredJitterBufferFramesValue(int desired) const; @@ -169,15 +199,21 @@ protected: SequenceNumberStats _incomingSequenceNumberStats; quint64 _lastFrameReceivedTime; - MovingMinMaxAvg _interframeTimeGapStatsForJitterCalc; - StDev _stdev; - MovingMinMaxAvg _interframeTimeGapStatsForStatsPacket; - + MovingMinMaxAvg _timeGapStatsForDesiredCalcOnTooManyStarves; + StDev _stdevStatsForDesiredCalcOnTooManyStarves; + MovingMinMaxAvg _timeGapStatsForDesiredReduction; + + int _starveHistoryWindowSeconds; + RingBufferHistory _starveHistory; + TimeWeightedAvg _framesAvailableStat; - // this value is based on the time-weighted avg from _framesAvailableStat. it is only used for + // this value is periodically updated with the time-weighted avg from _framesAvailableStat. it is only used for // dropping silent frames right now. int _currentJitterBufferFrames; + + + MovingMinMaxAvg _timeGapStatsForStatsPacket; }; #endif // hifi_InboundAudioStream_h diff --git a/libraries/shared/src/MovingEvent.h b/libraries/shared/src/MovingEvent.h new file mode 100644 index 0000000000..284ed9d890 --- /dev/null +++ b/libraries/shared/src/MovingEvent.h @@ -0,0 +1,36 @@ +// +// MovingPercentile.h +// libraries/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MovingPercentile_h +#define hifi_MovingPercentile_h + +#include + +class MovingPercentile { + +public: + MovingPercentile(int numSamples, float percentile = 0.5f); + + void updatePercentile(float sample); + float getValueAtPercentile() const { return _valueAtPercentile; } + +private: + const int _numSamples; + const float _percentile; + + QList _samplesSorted; + QList _sampleIds; // incrementally assigned, is cyclic + int _newSampleId; + + int _indexOfPercentile; + float _valueAtPercentile; +}; + +#endif diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 734018b469..9ac9ec9006 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -17,45 +17,62 @@ #include "RingBufferHistory.h" template -class MovingMinMaxAvg { +class MinMaxAvg { +public: + MinMaxAvg() + : _min(std::numeric_limits::max()), + _max(std::numeric_limits::min()), + _average(0.0), + _samples(0) + {} + + void reset() { + _min = std::numeric_limits::max(); + _max = std::numeric_limits::min(); + _average = 0.0; + _samples = 0; + } + + void update(T sample) { + if (sample < _min) { + _min = sample; + } + if (sample > _max) { + _max = sample; + } + double totalSamples = _samples + 1; + _average = _average * ((double)_samples / totalSamples) + + (double)sample / totalSamples; + _samples++; + } + + void update(const MinMaxAvg& other) { + if (other._min < _min) { + _min = other._min; + } + if (other._max > _max) { + _max = other._max; + } + double totalSamples = _samples + other._samples; + _average = _average * ((double)_samples / totalSamples) + + other._average * ((double)other._samples / totalSamples); + _samples += other._samples; + } + + T getMin() const { return _min; } + T getMax() const { return _max; } + double getAverage() const { return _average; } + int getSamples() const { return _samples; } private: - class Stats { - public: - Stats() - : _min(std::numeric_limits::max()), - _max(std::numeric_limits::min()), - _average(0.0) {} - - void updateWithSample(T sample, int& numSamplesInAverage) { - if (sample < _min) { - _min = sample; - } - if (sample > _max) { - _max = sample; - } - _average = _average * ((double)numSamplesInAverage / (numSamplesInAverage + 1)) - + (double)sample / (numSamplesInAverage + 1); - numSamplesInAverage++; - } - - void updateWithOtherStats(const Stats& other, int& numStatsInAverage) { - if (other._min < _min) { - _min = other._min; - } - if (other._max > _max) { - _max = other._max; - } - _average = _average * ((double)numStatsInAverage / (numStatsInAverage + 1)) - + other._average / (numStatsInAverage + 1); - numStatsInAverage++; - } - - T _min; - T _max; - double _average; - }; + T _min; + T _max; + double _average; + int _samples; +}; +template +class MovingMinMaxAvg { public: // This class collects 3 stats (min, max, avg) over a moving window of samples. // The moving window contains _windowIntervals * _intervalLength samples. @@ -65,66 +82,72 @@ public: // this class with MovingMinMaxAvg(100, 50). If you want a moving min of the past 100 samples updated on every // new sample, instantiate this class with MovingMinMaxAvg(1, 100). + + /// use intervalLength = 0 to use in manual mode, where the currentIntervalComplete() function must + /// be called to complete an interval MovingMinMaxAvg(int intervalLength, int windowIntervals) : _intervalLength(intervalLength), _windowIntervals(windowIntervals), _overallStats(), - _samplesCollected(0), _windowStats(), - _existingSamplesInCurrentInterval(0), _currentIntervalStats(), _intervalStats(windowIntervals), _newStatsAvailable(false) {} void reset() { - _overallStats = Stats(); - _samplesCollected = 0; - _windowStats = Stats(); - _existingSamplesInCurrentInterval = 0; - _currentIntervalStats = Stats(); + _overallStats.reset(); + _windowStats.reset(); + _currentIntervalStats.reset(); _intervalStats.clear(); _newStatsAvailable = false; } void update(T newSample) { // update overall stats - _overallStats.updateWithSample(newSample, _samplesCollected); + _overallStats.update(newSample); // update the current interval stats - _currentIntervalStats.updateWithSample(newSample, _existingSamplesInCurrentInterval); + _currentIntervalStats.update(newSample); // if the current interval of samples is now full, record its stats into our past intervals' stats - if (_existingSamplesInCurrentInterval == _intervalLength) { - - // record current interval's stats, then reset them - _intervalStats.insert(_currentIntervalStats); - _currentIntervalStats = Stats(); - _existingSamplesInCurrentInterval = 0; - - // update the window's stats by combining the intervals' stats - typename RingBufferHistory::Iterator i = _intervalStats.begin(); - typename RingBufferHistory::Iterator end = _intervalStats.end(); - _windowStats = Stats(); - int intervalsIncludedInWindowStats = 0; - while (i != end) { - _windowStats.updateWithOtherStats(*i, intervalsIncludedInWindowStats); - i++; - } - - _newStatsAvailable = true; + // NOTE: if _intervalLength is 0 (manual mode), currentIntervalComplete() will not be called here. + if (_currentIntervalStats.getSamples() == _intervalLength) { + currentIntervalComplete(); } } + /// This function can be called to manually control when each interval ends. For example, if each interval + /// needs to last T seconds as opposed to N samples, this function should be called every T seconds. + void currentIntervalComplete() { + // record current interval's stats, then reset them + _intervalStats.insert(_currentIntervalStats); + _currentIntervalStats.reset(); + + // update the window's stats by combining the intervals' stats + typename RingBufferHistory< MinMaxAvg >::Iterator i = _intervalStats.begin(); + typename RingBufferHistory< MinMaxAvg >::Iterator end = _intervalStats.end(); + _windowStats.reset(); + while (i != end) { + _windowStats.update(*i); + ++i; + } + + _newStatsAvailable = true; + } + bool getNewStatsAvailableFlag() const { return _newStatsAvailable; } void clearNewStatsAvailableFlag() { _newStatsAvailable = false; } - T getMin() const { return _overallStats._min; } - T getMax() const { return _overallStats._max; } - double getAverage() const { return _overallStats._average; } - T getWindowMin() const { return _windowStats._min; } - T getWindowMax() const { return _windowStats._max; } - double getWindowAverage() const { return _windowStats._average; } + T getMin() const { return _overallStats.getMin(); } + T getMax() const { return _overallStats.getMax(); } + double getAverage() const { return _overallStats.getAverage(); } + T getWindowMin() const { return _windowStats.getMin(); } + T getWindowMax() const { return _windowStats.getMax(); } + double getWindowAverage() const { return _windowStats.getAverage(); } + + const MinMaxAvg& getOverallStats() const{ return _overallStats; } + const MinMaxAvg& getWindowStats() const{ return _windowStats; } bool isWindowFilled() const { return _intervalStats.isFilled(); } @@ -133,18 +156,16 @@ private: int _windowIntervals; // these are min/max/avg stats for all samples collected. - Stats _overallStats; - int _samplesCollected; + MinMaxAvg _overallStats; // these are the min/max/avg stats for the samples in the moving window - Stats _windowStats; - int _existingSamplesInCurrentInterval; + MinMaxAvg _windowStats; - // these are the min/max/avg stats for the current interval - Stats _currentIntervalStats; + // these are the min/max/avg stats for the samples in the current interval + MinMaxAvg _currentIntervalStats; // these are stored stats for the past intervals in the window - RingBufferHistory _intervalStats; + RingBufferHistory< MinMaxAvg > _intervalStats; bool _newStatsAvailable; }; diff --git a/libraries/shared/src/MovingPercentile - Copy.cpp b/libraries/shared/src/MovingPercentile - Copy.cpp new file mode 100644 index 0000000000..ec007b5c22 --- /dev/null +++ b/libraries/shared/src/MovingPercentile - Copy.cpp @@ -0,0 +1,61 @@ +// +// MovingPercentile.cpp +// libraries/shared/src +// +// Created by Yixin Wang on 6/4/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MovingPercentile.h" + +MovingPercentile::MovingPercentile(int numSamples, float percentile) + : _numSamples(numSamples), + _percentile(percentile), + _samplesSorted(), + _sampleIds(), + _newSampleId(0), + _indexOfPercentile(0), + _valueAtPercentile(0.0f) +{ +} + +void MovingPercentile::updatePercentile(float sample) { + + // insert the new sample into _samplesSorted + int newSampleIndex; + if (_samplesSorted.size() < _numSamples) { + // if not all samples have been filled yet, simply append it + newSampleIndex = _samplesSorted.size(); + _samplesSorted.append(sample); + _sampleIds.append(_newSampleId); + + // update _indexOfPercentile + float index = _percentile * (float)(_samplesSorted.size() - 1); + _indexOfPercentile = (int)(index + 0.5f); // round to int + } else { + // find index of sample with id = _newSampleId and replace it with new sample + newSampleIndex = _sampleIds.indexOf(_newSampleId); + _samplesSorted[newSampleIndex] = sample; + } + + // increment _newSampleId. cycles from 0 thru N-1 + _newSampleId = (_newSampleId == _numSamples - 1) ? 0 : _newSampleId + 1; + + // swap new sample with neighbors in _samplesSorted until it's in sorted order + // try swapping up first, then down. element will only be swapped one direction. + while (newSampleIndex < _samplesSorted.size() - 1 && sample > _samplesSorted[newSampleIndex + 1]) { + _samplesSorted.swap(newSampleIndex, newSampleIndex + 1); + _sampleIds.swap(newSampleIndex, newSampleIndex + 1); + newSampleIndex++; + } + while (newSampleIndex > 0 && sample < _samplesSorted[newSampleIndex - 1]) { + _samplesSorted.swap(newSampleIndex, newSampleIndex - 1); + _sampleIds.swap(newSampleIndex, newSampleIndex - 1); + newSampleIndex--; + } + + // find new value at percentile + _valueAtPercentile = _samplesSorted[_indexOfPercentile]; +} diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index 27a78c0055..339f390cd5 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -83,9 +83,14 @@ private: QVector _buffer; public: - class Iterator : public std::iterator < std::forward_iterator_tag, T > { + class Iterator : public std::iterator < std::random_access_iterator_tag, T > { public: - Iterator(T* bufferFirst, T* bufferLast, T* at) : _bufferFirst(bufferFirst), _bufferLast(bufferLast), _at(at) {} + Iterator(T* bufferFirst, T* bufferLast, T* newestAt, T* at) + : _bufferFirst(bufferFirst), + _bufferLast(bufferLast), + _bufferLength(bufferLast - bufferFirst + 1), + _newestAt(newestAt), + _at(at) {} bool operator==(const Iterator& rhs) { return _at == rhs._at; } bool operator!=(const Iterator& rhs) { return _at != rhs._at; } @@ -103,20 +108,95 @@ public: return tmp; } + Iterator& operator--() { + _at = (_at == _bufferLast) ? _bufferFirst : _at + 1; + return *this; + } + + Iterator operator--(int) { + Iterator tmp(*this); + --(*this); + return tmp; + } + + Iterator operator+(int add) { + Iterator sum(*this); + sum._at = atShiftedBy(add); + return sum; + } + + Iterator operator-(int sub) { + Iterator sum(*this); + sum._at = atShiftedBy(-sub); + return sum; + } + + Iterator& operator+=(int add) { + _at = atShiftedBy(add); + return *this; + } + + Iterator& operator-=(int sub) { + _at = atShiftedBy(-sub); + return *this; + } + + T& operator[](int i) { + return *(atShiftedBy(i)); + } + + bool operator<(const Iterator& rhs) { + return age() < rhs.age(); + } + + bool operator>(const Iterator& rhs) { + return age() > rhs.age(); + } + + bool operator<=(const Iterator& rhs) { + return age() < rhs.age(); + } + + bool operator>=(const Iterator& rhs) { + return age() >= rhs.age(); + } + + int operator-(const Iterator& rhs) { + return age() - rhs.age(); + } + private: - T* const _bufferFirst; - T* const _bufferLast; + T* atShiftedBy(int i) { // shifts i places towards _bufferFirst (towards older entries) + i = (_at - _bufferFirst - i) % _bufferLength; + if (i < 0) { + i += _bufferLength; + } + return _bufferFirst + i; + } + + int age() { + int age = _newestAt - _at; + if (age < 0) { + age += _bufferLength; + } + return age; + } + + T* _bufferFirst; + T* _bufferLast; + int _bufferLength; + T* _newestAt; T* _at; }; - Iterator begin() { return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[_newestEntryAtIndex]); } + Iterator begin() { return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[_newestEntryAtIndex], &_buffer[_newestEntryAtIndex]); } Iterator end() { int endAtIndex = _newestEntryAtIndex - _numEntries; if (endAtIndex < 0) { endAtIndex += _size; } - return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[endAtIndex]); + return Iterator(&_buffer.first(), &_buffer.last(), &_buffer[_newestEntryAtIndex], &_buffer[endAtIndex]); } }; diff --git a/tests/shared/src/main.cpp b/tests/shared/src/main.cpp index d4251eef7a..34ba515062 100644 --- a/tests/shared/src/main.cpp +++ b/tests/shared/src/main.cpp @@ -16,6 +16,7 @@ int main(int argc, char** argv) { MovingMinMaxAvgTests::runAllTests(); MovingPercentileTests::runAllTests(); AngularConstraintTests::runAllTests(); + printf("tests complete, press enter to exit\n"); getchar(); return 0; } From 1153a76ab1005686716b048113be0fb4d582b80d Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 7 Aug 2014 16:37:36 -0700 Subject: [PATCH 04/34] new option knobs added, no new behavior yet --- assignment-client/src/Agent.cpp | 7 +- assignment-client/src/audio/AudioMixer.cpp | 74 ++- assignment-client/src/audio/AudioMixer.h | 8 +- .../src/audio/AudioMixerClientData.cpp | 9 +- .../src/audio/AvatarAudioStream.cpp | 4 +- .../src/audio/AvatarAudioStream.h | 2 +- .../resources/web/settings/describe.json | 26 +- interface/src/Application.cpp | 9 +- interface/src/Audio.cpp | 7 +- interface/src/Audio.h | 5 +- interface/src/Menu.cpp | 23 +- interface/src/Menu.h | 9 +- interface/src/ui/PreferencesDialog.cpp | 35 +- interface/ui/preferencesDialog.ui | 471 +++++++++++++++++- libraries/audio/src/InboundAudioStream.cpp | 20 + libraries/audio/src/InboundAudioStream.h | 37 +- libraries/audio/src/InjectedAudioStream.cpp | 4 +- libraries/audio/src/InjectedAudioStream.h | 2 +- libraries/audio/src/MixedAudioStream.cpp | 4 +- libraries/audio/src/MixedAudioStream.h | 2 +- .../audio/src/MixedProcessedAudioStream.cpp | 4 +- .../audio/src/MixedProcessedAudioStream.h | 2 +- libraries/audio/src/PositionalAudioStream.cpp | 5 +- libraries/audio/src/PositionalAudioStream.h | 3 +- libraries/shared/src/MovingMinMaxAvg.h | 9 + libraries/shared/src/RingBufferHistory.h | 8 + 26 files changed, 680 insertions(+), 109 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index dbb1620252..dd48c9daa7 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -33,12 +33,17 @@ #include "Agent.h" +static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES = 10; + Agent::Agent(const QByteArray& packet) : ThreadedAssignment(packet), _voxelEditSender(), _particleEditSender(), _modelEditSender(), - _receivedAudioStream(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, 1, false, 1, 0, false), + _receivedAudioStream(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, + InboundAudioStream::Settings(0, false, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, false, + DEFAULT_WINDOW_STARVE_THRESHOLD, DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES, + DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION)), _avatarHashMap() { // be the parent of the script engine so it gets moved when we do diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 5900e1f151..fc223ab850 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -67,9 +67,7 @@ void attachNewNodeDataToNode(Node *newNode) { } } -bool AudioMixer::_useDynamicJitterBuffers = false; -int AudioMixer::_staticDesiredJitterBufferFrames = 0; -int AudioMixer::_maxFramesOverDesired = 0; +InboundAudioStream::Settings AudioMixer::_streamSettings; AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), @@ -332,7 +330,7 @@ void AudioMixer::readPendingDatagrams() { void AudioMixer::sendStatsPacket() { static QJsonObject statsObject; - statsObject["useDynamicJitterBuffers"] = _useDynamicJitterBuffers; + statsObject["useDynamicJitterBuffers"] = _streamSettings._dynamicJitterBuffers; statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100.0f; statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; @@ -421,36 +419,62 @@ void AudioMixer::run() { if (settingsObject.contains(AUDIO_GROUP_KEY)) { QJsonObject audioGroupObject = settingsObject[AUDIO_GROUP_KEY].toObject(); - // check the payload to see if we have asked for dynamicJitterBuffer support - const QString DYNAMIC_JITTER_BUFFER_JSON_KEY = "A-dynamic-jitter-buffer"; - bool shouldUseDynamicJitterBuffers = audioGroupObject[DYNAMIC_JITTER_BUFFER_JSON_KEY].toBool(); - if (shouldUseDynamicJitterBuffers) { - qDebug() << "Enable dynamic jitter buffers."; - _useDynamicJitterBuffers = true; - } else { - qDebug() << "Dynamic jitter buffers disabled."; - _useDynamicJitterBuffers = false; - } - bool ok; - const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "B-desired-jitter-buffer-frames"; - _staticDesiredJitterBufferFrames = audioGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok); - if (!ok) { - _staticDesiredJitterBufferFrames = DEFAULT_DESIRED_JITTER_BUFFER_FRAMES; + // check the payload to see if we have asked for dynamicJitterBuffer support + const QString DYNAMIC_JITTER_BUFFER_JSON_KEY = "A-dynamic-jitter-buffer"; + _streamSettings._dynamicJitterBuffers = audioGroupObject[DYNAMIC_JITTER_BUFFER_JSON_KEY].toBool(); + if (_streamSettings._dynamicJitterBuffers) { + qDebug() << "Enable dynamic jitter buffers."; + } else { + qDebug() << "Dynamic jitter buffers disabled."; } - qDebug() << "Static desired jitter buffer frames:" << _staticDesiredJitterBufferFrames; + + const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "B-desired-jitter-buffer-frames"; + _streamSettings._staticDesiredJitterBufferFrames = audioGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok); + if (!ok) { + _streamSettings._staticDesiredJitterBufferFrames = DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES; + } + qDebug() << "Static desired jitter buffer frames:" << _streamSettings._staticDesiredJitterBufferFrames; const QString MAX_FRAMES_OVER_DESIRED_JSON_KEY = "C-max-frames-over-desired"; - _maxFramesOverDesired = audioGroupObject[MAX_FRAMES_OVER_DESIRED_JSON_KEY].toString().toInt(&ok); + _streamSettings._maxFramesOverDesired = audioGroupObject[MAX_FRAMES_OVER_DESIRED_JSON_KEY].toString().toInt(&ok); if (!ok) { - _maxFramesOverDesired = DEFAULT_MAX_FRAMES_OVER_DESIRED; + _streamSettings._maxFramesOverDesired = DEFAULT_MAX_FRAMES_OVER_DESIRED; } - qDebug() << "Max frames over desired:" << _maxFramesOverDesired; + qDebug() << "Max frames over desired:" << _streamSettings._maxFramesOverDesired; + + const QString USE_STDEV_FOR_DESIRED_CALC_JSON_KEY = "D-use-stdev-for-desired-calc"; + _streamSettings._useStDevForJitterCalc = audioGroupObject[USE_STDEV_FOR_DESIRED_CALC_JSON_KEY].toBool(); + if (_streamSettings._useStDevForJitterCalc) { + qDebug() << "Using Philip's stdev method for jitter calc if dynamic jitter buffers enabled"; + } else { + qDebug() << "Using Fred's max-gap method for jitter calc if dynamic jitter buffers enabled"; + } + + const QString WINDOW_STARVE_THRESHOLD_JSON_KEY = "E-window-starve-threshold"; + _streamSettings._windowStarveThreshold = audioGroupObject[WINDOW_STARVE_THRESHOLD_JSON_KEY].toString().toInt(&ok); + if (!ok) { + _streamSettings._windowStarveThreshold = DEFAULT_WINDOW_STARVE_THRESHOLD; + } + qDebug() << "Window A starve threshold:" << _streamSettings._windowStarveThreshold; + + const QString WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY = "F-window-seconds-for-desired-calc-on-too-many-starves"; + _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = audioGroupObject[WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES_JSON_KEY].toString().toInt(&ok); + if (!ok) { + _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES; + } + qDebug() << "Window A length:" << _streamSettings._windowSecondsForDesiredCalcOnTooManyStarves << "seconds"; + + const QString WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY = "G-window-seconds-for-desired-reduction"; + _streamSettings._windowSecondsForDesiredReduction = audioGroupObject[WINDOW_SECONDS_FOR_DESIRED_REDUCTION_JSON_KEY].toString().toInt(&ok); + if (!ok) { + _streamSettings._windowSecondsForDesiredReduction = DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION; + } + qDebug() << "Window B length:" << _streamSettings._windowSecondsForDesiredReduction << "seconds"; - - const QString UNATTENUATED_ZONE_KEY = "D-unattenuated-zone"; + const QString UNATTENUATED_ZONE_KEY = "Z-unattenuated-zone"; QString unattenuatedZoneString = audioGroupObject[UNATTENUATED_ZONE_KEY].toString(); if (!unattenuatedZoneString.isEmpty()) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 83769a4209..3df2bce682 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -37,9 +37,7 @@ public slots: void sendStatsPacket(); - static bool getUseDynamicJitterBuffers() { return _useDynamicJitterBuffers; } - static int getStaticDesiredJitterBufferFrames() { return _staticDesiredJitterBufferFrames; } - static int getMaxFramesOverDesired() { return _maxFramesOverDesired; } + static const InboundAudioStream::Settings& getStreamSettings() { return _streamSettings; } private: /// adds one stream to the mix for a listening node @@ -62,9 +60,7 @@ private: AABox* _sourceUnattenuatedZone; AABox* _listenerUnattenuatedZone; - static bool _useDynamicJitterBuffers; - static int _staticDesiredJitterBufferFrames; - static int _maxFramesOverDesired; + static InboundAudioStream::Settings _streamSettings; quint64 _lastSendAudioStreamStatsTime; }; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index ecb4d29171..333357fbf4 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -73,9 +73,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { quint8 channelFlag = *(reinterpret_cast(channelFlagAt)); bool isStereo = channelFlag == 1; - _audioStreams.insert(nullUUID, - matchingStream = new AvatarAudioStream(isStereo, AudioMixer::getUseDynamicJitterBuffers(), - AudioMixer::getStaticDesiredJitterBufferFrames(), AudioMixer::getMaxFramesOverDesired())); + _audioStreams.insert(nullUUID, matchingStream = new AvatarAudioStream(isStereo, AudioMixer::getStreamSettings())); } else { matchingStream = _audioStreams.value(nullUUID); } @@ -87,9 +85,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(bytesBeforeStreamIdentifier, NUM_BYTES_RFC4122_UUID)); if (!_audioStreams.contains(streamIdentifier)) { - _audioStreams.insert(streamIdentifier, - matchingStream = new InjectedAudioStream(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers(), - AudioMixer::getStaticDesiredJitterBufferFrames(), AudioMixer::getMaxFramesOverDesired())); + // we don't have this injected stream yet, so add it + _audioStreams.insert(streamIdentifier, matchingStream = new InjectedAudioStream(streamIdentifier, AudioMixer::getStreamSettings())); } else { matchingStream = _audioStreams.value(streamIdentifier); } diff --git a/assignment-client/src/audio/AvatarAudioStream.cpp b/assignment-client/src/audio/AvatarAudioStream.cpp index fcb78d7a6c..877d1a37c7 100644 --- a/assignment-client/src/audio/AvatarAudioStream.cpp +++ b/assignment-client/src/audio/AvatarAudioStream.cpp @@ -13,8 +13,8 @@ #include "AvatarAudioStream.h" -AvatarAudioStream::AvatarAudioStream(bool isStereo, bool dynamicJitterBuffer, int staticDesiredJitterBufferFrames, int maxFramesOverDesired) : - PositionalAudioStream(PositionalAudioStream::Microphone, isStereo, dynamicJitterBuffer, staticDesiredJitterBufferFrames, maxFramesOverDesired) +AvatarAudioStream::AvatarAudioStream(bool isStereo, const InboundAudioStream::Settings& settings) : + PositionalAudioStream(PositionalAudioStream::Microphone, isStereo, settings) { } diff --git a/assignment-client/src/audio/AvatarAudioStream.h b/assignment-client/src/audio/AvatarAudioStream.h index ebad4585e0..2eb0e674b0 100644 --- a/assignment-client/src/audio/AvatarAudioStream.h +++ b/assignment-client/src/audio/AvatarAudioStream.h @@ -18,7 +18,7 @@ class AvatarAudioStream : public PositionalAudioStream { public: - AvatarAudioStream(bool isStereo, bool dynamicJitterBuffer, int staticDesiredJitterBufferFrames, int maxFramesOverDesired); + AvatarAudioStream(bool isStereo, const InboundAudioStream::Settings& settings); private: // disallow copying of AvatarAudioStream objects diff --git a/domain-server/resources/web/settings/describe.json b/domain-server/resources/web/settings/describe.json index f4920a7b50..59d99eab11 100644 --- a/domain-server/resources/web/settings/describe.json +++ b/domain-server/resources/web/settings/describe.json @@ -21,7 +21,31 @@ "placeholder": "10", "default": "10" }, - "D-unattenuated-zone": { + "D-use-stdev": { + "type": "checkbox", + "label": "Use Stdev for Desired Jitter Frames Calc:", + "help": "If checked, Philip's method (stdev of timegaps) is used to calculate desired jitter frames. Otherwise, Fred's method (max timegap) is used", + "default": false + } + "E-window-starve-threshold": { + "label": "Window Starve Threshold", + "help": "If this many starves occur in an N-second window (N is the number in the next field), then the desired jitter frames will be re-evaluated using Window A.", + "placeholder": "3", + "default": "3" + }, + "F-window-seconds-for-desired-calc-on-too-many-starves": { + "label": "Timegaps Window (A) Seconds:", + "help": "Window A contains a history of timegaps. Its max timegap is used to re-evaluate the desired jitter frames when too many starves occur within it.", + "placeholder": "50", + "default": "50" + }, + "G-window-seconds-for-desired-reduction": { + "label": "Timegaps Window (B) Seconds:", + "help": "Window B contains a history of timegaps. Its max timegap is used as a ceiling for the desired jitter frames value.", + "placeholder": "10", + "default": "10" + }, + "Z-unattenuated-zone": { "label": "Unattenuated Zone", "help": "Boxes for source and listener (corner x, corner y, corner z, size x, size y, size z, corner x, corner y, corner z, size x, size y, size z)", "placeholder": "no zone", diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7eb087c531..a7d6cd9f3b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1763,14 +1763,7 @@ void Application::init() { _lastTimeUpdated.start(); Menu::getInstance()->loadSettings(); - if (Menu::getInstance()->getAudioJitterBufferFrames() != 0) { - _audio.setDynamicJitterBuffers(false); - _audio.setStaticDesiredJitterBufferFrames(Menu::getInstance()->getAudioJitterBufferFrames()); - } else { - _audio.setDynamicJitterBuffers(true); - } - - _audio.setMaxFramesOverDesired(Menu::getInstance()->getMaxFramesOverDesired()); + _audio.setReceivedAudioStreamSettings(Menu::getInstance()->getReceivedAudioStreamSettings()); qDebug("Loaded settings"); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 6ff053e5db..9244788de8 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -73,7 +73,7 @@ Audio::Audio(QObject* parent) : _proceduralAudioOutput(NULL), _proceduralOutputDevice(NULL), _inputRingBuffer(0), - _receivedAudioStream(0, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, true, 0, 0, true), + _receivedAudioStream(0, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, InboundAudioStream::Settings()), _isStereoInput(false), _averagedLatency(0.0), _lastInputLoudness(0), @@ -840,12 +840,11 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { void Audio::sendDownstreamAudioStatsPacket() { - // since this function is called every second, we'll sample some of our stats here - + // since this function is called every second, we'll sample for some of our stats here _inputRingBufferMsecsAvailableStats.update(getInputRingBufferMsecsAvailable()); - _audioOutputMsecsUnplayedStats.update(getAudioOutputMsecsUnplayed()); + // also, call _receivedAudioStream's per-second callback _receivedAudioStream.perSecondCallbackForUpdatingStats(); char packet[MAX_PACKET_SIZE]; diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 8fae6f3bdd..a93b8c5be7 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -71,10 +71,7 @@ public: virtual void startCollisionSound(float magnitude, float frequency, float noise, float duration, bool flashScreen); virtual void startDrumSound(float volume, float frequency, float duration, float decay); - void setDynamicJitterBuffers(bool dynamicJitterBuffers) { _receivedAudioStream.setDynamicJitterBuffers(dynamicJitterBuffers); } - void setStaticDesiredJitterBufferFrames(int staticDesiredJitterBufferFrames) { _receivedAudioStream.setStaticDesiredJitterBufferFrames(staticDesiredJitterBufferFrames); } - - void setMaxFramesOverDesired(int maxFramesOverDesired) { _receivedAudioStream.setMaxFramesOverDesired(maxFramesOverDesired); } + void setReceivedAudioStreamSettings(const InboundAudioStream::Settings& settings) { _receivedAudioStream.setSettings(settings); } int getDesiredJitterBufferFrames() const { return _receivedAudioStream.getDesiredJitterBufferFrames(); } diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 43d9fde01a..ecf28bcb17 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -82,8 +82,7 @@ const int CONSOLE_HEIGHT = 200; Menu::Menu() : _actionHash(), - _audioJitterBufferFrames(0), - _maxFramesOverDesired(0), + _receivedAudioStreamSettings(), _bandwidthDialog(NULL), _fieldOfView(DEFAULT_FIELD_OF_VIEW_DEGREES), _realWorldFieldOfView(DEFAULT_REAL_WORLD_FIELD_OF_VIEW_DEGREES), @@ -632,8 +631,14 @@ void Menu::loadSettings(QSettings* settings) { lockedSettings = true; } - _audioJitterBufferFrames = loadSetting(settings, "audioJitterBufferFrames", 0); - _maxFramesOverDesired = loadSetting(settings, "maxFramesOverDesired", DEFAULT_MAX_FRAMES_OVER_DESIRED); + _receivedAudioStreamSettings._dynamicJitterBuffers = settings->value("dynamicJitterBuffers", DEFAULT_DYNAMIC_JITTER_BUFFERS).toBool(); + _receivedAudioStreamSettings._maxFramesOverDesired = settings->value("maxFramesOverDesired", DEFAULT_MAX_FRAMES_OVER_DESIRED).toInt(); + _receivedAudioStreamSettings._staticDesiredJitterBufferFrames = settings->value("staticDesiredJitterBufferFrames", DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES).toInt(); + _receivedAudioStreamSettings._useStDevForJitterCalc = settings->value("useStDevForJitterCalc", DEFAULT_USE_STDEV_FOR_JITTER_CALC).toBool(); + _receivedAudioStreamSettings._windowStarveThreshold = settings->value("windowStarveThreshold", DEFAULT_WINDOW_STARVE_THRESHOLD).toInt(); + _receivedAudioStreamSettings._windowSecondsForDesiredCalcOnTooManyStarves = settings->value("windowSecondsForDesiredCalcOnTooManyStarves", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES).toInt(); + _receivedAudioStreamSettings._windowSecondsForDesiredReduction = settings->value("windowSecondsForDesiredReduction", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION).toInt(); + _fieldOfView = loadSetting(settings, "fieldOfView", DEFAULT_FIELD_OF_VIEW_DEGREES); _realWorldFieldOfView = loadSetting(settings, "realWorldFieldOfView", DEFAULT_REAL_WORLD_FIELD_OF_VIEW_DEGREES); _faceshiftEyeDeflection = loadSetting(settings, "faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION); @@ -683,8 +688,14 @@ void Menu::saveSettings(QSettings* settings) { lockedSettings = true; } - settings->setValue("audioJitterBufferFrames", _audioJitterBufferFrames); - settings->setValue("maxFramesOverDesired", _maxFramesOverDesired); + settings->setValue("dynamicJitterBuffers", _receivedAudioStreamSettings._dynamicJitterBuffers); + settings->setValue("maxFramesOverDesired", _receivedAudioStreamSettings._maxFramesOverDesired); + settings->setValue("staticDesiredJitterBufferFrames", _receivedAudioStreamSettings._staticDesiredJitterBufferFrames); + settings->setValue("useStDevForJitterCalc", _receivedAudioStreamSettings._useStDevForJitterCalc); + settings->setValue("windowStarveThreshold", _receivedAudioStreamSettings._windowStarveThreshold); + settings->setValue("windowSecondsForDesiredCalcOnTooManyStarves", _receivedAudioStreamSettings._windowSecondsForDesiredCalcOnTooManyStarves); + settings->setValue("windowSecondsForDesiredReduction", _receivedAudioStreamSettings._windowSecondsForDesiredReduction); + settings->setValue("fieldOfView", _fieldOfView); settings->setValue("faceshiftEyeDeflection", _faceshiftEyeDeflection); settings->setValue("maxVoxels", _maxVoxels); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 3bef306bef..20097989df 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -85,10 +85,8 @@ public: void triggerOption(const QString& menuOption); QAction* getActionForOption(const QString& menuOption); - float getAudioJitterBufferFrames() const { return _audioJitterBufferFrames; } - void setAudioJitterBufferFrames(float audioJitterBufferSamples) { _audioJitterBufferFrames = audioJitterBufferSamples; } - int getMaxFramesOverDesired() const { return _maxFramesOverDesired; } - void setMaxFramesOverDesired(int maxFramesOverDesired) { _maxFramesOverDesired = maxFramesOverDesired; } + const InboundAudioStream::Settings& getReceivedAudioStreamSettings() const { return _receivedAudioStreamSettings; } + void getReceivedAudioStreamSettings(const InboundAudioStream::Settings& receivedAudioStreamSettings) { _receivedAudioStreamSettings = receivedAudioStreamSettings; } float getFieldOfView() const { return _fieldOfView; } void setFieldOfView(float fieldOfView) { _fieldOfView = fieldOfView; } float getRealWorldFieldOfView() const { return _realWorldFieldOfView; } @@ -259,8 +257,7 @@ private: QHash _actionHash; - int _audioJitterBufferFrames; /// number of extra samples to wait before starting audio playback - int _maxFramesOverDesired; + InboundAudioStream::Settings _receivedAudioStreamSettings; BandwidthDialog* _bandwidthDialog; float _fieldOfView; /// in Degrees, doesn't apply to HMD like Oculus float _realWorldFieldOfView; // The actual FOV set by the user's monitor size and view distance diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 4ebd5f4c1a..7189a74579 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -149,9 +149,21 @@ void PreferencesDialog::loadPreferences() { ui.faceshiftEyeDeflectionSider->setValue(menuInstance->getFaceshiftEyeDeflection() * ui.faceshiftEyeDeflectionSider->maximum()); - ui.audioJitterSpin->setValue(menuInstance->getAudioJitterBufferFrames()); + const InboundAudioStream::Settings& streamSettings = menuInstance->getReceivedAudioStreamSettings(); - ui.maxFramesOverDesiredSpin->setValue(menuInstance->getMaxFramesOverDesired()); + ui.dynamicJitterBuffersCheckBox->setChecked(streamSettings._dynamicJitterBuffers); + + ui.staticDesiredJitterBufferFramesSpin->setValue(streamSettings._staticDesiredJitterBufferFrames); + + ui.maxFramesOverDesiredSpin->setValue(streamSettings._maxFramesOverDesired); + + ui.useStdevForJitterCalcCheckBox->setChecked(streamSettings._useStDevForJitterCalc); + + ui.windowStarveThresholdSpin->setValue(streamSettings._windowStarveThreshold); + + ui.windowSecondsForDesiredCalcOnTooManyStarvesSpin->setValue(streamSettings._windowSecondsForDesiredCalcOnTooManyStarves); + + ui.windowSecondsForDesiredReductionSpin->setValue(streamSettings._windowSecondsForDesiredReduction); ui.realWorldFieldOfViewSpin->setValue(menuInstance->getRealWorldFieldOfView()); @@ -241,16 +253,17 @@ void PreferencesDialog::savePreferences() { Menu::getInstance()->setInvertSixenseButtons(ui.invertSixenseButtonsCheckBox->isChecked()); - Menu::getInstance()->setAudioJitterBufferFrames(ui.audioJitterSpin->value()); - if (Menu::getInstance()->getAudioJitterBufferFrames() != 0) { - Application::getInstance()->getAudio()->setDynamicJitterBuffers(false); - Application::getInstance()->getAudio()->setStaticDesiredJitterBufferFrames(Menu::getInstance()->getAudioJitterBufferFrames()); - } else { - Application::getInstance()->getAudio()->setDynamicJitterBuffers(true); - } + InboundAudioStream::Settings streamSettings; + streamSettings._dynamicJitterBuffers = ui.dynamicJitterBuffersCheckBox->isChecked(); + streamSettings._staticDesiredJitterBufferFrames = ui.staticDesiredJitterBufferFramesSpin->value(); + streamSettings._maxFramesOverDesired = ui.maxFramesOverDesiredSpin->value(); + streamSettings._useStDevForJitterCalc = ui.useStdevForJitterCalcCheckBox->isChecked(); + streamSettings._windowStarveThreshold = ui.windowStarveThresholdSpin->value(); + streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = ui.windowSecondsForDesiredCalcOnTooManyStarvesSpin->value(); + streamSettings._windowSecondsForDesiredReduction = ui.windowSecondsForDesiredReductionSpin->value(); - Menu::getInstance()->setMaxFramesOverDesired(ui.maxFramesOverDesiredSpin->value()); - Application::getInstance()->getAudio()->setMaxFramesOverDesired(Menu::getInstance()->getMaxFramesOverDesired()); + Menu::getInstance()->getReceivedAudioStreamSettings(streamSettings); + Application::getInstance()->getAudio()->setReceivedAudioStreamSettings(streamSettings); Application::getInstance()->resizeGL(Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); diff --git a/interface/ui/preferencesDialog.ui b/interface/ui/preferencesDialog.ui index 566c24e4e3..cddc0f1299 100644 --- a/interface/ui/preferencesDialog.ui +++ b/interface/ui/preferencesDialog.ui @@ -1464,6 +1464,97 @@ padding: 10px;margin-top:10px + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Enable Dynamic Jitter Buffers + + + 15 + + + dynamicJitterBuffersCheckBox + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 32 + 0 + + + + + 0 + 0 + + + + + + + + 32 + 32 + + + + + + + + + @@ -1489,13 +1580,13 @@ padding: 10px;margin-top:10px color: rgb(51, 51, 51) - Audio Jitter Buffer Frames (0 for automatic) + Static Jitter Buffer Frames 15 - audioJitterSpin + staticDesiredJitterBufferFramesSpin @@ -1518,7 +1609,7 @@ padding: 10px;margin-top:10px - + 0 @@ -1555,6 +1646,7 @@ padding: 10px;margin-top:10px + @@ -1646,7 +1738,378 @@ padding: 10px;margin-top:10px - + + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Use Stdev for Dynamic Jitter Calc + + + 15 + + + useStdevForJitterCalcCheckBox + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 32 + 0 + + + + + 0 + 0 + + + + + + + + 32 + 32 + + + + + + + + + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Window A Starve Threshold + + + 15 + + + windowStarveThresholdSpin + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 95 + 36 + + + + + 70 + 16777215 + + + + + Arial + + + + 0 + + + 10000 + + + 1 + + + + + + + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Window A (raise desired on N starves) Seconds + + + 15 + + + windowSecondsForDesiredCalcOnTooManyStarvesSpin + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 95 + 36 + + + + + 70 + 16777215 + + + + + Arial + + + + 0 + + + 10000 + + + 1 + + + + + + + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Window B (desired ceiling) Seconds + + + 15 + + + windowSecondsForDesiredReductionSpin + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 95 + 36 + + + + + 70 + 16777215 + + + + + Arial + + + + 0 + + + 10000 + + + 1 + + + + + diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index f59070ee83..55322d6f32 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -38,6 +38,7 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _timeGapStatsForDesiredReduction(0, settings._windowSecondsForDesiredReduction), _starveHistoryWindowSeconds(settings._windowSecondsForDesiredCalcOnTooManyStarves), _starveHistory(STARVE_HISTORY_CAPACITY), + _starveThreshold(settings._windowStarveThreshold), _framesAvailableStat(), _currentJitterBufferFrames(0), _timeGapStatsForStatsPacket(0, STATS_FOR_STATS_PACKET_WINDOW_SECONDS) @@ -237,6 +238,16 @@ void InboundAudioStream::setToStarved() { _isStarved = (_ringBuffer.framesAvailable() < _desiredJitterBufferFrames); } +void InboundAudioStream::setSettings(const Settings& settings) { + setMaxFramesOverDesired(settings._maxFramesOverDesired); + setDynamicJitterBuffers(settings._dynamicJitterBuffers); + setStaticDesiredJitterBufferFrames(settings._staticDesiredJitterBufferFrames); + setUseStDevForJitterCalc(settings._useStDevForJitterCalc); + setWindowStarveThreshold(settings._windowStarveThreshold); + setWindowSecondsForDesiredCalcOnTooManyStarves(settings._windowSecondsForDesiredCalcOnTooManyStarves); + setWindowSecondsForDesiredReduction(settings._windowSecondsForDesiredReduction); +} + void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { if (!dynamicJitterBuffers) { _desiredJitterBufferFrames = _staticDesiredJitterBufferFrames; @@ -255,6 +266,15 @@ void InboundAudioStream::setStaticDesiredJitterBufferFrames(int staticDesiredJit } } +void InboundAudioStream::setWindowSecondsForDesiredCalcOnTooManyStarves(int windowSecondsForDesiredCalcOnTooManyStarves) { + _timeGapStatsForDesiredCalcOnTooManyStarves.setWindowIntervals(windowSecondsForDesiredCalcOnTooManyStarves); +} + +void InboundAudioStream::setWindowSecondsForDesiredReduction(int windowSecondsForDesiredReduction) { + _timeGapStatsForDesiredReduction.setWindowIntervals(windowSecondsForDesiredReduction); +} + + int InboundAudioStream::clampDesiredJitterBufferFramesValue(int desired) const { const int MIN_FRAMES_DESIRED = 0; const int MAX_FRAMES_DESIRED = _ringBuffer.getFrameCapacity(); diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 88760a35cb..791886ab5e 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -39,7 +39,9 @@ const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND; const int INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10; -const int DEFAULT_DESIRED_JITTER_BUFFER_FRAMES = 1; +const bool DEFAULT_DYNAMIC_JITTER_BUFFERS = true; +const int DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES = 1; +const bool DEFAULT_USE_STDEV_FOR_JITTER_CALC = false; const int DEFAULT_WINDOW_STARVE_THRESHOLD = 3; const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES = 50; @@ -52,14 +54,26 @@ public: public: Settings() : _maxFramesOverDesired(DEFAULT_MAX_FRAMES_OVER_DESIRED), - _dynamicJitterBuffers(true), - _staticDesiredJitterBufferFrames(DEFAULT_DESIRED_JITTER_BUFFER_FRAMES), - _useStDevForJitterCalc(false), + _dynamicJitterBuffers(DEFAULT_DYNAMIC_JITTER_BUFFERS), + _staticDesiredJitterBufferFrames(DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES), + _useStDevForJitterCalc(DEFAULT_USE_STDEV_FOR_JITTER_CALC), _windowStarveThreshold(DEFAULT_WINDOW_STARVE_THRESHOLD), _windowSecondsForDesiredCalcOnTooManyStarves(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES), _windowSecondsForDesiredReduction(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION) {} + Settings(int maxFramesOverDesired, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, + bool useStDevForJitterCalc, int windowStarveThreshold, int windowSecondsForDesiredCalcOnTooManyStarves, + int _windowSecondsForDesiredReduction) + : _maxFramesOverDesired(maxFramesOverDesired), + _dynamicJitterBuffers(dynamicJitterBuffers), + _staticDesiredJitterBufferFrames(staticDesiredJitterBufferFrames), + _useStDevForJitterCalc(useStDevForJitterCalc), + _windowStarveThreshold(windowStarveThreshold), + _windowSecondsForDesiredCalcOnTooManyStarves(windowSecondsForDesiredCalcOnTooManyStarves), + _windowSecondsForDesiredReduction(windowSecondsForDesiredCalcOnTooManyStarves) + {} + // max number of frames over desired in the ringbuffer. int _maxFramesOverDesired; @@ -95,15 +109,17 @@ public: void setToStarved(); - - void setDynamicJitterBuffers(bool dynamicJitterBuffers); - void setStaticDesiredJitterBufferFrames(int staticDesiredJitterBufferFrames); - - /// this function should be called once per second to ensure the seq num stats history spans ~30 seconds - //AudioStreamStats updateSeqHistoryAndGetAudioStreamStats(); + void setSettings(const Settings& settings); void setMaxFramesOverDesired(int maxFramesOverDesired) { _maxFramesOverDesired = maxFramesOverDesired; } + void setDynamicJitterBuffers(bool setDynamicJitterBuffers); + void setStaticDesiredJitterBufferFrames(int staticDesiredJitterBufferFrames); + void setUseStDevForJitterCalc(bool useStDevForJitterCalc) { _useStDevForJitterCalc = useStDevForJitterCalc; } + void setWindowStarveThreshold(int windowStarveThreshold) { _starveThreshold = windowStarveThreshold; } + void setWindowSecondsForDesiredCalcOnTooManyStarves(int windowSecondsForDesiredCalcOnTooManyStarves); + void setWindowSecondsForDesiredReduction(int windowSecondsForDesiredReduction); + virtual AudioStreamStats getAudioStreamStats() const; @@ -205,6 +221,7 @@ protected: int _starveHistoryWindowSeconds; RingBufferHistory _starveHistory; + int _starveThreshold; TimeWeightedAvg _framesAvailableStat; diff --git a/libraries/audio/src/InjectedAudioStream.cpp b/libraries/audio/src/InjectedAudioStream.cpp index 5c1c2ed269..9a757b774e 100644 --- a/libraries/audio/src/InjectedAudioStream.cpp +++ b/libraries/audio/src/InjectedAudioStream.cpp @@ -19,8 +19,8 @@ #include "InjectedAudioStream.h" -InjectedAudioStream::InjectedAudioStream(const QUuid& streamIdentifier, bool dynamicJitterBuffer, int staticDesiredJitterBufferFrames, int maxFramesOverDesired) : - PositionalAudioStream(PositionalAudioStream::Injector, false, dynamicJitterBuffer, staticDesiredJitterBufferFrames, maxFramesOverDesired), +InjectedAudioStream::InjectedAudioStream(const QUuid& streamIdentifier, const InboundAudioStream::Settings& settings) : + PositionalAudioStream(PositionalAudioStream::Injector, false, settings), _streamIdentifier(streamIdentifier), _radius(0.0f), _attenuationRatio(0) diff --git a/libraries/audio/src/InjectedAudioStream.h b/libraries/audio/src/InjectedAudioStream.h index d8d9a54c6e..f3840b1029 100644 --- a/libraries/audio/src/InjectedAudioStream.h +++ b/libraries/audio/src/InjectedAudioStream.h @@ -18,7 +18,7 @@ class InjectedAudioStream : public PositionalAudioStream { public: - InjectedAudioStream(const QUuid& streamIdentifier, bool dynamicJitterBuffer, int staticDesiredJitterBufferFrames, int maxFramesOverDesired); + InjectedAudioStream(const QUuid& streamIdentifier, const InboundAudioStream::Settings& settings); float getRadius() const { return _radius; } float getAttenuationRatio() const { return _attenuationRatio; } diff --git a/libraries/audio/src/MixedAudioStream.cpp b/libraries/audio/src/MixedAudioStream.cpp index 38c4ae641d..0041348d26 100644 --- a/libraries/audio/src/MixedAudioStream.cpp +++ b/libraries/audio/src/MixedAudioStream.cpp @@ -11,8 +11,8 @@ #include "MixedAudioStream.h" -MixedAudioStream::MixedAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc) - : InboundAudioStream(numFrameSamples, numFramesCapacity, dynamicJitterBuffers, staticDesiredJitterBufferFrames, maxFramesOverDesired, useStDevForJitterCalc) +MixedAudioStream::MixedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings) + : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) { } diff --git a/libraries/audio/src/MixedAudioStream.h b/libraries/audio/src/MixedAudioStream.h index d19f19af07..0b1979003d 100644 --- a/libraries/audio/src/MixedAudioStream.h +++ b/libraries/audio/src/MixedAudioStream.h @@ -17,7 +17,7 @@ class MixedAudioStream : public InboundAudioStream { public: - MixedAudioStream(int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc); + MixedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings); float getNextOutputFrameLoudness() const { return _ringBuffer.getNextOutputFrameLoudness(); } diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 49990dcd22..52581bd096 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -11,8 +11,8 @@ #include "MixedProcessedAudioStream.h" -MixedProcessedAudioStream ::MixedProcessedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc) - : InboundAudioStream(numFrameSamples, numFramesCapacity, dynamicJitterBuffers, staticDesiredJitterBufferFrames, maxFramesOverDesired, useStDevForJitterCalc) +MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings) + : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) { } diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 5a5b73115d..e033297362 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -17,7 +17,7 @@ class MixedProcessedAudioStream : public InboundAudioStream { Q_OBJECT public: - MixedProcessedAudioStream (int numFrameSamples, int numFramesCapacity, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, int maxFramesOverDesired, bool useStDevForJitterCalc); + MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings); signals: diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index 7b407ba62c..d2c1ade85c 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -21,10 +21,9 @@ #include #include -PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, bool dynamicJitterBuffers, - int staticDesiredJitterBufferFrames, int maxFramesOverDesired) : +PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, const InboundAudioStream::Settings& settings) : InboundAudioStream(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL, - AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, dynamicJitterBuffers, staticDesiredJitterBufferFrames, maxFramesOverDesired), + AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY, settings), _type(type), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index f99dc3a464..d1d5e013e7 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -27,8 +27,7 @@ public: Injector }; - PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, - int maxFramesOverDesired); + PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, const InboundAudioStream::Settings& settings); virtual AudioStreamStats getAudioStreamStats() const; diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 9ac9ec9006..469ec1b3ef 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -103,6 +103,15 @@ public: _newStatsAvailable = false; } + void setWindowIntervals(int windowIntervals) { + _windowIntervals = windowIntervals; + _overallStats.reset(); + _windowStats.reset(); + _currentIntervalStats.reset(); + _intervalStats.setCapacity(_windowIntervals); + _newStatsAvailable = false; + } + void update(T newSample) { // update overall stats _overallStats.update(newSample); diff --git a/libraries/shared/src/RingBufferHistory.h b/libraries/shared/src/RingBufferHistory.h index 339f390cd5..9534b2f1db 100644 --- a/libraries/shared/src/RingBufferHistory.h +++ b/libraries/shared/src/RingBufferHistory.h @@ -35,6 +35,14 @@ public: _numEntries = 0; } + void setCapacity(int capacity) { + _size = capacity + 1; + _capacity = capacity; + _newestEntryAtIndex = 0; + _numEntries = 0; + _buffer.resize(_size); + } + void insert(const T& entry) { // increment newest entry index cyclically _newestEntryAtIndex = (_newestEntryAtIndex == _size - 1) ? 0 : _newestEntryAtIndex + 1; From fd3425dfd1b53030db1eb49b65344877b65b4dc7 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 7 Aug 2014 17:05:23 -0700 Subject: [PATCH 05/34] knobs confirmed working --- assignment-client/src/audio/AudioMixer.cpp | 2 +- domain-server/resources/web/settings/describe.json | 10 +++++----- libraries/audio/src/InboundAudioStream.cpp | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index fc223ab850..e9bedc5433 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -430,7 +430,7 @@ void AudioMixer::run() { qDebug() << "Dynamic jitter buffers disabled."; } - const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "B-desired-jitter-buffer-frames"; + const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "B-static-desired-jitter-buffer-frames"; _streamSettings._staticDesiredJitterBufferFrames = audioGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok); if (!ok) { _streamSettings._staticDesiredJitterBufferFrames = DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES; diff --git a/domain-server/resources/web/settings/describe.json b/domain-server/resources/web/settings/describe.json index 59d99eab11..bb63c5f0a0 100644 --- a/domain-server/resources/web/settings/describe.json +++ b/domain-server/resources/web/settings/describe.json @@ -7,10 +7,10 @@ "type": "checkbox", "label": "Dynamic Jitter Buffers", "help": "Dynamically buffer client audio based on perceived jitter in packet receipt timing", - "default": false + "default": true }, - "B-desired-jitter-buffer-frames": { - "label": "Desired Jitter Buffer Frames", + "B-static-desired-jitter-buffer-frames": { + "label": "Static Desired Jitter Buffer Frames", "help": "If dynamic jitter buffers is disabled, this determines the target number of frames maintained by the AudioMixer's jitter buffers", "placeholder": "1", "default": "1" @@ -21,12 +21,12 @@ "placeholder": "10", "default": "10" }, - "D-use-stdev": { + "D-use-stdev-for-desired-calc": { "type": "checkbox", "label": "Use Stdev for Desired Jitter Frames Calc:", "help": "If checked, Philip's method (stdev of timegaps) is used to calculate desired jitter frames. Otherwise, Fred's method (max timegap) is used", "default": false - } + }, "E-window-starve-threshold": { "label": "Window Starve Threshold", "help": "If this many starves occur in an N-second window (N is the number in the next field), then the desired jitter frames will be re-evaluated using Window A.", diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 55322d6f32..c04f9d5df1 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -246,6 +246,15 @@ void InboundAudioStream::setSettings(const Settings& settings) { setWindowStarveThreshold(settings._windowStarveThreshold); setWindowSecondsForDesiredCalcOnTooManyStarves(settings._windowSecondsForDesiredCalcOnTooManyStarves); setWindowSecondsForDesiredReduction(settings._windowSecondsForDesiredReduction); + + + printf("\n\nmax frames over desired: %d\n", settings._maxFramesOverDesired); + printf("dynamic jitter buffers: %d\n", settings._dynamicJitterBuffers); + printf("static desired jbuffer frames: %d\n", settings._staticDesiredJitterBufferFrames); + printf("use stdev: %d\n", settings._useStDevForJitterCalc); + printf("starve threshold: %d\n", settings._windowStarveThreshold); + printf("window A seconds: %d\n", settings._windowSecondsForDesiredCalcOnTooManyStarves); + printf("window B seconds: %d\n", settings._windowSecondsForDesiredReduction); } void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { From 850a2d8d9463caa1a9db3fc0ecb70ade557e1979 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 09:59:33 -0700 Subject: [PATCH 06/34] inboundaudiostream behavior updated; needs testing --- libraries/audio/src/InboundAudioStream.cpp | 106 +++++++++++++++------ libraries/audio/src/InboundAudioStream.h | 9 +- 2 files changed, 82 insertions(+), 33 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index c04f9d5df1..6ab8e30c2a 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -21,8 +21,6 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _dynamicJitterBuffers(settings._dynamicJitterBuffers), _staticDesiredJitterBufferFrames(settings._staticDesiredJitterBufferFrames), _useStDevForJitterCalc(settings._useStDevForJitterCalc), - _calculatedJitterBufferFramesUsingMaxGap(0), - _calculatedJitterBufferFramesUsingStDev(0), _desiredJitterBufferFrames(settings._dynamicJitterBuffers ? 1 : settings._staticDesiredJitterBufferFrames), _maxFramesOverDesired(settings._maxFramesOverDesired), _isStarved(true), @@ -32,9 +30,10 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _silentFramesDropped(0), _oldFramesDropped(0), _incomingSequenceNumberStats(STATS_FOR_STATS_PACKET_WINDOW_SECONDS), - _lastFrameReceivedTime(0), + _lastPacketReceivedTime(0), _timeGapStatsForDesiredCalcOnTooManyStarves(0, settings._windowSecondsForDesiredCalcOnTooManyStarves), _stdevStatsForDesiredCalcOnTooManyStarves(), + _calculatedJitterBufferFramesUsingStDev(0), _timeGapStatsForDesiredReduction(0, settings._windowSecondsForDesiredReduction), _starveHistoryWindowSeconds(settings._windowSecondsForDesiredCalcOnTooManyStarves), _starveHistory(STARVE_HISTORY_CAPACITY), @@ -63,7 +62,7 @@ void InboundAudioStream::resetStats() { _silentFramesDropped = 0; _oldFramesDropped = 0; _incomingSequenceNumberStats.reset(); - _lastFrameReceivedTime = 0; + _lastPacketReceivedTime = 0; _timeGapStatsForDesiredCalcOnTooManyStarves.reset(); _stdevStatsForDesiredCalcOnTooManyStarves = StDev(); _timeGapStatsForDesiredReduction.reset(); @@ -230,12 +229,51 @@ void InboundAudioStream::framesAvailableChanged() { } void InboundAudioStream::setToStarved() { - _isStarved = true; _consecutiveNotMixedCount = 0; _starveCount++; // if we have more than the desired frames when setToStarved() is called, then we'll immediately // be considered refilled. in that case, there's no need to set _isStarved to true. _isStarved = (_ringBuffer.framesAvailable() < _desiredJitterBufferFrames); + + // if dynamic jitter buffers are enabled, we should check if this starve put us over the window + // starve threshold + if (_dynamicJitterBuffers) { + quint64 now = usecTimestampNow(); + _starveHistory.insert(now); + + quint64 windowEnd = now - _starveHistoryWindowSeconds * USECS_PER_SECOND; + RingBufferHistory::Iterator starvesIterator = _starveHistory.begin(); + RingBufferHistory::Iterator end = _starveHistory.end(); + int starvesInWindow = 1; + do { + ++starvesIterator; + if (*starvesIterator < windowEnd) { + break; + } + starvesInWindow++; + } while (starvesIterator != end); + + // this starve put us over the starve threshold. update _desiredJitterBufferFrames to + // value determined by window A. + if (starvesInWindow >= _starveThreshold) { + int calculatedJitterBufferFrames; + if (_useStDevForJitterCalc) { + calculatedJitterBufferFrames = _calculatedJitterBufferFramesUsingStDev; + } else { + // we don't know when the next packet will arrive, so it's possible the gap between the last packet and the + // next packet will exceed the max time gap in the window. If the time since the last packet has already exceeded + // the window max gap, then we should use that value to calculate desired frames. + if (now - _lastPacketReceivedTime) + + calculatedJitterBufferFrames = ceilf((float)_timeGapStatsForDesiredCalcOnTooManyStarves.getWindowMax() + / (float)BUFFER_SEND_INTERVAL_USECS); + } + // make sure _desiredJitterBufferFrames does not become lower here + if (calculatedJitterBufferFrames >= _desiredJitterBufferFrames) { + _desiredJitterBufferFrames = calculatedJitterBufferFrames; + } + } + } } void InboundAudioStream::setSettings(const Settings& settings) { @@ -259,9 +297,18 @@ void InboundAudioStream::setSettings(const Settings& settings) { void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { if (!dynamicJitterBuffers) { - _desiredJitterBufferFrames = _staticDesiredJitterBufferFrames; + if (_dynamicJitterBuffers) { + // if we're disabling dynamic jitter buffers, set desired frames to its static value + // and clear all the stats for calculating desired frames in dynamic mode. + _desiredJitterBufferFrames = _staticDesiredJitterBufferFrames; + _timeGapStatsForDesiredCalcOnTooManyStarves.reset(); + _stdevStatsForDesiredCalcOnTooManyStarves.reset(); + _timeGapStatsForDesiredReduction.reset(); + _starveHistory.clear(); + } } else { if (!_dynamicJitterBuffers) { + // if we're enabling dynamic jitter buffer frames, start desired frames at 1 _desiredJitterBufferFrames = 1; } } @@ -291,42 +338,45 @@ int InboundAudioStream::clampDesiredJitterBufferFramesValue(int desired) const { } void InboundAudioStream::frameReceivedUpdateTimingStats() { - /* + // update our timegap stats and desired jitter buffer frames if necessary // discard the first few packets we receive since they usually have gaps that aren't represensative of normal jitter const int NUM_INITIAL_PACKETS_DISCARD = 3; quint64 now = usecTimestampNow(); if (_incomingSequenceNumberStats.getReceived() > NUM_INITIAL_PACKETS_DISCARD) { - quint64 gap = now - _lastFrameReceivedTime; + quint64 gap = now - _lastPacketReceivedTime; _timeGapStatsForStatsPacket.update(gap); - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + if (_dynamicJitterBuffers) { - // update stats for Freddy's method of jitter calc - _interframeTimeGapStatsForJitterCalc.update(gap); - if (_interframeTimeGapStatsForJitterCalc.getNewStatsAvailableFlag()) { - _calculatedJitterBufferFramesUsingMaxGap = ceilf((float)_interframeTimeGapStatsForJitterCalc.getWindowMax() / USECS_PER_FRAME); - _interframeTimeGapStatsForJitterCalc.clearNewStatsAvailableFlag(); + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_dynamicJitterBuffers && !_useStDevForJitterCalc) { - _desiredJitterBufferFrames = clampDesiredJitterBufferFramesValue(_calculatedJitterBufferFramesUsingMaxGap); + // update all stats used for desired frames calculations under dynamic jitter buffer mode + _timeGapStatsForDesiredCalcOnTooManyStarves.update(gap); + _stdevStatsForDesiredCalcOnTooManyStarves.addValue(gap); + _timeGapStatsForDesiredReduction.update(gap); + + const int STANDARD_DEVIATION_SAMPLE_COUNT = 500; + if (_stdevStatsForDesiredCalcOnTooManyStarves.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) { + const float NUM_STANDARD_DEVIATIONS = 3.0f; + _calculatedJitterBufferFramesUsingStDev = (int)ceilf(NUM_STANDARD_DEVIATIONS * _stdevStatsForDesiredCalcOnTooManyStarves.getStDev() + / USECS_PER_FRAME); + _stdevStatsForDesiredCalcOnTooManyStarves.reset(); } - } - // update stats for Philip's method of jitter calc - _stdev.addValue(gap); - const int STANDARD_DEVIATION_SAMPLE_COUNT = 500; - if (_stdev.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) { - const float NUM_STANDARD_DEVIATIONS = 3.0f; - _calculatedJitterBufferFramesUsingStDev = (int)ceilf(NUM_STANDARD_DEVIATIONS * _stdev.getStDev() / USECS_PER_FRAME); - _stdev.reset(); - - if (_dynamicJitterBuffers && _useStDevForJitterCalc) { - _desiredJitterBufferFrames = clampDesiredJitterBufferFramesValue(_calculatedJitterBufferFramesUsingStDev); + // if the max gap in window B (_timeGapStatsForDesiredReduction) corresponds to a smaller number of frames than _desiredJitterBufferFrames, + // then reduce _desiredJitterBufferFrames to that number of frames. + if (_timeGapStatsForDesiredReduction.getNewStatsAvailableFlag() && _timeGapStatsForDesiredReduction.isWindowFilled()) { + int calculatedJitterBufferFrames = ceilf((float)_timeGapStatsForDesiredReduction.getWindowMax() / USECS_PER_FRAME); + if (calculatedJitterBufferFrames < _desiredJitterBufferFrames) { + _desiredJitterBufferFrames = calculatedJitterBufferFrames; + } + _timeGapStatsForDesiredReduction.clearNewStatsAvailableFlag(); } } } - _lastFrameReceivedTime = now;*/ + + _lastPacketReceivedTime = now; } int InboundAudioStream::writeDroppableSilentSamples(int numSilentSamples) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 791886ab5e..1e58eaa509 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -193,8 +193,6 @@ protected: // if jitter buffer is dynamic, this determines what method of calculating _desiredJitterBufferFrames // if true, Philip's timegap std dev calculation is used. Otherwise, Freddy's max timegap calculation is used bool _useStDevForJitterCalc; - int _calculatedJitterBufferFramesUsingMaxGap; - int _calculatedJitterBufferFramesUsingStDev; int _desiredJitterBufferFrames; @@ -214,9 +212,10 @@ protected: SequenceNumberStats _incomingSequenceNumberStats; - quint64 _lastFrameReceivedTime; - MovingMinMaxAvg _timeGapStatsForDesiredCalcOnTooManyStarves; - StDev _stdevStatsForDesiredCalcOnTooManyStarves; + quint64 _lastPacketReceivedTime; + MovingMinMaxAvg _timeGapStatsForDesiredCalcOnTooManyStarves; // for Freddy's method + StDev _stdevStatsForDesiredCalcOnTooManyStarves; // for Philip's method + int _calculatedJitterBufferFramesUsingStDev; // the most recent desired frames calculated by Philip's method MovingMinMaxAvg _timeGapStatsForDesiredReduction; int _starveHistoryWindowSeconds; From 3338102513fa21060d1b1b6afbc9307d022397bd Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 10:29:45 -0700 Subject: [PATCH 07/34] reverted some behavior to make getCalculated() to work again --- libraries/audio/src/InboundAudioStream.cpp | 68 ++++++++++------------ libraries/audio/src/InboundAudioStream.h | 3 +- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 6ab8e30c2a..64993af2b5 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -34,6 +34,7 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _timeGapStatsForDesiredCalcOnTooManyStarves(0, settings._windowSecondsForDesiredCalcOnTooManyStarves), _stdevStatsForDesiredCalcOnTooManyStarves(), _calculatedJitterBufferFramesUsingStDev(0), + _calculatedJitterBufferFramesUsingMaxGap(0), _timeGapStatsForDesiredReduction(0, settings._windowSecondsForDesiredReduction), _starveHistoryWindowSeconds(settings._windowSecondsForDesiredCalcOnTooManyStarves), _starveHistory(STARVE_HISTORY_CAPACITY), @@ -104,7 +105,7 @@ int InboundAudioStream::parseData(const QByteArray& packet) { readBytes += sizeof(quint16); SequenceNumberStats::ArrivalInfo arrivalInfo = _incomingSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); - frameReceivedUpdateTimingStats(); + packetReceivedUpdateTimingStats(); // TODO: handle generalized silent packet here????? @@ -235,12 +236,13 @@ void InboundAudioStream::setToStarved() { // be considered refilled. in that case, there's no need to set _isStarved to true. _isStarved = (_ringBuffer.framesAvailable() < _desiredJitterBufferFrames); - // if dynamic jitter buffers are enabled, we should check if this starve put us over the window - // starve threshold - if (_dynamicJitterBuffers) { - quint64 now = usecTimestampNow(); - _starveHistory.insert(now); + // record the time of this starve in the starve history + quint64 now = usecTimestampNow(); + _starveHistory.insert(now); + if (_dynamicJitterBuffers) { + // dynamic jitter buffers are enabled. check if this starve put us over the window + // starve threshold quint64 windowEnd = now - _starveHistoryWindowSeconds * USECS_PER_SECOND; RingBufferHistory::Iterator starvesIterator = _starveHistory.begin(); RingBufferHistory::Iterator end = _starveHistory.end(); @@ -263,10 +265,8 @@ void InboundAudioStream::setToStarved() { // we don't know when the next packet will arrive, so it's possible the gap between the last packet and the // next packet will exceed the max time gap in the window. If the time since the last packet has already exceeded // the window max gap, then we should use that value to calculate desired frames. - if (now - _lastPacketReceivedTime) - - calculatedJitterBufferFrames = ceilf((float)_timeGapStatsForDesiredCalcOnTooManyStarves.getWindowMax() - / (float)BUFFER_SEND_INTERVAL_USECS); + int framesSinceLastPacket = ceilf((float)(now - _lastPacketReceivedTime) / (float)BUFFER_SEND_INTERVAL_USECS); + calculatedJitterBufferFrames = std::max(_calculatedJitterBufferFramesUsingMaxGap, framesSinceLastPacket); } // make sure _desiredJitterBufferFrames does not become lower here if (calculatedJitterBufferFrames >= _desiredJitterBufferFrames) { @@ -297,15 +297,7 @@ void InboundAudioStream::setSettings(const Settings& settings) { void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { if (!dynamicJitterBuffers) { - if (_dynamicJitterBuffers) { - // if we're disabling dynamic jitter buffers, set desired frames to its static value - // and clear all the stats for calculating desired frames in dynamic mode. - _desiredJitterBufferFrames = _staticDesiredJitterBufferFrames; - _timeGapStatsForDesiredCalcOnTooManyStarves.reset(); - _stdevStatsForDesiredCalcOnTooManyStarves.reset(); - _timeGapStatsForDesiredReduction.reset(); - _starveHistory.clear(); - } + _desiredJitterBufferFrames = _staticDesiredJitterBufferFrames; } else { if (!_dynamicJitterBuffers) { // if we're enabling dynamic jitter buffer frames, start desired frames at 1 @@ -337,7 +329,7 @@ int InboundAudioStream::clampDesiredJitterBufferFramesValue(int desired) const { return glm::clamp(desired, MIN_FRAMES_DESIRED, MAX_FRAMES_DESIRED); } -void InboundAudioStream::frameReceivedUpdateTimingStats() { +void InboundAudioStream::packetReceivedUpdateTimingStats() { // update our timegap stats and desired jitter buffer frames if necessary // discard the first few packets we receive since they usually have gaps that aren't represensative of normal jitter @@ -347,27 +339,31 @@ void InboundAudioStream::frameReceivedUpdateTimingStats() { quint64 gap = now - _lastPacketReceivedTime; _timeGapStatsForStatsPacket.update(gap); + // update all stats used for desired frames calculations under dynamic jitter buffer mode + _timeGapStatsForDesiredCalcOnTooManyStarves.update(gap); + _stdevStatsForDesiredCalcOnTooManyStarves.addValue(gap); + _timeGapStatsForDesiredReduction.update(gap); + + if (_timeGapStatsForDesiredCalcOnTooManyStarves.getNewStatsAvailableFlag()) { + _calculatedJitterBufferFramesUsingMaxGap = ceilf((float)_timeGapStatsForDesiredCalcOnTooManyStarves.getWindowMax() + / (float)BUFFER_SEND_INTERVAL_USECS); + _timeGapStatsForDesiredCalcOnTooManyStarves.clearNewStatsAvailableFlag(); + } + + const int STANDARD_DEVIATION_SAMPLE_COUNT = 500; + if (_stdevStatsForDesiredCalcOnTooManyStarves.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) { + const float NUM_STANDARD_DEVIATIONS = 3.0f; + _calculatedJitterBufferFramesUsingStDev = ceilf(NUM_STANDARD_DEVIATIONS * _stdevStatsForDesiredCalcOnTooManyStarves.getStDev() + / (float)BUFFER_SEND_INTERVAL_USECS); + _stdevStatsForDesiredCalcOnTooManyStarves.reset(); + } + if (_dynamicJitterBuffers) { - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - - // update all stats used for desired frames calculations under dynamic jitter buffer mode - _timeGapStatsForDesiredCalcOnTooManyStarves.update(gap); - _stdevStatsForDesiredCalcOnTooManyStarves.addValue(gap); - _timeGapStatsForDesiredReduction.update(gap); - - const int STANDARD_DEVIATION_SAMPLE_COUNT = 500; - if (_stdevStatsForDesiredCalcOnTooManyStarves.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) { - const float NUM_STANDARD_DEVIATIONS = 3.0f; - _calculatedJitterBufferFramesUsingStDev = (int)ceilf(NUM_STANDARD_DEVIATIONS * _stdevStatsForDesiredCalcOnTooManyStarves.getStDev() - / USECS_PER_FRAME); - _stdevStatsForDesiredCalcOnTooManyStarves.reset(); - } - // if the max gap in window B (_timeGapStatsForDesiredReduction) corresponds to a smaller number of frames than _desiredJitterBufferFrames, // then reduce _desiredJitterBufferFrames to that number of frames. if (_timeGapStatsForDesiredReduction.getNewStatsAvailableFlag() && _timeGapStatsForDesiredReduction.isWindowFilled()) { - int calculatedJitterBufferFrames = ceilf((float)_timeGapStatsForDesiredReduction.getWindowMax() / USECS_PER_FRAME); + int calculatedJitterBufferFrames = ceilf((float)_timeGapStatsForDesiredReduction.getWindowMax() / (float)BUFFER_SEND_INTERVAL_USECS); if (calculatedJitterBufferFrames < _desiredJitterBufferFrames) { _desiredJitterBufferFrames = calculatedJitterBufferFrames; } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 1e58eaa509..c62cf957d7 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -157,7 +157,7 @@ public slots: void perSecondCallbackForUpdatingStats(); private: - void frameReceivedUpdateTimingStats(); + void packetReceivedUpdateTimingStats(); int clampDesiredJitterBufferFramesValue(int desired) const; int writeSamplesForDroppedPackets(int numSamples); @@ -214,6 +214,7 @@ protected: quint64 _lastPacketReceivedTime; MovingMinMaxAvg _timeGapStatsForDesiredCalcOnTooManyStarves; // for Freddy's method + int _calculatedJitterBufferFramesUsingMaxGap; StDev _stdevStatsForDesiredCalcOnTooManyStarves; // for Philip's method int _calculatedJitterBufferFramesUsingStDev; // the most recent desired frames calculated by Philip's method MovingMinMaxAvg _timeGapStatsForDesiredReduction; From 8183fd31995985eb0ff82e11100c404dad728ce3 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 11:17:32 -0700 Subject: [PATCH 08/34] forgot to update starveHistoryWindowSeconds when window A changes --- libraries/audio/src/InboundAudioStream.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 64993af2b5..e7713f9a22 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -316,6 +316,7 @@ void InboundAudioStream::setStaticDesiredJitterBufferFrames(int staticDesiredJit void InboundAudioStream::setWindowSecondsForDesiredCalcOnTooManyStarves(int windowSecondsForDesiredCalcOnTooManyStarves) { _timeGapStatsForDesiredCalcOnTooManyStarves.setWindowIntervals(windowSecondsForDesiredCalcOnTooManyStarves); + _starveHistoryWindowSeconds = windowSecondsForDesiredCalcOnTooManyStarves; } void InboundAudioStream::setWindowSecondsForDesiredReduction(int windowSecondsForDesiredReduction) { From b670226ee390a0c4cdb2b9e88e9852fe0c58cd60 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 11:22:04 -0700 Subject: [PATCH 09/34] removed printf, removed random copy of some file --- libraries/audio/src/InboundAudioStream.cpp | 9 --- .../shared/src/MovingPercentile - Copy.cpp | 61 ------------------- 2 files changed, 70 deletions(-) delete mode 100644 libraries/shared/src/MovingPercentile - Copy.cpp diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index e7713f9a22..54c9de6f2f 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -284,15 +284,6 @@ void InboundAudioStream::setSettings(const Settings& settings) { setWindowStarveThreshold(settings._windowStarveThreshold); setWindowSecondsForDesiredCalcOnTooManyStarves(settings._windowSecondsForDesiredCalcOnTooManyStarves); setWindowSecondsForDesiredReduction(settings._windowSecondsForDesiredReduction); - - - printf("\n\nmax frames over desired: %d\n", settings._maxFramesOverDesired); - printf("dynamic jitter buffers: %d\n", settings._dynamicJitterBuffers); - printf("static desired jbuffer frames: %d\n", settings._staticDesiredJitterBufferFrames); - printf("use stdev: %d\n", settings._useStDevForJitterCalc); - printf("starve threshold: %d\n", settings._windowStarveThreshold); - printf("window A seconds: %d\n", settings._windowSecondsForDesiredCalcOnTooManyStarves); - printf("window B seconds: %d\n", settings._windowSecondsForDesiredReduction); } void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { diff --git a/libraries/shared/src/MovingPercentile - Copy.cpp b/libraries/shared/src/MovingPercentile - Copy.cpp deleted file mode 100644 index ec007b5c22..0000000000 --- a/libraries/shared/src/MovingPercentile - Copy.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// -// MovingPercentile.cpp -// libraries/shared/src -// -// Created by Yixin Wang on 6/4/2014 -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "MovingPercentile.h" - -MovingPercentile::MovingPercentile(int numSamples, float percentile) - : _numSamples(numSamples), - _percentile(percentile), - _samplesSorted(), - _sampleIds(), - _newSampleId(0), - _indexOfPercentile(0), - _valueAtPercentile(0.0f) -{ -} - -void MovingPercentile::updatePercentile(float sample) { - - // insert the new sample into _samplesSorted - int newSampleIndex; - if (_samplesSorted.size() < _numSamples) { - // if not all samples have been filled yet, simply append it - newSampleIndex = _samplesSorted.size(); - _samplesSorted.append(sample); - _sampleIds.append(_newSampleId); - - // update _indexOfPercentile - float index = _percentile * (float)(_samplesSorted.size() - 1); - _indexOfPercentile = (int)(index + 0.5f); // round to int - } else { - // find index of sample with id = _newSampleId and replace it with new sample - newSampleIndex = _sampleIds.indexOf(_newSampleId); - _samplesSorted[newSampleIndex] = sample; - } - - // increment _newSampleId. cycles from 0 thru N-1 - _newSampleId = (_newSampleId == _numSamples - 1) ? 0 : _newSampleId + 1; - - // swap new sample with neighbors in _samplesSorted until it's in sorted order - // try swapping up first, then down. element will only be swapped one direction. - while (newSampleIndex < _samplesSorted.size() - 1 && sample > _samplesSorted[newSampleIndex + 1]) { - _samplesSorted.swap(newSampleIndex, newSampleIndex + 1); - _sampleIds.swap(newSampleIndex, newSampleIndex + 1); - newSampleIndex++; - } - while (newSampleIndex > 0 && sample < _samplesSorted[newSampleIndex - 1]) { - _samplesSorted.swap(newSampleIndex, newSampleIndex - 1); - _sampleIds.swap(newSampleIndex, newSampleIndex - 1); - newSampleIndex--; - } - - // find new value at percentile - _valueAtPercentile = _samplesSorted[_indexOfPercentile]; -} From 63624fae7df4c37808d3510c4b6a400e532de79a Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 11:40:48 -0700 Subject: [PATCH 10/34] cleaned up code, fixed typos --- interface/src/Menu.h | 2 +- interface/src/ui/PreferencesDialog.cpp | 2 +- libraries/audio/src/InboundAudioStream.cpp | 1 - libraries/audio/src/InboundAudioStream.h | 6 +--- libraries/shared/src/MovingEvent.h | 36 ---------------------- 5 files changed, 3 insertions(+), 44 deletions(-) delete mode 100644 libraries/shared/src/MovingEvent.h diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 20097989df..153b402089 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -86,7 +86,7 @@ public: QAction* getActionForOption(const QString& menuOption); const InboundAudioStream::Settings& getReceivedAudioStreamSettings() const { return _receivedAudioStreamSettings; } - void getReceivedAudioStreamSettings(const InboundAudioStream::Settings& receivedAudioStreamSettings) { _receivedAudioStreamSettings = receivedAudioStreamSettings; } + void setReceivedAudioStreamSettings(const InboundAudioStream::Settings& receivedAudioStreamSettings) { _receivedAudioStreamSettings = receivedAudioStreamSettings; } float getFieldOfView() const { return _fieldOfView; } void setFieldOfView(float fieldOfView) { _fieldOfView = fieldOfView; } float getRealWorldFieldOfView() const { return _realWorldFieldOfView; } diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 7189a74579..bde3cfd65a 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -262,7 +262,7 @@ void PreferencesDialog::savePreferences() { streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = ui.windowSecondsForDesiredCalcOnTooManyStarvesSpin->value(); streamSettings._windowSecondsForDesiredReduction = ui.windowSecondsForDesiredReductionSpin->value(); - Menu::getInstance()->getReceivedAudioStreamSettings(streamSettings); + Menu::getInstance()->setReceivedAudioStreamSettings(streamSettings); Application::getInstance()->getAudio()->setReceivedAudioStreamSettings(streamSettings); Application::getInstance()->resizeGL(Application::getInstance()->getGLWidget()->width(), diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 54c9de6f2f..c6e18b054e 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -351,7 +351,6 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { } if (_dynamicJitterBuffers) { - // if the max gap in window B (_timeGapStatsForDesiredReduction) corresponds to a smaller number of frames than _desiredJitterBufferFrames, // then reduce _desiredJitterBufferFrames to that number of frames. if (_timeGapStatsForDesiredReduction.getNewStatsAvailableFlag() && _timeGapStatsForDesiredReduction.isWindowFilled()) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index c62cf957d7..3275729850 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -31,18 +31,15 @@ const int DESIRED_JITTER_BUFFER_FRAMES_PADDING = 1; // _desiredJitterBufferFrames calculation) const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30; - // this controls the window size of the time-weighted avg of frames available. Every time the window fills up, // _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset. const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND; -const int INBOUND_RING_BUFFER_FRAME_CAPACITY = 100; - +// default values for members of the Settings struct const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10; const bool DEFAULT_DYNAMIC_JITTER_BUFFERS = true; const int DEFAULT_STATIC_DESIRED_JITTER_BUFFER_FRAMES = 1; const bool DEFAULT_USE_STDEV_FOR_JITTER_CALC = false; - const int DEFAULT_WINDOW_STARVE_THRESHOLD = 3; const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES = 50; const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION = 10; @@ -229,7 +226,6 @@ protected: // dropping silent frames right now. int _currentJitterBufferFrames; - MovingMinMaxAvg _timeGapStatsForStatsPacket; }; diff --git a/libraries/shared/src/MovingEvent.h b/libraries/shared/src/MovingEvent.h deleted file mode 100644 index 284ed9d890..0000000000 --- a/libraries/shared/src/MovingEvent.h +++ /dev/null @@ -1,36 +0,0 @@ -// -// MovingPercentile.h -// libraries/shared/src -// -// Created by Yixin Wang on 6/4/2014 -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_MovingPercentile_h -#define hifi_MovingPercentile_h - -#include - -class MovingPercentile { - -public: - MovingPercentile(int numSamples, float percentile = 0.5f); - - void updatePercentile(float sample); - float getValueAtPercentile() const { return _valueAtPercentile; } - -private: - const int _numSamples; - const float _percentile; - - QList _samplesSorted; - QList _sampleIds; // incrementally assigned, is cyclic - int _newSampleId; - - int _indexOfPercentile; - float _valueAtPercentile; -}; - -#endif From bf8188c6ffe02279b13a3a486bd2e5edbd0028e9 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 13:38:28 -0700 Subject: [PATCH 11/34] removed duplicate definition --- libraries/audio/src/InboundAudioStream.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index b0a087e9bb..dadc15392b 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -86,10 +86,6 @@ void InboundAudioStream::perSecondCallbackForUpdatingStats() { _timeGapStatsForStatsPacket.currentIntervalComplete(); } -int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { - return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); -} - int InboundAudioStream::parseData(const QByteArray& packet) { PacketType packetType = packetTypeForPacket(packet); From a2d66b9a8ffa802158146d787585e4a5e507407e Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 14:55:38 -0700 Subject: [PATCH 12/34] fixed audio scope; added parseSilentPacketStreamProperties() --- interface/src/Audio.cpp | 10 +++++----- libraries/audio/src/InboundAudioStream.cpp | 13 +++++++------ libraries/audio/src/InboundAudioStream.h | 4 ++++ .../audio/src/MixedProcessedAudioStream.cpp | 19 +++++++++++++++++-- .../audio/src/MixedProcessedAudioStream.h | 3 +++ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 29d6b8473e..c09d733cb5 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1481,17 +1481,17 @@ void Audio::renderScope(int width, int height) { return; static const float backgroundColor[4] = { 0.4f, 0.4f, 0.4f, 0.6f }; - static const float gridColor[4] = { 0.3f, 0.3f, 0.3f, 0.6f }; + static const float gridColor[4] = { 0.7f, 0.7f, 0.7f, 1.0f }; static const float inputColor[4] = { 0.3f, 1.0f, 0.3f, 1.0f }; static const float outputLeftColor[4] = { 1.0f, 0.3f, 0.3f, 1.0f }; static const float outputRightColor[4] = { 0.3f, 0.3f, 1.0f, 1.0f }; static const int gridRows = 2; int gridCols = _framesPerScope; - int x = (width - SCOPE_WIDTH) / 2; - int y = (height - SCOPE_HEIGHT) / 2; - int w = SCOPE_WIDTH; - int h = SCOPE_HEIGHT; + int x = (width - (int)SCOPE_WIDTH) / 2; + int y = (height - (int)SCOPE_HEIGHT) / 2; + int w = (int)SCOPE_WIDTH; + int h = (int)SCOPE_HEIGHT; renderBackground(backgroundColor, x, y, w, h); renderGrid(gridColor, x, y, w, h, gridRows, gridCols); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index dadc15392b..2686bd53c5 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -107,12 +107,7 @@ int InboundAudioStream::parseData(const QByteArray& packet) { int numAudioSamples; if (packetType == PacketTypeSilentAudioFrame) { - // this is a general silent packet; parse the number of silent samples - quint16 numSilentSamples = *(reinterpret_cast(dataAt)); - dataAt += sizeof(quint16); - readBytes += sizeof(quint16); - - numAudioSamples = numSilentSamples; + readBytes += parseSilentPacketStreamProperties(packet.mid(readBytes), numAudioSamples); } else { // parse the info after the seq number and before the audio data (the stream properties) readBytes += parseStreamProperties(packetType, packet.mid(readBytes), numAudioSamples); @@ -171,6 +166,12 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); } +int InboundAudioStream::parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples) { + // this is a general silent packet; parse the number of silent samples + quint16 numSilentSamples = *(reinterpret_cast(packetAfterSeqNum.data())); + numAudioSamples = numSilentSamples; + return sizeof(quint16); +} int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing, bool starveIfNoSamplesPopped) { int samplesPopped = 0; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index dee759042b..a3d8729120 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -172,6 +172,10 @@ protected: /// parses the info between the seq num and the audio data in the network packet and calculates /// how many audio samples this packet contains (used when filling in samples for dropped packets). virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0; + + /// parses a silent packet after the seq. default implementation assumes the number of silent samples + /// is the only thing in packetAfterSeqNum and should work in most cases + virtual int parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples); /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 52581bd096..fd3ecb3bc9 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -28,12 +28,22 @@ int MixedProcessedAudioStream::parseStreamProperties(PacketType type, const QByt // since numAudioSamples is used to know how many samples to add for each dropped packet before this one, // we want to set it to the number of device audio samples since this stream contains device audio samples, not network samples. - const int STEREO_DIVIDER = 2; - numAudioSamples = numNetworkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_DIVIDER * SAMPLE_RATE); + numAudioSamples = networkToDeviceSamples(numNetworkSamples); return 0; } +int MixedProcessedAudioStream::parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples) { + int numNetworkSamples; + int bytesRead = InboundAudioStream::parseSilentPacketStreamProperties(packetAfterSeqNum, numNetworkSamples); + + // since numAudioSamples is used to know how many samples to add for each dropped packet before this one, + // we want to set it to the number of device audio samples since this stream contains device audio samples, not network samples. + numAudioSamples = networkToDeviceSamples(numNetworkSamples); + + return bytesRead; +} + int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { QByteArray outputBuffer; @@ -43,3 +53,8 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& return packetAfterStreamProperties.size(); } + +int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { + const int STEREO_DIVIDER = 2; + return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_DIVIDER * SAMPLE_RATE); +} \ No newline at end of file diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index e033297362..7c89a5106d 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -28,8 +28,11 @@ public: protected: int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); + int parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples); int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); +private: + int networkToDeviceSamples(int networkSamples); private: int _outputFormatChannelsTimesSampleRate; }; From f71e1edd30ad29e5744f0312d29e00e81f9e9dde Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 16:46:12 -0700 Subject: [PATCH 13/34] removed parseSilentPacketStreamProperties() --- libraries/audio/src/InboundAudioStream.cpp | 79 ++++++++++--------- libraries/audio/src/InboundAudioStream.h | 14 ++-- libraries/audio/src/MixedAudioStream.cpp | 6 -- libraries/audio/src/MixedAudioStream.h | 3 - .../audio/src/MixedProcessedAudioStream.cpp | 30 ++----- .../audio/src/MixedProcessedAudioStream.h | 5 +- 6 files changed, 54 insertions(+), 83 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 2686bd53c5..67259c5d99 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -104,13 +104,15 @@ int InboundAudioStream::parseData(const QByteArray& packet) { packetReceivedUpdateTimingStats(); - int numAudioSamples; + int networkSamples; if (packetType == PacketTypeSilentAudioFrame) { - readBytes += parseSilentPacketStreamProperties(packet.mid(readBytes), numAudioSamples); + quint16 numSilentSamples = *(reinterpret_cast(dataAt)); + readBytes += sizeof(quint16); + networkSamples = (int)numSilentSamples; } else { // parse the info after the seq number and before the audio data (the stream properties) - readBytes += parseStreamProperties(packetType, packet.mid(readBytes), numAudioSamples); + readBytes += parseStreamProperties(packetType, packet.mid(readBytes), networkSamples); } // handle this packet based on its arrival status. @@ -120,16 +122,16 @@ int InboundAudioStream::parseData(const QByteArray& packet) { // NOTE: we assume that each dropped packet contains the same number of samples // as the packet we just received. int packetsDropped = arrivalInfo._seqDiffFromExpected; - writeSamplesForDroppedPackets(packetsDropped * numAudioSamples); + writeSamplesForDroppedPackets(packetsDropped * networkSamples); // fall through to OnTime case } case SequenceNumberStats::OnTime: { // Packet is on time; parse its data to the ringbuffer if (packetType == PacketTypeSilentAudioFrame) { - writeDroppableSilentSamples(numAudioSamples); + writeDroppableSilentSamples(networkSamples); } else { - readBytes += parseAudioData(packetType, packet.mid(readBytes), numAudioSamples); + readBytes += parseAudioData(packetType, packet.mid(readBytes), networkSamples); } break; } @@ -162,15 +164,40 @@ int InboundAudioStream::parseData(const QByteArray& packet) { return readBytes; } +int InboundAudioStream::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { + // mixed audio packets do not have any info between the seq num and the audio data. + numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t); + return 0; +} + int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { return _ringBuffer.writeData(packetAfterStreamProperties.data(), numAudioSamples * sizeof(int16_t)); } -int InboundAudioStream::parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples) { - // this is a general silent packet; parse the number of silent samples - quint16 numSilentSamples = *(reinterpret_cast(packetAfterSeqNum.data())); - numAudioSamples = numSilentSamples; - return sizeof(quint16); +int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { + + // calculate how many silent frames we should drop. + int samplesPerFrame = _ringBuffer.getNumFrameSamples(); + int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; + int numSilentFramesToDrop = 0; + + if (silentSamples >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { + + // our avg jitter buffer size exceeds its desired value, so ignore some silent + // frames to get that size as close to desired as possible + int numSilentFramesToDropDesired = _currentJitterBufferFrames - desiredJitterBufferFramesPlusPadding; + int numSilentFramesReceived = silentSamples / samplesPerFrame; + numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); + + // dont reset _currentJitterBufferFrames here; we want to be able to drop further silent frames + // without waiting for _framesAvailableStat to fill up to 10s of samples. + _currentJitterBufferFrames -= numSilentFramesToDrop; + _silentFramesDropped += numSilentFramesToDrop; + + _framesAvailableStat.reset(); + } + + return _ringBuffer.addSilentFrame(silentSamples - numSilentFramesToDrop * samplesPerFrame); } int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing, bool starveIfNoSamplesPopped) { @@ -386,34 +413,8 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { _lastPacketReceivedTime = now; } -int InboundAudioStream::writeDroppableSilentSamples(int numSilentSamples) { - - // calculate how many silent frames we should drop. - int samplesPerFrame = _ringBuffer.getNumFrameSamples(); - int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; - int numSilentFramesToDrop = 0; - - if (numSilentSamples >= samplesPerFrame && _currentJitterBufferFrames > desiredJitterBufferFramesPlusPadding) { - - // our avg jitter buffer size exceeds its desired value, so ignore some silent - // frames to get that size as close to desired as possible - int numSilentFramesToDropDesired = _currentJitterBufferFrames - desiredJitterBufferFramesPlusPadding; - int numSilentFramesReceived = numSilentSamples / samplesPerFrame; - numSilentFramesToDrop = std::min(numSilentFramesToDropDesired, numSilentFramesReceived); - - // dont reset _currentJitterBufferFrames here; we want to be able to drop further silent frames - // without waiting for _framesAvailableStat to fill up to 10s of samples. - _currentJitterBufferFrames -= numSilentFramesToDrop; - _silentFramesDropped += numSilentFramesToDrop; - - _framesAvailableStat.reset(); - } - - return _ringBuffer.addSilentFrame(numSilentSamples - numSilentFramesToDrop * samplesPerFrame); -} - -int InboundAudioStream::writeSamplesForDroppedPackets(int numSamples) { - return writeDroppableSilentSamples(numSamples); +int InboundAudioStream::writeSamplesForDroppedPackets(int networkSamples) { + return writeDroppableSilentSamples(networkSamples); } float InboundAudioStream::getLastPopOutputFrameLoudness() const { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index a3d8729120..62a22b61ab 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -159,7 +159,7 @@ private: void packetReceivedUpdateTimingStats(); int clampDesiredJitterBufferFramesValue(int desired) const; - int writeSamplesForDroppedPackets(int numSamples); + int writeSamplesForDroppedPackets(int networkSamples); void popSamplesNoCheck(int samples); void framesAvailableChanged(); @@ -171,17 +171,15 @@ protected: /// parses the info between the seq num and the audio data in the network packet and calculates /// how many audio samples this packet contains (used when filling in samples for dropped packets). - virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) = 0; + /// default implementation assumes no stream properties and raw audio samples after stream propertiess + virtual int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& networkSamples); - /// parses a silent packet after the seq. default implementation assumes the number of silent samples - /// is the only thing in packetAfterSeqNum and should work in most cases - virtual int parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples); - /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); + virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); - int writeDroppableSilentSamples(int numSilentSamples); + /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer + virtual int writeDroppableSilentSamples(int silentSamples); protected: diff --git a/libraries/audio/src/MixedAudioStream.cpp b/libraries/audio/src/MixedAudioStream.cpp index 0041348d26..85bf71747a 100644 --- a/libraries/audio/src/MixedAudioStream.cpp +++ b/libraries/audio/src/MixedAudioStream.cpp @@ -15,9 +15,3 @@ MixedAudioStream::MixedAudioStream(int numFrameSamples, int numFramesCapacity, c : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) { } - -int MixedAudioStream::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { - // mixed audio packets do not have any info between the seq num and the audio data. - numAudioSamples = packetAfterSeqNum.size() / sizeof(int16_t); - return 0; -} diff --git a/libraries/audio/src/MixedAudioStream.h b/libraries/audio/src/MixedAudioStream.h index 0b1979003d..edb26c486f 100644 --- a/libraries/audio/src/MixedAudioStream.h +++ b/libraries/audio/src/MixedAudioStream.h @@ -20,9 +20,6 @@ public: MixedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings); float getNextOutputFrameLoudness() const { return _ringBuffer.getNextOutputFrameLoudness(); } - -protected: - int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); }; #endif // hifi_MixedAudioStream_h diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index fd3ecb3bc9..2922459140 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -22,29 +22,7 @@ void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCount _ringBuffer.resizeForFrameSize(deviceOutputFrameSize); } -int MixedProcessedAudioStream::parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples) { - // mixed audio packets do not have any info between the seq num and the audio data. - int numNetworkSamples = packetAfterSeqNum.size() / sizeof(int16_t); - - // since numAudioSamples is used to know how many samples to add for each dropped packet before this one, - // we want to set it to the number of device audio samples since this stream contains device audio samples, not network samples. - numAudioSamples = networkToDeviceSamples(numNetworkSamples); - - return 0; -} - -int MixedProcessedAudioStream::parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples) { - int numNetworkSamples; - int bytesRead = InboundAudioStream::parseSilentPacketStreamProperties(packetAfterSeqNum, numNetworkSamples); - - // since numAudioSamples is used to know how many samples to add for each dropped packet before this one, - // we want to set it to the number of device audio samples since this stream contains device audio samples, not network samples. - numAudioSamples = networkToDeviceSamples(numNetworkSamples); - - return bytesRead; -} - -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples) { +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { QByteArray outputBuffer; emit processSamples(packetAfterStreamProperties, outputBuffer); @@ -54,7 +32,11 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& return packetAfterStreamProperties.size(); } +int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { + return InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); +} + int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { const int STEREO_DIVIDER = 2; return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_DIVIDER * SAMPLE_RATE); -} \ No newline at end of file +} diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 7c89a5106d..ec65c8f712 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -27,9 +27,8 @@ public: void outputFormatChanged(int outputFormatChannelCountTimesSampleRate); protected: - int parseStreamProperties(PacketType type, const QByteArray& packetAfterSeqNum, int& numAudioSamples); - int parseSilentPacketStreamProperties(const QByteArray& packetAfterSeqNum, int& numAudioSamples); - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int numAudioSamples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); + int writeDroppableSilentSamples(int silentSamples); private: int networkToDeviceSamples(int networkSamples); From 4ec84b32b076499ade294e297ef653b8aab12dc6 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 17:00:39 -0700 Subject: [PATCH 14/34] updated AudioRingBuffer bytes vs samples returns --- libraries/audio/src/AudioRingBuffer.cpp | 104 ++++++++++------------- libraries/audio/src/AudioRingBuffer.h | 60 +++++++------ tests/audio/src/AudioRingBufferTests.cpp | 37 ++++---- 3 files changed, 97 insertions(+), 104 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 2ad51e4a36..d9cb34ac1b 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -20,18 +20,16 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode, int numFramesCapacity) : - _frameCapacity(numFramesCapacity), - _sampleCapacity(numFrameSamples * numFramesCapacity), - _isFull(false), - _numFrameSamples(numFrameSamples), - _randomAccessMode(randomAccessMode), - _overflowCount(0) +_frameCapacity(numFramesCapacity), +_sampleCapacity(numFrameSamples * numFramesCapacity), +_bufferLength(numFrameSamples * (numFramesCapacity + 1)), +_numFrameSamples(numFrameSamples), +_randomAccessMode(randomAccessMode), +_overflowCount(0) { if (numFrameSamples) { - _buffer = new int16_t[_sampleCapacity]; - if (_randomAccessMode) { - memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); - } + _buffer = new int16_t[_bufferLength]; + memset(_buffer, 0, _bufferLength * sizeof(int16_t)); _nextOutput = _buffer; _endOfLastWrite = _buffer; } else { @@ -53,28 +51,29 @@ void AudioRingBuffer::reset() { void AudioRingBuffer::resizeForFrameSize(int numFrameSamples) { delete[] _buffer; _sampleCapacity = numFrameSamples * _frameCapacity; + _bufferLength = numFrameSamples * (_frameCapacity + 1); _numFrameSamples = numFrameSamples; - _buffer = new int16_t[_sampleCapacity]; + _buffer = new int16_t[_bufferLength]; + memset(_buffer, 0, _bufferLength * sizeof(int16_t)); if (_randomAccessMode) { - memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); + memset(_buffer, 0, _bufferLength * sizeof(int16_t)); } reset(); } void AudioRingBuffer::clear() { - _isFull = false; _endOfLastWrite = _buffer; _nextOutput = _buffer; } int AudioRingBuffer::readSamples(int16_t* destination, int maxSamples) { - return readData((char*) destination, maxSamples * sizeof(int16_t)); + return readData((char*)destination, maxSamples * sizeof(int16_t)) / sizeof(int16_t); } int AudioRingBuffer::readData(char *data, int maxSize) { // only copy up to the number of samples we have available - int numReadSamples = std::min((int) (maxSize / sizeof(int16_t)), samplesAvailable()); + int numReadSamples = std::min((int)(maxSize / sizeof(int16_t)), samplesAvailable()); // If we're in random access mode, then we consider our number of available read samples slightly // differently. Namely, if anything has been written, we say we have as many samples as they ask for @@ -83,16 +82,16 @@ int AudioRingBuffer::readData(char *data, int maxSize) { numReadSamples = _endOfLastWrite ? (maxSize / sizeof(int16_t)) : 0; } - if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { + if (_nextOutput + numReadSamples > _buffer + _bufferLength) { // we're going to need to do two reads to get this data, it wraps around the edge // read to the end of the buffer - int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; + int numSamplesToEnd = (_buffer + _bufferLength) - _nextOutput; memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); if (_randomAccessMode) { memset(_nextOutput, 0, numSamplesToEnd * sizeof(int16_t)); // clear it } - + // read the rest from the beginning of the buffer memcpy(data + (numSamplesToEnd * sizeof(int16_t)), _buffer, (numReadSamples - numSamplesToEnd) * sizeof(int16_t)); if (_randomAccessMode) { @@ -108,22 +107,19 @@ int AudioRingBuffer::readData(char *data, int maxSize) { // push the position of _nextOutput by the number of samples read _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); - if (numReadSamples > 0) { - _isFull = false; - } return numReadSamples * sizeof(int16_t); } -int AudioRingBuffer::writeSamples(const int16_t* source, int maxSamples) { - return writeData((const char*) source, maxSamples * sizeof(int16_t)); +int AudioRingBuffer::writeSamples(const int16_t* source, int maxSamples) { + return writeData((const char*)source, maxSamples * sizeof(int16_t)) / sizeof(int16_t); } int AudioRingBuffer::writeData(const char* data, int maxSize) { // make sure we have enough bytes left for this to be the right amount of audio // otherwise we should not copy that data, and leave the buffer pointers where they are int samplesToCopy = std::min((int)(maxSize / sizeof(int16_t)), _sampleCapacity); - + int samplesRoomFor = _sampleCapacity - samplesAvailable(); if (samplesToCopy > samplesRoomFor) { // there's not enough room for this write. erase old data to make room for this new data @@ -132,19 +128,16 @@ int AudioRingBuffer::writeData(const char* data, int maxSize) { _overflowCount++; qDebug() << "Overflowed ring buffer! Overwriting old data"; } - - if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { + + if (_endOfLastWrite + samplesToCopy <= _buffer + _bufferLength) { memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - if (samplesToCopy > 0 && _endOfLastWrite == _nextOutput) { - _isFull = true; - } return samplesToCopy * sizeof(int16_t); } @@ -158,61 +151,52 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { } void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { - if (numSamples > 0) { - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); - _isFull = false; - } + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); } int AudioRingBuffer::samplesAvailable() const { if (!_endOfLastWrite) { return 0; } - if (_isFull) { - return _sampleCapacity; - } int sampleDifference = _endOfLastWrite - _nextOutput; if (sampleDifference < 0) { - sampleDifference += _sampleCapacity; + sampleDifference += _bufferLength; } return sampleDifference; } -int AudioRingBuffer::addSilentFrame(int numSilentSamples) { +int AudioRingBuffer::addSilentSamples(int silentSamples) { int samplesRoomFor = _sampleCapacity - samplesAvailable(); - if (numSilentSamples > samplesRoomFor) { + if (silentSamples > samplesRoomFor) { // there's not enough room for this write. write as many silent samples as we have room for - numSilentSamples = samplesRoomFor; + silentSamples = samplesRoomFor; qDebug() << "Dropping some silent samples to prevent ring buffer overflow"; } // memset zeroes into the buffer, accomodate a wrap around the end // push the _endOfLastWrite to the correct spot - if (_endOfLastWrite + numSilentSamples <= _buffer + _sampleCapacity) { - memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); + if (_endOfLastWrite + silentSamples <= _buffer + _bufferLength) { + memset(_endOfLastWrite, 0, silentSamples * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _bufferLength) - _endOfLastWrite; memset(_endOfLastWrite, 0, numSamplesToEnd * sizeof(int16_t)); - memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); - } - _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numSilentSamples); - if (numSilentSamples > 0 && _nextOutput == _endOfLastWrite) { - _isFull = true; + memset(_buffer, 0, (silentSamples - numSamplesToEnd) * sizeof(int16_t)); } + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, silentSamples); - return numSilentSamples * sizeof(int16_t); + return silentSamples; } int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _bufferLength) { // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - _sampleCapacity; + return position + numSamplesShift - _bufferLength; } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { // this shift will go around to the end of the ring - return position + numSamplesShift + _sampleCapacity; + return position + numSamplesShift + _bufferLength; } else { return position + numSamplesShift; } @@ -221,22 +205,22 @@ int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int float AudioRingBuffer::getFrameLoudness(const int16_t* frameStart) const { float loudness = 0.0f; const int16_t* sampleAt = frameStart; - const int16_t* _bufferLastAt = _buffer + _sampleCapacity - 1; - + const int16_t* _bufferLastAt = _buffer + _bufferLength - 1; + for (int i = 0; i < _numFrameSamples; ++i) { loudness += fabsf(*sampleAt); sampleAt = sampleAt == _bufferLastAt ? _buffer : sampleAt + 1; } loudness /= _numFrameSamples; loudness /= MAX_SAMPLE_VALUE; - - return loudness; -} -float AudioRingBuffer::getNextOutputFrameLoudness() const { - return getFrameLoudness(_nextOutput); + return loudness; } float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { return getFrameLoudness(&(*frameStart)); } + +float AudioRingBuffer::getNextOutputFrameLoudness() const { + return getFrameLoudness(_nextOutput); +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 4dd258b2c5..be4dcaf545 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -30,7 +30,7 @@ const int NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL = 512; const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t); const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL - / (float) SAMPLE_RATE) * USECS_PER_SECOND); + / (float)SAMPLE_RATE) * USECS_PER_SECOND); const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); @@ -44,33 +44,33 @@ public: void reset(); void resizeForFrameSize(int numFrameSamples); - + void clear(); int getSampleCapacity() const { return _sampleCapacity; } int getFrameCapacity() const { return _frameCapacity; } - + int readSamples(int16_t* destination, int maxSamples); int writeSamples(const int16_t* source, int maxSamples); - + int readData(char* data, int maxSize); int writeData(const char* data, int maxSize); - + int16_t& operator[](const int index); const int16_t& operator[] (const int index) const; - + void shiftReadPosition(unsigned int numSamples); float getNextOutputFrameLoudness() const; - + int samplesAvailable() const; int framesAvailable() const { return samplesAvailable() / _numFrameSamples; } int getNumFrameSamples() const { return _numFrameSamples; } - + int getOverflowCount() const { return _overflowCount; } /// how many times has the ring buffer has overwritten old data - - int addSilentFrame(int numSilentSamples); + + int addSilentSamples(int samples); private: float getFrameLoudness(const int16_t* frameStart) const; @@ -79,12 +79,12 @@ protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); - + int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; int _frameCapacity; int _sampleCapacity; - bool _isFull; + int _bufferLength; // actual length of _buffer: will be one frame larger than _sampleCapacity int _numFrameSamples; int16_t* _nextOutput; int16_t* _endOfLastWrite; @@ -97,13 +97,13 @@ public: class ConstIterator { //public std::iterator < std::forward_iterator_tag, int16_t > { public: ConstIterator() - : _capacity(0), + : _bufferLength(0), _bufferFirst(NULL), _bufferLast(NULL), _at(NULL) {} ConstIterator(int16_t* bufferFirst, int capacity, int16_t* at) - : _capacity(capacity), + : _bufferLength(capacity), _bufferFirst(bufferFirst), _bufferLast(bufferFirst + capacity - 1), _at(at) {} @@ -113,7 +113,7 @@ public: const int16_t& operator*() { return *_at; } ConstIterator& operator=(const ConstIterator& rhs) { - _capacity = rhs._capacity; + _bufferLength = rhs._bufferLength; _bufferFirst = rhs._bufferFirst; _bufferLast = rhs._bufferLast; _at = rhs._at; @@ -147,40 +147,50 @@ public: } ConstIterator operator+(int i) { - return ConstIterator(_bufferFirst, _capacity, atShiftedBy(i)); + return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(i)); } ConstIterator operator-(int i) { - return ConstIterator(_bufferFirst, _capacity, atShiftedBy(-i)); + return ConstIterator(_bufferFirst, _bufferLength, atShiftedBy(-i)); } void readSamples(int16_t* dest, int numSamples) { + int16_t* at = _at; for (int i = 0; i < numSamples; i++) { - *dest = *(*this); + *dest = *at; ++dest; - ++(*this); + at = (at == _bufferLast) ? _bufferFirst : at + 1; } } - + + void readSamplesWithFade(int16_t* dest, int numSamples, float fade) { + int16_t* at = _at; + for (int i = 0; i < numSamples; i++) { + *dest = (float)*at * fade; + ++dest; + at = (at == _bufferLast) ? _bufferFirst : at + 1; + } + } + private: int16_t* atShiftedBy(int i) { - i = (_at - _bufferFirst + i) % _capacity; + i = (_at - _bufferFirst + i) % _bufferLength; if (i < 0) { - i += _capacity; + i += _bufferLength; } return _bufferFirst + i; } private: - int _capacity; + int _bufferLength; int16_t* _bufferFirst; int16_t* _bufferLast; int16_t* _at; }; - ConstIterator nextOutput() const { return ConstIterator(_buffer, _sampleCapacity, _nextOutput); } + ConstIterator nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); } float getFrameLoudness(ConstIterator frameStart) const; }; -#endif // hifi_AudioRingBuffer_h +#endif // hifi_AudioRingBuffer_h \ No newline at end of file diff --git a/tests/audio/src/AudioRingBufferTests.cpp b/tests/audio/src/AudioRingBufferTests.cpp index b9ed596e52..f31f9988d6 100644 --- a/tests/audio/src/AudioRingBufferTests.cpp +++ b/tests/audio/src/AudioRingBufferTests.cpp @@ -27,28 +27,28 @@ void AudioRingBufferTests::runAllTests() { int16_t readData[10000]; int readIndexAt; - + AudioRingBuffer ringBuffer(10, false, 10); // makes buffer of 100 int16_t samples for (int T = 0; T < 300; T++) { - + writeIndexAt = 0; readIndexAt = 0; // write 73 samples, 73 samples in buffer - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 73) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 73); assertBufferSize(ringBuffer, 73); // read 43 samples, 30 samples in buffer - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 43) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 43); assertBufferSize(ringBuffer, 30); // write 70 samples, 100 samples in buffer (full) - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 70) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 70); assertBufferSize(ringBuffer, 100); // read 100 samples, 0 samples in buffer (empty) - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100); assertBufferSize(ringBuffer, 0); @@ -65,15 +65,15 @@ void AudioRingBufferTests::runAllTests() { readIndexAt = 0; // write 59 samples, 59 samples in buffer - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 59) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 59); assertBufferSize(ringBuffer, 59); // write 99 samples, 100 samples in buffer - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 99) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 99); assertBufferSize(ringBuffer, 100); // read 100 samples, 0 samples in buffer - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100); assertBufferSize(ringBuffer, 0); // verify 100 samples of read data @@ -88,23 +88,23 @@ void AudioRingBufferTests::runAllTests() { readIndexAt = 0; // write 77 samples, 77 samples in buffer - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 77) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 77); assertBufferSize(ringBuffer, 77); // write 24 samples, 100 samples in buffer (overwrote one sample: "0") - writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 24) / sizeof(int16_t); + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 24); assertBufferSize(ringBuffer, 100); // write 29 silent samples, 100 samples in buffer, make sure non were added int samplesWritten; - if ((samplesWritten = ringBuffer.addSilentFrame(29)) != 0) { - qDebug("addSilentFrame(29) incorrect! Expected: 0 Actual: %d", samplesWritten); + if ((samplesWritten = ringBuffer.addSilentSamples(29)) != 0) { + qDebug("addSilentSamples(29) incorrect! Expected: 0 Actual: %d", samplesWritten); return; } assertBufferSize(ringBuffer, 100); // read 3 samples, 97 samples in buffer (expect to read "1", "2", "3") - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3); for (int i = 0; i < 3; i++) { if (readData[i] != i + 1) { qDebug("Second readData[%d] incorrect! Expcted: %d Actual: %d", i, i + 1, readData[i]); @@ -114,14 +114,14 @@ void AudioRingBufferTests::runAllTests() { assertBufferSize(ringBuffer, 97); // write 4 silent samples, 100 samples in buffer - if ((samplesWritten = ringBuffer.addSilentFrame(4) / sizeof(int16_t)) != 3) { - qDebug("addSilentFrame(4) incorrect! Exptected: 3 Actual: %d", samplesWritten); + if ((samplesWritten = ringBuffer.addSilentSamples(4)) != 3) { + qDebug("addSilentSamples(4) incorrect! Exptected: 3 Actual: %d", samplesWritten); return; } assertBufferSize(ringBuffer, 100); // read back 97 samples (the non-silent samples), 3 samples in buffer (expect to read "4" thru "100") - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 97) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 97); for (int i = 3; i < 100; i++) { if (readData[i] != i + 1) { qDebug("third readData[%d] incorrect! Expcted: %d Actual: %d", i, i + 1, readData[i]); @@ -131,7 +131,7 @@ void AudioRingBufferTests::runAllTests() { assertBufferSize(ringBuffer, 3); // read back 3 silent samples, 0 samples in buffer - readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3) / sizeof(int16_t); + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3); for (int i = 100; i < 103; i++) { if (readData[i] != 0) { qDebug("Fourth readData[%d] incorrect! Expcted: %d Actual: %d", i, 0, readData[i]); @@ -143,4 +143,3 @@ void AudioRingBufferTests::runAllTests() { qDebug() << "PASSED"; } - From 7aa5a1f830138c9b5dd0f5d5f970a11298f229a5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 17:02:13 -0700 Subject: [PATCH 15/34] fixed function name --- libraries/audio/src/InboundAudioStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 67259c5d99..15e940ac49 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -197,7 +197,7 @@ int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { _framesAvailableStat.reset(); } - return _ringBuffer.addSilentFrame(silentSamples - numSilentFramesToDrop * samplesPerFrame); + return _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); } int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing, bool starveIfNoSamplesPopped) { From fea97f8fe8b5ea7da42ea6ccfc1ead7eae135d65 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 17:17:30 -0700 Subject: [PATCH 16/34] added networkSamples buffer to MixedProcessedAudioStream --- interface/src/Audio.cpp | 2 +- .../audio/src/MixedProcessedAudioStream.cpp | 23 +++++++++++++++---- .../audio/src/MixedProcessedAudioStream.h | 4 ++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c09d733cb5..4b05ed9ac9 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -905,7 +905,7 @@ void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& s unsigned int delayCount = delay * _desiredOutputFormat.channelCount(); unsigned int silentCount = (remaining < delayCount) ? remaining : delayCount; if (silentCount) { - _spatialAudioRingBuffer.addSilentFrame(silentCount); + _spatialAudioRingBuffer.addSilentSamples(silentCount); } // Recalculate the number of remaining samples diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 2922459140..5693af7c6e 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -12,7 +12,8 @@ #include "MixedProcessedAudioStream.h" MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings) - : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) + : InboundAudioStream(numFrameSamples, numFramesCapacity, settings), + _networkSamplesWritten(0) { } @@ -24,6 +25,9 @@ void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCount int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { + memcpy(&_networkSamples[_networkSamplesWritten], packetAfterStreamProperties.data(), packetAfterStreamProperties.size()); + _networkSamplesWritten += packetAfterStreamProperties.size() / sizeof(int16_t); + QByteArray outputBuffer; emit processSamples(packetAfterStreamProperties, outputBuffer); @@ -33,10 +37,21 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& } int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { - return InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); + int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); + + int networkSilentSamplesWritten = deviceToNetworkSamples(deviceSilentSamplesWritten); + memset(&_networkSamples[_networkSamplesWritten], 0, networkSilentSamplesWritten * sizeof(int16_t)); + _networkSamplesWritten += networkSilentSamplesWritten; + + return deviceSilentSamplesWritten; } +static const int STEREO_FACTOR = 2; + int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { - const int STEREO_DIVIDER = 2; - return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_DIVIDER * SAMPLE_RATE); + return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_FACTOR * SAMPLE_RATE); +} + +int MixedProcessedAudioStream::deviceToNetworkSamples(int deviceSamples) { + return deviceSamples * (STEREO_FACTOR * SAMPLE_RATE) / _outputFormatChannelsTimesSampleRate; } diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index ec65c8f712..1ba8b7a29d 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -32,8 +32,12 @@ protected: private: int networkToDeviceSamples(int networkSamples); + int deviceToNetworkSamples(int deviceSamples); private: int _outputFormatChannelsTimesSampleRate; + + int16_t _networkSamples[10 * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; + int _networkSamplesWritten; }; #endif // hifi_MixedProcessedAudioStream_h From 7a063b8bc827d409e87e8ce78b50674a6bcf5920 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 8 Aug 2014 18:16:25 -0700 Subject: [PATCH 17/34] scope seems to be working now using networkSamples --- interface/src/Audio.cpp | 9 +++++++-- interface/src/Audio.h | 1 + libraries/audio/src/InboundAudioStream.cpp | 2 ++ libraries/audio/src/InboundAudioStream.h | 3 +++ libraries/audio/src/MixedProcessedAudioStream.h | 5 +++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 4b05ed9ac9..8aac32849e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -121,6 +121,7 @@ Audio::Audio(QObject* parent) : _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedAudioStreamSamples, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::dataParsed, this, &Audio::updateScopeBuffers, Qt::DirectConnection); } void Audio::init(QGLWidget *parent) { @@ -777,11 +778,13 @@ void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QBy numNetworkOutputSamples, numDeviceOutputSamples, _desiredOutputFormat, _outputFormat); +} - +void Audio::updateScopeBuffers() { if (_scopeEnabled && !_scopeEnabledPause) { unsigned int numAudioChannels = _desiredOutputFormat.channelCount(); - const int16_t* samples = receivedSamples; + const int16_t* samples = _receivedAudioStream.getNetworkSamples(); + int numNetworkOutputSamples = _receivedAudioStream.getNetworkSamplesWritten(); for (int numSamples = numNetworkOutputSamples / numAudioChannels; numSamples > 0; numSamples -= NETWORK_SAMPLES_PER_FRAME) { unsigned int audioChannel = 0; @@ -801,6 +804,8 @@ void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QBy samples += NETWORK_SAMPLES_PER_FRAME * numAudioChannels; } } + + _receivedAudioStream.clearNetworkSamples(); } void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index a93b8c5be7..c080557576 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -106,6 +106,7 @@ public slots: void parseAudioStreamStatsPacket(const QByteArray& packet); void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); + void updateScopeBuffers(); void handleAudioInput(); void reset(); void resetStats(); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 15e940ac49..f393a4af44 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -161,6 +161,8 @@ int InboundAudioStream::parseData(const QByteArray& packet) { framesAvailableChanged(); + emit dataParsed(); + return readBytes; } diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index 62a22b61ab..dac1bb138c 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -149,6 +149,9 @@ public: int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); } +signals: + void dataParsed(); + public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 1ba8b7a29d..e1a54e4449 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -26,6 +26,11 @@ signals: public: void outputFormatChanged(int outputFormatChannelCountTimesSampleRate); + const int16_t* getNetworkSamples() const { return _networkSamples; } + int getNetworkSamplesWritten() const { return _networkSamplesWritten; } + + void clearNetworkSamples() { _networkSamplesWritten = 0; } + protected: int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); int writeDroppableSilentSamples(int silentSamples); From a1ea3933242c959636d72aae376582161befa97a Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 11 Aug 2014 09:14:01 -0700 Subject: [PATCH 18/34] added a comment --- libraries/audio/src/MixedProcessedAudioStream.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index e1a54e4449..b85637a288 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -41,6 +41,8 @@ private: private: int _outputFormatChannelsTimesSampleRate; + // this buffer keeps a copy of the network samples written during parseData() for the sole purpose + // of passing it on to the audio scope int16_t _networkSamples[10 * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; int _networkSamplesWritten; }; From 1f011bfe9d559e2d9069a12107ea1fa0ec97ce5d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 11 Aug 2014 11:22:37 -0700 Subject: [PATCH 19/34] repetition-with-fade option added, not implemented --- assignment-client/src/Agent.cpp | 2 +- assignment-client/src/audio/AudioMixer.cpp | 8 ++ .../resources/web/settings/describe.json | 6 ++ interface/src/Menu.cpp | 2 + interface/src/ui/PreferencesDialog.cpp | 8 +- interface/ui/preferencesDialog.ui | 99 ++++++++++++++++++- libraries/audio/src/InboundAudioStream.cpp | 4 +- libraries/audio/src/InboundAudioStream.h | 18 +++- 8 files changed, 130 insertions(+), 17 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index ecc6414622..e7b11f6029 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -43,7 +43,7 @@ Agent::Agent(const QByteArray& packet) : _receivedAudioStream(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, InboundAudioStream::Settings(0, false, RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES, false, DEFAULT_WINDOW_STARVE_THRESHOLD, DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES, - DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION)), + DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION, false)), _avatarHashMap() { // be the parent of the script engine so it gets moved when we do diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0f76d17da1..bd8877a128 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -477,6 +477,14 @@ void AudioMixer::run() { } qDebug() << "Window B length:" << _streamSettings._windowSecondsForDesiredReduction << "seconds"; + const QString REPETITION_WITH_FADE_JSON_KEY = "H-repetition-with-fade"; + _streamSettings._repetitionWithFade = audioGroupObject[REPETITION_WITH_FADE_JSON_KEY].toBool(); + if (_streamSettings._repetitionWithFade) { + qDebug() << "Repetition with fade enabled"; + } else { + qDebug() << "Repetition with fade disabled"; + } + const QString UNATTENUATED_ZONE_KEY = "Z-unattenuated-zone"; diff --git a/domain-server/resources/web/settings/describe.json b/domain-server/resources/web/settings/describe.json index bb63c5f0a0..db2809bffb 100644 --- a/domain-server/resources/web/settings/describe.json +++ b/domain-server/resources/web/settings/describe.json @@ -45,6 +45,12 @@ "placeholder": "10", "default": "10" }, + "H-repetition-with-fade": { + "type": "checkbox", + "label": "Repetition with Fade:", + "help": "If enabled, dropped frames and mixing during starves will repeat the last frame, eventually fading to silence", + "default": true + }, "Z-unattenuated-zone": { "label": "Unattenuated Zone", "help": "Boxes for source and listener (corner x, corner y, corner z, size x, size y, size z, corner x, corner y, corner z, size x, size y, size z)", diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index ecf28bcb17..a0943ebb52 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -638,6 +638,7 @@ void Menu::loadSettings(QSettings* settings) { _receivedAudioStreamSettings._windowStarveThreshold = settings->value("windowStarveThreshold", DEFAULT_WINDOW_STARVE_THRESHOLD).toInt(); _receivedAudioStreamSettings._windowSecondsForDesiredCalcOnTooManyStarves = settings->value("windowSecondsForDesiredCalcOnTooManyStarves", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES).toInt(); _receivedAudioStreamSettings._windowSecondsForDesiredReduction = settings->value("windowSecondsForDesiredReduction", DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION).toInt(); + _receivedAudioStreamSettings._repetitionWithFade = settings->value("repetitionWithFade", DEFAULT_REPETITION_WITH_FADE).toBool(); _fieldOfView = loadSetting(settings, "fieldOfView", DEFAULT_FIELD_OF_VIEW_DEGREES); _realWorldFieldOfView = loadSetting(settings, "realWorldFieldOfView", DEFAULT_REAL_WORLD_FIELD_OF_VIEW_DEGREES); @@ -695,6 +696,7 @@ void Menu::saveSettings(QSettings* settings) { settings->setValue("windowStarveThreshold", _receivedAudioStreamSettings._windowStarveThreshold); settings->setValue("windowSecondsForDesiredCalcOnTooManyStarves", _receivedAudioStreamSettings._windowSecondsForDesiredCalcOnTooManyStarves); settings->setValue("windowSecondsForDesiredReduction", _receivedAudioStreamSettings._windowSecondsForDesiredReduction); + settings->setValue("repetitionWithFade", _receivedAudioStreamSettings._repetitionWithFade); settings->setValue("fieldOfView", _fieldOfView); settings->setValue("faceshiftEyeDeflection", _faceshiftEyeDeflection); diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index bde3cfd65a..c585b6ba0c 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -152,18 +152,13 @@ void PreferencesDialog::loadPreferences() { const InboundAudioStream::Settings& streamSettings = menuInstance->getReceivedAudioStreamSettings(); ui.dynamicJitterBuffersCheckBox->setChecked(streamSettings._dynamicJitterBuffers); - ui.staticDesiredJitterBufferFramesSpin->setValue(streamSettings._staticDesiredJitterBufferFrames); - ui.maxFramesOverDesiredSpin->setValue(streamSettings._maxFramesOverDesired); - ui.useStdevForJitterCalcCheckBox->setChecked(streamSettings._useStDevForJitterCalc); - ui.windowStarveThresholdSpin->setValue(streamSettings._windowStarveThreshold); - ui.windowSecondsForDesiredCalcOnTooManyStarvesSpin->setValue(streamSettings._windowSecondsForDesiredCalcOnTooManyStarves); - ui.windowSecondsForDesiredReductionSpin->setValue(streamSettings._windowSecondsForDesiredReduction); + ui.repetitionWithFadeCheckBox->setChecked(streamSettings._repetitionWithFade); ui.realWorldFieldOfViewSpin->setValue(menuInstance->getRealWorldFieldOfView()); @@ -261,6 +256,7 @@ void PreferencesDialog::savePreferences() { streamSettings._windowStarveThreshold = ui.windowStarveThresholdSpin->value(); streamSettings._windowSecondsForDesiredCalcOnTooManyStarves = ui.windowSecondsForDesiredCalcOnTooManyStarvesSpin->value(); streamSettings._windowSecondsForDesiredReduction = ui.windowSecondsForDesiredReductionSpin->value(); + streamSettings._repetitionWithFade = ui.repetitionWithFadeCheckBox->isChecked(); Menu::getInstance()->setReceivedAudioStreamSettings(streamSettings); Application::getInstance()->getAudio()->setReceivedAudioStreamSettings(streamSettings); diff --git a/interface/ui/preferencesDialog.ui b/interface/ui/preferencesDialog.ui index cddc0f1299..e35c66af5a 100644 --- a/interface/ui/preferencesDialog.ui +++ b/interface/ui/preferencesDialog.ui @@ -1683,7 +1683,7 @@ padding: 10px;margin-top:10px - + Arial @@ -1777,7 +1777,7 @@ padding: 10px;margin-top:10px - + Arial @@ -1867,7 +1867,7 @@ padding: 10px;margin-top:10px - + Arial @@ -1961,7 +1961,7 @@ padding: 10px;margin-top:10px - + Arial @@ -2055,7 +2055,7 @@ padding: 10px;margin-top:10px - + Arial @@ -2110,6 +2110,95 @@ padding: 10px;margin-top:10px + + + + + + + 0 + + + 10 + + + 0 + + + 10 + + + + + + Arial + + + + color: rgb(51, 51, 51) + + + Repetition with Fade + + + 15 + + + repetitionWithFadeCheckBox + + + + + + + + Arial + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 32 + 0 + + + + + 0 + 0 + + + + + + + + 32 + 32 + + + + + + diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index f393a4af44..39cd544b15 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -41,7 +41,8 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _starveThreshold(settings._windowStarveThreshold), _framesAvailableStat(), _currentJitterBufferFrames(0), - _timeGapStatsForStatsPacket(0, STATS_FOR_STATS_PACKET_WINDOW_SECONDS) + _timeGapStatsForStatsPacket(0, STATS_FOR_STATS_PACKET_WINDOW_SECONDS), + _repetitionWithFade(settings._repetitionWithFade) { } @@ -333,6 +334,7 @@ void InboundAudioStream::setSettings(const Settings& settings) { setWindowStarveThreshold(settings._windowStarveThreshold); setWindowSecondsForDesiredCalcOnTooManyStarves(settings._windowSecondsForDesiredCalcOnTooManyStarves); setWindowSecondsForDesiredReduction(settings._windowSecondsForDesiredReduction); + setRepetitionWithFade(settings._repetitionWithFade); } void InboundAudioStream::setDynamicJitterBuffers(bool dynamicJitterBuffers) { diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index dac1bb138c..f41d9255ff 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -43,6 +43,7 @@ const bool DEFAULT_USE_STDEV_FOR_JITTER_CALC = false; const int DEFAULT_WINDOW_STARVE_THRESHOLD = 3; const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES = 50; const int DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION = 10; +const bool DEFAULT_REPETITION_WITH_FADE = true; class InboundAudioStream : public NodeData { Q_OBJECT @@ -56,19 +57,21 @@ public: _useStDevForJitterCalc(DEFAULT_USE_STDEV_FOR_JITTER_CALC), _windowStarveThreshold(DEFAULT_WINDOW_STARVE_THRESHOLD), _windowSecondsForDesiredCalcOnTooManyStarves(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES), - _windowSecondsForDesiredReduction(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION) + _windowSecondsForDesiredReduction(DEFAULT_WINDOW_SECONDS_FOR_DESIRED_REDUCTION), + _repetitionWithFade(DEFAULT_REPETITION_WITH_FADE) {} Settings(int maxFramesOverDesired, bool dynamicJitterBuffers, int staticDesiredJitterBufferFrames, bool useStDevForJitterCalc, int windowStarveThreshold, int windowSecondsForDesiredCalcOnTooManyStarves, - int _windowSecondsForDesiredReduction) + int _windowSecondsForDesiredReduction, bool repetitionWithFade) : _maxFramesOverDesired(maxFramesOverDesired), _dynamicJitterBuffers(dynamicJitterBuffers), _staticDesiredJitterBufferFrames(staticDesiredJitterBufferFrames), _useStDevForJitterCalc(useStDevForJitterCalc), _windowStarveThreshold(windowStarveThreshold), _windowSecondsForDesiredCalcOnTooManyStarves(windowSecondsForDesiredCalcOnTooManyStarves), - _windowSecondsForDesiredReduction(windowSecondsForDesiredCalcOnTooManyStarves) + _windowSecondsForDesiredReduction(windowSecondsForDesiredCalcOnTooManyStarves), + _repetitionWithFade(repetitionWithFade) {} // max number of frames over desired in the ringbuffer. @@ -86,6 +89,10 @@ public: int _windowStarveThreshold; int _windowSecondsForDesiredCalcOnTooManyStarves; int _windowSecondsForDesiredReduction; + + // if true, the prev frame will be repeated (fading to silence) for dropped frames. + // otherwise, silence will be inserted. + bool _repetitionWithFade; }; public: @@ -116,7 +123,8 @@ public: void setWindowStarveThreshold(int windowStarveThreshold) { _starveThreshold = windowStarveThreshold; } void setWindowSecondsForDesiredCalcOnTooManyStarves(int windowSecondsForDesiredCalcOnTooManyStarves); void setWindowSecondsForDesiredReduction(int windowSecondsForDesiredReduction); - + void setRepetitionWithFade(bool repetitionWithFade) { _repetitionWithFade = repetitionWithFade; } + virtual AudioStreamStats getAudioStreamStats() const; @@ -234,6 +242,8 @@ protected: int _currentJitterBufferFrames; MovingMinMaxAvg _timeGapStatsForStatsPacket; + + bool _repetitionWithFade; }; #endif // hifi_InboundAudioStream_h From e276d15ed4a82070cdedd9cdeef870ca4ef92d1d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 11 Aug 2014 16:25:43 -0700 Subject: [PATCH 20/34] repetition-with-fade implemented; testing interface crash --- assignment-client/src/audio/AudioMixer.cpp | 37 ++++-- .../resources/web/settings/describe.json | 4 +- interface/src/Audio.cpp | 111 ++++++++++++++---- interface/src/Audio.h | 17 ++- libraries/audio/src/AudioRingBuffer.cpp | 42 +++++++ libraries/audio/src/AudioRingBuffer.h | 6 +- libraries/audio/src/InboundAudioStream.cpp | 46 +++++++- libraries/audio/src/InboundAudioStream.h | 11 +- .../audio/src/MixedProcessedAudioStream.cpp | 42 ++++--- .../audio/src/MixedProcessedAudioStream.h | 20 ++-- 10 files changed, 261 insertions(+), 75 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bd8877a128..509a965bf4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -95,6 +95,26 @@ const float ATTENUATION_EPSILON_DISTANCE = 0.1f; int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd, AvatarAudioStream* listeningNodeStream) { + // If repetition with fade is enabled: + // If streamToAdd could not provide a frame (it was starved), then we'll mix its previously-mixed frame + // This is preferable to not mixing it at all since that's equivalent to inserting silence. + // Basically, we'll repeat that last frame until it has a frame to mix. Depending on how many times + // we've repeated that frame in a row, we'll gradually fade that repeated frame into silence. + // This improves the perceived quality of the audio slightly. + + float repeatedFrameFadeFactor = 1.0f; + + if (!streamToAdd->lastPopSucceeded()) { + if (_streamSettings._repetitionWithFade) { + repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1); + if (repeatedFrameFadeFactor == 0.0f) { + return 0; + } + } else { + return 0; + } + } + float bearingRelativeAngleToSource = 0.0f; float attenuationCoefficient = 1.0f; int numSamplesDelay = 0; @@ -216,12 +236,13 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* int delayedChannelIndex = 0; const int SINGLE_STEREO_OFFSET = 2; + float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor; for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 4) { // setup the int16_t variables for the two sample sets - correctStreamSample[0] = streamPopOutput[s / 2] * attenuationCoefficient; - correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationCoefficient; + correctStreamSample[0] = streamPopOutput[s / 2] * attenuationAndFade; + correctStreamSample[1] = streamPopOutput[(s / 2) + 1] * attenuationAndFade; delayedChannelIndex = s + (numSamplesDelay * 2) + delayedChannelOffset; @@ -237,7 +258,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* if (numSamplesDelay > 0) { // if there was a sample delay for this stream, we need to pull samples prior to the popped output // to stick at the beginning - float attenuationAndWeakChannelRatio = attenuationCoefficient * weakChannelAmplitudeRatio; + float attenuationAndWeakChannelRatioAndFade = attenuationCoefficient * weakChannelAmplitudeRatio * repeatedFrameFadeFactor; AudioRingBuffer::ConstIterator delayStreamPopOutput = streamPopOutput - numSamplesDelay; // TODO: delayStreamPopOutput may be inside the last frame written if the ringbuffer is completely full @@ -245,7 +266,7 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* for (int i = 0; i < numSamplesDelay; i++) { int parentIndex = i * 2; - _clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatio; + _clientSamples[parentIndex + delayedChannelOffset] += *delayStreamPopOutput * attenuationAndWeakChannelRatioAndFade; ++delayStreamPopOutput; } } @@ -256,8 +277,10 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* attenuationCoefficient = 1.0f; } + float attenuationAndFade = attenuationCoefficient * repeatedFrameFadeFactor; + for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s++) { - _clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationCoefficient), + _clientSamples[s] = glm::clamp(_clientSamples[s] + (int)(streamPopOutput[s / stereoDivider] * attenuationAndFade), MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } } @@ -285,7 +308,6 @@ int AudioMixer::prepareMixForListeningNode(Node* node) { PositionalAudioStream* otherNodeStream = i.value(); if ((*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) - && otherNodeStream->lastPopSucceeded() && otherNodeStream->getLastPopOutputFrameLoudness() > 0.0f) { //&& otherNodeStream->getLastPopOutputTrailingLoudness() > 0.0f) { @@ -627,6 +649,7 @@ void AudioMixer::run() { } // send mixed audio packet + if (nodeData->getOutgoingSequenceNumber() % 100 < 50) nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); @@ -654,6 +677,4 @@ void AudioMixer::run() { usleep(usecToSleep); } } - - delete[] clientMixBuffer; } diff --git a/domain-server/resources/web/settings/describe.json b/domain-server/resources/web/settings/describe.json index db2809bffb..cfb7e1ed79 100644 --- a/domain-server/resources/web/settings/describe.json +++ b/domain-server/resources/web/settings/describe.json @@ -7,7 +7,7 @@ "type": "checkbox", "label": "Dynamic Jitter Buffers", "help": "Dynamically buffer client audio based on perceived jitter in packet receipt timing", - "default": true + "default": false }, "B-static-desired-jitter-buffer-frames": { "label": "Static Desired Jitter Buffer Frames", @@ -49,7 +49,7 @@ "type": "checkbox", "label": "Repetition with Fade:", "help": "If enabled, dropped frames and mixing during starves will repeat the last frame, eventually fading to silence", - "default": true + "default": false }, "Z-unattenuated-zone": { "label": "Unattenuated Zone", diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 8aac32849e..0e481e15c2 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -105,6 +105,7 @@ Audio::Audio(QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), + _scopeLastFrame(), _statsEnabled(false), _statsShowInjectedStreams(false), _outgoingAvatarAudioSequenceNumber(0), @@ -113,15 +114,17 @@ Audio::Audio(QObject* parent) : _audioOutputMsecsUnplayedStats(1, FRAMES_AVAILABLE_STATS_WINDOW_SECONDS), _lastSentAudioPacket(0), _packetSentTimeGaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS), - _audioOutputIODevice(*this) + _audioOutputIODevice(_receivedAudioStream) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); // Create the noise sample array _noiseSampleFrames = new float[NUMBER_OF_NOISE_SAMPLE_FRAMES]; - connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedAudioStreamSamples, Qt::DirectConnection); - connect(&_receivedAudioStream, &MixedProcessedAudioStream::dataParsed, this, &Audio::updateScopeBuffers, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedSilence, this, &Audio::addStereoSilenceToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedLastFrameRepeatedWithFade, this, &Audio::addLastFrameRepeatedWithFadeToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::addedStereoSamples, this, &Audio::addStereoSamplesToScope, Qt::DirectConnection); + connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, this, &Audio::processReceivedSamples, Qt::DirectConnection); } void Audio::init(QGLWidget *parent) { @@ -657,9 +660,7 @@ void Audio::handleAudioInput() { if (!_isStereoInput && _scopeEnabled && !_scopeEnabledPause) { unsigned int numMonoAudioChannels = 1; unsigned int monoAudioChannel = 0; - addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, monoAudioChannel, numMonoAudioChannels); - _scopeInputOffset += NETWORK_SAMPLES_PER_FRAME; - _scopeInputOffset %= _samplesPerScope; + _scopeInputOffset = addBufferToScope(_scopeInput, _scopeInputOffset, networkAudioSamples, NETWORK_SAMPLES_PER_FRAME, monoAudioChannel, numMonoAudioChannels); } NodeList* nodeList = NodeList::getInstance(); @@ -733,7 +734,48 @@ void Audio::handleAudioInput() { } } -void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { +const int STEREO_FACTOR = 2; + +void Audio::addStereoSilenceToScope(int silentSamplesPerChannel) { + if (!_scopeEnabled || _scopeEnabledPause) { + return; + } + addSilenceToScope(_scopeOutputLeft, _scopeOutputOffset, silentSamplesPerChannel); + _scopeOutputOffset = addSilenceToScope(_scopeOutputRight, _scopeOutputOffset, silentSamplesPerChannel); +} + +void Audio::addStereoSamplesToScope(const QByteArray& samples) { + if (!_scopeEnabled || _scopeEnabledPause) { + return; + } + const int16_t* samplesData = reinterpret_cast(samples.data()); + int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR; + + addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR); + _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR); + + _scopeLastFrame = samples.right(NETWORK_BUFFER_LENGTH_BYTES_STEREO); +} + +void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { + printf("addLastFrameRepeatedWithFadeToScope"); + const int16_t* lastFrameData = reinterpret_cast(_scopeLastFrame.data()); + + int samplesRemaining = samplesPerChannel; + int indexOfRepeat = 0; + do { + int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME); + float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); + printf("%f ", fade); + addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); + _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); + + samplesRemaining -= samplesToWriteThisIteration; + } while (samplesRemaining > 0); + printf("\n"); +} + +void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) @@ -780,7 +822,7 @@ void Audio::processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QBy _desiredOutputFormat, _outputFormat); } -void Audio::updateScopeBuffers() { +/*void Audio::updateScopeBuffers() { if (_scopeEnabled && !_scopeEnabledPause) { unsigned int numAudioChannels = _desiredOutputFormat.channelCount(); const int16_t* samples = _receivedAudioStream.getNetworkSamples(); @@ -794,7 +836,7 @@ void Audio::updateScopeBuffers() { samples, audioChannel, numAudioChannels); audioChannel = 1; - addBufferToScope( + _scopeOutputOffset = addBufferToScope( _scopeOutputRight, _scopeOutputOffset, samples, audioChannel, numAudioChannels); @@ -806,7 +848,7 @@ void Audio::updateScopeBuffers() { } _receivedAudioStream.clearNetworkSamples(); -} +}*/ void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { @@ -1259,12 +1301,15 @@ void Audio::freeScope() { } } -void Audio::addBufferToScope( - QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels) { +int Audio::addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamplesPerChannel, + unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade) { // Constant multiplier to map sample value to vertical size of scope float multiplier = (float)MULTIPLIER_SCOPE_HEIGHT / logf(2.0f); + // Used to scale each sample. (logf(sample) + fadeOffset) is same as logf(sample * fade). + float fadeOffset = logf(fade); + // Temporary variable receives sample value float sample; @@ -1275,17 +1320,41 @@ void Audio::addBufferToScope( // Short int pointer to mapped samples in byte array int16_t* destination = (int16_t*) byteArray->data(); - for (unsigned int i = 0; i < NETWORK_SAMPLES_PER_FRAME; i++) { + for (int i = 0; i < sourceSamplesPerChannel; i++) { sample = (float)source[i * sourceNumberOfChannels + sourceChannel]; - if (sample > 0) { - value = (int16_t)(multiplier * logf(sample)); - } else if (sample < 0) { - value = (int16_t)(-multiplier * logf(-sample)); + if (sample > 1) { + value = (int16_t)(multiplier * (logf(sample) + fadeOffset)); + } else if (sample < -1) { + value = (int16_t)(-multiplier * (logf(-sample) + fadeOffset)); } else { value = 0; } - destination[i + frameOffset] = value; + destination[frameOffset] = value; + frameOffset = (frameOffset == _samplesPerScope - 1) ? 0 : frameOffset + 1; } + return frameOffset; +} + +int Audio::addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples) { + + QMutexLocker lock(&_guard); + // Short int pointer to mapped samples in byte array + int16_t* destination = (int16_t*)byteArray->data(); + + if (silentSamples >= _samplesPerScope) { + memset(destination, 0, byteArray->size()); + return frameOffset; + } + + int samplesToBufferEnd = _samplesPerScope - frameOffset; + if (silentSamples > samplesToBufferEnd) { + memset(destination + frameOffset, 0, samplesToBufferEnd * sizeof(int16_t)); + memset(destination, 0, silentSamples - samplesToBufferEnd * sizeof(int16_t)); + } else { + memset(destination + frameOffset, 0, silentSamples * sizeof(int16_t)); + } + + return (frameOffset + silentSamples) % _samplesPerScope; } void Audio::renderStats(const float* color, int width, int height) { @@ -1761,13 +1830,11 @@ float Audio::getInputRingBufferMsecsAvailable() const { } qint64 Audio::AudioOutputIODevice::readData(char * data, qint64 maxSize) { - MixedProcessedAudioStream& receivedAUdioStream = _parent._receivedAudioStream; - int samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; - if ((samplesPopped = receivedAUdioStream.popSamples(samplesRequested, false)) > 0) { - AudioRingBuffer::ConstIterator lastPopOutput = receivedAUdioStream.getLastPopOutput(); + if ((samplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { + AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); bytesWritten = samplesPopped * sizeof(int16_t); } else { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index c080557576..a8ba46a35e 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -48,14 +48,14 @@ public: class AudioOutputIODevice : public QIODevice { public: - AudioOutputIODevice(Audio& parent) : _parent(parent) {}; + AudioOutputIODevice(MixedProcessedAudioStream& receivedAudioStream) : _receivedAudioStream(receivedAudioStream) {}; void start() { open(QIODevice::ReadOnly); } void stop() { close(); } qint64 readData(char * data, qint64 maxSize); qint64 writeData(const char * data, qint64 maxSize) { return 0; } private: - Audio& _parent; + MixedProcessedAudioStream& _receivedAudioStream; }; @@ -105,8 +105,6 @@ public slots: void addReceivedAudioToStream(const QByteArray& audioByteArray); void parseAudioStreamStatsPacket(const QByteArray& packet); void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); - void processReceivedAudioStreamSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); - void updateScopeBuffers(); void handleAudioInput(); void reset(); void resetStats(); @@ -123,6 +121,11 @@ public slots: void selectAudioScopeFiveFrames(); void selectAudioScopeTwentyFrames(); void selectAudioScopeFiftyFrames(); + + void addStereoSilenceToScope(int silentSamplesPerChannel); + void addLastFrameRepeatedWithFadeToScope(int samplesPerChannel); + void addStereoSamplesToScope(const QByteArray& samples); + void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); virtual void handleAudioByteArray(const QByteArray& audioByteArray); @@ -244,8 +247,9 @@ private: void reallocateScope(int frames); // Audio scope methods for data acquisition - void addBufferToScope( - QByteArray* byteArray, unsigned int frameOffset, const int16_t* source, unsigned int sourceChannel, unsigned int sourceNumberOfChannels); + int addBufferToScope(QByteArray* byteArray, int frameOffset, const int16_t* source, int sourceSamples, + unsigned int sourceChannel, unsigned int sourceNumberOfChannels, float fade = 1.0f); + int addSilenceToScope(QByteArray* byteArray, int frameOffset, int silentSamples); // Audio scope methods for rendering void renderBackground(const float* color, int x, int y, int width, int height); @@ -272,6 +276,7 @@ private: QByteArray* _scopeInput; QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + QByteArray _scopeLastFrame; #ifdef _WIN32 static const unsigned int STATS_WIDTH = 1500; #else diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d9cb34ac1b..31c714a1cc 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -224,3 +224,45 @@ float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { float AudioRingBuffer::getNextOutputFrameLoudness() const { return getFrameLoudness(_nextOutput); } + +int AudioRingBuffer::writeSamples(ConstIterator source, int maxSamples) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qDebug() << "Overflowed ring buffer! Overwriting old data"; + } + + int16_t* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = *source; + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} + +int AudioRingBuffer::writeSamplesWithFade(ConstIterator source, int maxSamples, float fade) { + int samplesToCopy = std::min(maxSamples, _sampleCapacity); + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; + qDebug() << "Overflowed ring buffer! Overwriting old data"; + } + + int16_t* bufferLast = _buffer + _bufferLength - 1; + for (int i = 0; i < samplesToCopy; i++) { + *_endOfLastWrite = (int16_t)((float)(*source) * fade); + _endOfLastWrite = (_endOfLastWrite == bufferLast) ? _buffer : _endOfLastWrite + 1; + ++source; + } + + return samplesToCopy; +} diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index be4dcaf545..65e6947115 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -189,8 +189,12 @@ public: }; ConstIterator nextOutput() const { return ConstIterator(_buffer, _bufferLength, _nextOutput); } - + ConstIterator lastFrameWritten() const { return ConstIterator(_buffer, _bufferLength, _endOfLastWrite) - _numFrameSamples; } + float getFrameLoudness(ConstIterator frameStart) const; + + int writeSamples(ConstIterator source, int maxSamples); + int writeSamplesWithFade(ConstIterator source, int maxSamples, float fade); }; #endif // hifi_AudioRingBuffer_h \ No newline at end of file diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 39cd544b15..70a4086eb0 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -162,8 +162,6 @@ int InboundAudioStream::parseData(const QByteArray& packet) { framesAvailableChanged(); - emit dataParsed(); - return readBytes; } @@ -418,9 +416,31 @@ void InboundAudioStream::packetReceivedUpdateTimingStats() { } int InboundAudioStream::writeSamplesForDroppedPackets(int networkSamples) { + if (_repetitionWithFade) { + return writeLastFrameRepeatedWithFade(networkSamples); + } return writeDroppableSilentSamples(networkSamples); } +int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { + AudioRingBuffer::ConstIterator frameToRepeat = _ringBuffer.lastFrameWritten(); + int frameSize = _ringBuffer.getNumFrameSamples(); + int samplesToWrite = samples; + int indexOfRepeat = 0; + do { + int samplesToWriteThisIteration = std::min(samplesToWrite, frameSize); + float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); + if (fade == 1.0f) { + samplesToWrite -= _ringBuffer.writeSamples(frameToRepeat, samplesToWriteThisIteration); + } else { + samplesToWrite -= _ringBuffer.writeSamplesWithFade(frameToRepeat, samplesToWriteThisIteration, fade); + } + indexOfRepeat++; + } while (samplesToWrite > 0); + + return samples; +} + float InboundAudioStream::getLastPopOutputFrameLoudness() const { return _ringBuffer.getFrameLoudness(_lastPopOutput); } @@ -448,3 +468,25 @@ AudioStreamStats InboundAudioStream::getAudioStreamStats() const { return streamStats; } + +float calculateRepeatedFrameFadeFactor(int indexOfRepeat) { + // fade factor scheme is from this paper: + // http://inst.eecs.berkeley.edu/~ee290t/sp04/lectures/packet_loss_recov_paper11.pdf + + const float INITIAL_MSECS_NO_FADE = 20.0f; + const float MSECS_FADE_TO_ZERO = 320.0f; + + const float INITIAL_FRAMES_NO_FADE = INITIAL_MSECS_NO_FADE * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS; + const float FRAMES_FADE_TO_ZERO = MSECS_FADE_TO_ZERO * (float)USECS_PER_MSEC / (float)BUFFER_SEND_INTERVAL_USECS; + + const float SAMPLE_RANGE = std::numeric_limits::max(); + + if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE) { + return 1.0f; + } else if (indexOfRepeat <= INITIAL_FRAMES_NO_FADE + FRAMES_FADE_TO_ZERO) { + return pow(SAMPLE_RANGE, -(indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO); + + //return 1.0f - ((indexOfRepeat - INITIAL_FRAMES_NO_FADE) / FRAMES_FADE_TO_ZERO); + } + return 0.0f; +} diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f41d9255ff..f8413f8d75 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -33,7 +33,7 @@ const int STATS_FOR_STATS_PACKET_WINDOW_SECONDS = 30; // this controls the window size of the time-weighted avg of frames available. Every time the window fills up, // _currentJitterBufferFrames is updated with the time-weighted avg and the running time-weighted avg is reset. -const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 2 * USECS_PER_SECOND; +const int FRAMES_AVAILABLE_STAT_WINDOW_USECS = 10 * USECS_PER_SECOND; // default values for members of the Settings struct const int DEFAULT_MAX_FRAMES_OVER_DESIRED = 10; @@ -157,9 +157,6 @@ public: int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); } -signals: - void dataParsed(); - public slots: /// This function should be called every second for all the stats to function properly. If dynamic jitter buffers /// is enabled, those stats are used to calculate _desiredJitterBufferFrames. @@ -191,6 +188,10 @@ protected: /// writes silent samples to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentSamples(int silentSamples); + + /// writes the last written frame repeatedly, gradually fading to silence. + /// used for writing samples for dropped packets. + virtual int writeLastFrameRepeatedWithFade(int samples); protected: @@ -246,4 +247,6 @@ protected: bool _repetitionWithFade; }; +float calculateRepeatedFrameFadeFactor(int indexOfRepeat); + #endif // hifi_InboundAudioStream_h diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 5693af7c6e..4b28e2f2c1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -11,9 +11,10 @@ #include "MixedProcessedAudioStream.h" +static const int STEREO_FACTOR = 2; + MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int numFramesCapacity, const InboundAudioStream::Settings& settings) - : InboundAudioStream(numFrameSamples, numFramesCapacity, settings), - _networkSamplesWritten(0) + : InboundAudioStream(numFrameSamples, numFramesCapacity, settings) { } @@ -23,30 +24,33 @@ void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCount _ringBuffer.resizeForFrameSize(deviceOutputFrameSize); } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { - - memcpy(&_networkSamples[_networkSamplesWritten], packetAfterStreamProperties.data(), packetAfterStreamProperties.size()); - _networkSamplesWritten += packetAfterStreamProperties.size() / sizeof(int16_t); - - QByteArray outputBuffer; - emit processSamples(packetAfterStreamProperties, outputBuffer); - - _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); - - return packetAfterStreamProperties.size(); -} - int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); - int networkSilentSamplesWritten = deviceToNetworkSamples(deviceSilentSamplesWritten); - memset(&_networkSamples[_networkSamplesWritten], 0, networkSilentSamplesWritten * sizeof(int16_t)); - _networkSamplesWritten += networkSilentSamplesWritten; + emit addedSilence(deviceToNetworkSamples(deviceSilentSamplesWritten) / STEREO_FACTOR); return deviceSilentSamplesWritten; } -static const int STEREO_FACTOR = 2; +int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { + int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(networkToDeviceSamples(samples)); + + emit addedLastFrameRepeatedWithFade(deviceToNetworkSamples(deviceSamplesWritten) / STEREO_FACTOR); + + return deviceSamplesWritten; +} + +int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples) { + + emit addedStereoSamples(packetAfterStreamProperties); + + QByteArray outputBuffer; + emit processSamples(packetAfterStreamProperties, outputBuffer); + + _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); + + return packetAfterStreamProperties.size(); +} int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_FACTOR * SAMPLE_RATE); diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index b85637a288..fd1f93a6a1 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -14,6 +14,8 @@ #include "InboundAudioStream.h" +class Audio; + class MixedProcessedAudioStream : public InboundAudioStream { Q_OBJECT public: @@ -21,30 +23,26 @@ public: signals: + void addedSilence(int silentSamplesPerChannel); + void addedLastFrameRepeatedWithFade(int samplesPerChannel); + void addedStereoSamples(const QByteArray& samples); + void processSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); public: void outputFormatChanged(int outputFormatChannelCountTimesSampleRate); - const int16_t* getNetworkSamples() const { return _networkSamples; } - int getNetworkSamplesWritten() const { return _networkSamplesWritten; } - - void clearNetworkSamples() { _networkSamplesWritten = 0; } - protected: - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); int writeDroppableSilentSamples(int silentSamples); + int writeLastFrameRepeatedWithFade(int samples); + int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties, int networkSamples); private: int networkToDeviceSamples(int networkSamples); int deviceToNetworkSamples(int deviceSamples); + private: int _outputFormatChannelsTimesSampleRate; - - // this buffer keeps a copy of the network samples written during parseData() for the sole purpose - // of passing it on to the audio scope - int16_t _networkSamples[10 * NETWORK_BUFFER_LENGTH_SAMPLES_STEREO]; - int _networkSamplesWritten; }; #endif // hifi_MixedProcessedAudioStream_h From a31d53544b3a78313ac2f7bf5168abf668cf83ca Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 11 Aug 2014 17:58:01 -0700 Subject: [PATCH 21/34] repetition-with-fade seems good; continue testing --- assignment-client/src/audio/AudioMixer.cpp | 1 - interface/src/Audio.cpp | 19 ++++++++++++++----- libraries/audio/src/InboundAudioStream.cpp | 5 +++-- .../audio/src/MixedProcessedAudioStream.cpp | 14 ++++++++------ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 509a965bf4..1372c92b1f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -649,7 +649,6 @@ void AudioMixer::run() { } // send mixed audio packet - if (nodeData->getOutgoingSequenceNumber() % 100 < 50) nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 0e481e15c2..662fa8f2d6 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -740,8 +740,11 @@ void Audio::addStereoSilenceToScope(int silentSamplesPerChannel) { if (!_scopeEnabled || _scopeEnabledPause) { return; } + printf("\t Audio::addStereoSilenceToScope %d per channel\n", silentSamplesPerChannel); addSilenceToScope(_scopeOutputLeft, _scopeOutputOffset, silentSamplesPerChannel); _scopeOutputOffset = addSilenceToScope(_scopeOutputRight, _scopeOutputOffset, silentSamplesPerChannel); + + printf("\t end\n"); } void Audio::addStereoSamplesToScope(const QByteArray& samples) { @@ -750,29 +753,35 @@ void Audio::addStereoSamplesToScope(const QByteArray& samples) { } const int16_t* samplesData = reinterpret_cast(samples.data()); int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR; + printf("\t Audio::addStereoSamplesToScope %d samples per channel\n", samplesPerChannel); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR); _scopeLastFrame = samples.right(NETWORK_BUFFER_LENGTH_BYTES_STEREO); + + printf("\t end\n"); } void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { - printf("addLastFrameRepeatedWithFadeToScope"); + printf("addLastFrameRepeatedWithFadeToScope %d per channel\n", samplesPerChannel); const int16_t* lastFrameData = reinterpret_cast(_scopeLastFrame.data()); int samplesRemaining = samplesPerChannel; int indexOfRepeat = 0; do { int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME); - float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - printf("%f ", fade); + float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); + printf("%f ", fade, samplesToWriteThisIteration); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); + printf("scopeOutputOffset %d\n", _scopeOutputOffset); + samplesRemaining -= samplesToWriteThisIteration; + indexOfRepeat++; } while (samplesRemaining > 0); - printf("\n"); + printf("\t end\n"); } void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { @@ -1755,7 +1764,7 @@ bool Audio::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo) // setup our general output device for audio-mixer audio _audioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this); _audioOutput->setBufferSize(AUDIO_OUTPUT_BUFFER_SIZE_FRAMES * _outputFrameSize * sizeof(int16_t)); - qDebug() << "Ring Buffer capacity in frames: " << _audioOutput->bufferSize() / sizeof(int16_t) / (float)_outputFrameSize; + qDebug() << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / sizeof(int16_t) / (float)_outputFrameSize; _audioOutputIODevice.start(); _audioOutput->start(&_audioOutputIODevice); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 70a4086eb0..d10c9da05c 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -176,7 +176,6 @@ int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packet } int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { - // calculate how many silent frames we should drop. int samplesPerFrame = _ringBuffer.getNumFrameSamples(); int desiredJitterBufferFramesPlusPadding = _desiredJitterBufferFrames + DESIRED_JITTER_BUFFER_FRAMES_PADDING; @@ -198,7 +197,9 @@ int InboundAudioStream::writeDroppableSilentSamples(int silentSamples) { _framesAvailableStat.reset(); } - return _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); + int ret = _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); + + return ret; } int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing, bool starveIfNoSamplesPopped) { diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 4b28e2f2c1..844adf36b3 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -20,23 +20,25 @@ MixedProcessedAudioStream::MixedProcessedAudioStream(int numFrameSamples, int nu void MixedProcessedAudioStream::outputFormatChanged(int outputFormatChannelCountTimesSampleRate) { _outputFormatChannelsTimesSampleRate = outputFormatChannelCountTimesSampleRate; - int deviceOutputFrameSize = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * _outputFormatChannelsTimesSampleRate / SAMPLE_RATE; + int deviceOutputFrameSize = networkToDeviceSamples(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); _ringBuffer.resizeForFrameSize(deviceOutputFrameSize); } int MixedProcessedAudioStream::writeDroppableSilentSamples(int silentSamples) { + int deviceSilentSamplesWritten = InboundAudioStream::writeDroppableSilentSamples(networkToDeviceSamples(silentSamples)); - + emit addedSilence(deviceToNetworkSamples(deviceSilentSamplesWritten) / STEREO_FACTOR); return deviceSilentSamplesWritten; } int MixedProcessedAudioStream::writeLastFrameRepeatedWithFade(int samples) { + int deviceSamplesWritten = InboundAudioStream::writeLastFrameRepeatedWithFade(networkToDeviceSamples(samples)); emit addedLastFrameRepeatedWithFade(deviceToNetworkSamples(deviceSamplesWritten) / STEREO_FACTOR); - + return deviceSamplesWritten; } @@ -48,14 +50,14 @@ int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& emit processSamples(packetAfterStreamProperties, outputBuffer); _ringBuffer.writeData(outputBuffer.data(), outputBuffer.size()); - + return packetAfterStreamProperties.size(); } int MixedProcessedAudioStream::networkToDeviceSamples(int networkSamples) { - return networkSamples * _outputFormatChannelsTimesSampleRate / (STEREO_FACTOR * SAMPLE_RATE); + return (quint64)networkSamples * (quint64)_outputFormatChannelsTimesSampleRate / (quint64)(STEREO_FACTOR * SAMPLE_RATE); } int MixedProcessedAudioStream::deviceToNetworkSamples(int deviceSamples) { - return deviceSamples * (STEREO_FACTOR * SAMPLE_RATE) / _outputFormatChannelsTimesSampleRate; + return (quint64)deviceSamples * (quint64)(STEREO_FACTOR * SAMPLE_RATE) / (quint64)_outputFormatChannelsTimesSampleRate; } From 8565e93ba49b4bba0d44973f58bc81b727c34373 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 12 Aug 2014 10:50:34 -0700 Subject: [PATCH 22/34] more test code (10% drop both directions) --- assignment-client/src/audio/AudioMixer.cpp | 1 + interface/src/Audio.cpp | 3 +-- libraries/script-engine/src/ScriptEngine.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1372c92b1f..f402050265 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -649,6 +649,7 @@ void AudioMixer::run() { } // send mixed audio packet + if (rand() % 100 < 90) nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 662fa8f2d6..84b3911771 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -724,6 +724,7 @@ void Audio::handleAudioInput() { } int packetBytes = currentPacketPtr - audioDataPacket; + if (rand() % 100 < 90) nodeList->writeDatagram(audioDataPacket, packetBytes, audioMixer); _outgoingAvatarAudioSequenceNumber++; @@ -776,8 +777,6 @@ void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); - printf("scopeOutputOffset %d\n", _scopeOutputOffset); - samplesRemaining -= samplesToWriteThisIteration; indexOfRepeat++; } while (samplesRemaining > 0); diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 2891055b65..58b0c90daa 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -519,6 +519,7 @@ void ScriptEngine::run() { memcpy(audioPacket.data() + numPreSequenceNumberBytes, &sequence, sizeof(quint16)); // send audio packet + if (rand() % 100 < 90) nodeList->writeDatagram(audioPacket, node); } } From 7f53ae0e4f9d0871976f371688a278d9a23eac71 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 12 Aug 2014 12:22:13 -0700 Subject: [PATCH 23/34] fixed warnings --- interface/src/Audio.cpp | 4 +--- libraries/audio/src/InboundAudioStream.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 84b3911771..9d00b4ffd6 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -672,8 +672,6 @@ void Audio::handleAudioInput() { glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); quint8 isStereo = _isStereoInput ? 1 : 0; - int numPacketBytes = 0; - PacketType packetType; if (_lastInputLoudness == 0) { packetType = PacketTypeSilentAudioFrame; @@ -773,7 +771,7 @@ void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { do { int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME); float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - printf("%f ", fade, samplesToWriteThisIteration); + printf("%f ", fade); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index d10c9da05c..7880602133 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -32,9 +32,9 @@ InboundAudioStream::InboundAudioStream(int numFrameSamples, int numFramesCapacit _incomingSequenceNumberStats(STATS_FOR_STATS_PACKET_WINDOW_SECONDS), _lastPacketReceivedTime(0), _timeGapStatsForDesiredCalcOnTooManyStarves(0, settings._windowSecondsForDesiredCalcOnTooManyStarves), + _calculatedJitterBufferFramesUsingMaxGap(0), _stdevStatsForDesiredCalcOnTooManyStarves(), _calculatedJitterBufferFramesUsingStDev(0), - _calculatedJitterBufferFramesUsingMaxGap(0), _timeGapStatsForDesiredReduction(0, settings._windowSecondsForDesiredReduction), _starveHistoryWindowSeconds(settings._windowSecondsForDesiredCalcOnTooManyStarves), _starveHistory(STARVE_HISTORY_CAPACITY), From 23b3d06260ed65d213dc1373cc3c5491203308d4 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 09:46:43 -0700 Subject: [PATCH 24/34] forgot null check for AudioMixer repeat mix of lastPopOutput --- assignment-client/src/audio/AudioMixer.cpp | 4 +++- libraries/audio/src/AudioRingBuffer.h | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f402050265..63b4716f82 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -105,7 +105,9 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* float repeatedFrameFadeFactor = 1.0f; if (!streamToAdd->lastPopSucceeded()) { - if (_streamSettings._repetitionWithFade) { + if (_streamSettings._repetitionWithFade && !streamToAdd->getLastPopOutput().isNull()) { + // reptition with fade is enabled, and we do have a valid previous frame to repeat. + // calculate its fade factor, which depends on how many times it's already been repeated. repeatedFrameFadeFactor = calculateRepeatedFrameFadeFactor(streamToAdd->getConsecutiveNotMixedCount() - 1); if (repeatedFrameFadeFactor == 0.0f) { return 0; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 65e6947115..522c1b8fff 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -108,6 +108,8 @@ public: _bufferLast(bufferFirst + capacity - 1), _at(at) {} + bool isNull() const { return _at == NULL; } + bool operator==(const ConstIterator& rhs) { return _at == rhs._at; } bool operator!=(const ConstIterator& rhs) { return _at != rhs._at; } const int16_t& operator*() { return *_at; } From f34f0a719b6389b8e540c062c44ffab3d091e2d7 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 10:26:01 -0700 Subject: [PATCH 25/34] fixed lastPopOutput null ptr errors for frame loudness check --- assignment-client/src/audio/AudioMixer.cpp | 12 +++--- .../src/audio/AudioMixerClientData.cpp | 19 ++++----- interface/src/Audio.cpp | 41 ------------------- libraries/audio/src/AudioRingBuffer.cpp | 3 ++ 4 files changed, 17 insertions(+), 58 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 63b4716f82..fc1f3a903b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -117,6 +117,12 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* } } + // at this point, we know streamToAdd's last pop output is valid + + if (streamToAdd->getLastPopOutputFrameLoudness() == 0.0f) { + return 0; + } + float bearingRelativeAngleToSource = 0.0f; float attenuationCoefficient = 1.0f; int numSamplesDelay = 0; @@ -309,10 +315,7 @@ int AudioMixer::prepareMixForListeningNode(Node* node) { for (i = otherNodeAudioStreams.constBegin(); i != otherNodeAudioStreams.constEnd(); i++) { PositionalAudioStream* otherNodeStream = i.value(); - if ((*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) - && otherNodeStream->getLastPopOutputFrameLoudness() > 0.0f) { - //&& otherNodeStream->getLastPopOutputTrailingLoudness() > 0.0f) { - + if (*otherNode != *node || otherNodeStream->shouldLoopbackForNode()) { streamsMixed += addStreamToMixForListeningNodeWithStream(otherNodeStream, nodeAudioStream); } } @@ -651,7 +654,6 @@ void AudioMixer::run() { } // send mixed audio packet - if (rand() % 100 < 90) nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); nodeData->incrementOutgoingMixedAudioSequenceNumber(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 333357fbf4..6799515e26 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -101,18 +101,13 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, A QHash::ConstIterator i; for (i = _audioStreams.constBegin(); i != _audioStreams.constEnd(); i++) { PositionalAudioStream* stream = i.value(); - if (stream->popFrames(1, true) > 0) { - // this is a ring buffer that is ready to go - - // calculate the trailing avg loudness for the next frame - // that would be mixed in - stream->updateLastPopOutputTrailingLoudness(); - - if (checkSourceZone && checkSourceZone->contains(stream->getPosition())) { - stream->setListenerUnattenuatedZone(listenerZone); - } else { - stream->setListenerUnattenuatedZone(NULL); - } + + stream->popFrames(1, true); + + if (checkSourceZone && checkSourceZone->contains(stream->getPosition())) { + stream->setListenerUnattenuatedZone(listenerZone); + } else { + stream->setListenerUnattenuatedZone(NULL); } } } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9d00b4ffd6..b14150fdd6 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -722,7 +722,6 @@ void Audio::handleAudioInput() { } int packetBytes = currentPacketPtr - audioDataPacket; - if (rand() % 100 < 90) nodeList->writeDatagram(audioDataPacket, packetBytes, audioMixer); _outgoingAvatarAudioSequenceNumber++; @@ -739,11 +738,8 @@ void Audio::addStereoSilenceToScope(int silentSamplesPerChannel) { if (!_scopeEnabled || _scopeEnabledPause) { return; } - printf("\t Audio::addStereoSilenceToScope %d per channel\n", silentSamplesPerChannel); addSilenceToScope(_scopeOutputLeft, _scopeOutputOffset, silentSamplesPerChannel); _scopeOutputOffset = addSilenceToScope(_scopeOutputRight, _scopeOutputOffset, silentSamplesPerChannel); - - printf("\t end\n"); } void Audio::addStereoSamplesToScope(const QByteArray& samples) { @@ -752,18 +748,14 @@ void Audio::addStereoSamplesToScope(const QByteArray& samples) { } const int16_t* samplesData = reinterpret_cast(samples.data()); int samplesPerChannel = samples.size() / sizeof(int16_t) / STEREO_FACTOR; - printf("\t Audio::addStereoSamplesToScope %d samples per channel\n", samplesPerChannel); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, samplesData, samplesPerChannel, 0, STEREO_FACTOR); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, samplesData, samplesPerChannel, 1, STEREO_FACTOR); _scopeLastFrame = samples.right(NETWORK_BUFFER_LENGTH_BYTES_STEREO); - - printf("\t end\n"); } void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { - printf("addLastFrameRepeatedWithFadeToScope %d per channel\n", samplesPerChannel); const int16_t* lastFrameData = reinterpret_cast(_scopeLastFrame.data()); int samplesRemaining = samplesPerChannel; @@ -771,14 +763,12 @@ void Audio::addLastFrameRepeatedWithFadeToScope(int samplesPerChannel) { do { int samplesToWriteThisIteration = std::min(samplesRemaining, (int)NETWORK_SAMPLES_PER_FRAME); float fade = calculateRepeatedFrameFadeFactor(indexOfRepeat); - printf("%f ", fade); addBufferToScope(_scopeOutputLeft, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 0, STEREO_FACTOR, fade); _scopeOutputOffset = addBufferToScope(_scopeOutputRight, _scopeOutputOffset, lastFrameData, samplesToWriteThisIteration, 1, STEREO_FACTOR, fade); samplesRemaining -= samplesToWriteThisIteration; indexOfRepeat++; } while (samplesRemaining > 0); - printf("\t end\n"); } void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { @@ -828,34 +818,6 @@ void Audio::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& ou _desiredOutputFormat, _outputFormat); } -/*void Audio::updateScopeBuffers() { - if (_scopeEnabled && !_scopeEnabledPause) { - unsigned int numAudioChannels = _desiredOutputFormat.channelCount(); - const int16_t* samples = _receivedAudioStream.getNetworkSamples(); - int numNetworkOutputSamples = _receivedAudioStream.getNetworkSamplesWritten(); - for (int numSamples = numNetworkOutputSamples / numAudioChannels; numSamples > 0; numSamples -= NETWORK_SAMPLES_PER_FRAME) { - - unsigned int audioChannel = 0; - addBufferToScope( - _scopeOutputLeft, - _scopeOutputOffset, - samples, audioChannel, numAudioChannels); - - audioChannel = 1; - _scopeOutputOffset = addBufferToScope( - _scopeOutputRight, - _scopeOutputOffset, - samples, audioChannel, numAudioChannels); - - _scopeOutputOffset += NETWORK_SAMPLES_PER_FRAME; - _scopeOutputOffset %= _samplesPerScope; - samples += NETWORK_SAMPLES_PER_FRAME * numAudioChannels; - } - } - - _receivedAudioStream.clearNetworkSamples(); -}*/ - void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { if (_audioOutput) { @@ -866,9 +828,6 @@ void Audio::addReceivedAudioToStream(const QByteArray& audioByteArray) { Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); } - - - void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { int numBytesPacketHeader = numBytesForPacketHeader(packet); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 31c714a1cc..baf40c530f 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -218,6 +218,9 @@ float AudioRingBuffer::getFrameLoudness(const int16_t* frameStart) const { } float AudioRingBuffer::getFrameLoudness(ConstIterator frameStart) const { + if (frameStart.isNull()) { + return 0.0f; + } return getFrameLoudness(&(*frameStart)); } From ddc8bec1ec74c0e7e98294664497119f1e84b5d1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 10:41:25 -0700 Subject: [PATCH 26/34] moved lastpopframeloudness calculation to checkBuffersBeforeFrameSend --- assignment-client/src/audio/AudioMixer.cpp | 4 +++- .../src/audio/AudioMixerClientData.cpp | 4 +++- libraries/audio/src/InboundAudioStream.cpp | 4 ---- libraries/audio/src/InboundAudioStream.h | 4 +--- libraries/audio/src/PositionalAudioStream.cpp | 18 ++++++++++++++---- libraries/audio/src/PositionalAudioStream.h | 6 ++++++ 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index fc1f3a903b..70b66e60b5 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -119,7 +119,9 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* // at this point, we know streamToAdd's last pop output is valid - if (streamToAdd->getLastPopOutputFrameLoudness() == 0.0f) { + // if the frame we're about to mix is silent, bail + if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { + printf("about to mix silent frame\n"); return 0; } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 6799515e26..3b9491a5ea 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -102,7 +102,9 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, A for (i = _audioStreams.constBegin(); i != _audioStreams.constEnd(); i++) { PositionalAudioStream* stream = i.value(); - stream->popFrames(1, true); + if (stream->popFrames(1, true) > 0) { + stream->updateLastPopOutputLoudness(); + } if (checkSourceZone && checkSourceZone->contains(stream->getPosition())) { stream->setListenerUnattenuatedZone(listenerZone); diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 7880602133..4c938b5bc2 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -442,10 +442,6 @@ int InboundAudioStream::writeLastFrameRepeatedWithFade(int samples) { return samples; } -float InboundAudioStream::getLastPopOutputFrameLoudness() const { - return _ringBuffer.getFrameLoudness(_lastPopOutput); -} - AudioStreamStats InboundAudioStream::getAudioStreamStats() const { AudioStreamStats streamStats; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index f8413f8d75..ca9591a746 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -99,7 +99,7 @@ public: InboundAudioStream(int numFrameSamples, int numFramesCapacity, const Settings& settings); void reset(); - void resetStats(); + virtual void resetStats(); void clearBuffer(); virtual int parseData(const QByteArray& packet); @@ -137,8 +137,6 @@ public: /// returns the desired number of jitter buffer frames using Freddy's method int getCalculatedJitterBufferFramesUsingMaxGap() const { return _calculatedJitterBufferFramesUsingMaxGap; } - - float getLastPopOutputFrameLoudness() const; int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } int getMaxFramesOverDesired() const { return _maxFramesOverDesired; } diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index d2c1ade85c..82b40cd2b7 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -30,22 +30,28 @@ PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, b _shouldLoopbackForNode(false), _isStereo(isStereo), _lastPopOutputTrailingLoudness(0.0f), + _lastPopOutputLoudness(0.0f), _listenerUnattenuatedZone(NULL) { } +void PositionalAudioStream::resetStats() { + _lastPopOutputTrailingLoudness = 0.0f; + _lastPopOutputLoudness = 0.0f; +} + void PositionalAudioStream::updateLastPopOutputTrailingLoudness() { - float lastPopLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput); + _lastPopOutputLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput); const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; const float LOUDNESS_EPSILON = 0.000001f; - if (lastPopLoudness >= _lastPopOutputTrailingLoudness) { - _lastPopOutputTrailingLoudness = lastPopLoudness; + if (_lastPopOutputLoudness >= _lastPopOutputTrailingLoudness) { + _lastPopOutputTrailingLoudness = _lastPopOutputLoudness; } else { - _lastPopOutputTrailingLoudness = (_lastPopOutputTrailingLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * lastPopLoudness); + _lastPopOutputTrailingLoudness = (_lastPopOutputTrailingLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * _lastPopOutputLoudness); if (_lastPopOutputTrailingLoudness < LOUDNESS_EPSILON) { _lastPopOutputTrailingLoudness = 0; @@ -53,6 +59,10 @@ void PositionalAudioStream::updateLastPopOutputTrailingLoudness() { } } +void PositionalAudioStream::updateLastPopOutputLoudness() { + _lastPopOutputLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput); +} + int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteArray) { QDataStream packetStream(positionalByteArray); diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index d1d5e013e7..c117046344 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -29,11 +29,16 @@ public: PositionalAudioStream(PositionalAudioStream::Type type, bool isStereo, const InboundAudioStream::Settings& settings); + virtual void resetStats(); + virtual AudioStreamStats getAudioStreamStats() const; void updateLastPopOutputTrailingLoudness(); float getLastPopOutputTrailingLoudness() const { return _lastPopOutputTrailingLoudness; } + void updateLastPopOutputLoudness(); + float getLastPopOutputLoudness() const { return _lastPopOutputLoudness; } + bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; } bool isStereo() const { return _isStereo; } PositionalAudioStream::Type getType() const { return _type; } @@ -59,6 +64,7 @@ protected: bool _isStereo; float _lastPopOutputTrailingLoudness; + float _lastPopOutputLoudness; AABox* _listenerUnattenuatedZone; }; From 746893cc94180f8730af1505c695f749bf5b8e8f Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 10:48:47 -0700 Subject: [PATCH 27/34] updateLastOutputLoudnessAndTrailingLoudness --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- libraries/audio/src/PositionalAudioStream.cpp | 6 +----- libraries/audio/src/PositionalAudioStream.h | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 3b9491a5ea..5c8bc4f5d3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -103,7 +103,7 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, A PositionalAudioStream* stream = i.value(); if (stream->popFrames(1, true) > 0) { - stream->updateLastPopOutputLoudness(); + stream->updateLastPopOutputLoudnessAndTrailingLoudness(); } if (checkSourceZone && checkSourceZone->contains(stream->getPosition())) { diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index 82b40cd2b7..ae30022268 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -40,7 +40,7 @@ void PositionalAudioStream::resetStats() { _lastPopOutputLoudness = 0.0f; } -void PositionalAudioStream::updateLastPopOutputTrailingLoudness() { +void PositionalAudioStream::updateLastPopOutputLoudnessAndTrailingLoudness() { _lastPopOutputLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput); const int TRAILING_AVERAGE_FRAMES = 100; @@ -59,10 +59,6 @@ void PositionalAudioStream::updateLastPopOutputTrailingLoudness() { } } -void PositionalAudioStream::updateLastPopOutputLoudness() { - _lastPopOutputLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput); -} - int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteArray) { QDataStream packetStream(positionalByteArray); diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index c117046344..2b615a575b 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -33,10 +33,8 @@ public: virtual AudioStreamStats getAudioStreamStats() const; - void updateLastPopOutputTrailingLoudness(); + void updateLastPopOutputLoudnessAndTrailingLoudness(); float getLastPopOutputTrailingLoudness() const { return _lastPopOutputTrailingLoudness; } - - void updateLastPopOutputLoudness(); float getLastPopOutputLoudness() const { return _lastPopOutputLoudness; } bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; } From a405cd9a7249458553dc3d19f6823e44c14e5ada Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 11:01:15 -0700 Subject: [PATCH 28/34] repetition-with-fade ready for commit --- assignment-client/src/audio/AudioMixer.cpp | 1 - libraries/script-engine/src/ScriptEngine.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 70b66e60b5..9ffebbc90b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -121,7 +121,6 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream* // if the frame we're about to mix is silent, bail if (streamToAdd->getLastPopOutputLoudness() == 0.0f) { - printf("about to mix silent frame\n"); return 0; } diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 58b0c90daa..2891055b65 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -519,7 +519,6 @@ void ScriptEngine::run() { memcpy(audioPacket.data() + numPreSequenceNumberBytes, &sequence, sizeof(quint16)); // send audio packet - if (rand() % 100 < 90) nodeList->writeDatagram(audioPacket, node); } } From 014346094bccd03cf9ff3290e5ce97b3583ced96 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 11:28:41 -0700 Subject: [PATCH 29/34] fixed compile error with LIBOVR --- interface/src/devices/OculusManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 7d7375fad5..b60f55636f 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -504,7 +504,7 @@ void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { } glm::vec3 OculusManager::getRelativePosition() { -#if defined(__APPLE__) || defined(_WIN32) +#if defined(HAVE_LIBOVR) && (defined(__APPLE__) || defined(_WIN32)) ovrTrackingState trackingState = ovrHmd_GetTrackingState(_ovrHmd, ovr_GetTimeInSeconds()); ovrVector3f headPosition = trackingState.HeadPose.ThePose.Position; From b17c9102c90155bb9b459fc680e565adc2924c90 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 13:30:02 -0700 Subject: [PATCH 30/34] added stats for readPendingDatagrams in audiomixer --- assignment-client/src/audio/AudioMixer.cpp | 44 +++++++++++++++++----- assignment-client/src/audio/AudioMixer.h | 17 ++++++++- libraries/shared/src/MovingMinMaxAvg.h | 2 + 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index dab41625bd..3fe872d57c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -81,7 +81,12 @@ AudioMixer::AudioMixer(const QByteArray& packet) : _sumMixes(0), _sourceUnattenuatedZone(NULL), _listenerUnattenuatedZone(NULL), - _lastSendAudioStreamStatsTime(usecTimestampNow()) + _lastPerSecondCallbackTime(usecTimestampNow()), + _sendAudioStreamStats(false), + _datagramsReadPerCallStats(0, READ_DATAGRAMS_STATS_WINDOW_SECONDS), + _timeSpentPerCallStats(0, READ_DATAGRAMS_STATS_WINDOW_SECONDS), + _timeSpentPerHashMatchCallStats(0, READ_DATAGRAMS_STATS_WINDOW_SECONDS), + _readPendingCallsPerSecondStats(1, READ_DATAGRAMS_STATS_WINDOW_SECONDS) { } @@ -328,12 +333,18 @@ int AudioMixer::prepareMixForListeningNode(Node* node) { } void AudioMixer::readPendingDatagrams() { + quint64 readPendingDatagramsStart = usecTimestampNow(); + QByteArray receivedPacket; HifiSockAddr senderSockAddr; NodeList* nodeList = NodeList::getInstance(); + int datagramsRead = 0; while (readAvailableDatagram(receivedPacket, senderSockAddr)) { - if (nodeList->packetVersionAndHashMatch(receivedPacket)) { + quint64 packetVersionAndHashMatchStart = usecTimestampNow(); + bool match = nodeList->packetVersionAndHashMatch(receivedPacket); + _timeSpentPerHashMatchCallStats.update(usecTimestampNow() - packetVersionAndHashMatchStart); + if (match) { // pull any new audio data from nodes off of the network stack PacketType mixerPacketType = packetTypeForPacket(receivedPacket); if (mixerPacketType == PacketTypeMicrophoneAudioNoEcho @@ -352,13 +363,16 @@ void AudioMixer::readPendingDatagrams() { nodeList->writeDatagram(packet, packet.size(), node); } } - } else { // let processNodeData handle it. nodeList->processNodeData(senderSockAddr, receivedPacket); } } + datagramsRead++; } + + _timeSpentPerCallStats.update(usecTimestampNow() - readPendingDatagramsStart); + _datagramsReadPerCallStats.update(datagramsRead); } void AudioMixer::sendStatsPacket() { @@ -609,12 +623,11 @@ void AudioMixer::run() { if (!hasRatioChanged) { ++framesSinceCutoffEvent; } - - bool sendAudioStreamStats = false; + quint64 now = usecTimestampNow(); - if (now - _lastSendAudioStreamStatsTime > TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS) { - _lastSendAudioStreamStatsTime = now; - sendAudioStreamStats = true; + if (now - _lastPerSecondCallbackTime > USECS_PER_SECOND) { + perSecondActions(); + _lastPerSecondCallbackTime = now; } bool streamStatsPrinted = false; @@ -667,14 +680,14 @@ void AudioMixer::run() { nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet if it's time - if (sendAudioStreamStats) { + if (_sendAudioStreamStats) { nodeData->sendAudioStreamStatsPackets(node); - if (_printStreamStats) { printf("\nStats for agent %s:\n", node->getUUID().toString().toLatin1().data()); nodeData->printUpstreamDownstreamStats(); streamStatsPrinted = true; } + _sendAudioStreamStats = false; } ++_sumListeners; @@ -700,3 +713,14 @@ void AudioMixer::run() { } } } + +void AudioMixer::perSecondActions() { + _sendAudioStreamStats = true; + + int callsLastSecond = _datagramsReadPerCallStats.getCurrentIntervalSamples(); + _readPendingCallsPerSecondStats.update(callsLastSecond); + + _datagramsReadPerCallStats.currentIntervalComplete(); + _timeSpentPerCallStats.currentIntervalComplete(); + _timeSpentPerHashMatchCallStats.currentIntervalComplete(); +} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index b620b1cd85..e47c5f3811 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -21,7 +21,8 @@ class AvatarAudioStream; const int SAMPLE_PHASE_DELAY_AT_90 = 20; -const quint64 TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS = 1 * USECS_PER_SECOND; +const int READ_DATAGRAMS_STATS_WINDOW_SECONDS = 30; + /// Handles assignments of type AudioMixer - mixing streams of audio and re-distributing to various clients. class AudioMixer : public ThreadedAssignment { @@ -50,6 +51,9 @@ private: // client samples capacity is larger than what will be sent to optimize mixing // we are MMX adding 4 samples at a time so we need client samples to have an extra 4 int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)]; + + void perSecondActions(); + float _trailingSleepRatio; float _minAudibilityThreshold; @@ -64,7 +68,16 @@ private: static bool _printStreamStats; - quint64 _lastSendAudioStreamStatsTime; + quint64 _lastPerSecondCallbackTime; + + bool _sendAudioStreamStats; + + // stats + MovingMinMaxAvg _datagramsReadPerCallStats; // update with # of datagrams read for each readPendingDatagrams call + MovingMinMaxAvg _timeSpentPerCallStats; // update with usecs spent inside each readPendingDatagrams call + MovingMinMaxAvg _timeSpentPerHashMatchCallStats; // update with usecs spent inside each packetVersionAndHashMatch call + + MovingMinMaxAvg _readPendingCallsPerSecondStats; // update with # of readPendingDatagrams calls in the last second }; #endif // hifi_AudioMixer_h diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 628d3b4353..16fcb94dcf 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -156,6 +156,8 @@ public: T getWindowMax() const { return _windowStats.getMax(); } double getWindowAverage() const { return _windowStats.getAverage(); } + int getCurrentIntervalSamples() const { return _windowStats._samples; } + const MinMaxAvg& getOverallStats() const{ return _overallStats; } const MinMaxAvg& getWindowStats() const{ return _windowStats; } From 81fa5ed41fd090864732516aab6d9e7239271866 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 16:48:06 -0700 Subject: [PATCH 31/34] readPendingDatagrams stats printed and sent to domain page jittertester now prints out send or receive error msgs --- assignment-client/src/audio/AudioMixer.cpp | 95 ++++++++++++++++++---- assignment-client/src/audio/AudioMixer.h | 1 + libraries/shared/src/MovingMinMaxAvg.h | 20 ++++- tests/jitter/src/main.cpp | 11 ++- 4 files changed, 108 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 3fe872d57c..d32f14ac0f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -404,9 +404,24 @@ void AudioMixer::sendStatsPacket() { int sizeOfStats = 0; int TOO_BIG_FOR_MTU = 1200; // some extra space for JSONification + QString property = "readPendingDatagramsStats"; + QString value = getReadPendingDatagramsStatsString(); + statsObject2[qPrintable(property)] = value; + somethingToSend = true; + sizeOfStats += property.size() + value.size(); + NodeList* nodeList = NodeList::getInstance(); int clientNumber = 0; foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { + + // if we're too large, send the packet + if (sizeOfStats > TOO_BIG_FOR_MTU) { + nodeList->sendStatsToDomainServer(statsObject2); + sizeOfStats = 0; + statsObject2 = QJsonObject(); // clear it + somethingToSend = false; + } + clientNumber++; AudioMixerClientData* clientData = static_cast(node->getLinkedData()); if (clientData) { @@ -416,14 +431,6 @@ void AudioMixer::sendStatsPacket() { somethingToSend = true; sizeOfStats += property.size() + value.size(); } - - // if we're too large, send the packet - if (sizeOfStats > TOO_BIG_FOR_MTU) { - nodeList->sendStatsToDomainServer(statsObject2); - sizeOfStats = 0; - statsObject2 = QJsonObject(); // clear it - somethingToSend = false; - } } if (somethingToSend) { @@ -682,11 +689,6 @@ void AudioMixer::run() { // send an audio stream stats packet if it's time if (_sendAudioStreamStats) { nodeData->sendAudioStreamStatsPackets(node); - if (_printStreamStats) { - printf("\nStats for agent %s:\n", node->getUUID().toString().toLatin1().data()); - nodeData->printUpstreamDownstreamStats(); - streamStatsPrinted = true; - } _sendAudioStreamStats = false; } @@ -694,9 +696,6 @@ void AudioMixer::run() { } } } - if (streamStatsPrinted) { - printf("\n----------------------------------------------------------------\n"); - } ++_numStatFrames; @@ -720,7 +719,71 @@ void AudioMixer::perSecondActions() { int callsLastSecond = _datagramsReadPerCallStats.getCurrentIntervalSamples(); _readPendingCallsPerSecondStats.update(callsLastSecond); + if (_printStreamStats) { + + printf("\n================================================================================\n\n"); + + printf(" readPendingDatagram() calls per second | avg: %.2f, avg_30s: %.2f, last_second: %d\n", + _readPendingCallsPerSecondStats.getAverage(), + _readPendingCallsPerSecondStats.getWindowAverage(), + callsLastSecond); + + printf(" Datagrams read per call | avg: %.2f, avg_30s: %.2f, last_second: %.2f\n", + _datagramsReadPerCallStats.getAverage(), + _datagramsReadPerCallStats.getWindowAverage(), + _datagramsReadPerCallStats.getCurrentIntervalAverage()); + + printf(" Usecs spent per readPendingDatagram() call | avg: %.2f, avg_30s: %.2f, last_second: %.2f\n", + _timeSpentPerCallStats.getAverage(), + _timeSpentPerCallStats.getWindowAverage(), + _timeSpentPerCallStats.getCurrentIntervalAverage()); + + printf(" Usecs spent per packetVersionAndHashMatch() call | avg: %.2f, avg_30s: %.2f, last_second: %.2f\n", + _timeSpentPerHashMatchCallStats.getAverage(), + _timeSpentPerHashMatchCallStats.getWindowAverage(), + _timeSpentPerHashMatchCallStats.getCurrentIntervalAverage()); + + double WINDOW_LENGTH_USECS = READ_DATAGRAMS_STATS_WINDOW_SECONDS * USECS_PER_SECOND; + + printf(" %% time spent in readPendingDatagram() calls | avg_30s: %.6f%%, last_second: %.6f%%\n", + _timeSpentPerCallStats.getWindowSum() / WINDOW_LENGTH_USECS * 100.0, + _timeSpentPerCallStats.getCurrentIntervalSum() / USECS_PER_SECOND * 100.0); + + printf("%% time spent in packetVersionAndHashMatch() calls: | avg_30s: %.6f%%, last_second: %.6f%%\n", + _timeSpentPerHashMatchCallStats.getWindowSum() / WINDOW_LENGTH_USECS * 100.0, + _timeSpentPerHashMatchCallStats.getCurrentIntervalSum() / USECS_PER_SECOND * 100.0); + + foreach(const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + if (node->getLinkedData()) { + AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); + + if (node->getType() == NodeType::Agent && node->getActiveSocket()) { + printf("\nStats for agent %s --------------------------------\n", + node->getUUID().toString().toLatin1().data()); + nodeData->printUpstreamDownstreamStats(); + } + } + } + } + _datagramsReadPerCallStats.currentIntervalComplete(); _timeSpentPerCallStats.currentIntervalComplete(); _timeSpentPerHashMatchCallStats.currentIntervalComplete(); } + +QString AudioMixer::getReadPendingDatagramsStatsString() const { + QString result + = "calls_per_sec_avg_30s: " + QString::number(_readPendingCallsPerSecondStats.getWindowAverage(), 'f', 2) + + " calls_last_sec: " + QString::number(_readPendingCallsPerSecondStats.getLastCompleteIntervalStats().getSum() + 0.5, 'f', 0) + + " pkts_per_call_avg_30s: " + QString::number(_datagramsReadPerCallStats.getWindowAverage(), 'f', 2) + + " pkts_per_call_avg_1s: " + QString::number(_datagramsReadPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) + + " usecs_per_call_avg_30s: " + QString::number(_timeSpentPerCallStats.getWindowAverage(), 'f', 2) + + " usecs_per_call_avg_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) + + " usecs_per_hashmatch_avg_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowAverage(), 'f', 2) + + " usecs_per_hashmatch_avg_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) + + " prct_time_in_call_30s: " + QString::number(_timeSpentPerCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" + + " prct_time_in_call_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%" + + " prct_time_in_hashmatch_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" + + " prct_time_in_hashmatch_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%"; + return result; +} \ No newline at end of file diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index e47c5f3811..0c1378d54f 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -54,6 +54,7 @@ private: void perSecondActions(); + QString getReadPendingDatagramsStatsString() const; float _trailingSleepRatio; float _minAudibilityThreshold; diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 16fcb94dcf..4a044392c1 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -64,6 +64,7 @@ public: T getMax() const { return _max; } double getAverage() const { return _average; } int getSamples() const { return _samples; } + double getSum() const { return _samples * _average; } private: T _min; @@ -152,14 +153,29 @@ public: T getMin() const { return _overallStats.getMin(); } T getMax() const { return _overallStats.getMax(); } double getAverage() const { return _overallStats.getAverage(); } + int getSamples() const { return _overallStats.getSamples(); } + double getSum() const { return _overallStats.getSum(); } + T getWindowMin() const { return _windowStats.getMin(); } T getWindowMax() const { return _windowStats.getMax(); } double getWindowAverage() const { return _windowStats.getAverage(); } + int getWindowSamples() const { return _windowStats.getSamples(); } + double getWindowSum() const { return _windowStats.getSum(); } - int getCurrentIntervalSamples() const { return _windowStats._samples; } - + T getCurrentIntervalMin() const { return _currentIntervalStats.getMin(); } + T getCurrentIntervalMax() const { return _currentIntervalStats.getMax(); } + double getCurrentIntervalAverage() const { return _currentIntervalStats.getAverage(); } + int getCurrentIntervalSamples() const { return _currentIntervalStats.getSamples(); } + double getCurrentIntervalSum() const { return _currentIntervalStats.getSum(); } + const MinMaxAvg& getOverallStats() const{ return _overallStats; } const MinMaxAvg& getWindowStats() const{ return _windowStats; } + const MinMaxAvg& getCurrentIntervalStats() const { return _currentIntervalStats; } + + MinMaxAvg getLastCompleteIntervalStats() const { + const MinMaxAvg* stats = _intervalStats.getNewestEntry(); + return stats == NULL ? MinMaxAvg() : *stats; + } bool isWindowFilled() const { return _intervalStats.isFilled(); } diff --git a/tests/jitter/src/main.cpp b/tests/jitter/src/main.cpp index a33347f9ef..07dc7062a8 100644 --- a/tests/jitter/src/main.cpp +++ b/tests/jitter/src/main.cpp @@ -13,6 +13,7 @@ #include #include #endif +#include #include #include // for MovingMinMaxAvg @@ -103,7 +104,10 @@ void runSend(const char* addressOption, int port, int gap, int size, int report) // pack seq num memcpy(outputBuffer, &outgoingSequenceNumber, sizeof(quint16)); - sendto(sockfd, outputBuffer, size, 0, (struct sockaddr *)&servaddr, sizeof(servaddr)); + int n = sendto(sockfd, outputBuffer, size, 0, (struct sockaddr *)&servaddr, sizeof(servaddr)); + if (n < 0) { + std::cout << "Send error: " << strerror(errno) << "\n"; + } outgoingSequenceNumber++; int gapDifferece = actualGap - gap; @@ -144,6 +148,7 @@ void runSend(const char* addressOption, int port, int gap, int size, int report) } } } + delete[] outputBuffer; } void runReceive(const char* addressOption, int port, int gap, int size, int report) { @@ -195,6 +200,9 @@ void runReceive(const char* addressOption, int port, int gap, int size, int repo while (true) { n = recvfrom(sockfd, inputBuffer, size, 0, NULL, NULL); // we don't care about where it came from + if (n < 0) { + std::cout << "Receive error: " << strerror(errno) << "\n"; + } // parse seq num quint16 incomingSequenceNumber = *(reinterpret_cast(inputBuffer)); @@ -260,5 +268,6 @@ void runReceive(const char* addressOption, int port, int gap, int size, int repo } } } + delete[] inputBuffer; } From 047c4dff655e9b2acd1a603d0581ff2f1f877b34 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 17:05:53 -0700 Subject: [PATCH 32/34] removed unused var --- assignment-client/src/audio/AudioMixer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d32f14ac0f..023a7a6ce9 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -636,8 +636,7 @@ void AudioMixer::run() { perSecondActions(); _lastPerSecondCallbackTime = now; } - - bool streamStatsPrinted = false; + foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); From e50bd1bed9d81868d8e725b8aa69d6b1b709369f Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 13 Aug 2014 17:57:04 -0700 Subject: [PATCH 33/34] separated readpendingdatagrams domain page stats --- assignment-client/src/audio/AudioMixer.cpp | 63 ++++++++++++++++------ assignment-client/src/audio/AudioMixer.h | 5 +- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 023a7a6ce9..036f1f4e50 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -404,12 +404,30 @@ void AudioMixer::sendStatsPacket() { int sizeOfStats = 0; int TOO_BIG_FOR_MTU = 1200; // some extra space for JSONification - QString property = "readPendingDatagramsStats"; - QString value = getReadPendingDatagramsStatsString(); + QString property = "readPendingDatagram_calls_stats"; + QString value = getReadPendingDatagramsCallsPerSecondsStatsString(); statsObject2[qPrintable(property)] = value; somethingToSend = true; sizeOfStats += property.size() + value.size(); + property = "readPendingDatagram_packets_per_call_stats"; + value = getReadPendingDatagramsPacketsPerCallStatsString(); + statsObject2[qPrintable(property)] = value; + somethingToSend = true; + sizeOfStats += property.size() + value.size(); + + property = "readPendingDatagram_packets_time_per_call_stats"; + value = getReadPendingDatagramsTimeStatsString(); + statsObject2[qPrintable(property)] = value; + somethingToSend = true; + sizeOfStats += property.size() + value.size(); + + property = "readPendingDatagram_hashmatch_time_per_call_stats"; + value = getReadPendingDatagramsHashMatchTimeStatsString(); + statsObject2[qPrintable(property)] = value; + somethingToSend = true; + sizeOfStats += property.size() + value.size(); + NodeList* nodeList = NodeList::getInstance(); int clientNumber = 0; foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { @@ -770,19 +788,30 @@ void AudioMixer::perSecondActions() { _timeSpentPerHashMatchCallStats.currentIntervalComplete(); } -QString AudioMixer::getReadPendingDatagramsStatsString() const { - QString result - = "calls_per_sec_avg_30s: " + QString::number(_readPendingCallsPerSecondStats.getWindowAverage(), 'f', 2) - + " calls_last_sec: " + QString::number(_readPendingCallsPerSecondStats.getLastCompleteIntervalStats().getSum() + 0.5, 'f', 0) - + " pkts_per_call_avg_30s: " + QString::number(_datagramsReadPerCallStats.getWindowAverage(), 'f', 2) - + " pkts_per_call_avg_1s: " + QString::number(_datagramsReadPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) - + " usecs_per_call_avg_30s: " + QString::number(_timeSpentPerCallStats.getWindowAverage(), 'f', 2) - + " usecs_per_call_avg_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) - + " usecs_per_hashmatch_avg_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowAverage(), 'f', 2) - + " usecs_per_hashmatch_avg_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) - + " prct_time_in_call_30s: " + QString::number(_timeSpentPerCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" - + " prct_time_in_call_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%" - + " prct_time_in_hashmatch_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" - + " prct_time_in_hashmatch_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%"; +QString AudioMixer::getReadPendingDatagramsCallsPerSecondsStatsString() const { + QString result = "calls_per_sec_avg_30s: " + QString::number(_readPendingCallsPerSecondStats.getWindowAverage(), 'f', 2) + + " calls_last_sec: " + QString::number(_readPendingCallsPerSecondStats.getLastCompleteIntervalStats().getSum() + 0.5, 'f', 0); return result; -} \ No newline at end of file +} + +QString AudioMixer::getReadPendingDatagramsPacketsPerCallStatsString() const { + QString result = "pkts_per_call_avg_30s: " + QString::number(_datagramsReadPerCallStats.getWindowAverage(), 'f', 2) + + " pkts_per_call_avg_1s: " + QString::number(_datagramsReadPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2); + return result; +} + +QString AudioMixer::getReadPendingDatagramsTimeStatsString() const { + QString result = "usecs_per_call_avg_30s: " + QString::number(_timeSpentPerCallStats.getWindowAverage(), 'f', 2) + + " usecs_per_call_avg_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) + + " prct_time_in_call_30s: " + QString::number(_timeSpentPerCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" + + " prct_time_in_call_1s: " + QString::number(_timeSpentPerCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%"; + return result; +} + +QString AudioMixer::getReadPendingDatagramsHashMatchTimeStatsString() const { + QString result = "usecs_per_hashmatch_avg_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowAverage(), 'f', 2) + + " usecs_per_hashmatch_avg_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getAverage(), 'f', 2) + + " prct_time_in_hashmatch_30s: " + QString::number(_timeSpentPerHashMatchCallStats.getWindowSum() / (READ_DATAGRAMS_STATS_WINDOW_SECONDS*USECS_PER_SECOND) * 100.0, 'f', 6) + "%" + + " prct_time_in_hashmatch_1s: " + QString::number(_timeSpentPerHashMatchCallStats.getLastCompleteIntervalStats().getSum() / USECS_PER_SECOND * 100.0, 'f', 6) + "%"; + return result; +} diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 0c1378d54f..7b8dc7af43 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -54,7 +54,10 @@ private: void perSecondActions(); - QString getReadPendingDatagramsStatsString() const; + QString getReadPendingDatagramsCallsPerSecondsStatsString() const; + QString getReadPendingDatagramsPacketsPerCallStatsString() const; + QString getReadPendingDatagramsTimeStatsString() const; + QString getReadPendingDatagramsHashMatchTimeStatsString() const; float _trailingSleepRatio; float _minAudibilityThreshold; From 7a8a8684d6f8c9956ca7e4f81eb8064b8dece58e Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 14 Aug 2014 15:56:13 -0700 Subject: [PATCH 34/34] fixed crash when audioscope frame size is reduced --- interface/src/Audio.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 3009262625..6a22f139b7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -1236,8 +1236,6 @@ void Audio::selectAudioFilterSmiley() { void Audio::toggleScope() { _scopeEnabled = !_scopeEnabled; if (_scopeEnabled) { - _scopeInputOffset = 0; - _scopeOutputOffset = 0; allocateScope(); } else { freeScope(); @@ -1275,6 +1273,8 @@ void Audio::selectAudioScopeFiftyFrames() { } void Audio::allocateScope() { + _scopeInputOffset = 0; + _scopeOutputOffset = 0; int num = _samplesPerScope * sizeof(int16_t); _scopeInput = new QByteArray(num, 0); _scopeOutputLeft = new QByteArray(num, 0);