From ff438a24344c63befa510a66d5f2bee168c8945a Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 11:23:50 -0700 Subject: [PATCH] added seq stats for Agent, fixed AudioInjector to do << (quint16)0 instead of skipRawData(sizeof(quint16) to correctly allocate space for seq number; added debug prints; --- assignment-client/src/Agent.cpp | 8 +++++++- assignment-client/src/Agent.h | 2 ++ assignment-client/src/audio/AudioMixer.cpp | 5 ++++- .../src/audio/AudioMixerClientData.cpp | 20 +++++++++++-------- .../src/audio/AudioMixerClientData.h | 10 +++++----- interface/src/Audio.cpp | 14 +++++++------ interface/src/Audio.h | 4 ++-- libraries/audio/src/AudioInjector.cpp | 15 ++++++++------ libraries/script-engine/src/ScriptEngine.cpp | 6 +++--- libraries/script-engine/src/ScriptEngine.h | 2 +- 10 files changed, 53 insertions(+), 33 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 561acc0eb0..a2c9b30ac1 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -147,7 +147,13 @@ void Agent::readPendingDatagrams() { } } else if (datagramPacketType == PacketTypeMixedAudio) { - // TODO: track sequence numbers for mixed audio??? + + // parse sequence number for this packet + int numBytesPacketHeader = numBytesForPacketHeader(receivedPacket); + const char* sequenceAt = receivedPacket.constData() + numBytesPacketHeader; + quint16 sequence = *(reinterpret_cast(sequenceAt)); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("mixed audio received %d\n", sequence); // parse the data and grab the average loudness _receivedAudioBuffer.parseData(receivedPacket); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 9af95e757c..ec8f7c88cb 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -71,6 +71,8 @@ private: ModelTreeHeadlessViewer _modelViewer; MixedAudioRingBuffer _receivedAudioBuffer; + SequenceNumberStats _incomingMixedAudioSequenceNumberStats; + AvatarHashMap _avatarHashMap; }; diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6b43fb37e0..094d6da371 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -457,6 +457,7 @@ void AudioMixer::sendStatsPacket() { // if we're too large, send the packet if (sizeOfStats > TOO_BIG_FOR_MTU) { +printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); sizeOfStats = 0; statsObject2 = QJsonObject(); // clear it @@ -465,6 +466,7 @@ void AudioMixer::sendStatsPacket() { } if (somethingToSend) { +printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); } } @@ -621,7 +623,8 @@ void AudioMixer::run() { dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; // send mixed audio packet - nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + //nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); +printf("mixed audio sent %d\n", sequence); nodeData->incrementOutgoingSequenceNumber(); // send an audio stream stats packet if it's time diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 8f50647463..6eec020991 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -21,8 +21,8 @@ AudioMixerClientData::AudioMixerClientData() : _ringBuffers(), - _outgoingSequenceNumber(0), - _incomingAvatarSequenceNumberStats() + _outgoingMixedAudioSequenceNumber(0), + _incomingAvatarAudioSequenceNumberStats() { } @@ -57,13 +57,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { || packetType == PacketTypeMicrophoneAudioNoEcho || packetType == PacketTypeSilentAudioFrame) { - _incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("avatar audio received %d\n", sequence); // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); // read the first byte after the header to see if this is a stereo or mono buffer - quint8 channelFlag = packet.at(numBytesForPacketHeader(packet)); + quint8 channelFlag = packet.at(numBytesForPacketHeader(packet) + sizeof(quint16)); bool isStereo = channelFlag == 1; if (avatarRingBuffer && avatarRingBuffer->isStereo() != isStereo) { @@ -86,9 +87,10 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // this is injected audio // grab the stream identifier for this injected audio - QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID)); + QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID)); - _incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); + _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); +printf("injected stream %s received seq %d\n", streamIdentifier.toString().toLatin1().data(), sequence); InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; @@ -96,6 +98,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector && ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) { matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i]; +printf("\t matching ring buffer found.\n"); } } @@ -104,6 +107,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()); _ringBuffers.push_back(matchingInjectedRingBuffer); +printf("\t no matching ring buffer, creating new one. _ringBuffer size now %d\n", _ringBuffers.size()); } matchingInjectedRingBuffer->parseData(packet); @@ -147,7 +151,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { // this is an empty audio buffer that has starved, safe to delete // also delete its sequence number stats QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier(); - _incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier); + _incomingInjectedAudioSequenceNumberStatsMap.remove(streamIdentifier); delete audioBuffer; i = _ringBuffers.erase(i); continue; @@ -212,7 +216,7 @@ QString AudioMixerClientData::getJitterBufferStatsString() const { } else { result = "mic unknown"; } - + printf("\tget jitter buffer stats string. _ringBuffer.size = %d\n", _ringBuffers.size()); for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index c418df7905..eb4e6f5dae 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -38,17 +38,17 @@ public: QString getJitterBufferStatsString() const; - void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; } + void incrementOutgoingSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } - quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; } + quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } private: QList _ringBuffers; - quint16 _outgoingSequenceNumber; - SequenceNumberStats _incomingAvatarSequenceNumberStats; - QHash _incomingInjectedSequenceNumberStatsMap; + quint16 _outgoingMixedAudioSequenceNumber; + SequenceNumberStats _incomingAvatarAudioSequenceNumberStats; + QHash _incomingInjectedAudioSequenceNumberStatsMap; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ff019712d7..024fb66cc1 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeOutputLeft(0), _scopeOutputRight(0), _audioMixerJitterBufferStats(), - _outgoingSequenceNumber(0) + _outgoingAvatarAudioSequenceNumber(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _outgoingSequenceNumber = 0; + _outgoingAvatarAudioSequenceNumber = 0; } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -657,7 +657,7 @@ void Audio::handleAudioInput() { char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); // pack sequence number - memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16)); + memcpy(currentPacketPtr, &_outgoingAvatarAudioSequenceNumber, sizeof(quint16)); currentPacketPtr += sizeof(quint16); // set the mono/stereo byte @@ -672,13 +672,14 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); - _outgoingSequenceNumber++; +printf("avatar audio sent %d\n", _outgoingAvatarAudioSequenceNumber); + _outgoingAvatarAudioSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); } else { // reset seq numbers if there's no connection with an audiomixer - _outgoingSequenceNumber = 0; + _outgoingAvatarAudioSequenceNumber = 0; } delete[] inputAudioSamples; } @@ -832,7 +833,8 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; quint16 sequence = *((quint16*)sequenceAt); - _incomingSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("mixed audio received %d\n", sequence); // parse audio data _ringBuffer.parseData(audioByteArray); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index cb04418eaf..f08cce9467 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -239,8 +239,8 @@ private: AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; - quint16 _outgoingSequenceNumber; - SequenceNumberStats _incomingSequenceNumberStats; + quint16 _outgoingAvatarAudioSequenceNumber; + SequenceNumberStats _incomingMixedAudioSequenceNumberStats; }; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index d9d4dc6d3a..01dfc8696a 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -61,12 +61,13 @@ void AudioInjector::injectAudio() { QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); QDataStream packetStream(&injectAudioPacket, QIODevice::Append); - // skip sequence number for now + // pack some placeholder sequence number for now int numPreSequenceNumberBytes = injectAudioPacket.size(); - packetStream.skipRawData(sizeof(quint16)); + packetStream << (quint16)0; // pack stream identifier (a generated UUID) - packetStream << QUuid::createUuid(); +QUuid streamID; + packetStream << (streamID = QUuid::createUuid()); // pack the flag for loopback uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface()); @@ -86,6 +87,7 @@ void AudioInjector::injectAudio() { quint8 volume = MAX_INJECTOR_VOLUME * _options.getVolume(); packetStream << volume; + QElapsedTimer timer; timer.start(); int nextFrame = 0; @@ -96,7 +98,7 @@ void AudioInjector::injectAudio() { bool shouldLoop = _options.getLoop(); // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks - quint16 outgoingSequenceNumber = 0; + quint16 outgoingInjectedAudioSequenceNumber = 0; while (currentSendPosition < soundByteArray.size() && !_shouldStop) { int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, @@ -106,7 +108,7 @@ void AudioInjector::injectAudio() { injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy); // pack the sequence number - memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16)); + memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingInjectedAudioSequenceNumber, sizeof(quint16)); // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy); @@ -116,7 +118,8 @@ void AudioInjector::injectAudio() { // send off this audio packet nodeList->writeDatagram(injectAudioPacket, audioMixer); - outgoingSequenceNumber++; +printf("injector stream %s sent seq %d\n", streamID.toString().toLatin1().data(), outgoingInjectedAudioSequenceNumber); + outgoingInjectedAudioSequenceNumber++; currentSendPosition += bytesToCopy; diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index b142819260..09b60d8e0d 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -501,9 +501,9 @@ void ScriptEngine::run() { // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { // pack sequence number - quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++; + quint16 sequence = _outgoingInjectedAudioSequenceNumbers[node->getUUID()]++; memcpy(sequenceAt, &sequence, sizeof(quint16)); - +printf("script engine audio sent %d\n", sequence); // send audio packet nodeList->writeDatagram(audioPacket, audioPacketSize, node); } @@ -683,5 +683,5 @@ void ScriptEngine::include(const QString& includeFile) { } void ScriptEngine::nodeKilled(SharedNodePointer node) { - _outgoingSequenceNumbers.remove(node->getUUID()); + _outgoingInjectedAudioSequenceNumbers.remove(node->getUUID()); } diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index f1aaf6f0bb..cd969e49cd 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -148,7 +148,7 @@ private: ScriptUUID _uuidLibrary; AnimationCache _animationCache; - QHash _outgoingSequenceNumbers; + QHash _outgoingInjectedAudioSequenceNumbers; }; #endif // hifi_ScriptEngine_h