From 6d7fab40530c781ffed6f6b6506352c678435033 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Tue, 21 Feb 2017 16:59:08 -0800 Subject: [PATCH 1/5] Agent Avatars sending loudness in AvatarData --- assignment-client/src/Agent.cpp | 53 +++++++++++++++++++++++++-------- assignment-client/src/Agent.h | 1 + 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index bea677aeb6..cd8c8189b2 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -88,9 +88,9 @@ void Agent::playAvatarSound(SharedSoundPointer sound) { QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound)); return; } else { - // TODO: seems to add occasional artifact in tests. I believe it is + // TODO: seems to add occasional artifact in tests. I believe it is // correct to do this, but need to figure out for sure, so commenting this - // out until I verify. + // out until I verify. // _numAvatarSoundSentBytes = 0; setAvatarSound(sound); } @@ -105,7 +105,7 @@ void Agent::handleOctreePacket(QSharedPointer message, SharedNo if (message->getSize() > statsMessageLength) { // pull out the piggybacked packet and create a new QSharedPointer for it int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength; - + auto buffer = std::unique_ptr(new char[piggyBackedSizeWithHeader]); memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader); @@ -284,7 +284,7 @@ void Agent::selectAudioFormat(const QString& selectedCodecName) { for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; - _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); + _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; @@ -424,16 +424,16 @@ void Agent::executeScript() { entityScriptingInterface->setEntityTree(_entityViewer.getTree()); DependencyManager::set(_entityViewer.getTree()); - + // 100Hz timer for audio AvatarAudioTimer* audioTimerWorker = new AvatarAudioTimer(); audioTimerWorker->moveToThread(&_avatarAudioTimerThread); connect(audioTimerWorker, &AvatarAudioTimer::avatarTick, this, &Agent::processAgentAvatarAudio); connect(this, &Agent::startAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::start); connect(this, &Agent::stopAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::stop); - connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); + connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); _avatarAudioTimerThread.start(); - + // 60Hz timer for avatar QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar); _scriptEngine->run(); @@ -448,14 +448,14 @@ QUuid Agent::getSessionUUID() const { return DependencyManager::get()->getSessionUUID(); } -void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { +void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { // this must happen on Agent's main thread if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "setIsListeningToAudioStream", Q_ARG(bool, isListeningToAudioStream)); return; } if (_isListeningToAudioStream) { - // have to tell just the audio mixer to KillAvatar. + // have to tell just the audio mixer to KillAvatar. auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode( @@ -471,7 +471,7 @@ void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { }); } - _isListeningToAudioStream = isListeningToAudioStream; + _isListeningToAudioStream = isListeningToAudioStream; } void Agent::setIsAvatar(bool isAvatar) { @@ -552,6 +552,7 @@ void Agent::processAgentAvatar() { nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); } } + void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) { _flushEncoder = false; static const QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0); @@ -562,6 +563,23 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) { } } +void Agent::computeLoudness(QByteArray* decodedBuffer) { + float loudness = 0.0f; + auto scriptedAvatar = DependencyManager::get(); + if (decodedBuffer) { + auto soundData = reinterpret_cast(decodedBuffer->constData()); + auto numFrames = decodedBuffer->size() / sizeof(int16_t); + // now iterate and come up with average + if (numFrames > 0) { + for(int i = 0; i < numFrames; i++) { + loudness += (float) std::abs(soundData[i]); + } + loudness /= numFrames; + } + } + scriptedAvatar->setAudioLoudness(loudness); +} + void Agent::processAgentAvatarAudio() { auto recordingInterface = DependencyManager::get(); bool isPlayingRecording = recordingInterface->isPlaying(); @@ -611,6 +629,9 @@ void Agent::processAgentAvatarAudio() { audioPacket->seek(sizeof(quint16)); if (silentFrame) { + // no matter what, the loudness should be set to 0 + computeLoudness(nullptr); + if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here return; @@ -618,7 +639,7 @@ void Agent::processAgentAvatarAudio() { // write the codec audioPacket->writeString(_selectedCodecName); - + // write the number of silent samples so the audio-mixer can uphold timing audioPacket->writePrimitive(numAvailableSamples); @@ -628,8 +649,9 @@ void Agent::processAgentAvatarAudio() { audioPacket->writePrimitive(headOrientation); audioPacket->writePrimitive(scriptedAvatar->getPosition()); audioPacket->writePrimitive(glm::vec3(0)); + } else if (nextSoundOutput) { - + // write the codec audioPacket->writeString(_selectedCodecName); @@ -646,6 +668,8 @@ void Agent::processAgentAvatarAudio() { QByteArray encodedBuffer; if (_flushEncoder) { encodeFrameOfZeros(encodedBuffer); + // loudness is 0 + computeLoudness(nullptr); } else { QByteArray decodedBuffer(reinterpret_cast(nextSoundOutput), numAvailableSamples*sizeof(int16_t)); if (_encoder) { @@ -654,10 +678,15 @@ void Agent::processAgentAvatarAudio() { } else { encodedBuffer = decodedBuffer; } + computeLoudness(&decodedBuffer); } audioPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } + // we should never have both nextSoundOutput being null and silentFrame being false, but lets + // assert on it in case things above change in a bad way + assert(nextSoundOutput || silentFrame); + // write audio packet to AudioMixer nodes auto nodeList = DependencyManager::get(); nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node) { diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index c9b1707101..7f04b4746f 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -82,6 +82,7 @@ private: void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); void encodeFrameOfZeros(QByteArray& encodedZeros); + void computeLoudness(QByteArray* decodedBuffer); std::unique_ptr _scriptEngine; EntityEditPacketSender _entityEditSender; From 397eb89c143d2e746ca6e5ecd3980c210ea95e04 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Wed, 22 Feb 2017 08:38:49 -0800 Subject: [PATCH 2/5] compiler warning - odd --- assignment-client/src/Agent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index cd8c8189b2..2f91571d8b 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -568,7 +568,7 @@ void Agent::computeLoudness(QByteArray* decodedBuffer) { auto scriptedAvatar = DependencyManager::get(); if (decodedBuffer) { auto soundData = reinterpret_cast(decodedBuffer->constData()); - auto numFrames = decodedBuffer->size() / sizeof(int16_t); + int numFrames = decodedBuffer->size() / sizeof(int16_t); // now iterate and come up with average if (numFrames > 0) { for(int i = 0; i < numFrames; i++) { From 2635657456a097bfc99d28eb44711f3b7a6b0797 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Tue, 28 Feb 2017 13:24:25 -0700 Subject: [PATCH 3/5] zappoman's feedback, plus added const corrrectness to computeLoudness --- assignment-client/src/Agent.cpp | 4 +++- assignment-client/src/Agent.h | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 5211c3beaa..9c830ef391 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -380,6 +380,8 @@ void Agent::executeScript() { audioTransform.setTranslation(scriptedAvatar->getPosition()); audioTransform.setRotation(headOrientation); + computeLoudness(&audio); + QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audio, encodedBuffer); @@ -571,7 +573,7 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) { } } -void Agent::computeLoudness(QByteArray* decodedBuffer) { +void Agent::computeLoudness(const QByteArray* decodedBuffer) { float loudness = 0.0f; auto scriptedAvatar = DependencyManager::get(); if (decodedBuffer) { diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 7f04b4746f..ce7393011f 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -68,10 +68,10 @@ private slots: void handleAudioPacket(QSharedPointer message); void handleOctreePacket(QSharedPointer message, SharedNodePointer senderNode); void handleJurisdictionPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleSelectedAudioFormat(QSharedPointer message); + void handleSelectedAudioFormat(QSharedPointer message); void nodeActivated(SharedNodePointer activatedNode); - + void processAgentAvatar(); void processAgentAvatarAudio(); @@ -82,7 +82,7 @@ private: void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); void encodeFrameOfZeros(QByteArray& encodedZeros); - void computeLoudness(QByteArray* decodedBuffer); + void computeLoudness(const QByteArray* decodedBuffer); std::unique_ptr _scriptEngine; EntityEditPacketSender _entityEditSender; @@ -104,10 +104,10 @@ private: bool _isAvatar = false; QTimer* _avatarIdentityTimer = nullptr; QHash _outgoingScriptAudioSequenceNumbers; - + CodecPluginPointer _codec; QString _selectedCodecName; - Encoder* _encoder { nullptr }; + Encoder* _encoder { nullptr }; QThread _avatarAudioTimerThread; bool _flushEncoder { false }; }; From 81451191c1fef266e2b78b9549b1bb855710caea Mon Sep 17 00:00:00 2001 From: David Kelly Date: Wed, 1 Mar 2017 18:06:03 -0700 Subject: [PATCH 4/5] no need to get the ScriptableAvatar from DependencyManager every time --- assignment-client/src/Agent.cpp | 14 ++++++-------- assignment-client/src/Agent.h | 3 ++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 9c830ef391..d20ef2e687 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -43,7 +43,6 @@ #include #include // TODO: consider moving to scriptengine.h -#include "avatars/ScriptableAvatar.h" #include "entities/AssignmentParentFinder.h" #include "RecordingScriptingInterface.h" #include "AbstractAudioInterface.h" @@ -380,7 +379,7 @@ void Agent::executeScript() { audioTransform.setTranslation(scriptedAvatar->getPosition()); audioTransform.setRotation(headOrientation); - computeLoudness(&audio); + computeLoudness(&audio, scriptedAvatar); QByteArray encodedBuffer; if (_encoder) { @@ -573,9 +572,8 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) { } } -void Agent::computeLoudness(const QByteArray* decodedBuffer) { +void Agent::computeLoudness(const QByteArray* decodedBuffer, QSharedPointer scriptableAvatar) { float loudness = 0.0f; - auto scriptedAvatar = DependencyManager::get(); if (decodedBuffer) { auto soundData = reinterpret_cast(decodedBuffer->constData()); int numFrames = decodedBuffer->size() / sizeof(int16_t); @@ -587,7 +585,7 @@ void Agent::computeLoudness(const QByteArray* decodedBuffer) { loudness /= numFrames; } } - scriptedAvatar->setAudioLoudness(loudness); + scriptableAvatar->setAudioLoudness(loudness); } void Agent::processAgentAvatarAudio() { @@ -640,7 +638,7 @@ void Agent::processAgentAvatarAudio() { if (silentFrame) { // no matter what, the loudness should be set to 0 - computeLoudness(nullptr); + computeLoudness(nullptr, scriptedAvatar); if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here @@ -679,7 +677,7 @@ void Agent::processAgentAvatarAudio() { if (_flushEncoder) { encodeFrameOfZeros(encodedBuffer); // loudness is 0 - computeLoudness(nullptr); + computeLoudness(nullptr, scriptedAvatar); } else { QByteArray decodedBuffer(reinterpret_cast(nextSoundOutput), numAvailableSamples*sizeof(int16_t)); if (_encoder) { @@ -688,7 +686,7 @@ void Agent::processAgentAvatarAudio() { } else { encodedBuffer = decodedBuffer; } - computeLoudness(&decodedBuffer); + computeLoudness(&decodedBuffer, scriptedAvatar); } audioPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index ce7393011f..0ce7b71d5d 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -30,6 +30,7 @@ #include #include "MixedAudioStream.h" +#include "avatars/ScriptableAvatar.h" class Agent : public ThreadedAssignment { Q_OBJECT @@ -82,7 +83,7 @@ private: void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); void encodeFrameOfZeros(QByteArray& encodedZeros); - void computeLoudness(const QByteArray* decodedBuffer); + void computeLoudness(const QByteArray* decodedBuffer, QSharedPointer); std::unique_ptr _scriptEngine; EntityEditPacketSender _entityEditSender; From 29f263a29668e5901865c8d60aadf772183b4026 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Thu, 2 Mar 2017 12:43:41 -0700 Subject: [PATCH 5/5] oh man, this was hard to find --- assignment-client/src/Agent.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index d20ef2e687..be23dcfa25 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -637,8 +637,6 @@ void Agent::processAgentAvatarAudio() { audioPacket->seek(sizeof(quint16)); if (silentFrame) { - // no matter what, the loudness should be set to 0 - computeLoudness(nullptr, scriptedAvatar); if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here @@ -658,6 +656,8 @@ void Agent::processAgentAvatarAudio() { audioPacket->writePrimitive(scriptedAvatar->getPosition()); audioPacket->writePrimitive(glm::vec3(0)); + // no matter what, the loudness should be set to 0 + computeLoudness(nullptr, scriptedAvatar); } else if (nextSoundOutput) { // write the codec