Merge pull request #9722 from davidkelly/dk/agentAvatarDataUpdates

Agent Avatars sending loudness in AvatarData

assignment-client/src/Agent.cpp

@@ -43,7 +43,6 @@
 #include <WebSocketServerClass.h>
 #include <EntityScriptingInterface.h> // TODO: consider moving to scriptengine.h
 
-#include "avatars/ScriptableAvatar.h"
 #include "entities/AssignmentParentFinder.h"
 #include "RecordingScriptingInterface.h"
 #include "AbstractAudioInterface.h"
@@ -88,9 +87,9 @@ void Agent::playAvatarSound(SharedSoundPointer sound) {
         QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound));
         return;
     } else {
-        // TODO: seems to add occasional artifact in tests.  I believe it is 
+        // TODO: seems to add occasional artifact in tests.  I believe it is
         // correct to do this, but need to figure out for sure, so commenting this
-        // out until I verify.  
+        // out until I verify.
         // _numAvatarSoundSentBytes = 0;
         setAvatarSound(sound);
     }
@@ -105,7 +104,7 @@ void Agent::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNo
         if (message->getSize() > statsMessageLength) {
             // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it
             int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength;
-            
+
             auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]);
             memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader);
 
@@ -284,7 +283,7 @@ void Agent::selectAudioFormat(const QString& selectedCodecName) {
     for (auto& plugin : codecPlugins) {
         if (_selectedCodecName == plugin->getName()) {
             _codec = plugin;
-            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); 
+            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO);
             _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
             qDebug() << "Selected Codec Plugin:" << _codec.get();
             break;
@@ -380,6 +379,8 @@ void Agent::executeScript() {
         audioTransform.setTranslation(scriptedAvatar->getPosition());
         audioTransform.setRotation(headOrientation);
 
+        computeLoudness(&audio, scriptedAvatar);
+
         QByteArray encodedBuffer;
         if (_encoder) {
             _encoder->encode(audio, encodedBuffer);
@@ -424,16 +425,16 @@ void Agent::executeScript() {
     entityScriptingInterface->setEntityTree(_entityViewer.getTree());
 
     DependencyManager::set<AssignmentParentFinder>(_entityViewer.getTree());
-    
+
     // 100Hz timer for audio
     AvatarAudioTimer* audioTimerWorker = new AvatarAudioTimer();
     audioTimerWorker->moveToThread(&_avatarAudioTimerThread);
     connect(audioTimerWorker, &AvatarAudioTimer::avatarTick, this, &Agent::processAgentAvatarAudio);
     connect(this, &Agent::startAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::start);
     connect(this, &Agent::stopAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::stop);
-    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); 
+    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater);
     _avatarAudioTimerThread.start();
-    
+
     // Agents should run at 45hz
     static const int AVATAR_DATA_HZ = 45;
     static const int AVATAR_DATA_IN_MSECS = MSECS_PER_SECOND / AVATAR_DATA_HZ;
@@ -456,14 +457,14 @@ QUuid Agent::getSessionUUID() const {
     return DependencyManager::get<NodeList>()->getSessionUUID();
 }
 
-void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { 
+void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
     // this must happen on Agent's main thread
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "setIsListeningToAudioStream", Q_ARG(bool, isListeningToAudioStream));
         return;
     }
     if (_isListeningToAudioStream) {
-        // have to tell just the audio mixer to KillAvatar. 
+        // have to tell just the audio mixer to KillAvatar.
 
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachMatchingNode(
@@ -479,7 +480,7 @@ void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
         });
 
     }
-    _isListeningToAudioStream = isListeningToAudioStream; 
+    _isListeningToAudioStream = isListeningToAudioStream;
 }
 
 void Agent::setIsAvatar(bool isAvatar) {
@@ -560,6 +561,7 @@ void Agent::processAgentAvatar() {
         nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer);
     }
 }
+
 void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     _flushEncoder = false;
     static const QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0);
@@ -570,6 +572,22 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     }
 }
 
+void Agent::computeLoudness(const QByteArray* decodedBuffer, QSharedPointer<ScriptableAvatar> scriptableAvatar) {
+    float loudness = 0.0f;
+    if (decodedBuffer) {
+        auto soundData = reinterpret_cast<const int16_t*>(decodedBuffer->constData());
+        int numFrames = decodedBuffer->size() / sizeof(int16_t);
+        // now iterate and come up with average
+        if (numFrames > 0) {
+            for(int i = 0; i < numFrames; i++) {
+                loudness += (float) std::abs(soundData[i]);
+            }
+            loudness /= numFrames;
+        }
+    }
+    scriptableAvatar->setAudioLoudness(loudness);
+}
+
 void Agent::processAgentAvatarAudio() {
     auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>();
     bool isPlayingRecording = recordingInterface->isPlaying();
@@ -619,6 +637,7 @@ void Agent::processAgentAvatarAudio() {
         audioPacket->seek(sizeof(quint16));
 
         if (silentFrame) {
+
             if (!_isListeningToAudioStream) {
                 // if we have a silent frame and we're not listening then just send nothing and break out of here
                 return;
@@ -626,7 +645,7 @@ void Agent::processAgentAvatarAudio() {
 
             // write the codec
             audioPacket->writeString(_selectedCodecName);
-            
+
             // write the number of silent samples so the audio-mixer can uphold timing
             audioPacket->writePrimitive(numAvailableSamples);
 
@@ -636,8 +655,11 @@ void Agent::processAgentAvatarAudio() {
             audioPacket->writePrimitive(headOrientation);
             audioPacket->writePrimitive(scriptedAvatar->getPosition());
             audioPacket->writePrimitive(glm::vec3(0));
+
+            // no matter what, the loudness should be set to 0
+            computeLoudness(nullptr, scriptedAvatar);
         } else if (nextSoundOutput) {
-            
+
             // write the codec
             audioPacket->writeString(_selectedCodecName);
 
@@ -654,6 +676,8 @@ void Agent::processAgentAvatarAudio() {
             QByteArray encodedBuffer;
             if (_flushEncoder) {
                 encodeFrameOfZeros(encodedBuffer);
+                // loudness is 0
+                computeLoudness(nullptr, scriptedAvatar);
             } else {
                 QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
                 if (_encoder) {
@@ -662,10 +686,15 @@ void Agent::processAgentAvatarAudio() {
                 } else {
                     encodedBuffer = decodedBuffer;
                 }
+                computeLoudness(&decodedBuffer, scriptedAvatar);
             }
             audioPacket->write(encodedBuffer.constData(), encodedBuffer.size());
         }
 
+        // we should never have both nextSoundOutput being null and silentFrame being false, but lets
+        // assert on it in case things above change in a bad way
+        assert(nextSoundOutput || silentFrame);
+
         // write audio packet to AudioMixer nodes
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node) {

assignment-client/src/Agent.h

@@ -30,6 +30,7 @@
 #include <plugins/CodecPlugin.h>
 
 #include "MixedAudioStream.h"
+#include "avatars/ScriptableAvatar.h"
 
 class Agent : public ThreadedAssignment {
     Q_OBJECT
@@ -68,10 +69,10 @@ private slots:
     void handleAudioPacket(QSharedPointer<ReceivedMessage> message);
     void handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleJurisdictionPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message); 
+    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
 
     void nodeActivated(SharedNodePointer activatedNode);
-    
+
     void processAgentAvatar();
     void processAgentAvatarAudio();
 
@@ -82,6 +83,7 @@ private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
     void encodeFrameOfZeros(QByteArray& encodedZeros);
+    void computeLoudness(const QByteArray* decodedBuffer, QSharedPointer<ScriptableAvatar>);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;
@@ -103,10 +105,10 @@ private:
     bool _isAvatar = false;
     QTimer* _avatarIdentityTimer = nullptr;
     QHash<QUuid, quint16> _outgoingScriptAudioSequenceNumbers;
-    
+
     CodecPluginPointer _codec;
     QString _selectedCodecName;
-    Encoder* _encoder { nullptr }; 
+    Encoder* _encoder { nullptr };
     QThread _avatarAudioTimerThread;
     bool _flushEncoder { false };
 };