Agent Avatars sending loudness in AvatarData

assignment-client/src/Agent.cpp

@@ -88,9 +88,9 @@ void Agent::playAvatarSound(SharedSoundPointer sound) {
         QMetaObject::invokeMethod(this, "playAvatarSound", Q_ARG(SharedSoundPointer, sound));
         return;
     } else {
-        // TODO: seems to add occasional artifact in tests.  I believe it is 
+        // TODO: seems to add occasional artifact in tests.  I believe it is
         // correct to do this, but need to figure out for sure, so commenting this
-        // out until I verify.  
+        // out until I verify.
         // _numAvatarSoundSentBytes = 0;
         setAvatarSound(sound);
     }
@@ -105,7 +105,7 @@ void Agent::handleOctreePacket(QSharedPointer<ReceivedMessage> message, SharedNo
         if (message->getSize() > statsMessageLength) {
             // pull out the piggybacked packet and create a new QSharedPointer<NLPacket> for it
             int piggyBackedSizeWithHeader = message->getSize() - statsMessageLength;
-            
+
             auto buffer = std::unique_ptr<char[]>(new char[piggyBackedSizeWithHeader]);
             memcpy(buffer.get(), message->getRawMessage() + statsMessageLength, piggyBackedSizeWithHeader);
 
@@ -284,7 +284,7 @@ void Agent::selectAudioFormat(const QString& selectedCodecName) {
     for (auto& plugin : codecPlugins) {
         if (_selectedCodecName == plugin->getName()) {
             _codec = plugin;
-            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); 
+            _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO);
             _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
             qDebug() << "Selected Codec Plugin:" << _codec.get();
             break;
@@ -424,16 +424,16 @@ void Agent::executeScript() {
     entityScriptingInterface->setEntityTree(_entityViewer.getTree());
 
     DependencyManager::set<AssignmentParentFinder>(_entityViewer.getTree());
-    
+
     // 100Hz timer for audio
     AvatarAudioTimer* audioTimerWorker = new AvatarAudioTimer();
     audioTimerWorker->moveToThread(&_avatarAudioTimerThread);
     connect(audioTimerWorker, &AvatarAudioTimer::avatarTick, this, &Agent::processAgentAvatarAudio);
     connect(this, &Agent::startAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::start);
     connect(this, &Agent::stopAvatarAudioTimer, audioTimerWorker, &AvatarAudioTimer::stop);
-    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater); 
+    connect(&_avatarAudioTimerThread, &QThread::finished, audioTimerWorker, &QObject::deleteLater);
     _avatarAudioTimerThread.start();
-    
+
     // 60Hz timer for avatar
     QObject::connect(_scriptEngine.get(), &ScriptEngine::update, this, &Agent::processAgentAvatar);
     _scriptEngine->run();
@@ -448,14 +448,14 @@ QUuid Agent::getSessionUUID() const {
     return DependencyManager::get<NodeList>()->getSessionUUID();
 }
 
-void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) { 
+void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
     // this must happen on Agent's main thread
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "setIsListeningToAudioStream", Q_ARG(bool, isListeningToAudioStream));
         return;
     }
     if (_isListeningToAudioStream) {
-        // have to tell just the audio mixer to KillAvatar. 
+        // have to tell just the audio mixer to KillAvatar.
 
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachMatchingNode(
@@ -471,7 +471,7 @@ void Agent::setIsListeningToAudioStream(bool isListeningToAudioStream) {
         });
 
     }
-    _isListeningToAudioStream = isListeningToAudioStream; 
+    _isListeningToAudioStream = isListeningToAudioStream;
 }
 
 void Agent::setIsAvatar(bool isAvatar) {
@@ -552,6 +552,7 @@ void Agent::processAgentAvatar() {
         nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer);
     }
 }
+
 void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     _flushEncoder = false;
     static const QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0);
@@ -562,6 +563,23 @@ void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     }
 }
 
+void Agent::computeLoudness(QByteArray* decodedBuffer) {
+    float loudness = 0.0f;
+    auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
+    if (decodedBuffer) {
+        auto soundData = reinterpret_cast<const int16_t*>(decodedBuffer->constData());
+        auto numFrames = decodedBuffer->size() / sizeof(int16_t);
+        // now iterate and come up with average
+        if (numFrames > 0) {
+            for(int i = 0; i < numFrames; i++) {
+                loudness += (float) std::abs(soundData[i]);
+            }
+            loudness /= numFrames;
+        }
+    }
+    scriptedAvatar->setAudioLoudness(loudness);
+}
+
 void Agent::processAgentAvatarAudio() {
     auto recordingInterface = DependencyManager::get<RecordingScriptingInterface>();
     bool isPlayingRecording = recordingInterface->isPlaying();
@@ -611,6 +629,9 @@ void Agent::processAgentAvatarAudio() {
         audioPacket->seek(sizeof(quint16));
 
         if (silentFrame) {
+            // no matter what, the loudness should be set to 0
+            computeLoudness(nullptr);
+
             if (!_isListeningToAudioStream) {
                 // if we have a silent frame and we're not listening then just send nothing and break out of here
                 return;
@@ -618,7 +639,7 @@ void Agent::processAgentAvatarAudio() {
 
             // write the codec
             audioPacket->writeString(_selectedCodecName);
-            
+
             // write the number of silent samples so the audio-mixer can uphold timing
             audioPacket->writePrimitive(numAvailableSamples);
 
@@ -628,8 +649,9 @@ void Agent::processAgentAvatarAudio() {
             audioPacket->writePrimitive(headOrientation);
             audioPacket->writePrimitive(scriptedAvatar->getPosition());
             audioPacket->writePrimitive(glm::vec3(0));
+
         } else if (nextSoundOutput) {
-            
+
             // write the codec
             audioPacket->writeString(_selectedCodecName);
 
@@ -646,6 +668,8 @@ void Agent::processAgentAvatarAudio() {
             QByteArray encodedBuffer;
             if (_flushEncoder) {
                 encodeFrameOfZeros(encodedBuffer);
+                // loudness is 0
+                computeLoudness(nullptr);
             } else {
                 QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
                 if (_encoder) {
@@ -654,10 +678,15 @@ void Agent::processAgentAvatarAudio() {
                 } else {
                     encodedBuffer = decodedBuffer;
                 }
+                computeLoudness(&decodedBuffer);
             }
             audioPacket->write(encodedBuffer.constData(), encodedBuffer.size());
         }
 
+        // we should never have both nextSoundOutput being null and silentFrame being false, but lets
+        // assert on it in case things above change in a bad way
+        assert(nextSoundOutput || silentFrame);
+
         // write audio packet to AudioMixer nodes
         auto nodeList = DependencyManager::get<NodeList>();
         nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node) {

assignment-client/src/Agent.h

@@ -82,6 +82,7 @@ private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
     void encodeFrameOfZeros(QByteArray& encodedZeros);
+    void computeLoudness(QByteArray* decodedBuffer);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;