Merge pull request #8927 from davidkelly/dk/moreACAudioArtifacts

More AC Audio Artifacts

assignment-client/src/Agent.cpp

@@ -474,24 +474,18 @@ void Agent::processAgentAvatar() {
         nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer);
     }
 }
-void Agent::flushEncoder() {
+void Agent::encodeFrameOfZeros(QByteArray& encodedZeros) {
     _flushEncoder = false;
-    static QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0);
+    static const QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0);
-    static QByteArray encodedZeros;
     if (_encoder) {
         _encoder->encode(zeros, encodedZeros);
+    } else {
+        encodedZeros = zeros;
     }
 }
 
 void Agent::processAgentAvatarAudio() {
     if (_isAvatar && (_isListeningToAudioStream || _avatarSound)) {
-        // after sound is done playing, encoder has a bit of state in it, 
-        // and needs some 0s to forget or you get a little click next time
-        // you play something
-        if (_flushEncoder) {
-            flushEncoder();
-        }
-
         // if we have an avatar audio stream then send it out to our audio-mixer
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
         bool silentFrame = true;
@@ -528,7 +522,7 @@ void Agent::processAgentAvatarAudio() {
             }
         }
 
-        auto audioPacket = NLPacket::create(silentFrame
+        auto audioPacket = NLPacket::create(silentFrame && !_flushEncoder
                 ? PacketType::SilentAudioFrame
                 : PacketType::MicrophoneAudioNoEcho);
 
@@ -564,13 +558,17 @@ void Agent::processAgentAvatarAudio() {
             glm::quat headOrientation = scriptedAvatar->getHeadOrientation();
             audioPacket->writePrimitive(headOrientation);
 
-            QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
             QByteArray encodedBuffer;
-            // encode it
+            if (_flushEncoder) {
-            if(_encoder) {
+                encodeFrameOfZeros(encodedBuffer);
-                _encoder->encode(decodedBuffer, encodedBuffer);
             } else {
-                encodedBuffer = decodedBuffer;
+                QByteArray decodedBuffer(reinterpret_cast<const char*>(nextSoundOutput), numAvailableSamples*sizeof(int16_t));
+                if (_encoder) {
+                    // encode it
+                    _encoder->encode(decodedBuffer, encodedBuffer);
+                } else {
+                    encodedBuffer = decodedBuffer;
+                }
             }
             audioPacket->write(encodedBuffer.constData(), encodedBuffer.size());
         }

assignment-client/src/Agent.h

@@ -81,7 +81,7 @@ signals:
 private:
     void negotiateAudioFormat();
     void selectAudioFormat(const QString& selectedCodecName);
-    void flushEncoder();
+    void encodeFrameOfZeros(QByteArray& encodedZeros);
 
     std::unique_ptr<ScriptEngine> _scriptEngine;
     EntityEditPacketSender _entityEditSender;

assignment-client/src/audio/AudioMixer.cpp

@@ -810,7 +810,8 @@ void AudioMixer::broadcastMixes() {
 
                     std::unique_ptr<NLPacket> mixPacket;
 
-                    if (mixHasAudio) {
+                    if (mixHasAudio || nodeData->shouldFlushEncoder()) {
+                        
                         int mixPacketBytes = sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE 
                                                              + AudioConstants::NETWORK_FRAME_BYTES_STEREO;
                         mixPacket = NLPacket::create(PacketType::MixedAudio, mixPacketBytes);
@@ -823,12 +824,17 @@ void AudioMixer::broadcastMixes() {
                         QString codecInPacket = nodeData->getCodecName();
                         mixPacket->writeString(codecInPacket);
 
-                        QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
                         QByteArray encodedBuffer;
-                        nodeData->encode(decodedBuffer, encodedBuffer);
+                        if (mixHasAudio) {
-
+                            QByteArray decodedBuffer(reinterpret_cast<char*>(_clampedSamples), AudioConstants::NETWORK_FRAME_BYTES_STEREO);
+                            nodeData->encode(decodedBuffer, encodedBuffer);
+                        } else {
+                            // time to flush, which resets the shouldFlush until next time we encode something
+                            nodeData->encodeFrameOfZeros(encodedBuffer);
+                        }
                         // pack mixed audio samples
                         mixPacket->write(encodedBuffer.constData(), encodedBuffer.size());
+                    
                     } else {
                         int silentPacketBytes = sizeof(quint16) + sizeof(quint16) + AudioConstants::MAX_CODEC_NAME_LENGTH_ON_WIRE;
                         mixPacket = NLPacket::create(PacketType::SilentAudioFrame, silentPacketBytes);

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -371,6 +371,17 @@ void AudioMixerClientData::sendSelectAudioFormat(SharedNodePointer node, const Q
     nodeList->sendPacket(std::move(replyPacket), *node);
 }
 
+void AudioMixerClientData::encodeFrameOfZeros(QByteArray& encodedZeros) {
+    static QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_STEREO, 0);
+    if (_shouldFlushEncoder) {
+        if (_encoder) {
+            _encoder->encode(zeros, encodedZeros);
+        } else {
+            encodedZeros = zeros;
+        }
+    }
+    _shouldFlushEncoder = false;
+}
 
 void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& codecName) {
     cleanupCodec(); // cleanup any previously allocated coders first

assignment-client/src/audio/AudioMixerClientData.h

@@ -77,7 +77,11 @@ public:
         } else {
             encodedBuffer = decodedBuffer;
         }
+        // once you have encoded, you need to flush eventually.
+        _shouldFlushEncoder = true;
     }
+    void encodeFrameOfZeros(QByteArray& encodedZeros);
+    bool shouldFlushEncoder() { return _shouldFlushEncoder; }
 
     QString getCodecName() { return _selectedCodecName; }
 
@@ -106,6 +110,8 @@ private:
     QString _selectedCodecName;
     Encoder* _encoder{ nullptr }; // for outbound mixed stream
     Decoder* _decoder{ nullptr }; // for mic stream
+
+    bool _shouldFlushEncoder { false };
 };
 
 #endif // hifi_AudioMixerClientData_h