From 86e01bbae6f4e6179510d5d4d09186b1049ae98f Mon Sep 17 00:00:00 2001 From: David Kelly Date: Mon, 24 Oct 2016 15:18:29 -0700 Subject: [PATCH 1/3] First cut Started with the horrible screaming when avatar goes away, seems due to not putting a codec string in the packets. This was an existing issue, not due to recent changes. Also, some weird indenting was fixed, etc... Still hear an artifact when audio starts, but only when a codec was negotiated. Hoping to fix that too. --- assignment-client/src/Agent.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index eebd2a81e7..5c97385ea4 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -529,14 +529,16 @@ void Agent::processAgentAvatarAudio() { return; } + // write the codec + audioPacket->writeString(_selectedCodecName); + // write the number of silent samples so the audio-mixer can uphold timing - audioPacket->writePrimitive(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + audioPacket->writePrimitive(numAvailableSamples); // use the orientation and position of this avatar for the source of this audio audioPacket->writePrimitive(scriptedAvatar->getPosition()); glm::quat headOrientation = scriptedAvatar->getHeadOrientation(); audioPacket->writePrimitive(headOrientation); - } else if (nextSoundOutput) { // write the codec @@ -550,30 +552,28 @@ void Agent::processAgentAvatarAudio() { glm::quat headOrientation = scriptedAvatar->getHeadOrientation(); audioPacket->writePrimitive(headOrientation); + QByteArray decodedBuffer(reinterpret_cast(nextSoundOutput), numAvailableSamples*sizeof(int16_t)); + QByteArray encodedBuffer; // encode it if(_encoder) { - QByteArray decodedBuffer(reinterpret_cast(nextSoundOutput), numAvailableSamples*sizeof(int16_t)); - QByteArray encodedBuffer; _encoder->encode(decodedBuffer, encodedBuffer); - audioPacket->write(encodedBuffer.data(), encodedBuffer.size()); - } else { - audioPacket->write(reinterpret_cast(nextSoundOutput), numAvailableSamples*sizeof(int16_t)); + } else { + encodedBuffer = decodedBuffer; } - + audioPacket->write(encodedBuffer.constData(), encodedBuffer.size()); } - // write audio packet to AudioMixer nodes + // write audio packet to AudioMixer nodes auto nodeList = DependencyManager::get(); nodeList->eachNode([this, &nodeList, &audioPacket](const SharedNodePointer& node) { // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { - // pack sequence number - quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++; - audioPacket->seek(0); - audioPacket->writePrimitive(sequence); - - // send audio packet - nodeList->sendUnreliablePacket(*audioPacket, *node); + // pack sequence number + quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++; + audioPacket->seek(0); + audioPacket->writePrimitive(sequence); + // send audio packet + nodeList->sendUnreliablePacket(*audioPacket, *node); } }); } From 30d8f1967c3263de9b05faf21b0fe416851deecf Mon Sep 17 00:00:00 2001 From: David Kelly Date: Tue, 25 Oct 2016 12:15:48 -0700 Subject: [PATCH 2/3] Flushing encoder Seems they have some state. So if you play a sound, then silence that doesn't go through the encoder (SilentAudioFrame), then a sound again, you will hear a click. Now, I just pop a frame of 0s through the encoder at the end of each .wav it plays. Poof! --- assignment-client/src/Agent.cpp | 22 +++++++++++++++++----- assignment-client/src/Agent.h | 4 +++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 5c97385ea4..d283262302 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -349,7 +349,7 @@ void Agent::executeScript() { audioTransform.setRotation(scriptedAvatar->getOrientation()); AbstractAudioInterface::emitAudioPacket(audio.data(), audio.size(), audioSequenceNumber, audioTransform, PacketType::MicrophoneAudioNoEcho); }); - + auto avatarHashMap = DependencyManager::set(); _scriptEngine->registerGlobalObject("AvatarList", avatarHashMap.data()); @@ -361,10 +361,6 @@ void Agent::executeScript() { // register ourselves to the script engine _scriptEngine->registerGlobalObject("Agent", this); - // FIXME -we shouldn't be calling this directly, it's normally called by run(), not sure why - // viewers would need this called. - //_scriptEngine->init(); // must be done before we set up the viewers - _scriptEngine->registerGlobalObject("SoundCache", DependencyManager::get().data()); QScriptValue webSocketServerConstructorValue = _scriptEngine->newFunction(WebSocketServerClass::constructor); @@ -478,9 +474,24 @@ void Agent::processAgentAvatar() { nodeList->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); } } +void Agent::flushEncoder() { + _flushEncoder = false; + static QByteArray zeros(AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL, 0); + static QByteArray encodedZeros; + if (_encoder) { + _encoder->encode(zeros, encodedZeros); + } +} void Agent::processAgentAvatarAudio() { if (_isAvatar && (_isListeningToAudioStream || _avatarSound)) { + // after sound is done playing, encoder has a bit of state in it, + // and needs some 0s to forget or you get a little click next time + // you play something + if (_flushEncoder) { + flushEncoder(); + } + // if we have an avatar audio stream then send it out to our audio-mixer auto scriptedAvatar = DependencyManager::get(); bool silentFrame = true; @@ -513,6 +524,7 @@ void Agent::processAgentAvatarAudio() { // and our sent bytes back to zero _avatarSound.clear(); _numAvatarSoundSentBytes = 0; + _flushEncoder = true; } } diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 67dba10de2..b882ac3125 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -81,7 +81,8 @@ signals: private: void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); - + void flushEncoder(); + std::unique_ptr _scriptEngine; EntityEditPacketSender _entityEditSender; EntityTreeHeadlessViewer _entityViewer; @@ -107,6 +108,7 @@ private: QString _selectedCodecName; Encoder* _encoder { nullptr }; QThread _avatarAudioTimerThread; + bool _flushEncoder { false }; }; #endif // hifi_Agent_h From b09364c288ac2a53aebf0dd8cadbdc0045a9698b Mon Sep 17 00:00:00 2001 From: David Kelly Date: Tue, 25 Oct 2016 12:22:26 -0700 Subject: [PATCH 3/3] whitespace --- assignment-client/src/Agent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index d283262302..9e6ec1209c 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -349,7 +349,7 @@ void Agent::executeScript() { audioTransform.setRotation(scriptedAvatar->getOrientation()); AbstractAudioInterface::emitAudioPacket(audio.data(), audio.size(), audioSequenceNumber, audioTransform, PacketType::MicrophoneAudioNoEcho); }); - + auto avatarHashMap = DependencyManager::set(); _scriptEngine->registerGlobalObject("AvatarList", avatarHashMap.data());