From 900f425f359fe59300d109bb1b96523720ba61c3 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Tue, 17 Nov 2015 11:58:46 -0800 Subject: [PATCH] Recording fixes --- .../scripting/RecordingScriptingInterface.cpp | 15 +- libraries/audio-client/src/AudioClient.cpp | 176 +++++++++--------- libraries/audio-client/src/AudioClient.h | 1 + libraries/avatars/src/AvatarData.cpp | 11 +- 4 files changed, 94 insertions(+), 109 deletions(-) diff --git a/interface/src/scripting/RecordingScriptingInterface.cpp b/interface/src/scripting/RecordingScriptingInterface.cpp index 32bd6fde97..4ca3881e8d 100644 --- a/interface/src/scripting/RecordingScriptingInterface.cpp +++ b/interface/src/scripting/RecordingScriptingInterface.cpp @@ -162,26 +162,15 @@ void RecordingScriptingInterface::startRecording() { } _recordingEpoch = Frame::epochForFrameTime(0); - - auto myAvatar = DependencyManager::get()->getMyAvatar(); - myAvatar->setRecordingBasis(); + DependencyManager::get()->getMyAvatar()->setRecordingBasis(); _recorder->start(); } void RecordingScriptingInterface::stopRecording() { _recorder->stop(); - _lastClip = _recorder->getClip(); - // post-process the audio into discreet chunks based on times of received samples _lastClip->seek(0); - Frame::ConstPointer frame; - while (frame = _lastClip->nextFrame()) { - qDebug() << "Frame time " << frame->timeOffset << " size " << frame->data.size(); - } - _lastClip->seek(0); - - auto myAvatar = DependencyManager::get()->getMyAvatar(); - myAvatar->clearRecordingBasis(); + DependencyManager::get()->getMyAvatar()->clearRecordingBasis(); } void RecordingScriptingInterface::saveRecording(const QString& filename) { diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 50bfd995f2..d4e571ade5 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -743,19 +743,9 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { } void AudioClient::handleAudioInput() { - if (!_audioPacket) { - // we don't have an audioPacket yet - set that up now - _audioPacket = NLPacket::create(PacketType::MicrophoneAudioNoEcho); - } - const float inputToNetworkInputRatio = calculateDeviceToNetworkInputRatio(); - const int inputSamplesRequired = (int)((float)AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio); const auto inputAudioSamples = std::unique_ptr(new int16_t[inputSamplesRequired]); - - static const int leadingBytes = sizeof(quint16) + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); - int16_t* const networkAudioSamples = (int16_t*)(_audioPacket->getPayload() + leadingBytes); - QByteArray inputByteArray = _inputDevice->readAll(); // Add audio source injection if enabled @@ -784,30 +774,30 @@ void AudioClient::handleAudioInput() { float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); _stats.updateInputMsecsRead(audioInputMsecsRead); - while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { + const int numNetworkBytes = _isStereoInput + ? AudioConstants::NETWORK_FRAME_BYTES_STEREO + : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkSamples = _isStereoInput + ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO + : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - const int numNetworkBytes = _isStereoInput - ? AudioConstants::NETWORK_FRAME_BYTES_STEREO - : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; - const int numNetworkSamples = _isStereoInput - ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO - : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + static int16_t networkAudioSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + + while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { if (!_muted) { - // zero out the monoAudioSamples array and the locally injected audio - memset(networkAudioSamples, 0, numNetworkBytes); // Increment the time since the last clip if (_timeSinceLastClip >= 0.0f) { - _timeSinceLastClip += (float) numNetworkSamples / (float) AudioConstants::SAMPLE_RATE; + _timeSinceLastClip += (float)numNetworkSamples / (float)AudioConstants::SAMPLE_RATE; } _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); possibleResampling(_inputToNetworkResampler, - inputAudioSamples.get(), networkAudioSamples, - inputSamplesRequired, numNetworkSamples, - _inputFormat, _desiredInputFormat); + inputAudioSamples.get(), networkAudioSamples, + inputSamplesRequired, numNetworkSamples, + _inputFormat, _desiredInputFormat); // Remove DC offset if (!_isStereoInput && !_audioSourceInjectEnabled) { @@ -829,7 +819,7 @@ void AudioClient::handleAudioInput() { for (int i = 0; i < numNetworkSamples; i++) { int thisSample = std::abs(networkAudioSamples[i]); - loudness += (float) thisSample; + loudness += (float)thisSample; if (thisSample > (AudioConstants::MAX_SAMPLE_VALUE * AudioNoiseGate::CLIPPING_THRESHOLD)) { _timeSinceLastClip = 0.0f; @@ -839,7 +829,7 @@ void AudioClient::handleAudioInput() { _lastInputLoudness = fabs(loudness / numNetworkSamples); } - emit inputReceived({reinterpret_cast(networkAudioSamples), numNetworkBytes}); + emit inputReceived({ reinterpret_cast(networkAudioSamples), numNetworkBytes }); } else { // our input loudness is 0, since we're muted @@ -849,14 +839,38 @@ void AudioClient::handleAudioInput() { _inputRingBuffer.shiftReadPosition(inputSamplesRequired); } - auto nodeList = DependencyManager::get(); - SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + emitAudioPacket(networkAudioSamples); + } +} - if (audioMixer && audioMixer->getActiveSocket()) { - glm::vec3 headPosition = _positionGetter(); - glm::quat headOrientation = _orientationGetter(); - quint8 isStereo = _isStereoInput ? 1 : 0; +void AudioClient::emitAudioPacket(const int16_t* audioData, PacketType packetType) { + static std::mutex _mutex; + using Locker = std::unique_lock; + // FIXME recorded audio isn't guaranteed to have the same stereo state + // as the current system + const int numNetworkBytes = _isStereoInput + ? AudioConstants::NETWORK_FRAME_BYTES_STEREO + : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkSamples = _isStereoInput + ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO + : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + + auto nodeList = DependencyManager::get(); + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer && audioMixer->getActiveSocket()) { + Locker lock(_mutex); + if (!_audioPacket) { + // we don't have an audioPacket yet - set that up now + _audioPacket = NLPacket::create(PacketType::MicrophoneAudioWithEcho); + } + + glm::vec3 headPosition = _positionGetter(); + glm::quat headOrientation = _orientationGetter(); + quint8 isStereo = _isStereoInput ? 1 : 0; + + if (packetType == PacketType::Unknown) { if (_lastInputLoudness == 0) { _audioPacket->setType(PacketType::SilentAudioFrame); } else { @@ -866,70 +880,52 @@ void AudioClient::handleAudioInput() { _audioPacket->setType(PacketType::MicrophoneAudioNoEcho); } } - - // reset the audio packet so we can start writing - _audioPacket->reset(); - - // write sequence number - _audioPacket->writePrimitive(_outgoingAvatarAudioSequenceNumber); - - if (_audioPacket->getType() == PacketType::SilentAudioFrame) { - // pack num silent samples - quint16 numSilentSamples = numNetworkSamples; - _audioPacket->writePrimitive(numSilentSamples); - } else { - // set the mono/stereo byte - _audioPacket->writePrimitive(isStereo); - } - - // pack the three float positions - _audioPacket->writePrimitive(headPosition); - - // pack the orientation - _audioPacket->writePrimitive(headOrientation); - - if (_audioPacket->getType() != PacketType::SilentAudioFrame) { - // audio samples have already been packed (written to networkAudioSamples) - _audioPacket->setPayloadSize(_audioPacket->getPayloadSize() + numNetworkBytes); - } - - _stats.sentPacket(); - - nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::SendAudioPacket); - - nodeList->sendUnreliablePacket(*_audioPacket, *audioMixer); - - _outgoingAvatarAudioSequenceNumber++; + } else { + _audioPacket->setType(packetType); } + + // reset the audio packet so we can start writing + _audioPacket->reset(); + + // write sequence number + _audioPacket->writePrimitive(_outgoingAvatarAudioSequenceNumber); + + if (_audioPacket->getType() == PacketType::SilentAudioFrame) { + // pack num silent samples + quint16 numSilentSamples = numNetworkSamples; + _audioPacket->writePrimitive(numSilentSamples); + } else { + // set the mono/stereo byte + _audioPacket->writePrimitive(isStereo); + } + + // pack the three float positions + _audioPacket->writePrimitive(headPosition); + + // pack the orientation + _audioPacket->writePrimitive(headOrientation); + + if (_audioPacket->getType() != PacketType::SilentAudioFrame) { + // audio samples have already been packed (written to networkAudioSamples) + _audioPacket->setPayloadSize(_audioPacket->getPayloadSize() + numNetworkBytes); + } + + static const int leadingBytes = sizeof(quint16) + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); + int16_t* const networkAudioSamples = (int16_t*)(_audioPacket->getPayload() + leadingBytes); + memcpy(networkAudioSamples, audioData, numNetworkBytes); + + _stats.sentPacket(); + + nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::SendAudioPacket); + + nodeList->sendUnreliablePacket(*_audioPacket, *audioMixer); + + _outgoingAvatarAudioSequenceNumber++; } } void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { - if (!_audioPacket) { - // we don't have an audioPacket yet - set that up now - _audioPacket = NLPacket::create(PacketType::MicrophoneAudioWithEcho); - } - - // FIXME either discard stereo in the recording or record a stereo flag - - auto nodeList = DependencyManager::get(); - SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); - if (audioMixer && audioMixer->getActiveSocket()) { - glm::vec3 headPosition = _positionGetter(); - glm::quat headOrientation = _orientationGetter(); - quint8 isStereo = _isStereoInput ? 1 : 0; - _audioPacket->reset(); - _audioPacket->setType(PacketType::MicrophoneAudioWithEcho); - _audioPacket->writePrimitive(_outgoingAvatarAudioSequenceNumber); - _audioPacket->writePrimitive(isStereo); - _audioPacket->writePrimitive(headPosition); - _audioPacket->writePrimitive(headOrientation); - _audioPacket->write(audio); - _stats.sentPacket(); - nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::SendAudioPacket); - nodeList->sendUnreliablePacket(*_audioPacket, *audioMixer); - _outgoingAvatarAudioSequenceNumber++; - } + emitAudioPacket((int16_t*)audio.data(), PacketType::MicrophoneAudioWithEcho); } void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 7d2b5a783f..9d46ad9d26 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -212,6 +212,7 @@ protected: } private: + void emitAudioPacket(const int16_t* audioData, PacketType packetType = PacketType::Unknown); void outputFormatChanged(); QByteArray firstInputFrame; diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 017ef7578a..fdfc6c1893 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1443,14 +1443,10 @@ QByteArray AvatarData::toFrame(const AvatarData& avatar) { auto recordingBasis = avatar.getRecordingBasis(); if (recordingBasis) { + root[JSON_AVATAR_BASIS] = Transform::toJson(*recordingBasis); // Find the relative transform auto relativeTransform = recordingBasis->relativeTransform(avatar.getTransform()); - - // if the resulting relative basis is identity, we shouldn't record anything - if (!relativeTransform.isIdentity()) { - root[JSON_AVATAR_RELATIVE] = Transform::toJson(relativeTransform); - root[JSON_AVATAR_BASIS] = Transform::toJson(*recordingBasis); - } + root[JSON_AVATAR_RELATIVE] = Transform::toJson(relativeTransform); } else { root[JSON_AVATAR_RELATIVE] = Transform::toJson(avatar.getTransform()); } @@ -1484,6 +1480,9 @@ QByteArray AvatarData::toFrame(const AvatarData& avatar) { void AvatarData::fromFrame(const QByteArray& frameData, AvatarData& result) { QJsonDocument doc = QJsonDocument::fromBinaryData(frameData); +#ifdef WANT_JSON_DEBUG + qDebug() << doc.toJson(QJsonDocument::JsonFormat::Indented); +#endif QJsonObject root = doc.object(); if (root.contains(JSON_AVATAR_HEAD_MODEL)) {