Updating audio record/playback mechanism to more closely match actual audio input

interface/src/scripting/RecordingScriptingInterface.cpp

@@ -156,7 +156,6 @@ void RecordingScriptingInterface::startRecording() {
         return;
     }
 
-
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "startRecording", Qt::BlockingQueuedConnection);
         return;
@@ -164,78 +163,16 @@ void RecordingScriptingInterface::startRecording() {
 
     _recordingEpoch = Frame::epochForFrameTime(0);
 
-    _audioRecordingBuffer.clear();
     auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
     myAvatar->setRecordingBasis();
     _recorder->start();
 }
 
-float calculateAudioTime(const QByteArray& audio) {
-    static const float AUDIO_BYTES_PER_SECOND = AudioConstants::SAMPLE_RATE * sizeof(AudioConstants::AudioSample);
-    return (float)audio.size() / AUDIO_BYTES_PER_SECOND;
-}
-
-void injectAudioFrame(Clip::Pointer& clip, Frame::Time time, const QByteArray& audio) {
-    static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME);
-    clip->addFrame(std::make_shared<Frame>(AUDIO_FRAME_TYPE, time, audio));
-}
-
-// Detect too much audio in a single frame, or too much deviation between 
-// the expected audio length and the computed audio length
-bool shouldStartNewAudioFrame(const QByteArray& currentAudioFrame, float expectedAudioLength) {
-    if (currentAudioFrame.isEmpty()) {
-        return true;
-    }
-
-    // 100 milliseconds
-    float actualAudioLength = calculateAudioTime(currentAudioFrame);
-    static const float MAX_AUDIO_PACKET_DURATION = 1.0f;
-    if (actualAudioLength >= MAX_AUDIO_PACKET_DURATION) {
-        return true;
-    }
-
-
-    float deviation = std::abs(actualAudioLength - expectedAudioLength);
-
-    qDebug() << "Checking buffer deviation current length ";
-    qDebug() << "Actual:    " << actualAudioLength;
-    qDebug() << "Expected:  " << expectedAudioLength;
-    qDebug() << "Deviation: " << deviation;
-
-    static const float MAX_AUDIO_DEVIATION = 0.1f;
-    if (deviation >= MAX_AUDIO_PACKET_DURATION) {
-        return true;
-    }
-
-    return false;
-}
-
-
-void injectAudioFrames(Clip::Pointer& clip, const QList<QPair<recording::Frame::Time, QByteArray>>& audioBuffer) {
-    Frame::Time lastAudioStartTime = 0;
-    QByteArray audioFrameBuffer;
-    for (const auto& audioPacket : audioBuffer) {
-        float expectedAudioLength = Frame::frameTimeToSeconds(audioPacket.first - lastAudioStartTime);
-        if (shouldStartNewAudioFrame(audioFrameBuffer, expectedAudioLength)) {
-            // Time to start a new frame, inject the old one if it exists
-            if (audioFrameBuffer.size()) {
-                injectAudioFrame(clip, lastAudioStartTime, audioFrameBuffer);
-                audioFrameBuffer.clear();
-            }
-            lastAudioStartTime = audioPacket.first;
-        }
-        audioFrameBuffer.append(audioPacket.second);
-    }
-}
-
-
 void RecordingScriptingInterface::stopRecording() {
     _recorder->stop();
 
     _lastClip = _recorder->getClip();
     // post-process the audio into discreet chunks based on times of received samples
-    injectAudioFrames(_lastClip, _audioRecordingBuffer);
-    _audioRecordingBuffer.clear();
     _lastClip->seek(0);
     Frame::ConstPointer frame;
     while (frame = _lastClip->nextFrame()) {
@@ -310,19 +247,12 @@ void RecordingScriptingInterface::processAvatarFrame(const Frame::ConstPointer&
 
 void RecordingScriptingInterface::processAudioInput(const QByteArray& audio) {
     if (_recorder->isRecording()) {
-        auto audioFrameTime = Frame::frameTimeFromEpoch(_recordingEpoch);
+        static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME);
-        _audioRecordingBuffer.push_back({ audioFrameTime, audio });
+        _recorder->recordFrame(AUDIO_FRAME_TYPE, audio);
-        qDebug() << "Got sound packet of size " << audio.size() << " At time " << audioFrameTime;
     }
 }
 
 void RecordingScriptingInterface::processAudioFrame(const recording::FrameConstPointer& frame) {
-    AudioInjectorOptions options;
+    auto audioClient = DependencyManager::get<AudioClient>();
-    auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
+    audioClient->handleRecordedAudioInput(frame->data);
-    options.position = myAvatar->getPosition();
-    options.orientation = myAvatar->getOrientation();
-    // FIXME store the audio format (sample rate, bits, stereo) in the frame
-    options.stereo = false;
-    // FIXME move audio injector to a thread pool model?
-    AudioInjector::playSoundAndDelete(frame->data, options, nullptr);
 }

interface/src/scripting/RecordingScriptingInterface.h

@@ -65,7 +65,6 @@ private:
     void processAudioInput(const QByteArray& audioData);
     QSharedPointer<recording::Deck> _player;
     QSharedPointer<recording::Recorder> _recorder;
-    QList<QPair<recording::Frame::Time, QByteArray>> _audioRecordingBuffer;
     quint64 _recordingEpoch { 0 };
     
     Flag _playFromCurrentLocation { true };

libraries/audio-client/src/AudioClient.cpp

@@ -904,6 +904,39 @@ void AudioClient::handleAudioInput() {
     }
 }
 
+void AudioClient::handleRecordedAudioInput(const QByteArray& audio) {
+    if (!_audioPacket) {
+        // we don't have an audioPacket yet - set that up now
+        _audioPacket = NLPacket::create(PacketType::MicrophoneAudioWithEcho);
+    }
+    // FIXME either discard stereo in the recording or record a stereo flag
+    const int numNetworkBytes = _isStereoInput
+        ? AudioConstants::NETWORK_FRAME_BYTES_STEREO
+        : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL;
+    const int numNetworkSamples = _isStereoInput
+        ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO
+        : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+    if (audioMixer && audioMixer->getActiveSocket()) {
+        glm::vec3 headPosition = _positionGetter();
+        glm::quat headOrientation = _orientationGetter();
+        quint8 isStereo = _isStereoInput ? 1 : 0;
+        _audioPacket->reset();
+        _audioPacket->setType(PacketType::MicrophoneAudioWithEcho);
+        _audioPacket->writePrimitive(_outgoingAvatarAudioSequenceNumber);
+        _audioPacket->writePrimitive(isStereo);
+        _audioPacket->writePrimitive(headPosition);
+        _audioPacket->writePrimitive(headOrientation);
+        _audioPacket->write(audio);
+        _stats.sentPacket();
+        nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::SendAudioPacket);
+        nodeList->sendUnreliablePacket(*_audioPacket, *audioMixer);
+        _outgoingAvatarAudioSequenceNumber++;
+    }
+}
+
 void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) {
     const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t);
     const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount())

libraries/audio-client/src/AudioClient.h

@@ -147,6 +147,7 @@ public slots:
 
     void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); }
     void handleAudioInput();
+    void handleRecordedAudioInput(const QByteArray& audio);
     void reset();
     void audioMixerKilled();
     void toggleMute();