// // AudioInjector.cpp // libraries/audio/src // // Created by Stephen Birarda on 1/2/2014. // Copyright 2014 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // #include "AudioInjector.h" #include #include #include #include #include #include #include "AbstractAudioInterface.h" #include "AudioInjectorManager.h" #include "AudioRingBuffer.h" #include "AudioLogging.h" #include "SoundCache.h" #include "AudioHelpers.h" int metaType = qRegisterMetaType("AudioInjectorPointer"); AbstractAudioInterface* AudioInjector::_localAudioInterface{ nullptr }; AudioInjectorState operator& (AudioInjectorState lhs, AudioInjectorState rhs) { return static_cast(static_cast(lhs) & static_cast(rhs)); }; AudioInjectorState& operator|= (AudioInjectorState& lhs, AudioInjectorState rhs) { lhs = static_cast(static_cast(lhs) | static_cast(rhs)); return lhs; }; AudioInjector::AudioInjector(SharedSoundPointer sound, const AudioInjectorOptions& injectorOptions) : _sound(sound), _audioData(sound->getAudioData()), _options(injectorOptions) { } AudioInjector::AudioInjector(AudioDataPointer audioData, const AudioInjectorOptions& injectorOptions) : _audioData(audioData), _options(injectorOptions) { } AudioInjector::~AudioInjector() {} bool AudioInjector::stateHas(AudioInjectorState state) const { return resultWithReadLock([&] { return (_state & state) == state; }); } void AudioInjector::setOptions(const AudioInjectorOptions& options) { // since options.stereo is computed from the audio stream, // we need to copy it from existing options just in case. withWriteLock([&] { bool currentlyStereo = _options.stereo; bool currentlyAmbisonic = _options.ambisonic; _options = options; _options.stereo = currentlyStereo; _options.ambisonic = currentlyAmbisonic; }); } void AudioInjector::finishNetworkInjection() { withWriteLock([&] { _state |= AudioInjectorState::NetworkInjectionFinished; }); // if we are already finished with local // injection, then we are finished if(stateHas(AudioInjectorState::LocalInjectionFinished)) { finish(); } } void AudioInjector::finishLocalInjection() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "finishLocalInjection"); return; } bool localOnly = false; withWriteLock([&] { _state |= AudioInjectorState::LocalInjectionFinished; localOnly = _options.localOnly; }); if(localOnly || stateHas(AudioInjectorState::NetworkInjectionFinished)) { finish(); } } void AudioInjector::finish() { withWriteLock([&] { _state |= AudioInjectorState::LocalInjectionFinished; _state |= AudioInjectorState::NetworkInjectionFinished; _state |= AudioInjectorState::Finished; }); emit finished(); _localBuffer = nullptr; } void AudioInjector::restart() { // reset the current send offset to zero _currentSendOffset = 0; // reset state to start sending from beginning again _nextFrame = 0; if (_frameTimer) { _frameTimer->invalidate(); } _hasSentFirstFrame = false; // check our state to decide if we need extra handling for the restart request if (stateHas(AudioInjectorState::Finished)) { if (!inject(&AudioInjectorManager::threadInjector)) { qWarning() << "AudioInjector::restart failed to thread injector"; } } } bool AudioInjector::inject(bool(AudioInjectorManager::*injection)(const AudioInjectorPointer&)) { AudioInjectorOptions options; uint32_t numBytes; withWriteLock([&] { _state = AudioInjectorState::NotFinished; options = _options; numBytes = _audioData->getNumBytes(); }); int byteOffset = 0; if (options.secondOffset > 0.0f) { int numChannels = options.ambisonic ? 4 : (options.stereo ? 2 : 1); byteOffset = (int)(AudioConstants::SAMPLE_RATE * options.secondOffset * numChannels); byteOffset *= AudioConstants::SAMPLE_SIZE; byteOffset = byteOffset % numBytes; } _currentSendOffset = byteOffset; if (!injectLocally()) { finishLocalInjection(); } bool success = true; if (!options.localOnly) { auto injectorManager = DependencyManager::get(); if (!(*injectorManager.*injection)(sharedFromThis())) { success = false; finishNetworkInjection(); } } return success; } bool AudioInjector::injectLocally() { bool success = false; if (_localAudioInterface) { if (_audioData->getNumBytes() > 0) { _localBuffer = QSharedPointer(new AudioInjectorLocalBuffer(_audioData), &AudioInjectorLocalBuffer::deleteLater); _localBuffer->moveToThread(thread()); _localBuffer->open(QIODevice::ReadOnly); _localBuffer->setShouldLoop(_options.loop); // give our current send position to the local buffer _localBuffer->setCurrentOffset(_currentSendOffset); // call this function on the AudioClient's thread // this will move the local buffer's thread to the LocalInjectorThread success = _localAudioInterface->outputLocalInjector(sharedFromThis()); if (!success) { qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface"; } } else { qCDebug(audio) << "AudioInjector::injectLocally called without any data in Sound QByteArray"; } } return success; } const uchar MAX_INJECTOR_VOLUME = packFloatGainToByte(1.0f); static const int64_t NEXT_FRAME_DELTA_ERROR_OR_FINISHED = -1; static const int64_t NEXT_FRAME_DELTA_IMMEDIATELY = 0; qint64 writeStringToStream(const QString& string, QDataStream& stream) { QByteArray data = string.toUtf8(); uint32_t length = data.length(); if (length == 0) { stream << static_cast(length); } else { // http://doc.qt.io/qt-5/datastreamformat.html // QDataStream << QByteArray - // If the byte array is null : 0xFFFFFFFF (quint32) // Otherwise : the array size(quint32) followed by the array bytes, i.e.size bytes stream << data; } return length + sizeof(uint32_t); } int64_t AudioInjector::injectNextFrame() { if (stateHas(AudioInjectorState::NetworkInjectionFinished)) { return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } // if we haven't setup the packet to send then do so now static int loopbackOptionOffset = -1; static int positionOptionOffset = -1; static int volumeOptionOffset = -1; static int audioDataOffset = -1; AudioInjectorOptions options = resultWithReadLock([&] { return _options; }); if (!_currentPacket) { if (_currentSendOffset < 0 || _currentSendOffset >= (int)_audioData->getNumBytes()) { _currentSendOffset = 0; } // make sure we actually have samples downloaded to inject if (_audioData && _audioData->getNumSamples() > 0) { _outgoingSequenceNumber = 0; _nextFrame = 0; if (!_frameTimer) { _frameTimer = std::unique_ptr(new QElapsedTimer); } _frameTimer->restart(); _currentPacket = NLPacket::create(PacketType::InjectAudio); // setup the packet for injected audio QDataStream audioPacketStream(_currentPacket.get()); // pack some placeholder sequence number for now audioPacketStream << (quint16) 0; // current injectors don't use codecs, so pack in the unknown codec name QString noCodecForInjectors(""); writeStringToStream(noCodecForInjectors, audioPacketStream); // pack stream identifier (a generated UUID) audioPacketStream << _streamID; // pack the stereo/mono type of the stream audioPacketStream << options.stereo; // pack the flag for loopback, if requested loopbackOptionOffset = _currentPacket->pos(); uchar loopbackFlag = (_localAudioInterface && _localAudioInterface->shouldLoopbackInjectors()); audioPacketStream << loopbackFlag; // pack the position for injected audio positionOptionOffset = _currentPacket->pos(); audioPacketStream.writeRawData(reinterpret_cast(&options.position), sizeof(options.position)); // pack our orientation for injected audio audioPacketStream.writeRawData(reinterpret_cast(&options.orientation), sizeof(options.orientation)); audioPacketStream.writeRawData(reinterpret_cast(&options.position), sizeof(options.position)); glm::vec3 boxCorner = glm::vec3(0); audioPacketStream.writeRawData(reinterpret_cast(&boxCorner), sizeof(glm::vec3)); // pack zero for radius float radius = 0; audioPacketStream << radius; // pack 255 for attenuation byte volumeOptionOffset = _currentPacket->pos(); quint8 volume = MAX_INJECTOR_VOLUME; audioPacketStream << volume; audioPacketStream << options.ignorePenumbra; audioDataOffset = _currentPacket->pos(); } else { // no samples to inject, return immediately qCDebug(audio) << "AudioInjector::injectNextFrame() called with no samples to inject. Returning."; return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } } if (!_frameTimer->isValid()) { // in the case where we have been restarted, the frame timer will be invalid and we need to start it back over here _frameTimer->restart(); } assert(loopbackOptionOffset != -1); assert(positionOptionOffset != -1); assert(volumeOptionOffset != -1); assert(audioDataOffset != -1); _currentPacket->seek(0); // pack the sequence number _currentPacket->writePrimitive(_outgoingSequenceNumber); _currentPacket->seek(loopbackOptionOffset); _currentPacket->writePrimitive((uchar)(_localAudioInterface && _localAudioInterface->shouldLoopbackInjectors())); _currentPacket->seek(positionOptionOffset); _currentPacket->writePrimitive(options.position); _currentPacket->writePrimitive(options.orientation); quint8 volume = packFloatGainToByte(options.volume); _currentPacket->seek(volumeOptionOffset); _currentPacket->writePrimitive(volume); _currentPacket->seek(audioDataOffset); // This code is copying bytes from the _sound directly into the packet, handling looping appropriately. // Might be a reasonable place to do the encode step here. QByteArray decodedAudio; int totalBytesLeftToCopy = (options.stereo ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; if (!options.loop) { // If we aren't looping, let's make sure we don't read past the end int bytesLeftToRead = _audioData->getNumBytes() - _currentSendOffset; totalBytesLeftToCopy = std::min(totalBytesLeftToCopy, bytesLeftToRead); } auto samples = _audioData->data(); auto currentSample = _currentSendOffset / AudioConstants::SAMPLE_SIZE; auto samplesLeftToCopy = totalBytesLeftToCopy / AudioConstants::SAMPLE_SIZE; using AudioConstants::AudioSample; decodedAudio.resize(totalBytesLeftToCopy); auto samplesOut = reinterpret_cast(decodedAudio.data()); // Copy and Measure the loudness of this frame withWriteLock([&] { _loudness = 0.0f; for (int i = 0; i < samplesLeftToCopy; ++i) { auto index = (currentSample + i) % _audioData->getNumSamples(); auto sample = samples[index]; samplesOut[i] = sample; _loudness += abs(sample) / (AudioConstants::MAX_SAMPLE_VALUE / 2.0f); } _loudness /= (float)samplesLeftToCopy; }); _currentSendOffset = (_currentSendOffset + totalBytesLeftToCopy) % _audioData->getNumBytes(); // FIXME -- good place to call codec encode here. We need to figure out how to tell the AudioInjector which // codec to use... possible through AbstractAudioInterface. QByteArray encodedAudio = decodedAudio; _currentPacket->write(encodedAudio.data(), encodedAudio.size()); // set the correct size used for this packet _currentPacket->setPayloadSize(_currentPacket->pos()); // grab our audio mixer from the NodeList, if it exists auto nodeList = DependencyManager::get(); SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer) { // send off this audio packet nodeList->sendUnreliablePacket(*_currentPacket, *audioMixer); _outgoingSequenceNumber++; } if (_currentSendOffset == 0 && !options.loop) { finishNetworkInjection(); return NEXT_FRAME_DELTA_ERROR_OR_FINISHED; } if (!_hasSentFirstFrame) { _hasSentFirstFrame = true; // ask AudioInjectorManager to call us right away again to // immediately send the first two frames so the mixer can start using the audio right away return NEXT_FRAME_DELTA_IMMEDIATELY; } const int MAX_ALLOWED_FRAMES_TO_FALL_BEHIND = 7; int64_t currentTime = _frameTimer->nsecsElapsed() / 1000; auto currentFrameBasedOnElapsedTime = currentTime / AudioConstants::NETWORK_FRAME_USECS; if (currentFrameBasedOnElapsedTime - _nextFrame > MAX_ALLOWED_FRAMES_TO_FALL_BEHIND) { // If we are falling behind by more frames than our threshold, let's skip the frames ahead qCDebug(audio) << this << "injectNextFrame() skipping ahead, fell behind by " << (currentFrameBasedOnElapsedTime - _nextFrame) << " frames"; _nextFrame = currentFrameBasedOnElapsedTime; _currentSendOffset = _nextFrame * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL * (options.stereo ? 2 : 1) % _audioData->getNumBytes(); } int64_t playNextFrameAt = ++_nextFrame * AudioConstants::NETWORK_FRAME_USECS; return std::max(INT64_C(0), playNextFrameAt - currentTime); } void AudioInjector::sendStopInjectorPacket() { auto nodeList = DependencyManager::get(); if (auto audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer)) { // Build packet auto stopInjectorPacket = NLPacket::create(PacketType::StopInjector); stopInjectorPacket->write(_streamID.toRfc4122()); // Send packet nodeList->sendUnreliablePacket(*stopInjectorPacket, *audioMixer); } }