diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 2fc905c6fd..d1c2c6597b 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -439,7 +439,7 @@ void Agent::executeScript() { encodedBuffer = audio; } - AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, + AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false, audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0), packetType, _selectedCodecName); }); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 49453c6fc6..2560f43337 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -275,17 +275,28 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { if (micStreamIt == _audioStreams.end()) { // we don't have a mic stream yet, so add it - // read the channel flag to see if our stream is stereo or not + // hop past the sequence number that leads the packet message.seek(sizeof(quint16)); - quint8 channelFlag; - message.readPrimitive(&channelFlag); + // pull the codec string from the packet + auto codecString = message.readString(); - bool isStereo = channelFlag == 1; + // determine if the stream is stereo or not + bool isStereo; + if (packetType == PacketType::SilentAudioFrame + || packetType == PacketType::ReplicatedSilentAudioFrame) { + quint16 numSilentSamples; + message.readPrimitive(&numSilentSamples); + isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + } else { + quint8 channelFlag; + message.readPrimitive(&channelFlag); + isStereo = channelFlag == 1; + } auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames()); - avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO); - qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName; + avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioMixerClientData::handleMismatchAudioFormat); @@ -324,7 +335,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { #if INJECTORS_SUPPORT_CODECS injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); - qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName; + qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; #endif auto emplaced = _audioStreams.emplace( @@ -567,7 +578,8 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c auto avatarAudioStream = getAvatarAudioStream(); if (avatarAudioStream) { - avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO); + avatarAudioStream->setupCodec(codec, codecName, avatarAudioStream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO); + qCDebug(audio) << "setting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << avatarAudioStream->isStereo(); } #if INJECTORS_SUPPORT_CODECS diff --git a/assignment-client/src/audio/AvatarAudioStream.cpp b/assignment-client/src/audio/AvatarAudioStream.cpp index 1e0c3ed9e6..42495b4dd0 100644 --- a/assignment-client/src/audio/AvatarAudioStream.cpp +++ b/assignment-client/src/audio/AvatarAudioStream.cpp @@ -11,6 +11,7 @@ #include +#include "AudioLogging.h" #include "AvatarAudioStream.h" AvatarAudioStream::AvatarAudioStream(bool isStereo, int numStaticJitterFrames) : @@ -41,6 +42,15 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray& _ringBuffer.resizeForFrameSize(isStereo ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + // restart the codec + if (_codec) { + if (_decoder) { + _codec->releaseDecoder(_decoder); + } + _decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, isStereo ? AudioConstants::STEREO : AudioConstants::MONO); + } + qCDebug(audio) << "resetting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo; + _isStereo = isStereo; } diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index af86499101..579910d9f7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -782,7 +782,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { _selectedCodecName = selectedCodecName; - qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName; + qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput; // release any old codec encoder/decoder first... if (_codec && _encoder) { @@ -797,7 +797,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); - _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); qCDebug(audioclient) << "Selected Codec Plugin:" << _codec.get(); break; } @@ -1079,7 +1079,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { encodedBuffer = audioBuffer; } - emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, + emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, _isStereoInput, audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, packetType, _selectedCodecName); _stats.sentPacket(); @@ -1382,7 +1382,16 @@ void AudioClient::setIsStereoInput(bool isStereoInput) { _desiredInputFormat.setChannelCount(1); } - // change in channel count for desired input format, restart the input device + // restart the codec + if (_codec) { + if (_encoder) { + _codec->releaseEncoder(_encoder); + } + _encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); + } + qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput; + + // restart the input device switchInputToAudioDevice(_inputDeviceInfo); } } @@ -1418,7 +1427,7 @@ void AudioClient::outputFormatChanged() { _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT); } -bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo, bool isShutdownRequest) { +bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) { qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]"; bool supportedFormat = false; @@ -1601,7 +1610,7 @@ void AudioClient::outputNotify() { } } -bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo, bool isShutdownRequest) { +bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) { qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName() << "]"; bool supportedFormat = false; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 0ceb9c4dc3..0643b8e52a 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -378,8 +378,8 @@ private: void handleLocalEchoAndReverb(QByteArray& inputByteArray); - bool switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo, bool isShutdownRequest = false); - bool switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo, bool isShutdownRequest = false); + bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false); + bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false); // Callback acceleration dependent calculations int calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const; diff --git a/libraries/audio/src/AbstractAudioInterface.cpp b/libraries/audio/src/AbstractAudioInterface.cpp index 4def97596f..376ecddd34 100644 --- a/libraries/audio/src/AbstractAudioInterface.cpp +++ b/libraries/audio/src/AbstractAudioInterface.cpp @@ -19,7 +19,7 @@ #include "AudioConstants.h" -void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, +void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo, const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, PacketType packetType, QString codecName) { static std::mutex _mutex; @@ -30,9 +30,6 @@ void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes Locker lock(_mutex); auto audioPacket = NLPacket::create(packetType); - // FIXME - this is not a good way to determine stereoness with codecs.... - quint8 isStereo = bytes == AudioConstants::NETWORK_FRAME_BYTES_STEREO ? 1 : 0; - // write sequence number auto sequence = sequenceNumber++; audioPacket->writePrimitive(sequence); @@ -48,7 +45,8 @@ void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes audioPacket->writePrimitive(numSilentSamples); } else { // set the mono/stereo byte - audioPacket->writePrimitive(isStereo); + quint8 channelFlag = isStereo ? 1 : 0; + audioPacket->writePrimitive(channelFlag); } // pack the three float positions diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index 8b48b55206..37731c31f7 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -29,7 +29,7 @@ class AbstractAudioInterface : public QObject { public: AbstractAudioInterface(QObject* parent = 0) : QObject(parent) {}; - static void emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, + static void emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo, const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, PacketType packetType, QString codecName = QString(""));