Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-24 09:23:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #12162 from kencooke/audio-stereo-mic

Browse source 
Support for stereo audio input/mic
This commit is contained in:
Brad Hefta-Gaub 2018-01-12 13:29:28 -08:00 committed by GitHub
commit 68de7f864e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 52 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
assignment-client/src/Agent.cpp
View file

@ -439,7 +439,7 @@ void Agent::executeScript() {
encodedBuffer = audio;
}
AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber,
AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false,
audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0),
packetType, _selectedCodecName);
});

28
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -275,17 +275,28 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
if (micStreamIt == _audioStreams.end()) {
// we don't have a mic stream yet, so add it
// read the channel flag to see if our stream is stereo or not
// hop past the sequence number that leads the packet
message.seek(sizeof(quint16));
quint8 channelFlag;
message.readPrimitive(&channelFlag);
// pull the codec string from the packet
auto codecString = message.readString();
bool isStereo = channelFlag == 1;
// determine if the stream is stereo or not
bool isStereo;
if (packetType == PacketType::SilentAudioFrame
|| packetType == PacketType::ReplicatedSilentAudioFrame) {
quint16 numSilentSamples;
message.readPrimitive(&numSilentSamples);
isStereo = numSilentSamples == AudioConstants::NETWORK_FRAME_SAMPLES_STEREO;
} else {
quint8 channelFlag;
message.readPrimitive(&channelFlag);
isStereo = channelFlag == 1;
}
auto avatarAudioStream = new AvatarAudioStream(isStereo, AudioMixer::getStaticJitterFrames());
avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO);
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName;
avatarAudioStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec,
this, &AudioMixerClientData::handleMismatchAudioFormat);
@ -324,7 +335,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) {
#if INJECTORS_SUPPORT_CODECS
injectorStream->setupCodec(_codec, _selectedCodecName, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName;
qCDebug(audio) << "creating new injectorStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
#endif
auto emplaced = _audioStreams.emplace(
@ -567,7 +578,8 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c
auto avatarAudioStream = getAvatarAudioStream();
if (avatarAudioStream) {
avatarAudioStream->setupCodec(codec, codecName, AudioConstants::MONO);
avatarAudioStream->setupCodec(codec, codecName, avatarAudioStream->isStereo() ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audio) << "setting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << avatarAudioStream->isStereo();
}
#if INJECTORS_SUPPORT_CODECS

10
assignment-client/src/audio/AvatarAudioStream.cpp
View file

@ -11,6 +11,7 @@
#include <udt/PacketHeaders.h>
#include "AudioLogging.h"
#include "AvatarAudioStream.h"
AvatarAudioStream::AvatarAudioStream(bool isStereo, int numStaticJitterFrames) :
@ -41,6 +42,15 @@ int AvatarAudioStream::parseStreamProperties(PacketType type, const QByteArray&
_ringBuffer.resizeForFrameSize(isStereo
? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO
: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL);
// restart the codec
if (_codec) {
if (_decoder) {
_codec->releaseDecoder(_decoder);
}
_decoder = _codec->createDecoder(AudioConstants::SAMPLE_RATE, isStereo ? AudioConstants::STEREO : AudioConstants::MONO);
}
qCDebug(audio) << "resetting AvatarAudioStream... codec:" << _selectedCodecName << "isStereo:" << isStereo;
_isStereo = isStereo;
}

21
libraries/audio-client/src/AudioClient.cpp
View file

@ -782,7 +782,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) {
_selectedCodecName = selectedCodecName;
qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName;
qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;
// release any old codec encoder/decoder first...
if (_codec && _encoder) {
@ -797,7 +797,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) {
if (_selectedCodecName == plugin->getName()) {
_codec = plugin;
_receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO);
_encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO);
_encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);
qCDebug(audioclient) << "Selected Codec Plugin:" << _codec.get();
break;
}
@ -1079,7 +1079,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) {
encodedBuffer = audioBuffer;
}
emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber,
emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, _isStereoInput,
audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale,
packetType, _selectedCodecName);
_stats.sentPacket();
@ -1382,7 +1382,16 @@ void AudioClient::setIsStereoInput(bool isStereoInput) {
_desiredInputFormat.setChannelCount(1);
}
// change in channel count for desired input format, restart the input device
// restart the codec
if (_codec) {
if (_encoder) {
_codec->releaseEncoder(_encoder);
}
_encoder = _codec->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO);
}
qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput;
// restart the input device
switchInputToAudioDevice(_inputDeviceInfo);
}
}
@ -1418,7 +1427,7 @@ void AudioClient::outputFormatChanged() {
_receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate(), OUTPUT_CHANNEL_COUNT);
}
bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo, bool isShutdownRequest) {
bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) {
qCDebug(audioclient) << __FUNCTION__ << "inputDeviceInfo: [" << inputDeviceInfo.deviceName() << "]";
bool supportedFormat = false;
@ -1601,7 +1610,7 @@ void AudioClient::outputNotify() {
}
}
bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo, bool isShutdownRequest) {
bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) {
qCDebug(audioclient) << "AudioClient::switchOutputToAudioDevice() outputDeviceInfo: [" << outputDeviceInfo.deviceName() << "]";
bool supportedFormat = false;

4
libraries/audio-client/src/AudioClient.h
View file

@ -378,8 +378,8 @@ private:
void handleLocalEchoAndReverb(QByteArray& inputByteArray);
bool switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo, bool isShutdownRequest = false);
bool switchOutputToAudioDevice(const QAudioDeviceInfo& outputDeviceInfo, bool isShutdownRequest = false);
bool switchInputToAudioDevice(const QAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);
bool switchOutputToAudioDevice(const QAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);
// Callback acceleration dependent calculations
int calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const;

8
libraries/audio/src/AbstractAudioInterface.cpp
View file

@ -19,7 +19,7 @@
#include "AudioConstants.h"
void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber,
void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo,
const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale,
PacketType packetType, QString codecName) {
static std::mutex _mutex;
@ -30,9 +30,6 @@ void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes
Locker lock(_mutex);
auto audioPacket = NLPacket::create(packetType);
// FIXME - this is not a good way to determine stereoness with codecs....
quint8 isStereo = bytes == AudioConstants::NETWORK_FRAME_BYTES_STEREO ? 1 : 0;
// write sequence number
auto sequence = sequenceNumber++;
audioPacket->writePrimitive(sequence);
@ -48,7 +45,8 @@ void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes
audioPacket->writePrimitive(numSilentSamples);
} else {
// set the mono/stereo byte
audioPacket->writePrimitive(isStereo);
quint8 channelFlag = isStereo ? 1 : 0;
audioPacket->writePrimitive(channelFlag);
}
// pack the three float positions

2
libraries/audio/src/AbstractAudioInterface.h
View file

@ -29,7 +29,7 @@ class AbstractAudioInterface : public QObject {
public:
AbstractAudioInterface(QObject* parent = 0) : QObject(parent) {};
static void emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber,
static void emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo,
const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale,
PacketType packetType, QString codecName = QString(""));