immediately send ICE heartbeat once data present

This commit is contained in:
Stephen Birarda 2015-05-22 16:29:01 -07:00
parent 603528eede
commit 69f1cd80e6
7 changed files with 167 additions and 121 deletions

View file

@ -68,6 +68,7 @@ void DatagramProcessor::processDatagrams() {
Qt::QueuedConnection,
Q_ARG(QByteArray, incomingPacket));
} else {
// qDebug() << "Received audio data packet at" << usecTimestampNow();
QMetaObject::invokeMethod(DependencyManager::get<AudioClient>().data(), "addReceivedAudioToStream",
Qt::QueuedConnection,
Q_ARG(QByteArray, incomingPacket));

View file

@ -116,7 +116,7 @@ AudioClient::AudioClient() :
{
// clear the array of locally injected samples
memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL);
connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples,
this, &AudioClient::processReceivedSamples, Qt::DirectConnection);
@ -127,7 +127,7 @@ AudioClient::AudioClient() :
QTimer* updateTimer = new QTimer(this);
connect(updateTimer, &QTimer::timeout, this, &AudioClient::checkDevices);
updateTimer->start(DEVICE_CHECK_INTERVAL_MSECS);
// create GVerb filter
_gverb = createGverbFilter();
configureGverbFilter(_gverb);
@ -135,7 +135,7 @@ AudioClient::AudioClient() :
AudioClient::~AudioClient() {
stop();
if (_gverb) {
gverb_free(_gverb);
}
@ -148,7 +148,7 @@ void AudioClient::reset() {
_toneSource.reset();
_sourceGain.reset();
_inputGain.reset();
gverb_flush(_gverb);
}
@ -186,7 +186,7 @@ int numDestinationSamplesRequired(const QAudioFormat& sourceFormat, const QAudio
int numSourceSamples) {
float ratio = (float) destinationFormat.channelCount() / sourceFormat.channelCount();
ratio *= (float) destinationFormat.sampleRate() / sourceFormat.sampleRate();
return (numSourceSamples * ratio) + 0.5f;
}
@ -287,7 +287,7 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) {
}
qCDebug(audioclient) << "DEBUG [" << deviceName << "] [" << getNamedAudioDeviceForMode(mode, deviceName).deviceName() << "]";
return getNamedAudioDeviceForMode(mode, deviceName);
#endif
@ -302,7 +302,7 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
if (!audioDevice.isFormatSupported(desiredAudioFormat)) {
qCDebug(audioclient) << "The desired format for audio I/O is" << desiredAudioFormat;
qCDebug(audioclient, "The desired audio format is not supported by this device");
if (desiredAudioFormat.channelCount() == 1) {
adjustedAudioFormat = desiredAudioFormat;
adjustedAudioFormat.setChannelCount(2);
@ -313,17 +313,17 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
adjustedAudioFormat.setChannelCount(1);
}
}
const int FORTY_FOUR = 44100;
adjustedAudioFormat = desiredAudioFormat;
#ifdef Q_OS_ANDROID
adjustedAudioFormat.setSampleRate(FORTY_FOUR);
#else
const int HALF_FORTY_FOUR = FORTY_FOUR / 2;
if (audioDevice.supportedSampleRates().contains(AudioConstants::SAMPLE_RATE * 2)) {
// use 48, which is a sample downsample, upsample
adjustedAudioFormat.setSampleRate(AudioConstants::SAMPLE_RATE * 2);
@ -335,7 +335,7 @@ bool adjustedFormatForAudioDevice(const QAudioDeviceInfo& audioDevice,
adjustedAudioFormat.setSampleRate(FORTY_FOUR);
}
#endif
if (adjustedAudioFormat != desiredAudioFormat) {
// return the nearest in case it needs 2 channels
adjustedAudioFormat = audioDevice.nearestFormat(adjustedAudioFormat);
@ -357,18 +357,18 @@ bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationS
for (uint i = 0; i < numSourceSamples; i += 2) {
destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 1] / 2);
}
return true;
} else if (sourceAudioFormat.channelCount() == 1 && destinationAudioFormat.channelCount() == 2) {
// loop through the mono input audio and repeat each sample twice
for (uint i = 0; i < numSourceSamples; ++i) {
destinationSamples[i * 2] = destinationSamples[(i * 2) + 1] = sourceSamples[i];
}
return true;
}
return false;
}
@ -376,7 +376,7 @@ soxr_error_t possibleResampling(soxr_t resampler,
const int16_t* sourceSamples, int16_t* destinationSamples,
unsigned int numSourceSamples, unsigned int numDestinationSamples,
const QAudioFormat& sourceAudioFormat, const QAudioFormat& destinationAudioFormat) {
if (numSourceSamples > 0) {
if (!resampler) {
if (!sampleChannelConversion(sourceSamples, destinationSamples, numSourceSamples,
@ -384,41 +384,41 @@ soxr_error_t possibleResampling(soxr_t resampler,
// no conversion, we can copy the samples directly across
memcpy(destinationSamples, sourceSamples, numSourceSamples * sizeof(int16_t));
}
return 0;
} else {
soxr_error_t resampleError = 0;
if (sourceAudioFormat.channelCount() != destinationAudioFormat.channelCount()) {
float channelCountRatio = (float) destinationAudioFormat.channelCount() / sourceAudioFormat.channelCount();
int numChannelCoversionSamples = (int) (numSourceSamples * channelCountRatio);
int16_t* channelConversionSamples = new int16_t[numChannelCoversionSamples];
sampleChannelConversion(sourceSamples, channelConversionSamples,
numSourceSamples,
sourceAudioFormat, destinationAudioFormat);
resampleError = soxr_process(resampler,
channelConversionSamples, numChannelCoversionSamples, NULL,
destinationSamples, numDestinationSamples, NULL);
delete[] channelConversionSamples;
} else {
unsigned int numAdjustedSourceSamples = numSourceSamples;
unsigned int numAdjustedDestinationSamples = numDestinationSamples;
if (sourceAudioFormat.channelCount() == 2 && destinationAudioFormat.channelCount() == 2) {
numAdjustedSourceSamples /= 2;
numAdjustedDestinationSamples /= 2;
}
resampleError = soxr_process(resampler,
sourceSamples, numAdjustedSourceSamples, NULL,
destinationSamples, numAdjustedDestinationSamples, NULL);
}
return resampleError;
}
} else {
@ -429,30 +429,30 @@ soxr_error_t possibleResampling(soxr_t resampler,
soxr_t soxrResamplerFromInputFormatToOutputFormat(const QAudioFormat& sourceAudioFormat,
const QAudioFormat& destinationAudioFormat) {
soxr_error_t soxrError;
// setup soxr_io_spec_t for input and output
soxr_io_spec_t inputToNetworkSpec = soxr_io_spec(soxrDataTypeFromQAudioFormat(sourceAudioFormat),
soxrDataTypeFromQAudioFormat(destinationAudioFormat));
// setup soxr_quality_spec_t for quality options
soxr_quality_spec_t qualitySpec = soxr_quality_spec(SOXR_MQ, 0);
int channelCount = (sourceAudioFormat.channelCount() == 2 && destinationAudioFormat.channelCount() == 2)
? 2 : 1;
soxr_t newResampler = soxr_create(sourceAudioFormat.sampleRate(),
destinationAudioFormat.sampleRate(),
channelCount,
&soxrError, &inputToNetworkSpec, &qualitySpec, 0);
if (soxrError) {
qCDebug(audioclient) << "There was an error setting up the soxr resampler -" << "soxr error code was " << soxrError;
soxr_delete(newResampler);
return NULL;
}
return newResampler;
}
@ -476,7 +476,7 @@ void AudioClient::start() {
QAudioDeviceInfo outputDeviceInfo = defaultAudioDeviceForMode(QAudio::AudioOutput);
qCDebug(audioclient) << "The default audio output device is" << outputDeviceInfo.deviceName();
bool outputFormatSupported = switchOutputToAudioDevice(outputDeviceInfo);
if (!inputFormatSupported) {
qCDebug(audioclient) << "Unable to set up audio input because of a problem with input format.";
qCDebug(audioclient) << "The closest format available is" << inputDeviceInfo.nearestFormat(_desiredInputFormat);
@ -503,11 +503,11 @@ void AudioClient::stop() {
_sourceGain.finalize();
_noiseSource.finalize();
_toneSource.finalize();
// "switch" to invalid devices in order to shut down the state
switchInputToAudioDevice(QAudioDeviceInfo());
switchOutputToAudioDevice(QAudioDeviceInfo());
if (_loopbackResampler) {
soxr_delete(_loopbackResampler);
_loopbackResampler = NULL;
@ -543,7 +543,7 @@ ty_gverb* AudioClient::createGverbFilter() {
_reverbOptions->getReverbTime(), _reverbOptions->getDamping(), _reverbOptions->getSpread(),
_reverbOptions->getInputBandwidth(), _reverbOptions->getEarlyLevel(),
_reverbOptions->getTailLevel());
return filter;
}
@ -560,7 +560,7 @@ void AudioClient::configureGverbFilter(ty_gverb* filter) {
void AudioClient::updateGverbOptions() {
bool reverbChanged = false;
if (_receivedAudioStream.hasReverb()) {
if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) {
_zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime());
reverbChanged = true;
@ -569,7 +569,7 @@ void AudioClient::updateGverbOptions() {
_zoneReverbOptions.setWetLevel(_receivedAudioStream.getWetLevel());
// Not part of actual filter config, no need to set reverbChanged to true
}
if (_reverbOptions != &_zoneReverbOptions) {
_reverbOptions = &_zoneReverbOptions;
reverbChanged = true;
@ -578,7 +578,7 @@ void AudioClient::updateGverbOptions() {
_reverbOptions = &_scriptReverbOptions;
reverbChanged = true;
}
if (reverbChanged) {
gverb_free(_gverb);
_gverb = createGverbFilter();
@ -588,7 +588,7 @@ void AudioClient::updateGverbOptions() {
void AudioClient::setReverb(bool reverb) {
_reverb = reverb;
if (!_reverb) {
gverb_flush(_gverb);
}
@ -620,7 +620,7 @@ void AudioClient::addReverb(ty_gverb* gverb, int16_t* samplesData, int16_t* reve
QAudioFormat& audioFormat, bool noEcho) {
float wetFraction = DB_CO(_reverbOptions->getWetLevel());
float dryFraction = 1.0f - wetFraction;
float lValue,rValue;
for (int sample = 0; sample < numSamples; sample += audioFormat.channelCount()) {
// Run GVerb
@ -634,7 +634,7 @@ void AudioClient::addReverb(ty_gverb* gverb, int16_t* samplesData, int16_t* reve
int lResult = glm::clamp((int)(samplesData[j] * dryFraction + lValue * wetFraction),
AudioConstants::MIN_SAMPLE_VALUE, AudioConstants::MAX_SAMPLE_VALUE);
samplesData[j] = (int16_t)lResult;
if (noEcho) {
reverbAlone[j] = (int16_t)lValue * wetFraction;
}
@ -643,7 +643,7 @@ void AudioClient::addReverb(ty_gverb* gverb, int16_t* samplesData, int16_t* reve
int rResult = glm::clamp((int)(samplesData[j] * dryFraction + rValue * wetFraction),
AudioConstants::MIN_SAMPLE_VALUE, AudioConstants::MAX_SAMPLE_VALUE);
samplesData[j] = (int16_t)rResult;
if (noEcho) {
reverbAlone[j] = (int16_t)rValue * wetFraction;
}
@ -660,53 +660,53 @@ void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) {
if (_muted || !_audioOutput || (!_shouldEchoLocally && !hasReverb)) {
return;
}
// if this person wants local loopback add that to the locally injected audio
// if there is reverb apply it to local audio and substract the origin samples
if (!_loopbackOutputDevice && _loopbackAudioOutput) {
// we didn't have the loopback output device going so set that up now
_loopbackOutputDevice = _loopbackAudioOutput->start();
if (!_loopbackOutputDevice) {
return;
}
}
// do we need to setup a resampler?
if (_inputFormat.sampleRate() != _outputFormat.sampleRate() && !_loopbackResampler) {
qCDebug(audioclient) << "Attemping to create a resampler for input format to output format for audio loopback.";
_loopbackResampler = soxrResamplerFromInputFormatToOutputFormat(_inputFormat, _outputFormat);
if (!_loopbackResampler) {
return;
}
}
static QByteArray reverbAlone; // Intermediary for local reverb with no echo
static QByteArray loopBackByteArray;
int numInputSamples = inputByteArray.size() / sizeof(int16_t);
int numLoopbackSamples = numDestinationSamplesRequired(_inputFormat, _outputFormat, numInputSamples);
reverbAlone.resize(numInputSamples * sizeof(int16_t));
loopBackByteArray.resize(numLoopbackSamples * sizeof(int16_t));
int16_t* inputSamples = reinterpret_cast<int16_t*>(inputByteArray.data());
int16_t* reverbAloneSamples = reinterpret_cast<int16_t*>(reverbAlone.data());
int16_t* loopbackSamples = reinterpret_cast<int16_t*>(loopBackByteArray.data());
if (hasReverb) {
updateGverbOptions();
addReverb(_gverb, inputSamples, reverbAloneSamples, numInputSamples,
_inputFormat, !_shouldEchoLocally);
}
possibleResampling(_loopbackResampler,
(_shouldEchoLocally) ? inputSamples : reverbAloneSamples, loopbackSamples,
numInputSamples, numLoopbackSamples,
_inputFormat, _outputFormat);
_loopbackOutputDevice->write(loopBackByteArray);
}
@ -726,7 +726,7 @@ void AudioClient::handleAudioInput() {
int inputSamplesRequired = (int)((float)AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * inputToNetworkInputRatio);
QByteArray inputByteArray = _inputDevice->readAll();
// Add audio source injection if enabled
if (!_muted && _audioSourceInjectEnabled) {
int16_t* inputFrameData = (int16_t*)inputByteArray.data();
@ -745,11 +745,11 @@ void AudioClient::handleAudioInput() {
_sourceGain.render(_inputFrameBuffer); // post gain
_inputFrameBuffer.copyFrames(1, inputFrameCount, inputFrameData, true /*copy out*/);
}
handleLocalEchoAndReverb(inputByteArray);
_inputRingBuffer.writeData(inputByteArray.data(), inputByteArray.size());
float audioInputMsecsRead = inputByteArray.size() / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC));
_stats.updateInputMsecsRead(audioInputMsecsRead);
@ -763,50 +763,50 @@ void AudioClient::handleAudioInput() {
: AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL;
if (!_muted) {
// zero out the monoAudioSamples array and the locally injected audio
memset(networkAudioSamples, 0, numNetworkBytes);
// Increment the time since the last clip
if (_timeSinceLastClip >= 0.0f) {
_timeSinceLastClip += (float) numNetworkSamples / (float) AudioConstants::SAMPLE_RATE;
}
int16_t* inputAudioSamples = new int16_t[inputSamplesRequired];
_inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired);
possibleResampling(_inputToNetworkResampler,
inputAudioSamples, networkAudioSamples,
inputSamplesRequired, numNetworkSamples,
_inputFormat, _desiredInputFormat);
delete[] inputAudioSamples;
// only impose the noise gate and perform tone injection if we are sending mono audio
if (!_isStereoInput && !_audioSourceInjectEnabled && _isNoiseGateEnabled) {
_inputGate.gateSamples(networkAudioSamples, numNetworkSamples);
// if we performed the noise gate we can get values from it instead of enumerating the samples again
_lastInputLoudness = _inputGate.getLastLoudness();
if (_inputGate.clippedInLastFrame()) {
_timeSinceLastClip = 0.0f;
}
} else {
float loudness = 0.0f;
for (int i = 0; i < numNetworkSamples; i++) {
int thisSample = std::abs(networkAudioSamples[i]);
loudness += (float) thisSample;
if (thisSample > (AudioConstants::MAX_SAMPLE_VALUE * AudioNoiseGate::CLIPPING_THRESHOLD)) {
_timeSinceLastClip = 0.0f;
}
}
_lastInputLoudness = fabs(loudness / numNetworkSamples);
}
emit inputReceived(QByteArray(reinterpret_cast<const char*>(networkAudioSamples),
AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * sizeof(AudioConstants::AudioSample)));
@ -814,13 +814,13 @@ void AudioClient::handleAudioInput() {
// our input loudness is 0, since we're muted
_lastInputLoudness = 0;
_timeSinceLastClip = 0.0f;
_inputRingBuffer.shiftReadPosition(inputSamplesRequired);
}
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
if (audioMixer && audioMixer->getActiveSocket()) {
glm::vec3 headPosition = _positionGetter();
glm::quat headOrientation = _orientationGetter();
@ -852,7 +852,7 @@ void AudioClient::handleAudioInput() {
// memcpy the three float positions
memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));
currentPacketPtr += (sizeof(headPosition));
// memcpy our orientation
memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));
currentPacketPtr += sizeof(headOrientation);
@ -864,7 +864,7 @@ void AudioClient::handleAudioInput() {
// memcpy the three float positions
memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));
currentPacketPtr += (sizeof(headPosition));
// memcpy our orientation
memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));
currentPacketPtr += sizeof(headOrientation);
@ -875,6 +875,8 @@ void AudioClient::handleAudioInput() {
_stats.sentPacket();
// qDebug() << "Sending audio packet at" << usecTimestampNow();
int packetBytes = currentPacketPtr - audioDataPacket;
nodeList->writeDatagram(audioDataPacket, packetBytes, audioMixer);
_outgoingAvatarAudioSequenceNumber++;
@ -890,7 +892,7 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr
outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t));
const int16_t* receivedSamples = reinterpret_cast<const int16_t*>(inputBuffer.data());
// copy the packet from the RB to the output
possibleResampling(_networkToOutputResampler, receivedSamples,
reinterpret_cast<int16_t*>(outputBuffer.data()),
@ -900,20 +902,20 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr
void AudioClient::sendMuteEnvironmentPacket() {
auto nodeList = DependencyManager::get<NodeList>();
QByteArray mutePacket = nodeList->byteArrayWithPopulatedHeader(PacketTypeMuteEnvironment);
int headerSize = mutePacket.size();
const float MUTE_RADIUS = 50;
glm::vec3 currentSourcePosition = _positionGetter();
mutePacket.resize(mutePacket.size() + sizeof(glm::vec3) + sizeof(float));
memcpy(mutePacket.data() + headerSize, &currentSourcePosition, sizeof(glm::vec3));
memcpy(mutePacket.data() + headerSize + sizeof(glm::vec3), &MUTE_RADIUS, sizeof(float));
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
if (audioMixer) {
// send off this mute packet
nodeList->writeDatagram(mutePacket, audioMixer);
@ -922,6 +924,7 @@ void AudioClient::sendMuteEnvironmentPacket() {
void AudioClient::addReceivedAudioToStream(const QByteArray& audioByteArray) {
if (_audioOutput) {
// qDebug() << "Adding received audio to stream at" << usecTimestampNow();
// Audio output must exist and be correctly set up if we're going to process received audio
_receivedAudioStream.parseData(audioByteArray);
}
@ -930,11 +933,11 @@ void AudioClient::addReceivedAudioToStream(const QByteArray& audioByteArray) {
void AudioClient::parseAudioEnvironmentData(const QByteArray &packet) {
int numBytesPacketHeader = numBytesForPacketHeader(packet);
const char* dataAt = packet.constData() + numBytesPacketHeader;
char bitset;
memcpy(&bitset, dataAt, sizeof(char));
dataAt += sizeof(char);
bool hasReverb = oneAtBit(bitset, HAS_REVERB_BIT);;
if (hasReverb) {
float reverbTime, wetLevel;
@ -956,13 +959,13 @@ void AudioClient::toggleMute() {
void AudioClient::setIsStereoInput(bool isStereoInput) {
if (isStereoInput != _isStereoInput) {
_isStereoInput = isStereoInput;
if (_isStereoInput) {
_desiredInputFormat.setChannelCount(2);
} else {
_desiredInputFormat.setChannelCount(1);
}
// change in channel count for desired input format, restart the input device
switchInputToAudioDevice(_inputAudioDeviceName);
}
@ -976,7 +979,7 @@ void AudioClient::selectAudioSourcePinkNoise() {
_noiseSourceEnabled = true;
_toneSourceEnabled = false;
}
void AudioClient::selectAudioSourceSine440() {
_toneSourceEnabled = true;
_noiseSourceEnabled = false;
@ -986,24 +989,24 @@ bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) {
if (injector->getLocalBuffer()) {
QAudioFormat localFormat = _desiredOutputFormat;
localFormat.setChannelCount(isStereo ? 2 : 1);
QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName),
localFormat,
injector->getLocalBuffer());
// move the localOutput to the same thread as the local injector buffer
localOutput->moveToThread(injector->getLocalBuffer()->thread());
// have it be stopped when that local buffer is about to close
connect(injector->getLocalBuffer(), &AudioInjectorLocalBuffer::bufferEmpty, localOutput, &QAudioOutput::stop);
connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop);
qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput;
localOutput->start(injector->getLocalBuffer());
return localOutput->state() == QAudio::ActiveState;
}
return false;
}
@ -1015,7 +1018,7 @@ void AudioClient::outputFormatChanged() {
bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceInfo) {
bool supportedFormat = false;
// cleanup any previously initialized device
if (_audioInput) {
// The call to stop() causes _inputDevice to be destructed.
@ -1030,7 +1033,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn
_inputAudioDeviceName = "";
}
if (_inputToNetworkResampler) {
// if we were using an input to network resampler, delete it here
soxr_delete(_inputToNetworkResampler);
@ -1040,36 +1043,36 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn
if (!inputDeviceInfo.isNull()) {
qCDebug(audioclient) << "The audio input device " << inputDeviceInfo.deviceName() << "is available.";
_inputAudioDeviceName = inputDeviceInfo.deviceName().trimmed();
if (adjustedFormatForAudioDevice(inputDeviceInfo, _desiredInputFormat, _inputFormat)) {
qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat;
// we've got the best we can get for input
// if required, setup a soxr resampler for this input to our desired network format
if (_inputFormat != _desiredInputFormat
&& _inputFormat.sampleRate() != _desiredInputFormat.sampleRate()) {
qCDebug(audioclient) << "Attemping to create a soxr resampler for input format to network format.";
_inputToNetworkResampler = soxrResamplerFromInputFormatToOutputFormat(_inputFormat, _desiredInputFormat);
if (!_inputToNetworkResampler) {
return false;
}
} else {
qCDebug(audioclient) << "No resampling required for audio input to match desired network format.";
}
// if the user wants stereo but this device can't provide then bail
if (!_isStereoInput || _inputFormat.channelCount() == 2) {
if (!_isStereoInput || _inputFormat.channelCount() == 2) {
_audioInput = new QAudioInput(inputDeviceInfo, _inputFormat, this);
_numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat);
_audioInput->setBufferSize(_numInputCallbackBytes);
// how do we want to handle input working, but output not working?
int numFrameSamples = calculateNumberOfFrameSamples(_numInputCallbackBytes);
_inputRingBuffer.resizeForFrameSize(numFrameSamples);
_inputDevice = _audioInput->start();
if (_inputDevice) {
connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleAudioInput()));
supportedFormat = true;
@ -1079,7 +1082,7 @@ bool AudioClient::switchInputToAudioDevice(const QAudioDeviceInfo& inputDeviceIn
}
}
}
return supportedFormat;
}
@ -1117,7 +1120,7 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
// cleanup any previously initialized device
if (_audioOutput) {
_audioOutput->stop();
delete _audioOutput;
_audioOutput = NULL;
@ -1125,13 +1128,13 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
delete _loopbackAudioOutput;
_loopbackAudioOutput = NULL;
}
if (_networkToOutputResampler) {
// if we were using an input to network resampler, delete it here
soxr_delete(_networkToOutputResampler);
_networkToOutputResampler = NULL;
}
if (_loopbackResampler) {
// if we were using an input to output resample, delete it here
soxr_delete(_loopbackResampler);
@ -1141,24 +1144,24 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
if (!outputDeviceInfo.isNull()) {
qCDebug(audioclient) << "The audio output device " << outputDeviceInfo.deviceName() << "is available.";
_outputAudioDeviceName = outputDeviceInfo.deviceName().trimmed();
if (adjustedFormatForAudioDevice(outputDeviceInfo, _desiredOutputFormat, _outputFormat)) {
qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat;
// we've got the best we can get for input
// if required, setup a soxr resampler for this input to our desired network format
if (_desiredOutputFormat != _outputFormat
&& _desiredOutputFormat.sampleRate() != _outputFormat.sampleRate()) {
qCDebug(audioclient) << "Attemping to create a resampler for network format to output format.";
_networkToOutputResampler = soxrResamplerFromInputFormatToOutputFormat(_desiredOutputFormat, _outputFormat);
if (!_networkToOutputResampler) {
return false;
}
} else {
qCDebug(audioclient) << "No resampling required for network output to match actual output format.";
}
outputFormatChanged();
// setup our general output device for audio-mixer audio
@ -1168,20 +1171,20 @@ bool AudioClient::switchOutputToAudioDevice(const QAudioDeviceInfo& outputDevice
connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify);
qCDebug(audioclient) << "Output Buffer capacity in frames: " << _audioOutput->bufferSize() / sizeof(int16_t) / (float)_outputFrameSize;
_audioOutputIODevice.start();
_audioOutput->start(&_audioOutputIODevice);
// setup a loopback audio output device
_loopbackAudioOutput = new QAudioOutput(outputDeviceInfo, _outputFormat, this);
_timeSinceLastReceived.start();
supportedFormat = true;
}
}
return supportedFormat;
}
@ -1202,7 +1205,7 @@ void AudioClient::setOutputBufferSize(int numFrames) {
// The following constant is operating system dependent due to differences in
// the way input audio is handled. The audio input buffer size is inversely
// proportional to the accelerator ratio.
// proportional to the accelerator ratio.
#ifdef Q_OS_WIN
const float AudioClient::CALLBACK_ACCELERATOR_RATIO = 0.1f;
@ -1226,7 +1229,7 @@ int AudioClient::calculateNumberOfInputCallbackBytes(const QAudioFormat& format)
}
float AudioClient::calculateDeviceToNetworkInputRatio() const {
float inputToNetworkInputRatio = (int)((_numInputCallbackBytes
float inputToNetworkInputRatio = (int)((_numInputCallbackBytes
* CALLBACK_ACCELERATOR_RATIO
/ AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL) + 0.5f);
@ -1257,7 +1260,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) {
int samplesRequested = maxSize / sizeof(int16_t);
int samplesPopped;
int bytesWritten;
if ((samplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) {
AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput();
lastPopOutput.readSamples((int16_t*)data, samplesPopped);

View file

@ -106,6 +106,8 @@ bool AddressManager::handleUrl(const QUrl& lookupUrl) {
qCDebug(networking) << "Trying to go to URL" << lookupUrl.toString();
qDebug() << "Lookup of HF URL at" << usecTimestampNow();
// there are 4 possible lookup strings
// 1. global place name (name of domain or place) - example: sanfrancisco
@ -163,6 +165,8 @@ void AddressManager::handleAPIResponse(QNetworkReply& requestReply) {
QJsonObject responseObject = QJsonDocument::fromJson(requestReply.readAll()).object();
QJsonObject dataObject = responseObject["data"].toObject();
qDebug() << "Go to address from API response at" << usecTimestampNow();
goToAddressFromObject(dataObject.toVariantMap(), requestReply);
emit lookupResultsFinished();

View file

@ -135,10 +135,21 @@ void DomainHandler::setIceServerHostnameAndID(const QString& iceServerHostname,
replaceableSockAddr->~HifiSockAddr();
replaceableSockAddr = new (replaceableSockAddr) HifiSockAddr(iceServerHostname, ICE_SERVER_DEFAULT_PORT);
if (_iceServerSockAddr.getAddress().isNull()) {
// connect to lookup completed for ice-server socket so we can request a heartbeat once hostname is looked up
connect(&_iceServerSockAddr, &HifiSockAddr::lookupCompleted, this, &DomainHandler::completedIceServerHostnameLookup);
} else {
completedIceServerHostnameLookup();
}
// refresh our ICE client UUID to something new
_iceClientID = QUuid::createUuid();
qCDebug(networking) << "ICE required to connect to domain via ice server at" << iceServerHostname;
qDebug() << "ICE server info set at" << usecTimestampNow();
}
}
@ -172,6 +183,14 @@ void DomainHandler::completedHostnameLookup(const QHostInfo& hostInfo) {
qCDebug(networking, "Failed domain server lookup");
}
void DomainHandler::completedIceServerHostnameLookup() {
qDebug() << "ICE server socket is at" << _iceServerSockAddr;
qDebug() << "ICE server socket lookup completed at" << usecTimestampNow();
// emit our signal so we can send a heartbeat to ice-server immediately
emit iceSocketAndIDReceived();
}
void DomainHandler::setIsConnected(bool isConnected) {
if (_isConnected != isConnected) {
_isConnected = isConnected;
@ -270,6 +289,8 @@ void DomainHandler::processICEResponsePacket(const QByteArray& icePacket) {
NetworkPeer packetPeer;
iceResponseStream >> packetPeer;
qDebug() << "Recieved response for network peer from ICE server at" << usecTimestampNow();
if (packetPeer.getUUID() != _iceDomainID) {
qCDebug(networking) << "Received a network peer with ID that does not match current domain. Will not attempt connection.";
} else {

View file

@ -85,7 +85,9 @@ public slots:
private slots:
void completedHostnameLookup(const QHostInfo& hostInfo);
void completedIceServerHostnameLookup();
void settingsRequestFinished();
signals:
void hostnameChanged(const QString& hostname);
@ -96,6 +98,7 @@ signals:
void connectedToDomain(const QString& hostname);
void disconnectedFromDomain();
void iceSocketAndIDReceived();
void requestICEConnectionAttempt();
void settingsReceived(const QJsonObject& domainSettingsObject);

View file

@ -64,6 +64,9 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned
// clear our NodeList when the domain changes
connect(&_domainHandler, &DomainHandler::disconnectedFromDomain, this, &NodeList::reset);
// send an ICE heartbeat as soon as we get ice server information
connect(&_domainHandler, &DomainHandler::iceSocketAndIDReceived, this, &NodeList::handleICEConnectionToDomainServer);
// handle ICE signal from DS so connection is attempted immediately
connect(&_domainHandler, &DomainHandler::requestICEConnectionAttempt, this, &NodeList::handleICEConnectionToDomainServer);
@ -160,6 +163,7 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr
if (!_domainHandler.getSockAddr().isNull()) {
// only process a list from domain-server if we're talking to a domain
// TODO: how do we make sure this is actually the domain we want the list from (DTLS probably)
qDebug() << "Processing domain server list at" << usecTimestampNow();
processDomainServerList(packet);
}
break;
@ -198,6 +202,8 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr
if (sendingNode) {
sendingNode->setLastHeardMicrostamp(usecTimestampNow());
qDebug() << "Activating socket for node" << sendingNode->getUUID() << "at" << usecTimestampNow();
// activate the appropriate socket for this node, if not yet updated
activateSocketFromNodeCommunication(packet, sendingNode);
@ -216,6 +222,8 @@ void NodeList::processNodeData(const HifiSockAddr& senderSockAddr, const QByteAr
case PacketTypeUnverifiedPingReply: {
qCDebug(networking) << "Received reply from domain-server on" << senderSockAddr;
qDebug() << "Received reply from domain server at" << usecTimestampNow();
// for now we're unsafely assuming this came back from the domain
if (senderSockAddr == _domainHandler.getICEPeer().getLocalSocket()) {
qCDebug(networking) << "Connecting to domain using local socket";
@ -379,6 +387,8 @@ void NodeList::sendDomainServerCheckIn() {
}
}
qDebug() << "Sending domain server check in at" << usecTimestampNow();
if (!isUsingDTLS) {
writeUnverifiedDatagram(domainServerPacket, _domainHandler.getSockAddr());
}
@ -504,6 +514,8 @@ void NodeList::handleICEConnectionToDomainServer() {
_domainHandler.getICEPeer().resetConnectionAttemps();
qDebug() << "Sending heartbeat to ice server at" << usecTimestampNow();
LimitedNodeList::sendHeartbeatToIceServer(_domainHandler.getICEServerSockAddr(),
_domainHandler.getICEClientID(),
_domainHandler.getICEDomainID());
@ -511,6 +523,8 @@ void NodeList::handleICEConnectionToDomainServer() {
qCDebug(networking) << "Sending ping packets to establish connectivity with domain-server with ID"
<< uuidStringWithoutCurlyBraces(_domainHandler.getICEDomainID());
qDebug() << "Sending ping packet to domain server at" << usecTimestampNow();
// send the ping packet to the local and public sockets for this node
QByteArray localPingPacket = constructPingPacket(PingType::Local, false, _domainHandler.getICEClientID());
writeUnverifiedDatagram(localPingPacket, _domainHandler.getICEPeer().getLocalSocket());
@ -596,6 +610,7 @@ void NodeList::sendAssignment(Assignment& assignment) {
}
void NodeList::pingPunchForInactiveNode(const SharedNodePointer& node) {
qDebug() << "Sending ping punch to node" << node->getUUID() << "at" << usecTimestampNow();
// send the ping packet to the local and public sockets for this node
QByteArray localPingPacket = constructPingPacket(PingType::Local);

View file

@ -76,6 +76,7 @@ signals:
void limitOfSilentDomainCheckInsReached();
private slots:
void sendPendingDSPathQuery();
void handleICEConnectionToDomainServer();
private:
NodeList() : LimitedNodeList(0, 0) { assert(false); } // Not implemented, needed for DependencyManager templates compile
NodeList(char ownerType, unsigned short socketListenPort = 0, unsigned short dtlsListenPort = 0);
@ -85,8 +86,6 @@ private:
void sendSTUNRequest();
bool processSTUNResponse(const QByteArray& packet);
void handleICEConnectionToDomainServer();
void processDomainServerAuthRequest(const QByteArray& packet);
void requestAuthForDomainServer();
void activateSocketFromNodeCommunication(const QByteArray& packet, const SharedNodePointer& sendingNode);