From 600348bf102fdef7bbb28e16bffd104b208d6838 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 1 Jul 2016 19:27:00 -0700 Subject: [PATCH 01/40] Initial cut of htrf for mono localOnly injectors Probably need to clean up a bit, but wanted to get this out there for comment on more general issues, etc... To test, I added a localOnly: true to the cow in the tutorial. --- libraries/audio-client/src/AudioClient.cpp | 263 +++++++++++++++++++-- libraries/audio-client/src/AudioClient.h | 11 +- libraries/audio/src/AudioInjector.h | 14 +- 3 files changed, 262 insertions(+), 26 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 7628c09748..01bd0d3c22 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #ifdef __APPLE__ #include @@ -42,6 +44,8 @@ #include #include "AudioInjector.h" +#include "AudioLimiter.h" +#include "AudioHRTF.h" #include "AudioConstants.h" #include "PositionalAudioStream.h" #include "AudioClientLogging.h" @@ -852,33 +856,43 @@ void AudioClient::setIsStereoInput(bool isStereoInput) { bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) { - if (injector->getLocalBuffer()) { - QAudioFormat localFormat = _desiredOutputFormat; - localFormat.setChannelCount(isStereo ? 2 : 1); + if (injector->getLocalBuffer() && _audioInput ) { + if(isStereo) { + QAudioFormat localFormat = _desiredOutputFormat; + localFormat.setChannelCount(isStereo ? 2 : 1); - QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName), - localFormat, - injector->getLocalBuffer()); + QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName), + localFormat, + injector->getLocalBuffer()); - // move the localOutput to the same thread as the local injector buffer - localOutput->moveToThread(injector->getLocalBuffer()->thread()); + // move the localOutput to the same thread as the local injector buffer + localOutput->moveToThread(injector->getLocalBuffer()->thread()); - // have it be stopped when that local buffer is about to close - // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle, - // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread. - connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) { - if (state == QAudio::IdleState) { - localOutput->stop(); - injector->stop(); - } - }); + // have it be stopped when that local buffer is about to close + // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle, + // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread. + connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) { + if (state == QAudio::IdleState) { + localOutput->stop(); + injector->stop(); + } + }); - connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop); + connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop); - qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput; + qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput; + + localOutput->start(injector->getLocalBuffer()); + return localOutput->state() == QAudio::ActiveState; + } else { + // just add it to the vector of active local injectors + // TODO: deal with concurrency perhaps? Maybe not + qDebug() << "adding new injector!!!!!!!"; + + _activeLocalAudioInjectors.append(injector); + return true; + } - localOutput->start(injector->getLocalBuffer()); - return localOutput->state() == QAudio::ActiveState; } return false; @@ -1134,20 +1148,225 @@ float AudioClient::getAudioOutputMsecsUnplayed() const { return msecsAudioOutputUnplayed; } +void AudioClient::AudioOutputIODevice::renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples) { + qint64 numFrames = numSamples/AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + int16_t* dataPtr = data; + float* hrtfPtr = hrtfBuffer; + for(qint64 i=0; i < numFrames; i++) { + hrtf.render(dataPtr, hrtfPtr, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + //qDebug() << "processed frame " << i; + + dataPtr += AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + hrtfPtr += 2*AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + + } + assert(dataPtr - data <= numSamples); + assert(hrtfPtr - hrtfBuffer <= 2*numSamples); +} + +float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { + // copied from AudioMixer, more or less + glm::quat inverseOrientation = glm::inverse(_orientationGetter()); + + // compute sample delay for the 2 ears to create phase panning + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + // project the rotated source position vector onto x-y plane + rotatedSourcePosition.y = 0.0f; + + static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + + if (glm::length2(rotatedSourcePosition) > SOURCE_DISTANCE_THRESHOLD) { + + // produce an oriented angle about the y-axis + return glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), glm::normalize(rotatedSourcePosition), glm::vec3(0.0f, -1.0f, 0.0f)); + } else { + + // no azimuth if they are in same spot + return 0.0f; + } +} + +float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) { + // TODO: put these in a place where we can share with AudioMixer! + const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; + const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f; + const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f; + const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; + + + //qDebug() << "initial gain is " << volume; + + // I'm assuming that the AudioMixer's getting of the stream's attenuation + // factor is basically same as getting volume + float gain = volume; + float distanceBetween = glm::length(relativePosition); + if (distanceBetween < EPSILON ) { + distanceBetween = EPSILON; + } + + // audio mixer has notion of zones. Unsure how to map that across here... + + // attenuate based on distance now + if (distanceBetween >= ATTENUATION_BEGINS_AT_DISTANCE) { + float distanceCoefficient = 1.0f - (logf(distanceBetween/ATTENUATION_BEGINS_AT_DISTANCE) / logf(2.0f) + * DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE); + if (distanceCoefficient < 0.0f) { + distanceCoefficient = 0.0f; + } + + gain *= distanceCoefficient; + } + + //qDebug() << "calculated gain as " << gain; + + return gain; +} + qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { auto samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; + // limit the number of NETWORK_FRAME_SAMPLES_PER_CHANNEL to return regardless + // of what maxSize requests. + static const qint64 MAX_FRAMES = 256; + static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; + static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; + + // injectors are 24Khz, use this to limit the injector audio only + static AudioLimiter audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + + // final output is 48Khz, so use this to limit network audio plus upsampled injectors + static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + + // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample + static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); + + // limit maxSize + if (maxSize > AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES) { + maxSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES; + } + + // initialize stuff + memset(hrtfBuffer, 0, sizeof(hrtfBuffer)); + QVector injectorsToRemove; + qint64 framesReadFromInjectors = 0; + + //qDebug() << "maxSize=" << maxSize << "data ptr: "<< (qint64)data; + + for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) { + // pop off maxSize/4 (since it is mono, and 24Khz instead of 48Khz) + // bytes from the injector's localBuffer, using the data ptr as a temporary buffer + // note we are only hrtf-ing mono buffers, so we request half of what maxSize is, since it + // wants that many stereo bytes + if (injector->getLocalBuffer() ) { + + glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); + + qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData(data, maxSize/4); + qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t); + if (framesReadFromInjector > framesReadFromInjectors) { + framesReadFromInjectors = framesReadFromInjector; + } + if (framesReadFromInjector > 0) { + + //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; + + // calculate these shortly + float gain = _audio->gainForSource(relativePosition, injector->getVolume()); + float azimuth = _audio->azimuthForSource(relativePosition); + + // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time + this->renderHRTF(injector->getLocalHRTF(), (int16_t*)data, hrtfBuffer, azimuth, gain, framesReadFromInjector); + + } else { + + // probably need to be more clever here than just getting rid of the injector? + //qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove"; + injectorsToRemove.append(injector); + injector->finish(); + } + } else { + + // probably need to be more clever here then just getting rid of injector? + //qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove"; + injectorsToRemove.append(injector); + injector->finish(); + } + } + + if (framesReadFromInjectors > 0) { + // resample the 24Khz injector audio to 48Khz + // but first, hit with limiter :( + audioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors); + audioSRC.render((int16_t*)data, scratchBuffer, framesReadFromInjectors); + + // now, lets move this back into the hrtfBuffer + for(int i=0; igetActiveLocalAudioInjectors().removeOne(injector); + + //qDebug() << "removed injector " << injector << " from active injector list!"; + } + + // now grab the stuff from the network, and mix it in... if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); + + //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples" << "to " << (qint64)data; + bytesWritten = samplesPopped * sizeof(int16_t); - } else { + if (framesReadFromInjectors > 0) { + + int16_t* dataPtr = (int16_t*)data; + // convert audio to floats for mixing with limiter... + //qDebug() << "AudioClient::AudioOUtputIODevice::readData converting to floats..."; + for (int i=0; i bytesWritten) { + bytesWritten = 8*framesReadFromInjectors; + } + + //qDebug() << "done, bytes written = " << bytesWritten; + } + } else if (framesReadFromInjectors == 0) { + // if nobody has data, 0 out buffer... memset(data, 0, maxSize); bytesWritten = maxSize; + } else { + // only sound from injectors, multiply by 2 as the htrf is stereo + finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors*2); + bytesWritten = 8*framesReadFromInjectors; } + int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); if (!bytesAudioOutputUnplayed) { qCDebug(audioclient) << "empty audio buffer"; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index dc46db5657..91987178b0 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -37,6 +37,7 @@ #include #include #include +#include #include "AudioIOStats.h" #include "AudioNoiseGate.h" @@ -86,12 +87,12 @@ public: void stop() { close(); } qint64 readData(char * data, qint64 maxSize); qint64 writeData(const char * data, qint64 maxSize) { return 0; } - int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } private: MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; + void renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples); }; const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } @@ -124,6 +125,8 @@ public: void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } + + QVector& getActiveLocalAudioInjectors() { return _activeLocalAudioInjectors; } static const float CALLBACK_ACCELERATOR_RATIO; @@ -291,6 +294,12 @@ private: void checkDevices(); bool _hasReceivedFirstPacket = false; + + QVector _activeLocalAudioInjectors; + + float azimuthForSource(const glm::vec3& relativePosition); + float gainForSource(const glm::vec3& relativePosition, float volume); + }; diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index c90256429d..a4cde77377 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -26,6 +26,7 @@ #include "AudioInjectorLocalBuffer.h" #include "AudioInjectorOptions.h" +#include "AudioHRTF.h" #include "Sound.h" class AbstractAudioInterface; @@ -54,8 +55,11 @@ public: void setCurrentSendOffset(int currentSendOffset) { _currentSendOffset = currentSendOffset; } AudioInjectorLocalBuffer* getLocalBuffer() const { return _localBuffer; } + AudioHRTF& getLocalHRTF() { return _localHRTF; } + bool isLocalOnly() const { return _options.localOnly; } - + float getVolume() const { return _options.volume; } + glm::vec3 getPosition() const { return _options.position; } void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; } static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface); @@ -74,14 +78,15 @@ public slots: float getLoudness() const { return _loudness; } bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; } + void finish(); signals: void finished(); void restarting(); -private slots: +/*private slots: void finish(); - +*/ private: void setupInjection(); int64_t injectNextFrame(); @@ -103,6 +108,9 @@ private: std::unique_ptr _frameTimer { nullptr }; quint16 _outgoingSequenceNumber { 0 }; + // when the injector is local, we need this + AudioHRTF _localHRTF; + // for local injection, friend class AudioInjectorManager; }; From 7a9b3978e96908034caee4c92c73082cd38bfafa Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 8 Jul 2016 13:01:06 -0700 Subject: [PATCH 02/40] Better processing of audio A couple things Ken suggested. First off, there are now AudioSRC calls for floats, which simplfied stuff a bit. Then, I switched to grabbing network packets first, and only pulling at most that amount of audio from the local injectors. That improves things - the occasional artifacts (when the injectors got more data than the network for instance) are gone. Also, fixed build issue (unused variable warning) for mac and android. --- libraries/audio-client/src/AudioClient.cpp | 168 +++++++++------------ 1 file changed, 71 insertions(+), 97 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 01bd0d3c22..0943585ab7 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1190,9 +1190,7 @@ float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { float AudioClient::gainForSource(const glm::vec3& relativePosition, float volume) { // TODO: put these in a place where we can share with AudioMixer! - const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18f; - const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f; const float ATTENUATION_BEGINS_AT_DISTANCE = 1.0f; @@ -1233,15 +1231,14 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // of what maxSize requests. static const qint64 MAX_FRAMES = 256; static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; + static float mixed48Khz[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES*2]; static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; - // injectors are 24Khz, use this to limit the injector audio only - static AudioLimiter audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - // final output is 48Khz, so use this to limit network audio plus upsampled injectors static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample + // TODO: probably an existing src the AudioClient is appropriate, check! static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); // limit maxSize @@ -1251,121 +1248,98 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // initialize stuff memset(hrtfBuffer, 0, sizeof(hrtfBuffer)); + memset(mixed48Khz, 0, sizeof(mixed48Khz)); QVector injectorsToRemove; qint64 framesReadFromInjectors = 0; - //qDebug() << "maxSize=" << maxSize << "data ptr: "<< (qint64)data; + qDebug() << "maxSize=" << maxSize; - for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) { - // pop off maxSize/4 (since it is mono, and 24Khz instead of 48Khz) - // bytes from the injector's localBuffer, using the data ptr as a temporary buffer - // note we are only hrtf-ing mono buffers, so we request half of what maxSize is, since it - // wants that many stereo bytes - if (injector->getLocalBuffer() ) { - - glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); + // first, grab the stuff from the network. Read into a + if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { + AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); + lastPopOutput.readSamples((int16_t*)data, samplesPopped); - qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData(data, maxSize/4); - qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t); - if (framesReadFromInjector > framesReadFromInjectors) { - framesReadFromInjectors = framesReadFromInjector; - } - if (framesReadFromInjector > 0) { + qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; + + // no matter what, this is how many bytes will be written + bytesWritten = samplesPopped * sizeof(int16_t); + + + // now, grab same number of frames (samplesPopped/2 since network is stereo) + // and divide by 2 again because these are 24Khz not 48Khz + // from each of the localInjectors, if any + for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) { + if (injector->getLocalBuffer() ) { + + // we want to get the same number of frames that we got from + // the network. Since network is stereo, that is samplesPopped/2. + // Since each frame is 2 bytes, we read samplesPopped bytes + qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData((char*)scratchBuffer, samplesPopped/2); + qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t); - //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; + // keep track of max frames read for all injectors + if (framesReadFromInjector > framesReadFromInjectors) { + framesReadFromInjectors = framesReadFromInjector; + } - // calculate these shortly - float gain = _audio->gainForSource(relativePosition, injector->getVolume()); - float azimuth = _audio->azimuthForSource(relativePosition); + if (framesReadFromInjector > 0) { - // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time - this->renderHRTF(injector->getLocalHRTF(), (int16_t*)data, hrtfBuffer, azimuth, gain, framesReadFromInjector); + qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; + // calculate gain and azimuth for hrtf + glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); + float gain = _audio->gainForSource(relativePosition, injector->getVolume()); + float azimuth = _audio->azimuthForSource(relativePosition); + + // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time + this->renderHRTF(injector->getLocalHRTF(), scratchBuffer, hrtfBuffer, azimuth, gain, framesReadFromInjector); + + } else { + + qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove"; + injectorsToRemove.append(injector); + injector->finish(); + } } else { - // probably need to be more clever here than just getting rid of the injector? - //qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove"; + qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove"; injectorsToRemove.append(injector); injector->finish(); } - } else { - - // probably need to be more clever here then just getting rid of injector? - //qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove"; - injectorsToRemove.append(injector); - injector->finish(); } - } - - if (framesReadFromInjectors > 0) { - // resample the 24Khz injector audio to 48Khz - // but first, hit with limiter :( - audioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors); - audioSRC.render((int16_t*)data, scratchBuffer, framesReadFromInjectors); - - // now, lets move this back into the hrtfBuffer - for(int i=0; i 0) { + // resample the 24Khz injector audio in hrtfBuffer to 48Khz + audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors); + + qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo"; + + int16_t* dataPtr = (int16_t*)data; + + // now mix in the network data + for (int i=0; igetActiveLocalAudioInjectors().removeOne(injector); - - //qDebug() << "removed injector " << injector << " from active injector list!"; - } - - // now grab the stuff from the network, and mix it in... - if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { - AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); - lastPopOutput.readSamples((int16_t*)data, samplesPopped); - - //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples" << "to " << (qint64)data; - - bytesWritten = samplesPopped * sizeof(int16_t); - if (framesReadFromInjectors > 0) { - - int16_t* dataPtr = (int16_t*)data; - // convert audio to floats for mixing with limiter... - //qDebug() << "AudioClient::AudioOUtputIODevice::readData converting to floats..."; - for (int i=0; i bytesWritten) { - bytesWritten = 8*framesReadFromInjectors; - } - - //qDebug() << "done, bytes written = " << bytesWritten; - } - } else if (framesReadFromInjectors == 0) { - // if nobody has data, 0 out buffer... - memset(data, 0, maxSize); - bytesWritten = maxSize; - } else { - // only sound from injectors, multiply by 2 as the htrf is stereo - finalAudioLimiter.render(hrtfBuffer, (int16_t*)data, framesReadFromInjectors*2); - bytesWritten = 8*framesReadFromInjectors; + qDebug() << "removed injector " << injector << " from active injector list!"; } + qDebug() << "bytesWritten=" << bytesWritten; int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); if (!bytesAudioOutputUnplayed) { From bc7123d701ba33135c3c0feb9e2e09482153c1fe Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 8 Jul 2016 13:59:49 -0700 Subject: [PATCH 03/40] comment out logging for now --- libraries/audio-client/src/AudioClient.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0943585ab7..67fb1ff561 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1252,14 +1252,14 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { QVector injectorsToRemove; qint64 framesReadFromInjectors = 0; - qDebug() << "maxSize=" << maxSize; + //qDebug() << "maxSize=" << maxSize; // first, grab the stuff from the network. Read into a if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); - qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; + //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; // no matter what, this is how many bytes will be written bytesWritten = samplesPopped * sizeof(int16_t); @@ -1284,7 +1284,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { if (framesReadFromInjector > 0) { - qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; + //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; // calculate gain and azimuth for hrtf glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); @@ -1312,7 +1312,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // resample the 24Khz injector audio in hrtfBuffer to 48Khz audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors); - qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo"; + //qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo"; int16_t* dataPtr = (int16_t*)data; @@ -1321,7 +1321,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { mixed48Khz[i] += (float)(*dataPtr++) * 1/32676.0f; } - qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ; + //qDebug() << "mixed network data into upsampled hrtf'd mixed injector data" ; finalAudioLimiter.render(mixed48Khz, (int16_t*)data, samplesPopped/2); } @@ -1336,10 +1336,10 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // k get rid of finished injectors for (AudioInjector* injector : injectorsToRemove) { _audio->getActiveLocalAudioInjectors().removeOne(injector); - qDebug() << "removed injector " << injector << " from active injector list!"; + //qDebug() << "removed injector " << injector << " from active injector list!"; } - qDebug() << "bytesWritten=" << bytesWritten; + //qDebug() << "bytesWritten=" << bytesWritten; int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); if (!bytesAudioOutputUnplayed) { From 80d33ee2516616beb3b08d9279e812686e37a866 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Fri, 8 Jul 2016 13:59:49 -0700 Subject: [PATCH 04/40] Working now as frame-at-a-time So localOnly sounds get HRTF'd, one network frame at a time. Less processing (upsampling, limiting, etc...) than doing this at the end of the pipeline (in the AudioOutputIODevice::readData call). --- libraries/audio-client/src/AudioClient.cpp | 199 ++++++++------------- libraries/audio-client/src/AudioClient.h | 20 ++- 2 files changed, 87 insertions(+), 132 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 67fb1ff561..190dd1583b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -43,10 +43,6 @@ #include #include -#include "AudioInjector.h" -#include "AudioLimiter.h" -#include "AudioHRTF.h" -#include "AudioConstants.h" #include "PositionalAudioStream.h" #include "AudioClientLogging.h" @@ -111,7 +107,8 @@ AudioClient::AudioClient() : _stats(&_receivedAudioStream), _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), - _orientationGetter(DEFAULT_ORIENTATION_GETTER) + _orientationGetter(DEFAULT_ORIENTATION_GETTER), + _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); @@ -787,15 +784,84 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { emitAudioPacket(audio.data(), audio.size(), _outgoingAvatarAudioSequenceNumber, audioTransform, PacketType::MicrophoneAudioWithEcho); } +void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { + + memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer)); + QVector injectorsToRemove; + + bool injectorsHaveData = false; + + for (AudioInjector* injector : getActiveLocalAudioInjectors()) { + if (injector->getLocalBuffer()) { + + // get one (mono) frame from the injector + if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*sizeof(int16_t)) ) { + + injectorsHaveData = true; + + // calculate gain and azimuth for hrtf + glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); + float gain = gainForSource(relativePosition, injector->getVolume()); + float azimuth = azimuthForSource(relativePosition); + + + injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + } else { + + qDebug() << "injector has no more data, marking finished for removal"; + + injector->finish(); + injectorsToRemove.append(injector); + } + + } else { + + qDebug() << "injector has no local buffer, marking as finished for removal"; + + injector->finish(); + injectorsToRemove.append(injector); + } + } + + if(injectorsHaveData) { + + // mix network into the hrtfBuffer + for(int i=0; i(inputBuffer.data()); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); + const int16_t* receivedSamples; + char inputBufferCopy[AudioConstants::NETWORK_FRAME_BYTES_STEREO]; + assert(inputBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); + + if(getActiveLocalAudioInjectors().size() > 0) { + // gotta copy the since it is const + for(int i = 0; i < sizeof(inputBufferCopy); i++) { + inputBufferCopy[i] = inputBuffer.data()[i]; + } + mixLocalAudioInjectors((int16_t*)inputBufferCopy); + receivedSamples = reinterpret_cast(inputBufferCopy); + } else { + receivedSamples = reinterpret_cast(inputBuffer.data()); + } // copy the packet from the RB to the output possibleResampling(_networkToOutputResampler, receivedSamples, outputSamples, @@ -807,6 +873,7 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr if (hasReverb) { assert(_outputFormat.channelCount() == 2); updateReverbOptions(); + qDebug() << "handling reverb"; _listenerReverb.render(outputSamples, outputSamples, numDeviceOutputSamples/2); } } @@ -1148,22 +1215,6 @@ float AudioClient::getAudioOutputMsecsUnplayed() const { return msecsAudioOutputUnplayed; } -void AudioClient::AudioOutputIODevice::renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples) { - qint64 numFrames = numSamples/AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - int16_t* dataPtr = data; - float* hrtfPtr = hrtfBuffer; - for(qint64 i=0; i < numFrames; i++) { - hrtf.render(dataPtr, hrtfPtr, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); - - //qDebug() << "processed frame " << i; - - dataPtr += AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - hrtfPtr += 2*AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; - - } - assert(dataPtr - data <= numSamples); - assert(hrtfPtr - hrtfBuffer <= 2*numSamples); -} float AudioClient::azimuthForSource(const glm::vec3& relativePosition) { // copied from AudioMixer, more or less @@ -1226,105 +1277,11 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { auto samplesRequested = maxSize / sizeof(int16_t); int samplesPopped; int bytesWritten; - - // limit the number of NETWORK_FRAME_SAMPLES_PER_CHANNEL to return regardless - // of what maxSize requests. - static const qint64 MAX_FRAMES = 256; - static float hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; - static float mixed48Khz[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES*2]; - static int16_t scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*MAX_FRAMES]; - // final output is 48Khz, so use this to limit network audio plus upsampled injectors - static AudioLimiter finalAudioLimiter(2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - - // Injectors are 24khz, but we are running at 48Khz here, so need an AudioSRC to upsample - // TODO: probably an existing src the AudioClient is appropriate, check! - static AudioSRC audioSRC(AudioConstants::SAMPLE_RATE, 2*AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); - - // limit maxSize - if (maxSize > AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES) { - maxSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * MAX_FRAMES; - } - - // initialize stuff - memset(hrtfBuffer, 0, sizeof(hrtfBuffer)); - memset(mixed48Khz, 0, sizeof(mixed48Khz)); - QVector injectorsToRemove; - qint64 framesReadFromInjectors = 0; - - //qDebug() << "maxSize=" << maxSize; - - // first, grab the stuff from the network. Read into a if ((samplesPopped = _receivedAudioStream.popSamples((int)samplesRequested, false)) > 0) { AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); lastPopOutput.readSamples((int16_t*)data, samplesPopped); - - //qDebug() << "AudioClient::AudioOutputIODevice::readData popped " << samplesPopped << "samples"; - - // no matter what, this is how many bytes will be written bytesWritten = samplesPopped * sizeof(int16_t); - - - // now, grab same number of frames (samplesPopped/2 since network is stereo) - // and divide by 2 again because these are 24Khz not 48Khz - // from each of the localInjectors, if any - for (AudioInjector* injector : _audio->getActiveLocalAudioInjectors()) { - if (injector->getLocalBuffer() ) { - - // we want to get the same number of frames that we got from - // the network. Since network is stereo, that is samplesPopped/2. - // Since each frame is 2 bytes, we read samplesPopped bytes - qint64 bytesReadFromInjector = injector->getLocalBuffer()->readData((char*)scratchBuffer, samplesPopped/2); - qint64 framesReadFromInjector = bytesReadFromInjector/sizeof(int16_t); - - // keep track of max frames read for all injectors - if (framesReadFromInjector > framesReadFromInjectors) { - framesReadFromInjectors = framesReadFromInjector; - } - - if (framesReadFromInjector > 0) { - - //qDebug() << "AudioClient::AudioOutputIODevice::readData found " << framesReadFromInjector << " frames"; - - // calculate gain and azimuth for hrtf - glm::vec3 relativePosition = injector->getPosition() - _audio->_positionGetter(); - float gain = _audio->gainForSource(relativePosition, injector->getVolume()); - float azimuth = _audio->azimuthForSource(relativePosition); - - // now hrtf this guy, one NETWORK_FRAME_SAMPLES_PER_CHANNEL at a time - this->renderHRTF(injector->getLocalHRTF(), scratchBuffer, hrtfBuffer, azimuth, gain, framesReadFromInjector); - - } else { - - qDebug() << "AudioClient::AudioOutputIODevice::readData no more data, adding to list of injectors to remove"; - injectorsToRemove.append(injector); - injector->finish(); - } - } else { - - qDebug() << "AudioClient::AudioOutputIODevice::readData injector " << injector << " has no local buffer, adding to list of injectors to remove"; - injectorsToRemove.append(injector); - injector->finish(); - } - } - - if (framesReadFromInjectors > 0) { - // resample the 24Khz injector audio in hrtfBuffer to 48Khz - audioSRC.render(hrtfBuffer, mixed48Khz, framesReadFromInjectors); - - //qDebug() << "upsampled to" << framesReadFromInjectors*2 << " frames, stereo"; - - int16_t* dataPtr = (int16_t*)data; - - // now mix in the network data - for (int i=0; igetActiveLocalAudioInjectors().removeOne(injector); - //qDebug() << "removed injector " << injector << " from active injector list!"; - } - - //qDebug() << "bytesWritten=" << bytesWritten; - int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); if (!bytesAudioOutputUnplayed) { qCDebug(audioclient) << "empty audio buffer"; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 91987178b0..3630358771 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -38,11 +38,14 @@ #include #include #include +#include +#include +#include +#include +#include #include "AudioIOStats.h" #include "AudioNoiseGate.h" -#include "AudioSRC.h" -#include "AudioReverb.h" #ifdef _WIN32 #pragma warning( push ) @@ -92,7 +95,6 @@ public: MixedProcessedAudioStream& _receivedAudioStream; AudioClient* _audio; int _unfulfilledReads; - void renderHRTF(AudioHRTF& hrtf, int16_t* data, float* hrtfBuffer, float azimuth, float gain, qint64 numSamples); }; const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } @@ -208,6 +210,9 @@ protected: private: void outputFormatChanged(); + void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer); + float azimuthForSource(const glm::vec3& relativePosition); + float gainForSource(const glm::vec3& relativePosition, float volume); QByteArray firstInputFrame; QAudioInput* _audioInput; @@ -264,6 +269,11 @@ private: AudioSRC* _inputToNetworkResampler; AudioSRC* _networkToOutputResampler; + // for local hrtf-ing + float _hrtfBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _scratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + AudioLimiter _audioLimiter; + // Adds Reverb void configureReverb(); void updateReverbOptions(); @@ -296,10 +306,6 @@ private: bool _hasReceivedFirstPacket = false; QVector _activeLocalAudioInjectors; - - float azimuthForSource(const glm::vec3& relativePosition); - float gainForSource(const glm::vec3& relativePosition, float volume); - }; From 46dc5bea0404526d1e487c43d19ee7a325c8ea89 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Mon, 11 Jul 2016 09:31:05 -0700 Subject: [PATCH 05/40] cleanup from PR comments --- libraries/audio-client/src/AudioClient.cpp | 6 +++--- libraries/audio/src/AudioInjector.h | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 190dd1583b..b83165eac6 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -939,11 +939,11 @@ bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) { // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle, // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread. connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) { - if (state == QAudio::IdleState) { + if (state == QAudio::IdleState) { localOutput->stop(); injector->stop(); - } - }); + } + }); connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop); diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index a4cde77377..7560863b03 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -84,9 +84,6 @@ signals: void finished(); void restarting(); -/*private slots: - void finish(); -*/ private: void setupInjection(); int64_t injectNextFrame(); @@ -110,7 +107,6 @@ private: // when the injector is local, we need this AudioHRTF _localHRTF; - // for local injection, friend class AudioInjectorManager; }; From 4f80c77b77b6ae808e4b8a33eb6405a5092e6f19 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Mon, 11 Jul 2016 10:36:55 -0700 Subject: [PATCH 06/40] PR feedback simple enough - not sure why I had an issue with this in the first place. --- libraries/audio-client/src/AudioClient.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index b83165eac6..fad34f01de 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -849,16 +849,12 @@ void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArr outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t)); int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); const int16_t* receivedSamples; - char inputBufferCopy[AudioConstants::NETWORK_FRAME_BYTES_STEREO]; + QByteArray inputBufferCopy = inputBuffer; assert(inputBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); if(getActiveLocalAudioInjectors().size() > 0) { - // gotta copy the since it is const - for(int i = 0; i < sizeof(inputBufferCopy); i++) { - inputBufferCopy[i] = inputBuffer.data()[i]; - } - mixLocalAudioInjectors((int16_t*)inputBufferCopy); - receivedSamples = reinterpret_cast(inputBufferCopy); + mixLocalAudioInjectors((int16_t*)inputBufferCopy.data()); + receivedSamples = reinterpret_cast(inputBufferCopy.data()); } else { receivedSamples = reinterpret_cast(inputBuffer.data()); } From c84ef5f62688f181b6e99ee9ceb19d5205c003da Mon Sep 17 00:00:00 2001 From: David Kelly Date: Mon, 11 Jul 2016 11:31:47 -0700 Subject: [PATCH 07/40] Stereo injectors now handled same as mono Except of course no HRTF. --- libraries/audio-client/src/AudioClient.cpp | 65 ++++++++-------------- libraries/audio/src/AudioInjector.h | 3 +- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index fad34f01de..3178a53e94 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -788,24 +788,37 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { memset(_hrtfBuffer, 0, sizeof(_hrtfBuffer)); QVector injectorsToRemove; + static const float INT16_TO_FLOAT_SCALE_FACTOR = 1/32768.0f; bool injectorsHaveData = false; for (AudioInjector* injector : getActiveLocalAudioInjectors()) { if (injector->getLocalBuffer()) { - // get one (mono) frame from the injector - if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL*sizeof(int16_t)) ) { + qint64 samplesToRead = injector->isStereo() ? + AudioConstants::NETWORK_FRAME_BYTES_STEREO : + AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + + // get one frame from the injector (mono or stereo) + if (0 < injector->getLocalBuffer()->readData((char*)_scratchBuffer, samplesToRead)) { injectorsHaveData = true; - // calculate gain and azimuth for hrtf - glm::vec3 relativePosition = injector->getPosition() - _positionGetter(); - float gain = gainForSource(relativePosition, injector->getVolume()); - float azimuth = azimuthForSource(relativePosition); + if (injector->isStereo() ) { + for(int i=0; igetPosition() - _positionGetter(); + float gain = gainForSource(relativePosition, injector->getVolume()); + float azimuth = azimuthForSource(relativePosition); - injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + injector->getLocalHRTF().render(_scratchBuffer, _hrtfBuffer, 1, azimuth, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } } else { @@ -828,7 +841,7 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { // mix network into the hrtfBuffer for(int i=0; igetLocalBuffer() && _audioInput ) { - if(isStereo) { - QAudioFormat localFormat = _desiredOutputFormat; - localFormat.setChannelCount(isStereo ? 2 : 1); - - QAudioOutput* localOutput = new QAudioOutput(getNamedAudioDeviceForMode(QAudio::AudioOutput, _outputAudioDeviceName), - localFormat, - injector->getLocalBuffer()); - - // move the localOutput to the same thread as the local injector buffer - localOutput->moveToThread(injector->getLocalBuffer()->thread()); - - // have it be stopped when that local buffer is about to close - // We don't want to stop this localOutput and injector whenever this AudioClient singleton goes idle, - // only when the localOutput does. But the connection is to localOutput, so that it happens on the right thread. - connect(localOutput, &QAudioOutput::stateChanged, localOutput, [=](QAudio::State state) { - if (state == QAudio::IdleState) { - localOutput->stop(); - injector->stop(); - } - }); - - connect(injector->getLocalBuffer(), &QIODevice::aboutToClose, localOutput, &QAudioOutput::stop); - - qCDebug(audioclient) << "Starting QAudioOutput for local injector" << localOutput; - - localOutput->start(injector->getLocalBuffer()); - return localOutput->state() == QAudio::ActiveState; - } else { // just add it to the vector of active local injectors // TODO: deal with concurrency perhaps? Maybe not - qDebug() << "adding new injector!!!!!!!"; - - _activeLocalAudioInjectors.append(injector); - return true; - } + qDebug() << "adding new injector!!!!!!!"; + _activeLocalAudioInjectors.append(injector); + return true; } return false; diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 7560863b03..c74497ce4d 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -59,7 +59,8 @@ public: bool isLocalOnly() const { return _options.localOnly; } float getVolume() const { return _options.volume; } - glm::vec3 getPosition() const { return _options.position; } + glm::vec3 getPosition() const { return _options.position; } + bool isStereo() const { return _options.stereo; } void setLocalAudioInterface(AbstractAudioInterface* localAudioInterface) { _localAudioInterface = localAudioInterface; } static AudioInjector* playSoundAndDelete(const QByteArray& buffer, const AudioInjectorOptions options, AbstractAudioInterface* localInterface); From 3955e03aa2fa5a770e034c3729477222cc77b2e8 Mon Sep 17 00:00:00 2001 From: David Kelly Date: Tue, 12 Jul 2016 11:38:30 -0700 Subject: [PATCH 08/40] Fix for build error doh! --- libraries/audio-client/src/AudioClient.cpp | 4 ++-- libraries/audio-client/src/AudioClient.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 3178a53e94..cb53e8f5de 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -102,13 +102,13 @@ AudioClient::AudioClient() : _reverbOptions(&_scriptReverbOptions), _inputToNetworkResampler(NULL), _networkToOutputResampler(NULL), + _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO), _outgoingAvatarAudioSequenceNumber(0), _audioOutputIODevice(_receivedAudioStream, this), _stats(&_receivedAudioStream), _inputGate(), _positionGetter(DEFAULT_POSITION_GETTER), - _orientationGetter(DEFAULT_ORIENTATION_GETTER), - _audioLimiter(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO) + _orientationGetter(DEFAULT_ORIENTATION_GETTER) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 3630358771..49c3a40627 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -210,7 +210,7 @@ protected: private: void outputFormatChanged(); - void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer); + void mixLocalAudioInjectors(int16_t* inputBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(const glm::vec3& relativePosition, float volume); From c951f507e3be2383a88b43c48e69201ce653ecdc Mon Sep 17 00:00:00 2001 From: David Kelly Date: Wed, 13 Jul 2016 11:28:36 -0700 Subject: [PATCH 09/40] Fixed issue with calling setOptions Since the stereo option is computed from the .wav file, if you call setOptions later (like the cow.js does), it resets stereo to false. So, I now just copy the stereo flag into the new options, since the sound file is the same. Also, calling AudioClient::outputLocalInjector on the AudioClient thread now to avoid potential concurrency issues accessing the vector of injectors. --- libraries/audio-client/src/AudioClient.cpp | 25 ++++++++++++++-------- libraries/audio/src/AudioInjector.cpp | 20 +++++++++++++---- libraries/audio/src/AudioInjector.h | 3 +-- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index cb53e8f5de..799c357944 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -823,7 +823,6 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { } else { qDebug() << "injector has no more data, marking finished for removal"; - injector->finish(); injectorsToRemove.append(injector); } @@ -831,7 +830,6 @@ void AudioClient::mixLocalAudioInjectors(int16_t* inputBuffer) { } else { qDebug() << "injector has no local buffer, marking as finished for removal"; - injector->finish(); injectorsToRemove.append(injector); } @@ -933,15 +931,24 @@ void AudioClient::setIsStereoInput(bool isStereoInput) { bool AudioClient::outputLocalInjector(bool isStereo, AudioInjector* injector) { if (injector->getLocalBuffer() && _audioInput ) { - // just add it to the vector of active local injectors - // TODO: deal with concurrency perhaps? Maybe not - qDebug() << "adding new injector!!!!!!!"; - - _activeLocalAudioInjectors.append(injector); + // just add it to the vector of active local injectors, if + // not already there. + // Since this is invoked with invokeMethod, there _should_ be + // no reason to lock access to the vector of injectors. + if (!_activeLocalAudioInjectors.contains(injector)) { + qDebug() << "adding new injector"; + + _activeLocalAudioInjectors.append(injector); + } else { + qDebug() << "injector exists in active list already"; + } + return true; - } - return false; + } else { + // no local buffer or audio + return false; + } } void AudioClient::outputFormatChanged() { diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 878a4c627c..d4382340bf 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -26,6 +26,8 @@ #include "AudioInjector.h" +int audioInjectorPtrMetaTypeId = qRegisterMetaType(); + AudioInjector::AudioInjector(QObject* parent) : QObject(parent) { @@ -41,11 +43,20 @@ AudioInjector::AudioInjector(const Sound& sound, const AudioInjectorOptions& inj AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) : _audioData(audioData), - _options(injectorOptions) + _options(injectorOptions) { } +void AudioInjector::setOptions(const AudioInjectorOptions& options) { + // since options.stereo is computed from the audio stream, + // we need to copy it from existing options just in case. + bool currentlyStereo = _options.stereo; + _options = options; + _options.stereo = currentlyStereo; +} + + void AudioInjector::finish() { bool shouldDelete = (_state == State::NotFinishedWithPendingDelete); _state = State::Finished; @@ -115,11 +126,11 @@ void AudioInjector::restart() { _hasSetup = false; _shouldStop = false; _state = State::NotFinished; - + // call inject audio to start injection over again setupInjection(); - // if we're a local injector call inject locally to start injecting again + // if we're a local injector, just inject again if (_options.localOnly) { injectLocally(); } else { @@ -145,7 +156,8 @@ bool AudioInjector::injectLocally() { // give our current send position to the local buffer _localBuffer->setCurrentOffset(_currentSendOffset); - success = _localAudioInterface->outputLocalInjector(_options.stereo, this); + // call this function on the AudioClient's thread + success = QMetaObject::invokeMethod(_localAudioInterface, "outputLocalInjector", Q_ARG(bool, _options.stereo), Q_ARG(AudioInjector*, this)); if (!success) { qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface"; diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index c74497ce4d..1af733ace6 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -75,7 +75,7 @@ public slots: void stopAndDeleteLater(); const AudioInjectorOptions& getOptions() const { return _options; } - void setOptions(const AudioInjectorOptions& options) { _options = options; } + void setOptions(const AudioInjectorOptions& options); float getLoudness() const { return _loudness; } bool isPlaying() const { return _state == State::NotFinished || _state == State::NotFinishedWithPendingDelete; } @@ -111,5 +111,4 @@ private: friend class AudioInjectorManager; }; - #endif // hifi_AudioInjector_h From a134ac72de1c9f08b46c916f6d8825d56487e59a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 12:08:26 -0700 Subject: [PATCH 10/40] add packet type and plumbing for node ignore --- assignment-client/src/audio/AudioMixer.cpp | 8 +++++++- assignment-client/src/audio/AudioMixer.h | 1 + assignment-client/src/avatars/AvatarMixer.cpp | 18 ++++++++++++------ assignment-client/src/avatars/AvatarMixer.h | 2 ++ libraries/networking/src/LimitedNodeList.cpp | 16 ++++++++++++---- libraries/networking/src/LimitedNodeList.h | 1 + libraries/networking/src/Node.cpp | 8 ++++++++ libraries/networking/src/Node.h | 8 ++++++++ libraries/networking/src/udt/PacketHeaders.h | 2 +- 9 files changed, 52 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 51c4e25410..002f32a60b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -93,6 +93,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : this, "handleNodeAudioPacket"); packetReceiver.registerListener(PacketType::MuteEnvironment, this, "handleMuteEnvironmentPacket"); packetReceiver.registerListener(PacketType::NegotiateAudioFormat, this, "handleNegotiateAudioFormat"); + packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -324,7 +325,8 @@ bool AudioMixer::prepareMixForListeningNode(Node* node) { // loop through all other nodes that have sufficient audio to mix DependencyManager::get()->eachNode([&](const SharedNodePointer& otherNode){ - if (otherNode->getLinkedData()) { + // make sure that we have audio data for this other node and that it isn't being ignored by our listening node + if (otherNode->getLinkedData() && !node->isIgnoringNodeWithID(otherNode->getUUID())) { AudioMixerClientData* otherNodeClientData = (AudioMixerClientData*) otherNode->getLinkedData(); // enumerate the ARBs attached to the otherNode and add all that should be added to mix @@ -554,6 +556,10 @@ void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { }); } +void AudioMixer::handleNodeIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { + sendingNode->handleNodeIgnoreRequest(packet); +} + void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { auto injectorClientData = qobject_cast(sender()); if (injectorClientData) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 4b2a27120d..9b26989847 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -47,6 +47,7 @@ private slots: void handleMuteEnvironmentPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleNegotiateAudioFormat(QSharedPointer message, SharedNodePointer sendingNode); void handleNodeKilled(SharedNodePointer killedNode); + void handleNodeIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void removeHRTFsForFinishedInjector(const QUuid& streamID); diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 46599396ca..7930ab843f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -45,6 +45,7 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::AvatarData, this, "handleAvatarDataPacket"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "handleAvatarIdentityPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); + packetReceiver.registerListener(PacketType::NodeIgnoreRequest, this, "handleNodeIgnoreRequestPacket"); auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); @@ -227,14 +228,15 @@ void AvatarMixer::broadcastAvatarData() { // send back a packet with other active node data to this node nodeList->eachMatchingNode( [&](const SharedNodePointer& otherNode)->bool { - if (!otherNode->getLinkedData()) { + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + if (!otherNode->getLinkedData() + || otherNode->getUUID() == node->getUUID() + || node->isIgnoringNodeWithID(otherNode->getUUID())) { return false; + } else { + return true; } - if (otherNode->getUUID() == node->getUUID()) { - return false; - } - - return true; }, [&](const SharedNodePointer& otherNode) { ++numOtherAvatars; @@ -431,6 +433,10 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message DependencyManager::get()->processKillNode(*message); } +void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { + senderNode->handleNodeIgnoreRequest(message); +} + void AvatarMixer::sendStatsPacket() { QJsonObject statsObject; statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index d1a9249c83..00cf457d40 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -37,9 +37,11 @@ private slots: void handleAvatarDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); void handleKillAvatarPacket(QSharedPointer message); + void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); + private: void broadcastAvatarData(); void parseDomainServerSettings(const QJsonObject& domainSettings); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index a03fa43093..2f085dc3cb 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -427,10 +427,7 @@ qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& int LimitedNodeList::updateNodeWithDataFromPacket(QSharedPointer message, SharedNodePointer sendingNode) { QMutexLocker locker(&sendingNode->getMutex()); - NodeData* linkedData = sendingNode->getLinkedData(); - if (!linkedData && linkedDataCreateCallback) { - linkedDataCreateCallback(sendingNode.data()); - } + NodeData* linkedData = getOrCreateLinkedData(sendingNode); if (linkedData) { QMutexLocker linkedDataLocker(&linkedData->getMutex()); @@ -440,6 +437,17 @@ int LimitedNodeList::updateNodeWithDataFromPacket(QSharedPointergetMutex()); + + NodeData* linkedData = node->getLinkedData(); + if (!linkedData && linkedDataCreateCallback) { + linkedDataCreateCallback(node.data()); + } + + return node->getLinkedData(); +} + SharedNodePointer LimitedNodeList::nodeWithUUID(const QUuid& nodeUUID) { QReadLocker readLocker(&_nodeMutex); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index c9054ac6d7..03e82f053f 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -149,6 +149,7 @@ public: void processKillNode(ReceivedMessage& message); int updateNodeWithDataFromPacket(QSharedPointer packet, SharedNodePointer matchingNode); + NodeData* getOrCreateLinkedData(SharedNodePointer node); unsigned int broadcastToNodes(std::unique_ptr packet, const NodeSet& destinationNodeTypes); SharedNodePointer soloNodeOfType(NodeType_t nodeType); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 7201b2fd9a..c95b9d5129 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -78,6 +78,14 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { _clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile(); } +void Node::handleNodeIgnoreRequest(QSharedPointer packet) { + // parse out the UUID being ignored from the packet + QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + // add the session UUID to the set of ignored ones for this listening node + _ignoredNodeIDSet.insert(ignoredUUID); +} + QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._type; out << node._uuid; diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index b277ac0083..daebc1fcfb 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -21,6 +21,10 @@ #include #include +#include + +#include + #include "HifiSockAddr.h" #include "NetworkPeer.h" #include "NodeData.h" @@ -66,6 +70,9 @@ public: bool getCanRezTmp() const { return _permissions.canRezTemporaryEntities; } bool getCanWriteToAssetServer() const { return _permissions.canWriteToAssetServer; } + void handleNodeIgnoreRequest(QSharedPointer packet); + bool isIgnoringNodeWithID(const QUuid& nodeID) const { return _ignoredNodeIDSet.find(nodeID) != _ignoredNodeIDSet.cend(); } + friend QDataStream& operator<<(QDataStream& out, const Node& node); friend QDataStream& operator>>(QDataStream& in, Node& node); @@ -84,6 +91,7 @@ private: QMutex _mutex; MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; + tbb::concurrent_unordered_set _ignoredNodeIDSet; }; Q_DECLARE_METATYPE(Node*) diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 296ebb8e68..ac40f8100b 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -61,7 +61,7 @@ public: AssignmentClientStatus, NoisyMute, AvatarIdentity, - TYPE_UNUSED_1, + NodeIgnoreRequest, DomainConnectRequest, DomainServerRequireDTLS, NodeJsonStats, From bb68e777e634aab8729b80b9cfb4061de20be6e8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 13:49:31 -0700 Subject: [PATCH 11/40] add a scripting interface to ignore users --- interface/src/Application.cpp | 3 ++ libraries/networking/src/Node.cpp | 2 + .../networking/src/udt/PacketHeaders.cpp | 2 - libraries/networking/src/udt/PacketHeaders.h | 1 - .../src/UsersScriptingInterface.cpp | 39 +++++++++++++++++++ .../src/UsersScriptingInterface.h | 28 +++++++++++++ 6 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 libraries/script-engine/src/UsersScriptingInterface.cpp create mode 100644 libraries/script-engine/src/UsersScriptingInterface.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 48bbccc3b6..a4ae0af1c5 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -98,6 +98,7 @@ #include #include #include +#include #include #include #include @@ -454,6 +455,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); #if defined(Q_OS_MAC) || defined(Q_OS_WIN) DependencyManager::set(); @@ -4755,6 +4757,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerGlobalObject("Reticle", getApplicationCompositor().getReticleInterface()); scriptEngine->registerGlobalObject("UserActivityLogger", DependencyManager::get().data()); + scriptEngine->registerGlobalObject("Users", DependencyManager::get().data()); } bool Application::canAcceptURL(const QString& urlString) const { diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index c95b9d5129..27598d10ae 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -82,6 +82,8 @@ void Node::handleNodeIgnoreRequest(QSharedPointer packet) { // parse out the UUID being ignored from the packet QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + qDebug() << "Adding" << ignoredUUID << "to ignore set for" << _uuid; + // add the session UUID to the set of ignored ones for this listening node _ignoredNodeIDSet.insert(ignoredUUID); } diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 6359ad0aff..99e10caabd 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -40,8 +40,6 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode; -const QSet RELIABLE_PACKETS = QSet(); - PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { case PacketType::DomainList: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index ac40f8100b..9581f3ca20 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -109,7 +109,6 @@ typedef char PacketVersion; extern const QSet NON_VERIFIED_PACKETS; extern const QSet NON_SOURCED_PACKETS; -extern const QSet RELIABLE_PACKETS; PacketVersion versionForPacketType(PacketType packetType); QByteArray protocolVersionsSignature(); /// returns a unqiue signature for all the current protocols diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp new file mode 100644 index 0000000000..3a05d81de1 --- /dev/null +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -0,0 +1,39 @@ +// +// UsersScriptingInterface.cpp +// libraries/script-engine/src +// +// Created by Stephen Birarda on 2016-07-11. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "UsersScriptingInterface.h" + +#include + +void UsersScriptingInterface::ignore(const QUuid& nodeID) { + // setup the ignore packet we send to all nodes (that currently handle it) + // to ignore the data (audio/avatar) for this user + + // enumerate the nodes to send a reliable ignore packet to each that can leverage it + auto nodeList = DependencyManager::get(); + + nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { + if (node->getType() != NodeType::AudioMixer || node->getType() != NodeType::AvatarMixer) { + return false; + } else { + return true; + } + }, [&nodeID, &nodeList](const SharedNodePointer& destinationNode) { + // create a reliable NLPacket with space for the ignore UUID + auto ignorePacket = NLPacket::create(PacketType::NodeIgnoreRequest, NUM_BYTES_RFC4122_UUID, true); + + // write the node ID to the packet + ignorePacket->write(nodeID.toRfc4122()); + + // send off this ignore packet reliably to the matching node + nodeList->sendPacket(std::move(ignorePacket), *destinationNode); + }); +} diff --git a/libraries/script-engine/src/UsersScriptingInterface.h b/libraries/script-engine/src/UsersScriptingInterface.h new file mode 100644 index 0000000000..0dc62c088c --- /dev/null +++ b/libraries/script-engine/src/UsersScriptingInterface.h @@ -0,0 +1,28 @@ +// +// UsersScriptingInterface.h +// libraries/script-engine/src +// +// Created by Stephen Birarda on 2016-07-11. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once + +#ifndef hifi_UsersScriptingInterface_h +#define hifi_UsersScriptingInterface_h + +#include + +class UsersScriptingInterface : public QObject, public Dependency { + Q_OBJECT + SINGLETON_DEPENDENCY + +public slots: + void ignore(const QUuid& nodeID); +}; + + +#endif // hifi_UsersScriptingInterface_h From 441b6d2813fcbdb672ee9f652c96dc07d04ea619 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 14:05:48 -0700 Subject: [PATCH 12/40] fix recursive mutex lock, conditional, logging --- libraries/networking/src/LimitedNodeList.cpp | 1 - libraries/networking/src/Node.cpp | 3 ++- libraries/script-engine/src/UsersScriptingInterface.cpp | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 2f085dc3cb..e2d6b277a7 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -425,7 +425,6 @@ qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& } int LimitedNodeList::updateNodeWithDataFromPacket(QSharedPointer message, SharedNodePointer sendingNode) { - QMutexLocker locker(&sendingNode->getMutex()); NodeData* linkedData = getOrCreateLinkedData(sendingNode); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 27598d10ae..4615c61506 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -82,7 +82,8 @@ void Node::handleNodeIgnoreRequest(QSharedPointer packet) { // parse out the UUID being ignored from the packet QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - qDebug() << "Adding" << ignoredUUID << "to ignore set for" << _uuid; + qDebug() << "Adding" << uuidStringWithoutCurlyBraces(ignoredUUID) << "to ignore set for" + << uuidStringWithoutCurlyBraces(_uuid); // add the session UUID to the set of ignored ones for this listening node _ignoredNodeIDSet.insert(ignoredUUID); diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp index 3a05d81de1..94c448ac86 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.cpp +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -21,10 +21,10 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { - if (node->getType() != NodeType::AudioMixer || node->getType() != NodeType::AvatarMixer) { - return false; - } else { + if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { return true; + } else { + return false; } }, [&nodeID, &nodeList](const SharedNodePointer& destinationNode) { // create a reliable NLPacket with space for the ignore UUID @@ -33,6 +33,8 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { // write the node ID to the packet ignorePacket->write(nodeID.toRfc4122()); + qDebug() << "Sending packet to ignore node" << uuidStringWithoutCurlyBraces(nodeID); + // send off this ignore packet reliably to the matching node nodeList->sendPacket(std::move(ignorePacket), *destinationNode); }); From 095bd7e6c8384a6b565535c9c25c2dde47ca49d9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 14:20:03 -0700 Subject: [PATCH 13/40] cleanup logging, move packet parsing to mixers --- assignment-client/src/audio/AudioMixer.cpp | 5 +++- assignment-client/src/avatars/AvatarMixer.cpp | 5 +++- libraries/networking/src/Node.cpp | 29 ++++++++++--------- libraries/networking/src/Node.h | 2 +- .../src/UsersScriptingInterface.cpp | 2 ++ .../src/UsersScriptingInterface.h | 2 ++ 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 002f32a60b..e9c4b9628a 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -557,7 +557,10 @@ void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { } void AudioMixer::handleNodeIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { - sendingNode->handleNodeIgnoreRequest(packet); + // parse out the UUID being ignored from the packet + QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + sendingNode->addIgnoredNode(ignoredUUID); } void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 7930ab843f..7b2b98d3e2 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -434,7 +434,10 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { - senderNode->handleNodeIgnoreRequest(message); + // parse out the UUID being ignored from the packet + QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + senderNode->addIgnoredNode(ignoredUUID); } void AvatarMixer::sendStatsPacket() { diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 4615c61506..f7fa898fbe 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -12,15 +12,17 @@ #include #include -#include - -#include "Node.h" -#include "SharedUtil.h" -#include "NodePermissions.h" - #include #include +#include + +#include "NetworkLogging.h" +#include "NodePermissions.h" +#include "SharedUtil.h" + +#include "Node.h" + const QString UNKNOWN_NodeType_t_NAME = "Unknown"; int NodePtrMetaTypeId = qRegisterMetaType("Node*"); @@ -78,15 +80,16 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { _clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile(); } -void Node::handleNodeIgnoreRequest(QSharedPointer packet) { - // parse out the UUID being ignored from the packet - QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - - qDebug() << "Adding" << uuidStringWithoutCurlyBraces(ignoredUUID) << "to ignore set for" +void Node::addIgnoredNode(const QUuid& otherNodeID) { + if (otherNodeID != _uuid) { + qCDebug(networking) << "Adding" << uuidStringWithoutCurlyBraces(otherNodeID) << "to ignore set for" << uuidStringWithoutCurlyBraces(_uuid); - // add the session UUID to the set of ignored ones for this listening node - _ignoredNodeIDSet.insert(ignoredUUID); + // add the session UUID to the set of ignored ones for this listening node + _ignoredNodeIDSet.insert(otherNodeID); + } else { + qCWarning(networking) << "Node::addIgnoredNode called with ID of ignoring node - nodes cannot self-ignore."; + } } QDataStream& operator<<(QDataStream& out, const Node& node) { diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index daebc1fcfb..ae775e50b2 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -70,7 +70,7 @@ public: bool getCanRezTmp() const { return _permissions.canRezTemporaryEntities; } bool getCanWriteToAssetServer() const { return _permissions.canWriteToAssetServer; } - void handleNodeIgnoreRequest(QSharedPointer packet); + void addIgnoredNode(const QUuid& otherNodeID); bool isIgnoringNodeWithID(const QUuid& nodeID) const { return _ignoredNodeIDSet.find(nodeID) != _ignoredNodeIDSet.cend(); } friend QDataStream& operator<<(QDataStream& out, const Node& node); diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp index 94c448ac86..6f863268c6 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.cpp +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -38,4 +38,6 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { // send off this ignore packet reliably to the matching node nodeList->sendPacket(std::move(ignorePacket), *destinationNode); }); + + emit ignoredNode(nodeID); } diff --git a/libraries/script-engine/src/UsersScriptingInterface.h b/libraries/script-engine/src/UsersScriptingInterface.h index 0dc62c088c..a63d375f3f 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.h +++ b/libraries/script-engine/src/UsersScriptingInterface.h @@ -22,6 +22,8 @@ class UsersScriptingInterface : public QObject, public Dependency { public slots: void ignore(const QUuid& nodeID); +signals: + void ignoredNode(const QUuid& nodeID); }; From d5af323057ef00ba229b542ea3afb7a4c2e910ac Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 14:23:22 -0700 Subject: [PATCH 14/40] don't self-ignore from UsersScriptingInterface --- .../src/UsersScriptingInterface.cpp | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp index 6f863268c6..fab6f453e5 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.cpp +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -20,24 +20,29 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { // enumerate the nodes to send a reliable ignore packet to each that can leverage it auto nodeList = DependencyManager::get(); - nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { - if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { - return true; - } else { - return false; - } - }, [&nodeID, &nodeList](const SharedNodePointer& destinationNode) { - // create a reliable NLPacket with space for the ignore UUID - auto ignorePacket = NLPacket::create(PacketType::NodeIgnoreRequest, NUM_BYTES_RFC4122_UUID, true); + if (nodeList->getSessionUUID() != nodeID) { + nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { + if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { + return true; + } else { + return false; + } + }, [&nodeID, &nodeList](const SharedNodePointer& destinationNode) { + // create a reliable NLPacket with space for the ignore UUID + auto ignorePacket = NLPacket::create(PacketType::NodeIgnoreRequest, NUM_BYTES_RFC4122_UUID, true); - // write the node ID to the packet - ignorePacket->write(nodeID.toRfc4122()); + // write the node ID to the packet + ignorePacket->write(nodeID.toRfc4122()); - qDebug() << "Sending packet to ignore node" << uuidStringWithoutCurlyBraces(nodeID); + qDebug() << "Sending packet to ignore node" << uuidStringWithoutCurlyBraces(nodeID); - // send off this ignore packet reliably to the matching node - nodeList->sendPacket(std::move(ignorePacket), *destinationNode); - }); - - emit ignoredNode(nodeID); + // send off this ignore packet reliably to the matching node + nodeList->sendPacket(std::move(ignorePacket), *destinationNode); + }); + + emit ignoredNode(nodeID); + } else { + qWarning() << "UsersScriptingInterface was asked to ignore a node ID which matches the current session ID, " + << "but self-ignore has no meaning."; + } } From 6b6513d5f9582469131c01f4c753d385e2f8f404 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 14:46:21 -0700 Subject: [PATCH 15/40] immediately fade out ignored avatars --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- interface/src/Application.cpp | 2 +- interface/src/avatar/AvatarManager.cpp | 9 ++++++++- interface/src/avatar/AvatarManager.h | 4 +++- libraries/networking/src/Node.cpp | 4 ++-- libraries/script-engine/src/UsersScriptingInterface.cpp | 5 ++--- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 7b2b98d3e2..64de015c9f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -435,7 +435,7 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { // parse out the UUID being ignored from the packet - QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); senderNode->addIgnoredNode(ignoredUUID); } diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a4ae0af1c5..0e0f56438e 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -441,6 +441,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -455,7 +456,6 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); - DependencyManager::set(); #if defined(Q_OS_MAC) || defined(Q_OS_WIN) DependencyManager::set(); diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 31a77df0cf..3f891ac207 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "Application.h" @@ -73,6 +74,11 @@ AvatarManager::AvatarManager(QObject* parent) : packetReceiver.registerListener(PacketType::BulkAvatarData, this, "processAvatarDataPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "processKillAvatar"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "processAvatarIdentityPacket"); + + // when we hear that the user has ignored an avatar by session UUID + // immediately remove that avatar instead of waiting for the absence of packets from avatar mixer + auto usersScriptingInterface = DependencyManager::get(); + connect(usersScriptingInterface.data(), &UsersScriptingInterface::ignoredNode, this, &AvatarManager::removeAvatar); } AvatarManager::~AvatarManager() { @@ -85,7 +91,8 @@ void AvatarManager::init() { _avatarHash.insert(MY_AVATAR_KEY, _myAvatar); } - connect(DependencyManager::get().data(), &SceneScriptingInterface::shouldRenderAvatarsChanged, this, &AvatarManager::updateAvatarRenderStatus, Qt::QueuedConnection); + connect(DependencyManager::get().data(), &SceneScriptingInterface::shouldRenderAvatarsChanged, + this, &AvatarManager::updateAvatarRenderStatus, Qt::QueuedConnection); render::ScenePointer scene = qApp->getMain3DScene(); render::PendingChanges pendingChanges; diff --git a/interface/src/avatar/AvatarManager.h b/interface/src/avatar/AvatarManager.h index c49d566aa8..f09aa9791c 100644 --- a/interface/src/avatar/AvatarManager.h +++ b/interface/src/avatar/AvatarManager.h @@ -78,6 +78,9 @@ public slots: void setShouldShowReceiveStats(bool shouldShowReceiveStats) { _shouldShowReceiveStats = shouldShowReceiveStats; } void updateAvatarRenderStatus(bool shouldRenderAvatars); +private slots: + virtual void removeAvatar(const QUuid& sessionUUID) override; + private: explicit AvatarManager(QObject* parent = 0); explicit AvatarManager(const AvatarManager& other); @@ -88,7 +91,6 @@ private: virtual AvatarSharedPointer newSharedAvatar() override; virtual AvatarSharedPointer addAvatar(const QUuid& sessionUUID, const QWeakPointer& mixerWeakPointer) override; - virtual void removeAvatar(const QUuid& sessionUUID) override; virtual void handleRemovedAvatar(const AvatarSharedPointer& removedAvatar) override; QVector _avatarFades; diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index f7fa898fbe..98249e3557 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -81,14 +81,14 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { } void Node::addIgnoredNode(const QUuid& otherNodeID) { - if (otherNodeID != _uuid) { + if (!otherNodeID.isNull() && otherNodeID != _uuid) { qCDebug(networking) << "Adding" << uuidStringWithoutCurlyBraces(otherNodeID) << "to ignore set for" << uuidStringWithoutCurlyBraces(_uuid); // add the session UUID to the set of ignored ones for this listening node _ignoredNodeIDSet.insert(otherNodeID); } else { - qCWarning(networking) << "Node::addIgnoredNode called with ID of ignoring node - nodes cannot self-ignore."; + qCWarning(networking) << "Node::addIgnoredNode called with null ID or ID of ignoring node."; } } diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp index fab6f453e5..c7d45503f0 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.cpp +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -20,7 +20,7 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { // enumerate the nodes to send a reliable ignore packet to each that can leverage it auto nodeList = DependencyManager::get(); - if (nodeList->getSessionUUID() != nodeID) { + if (!nodeID.isNull() && nodeList->getSessionUUID() != nodeID) { nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { return true; @@ -42,7 +42,6 @@ void UsersScriptingInterface::ignore(const QUuid& nodeID) { emit ignoredNode(nodeID); } else { - qWarning() << "UsersScriptingInterface was asked to ignore a node ID which matches the current session ID, " - << "but self-ignore has no meaning."; + qWarning() << "UsersScriptingInterface::ignore called with an invalid ID or an ID which matches the current session ID."; } } From 609900f24634aec3e1fd7fe11c4e33890dff4dd5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 15:12:14 -0700 Subject: [PATCH 16/40] move ignore set handling to NodeList --- interface/src/avatar/AvatarManager.cpp | 6 +-- libraries/avatars/src/AvatarHashMap.cpp | 4 +- libraries/avatars/src/AvatarHashMap.h | 4 +- libraries/networking/src/NodeList.cpp | 45 +++++++++++++++++++ libraries/networking/src/NodeList.h | 10 +++++ .../src/UsersScriptingInterface.cpp | 32 +------------ .../src/UsersScriptingInterface.h | 2 - 7 files changed, 66 insertions(+), 37 deletions(-) diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 3f891ac207..bd76d2bd81 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -70,15 +70,15 @@ AvatarManager::AvatarManager(QObject* parent) : // register a meta type for the weak pointer we'll use for the owning avatar mixer for each avatar qRegisterMetaType >("NodeWeakPointer"); - auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + auto nodeList = DependencyManager::get(); + auto& packetReceiver = nodeList->getPacketReceiver(); packetReceiver.registerListener(PacketType::BulkAvatarData, this, "processAvatarDataPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "processKillAvatar"); packetReceiver.registerListener(PacketType::AvatarIdentity, this, "processAvatarIdentityPacket"); // when we hear that the user has ignored an avatar by session UUID // immediately remove that avatar instead of waiting for the absence of packets from avatar mixer - auto usersScriptingInterface = DependencyManager::get(); - connect(usersScriptingInterface.data(), &UsersScriptingInterface::ignoredNode, this, &AvatarManager::removeAvatar); + connect(nodeList.data(), &NodeList::ignoredNode, this, &AvatarManager::removeAvatar); } AvatarManager::~AvatarManager() { diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index d153cfd977..f1f4b4eae2 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -19,7 +19,9 @@ #include "AvatarHashMap.h" AvatarHashMap::AvatarHashMap() { - connect(DependencyManager::get().data(), &NodeList::uuidChanged, this, &AvatarHashMap::sessionUUIDChanged); + auto nodeList = DependencyManager::get(); + + connect(nodeList.data(), &NodeList::uuidChanged, this, &AvatarHashMap::sessionUUIDChanged); } QVector AvatarHashMap::getAvatarIdentifiers() { diff --git a/libraries/avatars/src/AvatarHashMap.h b/libraries/avatars/src/AvatarHashMap.h index 9d3ebb60f5..a59cc4fa96 100644 --- a/libraries/avatars/src/AvatarHashMap.h +++ b/libraries/avatars/src/AvatarHashMap.h @@ -19,12 +19,14 @@ #include #include +#include + #include #include #include #include "AvatarData.h" -#include + class AvatarHashMap : public QObject, public Dependency { Q_OBJECT diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index d3fc93b991..cde141fdad 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -215,6 +215,11 @@ void NodeList::reset() { _numNoReplyDomainCheckIns = 0; + // lock and clear our set of ignored IDs + _ignoredSetLock.lockForWrite(); + _ignoredNodeIDs.clear(); + _ignoredSetLock.unlock(); + // refresh the owner UUID to the NULL UUID setSessionUUID(QUuid()); @@ -692,3 +697,43 @@ void NodeList::sendKeepAlivePings() { sendPacket(constructPingPacket(), *node); }); } + +void NodeList::ignoreNodeBySessionID(const QUuid& nodeID) { + // enumerate the nodes to send a reliable ignore packet to each that can leverage it + + if (!nodeID.isNull() && _sessionUUID != nodeID) { + eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { + if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { + return true; + } else { + return false; + } + }, [&nodeID, this](const SharedNodePointer& destinationNode) { + // create a reliable NLPacket with space for the ignore UUID + auto ignorePacket = NLPacket::create(PacketType::NodeIgnoreRequest, NUM_BYTES_RFC4122_UUID, true); + + // write the node ID to the packet + ignorePacket->write(nodeID.toRfc4122()); + + qDebug() << "Sending packet to ignore node" << uuidStringWithoutCurlyBraces(nodeID); + + // send off this ignore packet reliably to the matching node + sendPacket(std::move(ignorePacket), *destinationNode); + }); + + QReadLocker setLocker { &_ignoredSetLock }; + + // add this nodeID to our set of ignored IDs + _ignoredNodeIDs.insert(nodeID); + + emit ignoredNode(nodeID); + + } else { + qWarning() << "UsersScriptingInterface::ignore called with an invalid ID or an ID which matches the current session ID."; + } +} + +bool NodeList::isIgnoringNode(const QUuid& nodeID) const { + QReadLocker setLocker { &_ignoredSetLock }; + return _ignoredNodeIDs.find(nodeID) != _ignoredNodeIDs.cend(); +} diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index 3fbc86c736..8d8f19ce8c 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -20,6 +20,8 @@ #include // not on windows, not needed for mac or windows #endif +#include + #include #include #include @@ -68,6 +70,9 @@ public: void setIsShuttingDown(bool isShuttingDown) { _isShuttingDown = isShuttingDown; } + void ignoreNodeBySessionID(const QUuid& nodeID); + bool isIgnoringNode(const QUuid& nodeID) const; + public slots: void reset(); void sendDomainServerCheckIn(); @@ -92,6 +97,8 @@ public slots: signals: void limitOfSilentDomainCheckInsReached(); void receivedDomainServerList(); + void ignoredNode(const QUuid& nodeID); + private slots: void stopKeepalivePingTimer(); void sendPendingDSPathQuery(); @@ -129,6 +136,9 @@ private: bool _isShuttingDown { false }; QTimer _keepAlivePingTimer; + mutable QReadWriteLock _ignoredSetLock; + tbb::concurrent_unordered_set _ignoredNodeIDs; + #if (PR_BUILD || DEV_BUILD) bool _shouldSendNewerVersion { false }; #endif diff --git a/libraries/script-engine/src/UsersScriptingInterface.cpp b/libraries/script-engine/src/UsersScriptingInterface.cpp index c7d45503f0..ff7ccb0164 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.cpp +++ b/libraries/script-engine/src/UsersScriptingInterface.cpp @@ -14,34 +14,6 @@ #include void UsersScriptingInterface::ignore(const QUuid& nodeID) { - // setup the ignore packet we send to all nodes (that currently handle it) - // to ignore the data (audio/avatar) for this user - - // enumerate the nodes to send a reliable ignore packet to each that can leverage it - auto nodeList = DependencyManager::get(); - - if (!nodeID.isNull() && nodeList->getSessionUUID() != nodeID) { - nodeList->eachMatchingNode([&nodeID](const SharedNodePointer& node)->bool { - if (node->getType() == NodeType::AudioMixer || node->getType() == NodeType::AvatarMixer) { - return true; - } else { - return false; - } - }, [&nodeID, &nodeList](const SharedNodePointer& destinationNode) { - // create a reliable NLPacket with space for the ignore UUID - auto ignorePacket = NLPacket::create(PacketType::NodeIgnoreRequest, NUM_BYTES_RFC4122_UUID, true); - - // write the node ID to the packet - ignorePacket->write(nodeID.toRfc4122()); - - qDebug() << "Sending packet to ignore node" << uuidStringWithoutCurlyBraces(nodeID); - - // send off this ignore packet reliably to the matching node - nodeList->sendPacket(std::move(ignorePacket), *destinationNode); - }); - - emit ignoredNode(nodeID); - } else { - qWarning() << "UsersScriptingInterface::ignore called with an invalid ID or an ID which matches the current session ID."; - } + // ask the NodeList to ignore this user (based on the session ID of their node) + DependencyManager::get()->ignoreNodeBySessionID(nodeID); } diff --git a/libraries/script-engine/src/UsersScriptingInterface.h b/libraries/script-engine/src/UsersScriptingInterface.h index a63d375f3f..0dc62c088c 100644 --- a/libraries/script-engine/src/UsersScriptingInterface.h +++ b/libraries/script-engine/src/UsersScriptingInterface.h @@ -22,8 +22,6 @@ class UsersScriptingInterface : public QObject, public Dependency { public slots: void ignore(const QUuid& nodeID); -signals: - void ignoredNode(const QUuid& nodeID); }; From 39c7805ca23b865b7fe510106694e6d63cae827f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Jul 2016 15:15:59 -0700 Subject: [PATCH 17/40] don't process packets for ignored avatars --- libraries/avatars/src/AvatarHashMap.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index f1f4b4eae2..48b701d142 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -107,7 +107,10 @@ void AvatarHashMap::processAvatarDataPacket(QSharedPointer mess QByteArray byteArray = message->readWithoutCopy(message->getBytesLeftToRead()); - if (sessionUUID != _lastOwnerSessionUUID) { + // make sure this isn't our own avatar data or for a previously ignored node + auto nodeList = DependencyManager::get(); + + if (sessionUUID != _lastOwnerSessionUUID && !nodeList->isIgnoringNode(sessionUUID)) { auto avatar = newOrExistingAvatar(sessionUUID, sendingNode); // have the matching (or new) avatar parse the data from the packet @@ -126,9 +129,13 @@ void AvatarHashMap::processAvatarIdentityPacket(QSharedPointer AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); - // mesh URL for a UUID, find avatar in our list - auto avatar = newOrExistingAvatar(identity.uuid, sendingNode); - avatar->processAvatarIdentity(identity); + // make sure this isn't for an ignored avatar + auto nodeList = DependencyManager::get(); + if (!nodeList->isIgnoringNode(identity.uuid)) { + // mesh URL for a UUID, find avatar in our list + auto avatar = newOrExistingAvatar(identity.uuid, sendingNode); + avatar->processAvatarIdentity(identity); + } } void AvatarHashMap::processKillAvatar(QSharedPointer message, SharedNodePointer sendingNode) { From 345478eb36997a1e13c76b3a49676de42e74d284 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 13:36:03 -0700 Subject: [PATCH 18/40] add removeButton API to ToolbarScriptingInterface --- interface/src/scripting/ToolbarScriptingInterface.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/interface/src/scripting/ToolbarScriptingInterface.cpp b/interface/src/scripting/ToolbarScriptingInterface.cpp index 82332b3187..175a7fd539 100644 --- a/interface/src/scripting/ToolbarScriptingInterface.cpp +++ b/interface/src/scripting/ToolbarScriptingInterface.cpp @@ -14,7 +14,7 @@ class QmlWrapper : public QObject { Q_OBJECT public: QmlWrapper(QObject* qmlObject, QObject* parent = nullptr) - : QObject(parent), _qmlObject(qmlObject) { + : QObject(parent), _qmlObject(qmlObject) { } Q_INVOKABLE void writeProperty(QString propertyName, QVariant propertyValue) { @@ -91,6 +91,10 @@ public: return new ToolbarButtonProxy(rawButton, this); } + + Q_INVOKABLE void removeButton(const QVariant& name) { + QMetaObject::invokeMethod(_qmlObject, "removeButton", Qt::AutoConnection, Q_ARG(QVariant, name)); + } }; @@ -112,4 +116,4 @@ QObject* ToolbarScriptingInterface::getToolbar(const QString& toolbarId) { } -#include "ToolbarScriptingInterface.moc" \ No newline at end of file +#include "ToolbarScriptingInterface.moc" From 8724f0d0d9240b1e5c38fab6e7b838a913bacab1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 13:40:29 -0700 Subject: [PATCH 19/40] add a stubbed version of the ignore script --- scripts/system/ignore.js | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 scripts/system/ignore.js diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js new file mode 100644 index 0000000000..eaeecb2e96 --- /dev/null +++ b/scripts/system/ignore.js @@ -0,0 +1,40 @@ +// +// ignore.js +// scripts/system/ +// +// Created by Stephen Birarda on 07/11/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +// grab the toolbar +var toolbar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); + +// setup the ignore button and add it to the toolbar +var button = toolbar.addButton({ + objectName: 'ignore', + imageURL: Script.resolvePath("assets/images/tools/mic.svg"), + visible: true, + buttonState: 1, + alpha: 0.9 +}); + +var isShowingOverlays = false; + +// handle clicks on the toolbar button +function buttonClicked(){ + if (isShowingOverlays) { + hideOverlays(); + } else { + showOverlays(); + } +} + +button.clicked.connect(buttonClicked); + +// remove the toolbar button when script is stopped +Script.scriptEnding.connect(function() { + toolbar.removeButton('ignore'); +}); From 3c330d0c48f6b9c5fa350c8ce6463cf9b0c0b814 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 13:59:31 -0700 Subject: [PATCH 20/40] show simple overlay for script testing --- scripts/system/ignore.js | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index eaeecb2e96..f8939d8e25 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -22,14 +22,56 @@ var button = toolbar.addButton({ }); var isShowingOverlays = false; +var currentOverlays = []; + +function hideOverlays() { + // enumerate the overlays and remove them + while (currentOverlays.length) { + // shift the element to remove it from the current overlays array when it is removed + Overlays.deleteOverlay(currentOverlays.shift()); + } +} + +var OVERLAY_SIZE = 0.25; + +function showOverlays() { + var identifiers = AvatarList.getAvatarIdentifiers(); + + for (i = 0; i < identifiers.length; ++i) { + // get the position for this avatar + var avatar = AvatarList.getAvatar(identifiers[i]); + var avatarPosition = avatar && avatar.position; + + if (!avatarPosition) { + // we don't have a valid position for this avatar, skip it + break; + } + + // add the overlay above this avatar + var newOverlay = Overlays.addOverlay("cube", { + position: avatarPosition, + size: 0.25, + color: { red: 0, green: 0, blue: 255}, + alpha: 1, + solid: true + }); + + // push this overlay to our array of overlays + currentOverlays.push(newOverlay); + } +} // handle clicks on the toolbar button function buttonClicked(){ if (isShowingOverlays) { hideOverlays(); + isShowingOverlays = false; } else { showOverlays(); + isShowingOverlays = true; } + + button.writeProperty('buttonState', isShowingOverlays ? 0 : 1); } button.clicked.connect(buttonClicked); @@ -37,4 +79,5 @@ button.clicked.connect(buttonClicked); // remove the toolbar button when script is stopped Script.scriptEnding.connect(function() { toolbar.removeButton('ignore'); + hideOverlays(); }); From 154834b0ab91897c141e210ff5cb5a0f884c4a1d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 14:35:46 -0700 Subject: [PATCH 21/40] hook up ignoring of user to overlay --- scripts/system/ignore.js | 52 +++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index f8939d8e25..05622237a9 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -22,13 +22,13 @@ var button = toolbar.addButton({ }); var isShowingOverlays = false; -var currentOverlays = []; +var ignoreOverlays = []; function hideOverlays() { // enumerate the overlays and remove them - while (currentOverlays.length) { + while (ignoreOverlays.length) { // shift the element to remove it from the current overlays array when it is removed - Overlays.deleteOverlay(currentOverlays.shift()); + Overlays.deleteOverlay(ignoreOverlays.shift()['overlayID']); } } @@ -39,7 +39,14 @@ function showOverlays() { for (i = 0; i < identifiers.length; ++i) { // get the position for this avatar - var avatar = AvatarList.getAvatar(identifiers[i]); + var identifier = identifiers[i]; + + if (identifier === null) { + // this is our avatar, skip it + break; + } + + var avatar = AvatarList.getAvatar(identifier); var avatarPosition = avatar && avatar.position; if (!avatarPosition) { @@ -57,7 +64,10 @@ function showOverlays() { }); // push this overlay to our array of overlays - currentOverlays.push(newOverlay); + ignoreOverlays.push({ + avatarID: identifiers[i], + overlayID: newOverlay + }); } } @@ -76,7 +86,37 @@ function buttonClicked(){ button.clicked.connect(buttonClicked); -// remove the toolbar button when script is stopped +Controller.mousePressEvent.connect(function(event){ + // handle click events so we can detect when our overlays are clicked + + if (!event.isLeftButton && !that.triggered) { + // if another mouse button than left is pressed ignore it + return false; + } + + // compute the pick ray from the event + var pickRay = Camera.computePickRay(event.x, event.y); + + // grab the clicked overlay for the given pick ray + var clickedOverlay = Overlays.findRayIntersection(pickRay); + + // see this is one of our ignore overlays + for (i = 0; i < ignoreOverlays.length; ++i) { + var ignoreOverlay = ignoreOverlays[i]['overlayID'] + if (clickedOverlay.overlayID == ignoreOverlay) { + // matched to an overlay, ask for the matching avatar to be ignored + Users.ignore(ignoreOverlays[i]['avatarID']); + + // remove the actual overlay + Overlays.deleteOverlay(ignoreOverlay); + + // remove the overlay from our internal array + ignoreOverlays.splice(i, 1); + } + } +}); + +// cleanup the toolbar button and overlays when script is stopped Script.scriptEnding.connect(function() { toolbar.removeButton('ignore'); hideOverlays(); From 22ab3fc7b57ad568e53d1cca1d243413b4380ec1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 14:36:54 -0700 Subject: [PATCH 22/40] cleanup a couple of comments --- scripts/system/ignore.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index 05622237a9..a1b2445d85 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -38,7 +38,6 @@ function showOverlays() { var identifiers = AvatarList.getAvatarIdentifiers(); for (i = 0; i < identifiers.length; ++i) { - // get the position for this avatar var identifier = identifiers[i]; if (identifier === null) { @@ -46,6 +45,7 @@ function showOverlays() { break; } + // get the position for this avatar var avatar = AvatarList.getAvatar(identifier); var avatarPosition = avatar && avatar.position; @@ -107,10 +107,10 @@ Controller.mousePressEvent.connect(function(event){ // matched to an overlay, ask for the matching avatar to be ignored Users.ignore(ignoreOverlays[i]['avatarID']); - // remove the actual overlay + // remove the actual overlay so it is no longer rendered Overlays.deleteOverlay(ignoreOverlay); - // remove the overlay from our internal array + // remove the overlay ID and avatar ID from our internal array ignoreOverlays.splice(i, 1); } } From 7bd8c45098d557b783601a4f5b5330a94f3c4cd0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 14:59:42 -0700 Subject: [PATCH 23/40] add the correct ignore icon, cleanup data structure --- scripts/system/assets/images/tools/ignore.svg | 177 ++++++++++++++++++ scripts/system/ignore.js | 101 +++++----- 2 files changed, 234 insertions(+), 44 deletions(-) create mode 100644 scripts/system/assets/images/tools/ignore.svg diff --git a/scripts/system/assets/images/tools/ignore.svg b/scripts/system/assets/images/tools/ignore.svg new file mode 100644 index 0000000000..f315c5f249 --- /dev/null +++ b/scripts/system/assets/images/tools/ignore.svg @@ -0,0 +1,177 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index a1b2445d85..d0206710ee 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -15,69 +15,77 @@ var toolbar = Toolbars.getToolbar("com.highfidelity.interface.toolbar.system"); // setup the ignore button and add it to the toolbar var button = toolbar.addButton({ objectName: 'ignore', - imageURL: Script.resolvePath("assets/images/tools/mic.svg"), + imageURL: Script.resolvePath("assets/images/tools/ignore.svg"), visible: true, buttonState: 1, alpha: 0.9 }); var isShowingOverlays = false; -var ignoreOverlays = []; +var ignoreOverlays = {}; -function hideOverlays() { +function removeOverlays() { // enumerate the overlays and remove them - while (ignoreOverlays.length) { - // shift the element to remove it from the current overlays array when it is removed - Overlays.deleteOverlay(ignoreOverlays.shift()['overlayID']); + var ignoreOverlayKeys = Object.keys(ignoreOverlays); + + for (i = 0; i < ignoreOverlayKeys.length; ++i) { + var avatarID = ignoreOverlayKeys[i]; + Overlays.deleteOverlay(ignoreOverlays[avatarID]); } + + ignoreOverlays = {}; } var OVERLAY_SIZE = 0.25; -function showOverlays() { - var identifiers = AvatarList.getAvatarIdentifiers(); +function updateOverlays() { + if (isShowingOverlays) { - for (i = 0; i < identifiers.length; ++i) { - var identifier = identifiers[i]; + var identifiers = AvatarList.getAvatarIdentifiers(); - if (identifier === null) { - // this is our avatar, skip it - break; + for (i = 0; i < identifiers.length; ++i) { + var avatarID = identifiers[i]; + + if (avatarID === null) { + // this is our avatar, skip it + continue; + } + + // get the position for this avatar + var avatar = AvatarList.getAvatar(avatarID); + var avatarPosition = avatar && avatar.position; + + if (!avatarPosition) { + // we don't have a valid position for this avatar, skip it + continue; + } + + if (avatarID in ignoreOverlays) { + + } else { + + // add the overlay above this avatar + var newOverlay = Overlays.addOverlay("cube", { + position: avatarPosition, + size: 0.25, + color: { red: 0, green: 0, blue: 255}, + alpha: 1, + solid: true + }); + + // push this overlay to our array of overlays + ignoreOverlays[avatarID] = newOverlay; + } } - - // get the position for this avatar - var avatar = AvatarList.getAvatar(identifier); - var avatarPosition = avatar && avatar.position; - - if (!avatarPosition) { - // we don't have a valid position for this avatar, skip it - break; - } - - // add the overlay above this avatar - var newOverlay = Overlays.addOverlay("cube", { - position: avatarPosition, - size: 0.25, - color: { red: 0, green: 0, blue: 255}, - alpha: 1, - solid: true - }); - - // push this overlay to our array of overlays - ignoreOverlays.push({ - avatarID: identifiers[i], - overlayID: newOverlay - }); } } // handle clicks on the toolbar button function buttonClicked(){ if (isShowingOverlays) { - hideOverlays(); + removeOverlays(); isShowingOverlays = false; } else { - showOverlays(); isShowingOverlays = true; } @@ -101,23 +109,28 @@ Controller.mousePressEvent.connect(function(event){ var clickedOverlay = Overlays.findRayIntersection(pickRay); // see this is one of our ignore overlays - for (i = 0; i < ignoreOverlays.length; ++i) { - var ignoreOverlay = ignoreOverlays[i]['overlayID'] + var ignoreOverlayKeys = Object.keys(ignoreOverlays) + for (i = 0; i < ignoreOverlayKeys.length; ++i) { + var avatarID = ignoreOverlayKeys[i]; + var ignoreOverlay = ignoreOverlays[avatarID]; + if (clickedOverlay.overlayID == ignoreOverlay) { // matched to an overlay, ask for the matching avatar to be ignored - Users.ignore(ignoreOverlays[i]['avatarID']); + Users.ignore(avatarID); // remove the actual overlay so it is no longer rendered Overlays.deleteOverlay(ignoreOverlay); // remove the overlay ID and avatar ID from our internal array - ignoreOverlays.splice(i, 1); + delete ignoreOverlays[avatarID]; } } }); +Script.update.connect(updateOverlays); + // cleanup the toolbar button and overlays when script is stopped Script.scriptEnding.connect(function() { toolbar.removeButton('ignore'); - hideOverlays(); + removeOverlays(); }); From aeabae4faf76a9a8f55ff350834a20521be61b28 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 15:10:02 -0700 Subject: [PATCH 24/40] handle removal of an overlay for an avatar that is removed --- scripts/system/ignore.js | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index d0206710ee..cff640248a 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -36,6 +36,20 @@ function removeOverlays() { ignoreOverlays = {}; } +// handle clicks on the toolbar button +function buttonClicked(){ + if (isShowingOverlays) { + removeOverlays(); + isShowingOverlays = false; + } else { + isShowingOverlays = true; + } + + button.writeProperty('buttonState', isShowingOverlays ? 0 : 1); +} + +button.clicked.connect(buttonClicked); + var OVERLAY_SIZE = 0.25; function updateOverlays() { @@ -80,19 +94,19 @@ function updateOverlays() { } } -// handle clicks on the toolbar button -function buttonClicked(){ +Script.update.connect(updateOverlays); + +AvatarList.avatarRemovedEvent.connect(function(avatarID){ if (isShowingOverlays) { - removeOverlays(); - isShowingOverlays = false; - } else { - isShowingOverlays = true; + // we are currently showing overlays and an avatar just went away + + // first remove the rendered overlay + Overlays.deleteOverlay(ignoreOverlays[avatarID]); + + // delete the saved ID of the overlay from our ignored overlays object + delete ignoreOverlays[avatarID]; } - - button.writeProperty('buttonState', isShowingOverlays ? 0 : 1); -} - -button.clicked.connect(buttonClicked); +}) Controller.mousePressEvent.connect(function(event){ // handle click events so we can detect when our overlays are clicked @@ -118,16 +132,11 @@ Controller.mousePressEvent.connect(function(event){ // matched to an overlay, ask for the matching avatar to be ignored Users.ignore(avatarID); - // remove the actual overlay so it is no longer rendered - Overlays.deleteOverlay(ignoreOverlay); - - // remove the overlay ID and avatar ID from our internal array - delete ignoreOverlays[avatarID]; + // cleanup of the overlay is handled by the connection to avatarRemovedEvent } } }); -Script.update.connect(updateOverlays); // cleanup the toolbar button and overlays when script is stopped Script.scriptEnding.connect(function() { From 599d9c568aa027d0ad2e4cc29124dc17cc3a8992 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 15:43:57 -0700 Subject: [PATCH 25/40] remove unneeded check from copy-paste --- scripts/system/ignore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index cff640248a..da8c965540 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -111,7 +111,7 @@ AvatarList.avatarRemovedEvent.connect(function(avatarID){ Controller.mousePressEvent.connect(function(event){ // handle click events so we can detect when our overlays are clicked - if (!event.isLeftButton && !that.triggered) { + if (!event.isLeftButton) { // if another mouse button than left is pressed ignore it return false; } From 5fc2afe549dbac4095795292ac2af705b1a4b742 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jul 2016 17:42:42 -0700 Subject: [PATCH 26/40] switch to new icon for ignore target --- .../system/assets/images/ignore-target-01.svg | 37 +++++++++++++++++++ scripts/system/ignore.js | 26 ++++++++----- 2 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 scripts/system/assets/images/ignore-target-01.svg diff --git a/scripts/system/assets/images/ignore-target-01.svg b/scripts/system/assets/images/ignore-target-01.svg new file mode 100644 index 0000000000..98cee89ca1 --- /dev/null +++ b/scripts/system/assets/images/ignore-target-01.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index da8c965540..ee6292909a 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -50,8 +50,6 @@ function buttonClicked(){ button.clicked.connect(buttonClicked); -var OVERLAY_SIZE = 0.25; - function updateOverlays() { if (isShowingOverlays) { @@ -74,17 +72,27 @@ function updateOverlays() { continue; } + // setup a position for the overlay that is just above this avatar's head + var overlayPosition = avatar.getJointPosition("Head"); + overlayPosition.y += 0.45; + if (avatarID in ignoreOverlays) { - + // keep the overlay above the current position of this avatar + Overlays.editOverlay(ignoreOverlays[avatarID], { + position: overlayPosition + }); } else { - // add the overlay above this avatar - var newOverlay = Overlays.addOverlay("cube", { - position: avatarPosition, - size: 0.25, - color: { red: 0, green: 0, blue: 255}, + var newOverlay = Overlays.addOverlay("image3d", { + url: Script.resolvePath("assets/images/ignore-target-01.svg"), + position: overlayPosition, + size: 0.4, + scale: 0.4, + color: { red: 255, green: 255, blue: 255}, alpha: 1, - solid: true + solid: true, + isFacingAvatar: true, + drawInFront: true }); // push this overlay to our array of overlays From 07c177748506e5daad500728f9f63d3d3856a8cb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 09:54:30 -0700 Subject: [PATCH 27/40] push the packet version for NodeIgnoreRequest --- libraries/networking/src/udt/PacketHeaders.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 99e10caabd..fca006ae87 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -60,6 +60,8 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::AssetUpload: // Removal of extension from Asset requests return 18; + case PacketType::NodeIgnoreRequest: + return 18; // Introduction of node ignore request (which replaced an unused packet tpye) case PacketType::DomainConnectionDenied: return static_cast(DomainConnectionDeniedVersion::IncludesReasonCode); From 0918b55e7e679896fa60d177a2fb39d884f3e76c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 11:27:46 -0700 Subject: [PATCH 28/40] add ignore to defaultScripts --- scripts/defaultScripts.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/defaultScripts.js b/scripts/defaultScripts.js index 711e64f938..6880de99b5 100644 --- a/scripts/defaultScripts.js +++ b/scripts/defaultScripts.js @@ -17,6 +17,7 @@ Script.load("system/goto.js"); Script.load("system/hmd.js"); Script.load("system/examples.js"); Script.load("system/edit.js"); +Script.load("system/ignore.js"); Script.load("system/selectAudioDevice.js"); Script.load("system/notifications.js"); Script.load("system/controllers/handControllerGrab.js"); From a6f39d5e6860a7621af918f193b1c980f4520442 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 11:42:28 -0700 Subject: [PATCH 29/40] add code from entitySelectionTool to handle hmd overlay pick --- scripts/system/ignore.js | 78 ++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index ee6292909a..3acbda4666 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -114,7 +114,24 @@ AvatarList.avatarRemovedEvent.connect(function(avatarID){ // delete the saved ID of the overlay from our ignored overlays object delete ignoreOverlays[avatarID]; } -}) +}); + +function handleSelectedOverlay(clickedOverlay) { + // see this is one of our ignore overlays + + var ignoreOverlayKeys = Object.keys(ignoreOverlays) + for (i = 0; i < ignoreOverlayKeys.length; ++i) { + var avatarID = ignoreOverlayKeys[i]; + var ignoreOverlay = ignoreOverlays[avatarID]; + + if (clickedOverlay.overlayID == ignoreOverlay) { + // matched to an overlay, ask for the matching avatar to be ignored + Users.ignore(avatarID); + + // cleanup of the overlay is handled by the connection to avatarRemovedEvent + } + } +} Controller.mousePressEvent.connect(function(event){ // handle click events so we can detect when our overlays are clicked @@ -129,25 +146,58 @@ Controller.mousePressEvent.connect(function(event){ // grab the clicked overlay for the given pick ray var clickedOverlay = Overlays.findRayIntersection(pickRay); - - // see this is one of our ignore overlays - var ignoreOverlayKeys = Object.keys(ignoreOverlays) - for (i = 0; i < ignoreOverlayKeys.length; ++i) { - var avatarID = ignoreOverlayKeys[i]; - var ignoreOverlay = ignoreOverlays[avatarID]; - - if (clickedOverlay.overlayID == ignoreOverlay) { - // matched to an overlay, ask for the matching avatar to be ignored - Users.ignore(avatarID); - - // cleanup of the overlay is handled by the connection to avatarRemovedEvent - } + if (clickedOverlay.intersects) { + handleSelectedOverlay(clickedOverlay); } }); +// We get mouseMoveEvents from the handControllers, via handControllerPointer. +// But we dont' get mousePressEvents. +var triggerMapping = Controller.newMapping(Script.resolvePath('') + '-click'); + +var TRIGGER_GRAB_VALUE = 0.85; // From handControllerGrab/Pointer.js. Should refactor. +var TRIGGER_ON_VALUE = 0.4; +var TRIGGER_OFF_VALUE = 0.15; +var triggered = false; +var activeHand = Controller.Standard.RightHand; + +function controllerComputePickRay() { + var controllerPose = Controller.getPoseValue(activeHand); + if (controllerPose.valid && triggered) { + var controllerPosition = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, controllerPose.translation), + MyAvatar.position); + // This gets point direction right, but if you want general quaternion it would be more complicated: + var controllerDirection = Quat.getUp(Quat.multiply(MyAvatar.orientation, controllerPose.rotation)); + return {origin: controllerPosition, direction: controllerDirection}; + } +} + +function makeTriggerHandler(hand) { + return function (value) { + if (!triggered && (value > TRIGGER_GRAB_VALUE)) { // should we smooth? + triggered = true; + if (activeHand !== hand) { + // No switching while the other is already triggered, so no need to release. + activeHand = (activeHand === Controller.Standard.RightHand) ? Controller.Standard.LeftHand : Controller.Standard.RightHand; + } + + var pickRay = controllerComputePickRay(); + if (pickRay) { + var overlayIntersection = Overlays.findRayIntersection(pickRay); + if (overlayIntersection.intersects) { + handleClickedOverlay(overlayIntersection); + } + } + } + }; +} + +triggerMapping.from(Controller.Standard.RT).peek().to(makeTriggerHandler(Controller.Standard.RightHand)); +triggerMapping.from(Controller.Standard.LT).peek().to(makeTriggerHandler(Controller.Standard.LeftHand)); // cleanup the toolbar button and overlays when script is stopped Script.scriptEnding.connect(function() { toolbar.removeButton('ignore'); removeOverlays(); + triggerMapping.disable(); }); From b68958317a6ed83b27496f6ce04d648c4993cd54 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 13:40:22 -0700 Subject: [PATCH 30/40] re-send ignore requests when mixers are re-added --- assignment-client/src/audio/AudioMixer.cpp | 5 +--- assignment-client/src/avatars/AvatarMixer.cpp | 5 +--- libraries/networking/src/Node.cpp | 11 ++++++++ libraries/networking/src/Node.h | 1 + libraries/networking/src/NodeList.cpp | 26 +++++++++++++++++++ libraries/networking/src/NodeList.h | 2 ++ 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index e9c4b9628a..7ba9242306 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -557,10 +557,7 @@ void AudioMixer::handleNodeKilled(SharedNodePointer killedNode) { } void AudioMixer::handleNodeIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode) { - // parse out the UUID being ignored from the packet - QUuid ignoredUUID = QUuid::fromRfc4122(packet->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - - sendingNode->addIgnoredNode(ignoredUUID); + sendingNode->parseIgnoreRequestMessage(packet); } void AudioMixer::removeHRTFsForFinishedInjector(const QUuid& streamID) { diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 64de015c9f..65989b389e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -434,10 +434,7 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { - // parse out the UUID being ignored from the packet - QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - - senderNode->addIgnoredNode(ignoredUUID); + senderNode->parseIgnoreRequestMessage(message); } void AvatarMixer::sendStatsPacket() { diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 98249e3557..a7c1707648 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -80,6 +80,17 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { _clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile(); } +void Node::parseIgnoreRequestMessage(QSharedPointer message) { + while (message->getBytesLeftToRead()) { + // parse out the UUID being ignored from the packet + QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + addIgnoredNode(ignoredUUID); + + message->seek(message->getPosition() + NUM_BYTES_RFC4122_UUID); + } +} + void Node::addIgnoredNode(const QUuid& otherNodeID) { if (!otherNodeID.isNull() && otherNodeID != _uuid) { qCDebug(networking) << "Adding" << uuidStringWithoutCurlyBraces(otherNodeID) << "to ignore set for" diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index ae775e50b2..469a0c9755 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -70,6 +70,7 @@ public: bool getCanRezTmp() const { return _permissions.canRezTemporaryEntities; } bool getCanWriteToAssetServer() const { return _permissions.canWriteToAssetServer; } + void parseIgnoreRequestMessage(QSharedPointer message); void addIgnoredNode(const QUuid& otherNodeID); bool isIgnoringNodeWithID(const QUuid& nodeID) const { return _ignoredNodeIDSet.find(nodeID) != _ignoredNodeIDSet.cend(); } diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index cde141fdad..985c51a4f1 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -93,6 +93,9 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned // anytime we get a new node we will want to attempt to punch to it connect(this, &LimitedNodeList::nodeAdded, this, &NodeList::startNodeHolePunch); + + // anytime we get a new node we may need to re-send our set of ignored node IDs to it + connect(this, &LimitedNodeList::nodeAdded, this, &NodeList::maybeSendIgnoreSetToNode); // setup our timer to send keepalive pings (it's started and stopped on domain connect/disconnect) _keepAlivePingTimer.setInterval(KEEPALIVE_PING_INTERVAL_MS); @@ -737,3 +740,26 @@ bool NodeList::isIgnoringNode(const QUuid& nodeID) const { QReadLocker setLocker { &_ignoredSetLock }; return _ignoredNodeIDs.find(nodeID) != _ignoredNodeIDs.cend(); } + +void NodeList::maybeSendIgnoreSetToNode(SharedNodePointer newNode) { + if (newNode->getType() == NodeType::AudioMixer || newNode->getType() == NodeType::AvatarMixer) { + // this is a mixer that we just added - it's unlikely it knows who we were previously ignoring in this session, + // so send that list along now (assuming it isn't empty) + + QReadLocker setLocker { &_ignoredSetLock }; + if (_ignoredNodeIDs.size() > 0) { + // setup a packet list so we can send the stream of ignore IDs + auto ignorePacketList = NLPacketList::create(PacketType::NodeIgnoreRequest, QByteArray(), true); + + // enumerate the ignored IDs and write them to the packet list + auto it = _ignoredNodeIDs.cbegin(); + while (it != _ignoredNodeIDs.end()) { + ignorePacketList->write(it->toRfc4122()); + ++it; + } + + // send this NLPacketList to the new node + sendPacketList(std::move(ignorePacketList), *newNode); + } + } +} diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index 8d8f19ce8c..ff994ce612 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -110,6 +110,8 @@ private slots: void pingPunchForDomainServer(); void sendKeepAlivePings(); + + void maybeSendIgnoreSetToNode(SharedNodePointer node); private: NodeList() : LimitedNodeList(0, 0) { assert(false); } // Not implemented, needed for DependencyManager templates compile From cc9b72daa4aefb8cc7eb691d02d30bc1e98f096d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 14:11:56 -0700 Subject: [PATCH 31/40] fix seeking in packet, use nodeActivated for ignore list send --- libraries/networking/src/Node.cpp | 4 +--- libraries/networking/src/NodeList.cpp | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index a7c1707648..406498b025 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -80,14 +80,12 @@ void Node::updateClockSkewUsec(qint64 clockSkewSample) { _clockSkewUsec = (quint64)_clockSkewMovingPercentile.getValueAtPercentile(); } -void Node::parseIgnoreRequestMessage(QSharedPointer message) { +void Node::parseIgnoreRequestMessage(QSharedPointer message) { while (message->getBytesLeftToRead()) { // parse out the UUID being ignored from the packet QUuid ignoredUUID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); addIgnoredNode(ignoredUUID); - - message->seek(message->getPosition() + NUM_BYTES_RFC4122_UUID); } } diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 985c51a4f1..a73537aad0 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -95,7 +95,7 @@ NodeList::NodeList(char newOwnerType, unsigned short socketListenPort, unsigned connect(this, &LimitedNodeList::nodeAdded, this, &NodeList::startNodeHolePunch); // anytime we get a new node we may need to re-send our set of ignored node IDs to it - connect(this, &LimitedNodeList::nodeAdded, this, &NodeList::maybeSendIgnoreSetToNode); + connect(this, &LimitedNodeList::nodeActivated, this, &NodeList::maybeSendIgnoreSetToNode); // setup our timer to send keepalive pings (it's started and stopped on domain connect/disconnect) _keepAlivePingTimer.setInterval(KEEPALIVE_PING_INTERVAL_MS); @@ -747,6 +747,7 @@ void NodeList::maybeSendIgnoreSetToNode(SharedNodePointer newNode) { // so send that list along now (assuming it isn't empty) QReadLocker setLocker { &_ignoredSetLock }; + if (_ignoredNodeIDs.size() > 0) { // setup a packet list so we can send the stream of ignore IDs auto ignorePacketList = NLPacketList::create(PacketType::NodeIgnoreRequest, QByteArray(), true); From 30f55418db2e116fe8f01428713534e664999d27 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 14:17:42 -0700 Subject: [PATCH 32/40] only prepare packet list packets if they need a message number --- libraries/networking/src/udt/PacketQueue.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/PacketQueue.cpp b/libraries/networking/src/udt/PacketQueue.cpp index 4b8c0b187c..3684e5ba07 100644 --- a/libraries/networking/src/udt/PacketQueue.cpp +++ b/libraries/networking/src/udt/PacketQueue.cpp @@ -65,7 +65,9 @@ void PacketQueue::queuePacket(PacketPointer packet) { } void PacketQueue::queuePacketList(PacketListPointer packetList) { - packetList->preparePackets(getNextMessageNumber()); + if (packetList->isOrdered()) { + packetList->preparePackets(getNextMessageNumber()); + } LockGuard locker(_packetsLock); _channels.push_back(std::move(packetList->_packets)); From cd1c114807d4d81b0607822632fe8a5c2bd58f44 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 15:20:31 -0700 Subject: [PATCH 33/40] enable the trigger mapping so it's actually usable --- scripts/system/ignore.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index 3acbda4666..f99a45e684 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -191,10 +191,11 @@ function makeTriggerHandler(hand) { } }; } - triggerMapping.from(Controller.Standard.RT).peek().to(makeTriggerHandler(Controller.Standard.RightHand)); triggerMapping.from(Controller.Standard.LT).peek().to(makeTriggerHandler(Controller.Standard.LeftHand)); +triggerMapping.enable(); + // cleanup the toolbar button and overlays when script is stopped Script.scriptEnding.connect(function() { toolbar.removeButton('ignore'); From 9deb9744c644e5b98107c68e495f99b42fe89c03 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 15:34:20 -0700 Subject: [PATCH 34/40] handle trigger off in controller mapping --- scripts/system/ignore.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index f99a45e684..46040ba2bc 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -188,6 +188,8 @@ function makeTriggerHandler(hand) { handleClickedOverlay(overlayIntersection); } } + } else if (triggered && (value < TRIGGER_OFF_VALUE)) { + triggered = false; } }; } From 61975fe33a583eef5861378d6fc3ed9cbae2eeef Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 15:47:52 -0700 Subject: [PATCH 35/40] use correct function for peeked overlay --- scripts/system/ignore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index 46040ba2bc..920c8fc1de 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -185,7 +185,7 @@ function makeTriggerHandler(hand) { if (pickRay) { var overlayIntersection = Overlays.findRayIntersection(pickRay); if (overlayIntersection.intersects) { - handleClickedOverlay(overlayIntersection); + handleSelectedOverlay(overlayIntersection); } } } else if (triggered && (value < TRIGGER_OFF_VALUE)) { From 1d77cec125876844b3c4afced504aa475db9a094 Mon Sep 17 00:00:00 2001 From: Anthony Thibault Date: Wed, 13 Jul 2016 07:32:05 -0700 Subject: [PATCH 36/40] Support for scale property on model overlays If "scale" property is present but "dimensions" are not, the model is NOT scale to fit. And the scale of the model's natural dimensions will be affected by the scale properties. If only the "dimensions" property is present, the model will "scale to fit" the dimensions. If both properties are present, the model still be "scale to fit" but the dimension will be scaled by the scale factor. For example: If a model is loaded that is 2cm tall, is loaded with no "dimensions" or "scale" properties. It will be displayed as 2cm tall. {"scale": 2} The above properties will result in a model that is 4cm tall. {"dimensions": 1} This will result in a model that is 1cm tall. {"scale": 2, "dimensions" 2} Will result in a model that is 2cm tall. --- interface/src/ui/overlays/Base3DOverlay.h | 16 +++--- interface/src/ui/overlays/ModelOverlay.cpp | 55 +++++++++++++------ interface/src/ui/overlays/ModelOverlay.h | 5 +- interface/src/ui/overlays/Volume3DOverlay.cpp | 6 +- interface/src/ui/overlays/Volume3DOverlay.h | 14 ++--- 5 files changed, 60 insertions(+), 36 deletions(-) diff --git a/interface/src/ui/overlays/Base3DOverlay.h b/interface/src/ui/overlays/Base3DOverlay.h index 41e7e517b7..e602dec48c 100644 --- a/interface/src/ui/overlays/Base3DOverlay.h +++ b/interface/src/ui/overlays/Base3DOverlay.h @@ -17,7 +17,7 @@ class Base3DOverlay : public Overlay { Q_OBJECT - + public: Base3DOverlay(); Base3DOverlay(const Base3DOverlay* base3DOverlay); @@ -27,10 +27,10 @@ public: const glm::vec3& getPosition() const { return _transform.getTranslation(); } const glm::quat& getRotation() const { return _transform.getRotation(); } const glm::vec3& getScale() const { return _transform.getScale(); } - + // TODO: consider implementing registration points in this class const glm::vec3& getCenter() const { return getPosition(); } - + float getLineWidth() const { return _lineWidth; } bool getIsSolid() const { return _isSolid; } bool getIsDashedLine() const { return _isDashedLine; } @@ -43,7 +43,7 @@ public: void setRotation(const glm::quat& value) { _transform.setRotation(value); } void setScale(float value) { _transform.setScale(value); } void setScale(const glm::vec3& value) { _transform.setScale(value); } - + void setLineWidth(float lineWidth) { _lineWidth = lineWidth; } void setIsSolid(bool isSolid) { _isSolid = isSolid; } void setIsDashedLine(bool isDashedLine) { _isDashedLine = isDashedLine; } @@ -55,22 +55,22 @@ public: void setProperties(const QVariantMap& properties) override; QVariant getProperty(const QString& property) override; - virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, + virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal); - virtual bool findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction, + virtual bool findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal, QString& extraInfo) { return findRayIntersection(origin, direction, distance, face, surfaceNormal); } protected: Transform _transform; - + float _lineWidth; bool _isSolid; bool _isDashedLine; bool _ignoreRayIntersection; bool _drawInFront; }; - + #endif // hifi_Base3DOverlay_h diff --git a/interface/src/ui/overlays/ModelOverlay.cpp b/interface/src/ui/overlays/ModelOverlay.cpp index a857dc39e0..2897c07e64 100644 --- a/interface/src/ui/overlays/ModelOverlay.cpp +++ b/interface/src/ui/overlays/ModelOverlay.cpp @@ -19,8 +19,7 @@ QString const ModelOverlay::TYPE = "model"; ModelOverlay::ModelOverlay() : _model(std::make_shared(std::make_shared())), - _modelTextures(QVariantMap()), - _updateModel(false) + _modelTextures(QVariantMap()) { _model->init(); _isLoaded = false; @@ -44,7 +43,11 @@ void ModelOverlay::update(float deltatime) { if (_updateModel) { _updateModel = false; _model->setSnapModelToCenter(true); - _model->setScaleToFit(true, getDimensions()); + if (_scaleToFit) { + _model->setScaleToFit(true, getScale() * getDimensions()); + } else { + _model->setScale(getScale()); + } _model->setRotation(getRotation()); _model->setTranslation(getPosition()); _model->setURL(_url); @@ -84,16 +87,33 @@ void ModelOverlay::render(RenderArgs* args) { } void ModelOverlay::setProperties(const QVariantMap& properties) { - auto position = getPosition(); - auto rotation = getRotation(); + auto origPosition = getPosition(); + auto origRotation = getRotation(); + auto origDimensions = getDimensions(); + auto origScale = getScale(); - Volume3DOverlay::setProperties(properties); + Base3DOverlay::setProperties(properties); - if (position != getPosition() || rotation != getRotation()) { - _updateModel = true; + auto scale = properties["scale"]; + if (scale.isValid()) { + setScale(vec3FromVariant(scale)); } - _updateModel = true; + auto dimensions = properties["dimensions"]; + if (dimensions.isValid()) { + _scaleToFit = true; + setDimensions(vec3FromVariant(dimensions)); + } else if (scale.isValid()) { + // if "scale" property is set but "dimentions" is not. + // do NOT scale to fit. + if (scale.isValid()) { + _scaleToFit = false; + } + } + + if (origPosition != getPosition() || origRotation != getRotation() || origDimensions != getDimensions() || origScale != getScale()) { + _updateModel = true; + } auto urlValue = properties["url"]; if (urlValue.isValid() && urlValue.canConvert()) { @@ -101,15 +121,15 @@ void ModelOverlay::setProperties(const QVariantMap& properties) { _updateModel = true; _isLoaded = false; } - + auto texturesValue = properties["textures"]; if (texturesValue.isValid() && texturesValue.canConvert(QVariant::Map)) { QVariantMap textureMap = texturesValue.toMap(); foreach(const QString& key, textureMap.keys()) { - + QUrl newTextureURL = textureMap[key].toUrl(); qDebug() << "Updating texture named" << key << "to texture at URL" << newTextureURL; - + QMetaObject::invokeMethod(_model.get(), "setTextureWithNameToURL", Qt::AutoConnection, Q_ARG(const QString&, key), Q_ARG(const QUrl&, newTextureURL)); @@ -123,8 +143,11 @@ QVariant ModelOverlay::getProperty(const QString& property) { if (property == "url") { return _url.toString(); } - if (property == "dimensions" || property == "scale" || property == "size") { - return vec3toVariant(_model->getScaleToFitDimensions()); + if (property == "dimensions" || property == "size") { + return vec3toVariant(getDimensions()); + } + if (property == "scale") { + return vec3toVariant(getScale()); } if (property == "textures") { if (_modelTextures.size() > 0) { @@ -143,14 +166,14 @@ QVariant ModelOverlay::getProperty(const QString& property) { bool ModelOverlay::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal) { - + QString subMeshNameTemp; return _model->findRayIntersectionAgainstSubMeshes(origin, direction, distance, face, surfaceNormal, subMeshNameTemp); } bool ModelOverlay::findRayIntersectionExtraInfo(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face, glm::vec3& surfaceNormal, QString& extraInfo) { - + return _model->findRayIntersectionAgainstSubMeshes(origin, direction, distance, face, surfaceNormal, extraInfo); } diff --git a/interface/src/ui/overlays/ModelOverlay.h b/interface/src/ui/overlays/ModelOverlay.h index dc4b4a853b..091cab44c9 100644 --- a/interface/src/ui/overlays/ModelOverlay.h +++ b/interface/src/ui/overlays/ModelOverlay.h @@ -43,9 +43,10 @@ private: ModelPointer _model; QVariantMap _modelTextures; - + QUrl _url; - bool _updateModel; + bool _updateModel = { false }; + bool _scaleToFit = { false }; }; #endif // hifi_ModelOverlay_h diff --git a/interface/src/ui/overlays/Volume3DOverlay.cpp b/interface/src/ui/overlays/Volume3DOverlay.cpp index c8078d35c6..563198c976 100644 --- a/interface/src/ui/overlays/Volume3DOverlay.cpp +++ b/interface/src/ui/overlays/Volume3DOverlay.cpp @@ -22,7 +22,7 @@ AABox Volume3DOverlay::getBounds() const { auto extents = Extents{_localBoundingBox}; extents.rotate(getRotation()); extents.shiftBy(getPosition()); - + return AABox(extents); } @@ -31,7 +31,7 @@ void Volume3DOverlay::setProperties(const QVariantMap& properties) { auto dimensions = properties["dimensions"]; - // if "dimensions" property was not there, check to see if they included aliases: scale + // if "dimensions" property was not there, check to see if they included aliases: scale, size if (!dimensions.isValid()) { dimensions = properties["scale"]; if (!dimensions.isValid()) { @@ -57,7 +57,7 @@ bool Volume3DOverlay::findRayIntersection(const glm::vec3& origin, const glm::ve // extents is the entity relative, scaled, centered extents of the entity glm::mat4 worldToEntityMatrix; _transform.getInverseMatrix(worldToEntityMatrix); - + glm::vec3 overlayFrameOrigin = glm::vec3(worldToEntityMatrix * glm::vec4(origin, 1.0f)); glm::vec3 overlayFrameDirection = glm::vec3(worldToEntityMatrix * glm::vec4(direction, 0.0f)); diff --git a/interface/src/ui/overlays/Volume3DOverlay.h b/interface/src/ui/overlays/Volume3DOverlay.h index 4d087615d2..04b694b2f8 100644 --- a/interface/src/ui/overlays/Volume3DOverlay.h +++ b/interface/src/ui/overlays/Volume3DOverlay.h @@ -15,13 +15,13 @@ class Volume3DOverlay : public Base3DOverlay { Q_OBJECT - + public: Volume3DOverlay() {} Volume3DOverlay(const Volume3DOverlay* volume3DOverlay); - + virtual AABox getBounds() const override; - + const glm::vec3& getDimensions() const { return _localBoundingBox.getDimensions(); } void setDimensions(float value) { _localBoundingBox.setBox(glm::vec3(-value / 2.0f), value); } void setDimensions(const glm::vec3& value) { _localBoundingBox.setBox(-value / 2.0f, value); } @@ -29,13 +29,13 @@ public: void setProperties(const QVariantMap& properties) override; QVariant getProperty(const QString& property) override; - virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, - BoxFace& face, glm::vec3& surfaceNormal) override; - + virtual bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, + BoxFace& face, glm::vec3& surfaceNormal) override; + protected: // Centered local bounding box AABox _localBoundingBox{ vec3(0.0f), 1.0f }; }; - + #endif // hifi_Volume3DOverlay_h From 946c7d46447fc1dfb82061de56495512289f9bec Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 16:21:12 -0700 Subject: [PATCH 37/40] only handle trigger events when overlays are shown --- scripts/system/ignore.js | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index 920c8fc1de..2216ecff7e 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -168,28 +168,30 @@ function controllerComputePickRay() { MyAvatar.position); // This gets point direction right, but if you want general quaternion it would be more complicated: var controllerDirection = Quat.getUp(Quat.multiply(MyAvatar.orientation, controllerPose.rotation)); - return {origin: controllerPosition, direction: controllerDirection}; + return { origin: controllerPosition, direction: controllerDirection }; } } function makeTriggerHandler(hand) { return function (value) { - if (!triggered && (value > TRIGGER_GRAB_VALUE)) { // should we smooth? - triggered = true; - if (activeHand !== hand) { - // No switching while the other is already triggered, so no need to release. - activeHand = (activeHand === Controller.Standard.RightHand) ? Controller.Standard.LeftHand : Controller.Standard.RightHand; - } - - var pickRay = controllerComputePickRay(); - if (pickRay) { - var overlayIntersection = Overlays.findRayIntersection(pickRay); - if (overlayIntersection.intersects) { - handleSelectedOverlay(overlayIntersection); + if (isShowingOverlays) { + if (!triggered && (value > TRIGGER_GRAB_VALUE)) { // should we smooth? + triggered = true; + if (activeHand !== hand) { + // No switching while the other is already triggered, so no need to release. + activeHand = (activeHand === Controller.Standard.RightHand) ? Controller.Standard.LeftHand : Controller.Standard.RightHand; } + + var pickRay = controllerComputePickRay(); + if (pickRay) { + var overlayIntersection = Overlays.findRayIntersection(pickRay); + if (overlayIntersection.intersects) { + handleSelectedOverlay(overlayIntersection); + } + } + } else if (triggered && (value < TRIGGER_OFF_VALUE)) { + triggered = false; } - } else if (triggered && (value < TRIGGER_OFF_VALUE)) { - triggered = false; } }; } From aa433e72332393c2bc44c17fe956073703714425 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Jul 2016 16:23:40 -0700 Subject: [PATCH 38/40] handle click event on overlays only when in ignore mode --- scripts/system/ignore.js | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/system/ignore.js b/scripts/system/ignore.js index 2216ecff7e..39405a2e57 100644 --- a/scripts/system/ignore.js +++ b/scripts/system/ignore.js @@ -134,20 +134,22 @@ function handleSelectedOverlay(clickedOverlay) { } Controller.mousePressEvent.connect(function(event){ - // handle click events so we can detect when our overlays are clicked + if (isShowingOverlays) { + // handle click events so we can detect when our overlays are clicked - if (!event.isLeftButton) { - // if another mouse button than left is pressed ignore it - return false; - } + if (!event.isLeftButton) { + // if another mouse button than left is pressed ignore it + return false; + } - // compute the pick ray from the event - var pickRay = Camera.computePickRay(event.x, event.y); + // compute the pick ray from the event + var pickRay = Camera.computePickRay(event.x, event.y); - // grab the clicked overlay for the given pick ray - var clickedOverlay = Overlays.findRayIntersection(pickRay); - if (clickedOverlay.intersects) { - handleSelectedOverlay(clickedOverlay); + // grab the clicked overlay for the given pick ray + var clickedOverlay = Overlays.findRayIntersection(pickRay); + if (clickedOverlay.intersects) { + handleSelectedOverlay(clickedOverlay); + } } }); From a267843e3ec55880130a820a48a1409847627667 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Wed, 13 Jul 2016 16:37:30 -0700 Subject: [PATCH 39/40] fix razor blades in your ears when switching between domains with no codecs installed --- libraries/audio-client/src/AudioClient.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 0cd01bb579..78e71907f8 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -534,14 +534,18 @@ void AudioClient::handleSelectedAudioFormat(QSharedPointer mess _selectedCodecName = message->readString(); qDebug() << "Selected Codec:" << _selectedCodecName; + + // release any old codec encoder/decoder first... + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + _codec = nullptr; + } + _receivedAudioStream.cleanupCodec(); + auto codecPlugins = PluginManager::getInstance()->getCodecPlugins(); for (auto& plugin : codecPlugins) { if (_selectedCodecName == plugin->getName()) { - // release any old codec encoder/decoder first... - if (_codec && _encoder) { - _codec->releaseEncoder(_encoder); - _encoder = nullptr; - } _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); @@ -821,7 +825,6 @@ void AudioClient::handleAudioInput() { audioTransform.setRotation(_orientationGetter()); // FIXME find a way to properly handle both playback audio and user audio concurrently - // TODO - codec encode goes here QByteArray decocedBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); QByteArray encodedBuffer; if (_encoder) { @@ -840,7 +843,6 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { audioTransform.setTranslation(_positionGetter()); audioTransform.setRotation(_orientationGetter()); - // TODO - codec encode goes here QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audio, encodedBuffer); From 789297d8496cbb9228d12123ffc691f3ae70ab02 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Wed, 13 Jul 2016 17:39:39 -0700 Subject: [PATCH 40/40] Remove redundant isValid check. --- interface/src/ui/overlays/ModelOverlay.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/ui/overlays/ModelOverlay.cpp b/interface/src/ui/overlays/ModelOverlay.cpp index 2897c07e64..9c203c0129 100644 --- a/interface/src/ui/overlays/ModelOverlay.cpp +++ b/interface/src/ui/overlays/ModelOverlay.cpp @@ -106,9 +106,7 @@ void ModelOverlay::setProperties(const QVariantMap& properties) { } else if (scale.isValid()) { // if "scale" property is set but "dimentions" is not. // do NOT scale to fit. - if (scale.isValid()) { - _scaleToFit = false; - } + _scaleToFit = false; } if (origPosition != getPosition() || origRotation != getRotation() || origDimensions != getDimensions() || origScale != getScale()) {