From fb73b6e1cea0d5ed064df6af695c27eb061f417e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 19 Mar 2014 17:57:01 -0700 Subject: [PATCH 1/8] collect min and max loudness for each frame --- assignment-client/src/audio/AudioMixer.cpp | 11 +++++++++-- assignment-client/src/audio/AudioMixer.h | 3 +++ .../src/audio/AudioMixerClientData.cpp | 14 +++++++++++++- assignment-client/src/audio/AudioMixerClientData.h | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 988bcc1da7..2bece55ac4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -62,7 +62,9 @@ void attachNewBufferToNode(Node *newNode) { } AudioMixer::AudioMixer(const QByteArray& packet) : - ThreadedAssignment(packet) + ThreadedAssignment(packet), + _minSourceLoudnessInFrame(1.0f), + _maxSourceLoudnessInFrame(0.0f) { } @@ -353,9 +355,14 @@ void AudioMixer::run() { while (!_isFinished) { + _minSourceLoudnessInFrame = 1.0f; + _maxSourceLoudnessInFrame = 0.0f; + foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { - ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES); + ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES, + _minSourceLoudnessInFrame, + _maxSourceLoudnessInFrame); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 5a68b0023f..6056dbd318 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -38,6 +38,9 @@ private: // client samples capacity is larger than what will be sent to optimize mixing int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; + + float _minSourceLoudnessInFrame; + float _maxSourceLoudnessInFrame; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b2da0a0aaa..9b8cdb8709 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -6,6 +6,8 @@ // Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // +#include + #include #include @@ -82,7 +84,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples) { +void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, + float& currentMinLoudness, + float& currentMaxLoudness) { for (unsigned int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) { // this is a ring buffer that is ready to go @@ -92,6 +96,14 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL // that would be mixed in _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + + if (_nextOutputLoudness < currentMinLoudness) { + currentMinLoudness = _nextOutputLoudness; + } + + if (_nextOutputLoudness > currentMaxLoudness) { + currentMaxLoudness = _nextOutputLoudness; + } } } } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index bb10098e23..701b2befba 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -27,7 +27,7 @@ public: float getNextOutputLoudness() const { return _nextOutputLoudness; } int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples); + void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, float& currentMinLoudness, float& currentMaxLoudness); void pushBuffersAfterFrameSend(); private: std::vector _ringBuffers; From f92f45df63f7460a81193c50b865fab76cb097ab Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 20 Mar 2014 11:17:36 -0700 Subject: [PATCH 2/8] Keep track of keys pressed, synthesize release events when focus is lost. Closes #2375. --- interface/interface_en.ts | 8 +-- interface/src/Application.cpp | 92 +++++++++++++++++++---------------- interface/src/Application.h | 5 ++ interface/src/GLCanvas.cpp | 4 ++ interface/src/GLCanvas.h | 2 + 5 files changed, 66 insertions(+), 45 deletions(-) diff --git a/interface/interface_en.ts b/interface/interface_en.ts index 75ada1910c..6ec3f65acb 100644 --- a/interface/interface_en.ts +++ b/interface/interface_en.ts @@ -4,22 +4,22 @@ Application - + Export Voxels - + Sparse Voxel Octree Files (*.svo) - + Open Script - + JavaScript Files (*.js) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 50dbf1b51a..3854247e22 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -692,6 +692,8 @@ bool Application::event(QEvent* event) { void Application::keyPressEvent(QKeyEvent* event) { + _keysPressed.insert(event->key()); + _controllerScriptingInterface.emitKeyPressEvent(event); // send events to any registered scripts // if one of our scripts have asked to capture this event, then stop processing it @@ -914,6 +916,8 @@ void Application::keyPressEvent(QKeyEvent* event) { void Application::keyReleaseEvent(QKeyEvent* event) { + _keysPressed.remove(event->key()); + _controllerScriptingInterface.emitKeyReleaseEvent(event); // send events to any registered scripts // if one of our scripts have asked to capture this event, then stop processing it @@ -921,60 +925,66 @@ void Application::keyReleaseEvent(QKeyEvent* event) { return; } + switch (event->key()) { + case Qt::Key_E: + _myAvatar->setDriveKeys(UP, 0.f); + break; - if (activeWindow() == _window) { - switch (event->key()) { - case Qt::Key_E: - _myAvatar->setDriveKeys(UP, 0.f); - break; + case Qt::Key_C: + _myAvatar->setDriveKeys(DOWN, 0.f); + break; - case Qt::Key_C: - _myAvatar->setDriveKeys(DOWN, 0.f); - break; + case Qt::Key_W: + _myAvatar->setDriveKeys(FWD, 0.f); + break; - case Qt::Key_W: - _myAvatar->setDriveKeys(FWD, 0.f); - break; + case Qt::Key_S: + _myAvatar->setDriveKeys(BACK, 0.f); + break; - case Qt::Key_S: - _myAvatar->setDriveKeys(BACK, 0.f); - break; + case Qt::Key_A: + _myAvatar->setDriveKeys(ROT_LEFT, 0.f); + break; - case Qt::Key_A: - _myAvatar->setDriveKeys(ROT_LEFT, 0.f); - break; + case Qt::Key_D: + _myAvatar->setDriveKeys(ROT_RIGHT, 0.f); + break; - case Qt::Key_D: - _myAvatar->setDriveKeys(ROT_RIGHT, 0.f); - break; + case Qt::Key_Up: + _myAvatar->setDriveKeys(FWD, 0.f); + _myAvatar->setDriveKeys(UP, 0.f); + break; - case Qt::Key_Up: - _myAvatar->setDriveKeys(FWD, 0.f); - _myAvatar->setDriveKeys(UP, 0.f); - break; + case Qt::Key_Down: + _myAvatar->setDriveKeys(BACK, 0.f); + _myAvatar->setDriveKeys(DOWN, 0.f); + break; - case Qt::Key_Down: - _myAvatar->setDriveKeys(BACK, 0.f); - _myAvatar->setDriveKeys(DOWN, 0.f); - break; + case Qt::Key_Left: + _myAvatar->setDriveKeys(LEFT, 0.f); + _myAvatar->setDriveKeys(ROT_LEFT, 0.f); + break; - case Qt::Key_Left: - _myAvatar->setDriveKeys(LEFT, 0.f); - _myAvatar->setDriveKeys(ROT_LEFT, 0.f); - break; + case Qt::Key_Right: + _myAvatar->setDriveKeys(RIGHT, 0.f); + _myAvatar->setDriveKeys(ROT_RIGHT, 0.f); + break; - case Qt::Key_Right: - _myAvatar->setDriveKeys(RIGHT, 0.f); - _myAvatar->setDriveKeys(ROT_RIGHT, 0.f); - break; - - default: - event->ignore(); - break; - } + default: + event->ignore(); + break; } } +void Application::focusOutEvent(QFocusEvent* event) { + // synthesize events for keys currently pressed, since we may not get their release events + foreach (int key, _keysPressed) { + QKeyEvent event(QEvent::KeyRelease, key, Qt::NoModifier); + keyReleaseEvent(&event); + } + _keysPressed.clear(); +} + void Application::mouseMoveEvent(QMouseEvent* event) { _controllerScriptingInterface.emitMouseMoveEvent(event); // send events to any registered scripts diff --git a/interface/src/Application.h b/interface/src/Application.h index cbfbf4166d..28060113a9 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -122,6 +123,8 @@ public: void keyPressEvent(QKeyEvent* event); void keyReleaseEvent(QKeyEvent* event); + void focusOutEvent(QFocusEvent* event); + void mouseMoveEvent(QMouseEvent* event); void mousePressEvent(QMouseEvent* event); void mouseReleaseEvent(QMouseEvent* event); @@ -432,6 +435,8 @@ private: bool _mousePressed; // true if mouse has been pressed (clear when finished) + QSet _keysPressed; + GeometryCache _geometryCache; TextureCache _textureCache; diff --git a/interface/src/GLCanvas.cpp b/interface/src/GLCanvas.cpp index a91452c06d..513dcfe40c 100644 --- a/interface/src/GLCanvas.cpp +++ b/interface/src/GLCanvas.cpp @@ -55,6 +55,10 @@ void GLCanvas::keyReleaseEvent(QKeyEvent* event) { Application::getInstance()->keyReleaseEvent(event); } +void GLCanvas::focusOutEvent(QFocusEvent* event) { + Application::getInstance()->focusOutEvent(event); +} + void GLCanvas::mouseMoveEvent(QMouseEvent* event) { Application::getInstance()->mouseMoveEvent(event); } diff --git a/interface/src/GLCanvas.h b/interface/src/GLCanvas.h index ad396a48ce..f7f7fb7c20 100644 --- a/interface/src/GLCanvas.h +++ b/interface/src/GLCanvas.h @@ -31,6 +31,8 @@ protected: virtual void keyPressEvent(QKeyEvent* event); virtual void keyReleaseEvent(QKeyEvent* event); + virtual void focusOutEvent(QFocusEvent* event); + virtual void mouseMoveEvent(QMouseEvent* event); virtual void mousePressEvent(QMouseEvent* event); virtual void mouseReleaseEvent(QMouseEvent* event); From 4513e1675d9bf9ea151963d3870a132204338547 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 20 Mar 2014 11:35:57 -0700 Subject: [PATCH 3/8] More line number changes in the translation file. --- interface/interface_en.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface/interface_en.ts b/interface/interface_en.ts index 6ec3f65acb..1ef7f1656f 100644 --- a/interface/interface_en.ts +++ b/interface/interface_en.ts @@ -113,18 +113,18 @@ Menu - + Open .ini config file - - + + Text files (*.ini) - + Save .ini config file From 20ae5c15f74f761e556019a864c67e5e18f89a17 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 20 Mar 2014 12:09:20 -0700 Subject: [PATCH 4/8] Added a "flat" mode for environments that makes them follow you around on the X/Z axes. The default environment is flat. Closes #2378. --- interface/interface_en.ts | 12 ++++++------ interface/src/Application.cpp | 6 ++++-- interface/src/Environment.cpp | 16 ++++++++++------ libraries/shared/src/PacketHeaders.cpp | 2 ++ libraries/voxels/src/EnvironmentData.cpp | 15 +++++++++++++++ libraries/voxels/src/EnvironmentData.h | 8 ++++++++ 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/interface/interface_en.ts b/interface/interface_en.ts index 6ec3f65acb..34e3614716 100644 --- a/interface/interface_en.ts +++ b/interface/interface_en.ts @@ -14,12 +14,12 @@ - + Open Script - + JavaScript Files (*.js) @@ -113,18 +113,18 @@ Menu - + Open .ini config file - - + + Text files (*.ini) - + Save .ini config file diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 3854247e22..2243c49ac8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2150,7 +2150,8 @@ void Application::loadViewFrustum(Camera& camera, ViewFrustum& viewFrustum) { } glm::vec3 Application::getSunDirection() { - return glm::normalize(_environment.getClosestData(_myCamera.getPosition()).getSunLocation() - _myCamera.getPosition()); + return glm::normalize(_environment.getClosestData(_myCamera.getPosition()).getSunLocation(_myCamera.getPosition()) - + _myCamera.getPosition()); } void Application::updateShadowMap() { @@ -2312,7 +2313,8 @@ void Application::displaySide(Camera& whichCamera, bool selfAvatarOnly) { float alpha = 1.0f; if (Menu::getInstance()->isOptionChecked(MenuOption::Atmosphere)) { const EnvironmentData& closestData = _environment.getClosestData(whichCamera.getPosition()); - float height = glm::distance(whichCamera.getPosition(), closestData.getAtmosphereCenter()); + float height = glm::distance(whichCamera.getPosition(), + closestData.getAtmosphereCenter(whichCamera.getPosition())); if (height < closestData.getAtmosphereInnerRadius()) { alpha = 0.0f; diff --git a/interface/src/Environment.cpp b/interface/src/Environment.cpp index 1f9e23bee1..096b8770fb 100644 --- a/interface/src/Environment.cpp +++ b/interface/src/Environment.cpp @@ -92,7 +92,7 @@ glm::vec3 Environment::getGravity (const glm::vec3& position) { foreach (const ServerData& serverData, _data) { foreach (const EnvironmentData& environmentData, serverData) { - glm::vec3 vector = environmentData.getAtmosphereCenter() - position; + glm::vec3 vector = environmentData.getAtmosphereCenter(position) - position; float surfaceRadius = environmentData.getAtmosphereInnerRadius(); if (glm::length(vector) <= surfaceRadius) { // At or inside a planet, gravity is as set for the planet @@ -116,7 +116,7 @@ const EnvironmentData Environment::getClosestData(const glm::vec3& position) { float closestDistance = FLT_MAX; foreach (const ServerData& serverData, _data) { foreach (const EnvironmentData& environmentData, serverData) { - float distance = glm::distance(position, environmentData.getAtmosphereCenter()) - + float distance = glm::distance(position, environmentData.getAtmosphereCenter(position)) - environmentData.getAtmosphereOuterRadius(); if (distance < closestDistance) { closest = environmentData; @@ -132,6 +132,8 @@ bool Environment::findCapsulePenetration(const glm::vec3& start, const glm::vec3 // collide with the "floor" bool found = findCapsulePlanePenetration(start, end, radius, glm::vec4(0.0f, 1.0f, 0.0f, 0.0f), penetration); + glm::vec3 middle = (start + end) * 0.5f; + // get the lock for the duration of the call QMutexLocker locker(&_mutex); @@ -141,7 +143,7 @@ bool Environment::findCapsulePenetration(const glm::vec3& start, const glm::vec3 continue; // don't bother colliding with gravity-less environments } glm::vec3 environmentPenetration; - if (findCapsuleSpherePenetration(start, end, radius, environmentData.getAtmosphereCenter(), + if (findCapsuleSpherePenetration(start, end, radius, environmentData.getAtmosphereCenter(middle), environmentData.getAtmosphereInnerRadius(), environmentPenetration)) { penetration = addPenetrations(penetration, environmentPenetration); found = true; @@ -203,10 +205,12 @@ ProgramObject* Environment::createSkyProgram(const char* from, int* locations) { } void Environment::renderAtmosphere(Camera& camera, const EnvironmentData& data) { + glm::vec3 center = data.getAtmosphereCenter(camera.getPosition()); + glPushMatrix(); - glTranslatef(data.getAtmosphereCenter().x, data.getAtmosphereCenter().y, data.getAtmosphereCenter().z); - - glm::vec3 relativeCameraPos = camera.getPosition() - data.getAtmosphereCenter(); + glTranslatef(center.x, center.y, center.z); + + glm::vec3 relativeCameraPos = camera.getPosition() - center; float height = glm::length(relativeCameraPos); // use the appropriate shader depending on whether we're inside or outside diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index 7d436b9ca6..c7518708ce 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -46,6 +46,8 @@ PacketVersion versionForPacketType(PacketType type) { switch (type) { case PacketTypeAvatarData: return 3; + case PacketTypeEnvironmentData: + return 1; case PacketTypeParticleData: return 1; case PacketTypeDomainList: diff --git a/libraries/voxels/src/EnvironmentData.cpp b/libraries/voxels/src/EnvironmentData.cpp index 5cee88e127..1c9af55abd 100644 --- a/libraries/voxels/src/EnvironmentData.cpp +++ b/libraries/voxels/src/EnvironmentData.cpp @@ -14,6 +14,7 @@ // GameEngine.cpp EnvironmentData::EnvironmentData(int id) : _id(id), + _flat(true), _gravity(0.0f), _atmosphereCenter(0, -1000, 0), _atmosphereInnerRadius(1000), @@ -25,12 +26,23 @@ EnvironmentData::EnvironmentData(int id) : _sunBrightness(20.0f) { } +glm::vec3 EnvironmentData::getAtmosphereCenter(const glm::vec3& cameraPosition) const { + return _atmosphereCenter + (_flat ? glm::vec3(cameraPosition.x, 0.0f, cameraPosition.z) : glm::vec3()); +} + +glm::vec3 EnvironmentData::getSunLocation(const glm::vec3& cameraPosition) const { + return _sunLocation + (_flat ? glm::vec3(cameraPosition.x, 0.0f, cameraPosition.z) : glm::vec3()); +} + int EnvironmentData::getBroadcastData(unsigned char* destinationBuffer) const { unsigned char* bufferStart = destinationBuffer; memcpy(destinationBuffer, &_id, sizeof(_id)); destinationBuffer += sizeof(_id); + memcpy(destinationBuffer, &_flat, sizeof(_flat)); + destinationBuffer += sizeof(_flat); + memcpy(destinationBuffer, &_gravity, sizeof(_gravity)); destinationBuffer += sizeof(_gravity); @@ -67,6 +79,9 @@ int EnvironmentData::parseData(const unsigned char* sourceBuffer, int numBytes) memcpy(&_id, sourceBuffer, sizeof(_id)); sourceBuffer += sizeof(_id); + memcpy(&_flat, sourceBuffer, sizeof(_flat)); + sourceBuffer += sizeof(_flat); + memcpy(&_gravity, sourceBuffer, sizeof(_gravity)); sourceBuffer += sizeof(_gravity); diff --git a/libraries/voxels/src/EnvironmentData.h b/libraries/voxels/src/EnvironmentData.h index 90cc0763fe..627a661e1c 100644 --- a/libraries/voxels/src/EnvironmentData.h +++ b/libraries/voxels/src/EnvironmentData.h @@ -19,6 +19,9 @@ public: void setID(int id) { _id = id; } int getID() const { return _id; } + void setFlat(bool flat) { _flat = flat; } + bool isFlat() const { return _flat; } + void setGravity(float gravity) { _gravity = gravity; } float getGravity() const { return _gravity; } @@ -42,6 +45,9 @@ public: const glm::vec3& getSunLocation() const { return _sunLocation; } float getSunBrightness() const { return _sunBrightness; } + glm::vec3 getAtmosphereCenter(const glm::vec3& cameraPosition) const; + glm::vec3 getSunLocation(const glm::vec3& cameraPosition) const; + int getBroadcastData(unsigned char* destinationBuffer) const; int parseData(const unsigned char* sourceBuffer, int numBytes); @@ -49,6 +55,8 @@ private: int _id; + bool _flat; + float _gravity; glm::vec3 _atmosphereCenter; From a8ef64e0cecd266ee46245002485d367c35dff0d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 20 Mar 2014 12:52:17 -0700 Subject: [PATCH 5/8] calculate a cutoff loudness for mixer recovery --- assignment-client/src/audio/AudioMixer.cpp | 42 ++++++++++++++++++- assignment-client/src/audio/AudioMixer.h | 5 ++- .../src/audio/AudioMixerClientData.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 1 + 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2bece55ac4..1f403e0f3d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -64,7 +64,9 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _minSourceLoudnessInFrame(1.0f), - _maxSourceLoudnessInFrame(0.0f) + _maxSourceLoudnessInFrame(0.0f), + _loudnessCutoffRatio(0.0f), + _minRequiredLoudness(0.0f) { } @@ -352,6 +354,9 @@ void AudioMixer::run() { char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; + + int usecToSleep = 0; + bool isFirstRun = true; while (!_isFinished) { @@ -365,6 +370,39 @@ void AudioMixer::run() { _maxSourceLoudnessInFrame); } } + + if (!isFirstRun) { + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30; + const float CUTOFF_EPSILON = 0.0001; + + float percentageSleep = (usecToSleep / (float) BUFFER_SEND_INTERVAL_USECS); + + float lastCutoffRatio = _loudnessCutoffRatio; + + if (percentageSleep <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD || usecToSleep < 0) { + // we're struggling - change our min required loudness to reduce some load + _loudnessCutoffRatio += (1 - _loudnessCutoffRatio) / 2; + + qDebug() << "Mixer is struggling, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + } else if (percentageSleep >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _loudnessCutoffRatio != 0) { + // we've recovered and can back off the required loudness + _loudnessCutoffRatio -= _loudnessCutoffRatio / 2; + + if (_loudnessCutoffRatio < CUTOFF_EPSILON) { + _loudnessCutoffRatio = 0; + } + + qDebug() << "Mixer is recovering, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + } + + // set out min required loudness from the new ratio + _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + } else { + isFirstRun = false; + } foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() @@ -391,7 +429,7 @@ void AudioMixer::run() { break; } - int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); + usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); if (usecToSleep > 0) { usleep(usecToSleep); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 6056dbd318..3827b2917a 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -37,10 +37,13 @@ private: void prepareMixForListeningNode(Node* node); // client samples capacity is larger than what will be sent to optimize mixing - int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + SAMPLE_PHASE_DELAY_AT_90]; + // we are MMX adding 4 samples at a time so we need client samples to have an extra 4 + int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)]; float _minSourceLoudnessInFrame; float _maxSourceLoudnessInFrame; + float _loudnessCutoffRatio; + float _minRequiredLoudness; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 9b8cdb8709..6830e67aa3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -97,7 +97,7 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam // that would be mixed in _nextOutputLoudness = _ringBuffers[i]->averageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - if (_nextOutputLoudness < currentMinLoudness) { + if (_nextOutputLoudness != 0 && _nextOutputLoudness < currentMinLoudness) { currentMinLoudness = _nextOutputLoudness; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 66a27647d6..ccbe5d3f23 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -50,6 +50,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { int16_t numSilentSamples; memcpy(&numSilentSamples, packet.data() + readBytes, sizeof(int16_t)); + readBytes += sizeof(int16_t); addSilentFrame(numSilentSamples); From 64f946b6405ce158ed75b3c61e762ec10b1c6bdd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 20 Mar 2014 13:01:52 -0700 Subject: [PATCH 6/8] require that buffers be above min loudness to be mixed in --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1f403e0f3d..d8017c0a02 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -379,13 +379,15 @@ void AudioMixer::run() { float percentageSleep = (usecToSleep / (float) BUFFER_SEND_INTERVAL_USECS); float lastCutoffRatio = _loudnessCutoffRatio; + bool hasRatioChanged = false; if (percentageSleep <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD || usecToSleep < 0) { // we're struggling - change our min required loudness to reduce some load _loudnessCutoffRatio += (1 - _loudnessCutoffRatio) / 2; qDebug() << "Mixer is struggling, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + hasRatioChanged = true; } else if (percentageSleep >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _loudnessCutoffRatio != 0) { // we've recovered and can back off the required loudness _loudnessCutoffRatio -= _loudnessCutoffRatio / 2; @@ -396,10 +398,17 @@ void AudioMixer::run() { qDebug() << "Mixer is recovering, sleeping" << percentageSleep * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << _loudnessCutoffRatio; + hasRatioChanged = true; } - // set out min required loudness from the new ratio - _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + if (hasRatioChanged) { + // set out min required loudness from the new ratio + _minRequiredLoudness = _loudnessCutoffRatio * (_maxSourceLoudnessInFrame - _minSourceLoudnessInFrame); + qDebug() << "Minimum loudness required to be mixed is now" << _minRequiredLoudness; + } + + + } else { isFirstRun = false; } From 56ff423cf5a535e0e840fcefb077c24ffdf24594 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 20 Mar 2014 14:27:38 -0700 Subject: [PATCH 7/8] Let's actually use the result of our format conversion. --- interface/src/renderer/TextureCache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/renderer/TextureCache.cpp b/interface/src/renderer/TextureCache.cpp index 2b43c89998..b3820abf25 100644 --- a/interface/src/renderer/TextureCache.cpp +++ b/interface/src/renderer/TextureCache.cpp @@ -313,7 +313,7 @@ void ImageReader::run() { int imageArea = image.width() * image.height(); if (opaquePixels == imageArea) { qDebug() << "Image with alpha channel is completely opaque:" << url; - image.convertToFormat(QImage::Format_RGB888); + image = image.convertToFormat(QImage::Format_RGB888); } QMetaObject::invokeMethod(texture.data(), "setImage", Q_ARG(const QImage&, image), Q_ARG(bool, translucentPixels >= imageArea / 2)); From b4f5a6d1ce7d6ac0189125a44aeb9da5ea467896 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 20 Mar 2014 14:44:47 -0700 Subject: [PATCH 8/8] add _isShuttingDown to OctreeQueryNode and bail as fast as possible when shutting down --- .../src/octree/OctreeQueryNode.cpp | 66 ++++++++++++++++++- .../src/octree/OctreeQueryNode.h | 5 ++ .../src/octree/OctreeSendThread.cpp | 25 +++++-- assignment-client/src/octree/OctreeServer.cpp | 16 ++++- 4 files changed, 101 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/octree/OctreeQueryNode.cpp b/assignment-client/src/octree/OctreeQueryNode.cpp index a1922f980d..2ceb9e1040 100644 --- a/assignment-client/src/octree/OctreeQueryNode.cpp +++ b/assignment-client/src/octree/OctreeQueryNode.cpp @@ -36,18 +36,34 @@ OctreeQueryNode::OctreeQueryNode() : _lodChanged(false), _lodInitialized(false), _sequenceNumber(0), - _lastRootTimestamp(0) + _lastRootTimestamp(0), + _myPacketType(PacketTypeUnknown), + _isShuttingDown(false) { } OctreeQueryNode::~OctreeQueryNode() { + _isShuttingDown = true; + const bool extraDebugging = false; + if (extraDebugging) { + qDebug() << "OctreeQueryNode::~OctreeQueryNode()"; + } if (_octreeSendThread) { + if (extraDebugging) { + qDebug() << "OctreeQueryNode::~OctreeQueryNode()... calling _octreeSendThread->terminate()"; + } _octreeSendThread->terminate(); + if (extraDebugging) { + qDebug() << "OctreeQueryNode::~OctreeQueryNode()... calling delete _octreeSendThread"; + } delete _octreeSendThread; } delete[] _octreePacket; delete[] _lastOctreePacket; + if (extraDebugging) { + qDebug() << "OctreeQueryNode::~OctreeQueryNode()... DONE..."; + } } @@ -59,9 +75,13 @@ void OctreeQueryNode::initializeOctreeSendThread(OctreeServer* octreeServer, con } bool OctreeQueryNode::packetIsDuplicate() const { + // if shutting down, return immediately + if (_isShuttingDown) { + return false; + } // since our packets now include header information, like sequence number, and createTime, we can't just do a memcmp // of the entire packet, we need to compare only the packet content... - int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(getMyPacketType()); + int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(_myPacketType); if (_lastOctreePacketLength == getPacketLength()) { if (memcmp(_lastOctreePacket + (numBytesPacketHeader + OCTREE_PACKET_EXTRA_HEADERS_SIZE), @@ -74,6 +94,11 @@ bool OctreeQueryNode::packetIsDuplicate() const { } bool OctreeQueryNode::shouldSuppressDuplicatePacket() { + // if shutting down, return immediately + if (_isShuttingDown) { + return true; + } + bool shouldSuppress = false; // assume we won't suppress // only consider duplicate packets @@ -107,7 +132,17 @@ bool OctreeQueryNode::shouldSuppressDuplicatePacket() { return shouldSuppress; } +void OctreeQueryNode::init() { + _myPacketType = getMyPacketType(); + resetOctreePacket(true); // don't bump sequence +} + void OctreeQueryNode::resetOctreePacket(bool lastWasSurpressed) { + // if shutting down, return immediately + if (_isShuttingDown) { + return; + } + // Whenever we call this, we will keep a copy of the last packet, so we can determine if the last packet has // changed since we last reset it. Since we know that no two packets can ever be identical without being the same // scene information, (e.g. the root node packet of a static scene), we can use this as a strategy for reducing @@ -128,7 +163,7 @@ void OctreeQueryNode::resetOctreePacket(bool lastWasSurpressed) { } _octreePacketAvailableBytes = MAX_PACKET_SIZE; - int numBytesPacketHeader = populatePacketHeader(reinterpret_cast(_octreePacket), getMyPacketType()); + int numBytesPacketHeader = populatePacketHeader(reinterpret_cast(_octreePacket), _myPacketType); _octreePacketAt = _octreePacket + numBytesPacketHeader; _octreePacketAvailableBytes -= numBytesPacketHeader; @@ -158,6 +193,11 @@ void OctreeQueryNode::resetOctreePacket(bool lastWasSurpressed) { } void OctreeQueryNode::writeToPacket(const unsigned char* buffer, unsigned int bytes) { + // if shutting down, return immediately + if (_isShuttingDown) { + return; + } + // compressed packets include lead bytes which contain compressed size, this allows packing of // multiple compressed portions together if (_currentPacketIsCompressed) { @@ -174,6 +214,11 @@ void OctreeQueryNode::writeToPacket(const unsigned char* buffer, unsigned int by } bool OctreeQueryNode::updateCurrentViewFrustum() { + // if shutting down, return immediately + if (_isShuttingDown) { + return false; + } + bool currentViewFrustumChanged = false; ViewFrustum newestViewFrustum; // get position and orientation details from the camera @@ -233,6 +278,11 @@ void OctreeQueryNode::setViewSent(bool viewSent) { } void OctreeQueryNode::updateLastKnownViewFrustum() { + // if shutting down, return immediately + if (_isShuttingDown) { + return; + } + bool frustumChanges = !_lastKnownViewFrustum.isVerySimilar(_currentViewFrustum); if (frustumChanges) { @@ -247,6 +297,11 @@ void OctreeQueryNode::updateLastKnownViewFrustum() { bool OctreeQueryNode::moveShouldDump() const { + // if shutting down, return immediately + if (_isShuttingDown) { + return false; + } + glm::vec3 oldPosition = _lastKnownViewFrustum.getPosition(); glm::vec3 newPosition = _currentViewFrustum.getPosition(); @@ -259,6 +314,11 @@ bool OctreeQueryNode::moveShouldDump() const { } void OctreeQueryNode::dumpOutOfView() { + // if shutting down, return immediately + if (_isShuttingDown) { + return; + } + int stillInView = 0; int outOfView = 0; OctreeElementBag tempBag; diff --git a/assignment-client/src/octree/OctreeQueryNode.h b/assignment-client/src/octree/OctreeQueryNode.h index 20fa2c5858..eab8cb5d0a 100644 --- a/assignment-client/src/octree/OctreeQueryNode.h +++ b/assignment-client/src/octree/OctreeQueryNode.h @@ -28,6 +28,7 @@ public: OctreeQueryNode(); virtual ~OctreeQueryNode(); + void init(); // called after creation to set up some virtual items virtual PacketType getMyPacketType() const = 0; void resetOctreePacket(bool lastWasSurpressed = false); // resets octree packet to after "V" header @@ -91,6 +92,7 @@ public: unsigned int getlastOctreePacketLength() const { return _lastOctreePacketLength; } int getDuplicatePacketCount() const { return _duplicatePacketCount; } + bool isShuttingDown() const { return _isShuttingDown; } private: OctreeQueryNode(const OctreeQueryNode &); @@ -127,6 +129,9 @@ private: OCTREE_PACKET_SEQUENCE _sequenceNumber; quint64 _lastRootTimestamp; + + PacketType _myPacketType; + bool _isShuttingDown; }; #endif /* defined(__hifi__OctreeQueryNode__) */ diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index caa729e340..9c04c4a1ad 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -65,7 +65,7 @@ bool OctreeSendThread::process() { nodeData = (OctreeQueryNode*) node->getLinkedData(); // Sometimes the node data has not yet been linked, in which case we can't really do anything - if (nodeData) { + if (nodeData && !nodeData->isShuttingDown()) { bool viewFrustumChanged = nodeData->updateCurrentViewFrustum(); packetDistributor(node, nodeData, viewFrustumChanged); } @@ -99,7 +99,14 @@ quint64 OctreeSendThread::_totalBytes = 0; quint64 OctreeSendThread::_totalWastedBytes = 0; quint64 OctreeSendThread::_totalPackets = 0; -int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent) { +int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, + OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent) { + + // if we're shutting down, then exit early + if (nodeData->isShuttingDown()) { + return 0; + } + bool debug = _myServer->wantsDebugSending(); quint64 now = usecTimestampNow(); @@ -136,7 +143,7 @@ int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, OctreeQuer // If we've got a stats message ready to send, then see if we can piggyback them together - if (nodeData->stats.isReadyToSend()) { + if (nodeData->stats.isReadyToSend() && !nodeData->isShuttingDown()) { // Send the stats message to the client unsigned char* statsMessage = nodeData->stats.getStatsMessage(); int statsMessageLength = nodeData->stats.getStatsMessageLength(); @@ -203,7 +210,7 @@ int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, OctreeQuer nodeData->stats.markAsSent(); } else { // If there's actually a packet waiting, then send it. - if (nodeData->isPacketWaiting()) { + if (nodeData->isPacketWaiting() && !nodeData->isShuttingDown()) { // just send the voxel packet NodeList::getInstance()->writeDatagram((char*) nodeData->getPacket(), nodeData->getPacketLength(), SharedNodePointer(node)); @@ -234,6 +241,12 @@ int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, OctreeQuer /// Version of voxel distributor that sends the deepest LOD level at once int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQueryNode* nodeData, bool viewFrustumChanged) { + + // if shutting down, exit early + if (nodeData->isShuttingDown()) { + return 0; + } + int truePacketsSent = 0; int trueBytesSent = 0; int packetsSentThisInterval = 0; @@ -336,7 +349,7 @@ int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQue int extraPackingAttempts = 0; bool completedScene = false; - while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval) { + while (somethingToSend && packetsSentThisInterval < maxPacketsPerInterval && !nodeData->isShuttingDown()) { float lockWaitElapsedUsec = OctreeServer::SKIP_TIME; float encodeElapsedUsec = OctreeServer::SKIP_TIME; float compressAndWriteElapsedUsec = OctreeServer::SKIP_TIME; @@ -503,7 +516,7 @@ int OctreeSendThread::packetDistributor(const SharedNodePointer& node, OctreeQue // Here's where we can/should allow the server to send other data... // send the environment packet // TODO: should we turn this into a while loop to better handle sending multiple special packets - if (_myServer->hasSpecialPacketToSend(node)) { + if (_myServer->hasSpecialPacketToSend(node) && !nodeData->isShuttingDown()) { trueBytesSent += _myServer->sendSpecialPacket(node); truePacketsSent++; packetsSentThisInterval++; diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 7e6ffe52da..496f9af1a0 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -168,7 +168,7 @@ void OctreeServer::trackPacketSendingTime(float time) { void OctreeServer::attachQueryNodeToNode(Node* newNode) { if (!newNode->getLinkedData()) { OctreeQueryNode* newQueryNodeData = _instance->createOctreeQueryNode(); - newQueryNodeData->resetOctreePacket(true); // don't bump sequence + newQueryNodeData->init(); newNode->setLinkedData(newQueryNodeData); } } @@ -784,16 +784,26 @@ void OctreeServer::readPendingDatagrams() { if (packetType == getMyQueryMessageType()) { bool debug = false; if (debug) { - qDebug() << "Got PacketTypeVoxelQuery at" << usecTimestampNow(); + if (matchingNode) { + qDebug() << "Got PacketTypeVoxelQuery at" << usecTimestampNow() << "node:" << *matchingNode; + } else { + qDebug() << "Got PacketTypeVoxelQuery at" << usecTimestampNow() << "node: ??????"; + } } // If we got a PacketType_VOXEL_QUERY, then we're talking to an NodeType_t_AVATAR, and we // need to make sure we have it in our nodeList. if (matchingNode) { + if (debug) { + qDebug() << "calling updateNodeWithDataFromPacket()... node:" << *matchingNode; + } nodeList->updateNodeWithDataFromPacket(matchingNode, receivedPacket); OctreeQueryNode* nodeData = (OctreeQueryNode*) matchingNode->getLinkedData(); if (nodeData && !nodeData->isOctreeSendThreadInitalized()) { + if (debug) { + qDebug() << "calling initializeOctreeSendThread()... node:" << *matchingNode; + } nodeData->initializeOctreeSendThread(this, matchingNode->getUUID()); } } @@ -999,6 +1009,8 @@ void OctreeServer::nodeKilled(SharedNodePointer node) { node->setLinkedData(NULL); // set this first in case another thread comes through and tryes to acces this qDebug() << qPrintable(_safeServerName) << "server deleting Linked Data for node:" << *node; nodeData->deleteLater(); + } else { + qDebug() << qPrintable(_safeServerName) << "server node missing linked data node:" << *node; } }