From 22aa9b075dbad703ccbb0093b2b061f307a37684 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 20 Mar 2014 16:52:40 -0700 Subject: [PATCH 01/27] datalength sanity checking for AvatarData packet --- interface/src/avatar/Avatar.h | 2 +- interface/src/avatar/MyAvatar.cpp | 8 + interface/src/avatar/MyAvatar.h | 2 + libraries/avatars/src/AvatarData.cpp | 285 ++++++++++++++++++--------- libraries/avatars/src/AvatarData.h | 4 +- 5 files changed, 209 insertions(+), 92 deletions(-) diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 30073c54d4..638bff6e32 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -133,7 +133,7 @@ public: void setShowDisplayName(bool showDisplayName); - int parseDataAtOffset(const QByteArray& packet, int offset); + virtual int parseDataAtOffset(const QByteArray& packet, int offset); static void renderJointConnectingCone(glm::vec3 position1, glm::vec3 position2, float radius1, float radius2); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index fb0d704c6a..e21c179b5b 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -551,6 +551,14 @@ void MyAvatar::loadData(QSettings* settings) { settings->endGroup(); } +int MyAvatar::parseDataAtOffset(const QByteArray& packet, int offset) { + qDebug() << "Error: ignoring update packet for MyAvatar" + << " packetLength = " << packet.size() + << " offset = " << offset; + // this packet is just bad, so we pretend that we unpacked it ALL + return packet.size() - offset; +} + void MyAvatar::sendKillAvatar() { QByteArray killPacket = byteArrayWithPopulatedHeader(PacketTypeKillAvatar); NodeList::getInstance()->broadcastToNodes(killPacket, NodeSet() << NodeType::AvatarMixer); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 3544fb1a61..2363ee26e1 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -71,6 +71,8 @@ public: void jump() { _shouldJump = true; }; bool isMyAvatar() { return true; } + + virtual int parseDataAtOffset(const QByteArray& packet, int offset); static void sendKillAvatar(); diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 4e57e311eb..35e9938dc7 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,8 @@ #include "AvatarData.h" +quint64 DEFAULT_FILTERED_LOG_EXPIRY = 20 * USECS_PER_SECOND; + using namespace std; QNetworkAccessManager* AvatarData::networkAccessManager = NULL; @@ -42,7 +45,8 @@ AvatarData::AvatarData() : _displayNameBoundingRect(), _displayNameTargetAlpha(0.0f), _displayNameAlpha(0.0f), - _billboard() + _billboard(), + _debugLogExpiry(0) { } @@ -176,7 +180,6 @@ QByteArray AvatarData::toByteArray() { // read data in packet starting at byte offset and return number of bytes parsed int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { - // lazily allocate memory for HeadData in case we're not an Avatar instance if (!_headData) { _headData = new HeadData(this); @@ -189,102 +192,204 @@ int AvatarData::parseDataAtOffset(const QByteArray& packet, int offset) { const unsigned char* startPosition = reinterpret_cast(packet.data()) + offset; const unsigned char* sourceBuffer = startPosition; + + // 50 bytes of "plain old data" (POD) + // 1 byte for messageSize (0) + // 1 byte for pupilSize + // 1 byte for numJoints (0) + int minPossibleSize = 53; - // Body world position - memcpy(&_position, sourceBuffer, sizeof(float) * 3); - sourceBuffer += sizeof(float) * 3; - - // Body rotation (NOTE: This needs to become a quaternion to save two bytes) - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyYaw); - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyPitch); - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyRoll); - - // Body scale - sourceBuffer += unpackFloatRatioFromTwoByte(sourceBuffer, _targetScale); - - // Head rotation (NOTE: This needs to become a quaternion to save two bytes) - float headYaw, headPitch, headRoll; - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headYaw); - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headPitch); - sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headRoll); - _headData->setYaw(headYaw); - _headData->setPitch(headPitch); - _headData->setRoll(headRoll); - - // Head position relative to pelvis - memcpy(&_headData->_leanSideways, sourceBuffer, sizeof(_headData->_leanSideways)); - sourceBuffer += sizeof(float); - memcpy(&_headData->_leanForward, sourceBuffer, sizeof(_headData->_leanForward)); - sourceBuffer += sizeof(_headData->_leanForward); - - // Lookat Position - memcpy(&_headData->_lookAtPosition, sourceBuffer, sizeof(_headData->_lookAtPosition)); - sourceBuffer += sizeof(_headData->_lookAtPosition); - - // Instantaneous audio loudness (used to drive facial animation) - memcpy(&_headData->_audioLoudness, sourceBuffer, sizeof(float)); - sourceBuffer += sizeof(float); - - // the rest is a chat message - int chatMessageSize = *sourceBuffer++; - _chatMessage = string((char*)sourceBuffer, chatMessageSize); - sourceBuffer += chatMessageSize * sizeof(char); - - // voxel sending features... - unsigned char bitItems = 0; - bitItems = (unsigned char)*sourceBuffer++; - - // key state, stored as a semi-nibble in the bitItems - _keyState = (KeyState)getSemiNibbleAt(bitItems,KEY_STATE_START_BIT); - - // hand state, stored as a semi-nibble in the bitItems - _handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT); - - _headData->_isFaceshiftConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED); - - _isChatCirclingEnabled = oneAtBit(bitItems, IS_CHAT_CIRCLING_ENABLED); - - // If it is connected, pack up the data - if (_headData->_isFaceshiftConnected) { - memcpy(&_headData->_leftEyeBlink, sourceBuffer, sizeof(float)); - sourceBuffer += sizeof(float); - - memcpy(&_headData->_rightEyeBlink, sourceBuffer, sizeof(float)); - sourceBuffer += sizeof(float); - - memcpy(&_headData->_averageLoudness, sourceBuffer, sizeof(float)); - sourceBuffer += sizeof(float); - - memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); - sourceBuffer += sizeof(float); - - _headData->_blendshapeCoefficients.resize(*sourceBuffer++); - memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, - _headData->_blendshapeCoefficients.size() * sizeof(float)); - sourceBuffer += _headData->_blendshapeCoefficients.size() * sizeof(float); + int maxAvailableSize = packet.size() - offset; + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet at the start: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; } + + { // Body world position, rotation, and scale + // position + memcpy(&_position, sourceBuffer, sizeof(float) * 3); + sourceBuffer += sizeof(float) * 3; + + // rotation (NOTE: This needs to become a quaternion to save two bytes) + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyYaw); + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyPitch); + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &_bodyRoll); + + // scale + sourceBuffer += unpackFloatRatioFromTwoByte(sourceBuffer, _targetScale); + } // 20 bytes - // pupil dilation - sourceBuffer += unpackFloatFromByte(sourceBuffer, _headData->_pupilDilation, 1.0f); + { // Head rotation + //(NOTE: This needs to become a quaternion to save two bytes) + float headYaw, headPitch, headRoll; + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headYaw); + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headPitch); + sourceBuffer += unpackFloatAngleFromTwoByte((uint16_t*) sourceBuffer, &headRoll); + _headData->setYaw(headYaw); + _headData->setPitch(headPitch); + _headData->setRoll(headRoll); + } // 6 bytes + + // Head lean (relative to pelvis) + { + memcpy(&_headData->_leanSideways, sourceBuffer, sizeof(_headData->_leanSideways)); + sourceBuffer += sizeof(float); + memcpy(&_headData->_leanForward, sourceBuffer, sizeof(_headData->_leanForward)); + sourceBuffer += sizeof(float); + } // 8 bytes + + { // Lookat Position + memcpy(&_headData->_lookAtPosition, sourceBuffer, sizeof(_headData->_lookAtPosition)); + sourceBuffer += sizeof(_headData->_lookAtPosition); + } // 12 bytes + + { // AudioLoudness + // Instantaneous audio loudness (used to drive facial animation) + memcpy(&_headData->_audioLoudness, sourceBuffer, sizeof(float)); + sourceBuffer += sizeof(float); + } // 4 bytes + + // chat + int chatMessageSize = *sourceBuffer++; + minPossibleSize += chatMessageSize; + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet before ChatMessage: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; + } + { // chat payload + _chatMessage = string((char*)sourceBuffer, chatMessageSize); + sourceBuffer += chatMessageSize * sizeof(char); + } // 1 + chatMessageSize bytes + + { // bitFlags and face data + unsigned char bitItems = 0; + bitItems = (unsigned char)*sourceBuffer++; + + // key state, stored as a semi-nibble in the bitItems + _keyState = (KeyState)getSemiNibbleAt(bitItems,KEY_STATE_START_BIT); + + // hand state, stored as a semi-nibble in the bitItems + _handState = getSemiNibbleAt(bitItems,HAND_STATE_START_BIT); + + _headData->_isFaceshiftConnected = oneAtBit(bitItems, IS_FACESHIFT_CONNECTED); + _isChatCirclingEnabled = oneAtBit(bitItems, IS_CHAT_CIRCLING_ENABLED); + + if (_headData->_isFaceshiftConnected) { + minPossibleSize += 4 * sizeof(float) + 1; // four floats + one byte for blendDataSize + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet after BitItems: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; + } + // unpack face data + memcpy(&_headData->_leftEyeBlink, sourceBuffer, sizeof(float)); + sourceBuffer += sizeof(float); + + memcpy(&_headData->_rightEyeBlink, sourceBuffer, sizeof(float)); + sourceBuffer += sizeof(float); + + memcpy(&_headData->_averageLoudness, sourceBuffer, sizeof(float)); + sourceBuffer += sizeof(float); + + memcpy(&_headData->_browAudioLift, sourceBuffer, sizeof(float)); + sourceBuffer += sizeof(float); + + int numCoefficients = (int)(*sourceBuffer++); + int blendDataSize = numCoefficients * sizeof(float); + minPossibleSize += blendDataSize; + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet after Blendshapes: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; + } + + _headData->_blendshapeCoefficients.resize(numCoefficients); + memcpy(_headData->_blendshapeCoefficients.data(), sourceBuffer, blendDataSize); + sourceBuffer += numCoefficients * sizeof(float); + + //bitItemsDataSize = 4 * sizeof(float) + 1 + blendDataSize; + } + } // 1 + bitItemsDataSize bytes + + { // pupil dilation + sourceBuffer += unpackFloatFromByte(sourceBuffer, _headData->_pupilDilation, 1.0f); + } // 1 byte // joint data - int jointCount = *sourceBuffer++; - _jointData.resize(jointCount); - unsigned char validity = 0; - int validityBit = 0; - for (int i = 0; i < jointCount; i++) { - if (validityBit == 0) { - validity = *sourceBuffer++; - } - _jointData[i].valid = (bool)(validity & (1 << validityBit)); - validityBit = (validityBit + 1) % BITS_IN_BYTE; + int numJoints = *sourceBuffer++; + int bytesOfValidity = (int)ceil((float)numJoints / 8.f); + minPossibleSize += bytesOfValidity; + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet after JointValidityBits: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; } - for (int i = 0; i < jointCount; i++) { - JointData& data = _jointData[i]; - if (data.valid) { - sourceBuffer += unpackOrientationQuatFromBytes(sourceBuffer, data.rotation); + int numValidJoints = 0; + _jointData.resize(numJoints); + { // validity bits + unsigned char validity = 0; + int validityBit = 0; + for (int i = 0; i < numJoints; i++) { + if (validityBit == 0) { + validity = *sourceBuffer++; + } + bool valid = (bool)(validity & (1 << validityBit)); + if (valid) { + ++numValidJoints; + } + _jointData[i].valid = valid; + validityBit = (validityBit + 1) % BITS_IN_BYTE; } } + // 1 + bytesOfValidity bytes + + minPossibleSize += numValidJoints * 8; // 8 bytes per quaternion + if (minPossibleSize > maxAvailableSize) { + // this packet is malformed so we pretend to read to the end + quint64 now = usecTimestampNow(); + if (now > _debugLogExpiry) { + qDebug() << "Malformed AvatarData packet after JointData: minPossibleSize = " << minPossibleSize + << " but maxAvailableSize = " << maxAvailableSize; + _debugLogExpiry = now + DEFAULT_FILTERED_LOG_EXPIRY; + } + return maxAvailableSize; + } + + { // joint data + for (int i = 0; i < numJoints; i++) { + JointData& data = _jointData[i]; + if (data.valid) { + sourceBuffer += unpackOrientationQuatFromBytes(sourceBuffer, data.rotation); + } + } + } // numJoints * 8 bytes return sourceBuffer - startPosition; } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index c7a93daef5..4048b9bebe 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -33,10 +33,10 @@ typedef unsigned long long quint64; #include #include #include -#include #include #include #include +#include #include #include @@ -256,6 +256,8 @@ protected: static QNetworkAccessManager* networkAccessManager; + quint64 _debugLogExpiry; + private: // privatize the copy constructor and assignment operator so they cannot be called AvatarData(const AvatarData&); From 1214794f050e5acf19c437448a41b468767230ad Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 20 Mar 2014 17:03:28 -0700 Subject: [PATCH 02/27] QMap is not used by AvatarData --- libraries/avatars/src/AvatarData.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 4048b9bebe..bcf32ec8e2 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -36,7 +36,6 @@ typedef unsigned long long quint64; #include #include #include -#include #include #include From 3cb3cb81c40e3d08d177cfcd7ae27bf9e57dabb7 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 21 Mar 2014 14:15:02 -0700 Subject: [PATCH 03/27] fixes to voxel server crashes on client shutdown --- .../src/octree/OctreeQueryNode.cpp | 8 +++++ .../src/octree/OctreeQueryNode.h | 1 + .../src/octree/OctreeSendThread.cpp | 35 +++++++++++-------- .../src/octree/OctreeSendThread.h | 4 +++ libraries/shared/src/NodeList.cpp | 18 ++++++++-- libraries/shared/src/NodeList.h | 3 +- 6 files changed, 50 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/octree/OctreeQueryNode.cpp b/assignment-client/src/octree/OctreeQueryNode.cpp index 2ceb9e1040..e0ff29effd 100644 --- a/assignment-client/src/octree/OctreeQueryNode.cpp +++ b/assignment-client/src/octree/OctreeQueryNode.cpp @@ -67,6 +67,14 @@ OctreeQueryNode::~OctreeQueryNode() { } +void OctreeQueryNode::deleteLater() { + _isShuttingDown = true; + if (_octreeSendThread) { + _octreeSendThread->setIsShuttingDown(); + } + OctreeQuery::deleteLater(); +} + void OctreeQueryNode::initializeOctreeSendThread(OctreeServer* octreeServer, const QUuid& nodeUUID) { // Create octree sending thread... diff --git a/assignment-client/src/octree/OctreeQueryNode.h b/assignment-client/src/octree/OctreeQueryNode.h index eab8cb5d0a..b7e68e805d 100644 --- a/assignment-client/src/octree/OctreeQueryNode.h +++ b/assignment-client/src/octree/OctreeQueryNode.h @@ -27,6 +27,7 @@ class OctreeQueryNode : public OctreeQuery { public: OctreeQueryNode(); virtual ~OctreeQueryNode(); + virtual void deleteLater(); void init(); // called after creation to set up some virtual items virtual PacketType getMyPacketType() const = 0; diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 9c04c4a1ad..a215d9b3c3 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -5,6 +5,8 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // +#include + #include #include #include @@ -21,7 +23,9 @@ OctreeSendThread::OctreeSendThread(const QUuid& nodeUUID, OctreeServer* myServer _nodeUUID(nodeUUID), _myServer(myServer), _packetData(), - _nodeMissingCount(0) + _nodeMissingCount(0), + _processLock(), + _isShuttingDown(false) { QString safeServerName("Octree"); if (_myServer) { @@ -43,8 +47,19 @@ OctreeSendThread::~OctreeSendThread() { OctreeServer::clientDisconnected(); } +void OctreeSendThread::setIsShuttingDown() { + QMutexLocker locker(&_processLock); // this will cause us to wait till the process loop is complete + _isShuttingDown = true; +} + bool OctreeSendThread::process() { + QMutexLocker locker(&_processLock); + + if (_isShuttingDown) { + return false; // exit early if we're shutting down + } + const int MAX_NODE_MISSING_CHECKS = 10; if (_nodeMissingCount > MAX_NODE_MISSING_CHECKS) { qDebug() << "our target node:" << _nodeUUID << "has been missing the last" << _nodeMissingCount @@ -56,7 +71,10 @@ bool OctreeSendThread::process() { // don't do any send processing until the initial load of the octree is complete... if (_myServer->isInitialLoadComplete()) { - SharedNodePointer node = NodeList::getInstance()->nodeWithUUID(_nodeUUID); + + // see if we can get access to our node, but don't wait on the lock, if the nodeList is busy + // it might not return a node that is known, but that's ok we can handle that case. + SharedNodePointer node = NodeList::getInstance()->nodeWithUUID(_nodeUUID, false); if (node) { _nodeMissingCount = 0; @@ -113,19 +131,6 @@ int OctreeSendThread::handlePacketSend(const SharedNodePointer& node, bool packetSent = false; // did we send a packet? int packetsSent = 0; - // double check that the node has an active socket, otherwise, don't send... - - quint64 lockWaitStart = usecTimestampNow(); - QMutexLocker locker(&node->getMutex()); - quint64 lockWaitEnd = usecTimestampNow(); - float lockWaitElapsedUsec = (float)(lockWaitEnd - lockWaitStart); - OctreeServer::trackNodeWaitTime(lockWaitElapsedUsec); - - const HifiSockAddr* nodeAddress = node->getActiveSocket(); - if (!nodeAddress) { - return packetsSent; // without sending... - } - // Here's where we check to see if this packet is a duplicate of the last packet. If it is, we will silently // obscure the packet and not send it. This allows the callers and upper level logic to not need to know about // this rate control savings. diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index 39c27911b0..ab88121ee8 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -23,6 +23,8 @@ class OctreeSendThread : public GenericThread { public: OctreeSendThread(const QUuid& nodeUUID, OctreeServer* myServer); virtual ~OctreeSendThread(); + + void setIsShuttingDown(); static quint64 _totalBytes; static quint64 _totalWastedBytes; @@ -45,6 +47,8 @@ private: OctreePacketData _packetData; int _nodeMissingCount; + QMutex _processLock; // don't allow us to have our nodeData, or our thread to be deleted while we're processing + bool _isShuttingDown; }; #endif // __octree_server__OctreeSendThread__ diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 6d699322de..797e90fa51 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -308,9 +308,21 @@ int NodeList::findNodeAndUpdateWithDataFromPacket(const QByteArray& packet) { return 0; } -SharedNodePointer NodeList::nodeWithUUID(const QUuid& nodeUUID) { - QMutexLocker locker(&_nodeHashMutex); - return _nodeHash.value(nodeUUID); +SharedNodePointer NodeList::nodeWithUUID(const QUuid& nodeUUID, bool blockingLock) { + SharedNodePointer node; + // if caller wants us to block and guarantee the correct answer, then honor that request + if (blockingLock) { + // this will block till we can get access + QMutexLocker locker(&_nodeHashMutex); + node = _nodeHash.value(nodeUUID); + } else { + // some callers are willing to get wrong answers but not block + if (_nodeHashMutex.tryLock()) { + node = _nodeHash.value(nodeUUID); + _nodeHashMutex.unlock(); + } + } + return node; } SharedNodePointer NodeList::sendingNodeForPacket(const QByteArray& packet) { diff --git a/libraries/shared/src/NodeList.h b/libraries/shared/src/NodeList.h index 590a2ce83f..d6328a1303 100644 --- a/libraries/shared/src/NodeList.h +++ b/libraries/shared/src/NodeList.h @@ -101,7 +101,8 @@ public: QByteArray constructPingReplyPacket(const QByteArray& pingPacket); void pingPublicAndLocalSocketsForInactiveNode(const SharedNodePointer& node); - SharedNodePointer nodeWithUUID(const QUuid& nodeUUID); + /// passing false for blockingLock, will tryLock, and may return NULL when a node with the UUID actually does exist + SharedNodePointer nodeWithUUID(const QUuid& nodeUUID, bool blockingLock = true); SharedNodePointer sendingNodeForPacket(const QByteArray& packet); SharedNodePointer addOrUpdateNode(const QUuid& uuid, char nodeType, From 9f96129fb17eeac1347dea772e4fa24c40f887a5 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 21 Mar 2014 14:22:26 -0700 Subject: [PATCH 04/27] Fixed #2379 --- examples/editVoxels.js | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/editVoxels.js b/examples/editVoxels.js index ac0b67407b..34635d6e7d 100644 --- a/examples/editVoxels.js +++ b/examples/editVoxels.js @@ -742,6 +742,7 @@ function trackKeyReleaseEvent(event) { if (event.text == "TAB") { editToolsOn = !editToolsOn; moveTools(); + showPreviewGuides(); Audio.playSound(clickSound, audioOptions); } From c10d78c3975869477c8dc2c682990cda9ca053c1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 21 Mar 2014 14:29:19 -0700 Subject: [PATCH 05/27] Fixed bug when mouse leaves window. --- examples/inspect.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/inspect.js b/examples/inspect.js index 9292450784..8ff759c127 100644 --- a/examples/inspect.js +++ b/examples/inspect.js @@ -165,7 +165,7 @@ function keyReleaseEvent(event) { } function mousePressEvent(event) { - if (alt) { + if (alt && !isActive) { isActive = true; mouseLastX = event.x; mouseLastY = event.y; From 2575b33662c62f1dc65576ba0ccaeb38df6418c9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 14:29:57 -0700 Subject: [PATCH 06/27] reinstate a cutoff when the audio-mixer is struggling --- assignment-client/src/audio/AudioMixer.cpp | 73 +++++++++++++++---- assignment-client/src/audio/AudioMixer.h | 5 +- .../src/audio/AudioMixerClientData.cpp | 6 +- .../src/audio/AudioMixerClientData.h | 2 +- 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f5bed48a86..ba659d8379 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -53,6 +53,8 @@ const short JITTER_BUFFER_MSECS = 12; const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); +const float LOUDNESS_TO_DISTANCE_RATIO = 0.00305f; + const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; void attachNewBufferToNode(Node *newNode) { @@ -64,10 +66,7 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _trailingSleepRatio(1.0f), - _minSourceLoudnessInFrame(1.0f), - _maxSourceLoudnessInFrame(0.0f), - _loudnessCutoffRatio(0.0f), - _minRequiredLoudness(0.0f) + _minAudabilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2) { } @@ -83,8 +82,15 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf // if the two buffer pointers do not match then these are different buffers glm::vec3 relativePosition = bufferToAdd->getPosition() - listeningNodeBuffer->getPosition(); + + if (bufferToAdd->getAverageLoudness() / glm::length(relativePosition) <= _minAudabilityThreshold) { + // according to mixer performance we have decided this does not get to be mixed in + // bail out + return; + } + glm::quat inverseOrientation = glm::inverse(listeningNodeBuffer->getOrientation()); - + float distanceSquareToSource = glm::dot(relativePosition, relativePosition); float radius = 0.0f; @@ -305,8 +311,7 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { if ((*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) - && otherNodeBuffer->willBeAddedToMix() - && otherNodeBuffer->getAverageLoudness() > _minRequiredLoudness) { + && otherNodeBuffer->willBeAddedToMix()) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } @@ -357,19 +362,61 @@ void AudioMixer::run() { + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; + float audabilityCutoffRatio = 0; while (!_isFinished) { - - _minSourceLoudnessInFrame = 1.0f; - _maxSourceLoudnessInFrame = 0.0f; foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { - ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES, - _minSourceLoudnessInFrame, - _maxSourceLoudnessInFrame); + ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES); } } + + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30; + const float CUTOFF_EPSILON = 0.0001; + + const int TRAILING_AVERAGE_FRAMES = 100; + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1 - CURRENT_FRAME_RATIO; + + if (usecToSleep < 0) { + usecToSleep = 0; + } + + _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + + (usecToSleep * CURRENT_FRAME_RATIO / (float) BUFFER_SEND_INTERVAL_USECS); + + float lastCutoffRatio = audabilityCutoffRatio; + bool hasRatioChanged = false; + + if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { + // we're struggling - change our min required loudness to reduce some load + audabilityCutoffRatio += (1 - audabilityCutoffRatio) / 2; + + qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << audabilityCutoffRatio; + hasRatioChanged = true; + } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { + // we've recovered and can back off the required loudness + audabilityCutoffRatio -= audabilityCutoffRatio / 2; + + if (audabilityCutoffRatio < CUTOFF_EPSILON) { + audabilityCutoffRatio = 0; + } + + qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << audabilityCutoffRatio; + hasRatioChanged = true; + } + + + + if (hasRatioChanged) { + // set out min required loudness from the new ratio + _minAudabilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2 * (1 - audabilityCutoffRatio)); + qDebug() << "Minimum loudness required to be mixed is now" << _minAudabilityThreshold; + } foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 4ba8cdebd3..93656aabff 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -41,10 +41,7 @@ private: int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)]; float _trailingSleepRatio; - float _minSourceLoudnessInFrame; - float _maxSourceLoudnessInFrame; - float _loudnessCutoffRatio; - float _minRequiredLoudness; + float _minAudabilityThreshold; }; #endif /* defined(__hifi__AudioMixer__) */ diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index f370a1509f..381c80cb09 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -83,20 +83,16 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, - float& currentMinLoudness, - float& currentMaxLoudness) { +void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples) { for (unsigned int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) { // this is a ring buffer that is ready to go // set its flag so we know to push its buffer when all is said and done _ringBuffers[i]->setWillBeAddedToMix(true); - // calculate the average loudness for the next NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL // that would be mixed in _ringBuffers[i]->updateAverageLoudnessForBoundarySamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - } } } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index d41563bbca..7f44390ec5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -25,7 +25,7 @@ public: AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, float& currentMinLoudness, float& currentMaxLoudness); + void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples); void pushBuffersAfterFrameSend(); private: std::vector _ringBuffers; From 3721ce8e438c06381803c3bef54a57b98f09fae6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 21 Mar 2014 14:30:12 -0700 Subject: [PATCH 07/27] Added boundaries to altitude and changed radius rate --- examples/inspect.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/inspect.js b/examples/inspect.js index 8ff759c127..2443eadf9b 100644 --- a/examples/inspect.js +++ b/examples/inspect.js @@ -14,9 +14,11 @@ // Dragging the mouse will move your camera according to the mode you are in. // +var PI = 3.14 // No need for something more precise + var AZIMUTH_RATE = 90.0; var ALTITUDE_RATE = 200.0; -var RADIUS_RATE = 20.0; +var RADIUS_RATE = 1.0 / 100.0; var PAN_RATE = 50.0; var alt = false; @@ -46,7 +48,7 @@ var altitude = 0.0; function handleRadialMode(dx, dy) { azimuth += dx / AZIMUTH_RATE; - radius += radius * dy / RADIUS_RATE; + radius += radius * dy * RADIUS_RATE; if (radius < 1) { radius = 1; } @@ -61,6 +63,12 @@ function handleRadialMode(dx, dy) { function handleOrbitMode(dx, dy) { azimuth += dx / AZIMUTH_RATE; altitude += dy / ALTITUDE_RATE; + if (altitude > PI / 2.0) { + altitude = PI / 2.0; + } + if (altitude < -PI / 2.0) { + altitude = -PI / 2.0; + } vector = { x:(Math.cos(altitude) * Math.cos(azimuth)) * radius, y:Math.sin(altitude) * radius, From 53eaea6a2215835f578e8b6a8a8b9c4fa84df2bf Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Fri, 21 Mar 2014 14:30:57 -0700 Subject: [PATCH 08/27] Removed rest of orbit/pan code in editVoxels.js --- examples/editVoxels.js | 136 +++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 73 deletions(-) diff --git a/examples/editVoxels.js b/examples/editVoxels.js index 34635d6e7d..a66d82f9ba 100644 --- a/examples/editVoxels.js +++ b/examples/editVoxels.js @@ -32,8 +32,6 @@ var MIN_PASTE_VOXEL_SCALE = .256; var zFightingSizeAdjust = 0.002; // used to adjust preview voxels to prevent z fighting var previewLineWidth = 1.5; -var oldMode = Camera.getMode(); -var trackAsOrbitOrPan = false; var isAdding = false; var isExtruding = false; var extrudeDirection = { x: 0, y: 0, z: 0 }; @@ -614,8 +612,6 @@ function showPreviewVoxel() { var guidePosition; if (trackAsRecolor || recolorToolSelected || trackAsEyedropper || eyedropperToolSelected) { Overlays.editOverlay(voxelPreview, { visible: true }); - } else if (trackAsOrbitOrPan) { - Overlays.editOverlay(voxelPreview, { visible: false }); } else if (voxelToolSelected && !isExtruding) { Overlays.editOverlay(voxelPreview, { visible: true }); } else if (isExtruding) { @@ -706,15 +702,12 @@ function showPreviewGuides() { } function trackMouseEvent(event) { - if (!trackAsOrbitOrPan) { - trackLastMouseX = event.x; - trackLastMouseY = event.y; - trackAsDelete = event.isControl; - trackAsRecolor = event.isShifted; - trackAsEyedropper = event.isMeta; - trackAsOrbitOrPan = event.isAlt; // TODO: double check this...?? - showPreviewGuides(); - } + trackLastMouseX = event.x; + trackLastMouseY = event.y; + trackAsDelete = event.isControl; + trackAsRecolor = event.isShifted; + trackAsEyedropper = event.isMeta; + showPreviewGuides(); } function trackKeyPressEvent(event) { @@ -789,67 +782,64 @@ function mousePressEvent(event) { return; } - // no clicking on overlays while in panning mode - if (!trackAsOrbitOrPan) { - var clickedOnSomething = false; - var clickedOverlay = Overlays.getOverlayAtPoint({x: event.x, y: event.y}); - - // If the user clicked on the thumb, handle the slider logic - if (clickedOverlay == thumb) { - isMovingSlider = true; - thumbClickOffsetX = event.x - (sliderX + thumbX); // this should be the position of the mouse relative to the thumb - clickedOnSomething = true; - - Overlays.editOverlay(thumb, { imageURL: toolIconUrl + "voxel-size-slider-handle.svg", }); - - } else if (clickedOverlay == voxelTool) { - voxelToolSelected = true; - recolorToolSelected = false; - eyedropperToolSelected = false; - moveTools(); - clickedOnSomething = true; - } else if (clickedOverlay == recolorTool) { - voxelToolSelected = false; - recolorToolSelected = true; - eyedropperToolSelected = false; - moveTools(); - clickedOnSomething = true; - } else if (clickedOverlay == eyedropperTool) { - voxelToolSelected = false; - recolorToolSelected = false; - eyedropperToolSelected = true; - moveTools(); - clickedOnSomething = true; - } else if (clickedOverlay == slider) { - - if (event.x < sliderX + minThumbX) { - thumbX -= thumbDeltaPerStep; - calcScaleFromThumb(thumbX); - } - - if (event.x > sliderX + maxThumbX) { - thumbX += thumbDeltaPerStep; - calcScaleFromThumb(thumbX); - } - - moveTools(); - clickedOnSomething = true; - } else { - // if the user clicked on one of the color swatches, update the selectedSwatch - for (s = 0; s < numColors; s++) { - if (clickedOverlay == swatches[s]) { - whichColor = s; - moveTools(); - clickedOnSomething = true; - break; - } - } + var clickedOnSomething = false; + var clickedOverlay = Overlays.getOverlayAtPoint({x: event.x, y: event.y}); + + // If the user clicked on the thumb, handle the slider logic + if (clickedOverlay == thumb) { + isMovingSlider = true; + thumbClickOffsetX = event.x - (sliderX + thumbX); // this should be the position of the mouse relative to the thumb + clickedOnSomething = true; + + Overlays.editOverlay(thumb, { imageURL: toolIconUrl + "voxel-size-slider-handle.svg", }); + + } else if (clickedOverlay == voxelTool) { + voxelToolSelected = true; + recolorToolSelected = false; + eyedropperToolSelected = false; + moveTools(); + clickedOnSomething = true; + } else if (clickedOverlay == recolorTool) { + voxelToolSelected = false; + recolorToolSelected = true; + eyedropperToolSelected = false; + moveTools(); + clickedOnSomething = true; + } else if (clickedOverlay == eyedropperTool) { + voxelToolSelected = false; + recolorToolSelected = false; + eyedropperToolSelected = true; + moveTools(); + clickedOnSomething = true; + } else if (clickedOverlay == slider) { + + if (event.x < sliderX + minThumbX) { + thumbX -= thumbDeltaPerStep; + calcScaleFromThumb(thumbX); } - if (clickedOnSomething) { - return; // no further processing + + if (event.x > sliderX + maxThumbX) { + thumbX += thumbDeltaPerStep; + calcScaleFromThumb(thumbX); + } + + moveTools(); + clickedOnSomething = true; + } else { + // if the user clicked on one of the color swatches, update the selectedSwatch + for (s = 0; s < numColors; s++) { + if (clickedOverlay == swatches[s]) { + whichColor = s; + moveTools(); + clickedOnSomething = true; + break; + } } } - + if (clickedOnSomething) { + return; // no further processing + } + // TODO: does any of this stuff need to execute if we're panning or orbiting? trackMouseEvent(event); // used by preview support mouseX = event.x; @@ -1072,7 +1062,7 @@ function mouseMoveEvent(event) { } - if (!trackAsOrbitOrPan && isMovingSlider) { + if (isMovingSlider) { thumbX = (event.x - thumbClickOffsetX) - sliderX; if (thumbX < minThumbX) { thumbX = minThumbX; @@ -1082,7 +1072,7 @@ function mouseMoveEvent(event) { } calcScaleFromThumb(thumbX); - } else if (!trackAsOrbitOrPan && isAdding) { + } else if (isAdding) { // Watch the drag direction to tell which way to 'extrude' this voxel if (!isExtruding) { var pickRay = Camera.computePickRay(event.x, event.y); From 18e563797636cfc29bfec3a4d2441d0c0d83db4c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 14:33:11 -0700 Subject: [PATCH 09/27] remove some extra spaces --- assignment-client/src/audio/AudioMixer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index ba659d8379..8c504be763 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -410,8 +410,6 @@ void AudioMixer::run() { hasRatioChanged = true; } - - if (hasRatioChanged) { // set out min required loudness from the new ratio _minAudabilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2 * (1 - audabilityCutoffRatio)); From 3d9aa6cc9f7275f8bc6cf6185b53a7b288930965 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 14:36:44 -0700 Subject: [PATCH 10/27] fix a spelling mistake and convert some ints to floats --- assignment-client/src/audio/AudioMixer.cpp | 22 +++++++++++----------- assignment-client/src/audio/AudioMixer.h | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 8c504be763..5b889b979c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -66,7 +66,7 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _trailingSleepRatio(1.0f), - _minAudabilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2) + _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2) { } @@ -83,7 +83,7 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf glm::vec3 relativePosition = bufferToAdd->getPosition() - listeningNodeBuffer->getPosition(); - if (bufferToAdd->getAverageLoudness() / glm::length(relativePosition) <= _minAudabilityThreshold) { + if (bufferToAdd->getAverageLoudness() / glm::length(relativePosition) <= _minAudibilityThreshold) { // according to mixer performance we have decided this does not get to be mixed in // bail out return; @@ -372,13 +372,13 @@ void AudioMixer::run() { } } - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30; - const float CUTOFF_EPSILON = 0.0001; + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30f; + const float CUTOFF_EPSILON = 0.0001f; const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1 - CURRENT_FRAME_RATIO; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; if (usecToSleep < 0) { usecToSleep = 0; @@ -392,17 +392,17 @@ void AudioMixer::run() { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load - audabilityCutoffRatio += (1 - audabilityCutoffRatio) / 2; + audabilityCutoffRatio += (1.0f - audabilityCutoffRatio) / 2.0f; qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << audabilityCutoffRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { // we've recovered and can back off the required loudness - audabilityCutoffRatio -= audabilityCutoffRatio / 2; + audabilityCutoffRatio -= audabilityCutoffRatio / 2.0f; if (audabilityCutoffRatio < CUTOFF_EPSILON) { - audabilityCutoffRatio = 0; + audabilityCutoffRatio = 0.0f; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" @@ -412,8 +412,8 @@ void AudioMixer::run() { if (hasRatioChanged) { // set out min required loudness from the new ratio - _minAudabilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2 * (1 - audabilityCutoffRatio)); - qDebug() << "Minimum loudness required to be mixed is now" << _minAudabilityThreshold; + _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - audabilityCutoffRatio)); + qDebug() << "Minimum loudness required to be mixed is now" << _minAudibilityThreshold; } foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 93656aabff..9d731a5c9c 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -41,7 +41,7 @@ private: int16_t _clientSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)]; float _trailingSleepRatio; - float _minAudabilityThreshold; + float _minAudibilityThreshold; }; #endif /* defined(__hifi__AudioMixer__) */ From 98f014d13d1c7251a2b092b31283c1c6076d0adb Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 14:37:31 -0700 Subject: [PATCH 11/27] add another check to avoid any mixing for a silent frame --- assignment-client/src/audio/AudioMixer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 5b889b979c..a63fad4cd1 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -311,7 +311,8 @@ void AudioMixer::prepareMixForListeningNode(Node* node) { if ((*otherNode != *node || otherNodeBuffer->shouldLoopbackForNode()) - && otherNodeBuffer->willBeAddedToMix()) { + && otherNodeBuffer->willBeAddedToMix() + && otherNodeBuffer->getAverageLoudness() > 0) { addBufferToMixForListeningNodeWithBuffer(otherNodeBuffer, nodeRingBuffer); } } From c99f5a4ae6cb1ffac8b9e2b033fcd41584dd1861 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 14:38:01 -0700 Subject: [PATCH 12/27] int to float conversion in the constructor --- assignment-client/src/audio/AudioMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index a63fad4cd1..15e1b864eb 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -66,7 +66,7 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _trailingSleepRatio(1.0f), - _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2) + _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f) { } From 2c8c43094faa9166de5ba233229736524fc3e6a2 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 21 Mar 2014 14:46:46 -0700 Subject: [PATCH 13/27] CR feedback --- libraries/shared/src/NodeList.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 797e90fa51..2b131ab7ff 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -315,12 +315,9 @@ SharedNodePointer NodeList::nodeWithUUID(const QUuid& nodeUUID, bool blockingLoc // this will block till we can get access QMutexLocker locker(&_nodeHashMutex); node = _nodeHash.value(nodeUUID); - } else { - // some callers are willing to get wrong answers but not block - if (_nodeHashMutex.tryLock()) { - node = _nodeHash.value(nodeUUID); - _nodeHashMutex.unlock(); - } + } else if (_nodeHashMutex.tryLock()) { // some callers are willing to get wrong answers but not block + node = _nodeHash.value(nodeUUID); + _nodeHashMutex.unlock(); } return node; } From 4661553acb865d1d887e5256cac870d5298320e2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 15:40:08 -0700 Subject: [PATCH 14/27] check for cutoff change only every TRAILING_AVERAGE_FRAMES --- assignment-client/src/audio/AudioMixer.cpp | 54 +++++++++++++--------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 15e1b864eb..cab7af5fd0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -364,6 +364,9 @@ void AudioMixer::run() { int usecToSleep = BUFFER_SEND_INTERVAL_USECS; float audabilityCutoffRatio = 0; + + const int TRAILING_AVERAGE_FRAMES = 100; + int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; while (!_isFinished) { @@ -374,10 +377,9 @@ void AudioMixer::run() { } const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.30f; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float CUTOFF_EPSILON = 0.0001f; - const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; @@ -391,30 +393,36 @@ void AudioMixer::run() { float lastCutoffRatio = audabilityCutoffRatio; bool hasRatioChanged = false; - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - // we're struggling - change our min required loudness to reduce some load - audabilityCutoffRatio += (1.0f - audabilityCutoffRatio) / 2.0f; - - qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { + if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { + // we're struggling - change our min required loudness to reduce some load + audabilityCutoffRatio += (1.0f - audabilityCutoffRatio) / 2.0f; + + qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << audabilityCutoffRatio; - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { - // we've recovered and can back off the required loudness - audabilityCutoffRatio -= audabilityCutoffRatio / 2.0f; - - if (audabilityCutoffRatio < CUTOFF_EPSILON) { - audabilityCutoffRatio = 0.0f; + hasRatioChanged = true; + } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { + // we've recovered and can back off the required loudness + audabilityCutoffRatio -= audabilityCutoffRatio / 2.0f; + + if (audabilityCutoffRatio < CUTOFF_EPSILON) { + audabilityCutoffRatio = 0.0f; + } + + qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << audabilityCutoffRatio; + hasRatioChanged = true; } - qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << audabilityCutoffRatio; - hasRatioChanged = true; - } - - if (hasRatioChanged) { - // set out min required loudness from the new ratio - _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - audabilityCutoffRatio)); - qDebug() << "Minimum loudness required to be mixed is now" << _minAudibilityThreshold; + if (hasRatioChanged) { + // set out min required loudness from the new ratio + _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - audabilityCutoffRatio)); + qDebug() << "Minimum loudness required to be mixed is now" << _minAudibilityThreshold; + + framesSinceCutoffEvent = 0; + } + } else { + framesSinceCutoffEvent++; } foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { From 75bc64010434f4b8b61dc18ebf9f7884fa30eae4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 15:42:58 -0700 Subject: [PATCH 15/27] fix some indentation is audio mixer cutoff code --- assignment-client/src/audio/AudioMixer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index cab7af5fd0..fd7e5e360d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -399,7 +399,7 @@ void AudioMixer::run() { audabilityCutoffRatio += (1.0f - audabilityCutoffRatio) / 2.0f; qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << audabilityCutoffRatio; + << lastCutoffRatio << "and is now" << audabilityCutoffRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { // we've recovered and can back off the required loudness @@ -410,7 +410,7 @@ void AudioMixer::run() { } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << audabilityCutoffRatio; + << lastCutoffRatio << "and is now" << audabilityCutoffRatio; hasRatioChanged = true; } From 50007d7f4b8918839bfa2a410b53d9c3281ef775 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 15:43:39 -0700 Subject: [PATCH 16/27] fix a debug message for audability threshold --- assignment-client/src/audio/AudioMixer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index fd7e5e360d..dc066f8857 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -415,9 +415,9 @@ void AudioMixer::run() { } if (hasRatioChanged) { - // set out min required loudness from the new ratio + // set out min audability threshold from the new ratio _minAudibilityThreshold = LOUDNESS_TO_DISTANCE_RATIO / (2.0f * (1.0f - audabilityCutoffRatio)); - qDebug() << "Minimum loudness required to be mixed is now" << _minAudibilityThreshold; + qDebug() << "Minimum audability required to be mixed is now" << _minAudibilityThreshold; framesSinceCutoffEvent = 0; } From 438e5bd235ccc01f9be457e388c0e482d686c424 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 21 Mar 2014 15:46:06 -0700 Subject: [PATCH 17/27] fix crash on domain restart in Agent, also have Agent properly shutdown on call to Script.stop() --- assignment-client/src/Agent.cpp | 5 +++++ assignment-client/src/Agent.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 4eb6b17260..d32d6a3fd7 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -189,4 +189,9 @@ void Agent::run() { _scriptEngine.setScriptContents(scriptContents); _scriptEngine.run(); + setFinished(true); +} + +void Agent::aboutToFinish() { + _scriptEngine.stop(); } diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index b638c39356..0a61bd73f7 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -41,6 +41,8 @@ public: bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); } void setIsListeningToAudioStream(bool isListeningToAudioStream) { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); } + + virtual void aboutToFinish(); public slots: void run(); From dc2d050187568c8606a01c6aedc9b2af23d36da0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 15:47:21 -0700 Subject: [PATCH 18/27] remove sleep time debug now that the audio-mixer reports struggle --- assignment-client/src/audio/AudioMixer.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index dc066f8857..6754b1eee7 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -454,10 +454,7 @@ void AudioMixer::run() { if (usecToSleep > 0) { usleep(usecToSleep); - } else { - qDebug() << "AudioMixer loop took" << -usecToSleep << "of extra time. Not sleeping."; } - } delete[] clientMixBuffer; From bad209bf09f1b315ede15fe758bdd324ef0fa4a9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 15:53:00 -0700 Subject: [PATCH 19/27] use a trailing average for a quieter loudness --- libraries/audio/src/AudioRingBuffer.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 800f4f64ee..efe62de515 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -67,7 +67,15 @@ void AudioRingBuffer::updateAverageLoudnessForBoundarySamples(int numSamples) { nextLoudness /= numSamples; nextLoudness /= MAX_SAMPLE_VALUE; - _averageLoudness = nextLoudness; + const int TRAILING_AVERAGE_FRAMES = 100; + const float CURRENT_FRAME_RATIO = 1 / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1 - CURRENT_FRAME_RATIO; + + if (nextLoudness >= _averageLoudness) { + _averageLoudness = nextLoudness; + } else { + _averageLoudness = (_averageLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness); + } } qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { From 34f90759803f61eaa2ccfaca9d8fd6c140879711 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 16:47:41 -0700 Subject: [PATCH 20/27] use a linear change to audability ratio --- assignment-client/src/audio/AudioMixer.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6754b1eee7..f2c766460a 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -379,6 +379,7 @@ void AudioMixer::run() { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float CUTOFF_EPSILON = 0.0001f; + const float CUTOFF_DELTA = 0.05; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; @@ -396,17 +397,21 @@ void AudioMixer::run() { if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { // we're struggling - change our min required loudness to reduce some load - audabilityCutoffRatio += (1.0f - audabilityCutoffRatio) / 2.0f; + audabilityCutoffRatio += CUTOFF_DELTA; + + if (audabilityCutoffRatio > 1) { + audabilityCutoffRatio = 1; + } qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" << lastCutoffRatio << "and is now" << audabilityCutoffRatio; hasRatioChanged = true; } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && audabilityCutoffRatio != 0) { // we've recovered and can back off the required loudness - audabilityCutoffRatio -= audabilityCutoffRatio / 2.0f; + audabilityCutoffRatio -= CUTOFF_DELTA; - if (audabilityCutoffRatio < CUTOFF_EPSILON) { - audabilityCutoffRatio = 0.0f; + if (audabilityCutoffRatio < 0) { + audabilityCutoffRatio = 0; } qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" From 8ecd9aa0c04d024506ea7c6631b0fa92fde05251 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 17:16:54 -0700 Subject: [PATCH 21/27] change audibility delta to 2%, don't hit 1.0 --- assignment-client/src/audio/AudioMixer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f2c766460a..e95dcb0ec3 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -379,7 +379,7 @@ void AudioMixer::run() { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float CUTOFF_EPSILON = 0.0001f; - const float CUTOFF_DELTA = 0.05; + const float CUTOFF_DELTA = 0.02; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; @@ -399,8 +399,8 @@ void AudioMixer::run() { // we're struggling - change our min required loudness to reduce some load audabilityCutoffRatio += CUTOFF_DELTA; - if (audabilityCutoffRatio > 1) { - audabilityCutoffRatio = 1; + if (audabilityCutoffRatio >= 1) { + audabilityCutoffRatio = 1 - CUTOFF_DELTA; } qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" From 88b91b9eb70673c219c6297b5ff9a11fa5f1ceaa Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 17:17:11 -0700 Subject: [PATCH 22/27] change an int to a float --- assignment-client/src/audio/AudioMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index e95dcb0ec3..bb60abacc0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -379,7 +379,7 @@ void AudioMixer::run() { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; const float CUTOFF_EPSILON = 0.0001f; - const float CUTOFF_DELTA = 0.02; + const float CUTOFF_DELTA = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; From c70bbda10a1249f40464384ba070d335bf4e7cb0 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Fri, 21 Mar 2014 18:16:28 -0700 Subject: [PATCH 23/27] kill local voxels on domain change --- interface/src/Application.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 030e3bc6fd..9563cbf3b4 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3252,6 +3252,9 @@ void Application::domainChanged(const QString& domainHostname) { // reset the particle renderer _particles.clear(); + + // reset the voxels renderer + _voxels.killLocalVoxels(); } void Application::connectedToDomain(const QString& hostname) { From 88348b12d2158104fcfc5c7283584b2960aa30b0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 18:16:47 -0700 Subject: [PATCH 24/27] fix a divide by zero and output number of clients mixed in last frame --- assignment-client/src/audio/AudioMixer.cpp | 17 ++++++++++++++--- assignment-client/src/audio/AudioMixer.h | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bb60abacc0..d2a498eef1 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -66,7 +66,8 @@ void attachNewBufferToNode(Node *newNode) { AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _trailingSleepRatio(1.0f), - _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f) + _minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f), + _numClientsMixedInFrame(0) { } @@ -80,15 +81,22 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf if (bufferToAdd != listeningNodeBuffer) { // if the two buffer pointers do not match then these are different buffers - glm::vec3 relativePosition = bufferToAdd->getPosition() - listeningNodeBuffer->getPosition(); - if (bufferToAdd->getAverageLoudness() / glm::length(relativePosition) <= _minAudibilityThreshold) { + float distanceBetween = glm::length(relativePosition); + + if (distanceBetween < EPSILON) { + distanceBetween = EPSILON; + } + + if (bufferToAdd->getAverageLoudness() / distanceBetween <= _minAudibilityThreshold) { // according to mixer performance we have decided this does not get to be mixed in // bail out return; } + ++_numClientsMixedInFrame; + glm::quat inverseOrientation = glm::inverse(listeningNodeBuffer->getOrientation()); float distanceSquareToSource = glm::dot(relativePosition, relativePosition); @@ -441,6 +449,9 @@ void AudioMixer::run() { nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); } } + + qDebug() << "There were" << _numClientsMixedInFrame << "clients mixed in the last frame"; + _numClientsMixedInFrame = 0; // push forward the next output pointers for any audio buffers we used foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 9d731a5c9c..d8f1a8eb85 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -42,6 +42,7 @@ private: float _trailingSleepRatio; float _minAudibilityThreshold; + int _numClientsMixedInFrame; }; #endif /* defined(__hifi__AudioMixer__) */ From abd6c8a747cfdcbdfc22b5fd6bb472388d5807ef Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 18:28:12 -0700 Subject: [PATCH 25/27] fix trailing average on ring buffer, remove unused constant --- assignment-client/src/audio/AudioMixer.cpp | 1 - libraries/audio/src/AudioRingBuffer.cpp | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d2a498eef1..94fdbcea9d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -386,7 +386,6 @@ void AudioMixer::run() { const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; - const float CUTOFF_EPSILON = 0.0001f; const float CUTOFF_DELTA = 0.02f; const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index efe62de515..4f0df744ee 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -70,11 +70,16 @@ void AudioRingBuffer::updateAverageLoudnessForBoundarySamples(int numSamples) { const int TRAILING_AVERAGE_FRAMES = 100; const float CURRENT_FRAME_RATIO = 1 / TRAILING_AVERAGE_FRAMES; const float PREVIOUS_FRAMES_RATIO = 1 - CURRENT_FRAME_RATIO; + const float LOUDNESS_EPSILON = 0.001; if (nextLoudness >= _averageLoudness) { _averageLoudness = nextLoudness; } else { _averageLoudness = (_averageLoudness * PREVIOUS_FRAMES_RATIO) + (CURRENT_FRAME_RATIO * nextLoudness); + + if (_averageLoudness < LOUDNESS_EPSILON) { + _averageLoudness = 0; + } } } From af4fb2603135dbbffa28e85a798d7d62b43f88b6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 21 Mar 2014 18:44:59 -0700 Subject: [PATCH 26/27] floats are important --- assignment-client/src/audio/AudioMixer.cpp | 1 - libraries/audio/src/AudioRingBuffer.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 94fdbcea9d..ba17847773 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -449,7 +449,6 @@ void AudioMixer::run() { } } - qDebug() << "There were" << _numClientsMixedInFrame << "clients mixed in the last frame"; _numClientsMixedInFrame = 0; // push forward the next output pointers for any audio buffers we used diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 4f0df744ee..4732cc77d1 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -68,9 +68,9 @@ void AudioRingBuffer::updateAverageLoudnessForBoundarySamples(int numSamples) { nextLoudness /= MAX_SAMPLE_VALUE; const int TRAILING_AVERAGE_FRAMES = 100; - const float CURRENT_FRAME_RATIO = 1 / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1 - CURRENT_FRAME_RATIO; - const float LOUDNESS_EPSILON = 0.001; + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + const float LOUDNESS_EPSILON = 0.01f; if (nextLoudness >= _averageLoudness) { _averageLoudness = nextLoudness; From 98041b11564a6e092ffaa3d23ac4e17a9a8cde24 Mon Sep 17 00:00:00 2001 From: Leonardo Murillo Date: Fri, 21 Mar 2014 22:05:56 -0600 Subject: [PATCH 27/27] Trivial commit - testing build on new server --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c374901d2..f8a6725ed8 100644 --- a/README.md +++ b/README.md @@ -63,4 +63,5 @@ To test things out you'll want to run the Interface client. To access your local domain in Interface, open your Preferences -- on OS X this is available in the Interface menu, on Linux you'll find it in the File menu. Enter "localhost" in the "Domain server" field. -If everything worked you should see that you are connected to at least one server. Nice work! +If everything worked you should see that you are connected to at least one server. +Nice work!