diff --git a/audio-mixer/src/PositionalAudioRingBuffer.cpp b/audio-mixer/src/PositionalAudioRingBuffer.cpp index 60ec9a425e..9980b48a3e 100644 --- a/audio-mixer/src/PositionalAudioRingBuffer.cpp +++ b/audio-mixer/src/PositionalAudioRingBuffer.cpp @@ -8,6 +8,7 @@ #include +#include #include #include "PositionalAudioRingBuffer.h" @@ -16,25 +17,81 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer() : AudioRingBuffer(false), _position(0.0f, 0.0f, 0.0f), _orientation(0.0f, 0.0f, 0.0f, 0.0f), - _willBeAddedToMix(false) + _willBeAddedToMix(false), + _listenMode(AudioRingBuffer::NORMAL), + _listenRadius(0.0f) { } +PositionalAudioRingBuffer::~PositionalAudioRingBuffer() { +} + +bool PositionalAudioRingBuffer::isListeningToNode(Node& other) const { + switch (_listenMode) { + default: + case AudioRingBuffer::NORMAL: + return true; + break; + + case AudioRingBuffer::OMNI_DIRECTIONAL_POINT: { + PositionalAudioRingBuffer* otherNodeBuffer = (PositionalAudioRingBuffer*) other.getLinkedData(); + float distance = glm::distance(_position, otherNodeBuffer->_position); + return distance <= _listenRadius; + break; + } + case AudioRingBuffer::SELECTED_SOURCES: + for (int i = 0; i < _listenSources.size(); i++) { + if (other.getNodeID() == _listenSources[i]) { + return true; + } + } + return false; + break; + } +} + + int PositionalAudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) { unsigned char* currentBuffer = sourceBuffer + numBytesForPacketHeader(sourceBuffer); + currentBuffer += sizeof(uint16_t); // the source ID + currentBuffer += parseListenModeData(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); currentBuffer += parsePositionalData(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); currentBuffer += parseAudioSamples(currentBuffer, numBytes - (currentBuffer - sourceBuffer)); return currentBuffer - sourceBuffer; } +int PositionalAudioRingBuffer::parseListenModeData(unsigned char* sourceBuffer, int numBytes) { + unsigned char* currentBuffer = sourceBuffer; + + memcpy(&_listenMode, currentBuffer, sizeof(_listenMode)); + currentBuffer += sizeof(_listenMode); + + if (_listenMode == AudioRingBuffer::OMNI_DIRECTIONAL_POINT) { + memcpy(&_listenRadius, currentBuffer, sizeof(_listenRadius)); + currentBuffer += sizeof(_listenRadius); + } else if (_listenMode == AudioRingBuffer::SELECTED_SOURCES) { + int listenSourcesCount; + memcpy(&listenSourcesCount, currentBuffer, sizeof(listenSourcesCount)); + currentBuffer += sizeof(listenSourcesCount); + for (int i = 0; i < listenSourcesCount; i++) { + int sourceID; + memcpy(&sourceID, currentBuffer, sizeof(sourceID)); + currentBuffer += sizeof(sourceID); + _listenSources.push_back(sourceID); + } + } + + return currentBuffer - sourceBuffer; +} + int PositionalAudioRingBuffer::parsePositionalData(unsigned char* sourceBuffer, int numBytes) { unsigned char* currentBuffer = sourceBuffer; memcpy(&_position, currentBuffer, sizeof(_position)); currentBuffer += sizeof(_position); - + memcpy(&_orientation, currentBuffer, sizeof(_orientation)); currentBuffer += sizeof(_orientation); @@ -63,6 +120,6 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { return true; } } - + printf("packet mismatch...\n"); return false; } diff --git a/audio-mixer/src/PositionalAudioRingBuffer.h b/audio-mixer/src/PositionalAudioRingBuffer.h index 41e775e74a..6c7ee9ce3f 100644 --- a/audio-mixer/src/PositionalAudioRingBuffer.h +++ b/audio-mixer/src/PositionalAudioRingBuffer.h @@ -9,6 +9,7 @@ #ifndef __hifi__PositionalAudioRingBuffer__ #define __hifi__PositionalAudioRingBuffer__ +#include #include #include @@ -16,9 +17,11 @@ class PositionalAudioRingBuffer : public AudioRingBuffer { public: PositionalAudioRingBuffer(); + ~PositionalAudioRingBuffer(); int parseData(unsigned char* sourceBuffer, int numBytes); int parsePositionalData(unsigned char* sourceBuffer, int numBytes); + int parseListenModeData(unsigned char* sourceBuffer, int numBytes); bool shouldBeAddedToMix(int numJitterBufferSamples); @@ -27,6 +30,9 @@ public: const glm::vec3& getPosition() const { return _position; } const glm::quat& getOrientation() const { return _orientation; } + + bool isListeningToNode(Node& other) const; + ListenMode getListeningMode() const { return _listenMode; } protected: // disallow copying of PositionalAudioRingBuffer objects @@ -36,6 +42,10 @@ protected: glm::vec3 _position; glm::quat _orientation; bool _willBeAddedToMix; + + ListenMode _listenMode; + float _listenRadius; + std::vector _listenSources; }; #endif /* defined(__hifi__PositionalAudioRingBuffer__) */ diff --git a/audio-mixer/src/main.cpp b/audio-mixer/src/main.cpp index 239eb7a32d..9fc088a8bf 100644 --- a/audio-mixer/src/main.cpp +++ b/audio-mixer/src/main.cpp @@ -141,7 +141,6 @@ int main(int argc, const char* argv[]) { for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { PositionalAudioRingBuffer* positionalRingBuffer = (PositionalAudioRingBuffer*) node->getLinkedData(); - if (positionalRingBuffer && positionalRingBuffer->shouldBeAddedToMix(JITTER_BUFFER_SAMPLES)) { // this is a ring buffer that is ready to go // set its flag so we know to push its buffer when all is said and done @@ -159,168 +158,171 @@ int main(int argc, const char* argv[]) { // zero out the client mix for this node memset(clientSamples, 0, sizeof(clientSamples)); + // loop through all other nodes that have sufficient audio to mix for (NodeList::iterator otherNode = nodeList->begin(); otherNode != nodeList->end(); otherNode++) { if (((PositionalAudioRingBuffer*) otherNode->getLinkedData())->willBeAddedToMix() && (otherNode != node || (otherNode == node && nodeRingBuffer->shouldLoopbackForNode()))) { - PositionalAudioRingBuffer* otherNodeBuffer = (PositionalAudioRingBuffer*) otherNode->getLinkedData(); + // based on our listen mode we will do this mixing... + if (nodeRingBuffer->isListeningToNode(*otherNode)) { + float bearingRelativeAngleToSource = 0.0f; + float attenuationCoefficient = 1.0f; + int numSamplesDelay = 0; + float weakChannelAmplitudeRatio = 1.0f; - float bearingRelativeAngleToSource = 0.0f; - float attenuationCoefficient = 1.0f; - int numSamplesDelay = 0; - float weakChannelAmplitudeRatio = 1.0f; + stk::TwoPole* otherNodeTwoPole = NULL; - stk::TwoPole* otherNodeTwoPole = NULL; - - if (otherNode != node) { + // only do axis/distance attenuation when in normal mode + if (otherNode != node && nodeRingBuffer->getListeningMode() == AudioRingBuffer::NORMAL) { - glm::vec3 listenerPosition = nodeRingBuffer->getPosition(); - glm::vec3 relativePosition = otherNodeBuffer->getPosition() - nodeRingBuffer->getPosition(); - glm::quat inverseOrientation = glm::inverse(nodeRingBuffer->getOrientation()); + glm::vec3 listenerPosition = nodeRingBuffer->getPosition(); + glm::vec3 relativePosition = otherNodeBuffer->getPosition() - nodeRingBuffer->getPosition(); + glm::quat inverseOrientation = glm::inverse(nodeRingBuffer->getOrientation()); - float distanceSquareToSource = glm::dot(relativePosition, relativePosition); - float radius = 0.0f; + float distanceSquareToSource = glm::dot(relativePosition, relativePosition); + float radius = 0.0f; - if (otherNode->getType() == NODE_TYPE_AUDIO_INJECTOR) { - InjectedAudioRingBuffer* injectedBuffer = (InjectedAudioRingBuffer*) otherNodeBuffer; - radius = injectedBuffer->getRadius(); - attenuationCoefficient *= injectedBuffer->getAttenuationRatio(); - } + if (otherNode->getType() == NODE_TYPE_AUDIO_INJECTOR) { + InjectedAudioRingBuffer* injectedBuffer = (InjectedAudioRingBuffer*) otherNodeBuffer; + radius = injectedBuffer->getRadius(); + attenuationCoefficient *= injectedBuffer->getAttenuationRatio(); + } - if (radius == 0 || (distanceSquareToSource > radius * radius)) { - // this is either not a spherical source, or the listener is outside the sphere + if (radius == 0 || (distanceSquareToSource > radius * radius)) { + // this is either not a spherical source, or the listener is outside the sphere - if (radius > 0) { - // this is a spherical source - the distance used for the coefficient - // needs to be the closest point on the boundary to the source + if (radius > 0) { + // this is a spherical source - the distance used for the coefficient + // needs to be the closest point on the boundary to the source - // ovveride the distance to the node with the distance to the point on the - // boundary of the sphere - distanceSquareToSource -= (radius * radius); + // ovveride the distance to the node with the distance to the point on the + // boundary of the sphere + distanceSquareToSource -= (radius * radius); - } else { - // calculate the angle delivery for off-axis attenuation - glm::vec3 rotatedListenerPosition = glm::inverse(otherNodeBuffer->getOrientation()) - * relativePosition; + } else { + // calculate the angle delivery for off-axis attenuation + glm::vec3 rotatedListenerPosition = glm::inverse(otherNodeBuffer->getOrientation()) + * relativePosition; - float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedListenerPosition)); + float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedListenerPosition)); - const float MAX_OFF_AXIS_ATTENUATION = 0.2f; - const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; + const float MAX_OFF_AXIS_ATTENUATION = 0.2f; + const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; - float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / 90.0f)); + float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + + (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / 90.0f)); - // multiply the current attenuation coefficient by the calculated off axis coefficient - attenuationCoefficient *= offAxisCoefficient; + // multiply the current attenuation coefficient by the calculated off axis coefficient + attenuationCoefficient *= offAxisCoefficient; + } + + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + const float DISTANCE_SCALE = 2.5f; + const float GEOMETRIC_AMPLITUDE_SCALAR = 0.3f; + const float DISTANCE_LOG_BASE = 2.5f; + const float DISTANCE_SCALE_LOG = logf(DISTANCE_SCALE) / logf(DISTANCE_LOG_BASE); + + // calculate the distance coefficient using the distance to this node + float distanceCoefficient = powf(GEOMETRIC_AMPLITUDE_SCALAR, + DISTANCE_SCALE_LOG + + (0.5f * logf(distanceSquareToSource) / logf(DISTANCE_LOG_BASE)) - 1); + distanceCoefficient = std::min(1.0f, distanceCoefficient); + + // multiply the current attenuation coefficient by the distance coefficient + attenuationCoefficient *= distanceCoefficient; + + // project the rotated source position vector onto the XZ plane + rotatedSourcePosition.y = 0.0f; + + // produce an oriented angle about the y-axis + bearingRelativeAngleToSource = glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedSourcePosition), + glm::vec3(0.0f, 1.0f, 0.0f)); + + const float PHASE_AMPLITUDE_RATIO_AT_90 = 0.5; + + // figure out the number of samples of delay and the ratio of the amplitude + // in the weak channel for audio spatialization + float sinRatio = fabsf(sinf(glm::radians(bearingRelativeAngleToSource))); + numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; + weakChannelAmplitudeRatio = 1 - (PHASE_AMPLITUDE_RATIO_AT_90 * sinRatio); + + // grab the TwoPole object for this source, add it if it doesn't exist + TwoPoleNodeMap& nodeTwoPoles = nodeRingBuffer->getTwoPoles(); + TwoPoleNodeMap::iterator twoPoleIterator = nodeTwoPoles.find(otherNode->getNodeID()); + + if (twoPoleIterator == nodeTwoPoles.end()) { + // setup the freeVerb effect for this source for this client + otherNodeTwoPole = nodeTwoPoles[otherNode->getNodeID()] = new stk::TwoPole; + } else { + otherNodeTwoPole = twoPoleIterator->second; + } + + // calculate the reasonance for this TwoPole based on angle to source + float TWO_POLE_CUT_OFF_FREQUENCY = 800.0f; + float TWO_POLE_MAX_FILTER_STRENGTH = 0.4f; + + otherNodeTwoPole->setResonance(TWO_POLE_CUT_OFF_FREQUENCY, + TWO_POLE_MAX_FILTER_STRENGTH + * fabsf(bearingRelativeAngleToSource) / 180.0f, + true); } + } + + int16_t* sourceBuffer = otherNodeBuffer->getNextOutput(); + + int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) + ? clientSamples + : clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) + ? clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL + : clientSamples; + + int16_t* delaySamplePointer = otherNodeBuffer->getNextOutput() == otherNodeBuffer->getBuffer() + ? otherNodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay + : otherNodeBuffer->getNextOutput() - numSamplesDelay; + + for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { + // load up the stkFrameBuffer with this source's samples + stkFrameBuffer[s] = (stk::StkFloat) sourceBuffer[s]; + } + + // perform the TwoPole effect on the stkFrameBuffer + if (otherNodeTwoPole) { + otherNodeTwoPole->tick(stkFrameBuffer); + } + + for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { + if (s < numSamplesDelay) { + // pull the earlier sample for the delayed channel + int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; - glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; - - const float DISTANCE_SCALE = 2.5f; - const float GEOMETRIC_AMPLITUDE_SCALAR = 0.3f; - const float DISTANCE_LOG_BASE = 2.5f; - const float DISTANCE_SCALE_LOG = logf(DISTANCE_SCALE) / logf(DISTANCE_LOG_BASE); - - // calculate the distance coefficient using the distance to this node - float distanceCoefficient = powf(GEOMETRIC_AMPLITUDE_SCALAR, - DISTANCE_SCALE_LOG + - (0.5f * logf(distanceSquareToSource) / logf(DISTANCE_LOG_BASE)) - 1); - distanceCoefficient = std::min(1.0f, distanceCoefficient); - - // multiply the current attenuation coefficient by the distance coefficient - attenuationCoefficient *= distanceCoefficient; - - // project the rotated source position vector onto the XZ plane - rotatedSourcePosition.y = 0.0f; - - // produce an oriented angle about the y-axis - bearingRelativeAngleToSource = glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedSourcePosition), - glm::vec3(0.0f, 1.0f, 0.0f)); - - const float PHASE_AMPLITUDE_RATIO_AT_90 = 0.5; - - // figure out the number of samples of delay and the ratio of the amplitude - // in the weak channel for audio spatialization - float sinRatio = fabsf(sinf(glm::radians(bearingRelativeAngleToSource))); - numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; - weakChannelAmplitudeRatio = 1 - (PHASE_AMPLITUDE_RATIO_AT_90 * sinRatio); - - // grab the TwoPole object for this source, add it if it doesn't exist - TwoPoleNodeMap& nodeTwoPoles = nodeRingBuffer->getTwoPoles(); - TwoPoleNodeMap::iterator twoPoleIterator = nodeTwoPoles.find(otherNode->getNodeID()); - - if (twoPoleIterator == nodeTwoPoles.end()) { - // setup the freeVerb effect for this source for this client - otherNodeTwoPole = nodeTwoPoles[otherNode->getNodeID()] = new stk::TwoPole; - } else { - otherNodeTwoPole = twoPoleIterator->second; + delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); } - - // calculate the reasonance for this TwoPole based on angle to source - float TWO_POLE_CUT_OFF_FREQUENCY = 800.0f; - float TWO_POLE_MAX_FILTER_STRENGTH = 0.4f; - - otherNodeTwoPole->setResonance(TWO_POLE_CUT_OFF_FREQUENCY, - TWO_POLE_MAX_FILTER_STRENGTH - * fabsf(bearingRelativeAngleToSource) / 180.0f, - true); - } - } - - int16_t* sourceBuffer = otherNodeBuffer->getNextOutput(); - - int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) - ? clientSamples - : clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) - ? clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL - : clientSamples; - - int16_t* delaySamplePointer = otherNodeBuffer->getNextOutput() == otherNodeBuffer->getBuffer() - ? otherNodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay - : otherNodeBuffer->getNextOutput() - numSamplesDelay; - - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { - // load up the stkFrameBuffer with this source's samples - stkFrameBuffer[s] = (stk::StkFloat) sourceBuffer[s]; - } - - // perform the TwoPole effect on the stkFrameBuffer - if (otherNodeTwoPole) { - otherNodeTwoPole->tick(stkFrameBuffer); - } - - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { - if (s < numSamplesDelay) { - // pull the earlier sample for the delayed channel - int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; - - delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); - } - int16_t currentSample = stkFrameBuffer[s] * attenuationCoefficient; + int16_t currentSample = stkFrameBuffer[s] * attenuationCoefficient; - goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); + goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); - if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - int sumSample = delayedChannel[s + numSamplesDelay] - + (currentSample * weakChannelAmplitudeRatio); - delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); - } + if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + int sumSample = delayedChannel[s + numSamplesDelay] + + (currentSample * weakChannelAmplitudeRatio); + delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); + } - if (s >= BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PHASE_DELAY_AT_90) { - // this could be a delayed sample on the next pass - // so store the affected back in the ARB - otherNodeBuffer->getNextOutput()[s] = (int16_t) stkFrameBuffer[s]; + if (s >= BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PHASE_DELAY_AT_90) { + // this could be a delayed sample on the next pass + // so store the affected back in the ARB + otherNodeBuffer->getNextOutput()[s] = (int16_t) stkFrameBuffer[s]; + } } } } @@ -340,7 +342,6 @@ int main(int argc, const char* argv[]) { if (nodeBuffer->getNextOutput() >= nodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { nodeBuffer->setNextOutput(nodeBuffer->getBuffer()); } - nodeBuffer->setWillBeAddedToMix(false); } } @@ -350,14 +351,15 @@ int main(int argc, const char* argv[]) { packetVersionMatch(packetData)) { if (packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO || packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO) { + + unsigned char* currentBuffer = packetData + numBytesForPacketHeader(packetData); + uint16_t sourceID; + memcpy(&sourceID, currentBuffer, sizeof(sourceID)); + Node* avatarNode = nodeList->addOrUpdateNode(nodeAddress, nodeAddress, NODE_TYPE_AGENT, - nodeList->getLastNodeID()); - - if (avatarNode->getNodeID() == nodeList->getLastNodeID()) { - nodeList->increaseNodeID(); - } + sourceID); nodeList->updateNodeWithData(nodeAddress, packetData, receivedBytes); diff --git a/avatar-mixer/src/main.cpp b/avatar-mixer/src/main.cpp index f8bc36f6e6..0677ffbc7e 100644 --- a/avatar-mixer/src/main.cpp +++ b/avatar-mixer/src/main.cpp @@ -36,7 +36,7 @@ const int AVATAR_LISTEN_PORT = 55444; -unsigned char *addNodeToBroadcastPacket(unsigned char *currentPosition, Node *nodeToAdd) { +unsigned char* addNodeToBroadcastPacket(unsigned char *currentPosition, Node *nodeToAdd) { currentPosition += packNodeId(currentPosition, nodeToAdd->getNodeID()); AvatarData *nodeData = (AvatarData *)nodeToAdd->getLinkedData(); @@ -51,6 +51,53 @@ void attachAvatarDataToNode(Node* newNode) { } } +// NOTE: some additional optimizations to consider. +// 1) use the view frustum to cull those avatars that are out of view. Since avatar data doesn't need to be present +// if the avatar is not in view or in the keyhole. +// 2) after culling for view frustum, sort order the avatars by distance, send the closest ones first. +// 3) if we need to rate limit the amount of data we send, we can use a distance weighted "semi-random" function to +// determine which avatars are included in the packet stream +// 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful). +void broadcastAvatarData(NodeList* nodeList, sockaddr* nodeAddress) { + static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE]; + static unsigned char avatarDataBuffer[MAX_PACKET_SIZE]; + unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0]; + int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA); + unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes; + int packetLength = currentBufferPosition - broadcastPacket; + int packetsSent = 0; + + // send back a packet with other active node data to this node + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + if (node->getLinkedData() && !socketMatch(nodeAddress, node->getActiveSocket())) { + unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node); + int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer; + + if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) { + memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength); + packetLength += avatarDataLength; + currentBufferPosition += avatarDataLength; + } else { + packetsSent++; + //printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength); + nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket); + + // reset the packet + currentBufferPosition = broadcastPacket + numHeaderBytes; + packetLength = currentBufferPosition - broadcastPacket; + + // copy the avatar that didn't fit into the next packet + memcpy(currentBufferPosition, &avatarDataBuffer[0], avatarDataLength); + packetLength += avatarDataLength; + currentBufferPosition += avatarDataLength; + } + } + } + packetsSent++; + //printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength); + nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket); +} + int main(int argc, const char* argv[]) { NodeList* nodeList = NodeList::createInstance(NODE_TYPE_AVATAR_MIXER, AVATAR_LISTEN_PORT); @@ -67,14 +114,11 @@ int main(int argc, const char* argv[]) { nodeList->startSilentNodeRemovalThread(); - sockaddr *nodeAddress = new sockaddr; - unsigned char *packetData = new unsigned char[MAX_PACKET_SIZE]; + sockaddr* nodeAddress = new sockaddr; ssize_t receivedBytes = 0; - unsigned char *broadcastPacket = new unsigned char[MAX_PACKET_SIZE]; - int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA); + unsigned char* packetData = new unsigned char[MAX_PACKET_SIZE]; - unsigned char* currentBufferPosition = NULL; uint16_t nodeID = 0; Node* avatarNode = NULL; @@ -104,17 +148,7 @@ int main(int argc, const char* argv[]) { // parse positional data from an node nodeList->updateNodeWithData(avatarNode, packetData, receivedBytes); case PACKET_TYPE_INJECT_AUDIO: - currentBufferPosition = broadcastPacket + numHeaderBytes; - - // send back a packet with other active node data to this node - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - if (node->getLinkedData() && !socketMatch(nodeAddress, node->getActiveSocket())) { - currentBufferPosition = addNodeToBroadcastPacket(currentBufferPosition, &*node); - } - } - - nodeList->getNodeSocket()->send(nodeAddress, broadcastPacket, currentBufferPosition - broadcastPacket); - + broadcastAvatarData(nodeList, nodeAddress); break; case PACKET_TYPE_AVATAR_VOXEL_URL: case PACKET_TYPE_AVATAR_FACE_VIDEO: diff --git a/injector/src/main.cpp b/injector/src/main.cpp index e2c1effc15..9caad19e60 100644 --- a/injector/src/main.cpp +++ b/injector/src/main.cpp @@ -40,11 +40,13 @@ bool hasInjectedAudioOnce = false; float sleepIntervalMin = 1.00; float sleepIntervalMax = 2.00; char *sourceAudioFile = NULL; -const char *allowedParameters = ":sc::a::f::t::r:"; +const char *allowedParameters = ":sc::a::f::t::r:l"; float floatArguments[4] = {0.0f, 0.0f, 0.0f, 0.0f}; unsigned char volume = DEFAULT_INJECTOR_VOLUME; float triggerDistance = 0.0f; float radius = 0.0f; +bool wantsLocalDomain = false; + void usage(void) { std::cout << "High Fidelity - Interface audio injector" << std::endl; @@ -54,6 +56,7 @@ void usage(void) { std::cout << " -f FILENAME Name of audio source file. Required - RAW format, 22050hz 16bit signed mono" << std::endl; std::cout << " -t FLOAT Trigger distance for injection. If not specified will loop constantly" << std::endl; std::cout << " -r FLOAT Radius for spherical source. If not specified injected audio is point source" << std::endl; + std::cout << " -l Local domain mode." << std::endl; } bool processParameters(int parameterCount, char* parameterData[]) { @@ -96,6 +99,9 @@ bool processParameters(int parameterCount, char* parameterData[]) { ::radius = atof(optarg); std::cout << "[DEBUG] Injector radius: " << optarg << std::endl; break; + case 'l': + ::wantsLocalDomain = true; + break; default: usage(); return false; @@ -111,6 +117,7 @@ void createAvatarDataForNode(Node* node) { } int main(int argc, char* argv[]) { + // new seed for random audio sleep times srand(time(0)); @@ -126,6 +133,11 @@ int main(int argc, char* argv[]) { // create an NodeList instance to handle communication with other nodes NodeList* nodeList = NodeList::createInstance(NODE_TYPE_AUDIO_INJECTOR, AUDIO_UDP_SEND_PORT); + if (::wantsLocalDomain) { + printf("Local Domain MODE!\n"); + nodeList->setDomainIPToLocalhost(); + } + // start the node list thread that will kill off nodes when they stop talking nodeList->startSilentNodeRemovalThread(); diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 9f8a02a12a..ae87d53939 100755 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -77,7 +77,7 @@ include_directories(external/fervor/) # run qt moc on qt-enabled headers qt4_wrap_cpp(INTERFACE_SRCS src/Application.h src/Webcam.h src/avatar/AvatarVoxelSystem.h - src/avatar/Face.h src/ui/BandwidthDialog.h) + src/avatar/Face.h src/ui/BandwidthDialog.h src/ui/VoxelStatsDialog.h) # create the executable, make it a bundle on OS X add_executable(${TARGET_NAME} MACOSX_BUNDLE ${INTERFACE_SRCS}) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index d1fa5f1dc1..c56a58ce90 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -56,6 +56,8 @@ #include #include +#include + #include "Application.h" #include "InterfaceConfig.h" #include "LogDisplay.h" @@ -174,6 +176,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _window(new QMainWindow(desktop())), _glWidget(new GLCanvas()), _bandwidthDialog(NULL), + _voxelStatsDialog(NULL), _displayLevels(false), _frameCount(0), _fps(120.0f), @@ -200,6 +203,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _mouseVoxelScale(1.0f / 1024.0f), _justEditedVoxel(false), _isLookingAtOtherAvatar(false), + _lookatIndicatorScale(1.0f), _paintOn(false), _dominantColor(0), _perfStatsOn(false), @@ -1147,6 +1151,21 @@ void Application::bandwidthDetailsClosed() { delete dlg; } +void Application::voxelStatsDetails() { + if (!_voxelStatsDialog) { + _voxelStatsDialog = new VoxelStatsDialog(_glWidget, &_voxelSceneStats); + connect(_voxelStatsDialog, SIGNAL(closed()), SLOT(voxelStatsDetailsClosed())); + _voxelStatsDialog->show(); + } + _voxelStatsDialog->raise(); +} + +void Application::voxelStatsDetailsClosed() { + QDialog* dlg = _voxelStatsDialog; + _voxelStatsDialog = NULL; + delete dlg; +} + void Application::editPreferences() { QDialog dialog(_glWidget); dialog.setWindowTitle("Interface Preferences"); @@ -1751,6 +1770,7 @@ void Application::initMenu() { (_bandwidthDisplayOn = toolsMenu->addAction("Bandwidth Display"))->setCheckable(true); _bandwidthDisplayOn->setChecked(true); toolsMenu->addAction("Bandwidth Details", this, SLOT(bandwidthDetails())); + toolsMenu->addAction("Voxel Stats Details", this, SLOT(voxelStatsDetails())); QMenu* voxelMenu = menuBar->addMenu("Voxels"); @@ -1835,6 +1855,11 @@ void Application::initMenu() { (_simulateLeapHand = debugMenu->addAction("Simulate Leap Hand"))->setCheckable(true); (_testRaveGlove = debugMenu->addAction("Test RaveGlove"))->setCheckable(true); + QMenu* audioDebugMenu = debugMenu->addMenu("Audio Debugging Tools"); + audioDebugMenu->addAction("Listen Mode Normal", this, SLOT(setListenModeNormal()), Qt::CTRL | Qt::Key_1); + audioDebugMenu->addAction("Listen Mode Point/Radius", this, SLOT(setListenModePoint()), Qt::CTRL | Qt::Key_2); + audioDebugMenu->addAction("Listen Mode Single Source", this, SLOT(setListenModeSingleSource()), Qt::CTRL | Qt::Key_3); + QMenu* settingsMenu = menuBar->addMenu("Settings"); (_settingsAutosave = settingsMenu->addAction("Autosave"))->setCheckable(true); _settingsAutosave->setChecked(true); @@ -1846,6 +1871,30 @@ void Application::initMenu() { _networkAccessManager = new QNetworkAccessManager(this); } +void Application::setListenModeNormal() { + _audio.setListenMode(AudioRingBuffer::NORMAL); +} + +void Application::setListenModePoint() { + _audio.setListenMode(AudioRingBuffer::OMNI_DIRECTIONAL_POINT); + _audio.setListenRadius(1.0); +} + +void Application::setListenModeSingleSource() { + _audio.setListenMode(AudioRingBuffer::SELECTED_SOURCES); + _audio.clearListenSources(); + + glm::vec3 mouseRayOrigin = _myAvatar.getMouseRayOrigin(); + glm::vec3 mouseRayDirection = _myAvatar.getMouseRayDirection(); + glm::vec3 eyePositionIgnored; + uint16_t nodeID; + + if (isLookingAtOtherAvatar(mouseRayOrigin, mouseRayDirection, eyePositionIgnored, nodeID)) { + _audio.addListenSource(nodeID); + } +} + + void Application::updateFrustumRenderModeAction() { switch (_frustumDrawingMode) { default: @@ -1934,7 +1983,10 @@ const float MAX_AVATAR_EDIT_VELOCITY = 1.0f; const float MAX_VOXEL_EDIT_DISTANCE = 20.0f; const float HEAD_SPHERE_RADIUS = 0.07; -bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& mouseRayDirection, glm::vec3& eyePosition) { + +bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& mouseRayDirection, + glm::vec3& eyePosition, uint16_t& nodeID) { + NodeList* nodeList = NodeList::getInstance(); for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { if (node->getLinkedData() != NULL && node->getType() == NODE_TYPE_AGENT) { @@ -1942,7 +1994,9 @@ bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& m glm::vec3 headPosition = avatar->getHead().getPosition(); if (rayIntersectsSphere(mouseRayOrigin, mouseRayDirection, headPosition, HEAD_SPHERE_RADIUS)) { eyePosition = avatar->getHead().getEyeLevelPosition(); + _lookatIndicatorScale = avatar->getScale(); _lookatOtherPosition = headPosition; + nodeID = avatar->getOwningNode()->getNodeID(); return true; } } @@ -1952,11 +2006,13 @@ bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& m void Application::renderLookatIndicator(glm::vec3 pointOfInterest, Camera& whichCamera) { - const float DISTANCE_FROM_HEAD_SPHERE = 0.1f; + const float DISTANCE_FROM_HEAD_SPHERE = 0.1f * _lookatIndicatorScale; + const float INDICATOR_RADIUS = 0.1f * _lookatIndicatorScale; const float YELLOW[] = { 1.0f, 1.0f, 0.0f }; + const int NUM_SEGMENTS = 30; glm::vec3 haloOrigin(pointOfInterest.x, pointOfInterest.y + DISTANCE_FROM_HEAD_SPHERE, pointOfInterest.z); glColor3f(YELLOW[0], YELLOW[1], YELLOW[2]); - renderCircle(haloOrigin, 0.1f, glm::vec3(0.0f, 1.0f, 0.0f), 30); + renderCircle(haloOrigin, INDICATOR_RADIUS, IDENTITY_UP, NUM_SEGMENTS); } void Application::update(float deltaTime) { @@ -1986,7 +2042,9 @@ void Application::update(float deltaTime) { // Set where I am looking based on my mouse ray (so that other people can see) glm::vec3 eyePosition; - if (_isLookingAtOtherAvatar = isLookingAtOtherAvatar(mouseRayOrigin, mouseRayDirection, eyePosition)) { + uint16_t ignored; + _isLookingAtOtherAvatar = isLookingAtOtherAvatar(mouseRayOrigin, mouseRayDirection, eyePosition, ignored); + if (_isLookingAtOtherAvatar) { // If the mouse is over another avatar's head... glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); @@ -2003,7 +2061,7 @@ void Application::update(float deltaTime) { glm::vec3 front = orientation * IDENTITY_FRONT; glm::vec3 up = orientation * IDENTITY_UP; glm::vec3 towardVoxel = getMouseVoxelWorldCoordinates(_mouseVoxelDragging) - - _myAvatar.getCameraPosition(); // is this an error? getCameraPosition dne + - _myAvatar.getCameraPosition(); towardVoxel = front * glm::length(towardVoxel); glm::vec3 lateralToVoxel = glm::cross(up, glm::normalize(towardVoxel)) * glm::length(towardVoxel); _voxelThrust = glm::vec3(0, 0, 0); @@ -2103,10 +2161,8 @@ void Application::update(float deltaTime) { // Leap finger-sensing device LeapManager::enableFakeFingers(_simulateLeapHand->isChecked() || _testRaveGlove->isChecked()); - LeapManager::nextFrame(); _myAvatar.getHand().setRaveGloveActive(_testRaveGlove->isChecked()); - _myAvatar.getHand().setLeapFingers(LeapManager::getFingerTips(), LeapManager::getFingerRoots()); - _myAvatar.getHand().setLeapHands(LeapManager::getHandPositions(), LeapManager::getHandNormals()); + LeapManager::nextFrame(_myAvatar); // Read serial port interface devices if (_serialHeadSensor.isActive()) { @@ -2193,6 +2249,9 @@ void Application::update(float deltaTime) { if (_bandwidthDialog) { _bandwidthDialog->update(); } + if (_voxelStatsDialog) { + _voxelStatsDialog->update(); + } // Update audio stats for procedural sounds #ifndef _WIN32 @@ -2250,7 +2309,9 @@ void Application::updateAvatar(float deltaTime) { _viewFrustum.computePickRay(MIDPOINT_OF_SCREEN, MIDPOINT_OF_SCREEN, screenCenterRayOrigin, screenCenterRayDirection); glm::vec3 eyePosition; - if (_isLookingAtOtherAvatar = isLookingAtOtherAvatar(screenCenterRayOrigin, screenCenterRayDirection, eyePosition)) { + uint16_t ignored; + _isLookingAtOtherAvatar = isLookingAtOtherAvatar(screenCenterRayOrigin, screenCenterRayDirection, eyePosition, ignored); + if (_isLookingAtOtherAvatar) { glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); } @@ -2278,7 +2339,7 @@ void Application::updateAvatar(float deltaTime) { // actually need to calculate the view frustum planes to send these details // to the server. loadViewFrustum(_myCamera, _viewFrustum); - _myAvatar.setCameraPosition(_viewFrustum.getPosition()); // setCameraPosition() dne + _myAvatar.setCameraPosition(_viewFrustum.getPosition()); _myAvatar.setCameraOrientation(_viewFrustum.getOrientation()); _myAvatar.setCameraFov(_viewFrustum.getFieldOfView()); _myAvatar.setCameraAspectRatio(_viewFrustum.getAspectRatio()); @@ -2845,27 +2906,19 @@ void Application::displayStats() { drawtext(10, statsVerticalOffset + 230, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); voxelStats.str(""); - voxelStats << "Voxels Created: " << _voxels.getVoxelsCreated() / 1000.f << "K (" << _voxels.getVoxelsCreatedPerSecondAverage() / 1000.f - << "Kps) "; + char* voxelDetails = _voxelSceneStats.getItemValue(VoxelSceneStats::ITEM_VOXELS); + voxelStats << "Voxels Sent from Server: " << voxelDetails; drawtext(10, statsVerticalOffset + 250, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); - - voxelStats.str(""); - voxelStats << "Voxels Colored: " << _voxels.getVoxelsColored() / 1000.f << "K (" << _voxels.getVoxelsColoredPerSecondAverage() / 1000.f - << "Kps) "; - drawtext(10, statsVerticalOffset + 270, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); - - voxelStats.str(""); - voxelStats << "Voxel Bits Read: " << _voxels.getVoxelsBytesRead() * 8.f / 1000000.f - << "M (" << _voxels.getVoxelsBytesReadPerSecondAverage() * 8.f / 1000000.f << " Mbps)"; - drawtext(10, statsVerticalOffset + 290,0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); voxelStats.str(""); - float voxelsBytesPerColored = _voxels.getVoxelsColored() - ? ((float) _voxels.getVoxelsBytesRead() / _voxels.getVoxelsColored()) - : 0; - - voxelStats << "Voxels Bits per Colored: " << voxelsBytesPerColored * 8; - drawtext(10, statsVerticalOffset + 310, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); + voxelDetails = _voxelSceneStats.getItemValue(VoxelSceneStats::ITEM_ELAPSED); + voxelStats << "Scene Send Time from Server: " << voxelDetails; + drawtext(10, statsVerticalOffset + 270, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); + + voxelStats.str(""); + voxelDetails = _voxelSceneStats.getItemValue(VoxelSceneStats::ITEM_ENCODE); + voxelStats << "Encode Time on Server: " << voxelDetails; + drawtext(10, statsVerticalOffset + 290, 0.10f, 0, 1.0, 0, (char *)voxelStats.str().c_str()); Node *avatarMixer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_AVATAR_MIXER); char avatarMixerStats[200]; @@ -3420,16 +3473,37 @@ void* Application::networkReceive(void* args) { case PACKET_TYPE_VOXEL_DATA_MONOCHROME: case PACKET_TYPE_Z_COMMAND: case PACKET_TYPE_ERASE_VOXEL: + case PACKET_TYPE_VOXEL_STATS: case PACKET_TYPE_ENVIRONMENT_DATA: { + + unsigned char* messageData = app->_incomingPacket; + ssize_t messageLength = bytesReceived; + + // note: PACKET_TYPE_VOXEL_STATS can have PACKET_TYPE_VOXEL_DATA or PACKET_TYPE_VOXEL_DATA_MONOCHROME + // immediately following them inside the same packet. So, we process the PACKET_TYPE_VOXEL_STATS first + // then process any remaining bytes as if it was another packet + if (messageData[0] == PACKET_TYPE_VOXEL_STATS) { + int statsMessageLength = app->_voxelSceneStats.unpackFromMessage(messageData, messageLength); + if (messageLength > statsMessageLength) { + messageData += statsMessageLength; + messageLength -= statsMessageLength; + if (!packetVersionMatch(messageData)) { + break; // bail since piggyback data doesn't match our versioning + } + } else { + break; // bail since no piggyback data + } + } // fall through to piggyback message + if (app->_renderVoxels->isChecked()) { Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { voxelServer->lock(); - if (app->_incomingPacket[0] == PACKET_TYPE_ENVIRONMENT_DATA) { - app->_environment.parseData(&senderAddress, app->_incomingPacket, bytesReceived); + if (messageData[0] == PACKET_TYPE_ENVIRONMENT_DATA) { + app->_environment.parseData(&senderAddress, messageData, messageLength); } else { - app->_voxels.parseData(app->_incomingPacket, bytesReceived); + app->_voxels.parseData(messageData, messageLength); } voxelServer->unlock(); diff --git a/interface/src/Application.h b/interface/src/Application.h index d29328c0b5..bee9bb8b20 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -43,6 +43,7 @@ #include "avatar/HandControl.h" #include "ui/BandwidthDialog.h" #include "ui/ChatEntry.h" +#include "ui/VoxelStatsDialog.h" class QAction; class QActionGroup; @@ -120,6 +121,9 @@ private slots: void bandwidthDetails(); void editPreferences(); void bandwidthDetailsClosed(); + + void voxelStatsDetails(); + void voxelStatsDetailsClosed(); void pair(); @@ -170,6 +174,10 @@ private slots: void copyVoxels(); void pasteVoxels(); void runTests(); + void setListenModeNormal(); + void setListenModePoint(); + void setListenModeSingleSource(); + void renderCoverageMap(); void renderCoverageMapsRecursively(CoverageMap* map); @@ -199,7 +207,9 @@ private: void init(); void update(float deltaTime); - bool isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& mouseRayDirection, glm::vec3& eyePosition); + bool isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& mouseRayDirection, + glm::vec3& eyePosition, uint16_t& nodeID); + void renderLookatIndicator(glm::vec3 pointOfInterest, Camera& whichCamera); void updateAvatar(float deltaTime); void loadViewFrustum(Camera& camera, ViewFrustum& viewFrustum); @@ -284,6 +294,7 @@ private: BandwidthMeter _bandwidthMeter; BandwidthDialog* _bandwidthDialog; + VoxelStatsDialog* _voxelStatsDialog; SerialInterface _serialHeadSensor; QNetworkAccessManager* _networkAccessManager; @@ -373,6 +384,7 @@ private: bool _isLookingAtOtherAvatar; glm::vec3 _lookatOtherPosition; + float _lookatIndicatorScale; bool _paintOn; // Whether to paint voxels as you fly around unsigned char _dominantColor; // The dominant color of the voxel we're painting @@ -416,6 +428,8 @@ private: ToolsPalette _palette; Swatch _swatch; + + VoxelSceneStats _voxelSceneStats; }; #endif /* defined(__interface__Application__) */ diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 41090584f3..a9aa61a34e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -112,7 +112,7 @@ inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* o // we need the amount of bytes in the buffer + 1 for type // + 12 for 3 floats for position + float for bearing + 1 attenuation byte - unsigned char dataPacket[BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes]; + unsigned char dataPacket[MAX_PACKET_SIZE]; PACKET_TYPE packetType = (Application::getInstance()->shouldEchoAudio()) ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO @@ -120,6 +120,33 @@ inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* o unsigned char* currentPacketPtr = dataPacket + populateTypeAndVersion(dataPacket, packetType); + // pack Source Data + uint16_t ownerID = NodeList::getInstance()->getOwnerID(); + memcpy(currentPacketPtr, &ownerID, sizeof(ownerID)); + currentPacketPtr += (sizeof(ownerID)); + leadingBytes += (sizeof(ownerID)); + + // pack Listen Mode Data + memcpy(currentPacketPtr, &_listenMode, sizeof(_listenMode)); + currentPacketPtr += (sizeof(_listenMode)); + leadingBytes += (sizeof(_listenMode)); + + if (_listenMode == AudioRingBuffer::OMNI_DIRECTIONAL_POINT) { + memcpy(currentPacketPtr, &_listenRadius, sizeof(_listenRadius)); + currentPacketPtr += (sizeof(_listenRadius)); + leadingBytes += (sizeof(_listenRadius)); + } else if (_listenMode == AudioRingBuffer::SELECTED_SOURCES) { + int listenSourceCount = _listenSources.size(); + memcpy(currentPacketPtr, &listenSourceCount, sizeof(listenSourceCount)); + currentPacketPtr += (sizeof(listenSourceCount)); + leadingBytes += (sizeof(listenSourceCount)); + for (int i = 0; i < listenSourceCount; i++) { + memcpy(currentPacketPtr, &_listenSources[i], sizeof(_listenSources[i])); + currentPacketPtr += sizeof(_listenSources[i]); + leadingBytes += sizeof(_listenSources[i]); + } + } + // memcpy the three float positions memcpy(currentPacketPtr, &headPosition, sizeof(headPosition)); currentPacketPtr += (sizeof(headPosition)); @@ -309,6 +336,24 @@ void Audio::reset() { _ringBuffer.reset(); } +void Audio::addListenSource(int sourceID) { + _listenSources.push_back(sourceID); +} + +void Audio::clearListenSources() { + _listenSources.clear(); +} + +void Audio::removeListenSource(int sourceID) { + for (int i = 0; i < _listenSources.size(); i++) { + if (_listenSources[i] == sourceID) { + _listenSources.erase(_listenSources.begin() + i); + return; + } + } +} + + Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : _stream(NULL), _ringBuffer(true), @@ -338,7 +383,9 @@ Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) : _collisionSoundNoise(0.0f), _collisionSoundDuration(0.0f), _proceduralEffectSample(0), - _heartbeatMagnitude(0.0f) + _heartbeatMagnitude(0.0f), + _listenMode(AudioRingBuffer::NORMAL), + _listenRadius(0.0f) { outputPortAudioError(Pa_Initialize()); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 37db447381..2eb4e7ef70 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -9,6 +9,7 @@ #ifndef __interface__Audio__ #define __interface__Audio__ +#include #include #include #include @@ -54,6 +55,11 @@ public: // The results of the analysis are written to the log. bool eventuallyAnalyzePing(); + void setListenMode(AudioRingBuffer::ListenMode mode) { _listenMode = mode; }; + void setListenRadius(float radius) { _listenRadius = radius; }; + void addListenSource(int sourceID); + void removeListenSource(int sourceID); + void clearListenSources(); private: PaStream* _stream; @@ -90,6 +96,10 @@ private: float _collisionSoundDuration; int _proceduralEffectSample; float _heartbeatMagnitude; + + AudioRingBuffer::ListenMode _listenMode; + float _listenRadius; + std::vector _listenSources; // Audio callback in class context. inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); diff --git a/interface/src/LeapManager.cpp b/interface/src/LeapManager.cpp index fe47e527fa..883a9c1037 100755 --- a/interface/src/LeapManager.cpp +++ b/interface/src/LeapManager.cpp @@ -7,57 +7,28 @@ // #include "LeapManager.h" +#include "avatar/Avatar.h" #include #include // needed for RTLD_LAZY #include +// Uncomment the next line to use Leap-smoothed stabilized (slower) data. +//#define USE_STABILIZED_DATA + bool LeapManager::_libraryExists = false; bool LeapManager::_doFakeFingers = false; Leap::Controller* LeapManager::_controller = NULL; HifiLeapListener* LeapManager::_listener = NULL; -namespace { -glm::vec3 fakeHandOffset(0.0f, 50.0f, 50.0f); -} // end anonymous namespace - class HifiLeapListener : public Leap::Listener { public: HifiLeapListener() {} virtual ~HifiLeapListener() {} Leap::Frame lastFrame; - std::vector fingerTips; - std::vector fingerRoots; - std::vector handPositions; - std::vector handNormals; - virtual void onFrame(const Leap::Controller& controller) { #ifndef LEAP_STUBS - Leap::Frame frame = controller.frame(); - int numFingers = frame.fingers().count(); - fingerTips.resize(numFingers); - fingerRoots.resize(numFingers); - for (int i = 0; i < numFingers; ++i) { - const Leap::Finger& thisFinger = frame.fingers()[i]; - const Leap::Vector pos = thisFinger.stabilizedTipPosition(); - fingerTips[i] = glm::vec3(pos.x, pos.y, pos.z); - - const Leap::Vector root = pos - thisFinger.direction() * thisFinger.length(); - fingerRoots[i] = glm::vec3(root.x, root.y, root.z); - } - - int numHands = frame.hands().count(); - handPositions.resize(numHands); - handNormals.resize(numHands); - for (int i = 0; i < numHands; ++i) { - const Leap::Hand& thisHand = frame.hands()[i]; - const Leap::Vector pos = thisHand.palmPosition(); - handPositions[i] = glm::vec3(pos.x, pos.y, pos.z); - - const Leap::Vector norm = thisHand.palmNormal(); - handNormals[i] = glm::vec3(norm.x, norm.y, norm.z); - } - lastFrame = frame; + lastFrame = controller.frame(); #endif } @@ -80,10 +51,199 @@ void LeapManager::terminate() { _controller = NULL; } -void LeapManager::nextFrame() { +void LeapManager::nextFrame(Avatar& avatar) { + // Apply the frame data directly to the avatar. + Hand& hand = avatar.getHand(); + + // If we actually get valid Leap data, this will be set to true; + bool gotRealData = false; + if (controllersExist()) { _listener->onFrame(*_controller); } + +#ifndef LEAP_STUBS + if (controllersExist()) { + gotRealData = true; + // First, see which palms and fingers are still valid. + Leap::Frame& frame = _listener->lastFrame; + + // Note that this is O(n^2) at worst, but n is very small. + + // After this many frames of no data, assume the digit is lost. + const int assumeLostAfterFrameCount = 10; + + // Increment our frame data counters + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + palm.incrementFramesWithoutData(); + if (palm.getFramesWithoutData() > assumeLostAfterFrameCount) { + palm.setActive(false); + } + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.incrementFramesWithoutData(); + if (finger.getFramesWithoutData() > assumeLostAfterFrameCount) { + finger.setActive(false); + } + } + } + + size_t numLeapHands = frame.hands().count(); + std::vector palmAssignment(numLeapHands); + + // Look for matches + for (size_t index = 0; index < numLeapHands; ++index) { + PalmData* takeoverCandidate = NULL; + palmAssignment[index] = NULL; + Leap::Hand leapHand = frame.hands()[index]; + int id = leapHand.id(); + if (leapHand.isValid()) { + for (size_t i = 0; i < hand.getNumPalms() && palmAssignment[index] == NULL; ++i) { + PalmData& palm = hand.getPalms()[i]; + if (palm.getLeapID() == id) { + // Found hand with the same ID. We're set! + palmAssignment[index] = &palm; + palm.resetFramesWithoutData(); + } + else if (palm.getFramesWithoutData() > assumeLostAfterFrameCount) { + takeoverCandidate = &palm; + } + } + if (palmAssignment[index] == NULL) { + palmAssignment[index] = takeoverCandidate; + } + if (palmAssignment[index] == NULL) { + palmAssignment[index] = &hand.addNewPalm(); + } + } + } + + // Apply the assignments + for (size_t index = 0; index < numLeapHands; ++index) { + if (palmAssignment[index]) { + Leap::Hand leapHand = frame.hands()[index]; + PalmData& palm = *(palmAssignment[index]); + + palm.resetFramesWithoutData(); + palm.setLeapID(leapHand.id()); + palm.setActive(true); + const Leap::Vector pos = leapHand.palmPosition(); + const Leap::Vector normal = leapHand.palmNormal(); + palm.setRawPosition(glm::vec3(pos.x, pos.y, pos.z)); + palm.setRawNormal(glm::vec3(normal.x, normal.y, normal.z)); + } + } + + // Look for fingers per palm + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + if (palm.isActive()) { + Leap::Hand leapHand = frame.hand(palm.getLeapID()); + if (leapHand.isValid()) { + int numLeapFingers = leapHand.fingers().count(); + std::vector fingerAssignment(numLeapFingers); + + + // Look for matches + for (size_t index = 0; index < numLeapFingers; ++index) { + FingerData* takeoverCandidate = NULL; + fingerAssignment[index] = NULL; + Leap::Finger leapFinger = leapHand.fingers()[index]; + int id = leapFinger.id(); + if (leapFinger.isValid()) { + for (size_t f = 0; f < palm.getNumFingers() && fingerAssignment[index] == NULL; ++f) { + FingerData& finger = palm.getFingers()[f]; + if (finger.getLeapID() == id) { + // Found hand with the same ID. We're set! + fingerAssignment[index] = &finger; + } + else if (finger.getFramesWithoutData() > assumeLostAfterFrameCount) { + takeoverCandidate = &finger; + } + } + // If we didn't find a match, but we found an unused finger, us it. + if (fingerAssignment[index] == NULL) { + fingerAssignment[index] = takeoverCandidate; + } + } + } + + // Apply the assignments + for (size_t index = 0; index < numLeapFingers; ++index) { + if (fingerAssignment[index]) { + Leap::Finger leapFinger = leapHand.fingers()[index]; + FingerData& finger = *(fingerAssignment[index]); + + finger.resetFramesWithoutData(); + finger.setLeapID(leapFinger.id()); + finger.setActive(true); +#ifdef USE_STABILIZED_DATA + const Leap::Vector tip = leapFinger.stabilizedTipPosition(); +#else + const Leap::Vector tip = leapFinger.tipPosition(); +#endif + const Leap::Vector root = tip - leapFinger.direction() * leapFinger.length(); + finger.setRawTipPosition(glm::vec3(tip.x, tip.y, tip.z)); + finger.setRawRootPosition(glm::vec3(root.x, root.y, root.z)); + } + } + } + } + } + } +#endif + if (!gotRealData) { + if (_doFakeFingers) { + // There's no real Leap data and we need to fake it. + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + static const glm::vec3 fakeHandOffsets[] = { + glm::vec3( -500.0f, 50.0f, 50.0f), + glm::vec3( 0.0f, 50.0f, 50.0f) + }; + static const glm::vec3 fakeHandFingerMirrors[] = { + glm::vec3( -1.0f, 1.0f, 1.0f), + glm::vec3( 1.0f, 1.0f, 1.0f) + }; + static const glm::vec3 fakeFingerPositions[] = { + glm::vec3( -60.0f, 0.0f, -40.0f), + glm::vec3( -20.0f, 0.0f, -60.0f), + glm::vec3( 20.0f, 0.0f, -60.0f), + glm::vec3( 60.0f, 0.0f, -40.0f), + glm::vec3( -50.0f, 0.0f, 30.0f) + }; + + PalmData& palm = hand.getPalms()[i]; + palm.setActive(true); + // Simulated data + + palm.setRawPosition(glm::vec3( 0.0f, 0.0f, 0.0f) + fakeHandOffsets[i]); + palm.setRawNormal(glm::vec3(0.0f, 1.0f, 0.0f)); + + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); + const float tipScale = 1.5f; + const float rootScale = 0.75f; + glm::vec3 fingerPos = fakeFingerPositions[f] * fakeHandFingerMirrors[i]; + finger.setRawTipPosition(fingerPos * tipScale + fakeHandOffsets[i]); + finger.setRawRootPosition(fingerPos * rootScale + fakeHandOffsets[i]); + } + } + } + else { + // Just deactivate everything. + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + palm.setActive(false); + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setActive(false); + } + } + } + } + hand.updateFingerTrails(); } void LeapManager::enableFakeFingers(bool enable) { @@ -98,77 +258,6 @@ bool LeapManager::controllersExist() { #endif } -const std::vector& LeapManager::getFingerTips() { - if (controllersExist()) { - return _listener->fingerTips; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - float scale = 1.5f; - stubData.push_back(glm::vec3( -60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -50.0f, 0.0f, 30.0f) * scale + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getFingerRoots() { - if (controllersExist()) { - return _listener->fingerRoots; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - float scale = 0.75f; - stubData.push_back(glm::vec3( -60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -50.0f, 0.0f, 30.0f) * scale + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getHandPositions() { - if (controllersExist()) { - return _listener->handPositions; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - glm::vec3 handOffset(0.0f, 50.0f, 50.0f); - stubData.push_back(glm::vec3( 0.0f, 0.0f, 0.0f) + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getHandNormals() { - if (controllersExist()) { - return _listener->handNormals; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - stubData.push_back(glm::vec3(0.0f, 1.0f, 0.0f)); - } - return stubData; - } -} - std::string LeapManager::statusString() { std::stringstream leapString; #ifndef LEAP_STUBS diff --git a/interface/src/LeapManager.h b/interface/src/LeapManager.h index e6ac304677..11dbefe849 100755 --- a/interface/src/LeapManager.h +++ b/interface/src/LeapManager.h @@ -13,6 +13,7 @@ #include #include +class Avatar; class HifiLeapListener; namespace Leap { class Controller; @@ -20,7 +21,7 @@ namespace Leap { class LeapManager { public: - static void nextFrame(); // called once per frame to get new Leap data + static void nextFrame(Avatar& avatar); // called once per frame to get new Leap data static bool controllersExist(); // Returns true if there's at least one active Leap plugged in static void enableFakeFingers(bool enable); // put fake data in if there's no Leap plugged in static const std::vector& getFingerTips(); diff --git a/interface/src/avatar/Avatar.cpp b/interface/src/avatar/Avatar.cpp index ddb78f595b..c2a2d5aadd 100755 --- a/interface/src/avatar/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -830,13 +830,14 @@ void Avatar::updateHandMovementAndTouching(float deltaTime, bool enableHandMovem } // If there's a leap-interaction hand visible, use that as the endpoint - for (size_t i = 0; i < getHand().getPalms().size(); ++i) { - PalmData& palm = getHand().getPalms()[i]; - if (palm.isActive()) { - _skeleton.joint[ AVATAR_JOINT_RIGHT_FINGERTIPS ].position = palm.getPosition(); + if (!getHand().isRaveGloveActive()) { + for (size_t i = 0; i < getHand().getPalms().size(); ++i) { + PalmData& palm = getHand().getPalms()[i]; + if (palm.isActive()) { + _skeleton.joint[ AVATAR_JOINT_RIGHT_FINGERTIPS ].position = palm.getPosition(); + } } } - }//if (_isMine) //constrain right arm length and re-adjust elbow position as it bends diff --git a/interface/src/avatar/Avatar.h b/interface/src/avatar/Avatar.h index 75e0ab2f9b..ec0d5a2ce5 100755 --- a/interface/src/avatar/Avatar.h +++ b/interface/src/avatar/Avatar.h @@ -164,6 +164,10 @@ public: glm::quat getOrientation () const; glm::quat getWorldAlignedOrientation() const; + const glm::vec3& getMouseRayOrigin() const { return _mouseRayOrigin; } + const glm::vec3& getMouseRayDirection() const { return _mouseRayDirection; } + + glm::vec3 getGravity () const { return _gravity; } glm::vec3 getUprightHeadPosition() const; diff --git a/interface/src/avatar/Hand.cpp b/interface/src/avatar/Hand.cpp index 69d8ddfaac..97594d0f50 100755 --- a/interface/src/avatar/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -95,6 +95,7 @@ void Hand::render(bool lookingInMirror) { glEnable(GL_DEPTH_TEST); glEnable(GL_RESCALE_NORMAL); + renderFingerTrails(); renderHandSpheres(); } @@ -173,44 +174,35 @@ void Hand::renderHandSpheres() { glPopMatrix(); } -void Hand::setLeapFingers(const std::vector& fingerTips, - const std::vector& fingerRoots) { - // TODO: add id-checking here to increase finger stability - - size_t fingerIndex = 0; +void Hand::renderFingerTrails() { + // Draw the finger root cones for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; - for (size_t f = 0; f < palm.getNumFingers(); ++f) { - FingerData& finger = palm.getFingers()[f]; - if (fingerIndex < fingerTips.size()) { - finger.setActive(true); - finger.setRawTipPosition(fingerTips[fingerIndex]); - finger.setRawRootPosition(fingerRoots[fingerIndex]); - fingerIndex++; - } - else { - finger.setActive(false); + if (palm.isActive()) { + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + int numPositions = finger.getTrailNumPositions(); + if (numPositions > 0) { + glBegin(GL_TRIANGLE_STRIP); + for (int t = 0; t < numPositions; ++t) + { + const glm::vec3& center = finger.getTrailPosition(t); + const float halfWidth = 0.001f; + const glm::vec3 edgeDirection(1.0f, 0.0f, 0.0f); + glm::vec3 edge0 = center + edgeDirection * halfWidth; + glm::vec3 edge1 = center - edgeDirection * halfWidth; + float alpha = 1.0f - ((float)t / (float)(numPositions - 1)); + glColor4f(1.0f, 0.0f, 0.0f, alpha); + glVertex3fv((float*)&edge0); + glVertex3fv((float*)&edge1); + } + glEnd(); + } } } } } -void Hand::setLeapHands(const std::vector& handPositions, - const std::vector& handNormals) { - for (size_t i = 0; i < getNumPalms(); ++i) { - PalmData& palm = getPalms()[i]; - if (i < handPositions.size()) { - palm.setActive(true); - palm.setRawPosition(handPositions[i]); - palm.setRawNormal(handNormals[i]); - } - else { - palm.setActive(false); - } - } -} - - void Hand::updateFingerParticles(float deltaTime) { if (!_particleSystemInitialized) { diff --git a/interface/src/avatar/Hand.h b/interface/src/avatar/Hand.h index fb6b863458..f37046ed3a 100755 --- a/interface/src/avatar/Hand.h +++ b/interface/src/avatar/Hand.h @@ -42,10 +42,6 @@ public: void render(bool lookingInMirror); void setBallColor (glm::vec3 ballColor ) { _ballColor = ballColor; } - void setLeapFingers (const std::vector& fingerTips, - const std::vector& fingerRoots); - void setLeapHands (const std::vector& handPositions, - const std::vector& handNormals); void updateFingerParticles(float deltaTime); void setRaveGloveActive(bool active) { _isRaveGloveActive = active; } @@ -74,6 +70,7 @@ private: // private methods void renderRaveGloveStage(); void renderHandSpheres(); + void renderFingerTrails(); void calculateGeometry(); }; diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index b8393e26ab..76cf13a44d 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -272,7 +272,7 @@ void Head::calculateGeometry() { + up * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_UP_OFFSET + front * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_FRONT_OFFSET; - _eyeLevelPosition = _position + up * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_UP_OFFSET; + _eyeLevelPosition = _rightEyePosition - right * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_RIGHT_OFFSET; //calculate the eyebrow positions _leftEyeBrowPosition = _leftEyePosition; diff --git a/interface/src/ui/VoxelStatsDialog.cpp b/interface/src/ui/VoxelStatsDialog.cpp new file mode 100644 index 0000000000..bfe93ec119 --- /dev/null +++ b/interface/src/ui/VoxelStatsDialog.cpp @@ -0,0 +1,79 @@ +// +// VoxelStatsDialog.cpp +// interface +// +// Created by Brad Hefta-Gaub on 7/19/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include +#include + +#include +#include + +#include + +#include "ui/VoxelStatsDialog.h" + + +VoxelStatsDialog::VoxelStatsDialog(QWidget* parent, VoxelSceneStats* model) : + QDialog(parent, Qt::Window | Qt::WindowCloseButtonHint | Qt::WindowStaysOnTopHint), + _model(model) { + + char strBuf[64]; + + this->setWindowTitle("Voxel Statistics"); + + // Create layouter + QFormLayout* form = new QFormLayout(); + this->QDialog::setLayout(form); + + // Setup labels + for (int i = 0; i < VoxelSceneStats::ITEM_COUNT; ++i) { + VoxelSceneStats::ItemInfo& itemInfo = _model->getItemInfo(i); + QLabel* label = _labels[i] = new QLabel(); + label->setAlignment(Qt::AlignRight); + + // Set foreground color to 62.5% brightness of the meter (otherwise will be hard to read on the bright background) + QPalette palette = label->palette(); + unsigned rgb = itemInfo.colorRGBA >> 8; + const unsigned colorpart1 = 0xfefefeu; + const unsigned colorpart2 = 0xf8f8f8; + rgb = ((rgb & colorpart1) >> 1) + ((rgb & colorpart2) >> 3); + palette.setColor(QPalette::WindowText, QColor::fromRgb(rgb)); + label->setPalette(palette); + + // This is my hackery attempt at making QDialog auto-size to a width that will hold our info. It kinda works. + label->setText("123456789012345678901234567890123456789012345678901234567890"); + + snprintf(strBuf, sizeof(strBuf), " %s:", itemInfo.caption); + form->addRow(strBuf, label); + } +} + +void VoxelStatsDialog::paintEvent(QPaintEvent* event) { + + // Update labels + char strBuf[256]; + for (int i = 0; i < VoxelSceneStats::ITEM_COUNT; ++i) { + QLabel* label = _labels[i]; + snprintf(strBuf, sizeof(strBuf), "%s", _model->getItemValue(i)); + label->setText(strBuf); + } + + this->QDialog::paintEvent(event); + this->setFixedSize(this->width(), this->height()); +} + +void VoxelStatsDialog::reject() { + // Just regularly close upon ESC + this->QDialog::close(); +} + +void VoxelStatsDialog::closeEvent(QCloseEvent* event) { + this->QDialog::closeEvent(event); + emit closed(); +} + + diff --git a/interface/src/ui/VoxelStatsDialog.h b/interface/src/ui/VoxelStatsDialog.h new file mode 100644 index 0000000000..eab5b6a45a --- /dev/null +++ b/interface/src/ui/VoxelStatsDialog.h @@ -0,0 +1,42 @@ +// +// VoxelStatsDialog.h +// interface +// +// Created by Brad Hefta-Gaub on 7/19/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __hifi__VoxelStatsDialog__ +#define __hifi__VoxelStatsDialog__ + +#include +#include + +#include + +class VoxelStatsDialog : public QDialog { + Q_OBJECT +public: + // Sets up the UI + VoxelStatsDialog(QWidget* parent, VoxelSceneStats* model); + +signals: + void closed(); + +public slots: + void reject(); + +protected: + // State <- data model held by BandwidthMeter + void paintEvent(QPaintEvent*); + + // Emits a 'closed' signal when this dialog is closed. + void closeEvent(QCloseEvent*); + +private: + QLabel* _labels[VoxelSceneStats::ITEM_COUNT]; + VoxelSceneStats* _model; +}; + +#endif /* defined(__interface__VoxelStatsDialog__) */ + diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index dea4a13fec..4c5374f032 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -27,6 +27,14 @@ const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENG class AudioRingBuffer : public NodeData { public: + + static int const DEFAULT_LISTEN_LIST_SIZE = 100; + typedef enum { + NORMAL, + OMNI_DIRECTIONAL_POINT, + SELECTED_SOURCES + } ListenMode; + AudioRingBuffer(bool isStereo); ~AudioRingBuffer(); diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 32e83b352d..7bb6eebdc0 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -13,15 +13,22 @@ HandData::HandData(AvatarData* owningAvatar) : _baseOrientation(0.0f, 0.0f, 0.0f, 1.0f), _owningAvatarData(owningAvatar) { - for (int i = 0; i < 2; ++i) { - _palms.push_back(PalmData(this)); - } + // Start with two palms + addNewPalm(); + addNewPalm(); +} + +PalmData& HandData::addNewPalm() { + _palms.push_back(PalmData(this)); + return _palms.back(); } PalmData::PalmData(HandData* owningHandData) : _rawPosition(0, 0, 0), _rawNormal(0, 1, 0), _isActive(false), +_leapID(LEAPID_INVALID), +_numFramesWithoutData(0), _owningHandData(owningHandData) { for (int i = 0; i < NUM_FINGERS_PER_HAND; ++i) { @@ -33,9 +40,13 @@ FingerData::FingerData(PalmData* owningPalmData, HandData* owningHandData) : _tipRawPosition(0, 0, 0), _rootRawPosition(0, 0, 0), _isActive(false), +_leapID(LEAPID_INVALID), +_numFramesWithoutData(0), _owningPalmData(owningPalmData), _owningHandData(owningHandData) { + const int standardTrailLength = 30; + setTrailLength(standardTrailLength); } void HandData::encodeRemoteData(std::vector& fingerVectors) { @@ -80,3 +91,66 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { } } +void HandData::setFingerTrailLength(unsigned int length) { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setTrailLength(length); + } + } +} + +void HandData::updateFingerTrails() { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.updateTrail(); + } + } +} + +void FingerData::setTrailLength(unsigned int length) { + _tipTrailPositions.resize(length); + _tipTrailCurrentStartIndex = 0; + _tipTrailCurrentValidLength = 0; +} + +void FingerData::updateTrail() { + if (_tipTrailPositions.size() == 0) + return; + + if (_isActive) { + // Add the next point in the trail. + _tipTrailCurrentStartIndex--; + if (_tipTrailCurrentStartIndex < 0) + _tipTrailCurrentStartIndex = _tipTrailPositions.size() - 1; + + _tipTrailPositions[_tipTrailCurrentStartIndex] = getTipPosition(); + + if (_tipTrailCurrentValidLength < _tipTrailPositions.size()) + _tipTrailCurrentValidLength++; + } + else { + // It's not active, so just shorten the trail. + if (_tipTrailCurrentValidLength > 0) + _tipTrailCurrentValidLength--; + } +} + +int FingerData::getTrailNumPositions() { + return _tipTrailCurrentValidLength; +} + +const glm::vec3& FingerData::getTrailPosition(int index) { + if (index >= _tipTrailCurrentValidLength) { + static glm::vec3 zero(0,0,0); + return zero; + } + int posIndex = (index + _tipTrailCurrentStartIndex) % _tipTrailCurrentValidLength; + return _tipTrailPositions[posIndex]; +} + + + diff --git a/libraries/avatars/src/HandData.h b/libraries/avatars/src/HandData.h index 65b32ff5c6..d2b5cae90d 100755 --- a/libraries/avatars/src/HandData.h +++ b/libraries/avatars/src/HandData.h @@ -20,6 +20,7 @@ class FingerData; class PalmData; const int NUM_FINGERS_PER_HAND = 5; +const int LEAPID_INVALID = -1; class HandData { public: @@ -39,6 +40,10 @@ public: std::vector& getPalms() { return _palms; } size_t getNumPalms() { return _palms.size(); } + PalmData& addNewPalm(); + + void setFingerTrailLength(unsigned int length); + void updateFingerTrails(); // Use these for sending and receiving hand data void encodeRemoteData(std::vector& fingerVectors); @@ -65,15 +70,31 @@ public: const glm::vec3& getTipRawPosition() const { return _tipRawPosition; } const glm::vec3& getRootRawPosition() const { return _rootRawPosition; } bool isActive() const { return _isActive; } + int getLeapID() const { return _leapID; } void setActive(bool active) { _isActive = active; } + void setLeapID(int id) { _leapID = id; } void setRawTipPosition(const glm::vec3& pos) { _tipRawPosition = pos; } void setRawRootPosition(const glm::vec3& pos) { _rootRawPosition = pos; } + void setTrailLength(unsigned int length); + void updateTrail(); + + int getTrailNumPositions(); + const glm::vec3& getTrailPosition(int index); + + void incrementFramesWithoutData() { _numFramesWithoutData++; } + void resetFramesWithoutData() { _numFramesWithoutData = 0; } + int getFramesWithoutData() const { return _numFramesWithoutData; } private: glm::vec3 _tipRawPosition; glm::vec3 _rootRawPosition; - bool _isActive; // This has current valid data + bool _isActive; // This has current valid data + int _leapID; // the Leap's serial id for this tracked object + int _numFramesWithoutData; // after too many frames without data, this tracked object assumed lost. + std::vector _tipTrailPositions; + int _tipTrailCurrentStartIndex; + int _tipTrailCurrentValidLength; PalmData* _owningPalmData; HandData* _owningHandData; }; @@ -86,19 +107,27 @@ public: const glm::vec3& getRawPosition() const { return _rawPosition; } const glm::vec3& getRawNormal() const { return _rawNormal; } bool isActive() const { return _isActive; } + int getLeapID() const { return _leapID; } std::vector& getFingers() { return _fingers; } size_t getNumFingers() { return _fingers.size(); } void setActive(bool active) { _isActive = active; } + void setLeapID(int id) { _leapID = id; } void setRawPosition(const glm::vec3& pos) { _rawPosition = pos; } void setRawNormal(const glm::vec3& normal) { _rawNormal = normal; } + void incrementFramesWithoutData() { _numFramesWithoutData++; } + void resetFramesWithoutData() { _numFramesWithoutData = 0; } + int getFramesWithoutData() const { return _numFramesWithoutData; } + private: std::vector _fingers; glm::vec3 _rawPosition; glm::vec3 _rawNormal; - bool _isActive; // This has current valid data + bool _isActive; // This has current valid data + int _leapID; // the Leap's serial id for this tracked object + int _numFramesWithoutData; // after too many frames without data, this tracked object assumed lost. HandData* _owningHandData; }; diff --git a/libraries/shared/src/PacketHeaders.cpp b/libraries/shared/src/PacketHeaders.cpp index 821a2d0247..292c4bbc0a 100644 --- a/libraries/shared/src/PacketHeaders.cpp +++ b/libraries/shared/src/PacketHeaders.cpp @@ -14,6 +14,12 @@ PACKET_VERSION versionForPacketType(PACKET_TYPE type) { switch (type) { + + case PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO: + case PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO: + return 1; + break; + case PACKET_TYPE_HEAD_DATA: return 2; break; diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 9652ac14e7..2fbdc4d791 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -34,6 +34,7 @@ const PACKET_TYPE PACKET_TYPE_TRANSMITTER_DATA_V2 = 'T'; const PACKET_TYPE PACKET_TYPE_ENVIRONMENT_DATA = 'e'; const PACKET_TYPE PACKET_TYPE_DOMAIN_LIST_REQUEST = 'L'; const PACKET_TYPE PACKET_TYPE_DOMAIN_REPORT_FOR_DUTY = 'C'; +const PACKET_TYPE PACKET_TYPE_VOXEL_STATS = '#'; typedef char PACKET_VERSION; diff --git a/libraries/voxels/src/VoxelNode.cpp b/libraries/voxels/src/VoxelNode.cpp index 1268449ac6..34eb2b2261 100644 --- a/libraries/voxels/src/VoxelNode.cpp +++ b/libraries/voxels/src/VoxelNode.cpp @@ -44,6 +44,8 @@ void VoxelNode::init(unsigned char * octalCode) { _children[i] = NULL; } _childCount = 0; + _subtreeNodeCount = 1; // that's me + _subtreeLeafNodeCount = 0; // that's me _glBufferIndex = GLBUFFER_INDEX_UNKNOWN; _isDirty = true; @@ -79,6 +81,24 @@ void VoxelNode::handleSubtreeChanged(VoxelTree* myTree) { if (myTree->getShouldReaverage()) { setColorFromAverageOfChildren(); } + + recalculateSubTreeNodeCount(); +} + +void VoxelNode::recalculateSubTreeNodeCount() { + // Assuming the tree below me as changed, I need to recalculate my node count + _subtreeNodeCount = 1; // that's me + if (isLeaf()) { + _subtreeLeafNodeCount = 1; + } else { + _subtreeLeafNodeCount = 0; + for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { + if (_children[i]) { + _subtreeNodeCount += _children[i]->_subtreeNodeCount; + _subtreeLeafNodeCount += _children[i]->_subtreeLeafNodeCount; + } + } + } } diff --git a/libraries/voxels/src/VoxelNode.h b/libraries/voxels/src/VoxelNode.h index 431592c2f9..c84ce3c7d6 100644 --- a/libraries/voxels/src/VoxelNode.h +++ b/libraries/voxels/src/VoxelNode.h @@ -107,6 +107,12 @@ public: static int addDeleteHook(VoxelNodeDeleteHook hook, void* extraData = NULL); static void removeDeleteHook(int hookID); + + void recalculateSubTreeNodeCount(); + unsigned long getSubTreeNodeCount() const { return _subtreeNodeCount; }; + unsigned long getSubTreeInternalNodeCount() const { return _subtreeNodeCount - _subtreeLeafNodeCount; }; + unsigned long getSubTreeLeafNodeCount() const { return _subtreeLeafNodeCount; }; + private: void calculateAABox(); void init(unsigned char * octalCode); @@ -126,6 +132,8 @@ private: unsigned char* _octalCode; VoxelNode* _children[8]; int _childCount; + unsigned long _subtreeNodeCount; + unsigned long _subtreeLeafNodeCount; float _density; // If leaf: density = 1, if internal node: 0-1 density of voxels inside static VoxelNodeDeleteHook _hooks[VOXEL_NODE_MAX_DELETE_HOOKS]; diff --git a/libraries/voxels/src/VoxelSceneStats.cpp b/libraries/voxels/src/VoxelSceneStats.cpp new file mode 100644 index 0000000000..b226886a25 --- /dev/null +++ b/libraries/voxels/src/VoxelSceneStats.cpp @@ -0,0 +1,556 @@ +// +// VoxelSceneStats.cpp +// hifi +// +// Created by Brad Hefta-Gaub on 7/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// +// + +#include +#include + +#include "VoxelNode.h" +#include "VoxelSceneStats.h" + + +const int samples = 100; +VoxelSceneStats::VoxelSceneStats() : + _elapsedAverage(samples), + _bitsPerVoxelAverage(samples) +{ + reset(); + _isReadyToSend = false; + _isStarted = false; +} + +void VoxelSceneStats::sceneStarted(bool isFullScene, bool isMoving, VoxelNode* root) { + reset(); // resets packet and voxel stats + _isStarted = true; + _start = usecTimestampNow(); + _totalVoxels = root->getSubTreeNodeCount(); + _totalInternal = root->getSubTreeInternalNodeCount(); + _totalLeaves = root->getSubTreeLeafNodeCount(); + + _isFullScene = isFullScene; + _isMoving = isMoving; +} + +void VoxelSceneStats::sceneCompleted() { + if (_isStarted) { + _end = usecTimestampNow(); + _elapsed = _end - _start; + _elapsedAverage.updateAverage((float)_elapsed); + + _statsMessageLength = packIntoMessage(_statsMessage, sizeof(_statsMessage)); + _isReadyToSend = true; + _isStarted = false; + } +} + +void VoxelSceneStats::encodeStarted() { + _encodeStart = usecTimestampNow(); +} + +void VoxelSceneStats::encodeStopped() { + _totalEncodeTime += (usecTimestampNow() - _encodeStart); +} + +void VoxelSceneStats::reset() { + _totalEncodeTime = 0; + _encodeStart = 0; + + _packets = 0; + _bytes = 0; + _passes = 0; + + _totalVoxels = 0; + _totalInternal = 0; + _totalLeaves = 0; + + _traversed = 0; + _internal = 0; + _leaves = 0; + + _skippedDistance = 0; + _internalSkippedDistance = 0; + _leavesSkippedDistance = 0; + + _skippedOutOfView = 0; + _internalSkippedOutOfView = 0; + _leavesSkippedOutOfView = 0; + + _skippedWasInView = 0; + _internalSkippedWasInView = 0; + _leavesSkippedWasInView = 0; + + _skippedNoChange = 0; + _internalSkippedNoChange = 0; + _leavesSkippedNoChange = 0; + + _skippedOccluded = 0; + _internalSkippedOccluded = 0; + _leavesSkippedOccluded = 0; + + _colorSent = 0; + _internalColorSent = 0; + _leavesColorSent = 0; + + _didntFit = 0; + _internalDidntFit = 0; + _leavesDidntFit = 0; + + _colorBitsWritten = 0; + _existsBitsWritten = 0; + _existsInPacketBitsWritten = 0; + _treesRemoved = 0; +} + +void VoxelSceneStats::packetSent(int bytes) { + _packets++; + _bytes += bytes; +} + +void VoxelSceneStats::traversed(const VoxelNode* node) { + _traversed++; + if (node->isLeaf()) { + _leaves++; + } else { + _internal++; + } +} + +void VoxelSceneStats::skippedDistance(const VoxelNode* node) { + _skippedDistance++; + if (node->isLeaf()) { + _leavesSkippedDistance++; + } else { + _internalSkippedDistance++; + } +} + +void VoxelSceneStats::skippedOutOfView(const VoxelNode* node) { + _skippedOutOfView++; + if (node->isLeaf()) { + _leavesSkippedOutOfView++; + } else { + _internalSkippedOutOfView++; + } +} + +void VoxelSceneStats::skippedWasInView(const VoxelNode* node) { + _skippedWasInView++; + if (node->isLeaf()) { + _leavesSkippedWasInView++; + } else { + _internalSkippedWasInView++; + } +} + +void VoxelSceneStats::skippedNoChange(const VoxelNode* node) { + _skippedNoChange++; + if (node->isLeaf()) { + _leavesSkippedNoChange++; + } else { + _internalSkippedNoChange++; + } +} + +void VoxelSceneStats::skippedOccluded(const VoxelNode* node) { + _skippedOccluded++; + if (node->isLeaf()) { + _leavesSkippedOccluded++; + } else { + _internalSkippedOccluded++; + } +} + +void VoxelSceneStats::colorSent(const VoxelNode* node) { + _colorSent++; + if (node->isLeaf()) { + _leavesColorSent++; + } else { + _internalColorSent++; + } +} + +void VoxelSceneStats::didntFit(const VoxelNode* node) { + _didntFit++; + if (node->isLeaf()) { + _leavesDidntFit++; + } else { + _internalDidntFit++; + } +} + +void VoxelSceneStats::colorBitsWritten() { + _colorBitsWritten++; +} + +void VoxelSceneStats::existsBitsWritten() { + _existsBitsWritten++; +} + +void VoxelSceneStats::existsInPacketBitsWritten() { + _existsInPacketBitsWritten++; +} + +void VoxelSceneStats::childBitsRemoved(bool includesExistsBits, bool includesColors) { + _existsInPacketBitsWritten--; + if (includesExistsBits) { + _existsBitsWritten--; + } + if (includesColors) { + _colorBitsWritten--; + } + _treesRemoved++; +} + +int VoxelSceneStats::packIntoMessage(unsigned char* destinationBuffer, int availableBytes) { + unsigned char* bufferStart = destinationBuffer; + + int headerLength = populateTypeAndVersion(destinationBuffer, PACKET_TYPE_VOXEL_STATS); + destinationBuffer += headerLength; + + memcpy(destinationBuffer, &_start, sizeof(_start)); + destinationBuffer += sizeof(_start); + memcpy(destinationBuffer, &_end, sizeof(_end)); + destinationBuffer += sizeof(_end); + memcpy(destinationBuffer, &_elapsed, sizeof(_elapsed)); + destinationBuffer += sizeof(_elapsed); + memcpy(destinationBuffer, &_totalEncodeTime, sizeof(_totalEncodeTime)); + destinationBuffer += sizeof(_totalEncodeTime); + memcpy(destinationBuffer, &_isFullScene, sizeof(_isFullScene)); + destinationBuffer += sizeof(_isFullScene); + memcpy(destinationBuffer, &_isMoving, sizeof(_isMoving)); + destinationBuffer += sizeof(_isMoving); + memcpy(destinationBuffer, &_packets, sizeof(_packets)); + destinationBuffer += sizeof(_packets); + memcpy(destinationBuffer, &_bytes, sizeof(_bytes)); + destinationBuffer += sizeof(_bytes); + + memcpy(destinationBuffer, &_totalInternal, sizeof(_totalInternal)); + destinationBuffer += sizeof(_totalInternal); + memcpy(destinationBuffer, &_totalLeaves, sizeof(_totalLeaves)); + destinationBuffer += sizeof(_totalLeaves); + memcpy(destinationBuffer, &_internal, sizeof(_internal)); + destinationBuffer += sizeof(_internal); + memcpy(destinationBuffer, &_leaves, sizeof(_leaves)); + destinationBuffer += sizeof(_leaves); + memcpy(destinationBuffer, &_internalSkippedDistance, sizeof(_internalSkippedDistance)); + destinationBuffer += sizeof(_internalSkippedDistance); + memcpy(destinationBuffer, &_leavesSkippedDistance, sizeof(_leavesSkippedDistance)); + destinationBuffer += sizeof(_leavesSkippedDistance); + memcpy(destinationBuffer, &_internalSkippedOutOfView, sizeof(_internalSkippedOutOfView)); + destinationBuffer += sizeof(_internalSkippedOutOfView); + memcpy(destinationBuffer, &_leavesSkippedOutOfView, sizeof(_leavesSkippedOutOfView)); + destinationBuffer += sizeof(_leavesSkippedOutOfView); + memcpy(destinationBuffer, &_internalSkippedWasInView, sizeof(_internalSkippedWasInView)); + destinationBuffer += sizeof(_internalSkippedWasInView); + memcpy(destinationBuffer, &_leavesSkippedWasInView, sizeof(_leavesSkippedWasInView)); + destinationBuffer += sizeof(_leavesSkippedWasInView); + memcpy(destinationBuffer, &_internalSkippedNoChange, sizeof(_internalSkippedNoChange)); + destinationBuffer += sizeof(_internalSkippedNoChange); + memcpy(destinationBuffer, &_leavesSkippedNoChange, sizeof(_leavesSkippedNoChange)); + destinationBuffer += sizeof(_leavesSkippedNoChange); + memcpy(destinationBuffer, &_internalSkippedOccluded, sizeof(_internalSkippedOccluded)); + destinationBuffer += sizeof(_internalSkippedOccluded); + memcpy(destinationBuffer, &_leavesSkippedOccluded, sizeof(_leavesSkippedOccluded)); + destinationBuffer += sizeof(_leavesSkippedOccluded); + memcpy(destinationBuffer, &_internalColorSent, sizeof(_internalColorSent)); + destinationBuffer += sizeof(_internalColorSent); + memcpy(destinationBuffer, &_leavesColorSent, sizeof(_leavesColorSent)); + destinationBuffer += sizeof(_leavesColorSent); + memcpy(destinationBuffer, &_internalDidntFit, sizeof(_internalDidntFit)); + destinationBuffer += sizeof(_internalDidntFit); + memcpy(destinationBuffer, &_leavesDidntFit, sizeof(_leavesDidntFit)); + destinationBuffer += sizeof(_leavesDidntFit); + memcpy(destinationBuffer, &_colorBitsWritten, sizeof(_colorBitsWritten)); + destinationBuffer += sizeof(_colorBitsWritten); + memcpy(destinationBuffer, &_existsBitsWritten, sizeof(_existsBitsWritten)); + destinationBuffer += sizeof(_existsBitsWritten); + memcpy(destinationBuffer, &_existsInPacketBitsWritten, sizeof(_existsInPacketBitsWritten)); + destinationBuffer += sizeof(_existsInPacketBitsWritten); + memcpy(destinationBuffer, &_treesRemoved, sizeof(_treesRemoved)); + destinationBuffer += sizeof(_treesRemoved); + + return destinationBuffer - bufferStart; // includes header! +} + +int VoxelSceneStats::unpackFromMessage(unsigned char* sourceBuffer, int availableBytes) { + unsigned char* startPosition = sourceBuffer; + + // increment to push past the packet header + int numBytesPacketHeader = numBytesForPacketHeader(sourceBuffer); + sourceBuffer += numBytesPacketHeader; + + memcpy(&_start, sourceBuffer, sizeof(_start)); + sourceBuffer += sizeof(_start); + memcpy(&_end, sourceBuffer, sizeof(_end)); + sourceBuffer += sizeof(_end); + memcpy(&_elapsed, sourceBuffer, sizeof(_elapsed)); + sourceBuffer += sizeof(_elapsed); + memcpy(&_totalEncodeTime, sourceBuffer, sizeof(_totalEncodeTime)); + sourceBuffer += sizeof(_totalEncodeTime); + memcpy(&_isFullScene, sourceBuffer, sizeof(_isFullScene)); + sourceBuffer += sizeof(_isFullScene); + memcpy(&_isMoving, sourceBuffer, sizeof(_isMoving)); + sourceBuffer += sizeof(_isMoving); + memcpy(&_packets, sourceBuffer, sizeof(_packets)); + sourceBuffer += sizeof(_packets); + memcpy(&_bytes, sourceBuffer, sizeof(_bytes)); + sourceBuffer += sizeof(_bytes); + + memcpy(&_totalInternal, sourceBuffer, sizeof(_totalInternal)); + sourceBuffer += sizeof(_totalInternal); + memcpy(&_totalLeaves, sourceBuffer, sizeof(_totalLeaves)); + sourceBuffer += sizeof(_totalLeaves); + _totalVoxels = _totalInternal + _totalLeaves; + + memcpy(&_internal, sourceBuffer, sizeof(_internal)); + sourceBuffer += sizeof(_internal); + memcpy(&_leaves, sourceBuffer, sizeof(_leaves)); + sourceBuffer += sizeof(_leaves); + _traversed = _internal + _leaves; + + memcpy(&_internalSkippedDistance, sourceBuffer, sizeof(_internalSkippedDistance)); + sourceBuffer += sizeof(_internalSkippedDistance); + memcpy(&_leavesSkippedDistance, sourceBuffer, sizeof(_leavesSkippedDistance)); + sourceBuffer += sizeof(_leavesSkippedDistance); + _skippedDistance = _internalSkippedDistance + _leavesSkippedDistance; + + memcpy(&_internalSkippedOutOfView, sourceBuffer, sizeof(_internalSkippedOutOfView)); + sourceBuffer += sizeof(_internalSkippedOutOfView); + memcpy(&_leavesSkippedOutOfView, sourceBuffer, sizeof(_leavesSkippedOutOfView)); + sourceBuffer += sizeof(_leavesSkippedOutOfView); + _skippedOutOfView = _internalSkippedOutOfView + _leavesSkippedOutOfView; + + memcpy(&_internalSkippedWasInView, sourceBuffer, sizeof(_internalSkippedWasInView)); + sourceBuffer += sizeof(_internalSkippedWasInView); + memcpy(&_leavesSkippedWasInView, sourceBuffer, sizeof(_leavesSkippedWasInView)); + sourceBuffer += sizeof(_leavesSkippedWasInView); + _skippedWasInView = _internalSkippedWasInView + _leavesSkippedWasInView; + + memcpy(&_internalSkippedNoChange, sourceBuffer, sizeof(_internalSkippedNoChange)); + sourceBuffer += sizeof(_internalSkippedNoChange); + memcpy(&_leavesSkippedNoChange, sourceBuffer, sizeof(_leavesSkippedNoChange)); + sourceBuffer += sizeof(_leavesSkippedNoChange); + _skippedNoChange = _internalSkippedNoChange + _leavesSkippedNoChange; + + memcpy(&_internalSkippedOccluded, sourceBuffer, sizeof(_internalSkippedOccluded)); + sourceBuffer += sizeof(_internalSkippedOccluded); + memcpy(&_leavesSkippedOccluded, sourceBuffer, sizeof(_leavesSkippedOccluded)); + sourceBuffer += sizeof(_leavesSkippedOccluded); + _skippedOccluded = _internalSkippedOccluded + _leavesSkippedOccluded; + + memcpy(&_internalColorSent, sourceBuffer, sizeof(_internalColorSent)); + sourceBuffer += sizeof(_internalColorSent); + memcpy(&_leavesColorSent, sourceBuffer, sizeof(_leavesColorSent)); + sourceBuffer += sizeof(_leavesColorSent); + _colorSent = _internalColorSent + _leavesColorSent; + + memcpy(&_internalDidntFit, sourceBuffer, sizeof(_internalDidntFit)); + sourceBuffer += sizeof(_internalDidntFit); + memcpy(&_leavesDidntFit, sourceBuffer, sizeof(_leavesDidntFit)); + sourceBuffer += sizeof(_leavesDidntFit); + _didntFit = _internalDidntFit + _leavesDidntFit; + + memcpy(&_colorBitsWritten, sourceBuffer, sizeof(_colorBitsWritten)); + sourceBuffer += sizeof(_colorBitsWritten); + memcpy(&_existsBitsWritten, sourceBuffer, sizeof(_existsBitsWritten)); + sourceBuffer += sizeof(_existsBitsWritten); + memcpy(&_existsInPacketBitsWritten, sourceBuffer, sizeof(_existsInPacketBitsWritten)); + sourceBuffer += sizeof(_existsInPacketBitsWritten); + memcpy(&_treesRemoved, sourceBuffer, sizeof(_treesRemoved)); + sourceBuffer += sizeof(_treesRemoved); + + // running averages + _elapsedAverage.updateAverage((float)_elapsed); + unsigned long total = _existsInPacketBitsWritten + _colorSent; + float calculatedBPV = total == 0 ? 0 : (_bytes * 8) / total; + _bitsPerVoxelAverage.updateAverage(calculatedBPV); + + + return sourceBuffer - startPosition; // includes header! +} + + +void VoxelSceneStats::printDebugDetails() { + qDebug("\n------------------------------\n"); + qDebug("VoxelSceneStats:\n"); + qDebug(" start : %llu \n", _start); + qDebug(" end : %llu \n", _end); + qDebug(" elapsed : %llu \n", _elapsed); + qDebug(" encoding : %llu \n", _totalEncodeTime); + qDebug("\n"); + qDebug(" full scene: %s\n", debug::valueOf(_isFullScene)); + qDebug(" moving: %s\n", debug::valueOf(_isMoving)); + qDebug("\n"); + qDebug(" packets: %d\n", _packets); + qDebug(" bytes : %ld\n", _bytes); + qDebug("\n"); + qDebug(" total voxels : %lu\n", _totalVoxels ); + qDebug(" internal : %lu\n", _totalInternal ); + qDebug(" leaves : %lu\n", _totalLeaves ); + qDebug(" traversed : %lu\n", _traversed ); + qDebug(" internal : %lu\n", _internal ); + qDebug(" leaves : %lu\n", _leaves ); + qDebug(" skipped distance : %lu\n", _skippedDistance ); + qDebug(" internal : %lu\n", _internalSkippedDistance ); + qDebug(" leaves : %lu\n", _leavesSkippedDistance ); + qDebug(" skipped out of view : %lu\n", _skippedOutOfView ); + qDebug(" internal : %lu\n", _internalSkippedOutOfView ); + qDebug(" leaves : %lu\n", _leavesSkippedOutOfView ); + qDebug(" skipped was in view : %lu\n", _skippedWasInView ); + qDebug(" internal : %lu\n", _internalSkippedWasInView ); + qDebug(" leaves : %lu\n", _leavesSkippedWasInView ); + qDebug(" skipped no change : %lu\n", _skippedNoChange ); + qDebug(" internal : %lu\n", _internalSkippedNoChange ); + qDebug(" leaves : %lu\n", _leavesSkippedNoChange ); + qDebug(" skipped occluded : %lu\n", _skippedOccluded ); + qDebug(" internal : %lu\n", _internalSkippedOccluded ); + qDebug(" leaves : %lu\n", _leavesSkippedOccluded ); + + qDebug("\n"); + qDebug(" color sent : %lu\n", _colorSent ); + qDebug(" internal : %lu\n", _internalColorSent ); + qDebug(" leaves : %lu\n", _leavesColorSent ); + qDebug(" Didn't Fit : %lu\n", _didntFit ); + qDebug(" internal : %lu\n", _internalDidntFit ); + qDebug(" leaves : %lu\n", _leavesDidntFit ); + qDebug(" color bits : %lu\n", _colorBitsWritten ); + qDebug(" exists bits : %lu\n", _existsBitsWritten ); + qDebug(" in packet bit : %lu\n", _existsInPacketBitsWritten); + qDebug(" trees removed : %lu\n", _treesRemoved ); +} + +const unsigned greenish = 0x40ff40d0; +const unsigned yellowish = 0xffef40c0; +const unsigned greyish = 0xd0d0d0a0; + +VoxelSceneStats::ItemInfo VoxelSceneStats::_ITEMS[] = { + { "Elapsed" , greenish }, + { "Encode" , yellowish }, + { "Network" , greyish }, + { "Voxels on Server" , greenish }, + { "Voxels Sent" , yellowish }, + { "Colors Sent" , greyish }, + { "Bitmasks Sent" , greenish }, + { "Traversed" , yellowish }, + { "Skipped - Total" , greyish }, + { "Skipped - Distance" , greenish }, + { "Skipped - Out of View", yellowish }, + { "Skipped - Was in View", greyish }, + { "Skipped - No Change" , greenish }, + { "Skipped - Occluded" , yellowish }, + { "Didn't fit in packet" , greyish }, + { "Mode" , greenish }, +}; + +char* VoxelSceneStats::getItemValue(int item) { + const uint64_t USECS_PER_SECOND = 1000 * 1000; + int calcFPS, calcAverageFPS, calculatedKBPS; + switch(item) { + case ITEM_ELAPSED: { + calcFPS = (float)USECS_PER_SECOND / (float)_elapsed; + float elapsedAverage = _elapsedAverage.getAverage(); + calcAverageFPS = (float)USECS_PER_SECOND / (float)elapsedAverage; + + sprintf(_itemValueBuffer, "%llu usecs (%d fps) Average: %.0f usecs (%d fps)", + _elapsed, calcFPS, elapsedAverage, calcAverageFPS); + break; + } + case ITEM_ENCODE: + calcFPS = (float)USECS_PER_SECOND / (float)_totalEncodeTime; + sprintf(_itemValueBuffer, "%llu usecs (%d fps)", _totalEncodeTime, calcFPS); + break; + case ITEM_PACKETS: { + float elapsedSecs = ((float)_elapsed / (float)USECS_PER_SECOND); + calculatedKBPS = elapsedSecs == 0 ? 0 : ((_bytes * 8) / elapsedSecs) / 1000; + sprintf(_itemValueBuffer, "%d packets %lu bytes (%d kbps)", _packets, _bytes, calculatedKBPS); + break; + } + case ITEM_VOXELS_SERVER: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _totalVoxels, _totalInternal, _totalLeaves); + break; + } + case ITEM_VOXELS: { + unsigned long total = _existsInPacketBitsWritten + _colorSent; + float calculatedBPV = total == 0 ? 0 : (_bytes * 8) / total; + float averageBPV = _bitsPerVoxelAverage.getAverage(); + sprintf(_itemValueBuffer, "%lu (%.2f bits/voxel Average: %.2f bits/voxel) %lu internal %lu leaves", + total, calculatedBPV, averageBPV, _existsInPacketBitsWritten, _colorSent); + break; + } + case ITEM_TRAVERSED: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _traversed, _internal, _leaves); + break; + } + case ITEM_SKIPPED: { + unsigned long total = _skippedDistance + _skippedOutOfView + + _skippedWasInView + _skippedNoChange + _skippedOccluded; + + unsigned long internal = _internalSkippedDistance + _internalSkippedOutOfView + + _internalSkippedWasInView + _internalSkippedNoChange + _internalSkippedOccluded; + + unsigned long leaves = _leavesSkippedDistance + _leavesSkippedOutOfView + + _leavesSkippedWasInView + _leavesSkippedNoChange + _leavesSkippedOccluded; + + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + total, internal, leaves); + break; + } + case ITEM_SKIPPED_DISTANCE: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _skippedDistance, _internalSkippedDistance, _leavesSkippedDistance); + break; + } + case ITEM_SKIPPED_OUT_OF_VIEW: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _skippedOutOfView, _internalSkippedOutOfView, _leavesSkippedOutOfView); + break; + } + case ITEM_SKIPPED_WAS_IN_VIEW: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _skippedWasInView, _internalSkippedWasInView, _leavesSkippedWasInView); + break; + } + case ITEM_SKIPPED_NO_CHANGE: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _skippedNoChange, _internalSkippedNoChange, _leavesSkippedNoChange); + break; + } + case ITEM_SKIPPED_OCCLUDED: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _skippedOccluded, _internalSkippedOccluded, _leavesSkippedOccluded); + break; + } + case ITEM_COLORS: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves", + _colorSent, _internalColorSent, _leavesColorSent); + break; + } + case ITEM_DIDNT_FIT: { + sprintf(_itemValueBuffer, "%lu total %lu internal %lu leaves (removed: %lu)", + _didntFit, _internalDidntFit, _leavesDidntFit, _treesRemoved); + break; + } + case ITEM_BITS: { + sprintf(_itemValueBuffer, "colors: %lu, exists: %lu, in packets: %lu", + _colorBitsWritten, _existsBitsWritten, _existsInPacketBitsWritten); + break; + } + case ITEM_MODE: { + sprintf(_itemValueBuffer, "%s - %s", (_isFullScene ? "Full Scene" : "Partial Scene"), + (_isMoving ? "Moving" : "Stationary")); + break; + } + default: + sprintf(_itemValueBuffer, ""); + break; + } + return _itemValueBuffer; +} + diff --git a/libraries/voxels/src/VoxelSceneStats.h b/libraries/voxels/src/VoxelSceneStats.h new file mode 100644 index 0000000000..ded2061a6e --- /dev/null +++ b/libraries/voxels/src/VoxelSceneStats.h @@ -0,0 +1,170 @@ +// +// VoxelSceneStats.h +// hifi +// +// Created by Brad Hefta-Gaub on 7/18/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// +// + +#ifndef __hifi__VoxelSceneStats__ +#define __hifi__VoxelSceneStats__ + +#include +#include + +class VoxelNode; + +class VoxelSceneStats { +public: + VoxelSceneStats(); + void reset(); + void sceneStarted(bool fullScene, bool moving, VoxelNode* root); + void sceneCompleted(); + + void printDebugDetails(); + void packetSent(int bytes); + + void encodeStarted(); + void encodeStopped(); + + void traversed(const VoxelNode* node); + void skippedDistance(const VoxelNode* node); + void skippedOutOfView(const VoxelNode* node); + void skippedWasInView(const VoxelNode* node); + void skippedNoChange(const VoxelNode* node); + void skippedOccluded(const VoxelNode* node); + void colorSent(const VoxelNode* node); + void didntFit(const VoxelNode* node); + void colorBitsWritten(); + void existsBitsWritten(); + void existsInPacketBitsWritten(); + void childBitsRemoved(bool includesExistsBits, bool includesColors); + + int packIntoMessage(unsigned char* destinationBuffer, int availableBytes); + int unpackFromMessage(unsigned char* sourceBuffer, int availableBytes); + + bool isReadyToSend() const { return _isReadyToSend; } + void markAsSent() { _isReadyToSend = false; } + unsigned char* getStatsMessage() { return &_statsMessage[0]; } + int getStatsMessageLength() const { return _statsMessageLength; } + + enum { + ITEM_ELAPSED, + ITEM_ENCODE, + ITEM_PACKETS, + ITEM_VOXELS_SERVER, + ITEM_VOXELS, + ITEM_COLORS, + ITEM_BITS, + ITEM_TRAVERSED, + ITEM_SKIPPED, + ITEM_SKIPPED_DISTANCE, + ITEM_SKIPPED_OUT_OF_VIEW, + ITEM_SKIPPED_WAS_IN_VIEW, + ITEM_SKIPPED_NO_CHANGE, + ITEM_SKIPPED_OCCLUDED, + ITEM_DIDNT_FIT, + ITEM_MODE, + ITEM_COUNT + }; + + // Meta information about each stats item + struct ItemInfo { + char const* const caption; + unsigned colorRGBA; + }; + + ItemInfo& getItemInfo(int item) { return _ITEMS[item]; }; + char* getItemValue(int item); + +private: + bool _isReadyToSend; + unsigned char _statsMessage[MAX_PACKET_SIZE]; + int _statsMessageLength; + + // scene timing data in usecs + bool _isStarted; + uint64_t _start; + uint64_t _end; + uint64_t _elapsed; + + SimpleMovingAverage _elapsedAverage; + SimpleMovingAverage _bitsPerVoxelAverage; + + uint64_t _totalEncodeTime; + uint64_t _encodeStart; + + // scene voxel related data + unsigned long _totalVoxels; + unsigned long _totalInternal; + unsigned long _totalLeaves; + + unsigned long _traversed; + unsigned long _internal; + unsigned long _leaves; + + unsigned long _skippedDistance; + unsigned long _internalSkippedDistance; + unsigned long _leavesSkippedDistance; + + unsigned long _skippedOutOfView; + unsigned long _internalSkippedOutOfView; + unsigned long _leavesSkippedOutOfView; + + unsigned long _skippedWasInView; + unsigned long _internalSkippedWasInView; + unsigned long _leavesSkippedWasInView; + + unsigned long _skippedNoChange; + unsigned long _internalSkippedNoChange; + unsigned long _leavesSkippedNoChange; + + unsigned long _skippedOccluded; + unsigned long _internalSkippedOccluded; + unsigned long _leavesSkippedOccluded; + + unsigned long _colorSent; + unsigned long _internalColorSent; + unsigned long _leavesColorSent; + + unsigned long _didntFit; + unsigned long _internalDidntFit; + unsigned long _leavesDidntFit; + + unsigned long _colorBitsWritten; + unsigned long _existsBitsWritten; + unsigned long _existsInPacketBitsWritten; + unsigned long _treesRemoved; + + // Accounting Notes: + // + // 1) number of voxels sent can be calculated as _colorSent + _colorBitsWritten. This works because each internal + // node in a packet will have a _colorBitsWritten included for it and each "leaf" in the packet will have a + // _colorSent written for it. Note that these "leaf" nodes in the packets may not be actual leaves in the full + // tree, because LOD may cause us to send an average color for an internal node instead of recursing deeper to + // the leaves. + // + // 2) the stats balance if: (working assumption) + // if _colorSent > 0 + // _traversed = all skipped + _colorSent + _colorBitsWritten + // else + // _traversed = all skipped + _colorSent + _colorBitsWritten + _treesRemoved + // + + // scene network related data + unsigned int _packets; + unsigned long _bytes; + unsigned int _passes; + + // features related items + bool _isMoving; + bool _isFullScene; + + + static ItemInfo _ITEMS[]; + static int const MAX_ITEM_VALUE_LENGTH = 128; + char _itemValueBuffer[MAX_ITEM_VALUE_LENGTH]; +}; + +#endif /* defined(__hifi__VoxelSceneStats__) */ diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index 9cb6a85f93..fe67a50dee 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -673,6 +673,9 @@ void VoxelTree::reaverageVoxelColors(VoxelNode *startNode) { if (hasChildren && !startNode->collapseIdenticalLeaves()) { startNode->setColorFromAverageOfChildren(); } + + // this is also a good time to recalculateSubTreeNodeCount() + startNode->recalculateSubTreeNodeCount(); } } @@ -1037,6 +1040,13 @@ int VoxelTree::encodeTreeBitstream(VoxelNode* node, unsigned char* outputBuffer, availableBytes -= codeLength; // keep track or remaining space int currentEncodeLevel = 0; + + // record some stats, this is the one node that we won't record below in the recursion function, so we need to + // track it here + if (params.stats) { + params.stats->traversed(node); + } + int childBytesWritten = encodeTreeBitstreamRecursion(node, outputBuffer, availableBytes, bag, params, currentEncodeLevel); // if childBytesWritten == 1 then something went wrong... that's not possible @@ -1061,6 +1071,9 @@ int VoxelTree::encodeTreeBitstream(VoxelNode* node, unsigned char* outputBuffer, int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outputBuffer, int availableBytes, VoxelNodeBag& bag, EncodeBitstreamParams& params, int& currentEncodeLevel) const { + // you can't call this without a valid node + assert(node); + // How many bytes have we written so far at this level; int bytesAtThisLevel = 0; @@ -1081,6 +1094,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp // If we're too far away for our render level, then just return if (distance >= boundaryDistance) { + if (params.stats) { + params.stats->skippedDistance(node); + } return bytesAtThisLevel; } @@ -1088,6 +1104,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp // although technically, we really shouldn't ever be here, because our callers shouldn't be calling us if // we're out of view if (!node->isInView(*params.viewFrustum)) { + if (params.stats) { + params.stats->skippedOutOfView(node); + } return bytesAtThisLevel; } @@ -1110,6 +1129,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp // if we're in deltaViewFrustum mode, and this node has changed since it was last sent, then we do // need to send it. if (wasInView && !(params.deltaViewFrustum && node->hasChangedSince(params.lastViewFrustumSent - CHANGE_FUDGE))) { + if (params.stats) { + params.stats->skippedWasInView(node); + } return bytesAtThisLevel; } @@ -1117,6 +1139,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp // then we can also bail early and save bits if (!params.forceSendScene && !params.deltaViewFrustum && !node->hasChangedSince(params.lastViewFrustumSent - CHANGE_FUDGE)) { + if (params.stats) { + params.stats->skippedNoChange(node); + } return bytesAtThisLevel; } @@ -1136,6 +1161,9 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp CoverageMapStorageResult result = params.map->checkMap(voxelPolygon, false); delete voxelPolygon; // cleanup if (result == OCCLUDED) { + if (params.stats) { + params.stats->skippedOccluded(node); + } return bytesAtThisLevel; } } else { @@ -1201,6 +1229,13 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp distancesToChildren[i] = 0.0f; currentCount++; } + + // track stats + // must check childNode here, because it could be we got here with no childNode + if (params.stats && childNode) { + params.stats->traversed(childNode); + } + } // for each child node in Distance sorted order..., check to see if they exist, are colored, and in view, and if so @@ -1211,13 +1246,23 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp bool childIsInView = (childNode && (!params.viewFrustum || childNode->isInView(*params.viewFrustum))); - if (childIsInView) { + if (!childIsInView) { + // must check childNode here, because it could be we got here because there was no childNode + if (params.stats && childNode) { + params.stats->skippedOutOfView(childNode); + } + } else { // Before we determine consider this further, let's see if it's in our LOD scope... float distance = distancesToChildren[i]; // params.viewFrustum ? childNode->distanceToCamera(*params.viewFrustum) : 0; float boundaryDistance = !params.viewFrustum ? 1 : boundaryDistanceForRenderLevel(childNode->getLevel() + params.boundaryLevelAdjust); - if (distance < boundaryDistance) { + if (!(distance < boundaryDistance)) { + // don't need to check childNode here, because we can't get here with no childNode + if (params.stats) { + params.stats->skippedDistance(childNode); + } + } else { inViewCount++; // track children in view as existing and not a leaf, if they're a leaf, @@ -1261,7 +1306,21 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp } // wants occlusion culling & isLeaf() - bool shouldRender = !params.viewFrustum ? true : childNode->calculateShouldRender(params.viewFrustum, params.boundaryLevelAdjust); + bool shouldRender = !params.viewFrustum + ? true + : childNode->calculateShouldRender(params.viewFrustum, params.boundaryLevelAdjust); + + // track some stats + if (params.stats) { + // don't need to check childNode here, because we can't get here with no childNode + if (!shouldRender && childNode->isLeaf()) { + params.stats->skippedDistance(childNode); + } + // don't need to check childNode here, because we can't get here with no childNode + if (childIsOccluded) { + params.stats->skippedOccluded(childNode); + } + } // track children with actual color, only if the child wasn't previously in view! if (shouldRender && !childIsOccluded) { @@ -1288,7 +1347,15 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp inViewWithColorCount++; } else { // otherwise just track stats of the items we discarded - params.childWasInViewDiscarded++; + // don't need to check childNode here, because we can't get here with no childNode + if (params.stats) { + if (childWasInView) { + params.stats->skippedWasInView(childNode); + } else { + params.stats->skippedNoChange(childNode); + } + } + } } } @@ -1297,14 +1364,24 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp *writeToThisLevelBuffer = childrenColoredBits; writeToThisLevelBuffer += sizeof(childrenColoredBits); // move the pointer bytesAtThisLevel += sizeof(childrenColoredBits); // keep track of byte count + if (params.stats) { + params.stats->colorBitsWritten(); + } // write the color data... if (params.includeColor) { for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { if (oneAtBit(childrenColoredBits, i)) { - memcpy(writeToThisLevelBuffer, &node->getChildAtIndex(i)->getColor(), BYTES_PER_COLOR); + VoxelNode* childNode = node->getChildAtIndex(i); + memcpy(writeToThisLevelBuffer, &childNode->getColor(), BYTES_PER_COLOR); writeToThisLevelBuffer += BYTES_PER_COLOR; // move the pointer for color bytesAtThisLevel += BYTES_PER_COLOR; // keep track of byte count for color + + // don't need to check childNode here, because we can't get here with no childNode + if (params.stats) { + params.stats->colorSent(childNode); + } + } } } @@ -1315,12 +1392,18 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp *writeToThisLevelBuffer = childrenExistInTreeBits; writeToThisLevelBuffer += sizeof(childrenExistInTreeBits); // move the pointer bytesAtThisLevel += sizeof(childrenExistInTreeBits); // keep track of byte count + if (params.stats) { + params.stats->existsBitsWritten(); + } } // write the child exist bits *writeToThisLevelBuffer = childrenExistInPacketBits; writeToThisLevelBuffer += sizeof(childrenExistInPacketBits); // move the pointer bytesAtThisLevel += sizeof(childrenExistInPacketBits); // keep track of byte count + if (params.stats) { + params.stats->existsInPacketBitsWritten(); + } // We only need to keep digging, if there is at least one child that is inView, and not a leaf. keepDiggingDeeper = (inViewNotLeafCount > 0); @@ -1333,6 +1416,12 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp availableBytes -= bytesAtThisLevel; } else { bag.insert(node); + + // don't need to check node here, because we can't get here with no node + if (params.stats) { + params.stats->didntFit(node); + } + return 0; } @@ -1393,7 +1482,12 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp // so, if the child returns 2 bytes out, we can actually consider that an empty tree also!! // // we can make this act like no bytes out, by just resetting the bytes out in this case - if (params.includeColor && childTreeBytesOut == 2) { + if (params.includeColor && !params.includeExistsBits && childTreeBytesOut == 2) { + childTreeBytesOut = 0; // this is the degenerate case of a tree with no colors and no child trees + } + // If we've asked for existBits, this is also true, except that the tree will output 3 bytes + // NOTE: does this introduce a problem with detecting deletion?? + if (params.includeColor && params.includeExistsBits && childTreeBytesOut == 3) { childTreeBytesOut = 0; // this is the degenerate case of a tree with no colors and no child trees } @@ -1408,6 +1502,12 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp childrenExistInPacketBits -= (1 << (7 - originalIndex)); // repair the child exists mask *childExistsPlaceHolder = childrenExistInPacketBits; + + // If this is the last of the child exists bits, then we're actually be rolling out the entire tree + if (params.stats && childrenExistInPacketBits == 0) { + params.stats->childBitsRemoved(params.includeExistsBits, params.includeColor); + } + // Note: no need to move the pointer, cause we already stored this } // end if (childTreeBytesOut == 0) } // end if (oneAtBit(childrenExistInPacketBits, originalIndex)) diff --git a/libraries/voxels/src/VoxelTree.h b/libraries/voxels/src/VoxelTree.h index e5db6526e9..199942605e 100644 --- a/libraries/voxels/src/VoxelTree.h +++ b/libraries/voxels/src/VoxelTree.h @@ -9,12 +9,14 @@ #ifndef __hifi__VoxelTree__ #define __hifi__VoxelTree__ -#include "SimpleMovingAverage.h" +#include +#include + +#include "CoverageMap.h" #include "ViewFrustum.h" #include "VoxelNode.h" #include "VoxelNodeBag.h" -#include "CoverageMap.h" -#include "PointerStack.h" +#include "VoxelSceneStats.h" // Callback function, for recuseTreeWithOperation typedef bool (*RecurseVoxelTreeOperation)(VoxelNode* node, void* extraData); @@ -36,6 +38,7 @@ typedef enum {GRADIENT, RANDOM, NATURAL} creationMode; #define NO_BOUNDARY_ADJUST 0 #define LOW_RES_MOVING_ADJUST 1 #define IGNORE_LAST_SENT 0 +#define IGNORE_SCENE_STATS NULL class EncodeBitstreamParams { public: @@ -48,10 +51,10 @@ public: bool deltaViewFrustum; const ViewFrustum* lastViewFrustum; bool wantOcclusionCulling; - long childWasInViewDiscarded; int boundaryLevelAdjust; uint64_t lastViewFrustumSent; bool forceSendScene; + VoxelSceneStats* stats; CoverageMap* map; EncodeBitstreamParams( @@ -66,7 +69,8 @@ public: CoverageMap* map = IGNORE_COVERAGE_MAP, int boundaryLevelAdjust = NO_BOUNDARY_ADJUST, uint64_t lastViewFrustumSent = IGNORE_LAST_SENT, - bool forceSendScene = true) : + bool forceSendScene = true, + VoxelSceneStats* stats = IGNORE_SCENE_STATS) : maxEncodeLevel (maxEncodeLevel), maxLevelReached (0), viewFrustum (viewFrustum), @@ -76,10 +80,10 @@ public: deltaViewFrustum (deltaViewFrustum), lastViewFrustum (lastViewFrustum), wantOcclusionCulling (wantOcclusionCulling), - childWasInViewDiscarded (0), boundaryLevelAdjust (boundaryLevelAdjust), lastViewFrustumSent (lastViewFrustumSent), forceSendScene (forceSendScene), + stats (stats), map (map) {} }; diff --git a/voxel-server/src/VoxelNodeData.h b/voxel-server/src/VoxelNodeData.h index 96b61db963..746db6da93 100644 --- a/voxel-server/src/VoxelNodeData.h +++ b/voxel-server/src/VoxelNodeData.h @@ -12,9 +12,11 @@ #include #include #include -#include "VoxelNodeBag.h" -#include "VoxelConstants.h" -#include "CoverageMap.h" + +#include +#include +#include +#include class VoxelNodeData : public AvatarData { public: @@ -58,6 +60,9 @@ public: void setLastTimeBagEmpty(uint64_t lastTimeBagEmpty) { _lastTimeBagEmpty = lastTimeBagEmpty; }; bool getCurrentPacketIsColor() const { return _currentPacketIsColor; }; + + VoxelSceneStats stats; + private: VoxelNodeData(const VoxelNodeData &); VoxelNodeData& operator= (const VoxelNodeData&); diff --git a/voxel-server/src/main.cpp b/voxel-server/src/main.cpp index 8a8048035b..c009eec2a5 100644 --- a/voxel-server/src/main.cpp +++ b/voxel-server/src/main.cpp @@ -60,6 +60,7 @@ bool wantLocalDomain = false; bool wantColorRandomizer = false; bool debugVoxelSending = false; bool shouldShowAnimationDebug = false; +bool displayVoxelStats = false; EnvironmentData environmentData[3]; @@ -111,6 +112,44 @@ void eraseVoxelTreeAndCleanupNodeVisitData() { pthread_mutex_t treeLock; +void handlePacketSend(NodeList* nodeList, + NodeList::iterator& node, + VoxelNodeData* nodeData, + int& trueBytesSent, int& truePacketsSent) { + // If we've got a stats message ready to send, then see if we can piggyback them together + if (nodeData->stats.isReadyToSend()) { + // Send the stats message to the client + unsigned char* statsMessage = nodeData->stats.getStatsMessage(); + int statsMessageLength = nodeData->stats.getStatsMessageLength(); + + // If the size of the stats message and the voxel message will fit in a packet, then piggyback them + if (nodeData->getPacketLength() + statsMessageLength < MAX_PACKET_SIZE) { + + // copy voxel message to back of stats message + memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength()); + statsMessageLength += nodeData->getPacketLength(); + + // actually send it + nodeList->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength); + } else { + // not enough room in the packet, send two packets + nodeList->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength); + nodeList->getNodeSocket()->send(node->getActiveSocket(), + nodeData->getPacket(), nodeData->getPacketLength()); + } + } else { + // just send the voxel packet + nodeList->getNodeSocket()->send(node->getActiveSocket(), + nodeData->getPacket(), nodeData->getPacketLength()); + } + // remember to track our stats + nodeData->stats.packetSent(nodeData->getPacketLength()); + trueBytesSent += nodeData->getPacketLength(); + truePacketsSent++; + nodeData->resetVoxelPacket(); +} + + // Version of voxel distributor that sends the deepest LOD level at once void deepestLevelVoxelDistributor(NodeList* nodeList, NodeList::iterator& node, @@ -141,11 +180,9 @@ void deepestLevelVoxelDistributor(NodeList* nodeList, printf("wantColor=%s --- SENDING PARTIAL PACKET! nodeData->getCurrentPacketIsColor()=%s\n", debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor())); } - nodeList->getNodeSocket()->send(node->getActiveSocket(), - nodeData->getPacket(), nodeData->getPacketLength()); - trueBytesSent += nodeData->getPacketLength(); - truePacketsSent++; - nodeData->resetVoxelPacket(); + + handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); + } else { if (::debugVoxelSending) { printf("wantColor=%s --- FIXING HEADER! nodeData->getCurrentPacketIsColor()=%s\n", @@ -200,13 +237,20 @@ void deepestLevelVoxelDistributor(NodeList* nodeList, // only set our last sent time if we weren't resetting due to frustum change uint64_t now = usecTimestampNow(); nodeData->setLastTimeBagEmpty(now); - if (::debugVoxelSending) { - printf("ENTIRE SCENE SENT! nodeData->setLastTimeBagEmpty(now=[%lld])\n", now); - } } - + + nodeData->stats.sceneCompleted(); + + if (::displayVoxelStats) { + nodeData->stats.printDebugDetails(); + } + // This is the start of "resending" the scene. nodeData->nodeBag.insert(serverTree.rootNode); + + // start tracking our stats + bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging(); + nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, ::serverTree.rootNode); } // If we have something in our nodeBag, then turn them into packets and send them out... @@ -239,33 +283,32 @@ void deepestLevelVoxelDistributor(NodeList* nodeList, CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP; int boundaryLevelAdjust = viewFrustumChanged && nodeData->getWantLowResMoving() ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST; + + bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && + nodeData->getViewFrustumJustStoppedChanging(); EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor, WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum, wantOcclusionCulling, coverageMap, boundaryLevelAdjust, nodeData->getLastTimeBagEmpty(), - nodeData->getViewFrustumJustStoppedChanging()); - + isFullScene, &nodeData->stats); + + nodeData->stats.encodeStarted(); bytesWritten = serverTree.encodeTreeBitstream(subTree, &tempOutputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1, nodeData->nodeBag, params); + nodeData->stats.encodeStopped(); if (nodeData->getAvailable() >= bytesWritten) { nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten); } else { - nodeList->getNodeSocket()->send(node->getActiveSocket(), - nodeData->getPacket(), nodeData->getPacketLength()); - trueBytesSent += nodeData->getPacketLength(); - truePacketsSent++; + handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); packetsSentThisInterval++; nodeData->resetVoxelPacket(); nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten); } } else { if (nodeData->isPacketWaiting()) { - nodeList->getNodeSocket()->send(node->getActiveSocket(), - nodeData->getPacket(), nodeData->getPacketLength()); - trueBytesSent += nodeData->getPacketLength(); - truePacketsSent++; + handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent); nodeData->resetVoxelPacket(); } packetsSentThisInterval = PACKETS_PER_CLIENT_PER_INTERVAL; // done for now, no nodes left @@ -368,7 +411,9 @@ void *distributeVoxelsToListeners(void *args) { if (usecToSleep > 0) { usleep(usecToSleep); } else { - std::cout << "Last send took too much time, not sleeping!\n"; + if (::debugVoxelSending) { + std::cout << "Last send took too much time, not sleeping!\n"; + } } } @@ -401,6 +446,10 @@ int main(int argc, const char * argv[]) { nodeList->startSilentNodeRemovalThread(); srand((unsigned)time(0)); + + const char* DISPLAY_VOXEL_STATS = "--displayVoxelStats"; + ::displayVoxelStats = cmdOptionExists(argc, argv, DISPLAY_VOXEL_STATS); + printf("displayVoxelStats=%s\n", debug::valueOf(::displayVoxelStats)); const char* DEBUG_VOXEL_SENDING = "--debugVoxelSending"; ::debugVoxelSending = cmdOptionExists(argc, argv, DEBUG_VOXEL_SENDING); @@ -437,8 +486,10 @@ int main(int argc, const char * argv[]) { ::serverTree.clearDirtyBit(); // the tree is clean since we just loaded it printf("DONE loading voxels from file... fileRead=%s\n", debug::valueOf(persistantFileRead)); - unsigned long nodeCount = ::serverTree.getVoxelCount(); - printf("Nodes after loading scene %ld nodes\n", nodeCount); + unsigned long nodeCount = ::serverTree.rootNode->getSubTreeNodeCount(); + unsigned long internalNodeCount = ::serverTree.rootNode->getSubTreeInternalNodeCount(); + unsigned long leafNodeCount = ::serverTree.rootNode->getSubTreeLeafNodeCount(); + printf("Nodes after loading scene %lu nodes %lu internal %lu leaves\n", nodeCount, internalNodeCount, leafNodeCount); } // Check to see if the user passed in a command line option for loading an old style local