From bdbb12f4dcb71f2bbe93b9c5050e176842a6feec Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 14:35:40 -0700 Subject: [PATCH 01/38] switched edit nacks to non-verified --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 2 +- libraries/networking/src/PacketHeaders.h | 2 +- libraries/octree/src/OctreeEditPacketSender.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 44c9576943..78239bdf1f 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -256,7 +256,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { numSequenceNumbersAvailable -= numSequenceNumbers; // send it - qint64 bytesWritten = NodeList::getInstance()->writeDatagram(packet, dataAt - packet, destinationNode); + qint64 bytesWritten = NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); printf("\t\t wrote %lld bytes\n\n", bytesWritten); packetsSent++; diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 5ed4110627..0f87b0e607 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -79,7 +79,7 @@ const QSet NON_VERIFIED_PACKETS = QSet() << PacketTypeDomainList << PacketTypeDomainListRequest << PacketTypeDomainOAuthRequest << PacketTypeCreateAssignment << PacketTypeRequestAssignment << PacketTypeStunResponse << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery - << PacketTypeOctreeDataNack; + << PacketTypeOctreeDataNack << PacketTypeVoxelEditNack << PacketTypeParticleEditNack << PacketTypeModelEditNack; const int NUM_BYTES_MD5_HASH = 16; const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 01d2ecf464..5281617e72 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -379,7 +379,7 @@ void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { // read number of sequence numbers uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); dataAt += sizeof(uint16_t); - + // read sequence numbers and queue packets for resend for (int i = 0; i < numSequenceNumbers; i++) { unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); From 531c32fdd35bf7038cd6763a077c9c03874e02ba Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 18:07:50 -0700 Subject: [PATCH 02/38] More comments, stubbing out incremental streaming. --- libraries/metavoxels/src/Bitstream.h | 6 +++--- libraries/metavoxels/src/MetavoxelData.cpp | 8 ++++++++ libraries/metavoxels/src/MetavoxelData.h | 11 ++++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 0d9e516640..b116bd4b49 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -1104,7 +1104,7 @@ private: Q_DECLARE_METATYPE(const QMetaObject*) /// Macro for registering streamable meta-objects. Typically, one would use this macro at the top level of the source file -/// associated with the class. +/// associated with the class. The class should have a no-argument constructor flagged with Q_INVOKABLE. #define REGISTER_META_OBJECT(x) static int x##Registration = Bitstream::registerMetaObject(#x, &x::staticMetaObject); /// Contains a value along with a pointer to its streamer. This is stored in QVariants when using fallback generics and @@ -1563,8 +1563,8 @@ public: Bitstream::registerTypeStreamer(qMetaTypeId(), new CollectionTypeStreamer()); /// Declares the metatype and the streaming operators. Typically, one would use this immediately after the definition of a -/// type flagged as STREAMABLE in its header file. The last lines ensure that the generated file will be included in the link -/// phase. +/// type flagged as STREAMABLE in its header file. The type should have a no-argument constructor. The last lines of this +/// macro ensure that the generated file will be included in the link phase. #ifdef _WIN32 #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 0d52fc5ed6..503dcce8d2 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -602,6 +602,14 @@ void MetavoxelData::writeDelta(const MetavoxelData& reference, const MetavoxelLO } } +void MetavoxelData::readIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, + Bitstream& in, const MetavoxelLOD& lod) { +} + +void MetavoxelData::writeIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, + Bitstream& out, const MetavoxelLOD& lod) const { +} + MetavoxelNode* MetavoxelData::createRoot(const AttributePointer& attribute) { MetavoxelNode*& root = _roots[attribute]; if (root) { diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 2e6f6c4437..199b6a8aff 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -54,7 +54,8 @@ public: DECLARE_STREAMABLE_METATYPE(MetavoxelLOD) -/// The base metavoxel representation shared between server and client. +/// The base metavoxel representation shared between server and client. Contains a size (for all dimensions) and a set of +/// octrees for different attributes. class MetavoxelData { public: @@ -64,11 +65,14 @@ public: MetavoxelData& operator=(const MetavoxelData& other); + /// Sets the size in all dimensions. void setSize(float size) { _size = size; } float getSize() const { return _size; } + /// Returns the minimum extent of the octrees (which are centered about the origin). glm::vec3 getMinimum() const { return glm::vec3(_size, _size, _size) * -0.5f; } + /// Returns the bounds of the octrees. Box getBounds() const; /// Applies the specified visitor to the contained voxels. @@ -107,6 +111,11 @@ public: void writeDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, Bitstream& out, const MetavoxelLOD& lod) const; + void readIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, + Bitstream& in, const MetavoxelLOD& lod); + void writeIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, + Bitstream& out, const MetavoxelLOD& lod) const; + MetavoxelNode* getRoot(const AttributePointer& attribute) const { return _roots.value(attribute); } MetavoxelNode* createRoot(const AttributePointer& attribute); From 2dbf8d7daf6a45a18991f06dd6ec1ef08dfe8f15 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 19 Jun 2014 11:08:21 -0700 Subject: [PATCH 03/38] Fix fullscreen toggle shortcut Remove the "Meta" modifier so that the shortcut works on Windows. --- interface/src/Menu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 5c8c2e97aa..9df3e25bf8 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -251,7 +251,7 @@ Menu::Menu() : addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::Fullscreen, - Qt::CTRL | Qt::META | Qt::Key_F, + Qt::CTRL | Qt::Key_F, false, appInstance, SLOT(setFullscreen(bool))); From aa3602f0fb5fa0def615377e25dd52e2f33e5a3f Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 14:34:55 -0700 Subject: [PATCH 04/38] added InterframeTimeGapHistory and frameReceived() calls --- .../src/audio/AvatarAudioRingBuffer.cpp | 2 + libraries/audio/src/AudioRingBuffer.cpp | 57 +++++++++++++++++++ libraries/audio/src/AudioRingBuffer.h | 28 +++++++++ .../audio/src/InjectedAudioRingBuffer.cpp | 2 + 4 files changed, 89 insertions(+) diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 5613a64cc4..47a0ecbea8 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,6 +19,8 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { + _timeGapHistory.frameReceived(); + _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); } diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 2101fcb9cd..ea15e27ef5 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include "SharedUtil.h" #include @@ -19,6 +20,62 @@ #include "AudioRingBuffer.h" +InterframeTimeGapHistory::InterframeTimeGapHistory() + : _lastFrameReceivedTime(0), + _numSamplesInCurrentInterval(0), + _currentIntervalMaxGap(0), + _newestIntervalMaxGapAt(0), + _windowMaxGap(0), + _newWindowMaxGapAvailable(false) +{ + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); +} + +void InterframeTimeGapHistory::frameReceived() { + quint64 now = usecTimestampNow(); + + // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + if (_lastFrameReceivedTime != 0) { + quint64 gap = now - _lastFrameReceivedTime; + + // update the current interval max + if (gap > _currentIntervalMaxGap) { + _currentIntervalMaxGap = gap; + } + _numSamplesInCurrentInterval++; + + // if the current interval of samples is now full, record it in our interval maxes + if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { + + // find location to insert this interval's max (increment index cyclically) + _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; + + // record the current interval's max gap as the newest + _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; + + // update the window max gap, which is the max out of all the past intervals' max gaps + _windowMaxGap = 0; + for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { + if (_intervalMaxGaps[i] > _windowMaxGap) { + _windowMaxGap = _intervalMaxGaps[i]; + } + } + _newWindowMaxGapAvailable = true; + + // reset the current interval + _numSamplesInCurrentInterval = 0; + _currentIntervalMaxGap = 0; + } + } + _lastFrameReceivedTime = now; +} + +quint64 InterframeTimeGapHistory::getPastWindowMaxGap() { + _newWindowMaxGapAvailable = false; + return _windowMaxGap; +} + + AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 04cc67c8ac..105e3761ae 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -21,6 +21,30 @@ #include "NodeData.h" +// this means that every 500 samples, the max for the past 10*500 samples will be calculated +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; + +class InterframeTimeGapHistory { +public: + InterframeTimeGapHistory(); + + void frameReceived(); + bool isNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 getPastWindowMaxGap(); + +private: + quint64 _lastFrameReceivedTime; + + int _numSamplesInCurrentInterval; + quint64 _currentIntervalMaxGap; + quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; + int _newestIntervalMaxGapAt; + quint64 _windowMaxGap; + bool _newWindowMaxGapAvailable; +}; + + const int SAMPLE_RATE = 24000; const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; @@ -74,6 +98,8 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); + + InterframeTimeGapHistory& getInterframeTimeGapHistory() { return _timeGapHistory; } protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); @@ -89,6 +115,8 @@ protected: bool _isStarved; bool _hasStarted; bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing + + InterframeTimeGapHistory _timeGapHistory; }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 2658b4c336..103d0d8d2e 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,6 +31,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { + _timeGapHistory.frameReceived(); + // setup a data stream to read from this packet QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); From 7a9c0f0e6a504038bf7d19ef616ab4c5d3054945 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Jun 2014 14:50:19 -0700 Subject: [PATCH 05/38] Working on tests for metavoxel streaming. --- libraries/metavoxels/src/MetavoxelData.cpp | 69 +++++++++ libraries/metavoxels/src/MetavoxelData.h | 13 +- tests/metavoxels/src/MetavoxelTests.cpp | 164 +++++++++++++++++---- tests/metavoxels/src/MetavoxelTests.h | 16 +- 4 files changed, 229 insertions(+), 33 deletions(-) diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 503dcce8d2..cf645e6f5e 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -604,10 +604,79 @@ void MetavoxelData::writeDelta(const MetavoxelData& reference, const MetavoxelLO void MetavoxelData::readIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, Bitstream& in, const MetavoxelLOD& lod) { + + // shallow copy the reference + *this = reference; + + // if nothing changed, return + bool changed; + in >> changed; + if (!changed) { + return; + } + + // expand if the size has changed + bool sizeChanged; + in >> sizeChanged; + if (sizeChanged) { + float size; + in >> size; + while (_size < size) { + expand(); + } + } + + // read the nodes removed + forever { + AttributePointer attribute; + in >> attribute; + if (!attribute) { + break; + } + _roots.take(attribute)->decrementReferenceCount(attribute); + } } void MetavoxelData::writeIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, Bitstream& out, const MetavoxelLOD& lod) const { + + // first things first: there might be no change whatsoever + glm::vec3 minimum = getMinimum(); + bool becameSubdivided = lod.becameSubdivided(minimum, _size, referenceLOD); + if (_size == reference._size && _roots == reference._roots && !becameSubdivided) { + out << false; + return; + } + out << true; + + // compare the size; if changed (rare), we must compare to the expanded reference + const MetavoxelData* expandedReference = &reference; + if (_size == reference._size) { + out << false; + } else { + out << true; + out << _size; + + MetavoxelData* expanded = new MetavoxelData(reference); + while (expanded->_size < _size) { + expanded->expand(); + } + expandedReference = expanded; + } + + // write the nodes removed + for (QHash::const_iterator it = expandedReference->_roots.constBegin(); + it != expandedReference->_roots.constEnd(); it++) { + if (!_roots.contains(it.key())) { + out << it.key(); + } + } + out << AttributePointer(); + + // delete the expanded reference if we had to expand + if (expandedReference != &reference) { + delete expandedReference; + } } MetavoxelNode* MetavoxelData::createRoot(const AttributePointer& attribute) { diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 199b6a8aff..6b262d6892 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -34,7 +34,8 @@ class NetworkValue; class Spanner; class SpannerRenderer; -/// Determines whether to subdivide each node when traversing. +/// Determines whether to subdivide each node when traversing. Contains the position (presumed to be of the viewer) and a +/// threshold value, where lower thresholds cause smaller/more distant voxels to be subdivided. class MetavoxelLOD { STREAMABLE @@ -46,6 +47,7 @@ public: bool isValid() const { return threshold > 0.0f; } + /// Checks whether, according to this LOD, we should subdivide the described voxel. bool shouldSubdivide(const glm::vec3& minimum, float size, float multiplier = 1.0f) const; /// Checks whether the node or any of the nodes underneath it have had subdivision enabled as compared to the reference. @@ -78,20 +80,25 @@ public: /// Applies the specified visitor to the contained voxels. void guide(MetavoxelVisitor& visitor); + /// Inserts a spanner into the specified attribute layer. void insert(const AttributePointer& attribute, const SharedObjectPointer& object); void insert(const AttributePointer& attribute, const Box& bounds, float granularity, const SharedObjectPointer& object); + /// Removes a spanner from the specified attribute layer. void remove(const AttributePointer& attribute, const SharedObjectPointer& object); void remove(const AttributePointer& attribute, const Box& bounds, float granularity, const SharedObjectPointer& object); + /// Toggles the existence of a spanner in the specified attribute layer (removes if present, adds if not). void toggle(const AttributePointer& attribute, const SharedObjectPointer& object); void toggle(const AttributePointer& attribute, const Box& bounds, float granularity, const SharedObjectPointer& object); + /// Replaces a spanner in the specified attribute layer. void replace(const AttributePointer& attribute, const SharedObjectPointer& oldObject, const SharedObjectPointer& newObject); void replace(const AttributePointer& attribute, const Box& bounds, float granularity, const SharedObjectPointer& oldObject, const SharedObjectPointer& newObject); - + + /// Clears all data in the specified attribute layer. void clear(const AttributePointer& attribute); /// Convenience function that finds the first spanner intersecting the provided ray. @@ -101,7 +108,7 @@ public: /// Sets part of the data. void set(const glm::vec3& minimum, const MetavoxelData& data, bool blend = false); - /// Expands the tree, increasing its capacity in all dimensions. + /// Expands the tree, doubling its size in all dimensions (that is, increasing its volume eightfold). void expand(); void read(Bitstream& in, const MetavoxelLOD& lod = MetavoxelLOD()); diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index c9bce27cc3..daee1165f6 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -321,44 +321,76 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { } bool MetavoxelTests::run() { - - qDebug() << "Running transmission tests..."; - qDebug(); - // seed the random number generator so that our tests are reproducible srand(0xBAAAAABE); - // create two endpoints with the same header + // check for an optional command line argument specifying a single test + QStringList arguments = this->arguments(); + int test = (arguments.size() > 1) ? arguments.at(1).toInt() : 0; + QByteArray datagramHeader("testheader"); - Endpoint alice(datagramHeader), bob(datagramHeader); - - alice.setOther(&bob); - bob.setOther(&alice); - - // perform a large number of simulation iterations const int SIMULATION_ITERATIONS = 10000; - for (int i = 0; i < SIMULATION_ITERATIONS; i++) { - if (alice.simulate(i) || bob.simulate(i)) { + if (test == 0 || test == 1) { + qDebug() << "Running transmission tests..."; + qDebug(); + + // create two endpoints with the same header + Endpoint alice(datagramHeader), bob(datagramHeader); + + alice.setOther(&bob); + bob.setOther(&alice); + + // perform a large number of simulation iterations + for (int i = 0; i < SIMULATION_ITERATIONS; i++) { + if (alice.simulate(i) || bob.simulate(i)) { + return true; + } + } + + qDebug() << "Sent" << highPriorityMessagesSent << "high priority messages, received" << highPriorityMessagesReceived; + qDebug() << "Sent" << unreliableMessagesSent << "unreliable messages, received" << unreliableMessagesReceived; + qDebug() << "Sent" << reliableMessagesSent << "reliable messages, received" << reliableMessagesReceived; + qDebug() << "Sent" << streamedBytesSent << "streamed bytes, received" << streamedBytesReceived; + qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << + datagramsReceived << "with" << bytesReceived << "bytes"; + qDebug() << "Created" << sharedObjectsCreated << "shared objects, destroyed" << sharedObjectsDestroyed; + qDebug() << "Performed" << objectMutationsPerformed << "object mutations"; + qDebug() << "Created" << scriptObjectsCreated << "script objects, mutated" << scriptMutationsPerformed; + qDebug(); + } + + if (test == 0 || test == 2) { + qDebug() << "Running serialization tests..."; + qDebug(); + + if (testSerialization(Bitstream::HASH_METADATA) || testSerialization(Bitstream::FULL_METADATA)) { return true; } } - qDebug() << "Sent" << highPriorityMessagesSent << "high priority messages, received" << highPriorityMessagesReceived; - qDebug() << "Sent" << unreliableMessagesSent << "unreliable messages, received" << unreliableMessagesReceived; - qDebug() << "Sent" << reliableMessagesSent << "reliable messages, received" << reliableMessagesReceived; - qDebug() << "Sent" << streamedBytesSent << "streamed bytes, received" << streamedBytesReceived; - qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << - datagramsReceived << "with" << bytesReceived << "bytes"; - qDebug() << "Created" << sharedObjectsCreated << "shared objects, destroyed" << sharedObjectsDestroyed; - qDebug() << "Performed" << objectMutationsPerformed << "object mutations"; - qDebug() << "Created" << scriptObjectsCreated << "script objects, mutated" << scriptMutationsPerformed; - qDebug(); + if (test == 0 || test == 3) { + qDebug() << "Running metavoxel data tests..."; + qDebug(); - qDebug() << "Running serialization tests..."; - qDebug(); + // clear the stats + datagramsSent = bytesSent = datagramsReceived = bytesReceived = 0; - if (testSerialization(Bitstream::HASH_METADATA) || testSerialization(Bitstream::FULL_METADATA)) { - return true; + // create client and server endpoints + Endpoint client(datagramHeader, Endpoint::METAVOXEL_CLIENT_MODE); + Endpoint server(datagramHeader, Endpoint::METAVOXEL_SERVER_MODE); + + client.setOther(&server); + server.setOther(&client); + + // simulate + for (int i = 0; i < SIMULATION_ITERATIONS; i++) { + if (client.simulate(i) || server.simulate(i)) { + return true; + } + } + + qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << + datagramsReceived << "with" << bytesReceived << "bytes"; } qDebug() << "All tests passed!"; @@ -375,7 +407,8 @@ static SharedObjectPointer createRandomSharedObject() { } } -Endpoint::Endpoint(const QByteArray& datagramHeader) : +Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) : + _mode(mode), _sequencer(new DatagramSequencer(datagramHeader, this)), _highPriorityMessagesToSend(0.0f), _reliableMessagesToSend(0.0f) { @@ -396,6 +429,13 @@ Endpoint::Endpoint(const QByteArray& datagramHeader) : ReceiveRecord receiveRecord = { 0 }; _receiveRecords.append(receiveRecord); + if (mode == METAVOXEL_CLIENT_MODE) { + _lod = MetavoxelLOD(glm::vec3(), 0.01f); + return; + } + if (mode == METAVOXEL_SERVER_MODE) { + return; + } // create the object that represents out delta-encoded state _localState = new TestSharedObjectA(); @@ -415,7 +455,7 @@ Endpoint::Endpoint(const QByteArray& datagramHeader) : QByteArray bytes = createRandomBytes(MIN_STREAM_BYTES, MAX_STREAM_BYTES); _dataStreamed.append(bytes); output->getBuffer().write(bytes); - streamedBytesSent += bytes.size(); + streamedBytesSent += bytes.size(); } static QVariant createRandomMessage() { @@ -525,6 +565,34 @@ bool Endpoint::simulate(int iterationNumber) { } } + if (_mode == METAVOXEL_CLIENT_MODE) { + Bitstream& out = _sequencer->startPacket(); + + ClientStateMessage state = { _lod }; + out << QVariant::fromValue(state); + _sequencer->endPacket(); + + // record the send + SendRecord record = { _sequencer->getOutgoingPacketNumber(), SharedObjectPointer(), MetavoxelData(), _lod }; + _sendRecords.append(record); + return false; + } + if (_mode == METAVOXEL_SERVER_MODE) { + // wait until we have a valid lod + if (!_lod.isValid()) { + return false; + } + Bitstream& out = _sequencer->startPacket(); + out << QVariant::fromValue(MetavoxelDeltaMessage()); + _data.writeDelta(_sendRecords.first().data, _sendRecords.first().lod, out, _lod); + _sequencer->endPacket(); + + // record the send + SendRecord record = { _sequencer->getOutgoingPacketNumber(), SharedObjectPointer(), _data, _lod }; + _sendRecords.append(record); + return false; + } + // enqueue some number of high priority messages const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; const float MAX_HIGH_PRIORITY_MESSAGES = 2.0f; @@ -619,6 +687,28 @@ void Endpoint::handleHighPriorityMessage(const QVariant& message) { } void Endpoint::readMessage(Bitstream& in) { + if (_mode == METAVOXEL_CLIENT_MODE) { + QVariant message; + in >> message; + handleMessage(message, in); + + // record the receipt + ReceiveRecord record = { _sequencer->getIncomingPacketNumber(), SharedObjectPointer(), + _data, _sendRecords.first().lod }; + _receiveRecords.append(record); + return; + } + if (_mode == METAVOXEL_SERVER_MODE) { + QVariant message; + in >> message; + handleMessage(message, in); + + // record the receipt + ReceiveRecord record = { _sequencer->getIncomingPacketNumber() }; + _receiveRecords.append(record); + return; + } + SequencedTestMessage message; in >> message; @@ -682,6 +772,22 @@ void Endpoint::clearReceiveRecordsBefore(int index) { _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1); } +void Endpoint::handleMessage(const QVariant& message, Bitstream& in) { + int userType = message.userType(); + if (userType == ClientStateMessage::Type) { + ClientStateMessage state = message.value(); + _lod = state.lod; + + } else if (userType == MetavoxelDeltaMessage::Type) { + _data.readDelta(_receiveRecords.first().data, _receiveRecords.first().lod, in, _sendRecords.first().lod); + + } else if (userType == QMetaType::QVariantList) { + foreach (const QVariant& element, message.toList()) { + handleMessage(element, in); + } + } +} + TestSharedObjectA::TestSharedObjectA(float foo, TestEnum baz, TestFlags bong) : _foo(foo), _baz(baz), diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index ac9eda2659..1de355661d 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -16,6 +16,7 @@ #include #include +#include #include class SequencedTestMessage; @@ -39,7 +40,9 @@ class Endpoint : public QObject { public: - Endpoint(const QByteArray& datagramHeader); + enum Mode { BASIC_PEER_MODE, METAVOXEL_SERVER_MODE, METAVOXEL_CLIENT_MODE }; + + Endpoint(const QByteArray& datagramHeader, Mode mode = BASIC_PEER_MODE); void setOther(Endpoint* other) { _other = other; } @@ -60,18 +63,26 @@ private slots: private: + void handleMessage(const QVariant& message, Bitstream& in); + class SendRecord { public: int packetNumber; SharedObjectPointer localState; + MetavoxelData data; + MetavoxelLOD lod; }; class ReceiveRecord { public: int packetNumber; SharedObjectPointer remoteState; + MetavoxelData data; + MetavoxelLOD lod; }; + Mode _mode; + DatagramSequencer* _sequencer; QList _sendRecords; QList _receiveRecords; @@ -79,6 +90,9 @@ private: SharedObjectPointer _localState; SharedObjectPointer _remoteState; + MetavoxelData _data; + MetavoxelLOD _lod; + Endpoint* _other; QList > _delayedDatagrams; float _highPriorityMessagesToSend; From 8176aa129289dc99eeaa85a9727dceac6fc27cbc Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 14:54:22 -0700 Subject: [PATCH 06/38] added debug code for testing time gap history --- libraries/audio/src/AudioRingBuffer.cpp | 6 ++++++ libraries/audio/src/AudioRingBuffer.h | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ea15e27ef5..f42ba49380 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -38,6 +38,8 @@ void InterframeTimeGapHistory::frameReceived() { if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; +printf("new gap: %llu\n", gap); + // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; @@ -47,6 +49,8 @@ void InterframeTimeGapHistory::frameReceived() { // if the current interval of samples is now full, record it in our interval maxes if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { +printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); + // find location to insert this interval's max (increment index cyclically) _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; @@ -62,6 +66,8 @@ void InterframeTimeGapHistory::frameReceived() { } _newWindowMaxGapAvailable = true; +printf("\t\t new window max gap: %llu\n", _windowMaxGap); + // reset the current interval _numSamplesInCurrentInterval = 0; _currentIntervalMaxGap = 0; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 105e3761ae..b702abde7b 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -22,8 +22,8 @@ #include "NodeData.h" // this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 6; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 4; class InterframeTimeGapHistory { public: From 77c377711a28d735405141c305c7d72e7460b620 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 15:39:54 -0700 Subject: [PATCH 07/38] removed debug code --- libraries/audio/src/AudioRingBuffer.cpp | 6 ------ libraries/audio/src/AudioRingBuffer.h | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index f42ba49380..ea15e27ef5 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -38,8 +38,6 @@ void InterframeTimeGapHistory::frameReceived() { if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; -printf("new gap: %llu\n", gap); - // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; @@ -49,8 +47,6 @@ printf("new gap: %llu\n", gap); // if the current interval of samples is now full, record it in our interval maxes if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { -printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); - // find location to insert this interval's max (increment index cyclically) _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; @@ -66,8 +62,6 @@ printf("\t interval full: max interval gap: %llu\n", _currentIntervalMaxGap); } _newWindowMaxGapAvailable = true; -printf("\t\t new window max gap: %llu\n", _windowMaxGap); - // reset the current interval _numSamplesInCurrentInterval = 0; _currentIntervalMaxGap = 0; diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index b702abde7b..105e3761ae 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -22,8 +22,8 @@ #include "NodeData.h" // this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 6; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 4; +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; class InterframeTimeGapHistory { public: From 100bc022ec55a0851a6d5ffb6bff1d16f06bd03c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 16:42:51 -0700 Subject: [PATCH 08/38] fixed a for loop that erased elements while iterating --- assignment-client/src/audio/AudioMixerClientData.cpp | 9 ++++++--- libraries/audio/src/AudioRingBuffer.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2f78a4ac78..2329946e49 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -120,9 +120,10 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSam } void AudioMixerClientData::pushBuffersAfterFrameSend() { - for (int i = 0; i < _ringBuffers.size(); i++) { + QList::iterator i = _ringBuffers.begin(); + while (i != _ringBuffers.end()) { // this was a used buffer, push the output pointer forwards - PositionalAudioRingBuffer* audioBuffer = _ringBuffers[i]; + PositionalAudioRingBuffer* audioBuffer = *i; if (audioBuffer->willBeAddedToMix()) { audioBuffer->shiftReadPosition(audioBuffer->isStereo() @@ -133,7 +134,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; - _ringBuffers.erase(_ringBuffers.begin() + i); + i = _ringBuffers.erase(i); + continue; } + i++; } } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 105e3761ae..46bc333e4b 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -25,6 +25,7 @@ const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; +// class used to track time between incoming frames for the purpose of varying the jitter buffer length class InterframeTimeGapHistory { public: InterframeTimeGapHistory(); From 435b5094a03437ab9f942a1ab1753961bc63b8ad Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 19 Jun 2014 17:58:38 -0700 Subject: [PATCH 09/38] moved time gap history to PositionalAudioRingBuffer, untested removed hard-coded jitter buffer length. --- assignment-client/src/audio/AudioMixer.cpp | 6 +- .../src/audio/AudioMixerClientData.cpp | 5 +- .../src/audio/AudioMixerClientData.h | 3 +- .../src/audio/AvatarAudioRingBuffer.cpp | 3 +- libraries/audio/src/AudioRingBuffer.cpp | 57 ------------- libraries/audio/src/AudioRingBuffer.h | 32 +------- .../audio/src/InjectedAudioRingBuffer.cpp | 3 +- .../audio/src/PositionalAudioRingBuffer.cpp | 79 ++++++++++++++++++- .../audio/src/PositionalAudioRingBuffer.h | 34 +++++++- 9 files changed, 118 insertions(+), 104 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2dc51b44a0..42be1aea5a 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -54,9 +54,6 @@ #include "AudioMixer.h" -const short JITTER_BUFFER_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); - const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f; const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer"; @@ -487,8 +484,7 @@ void AudioMixer::run() { foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { - ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(JITTER_BUFFER_SAMPLES, - _sourceUnattenuatedZone, + ((AudioMixerClientData*) node->getLinkedData())->checkBuffersBeforeFrameSend(_sourceUnattenuatedZone, _listenerUnattenuatedZone); } } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2329946e49..3ee571fc46 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -98,10 +98,9 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { return 0; } -void AudioMixerClientData::checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, - AABox* checkSourceZone, AABox* listenerZone) { +void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, AABox* listenerZone) { for (int i = 0; i < _ringBuffers.size(); i++) { - if (_ringBuffers[i]->shouldBeAddedToMix(jitterBufferLengthSamples)) { + if (_ringBuffers[i]->shouldBeAddedToMix()) { // this is a ring buffer that is ready to go // set its flag so we know to push its buffer when all is said and done _ringBuffers[i]->setWillBeAddedToMix(true); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index e52b09e134..3c4ddd3459 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -27,8 +27,7 @@ public: AvatarAudioRingBuffer* getAvatarAudioRingBuffer() const; int parseData(const QByteArray& packet); - void checkBuffersBeforeFrameSend(int jitterBufferLengthSamples, - AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); + void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); private: QList _ringBuffers; diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 47a0ecbea8..6658e446cf 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,8 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - _timeGapHistory.frameReceived(); + _interframeTimeGapHistory.frameReceived(); + updateDesiredJitterBufferNumSamples(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ea15e27ef5..2101fcb9cd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -12,7 +12,6 @@ #include #include #include -#include "SharedUtil.h" #include @@ -20,62 +19,6 @@ #include "AudioRingBuffer.h" -InterframeTimeGapHistory::InterframeTimeGapHistory() - : _lastFrameReceivedTime(0), - _numSamplesInCurrentInterval(0), - _currentIntervalMaxGap(0), - _newestIntervalMaxGapAt(0), - _windowMaxGap(0), - _newWindowMaxGapAvailable(false) -{ - memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); -} - -void InterframeTimeGapHistory::frameReceived() { - quint64 now = usecTimestampNow(); - - // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. - if (_lastFrameReceivedTime != 0) { - quint64 gap = now - _lastFrameReceivedTime; - - // update the current interval max - if (gap > _currentIntervalMaxGap) { - _currentIntervalMaxGap = gap; - } - _numSamplesInCurrentInterval++; - - // if the current interval of samples is now full, record it in our interval maxes - if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { - - // find location to insert this interval's max (increment index cyclically) - _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; - - // record the current interval's max gap as the newest - _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; - - // update the window max gap, which is the max out of all the past intervals' max gaps - _windowMaxGap = 0; - for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { - if (_intervalMaxGaps[i] > _windowMaxGap) { - _windowMaxGap = _intervalMaxGaps[i]; - } - } - _newWindowMaxGapAvailable = true; - - // reset the current interval - _numSamplesInCurrentInterval = 0; - _currentIntervalMaxGap = 0; - } - } - _lastFrameReceivedTime = now; -} - -quint64 InterframeTimeGapHistory::getPastWindowMaxGap() { - _newWindowMaxGapAvailable = false; - return _windowMaxGap; -} - - AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 46bc333e4b..3d2864f373 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -21,31 +21,6 @@ #include "NodeData.h" -// this means that every 500 samples, the max for the past 10*500 samples will be calculated -const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; -const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; - -// class used to track time between incoming frames for the purpose of varying the jitter buffer length -class InterframeTimeGapHistory { -public: - InterframeTimeGapHistory(); - - void frameReceived(); - bool isNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } - quint64 getPastWindowMaxGap(); - -private: - quint64 _lastFrameReceivedTime; - - int _numSamplesInCurrentInterval; - quint64 _currentIntervalMaxGap; - quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; - int _newestIntervalMaxGapAt; - quint64 _windowMaxGap; - bool _newWindowMaxGapAvailable; -}; - - const int SAMPLE_RATE = 24000; const int NETWORK_BUFFER_LENGTH_BYTES_STEREO = 1024; @@ -99,15 +74,14 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); - - InterframeTimeGapHistory& getInterframeTimeGapHistory() { return _timeGapHistory; } + protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - + int _sampleCapacity; int _numFrameSamples; int16_t* _nextOutput; @@ -116,8 +90,6 @@ protected: bool _isStarved; bool _hasStarted; bool _randomAccessMode; /// will this ringbuffer be used for random access? if so, do some special processing - - InterframeTimeGapHistory _timeGapHistory; }; #endif // hifi_AudioRingBuffer_h diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 103d0d8d2e..ffe5876bfd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,8 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - _timeGapHistory.frameReceived(); + _interframeTimeGapHistory.frameReceived(); + updateDesiredJitterBufferNumSamples(); // setup a data stream to read from this packet QDataStream packetStream(packet); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 1cc4147175..b537e5c6d6 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -19,6 +19,63 @@ #include #include "PositionalAudioRingBuffer.h" +#include "SharedUtil.h" + +InterframeTimeGapHistory::InterframeTimeGapHistory() + : _lastFrameReceivedTime(0), + _numSamplesInCurrentInterval(0), + _currentIntervalMaxGap(0), + _newestIntervalMaxGapAt(0), + _windowMaxGap(0), + _newWindowMaxGapAvailable(false) +{ + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); +} + +void InterframeTimeGapHistory::frameReceived() { + quint64 now = usecTimestampNow(); + + // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + if (_lastFrameReceivedTime != 0) { + quint64 gap = now - _lastFrameReceivedTime; + + // update the current interval max + if (gap > _currentIntervalMaxGap) { + _currentIntervalMaxGap = gap; + } + _numSamplesInCurrentInterval++; + + // if the current interval of samples is now full, record it in our interval maxes + if (_numSamplesInCurrentInterval == TIME_GAP_NUM_SAMPLES_IN_INTERVAL) { + + // find location to insert this interval's max (increment index cyclically) + _newestIntervalMaxGapAt = _newestIntervalMaxGapAt == TIME_GAP_NUM_INTERVALS_IN_WINDOW - 1 ? 0 : _newestIntervalMaxGapAt + 1; + + // record the current interval's max gap as the newest + _intervalMaxGaps[_newestIntervalMaxGapAt] = _currentIntervalMaxGap; + + // update the window max gap, which is the max out of all the past intervals' max gaps + _windowMaxGap = 0; + for (int i = 0; i < TIME_GAP_NUM_INTERVALS_IN_WINDOW; i++) { + if (_intervalMaxGaps[i] > _windowMaxGap) { + _windowMaxGap = _intervalMaxGaps[i]; + } + } + _newWindowMaxGapAvailable = true; + + // reset the current interval + _numSamplesInCurrentInterval = 0; + _currentIntervalMaxGap = 0; + } + } + _lastFrameReceivedTime = now; +} + +quint64 InterframeTimeGapHistory::getWindowMaxGap() { + _newWindowMaxGapAvailable = false; + return _windowMaxGap; +} + PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo) : AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), @@ -29,7 +86,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _shouldLoopbackForNode(false), _shouldOutputStarveDebug(true), _isStereo(isStereo), - _listenerUnattenuatedZone(NULL) + _listenerUnattenuatedZone(NULL), + _desiredJitterBufferNumSamples(getNumSamplesPerFrame()) { } @@ -106,14 +164,17 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } } -bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { - if (!isNotStarvedOrHasMinimumSamples(NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL + numJitterBufferSamples)) { +bool PositionalAudioRingBuffer::shouldBeAddedToMix() { + + int samplesPerFrame = getNumSamplesPerFrame(); + + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferNumSamples)) { if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } return false; - } else if (samplesAvailable() < NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + } else if (samplesAvailable() < samplesPerFrame) { _isStarved = true; // reset our _shouldOutputStarveDebug to true so the next is printed @@ -132,3 +193,13 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix(int numJitterBufferSamples) { return false; } + +void PositionalAudioRingBuffer::updateDesiredJitterBufferNumSamples() { + + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + + if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { + int desiredJitterBufferNumFrames = (int)((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME + 0.5f); + _desiredJitterBufferNumSamples = desiredJitterBufferNumFrames * getNumSamplesPerFrame(); + } +} \ No newline at end of file diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 00362c245a..de731c6136 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -18,6 +18,31 @@ #include "AudioRingBuffer.h" +// this means that every 500 samples, the max for the past 10*500 samples will be calculated +const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; +const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; + +// class used to track time between incoming frames for the purpose of varying the jitter buffer length +class InterframeTimeGapHistory { +public: + InterframeTimeGapHistory(); + + void frameReceived(); + bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 getWindowMaxGap(); + +private: + quint64 _lastFrameReceivedTime; + + int _numSamplesInCurrentInterval; + quint64 _currentIntervalMaxGap; + quint64 _intervalMaxGaps[TIME_GAP_NUM_INTERVALS_IN_WINDOW]; + int _newestIntervalMaxGapAt; + quint64 _windowMaxGap; + bool _newWindowMaxGapAvailable; +}; + + class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { @@ -34,7 +59,7 @@ public: void updateNextOutputTrailingLoudness(); float getNextOutputTrailingLoudness() const { return _nextOutputTrailingLoudness; } - bool shouldBeAddedToMix(int numJitterBufferSamples); + bool shouldBeAddedToMix(); bool willBeAddedToMix() const { return _willBeAddedToMix; } void setWillBeAddedToMix(bool willBeAddedToMix) { _willBeAddedToMix = willBeAddedToMix; } @@ -50,10 +75,14 @@ public: AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } + int getNumSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; } + protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); + + void updateDesiredJitterBufferNumSamples(); PositionalAudioRingBuffer::Type _type; glm::vec3 _position; @@ -65,6 +94,9 @@ protected: float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; + + InterframeTimeGapHistory _interframeTimeGapHistory; + int _desiredJitterBufferNumSamples; }; #endif // hifi_PositionalAudioRingBuffer_h From 24f535e02d263502680f0b04ca5628de77023251 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 19 Jun 2014 18:15:17 -0700 Subject: [PATCH 10/38] Working on metavoxel streaming tests. --- libraries/metavoxels/src/MetavoxelData.cpp | 46 +++++ libraries/metavoxels/src/MetavoxelData.h | 11 ++ tests/metavoxels/src/MetavoxelTests.cpp | 193 +++++++++++++++------ 3 files changed, 196 insertions(+), 54 deletions(-) diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index cf645e6f5e..d7528d1959 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -687,6 +687,23 @@ MetavoxelNode* MetavoxelData::createRoot(const AttributePointer& attribute) { return root = new MetavoxelNode(attribute); } +bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod) const { + if (_size != other._size) { + return false; + } + if (_roots.size() != other._roots.size()) { + return false; + } + glm::vec3 minimum = getMinimum(); + for (QHash::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) { + MetavoxelNode* otherNode = other._roots.value(it.key()); + if (!(otherNode && it.value()->deepEquals(it.key(), *otherNode, minimum, _size, lod))) { + return false; + } + } + return true; +} + bool MetavoxelData::operator==(const MetavoxelData& other) const { return _size == other._size && _roots == other._roots; } @@ -1083,6 +1100,31 @@ void MetavoxelNode::clearChildren(const AttributePointer& attribute) { } } +bool MetavoxelNode::deepEquals(const AttributePointer& attribute, const MetavoxelNode& other, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod) const { + if (!attribute->equal(_attributeValue, other._attributeValue)) { + return false; + } + if (!lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) { + return true; + } + bool leaf = isLeaf(), otherLeaf = other.isLeaf(); + if (leaf && otherLeaf) { + return true; + } + if (leaf || otherLeaf) { + return false; + } + float nextSize = size * 0.5f; + for (int i = 0; i < CHILD_COUNT; i++) { + glm::vec3 nextMinimum = getNextMinimum(minimum, nextSize, i); + if (!_children[i]->deepEquals(attribute, *(other._children[i]), nextMinimum, nextSize, lod)) { + return false; + } + } + return true; +} + int MetavoxelVisitor::encodeOrder(int first, int second, int third, int fourth, int fifth, int sixth, int seventh, int eighth) { return first | (second << 3) | (third << 6) | (fourth << 9) | @@ -1111,6 +1153,10 @@ int MetavoxelVisitor::encodeOrder(const glm::vec3& direction) { indexDistances.at(6).index, indexDistances.at(7).index); } +int MetavoxelVisitor::encodeRandomOrder() { + return DEFAULT_ORDER; +} + const int MetavoxelVisitor::DEFAULT_ORDER = encodeOrder(0, 1, 2, 3, 4, 5, 6, 7); const int MetavoxelVisitor::STOP_RECURSION = 0; const int MetavoxelVisitor::SHORT_CIRCUIT = -1; diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 6b262d6892..0df97a7a4c 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -126,6 +126,10 @@ public: MetavoxelNode* getRoot(const AttributePointer& attribute) const { return _roots.value(attribute); } MetavoxelNode* createRoot(const AttributePointer& attribute); + /// Performs a deep comparison between this data and the specified other (as opposed to the == operator, which does a + /// shallow comparison). + bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const; + bool operator==(const MetavoxelData& other) const; bool operator!=(const MetavoxelData& other) const; @@ -214,6 +218,10 @@ public: void clearChildren(const AttributePointer& attribute); + /// Performs a deep comparison between this and the specified other node. + bool deepEquals(const AttributePointer& attribute, const MetavoxelNode& other, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod) const; + private: Q_DISABLE_COPY(MetavoxelNode) @@ -250,6 +258,9 @@ public: /// Encodes a visitation order sequence that visits each child as sorted along the specified direction. static int encodeOrder(const glm::vec3& direction); + /// Returns a random visitation order sequence. + static int encodeRandomOrder(); + /// The default visitation order. static const int DEFAULT_ORDER; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index daee1165f6..d261d9d926 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -32,6 +32,8 @@ static int datagramsSent = 0; static int datagramsReceived = 0; static int bytesSent = 0; static int bytesReceived = 0; +static int maxDatagramsPerPacket = 0; +static int maxBytesPerPacket = 0; static int highPriorityMessagesSent = 0; static int highPriorityMessagesReceived = 0; static int unreliableMessagesSent = 0; @@ -353,6 +355,7 @@ bool MetavoxelTests::run() { qDebug() << "Sent" << streamedBytesSent << "streamed bytes, received" << streamedBytesReceived; qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << datagramsReceived << "with" << bytesReceived << "bytes"; + qDebug() << "Max" << maxDatagramsPerPacket << "datagrams," << maxBytesPerPacket << "bytes per packet"; qDebug() << "Created" << sharedObjectsCreated << "shared objects, destroyed" << sharedObjectsDestroyed; qDebug() << "Performed" << objectMutationsPerformed << "object mutations"; qDebug() << "Created" << scriptObjectsCreated << "script objects, mutated" << scriptMutationsPerformed; @@ -373,7 +376,7 @@ bool MetavoxelTests::run() { qDebug(); // clear the stats - datagramsSent = bytesSent = datagramsReceived = bytesReceived = 0; + datagramsSent = bytesSent = datagramsReceived = bytesReceived = maxDatagramsPerPacket = maxBytesPerPacket = 0; // create client and server endpoints Endpoint client(datagramHeader, Endpoint::METAVOXEL_CLIENT_MODE); @@ -391,6 +394,7 @@ bool MetavoxelTests::run() { qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << datagramsReceived << "with" << bytesReceived << "bytes"; + qDebug() << "Max" << maxDatagramsPerPacket << "datagrams," << maxBytesPerPacket << "bytes per packet"; } qDebug() << "All tests passed!"; @@ -407,6 +411,34 @@ static SharedObjectPointer createRandomSharedObject() { } } +class RandomVisitor : public MetavoxelVisitor { +public: + + int leafCount; + + RandomVisitor(); + virtual int visit(MetavoxelInfo& info); +}; + +RandomVisitor::RandomVisitor() : + MetavoxelVisitor(QVector(), + QVector() << AttributeRegistry::getInstance()->getColorAttribute()), + leafCount(0) { +} + +const float MAXIMUM_LEAF_SIZE = 0.5f; +const float MINIMUM_LEAF_SIZE = 0.25f; + +int RandomVisitor::visit(MetavoxelInfo& info) { + if (info.size > MAXIMUM_LEAF_SIZE || (info.size > MINIMUM_LEAF_SIZE && randomBoolean())) { + return DEFAULT_ORDER; + } + info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randIntInRange(0, 255), + randIntInRange(0, 255), randIntInRange(0, 255)))); + leafCount++; + return STOP_RECURSION; +} + Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) : _mode(mode), _sequencer(new DatagramSequencer(datagramHeader, this)), @@ -434,6 +466,12 @@ Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) : return; } if (mode == METAVOXEL_SERVER_MODE) { + _data.expand(); + _data.expand(); + + RandomVisitor visitor; + _data.guide(visitor); + qDebug() << "Created" << visitor.leafCount << "base leaves"; return; } // create the object that represents out delta-encoded state @@ -552,6 +590,36 @@ static bool messagesEqual(const QVariant& firstMessage, const QVariant& secondMe } } +class MutateVisitor : public MetavoxelVisitor { +public: + + MutateVisitor(); + virtual int visit(MetavoxelInfo& info); + +private: + + bool _finished; +}; + +MutateVisitor::MutateVisitor() : + MetavoxelVisitor(QVector(), + QVector() << AttributeRegistry::getInstance()->getColorAttribute()), + _finished(false) { +} + +int MutateVisitor::visit(MetavoxelInfo& info) { + if (_finished) { + return STOP_RECURSION; + } + if (info.size > MAXIMUM_LEAF_SIZE || (info.size > MINIMUM_LEAF_SIZE && randomBoolean())) { + return encodeRandomOrder(); + } + info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randIntInRange(0, 255), + randIntInRange(0, 255), randIntInRange(0, 255)))); + _finished = true; + return STOP_RECURSION; +} + bool Endpoint::simulate(int iterationNumber) { // update/send our delayed datagrams for (QList >::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) { @@ -565,6 +633,8 @@ bool Endpoint::simulate(int iterationNumber) { } } + int oldDatagramsSent = datagramsSent; + int oldBytesSent = bytesSent; if (_mode == METAVOXEL_CLIENT_MODE) { Bitstream& out = _sequencer->startPacket(); @@ -575,69 +645,74 @@ bool Endpoint::simulate(int iterationNumber) { // record the send SendRecord record = { _sequencer->getOutgoingPacketNumber(), SharedObjectPointer(), MetavoxelData(), _lod }; _sendRecords.append(record); - return false; - } - if (_mode == METAVOXEL_SERVER_MODE) { - // wait until we have a valid lod + + } else if (_mode == METAVOXEL_SERVER_MODE) { + // make a random change + MutateVisitor visitor; + _data.guide(visitor); + + // wait until we have a valid lod before sending if (!_lod.isValid()) { return false; } Bitstream& out = _sequencer->startPacket(); out << QVariant::fromValue(MetavoxelDeltaMessage()); _data.writeDelta(_sendRecords.first().data, _sendRecords.first().lod, out, _lod); - _sequencer->endPacket(); // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber(), SharedObjectPointer(), _data, _lod }; + SendRecord record = { _sequencer->getOutgoingPacketNumber() + 1, SharedObjectPointer(), _data, _lod }; _sendRecords.append(record); - return false; - } - - // enqueue some number of high priority messages - const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; - const float MAX_HIGH_PRIORITY_MESSAGES = 2.0f; - _highPriorityMessagesToSend += randFloatInRange(MIN_HIGH_PRIORITY_MESSAGES, MAX_HIGH_PRIORITY_MESSAGES); - while (_highPriorityMessagesToSend >= 1.0f) { - QVariant message = createRandomMessage(); - _highPriorityMessagesSent.append(message); - _sequencer->sendHighPriorityMessage(message); - highPriorityMessagesSent++; - _highPriorityMessagesToSend -= 1.0f; - } - - // and some number of reliable messages - const float MIN_RELIABLE_MESSAGES = 0.0f; - const float MAX_RELIABLE_MESSAGES = 4.0f; - _reliableMessagesToSend += randFloatInRange(MIN_RELIABLE_MESSAGES, MAX_RELIABLE_MESSAGES); - while (_reliableMessagesToSend >= 1.0f) { - QVariant message = createRandomMessage(); - _reliableMessagesSent.append(message); - _sequencer->getReliableOutputChannel()->sendMessage(message); - reliableMessagesSent++; - _reliableMessagesToSend -= 1.0f; - } - - // tweak the local state - _localState = mutate(_localState); - - // send a packet - try { - Bitstream& out = _sequencer->startPacket(); - SequencedTestMessage message = { iterationNumber, createRandomMessage(), _localState }; - _unreliableMessagesSent.append(message); - unreliableMessagesSent++; - out << message; + _sequencer->endPacket(); + + } else { + // enqueue some number of high priority messages + const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; + const float MAX_HIGH_PRIORITY_MESSAGES = 2.0f; + _highPriorityMessagesToSend += randFloatInRange(MIN_HIGH_PRIORITY_MESSAGES, MAX_HIGH_PRIORITY_MESSAGES); + while (_highPriorityMessagesToSend >= 1.0f) { + QVariant message = createRandomMessage(); + _highPriorityMessagesSent.append(message); + _sequencer->sendHighPriorityMessage(message); + highPriorityMessagesSent++; + _highPriorityMessagesToSend -= 1.0f; + } + + // and some number of reliable messages + const float MIN_RELIABLE_MESSAGES = 0.0f; + const float MAX_RELIABLE_MESSAGES = 4.0f; + _reliableMessagesToSend += randFloatInRange(MIN_RELIABLE_MESSAGES, MAX_RELIABLE_MESSAGES); + while (_reliableMessagesToSend >= 1.0f) { + QVariant message = createRandomMessage(); + _reliableMessagesSent.append(message); + _sequencer->getReliableOutputChannel()->sendMessage(message); + reliableMessagesSent++; + _reliableMessagesToSend -= 1.0f; + } + + // tweak the local state + _localState = mutate(_localState); + + // send a packet + try { + Bitstream& out = _sequencer->startPacket(); + SequencedTestMessage message = { iterationNumber, createRandomMessage(), _localState }; + _unreliableMessagesSent.append(message); + unreliableMessagesSent++; + out << message; + _sequencer->endPacket(); + + } catch (const QString& message) { + qDebug() << message; + return true; + } - } catch (const QString& message) { - qDebug() << message; - return true; + // record the send + SendRecord record = { _sequencer->getOutgoingPacketNumber(), _localState }; + _sendRecords.append(record); } - - // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber(), _localState }; - _sendRecords.append(record); - + maxDatagramsPerPacket = qMax(maxDatagramsPerPacket, datagramsSent - oldDatagramsSent); + maxBytesPerPacket = qMax(maxBytesPerPacket, bytesSent - oldBytesSent); return false; } @@ -692,9 +767,19 @@ void Endpoint::readMessage(Bitstream& in) { in >> message; handleMessage(message, in); + // deep-compare data to sent version + int packetNumber = _sequencer->getIncomingPacketNumber(); + foreach (const SendRecord& sendRecord, _other->_sendRecords) { + if (sendRecord.packetNumber == packetNumber) { + if (!sendRecord.data.deepEquals(_data, _sendRecords.first().lod)) { + throw QString("Sent/received metavoxel data mismatch."); + } + break; + } + } + // record the receipt - ReceiveRecord record = { _sequencer->getIncomingPacketNumber(), SharedObjectPointer(), - _data, _sendRecords.first().lod }; + ReceiveRecord record = { packetNumber, SharedObjectPointer(), _data, _sendRecords.first().lod }; _receiveRecords.append(record); return; } From 0c87f0aabc677e27188c887d229390fdeac65ab5 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 19 Jun 2014 19:45:43 -0700 Subject: [PATCH 11/38] Update chat window behavior Lower opacity when inactive Move focus to main window when pressing escape, instead of hiding --- interface/src/Menu.cpp | 1 + interface/src/ui/ChatWindow.cpp | 14 +++++++++++++- interface/src/ui/ChatWindow.h | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 5c8c2e97aa..334a2e441b 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -1293,6 +1293,7 @@ void Menu::showChat() { if (_chatWindow->isHidden()) { _chatWindow->show(); } + _chatWindow->activateWindow(); } else { Application::getInstance()->getTrayIcon()->showMessage("Interface", "You need to login to be able to chat with others on this domain."); } diff --git a/interface/src/ui/ChatWindow.cpp b/interface/src/ui/ChatWindow.cpp index fde77334f4..b101b50b2f 100644 --- a/interface/src/ui/ChatWindow.cpp +++ b/interface/src/ui/ChatWindow.cpp @@ -30,6 +30,9 @@ const int NUM_MESSAGES_TO_TIME_STAMP = 20; +const float OPACITY_ACTIVE = 1.0; +const float OPACITY_INACTIVE = 0.6; + const QRegularExpression regexLinks("((?:(?:ftp)|(?:https?)|(?:hifi))://\\S+)"); const QRegularExpression regexHifiLinks("([#@]\\S+)"); const QString mentionSoundsPath("/mention-sounds/"); @@ -108,7 +111,7 @@ ChatWindow::~ChatWindow() { void ChatWindow::keyPressEvent(QKeyEvent* event) { if (event->key() == Qt::Key_Escape) { - hide(); + Application::getInstance()->getWindow()->activateWindow(); } else { FramelessDialog::keyPressEvent(event); } @@ -383,3 +386,12 @@ void ChatWindow::scrollToBottom() { QScrollBar* verticalScrollBar = ui->messagesScrollArea->verticalScrollBar(); verticalScrollBar->setValue(verticalScrollBar->maximum()); } + +bool ChatWindow::event(QEvent* event) { + if (event->type() == QEvent::WindowActivate) { + setWindowOpacity(OPACITY_ACTIVE); + } else if (event->type() == QEvent::WindowDeactivate) { + setWindowOpacity(OPACITY_INACTIVE); + } + return FramelessDialog::event(event); +} diff --git a/interface/src/ui/ChatWindow.h b/interface/src/ui/ChatWindow.h index 1e0f533e9e..652dcb5b08 100644 --- a/interface/src/ui/ChatWindow.h +++ b/interface/src/ui/ChatWindow.h @@ -50,6 +50,7 @@ protected: virtual void keyPressEvent(QKeyEvent *event); virtual void showEvent(QShowEvent* event); + virtual bool event(QEvent* event); private: #ifdef HAVE_QXMPP From 633746ab168fa5fc7cbb30895570f20066984696 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 20 Jun 2014 13:30:04 -0700 Subject: [PATCH 12/38] Different fullscreen shortcuts for Mac and Windows Revert to the original Mac OSX key combination and provide alternative for Windows. --- interface/src/Menu.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 9df3e25bf8..22a832278f 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -249,12 +249,21 @@ Menu::Menu() : QMenu* viewMenu = addMenu("View"); +#ifdef Q_OS_MAC + addCheckableActionToQMenuAndActionHash(viewMenu, + MenuOption::Fullscreen, + Qt::CTRL | Qt::META | Qt::Key_F, + false, + appInstance, + SLOT(setFullscreen(bool))); +#else addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::Fullscreen, Qt::CTRL | Qt::Key_F, false, appInstance, SLOT(setFullscreen(bool))); +#endif addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::FirstPerson, Qt::Key_P, true, appInstance,SLOT(cameraMenuChanged())); addCheckableActionToQMenuAndActionHash(viewMenu, MenuOption::Mirror, Qt::SHIFT | Qt::Key_H, true); From ce778f47b1c6fc29999a5f9a865ba1c1514f866f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Jun 2014 14:08:39 -0700 Subject: [PATCH 13/38] More work on stream testing, fixed bug with differently-ordered edits. --- libraries/metavoxels/src/MetavoxelData.cpp | 21 +++++++++++++++++---- tests/metavoxels/src/MetavoxelTests.cpp | 8 ++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index d7528d1959..6d1031d3cf 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -1153,8 +1153,23 @@ int MetavoxelVisitor::encodeOrder(const glm::vec3& direction) { indexDistances.at(6).index, indexDistances.at(7).index); } +const int ORDER_ELEMENT_BITS = 3; +const int ORDER_ELEMENT_MASK = (1 << ORDER_ELEMENT_BITS) - 1; + int MetavoxelVisitor::encodeRandomOrder() { - return DEFAULT_ORDER; + // see http://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_.22inside-out.22_algorithm + int order; + int randomValues = rand(); + for (int i = 0, iShift = 0; i < MetavoxelNode::CHILD_COUNT; i++, iShift += ORDER_ELEMENT_BITS) { + int j = (randomValues >> iShift) % (i + 1); + int jShift = j * ORDER_ELEMENT_BITS; + if (j != i) { + int jValue = (order >> jShift) & ORDER_ELEMENT_MASK; + order = (order & ~(ORDER_ELEMENT_MASK << iShift)) | (jValue << iShift); + } + order = (order & ~(ORDER_ELEMENT_MASK << jShift)) | (i << jShift); + } + return order; } const int MetavoxelVisitor::DEFAULT_ORDER = encodeOrder(0, 1, 2, 3, 4, 5, 6, 7); @@ -1350,8 +1365,6 @@ bool DefaultMetavoxelGuide::guide(MetavoxelVisitation& visitation) { QVector(visitation.outputNodes.size()) } }; for (int i = 0; i < MetavoxelNode::CHILD_COUNT; i++) { // the encoded order tells us the child indices for each iteration - const int ORDER_ELEMENT_BITS = 3; - const int ORDER_ELEMENT_MASK = (1 << ORDER_ELEMENT_BITS) - 1; int index = encodedOrder & ORDER_ELEMENT_MASK; encodedOrder >>= ORDER_ELEMENT_BITS; for (int j = 0; j < visitation.inputNodes.size(); j++) { @@ -1392,7 +1405,7 @@ bool DefaultMetavoxelGuide::guide(MetavoxelVisitation& visitation) { } } MetavoxelNode* node = visitation.outputNodes.at(j); - MetavoxelNode* child = node->getChild(i); + MetavoxelNode* child = node->getChild(index); if (child) { child->decrementReferenceCount(value.getAttribute()); } else { diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index d261d9d926..6aef40eab1 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -598,17 +598,17 @@ public: private: - bool _finished; + int _mutationsRemaining; }; MutateVisitor::MutateVisitor() : MetavoxelVisitor(QVector(), QVector() << AttributeRegistry::getInstance()->getColorAttribute()), - _finished(false) { + _mutationsRemaining(randIntInRange(2, 4)) { } int MutateVisitor::visit(MetavoxelInfo& info) { - if (_finished) { + if (_mutationsRemaining <= 0) { return STOP_RECURSION; } if (info.size > MAXIMUM_LEAF_SIZE || (info.size > MINIMUM_LEAF_SIZE && randomBoolean())) { @@ -616,7 +616,7 @@ int MutateVisitor::visit(MetavoxelInfo& info) { } info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randIntInRange(0, 255), randIntInRange(0, 255), randIntInRange(0, 255)))); - _finished = true; + _mutationsRemaining--; return STOP_RECURSION; } From a5457eb86b7e6d766b46d4106dfa098ba3a49a4d Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 14:43:18 -0700 Subject: [PATCH 14/38] fixed bug in Audio.cpp where numSilentSamples was written to wrong place in packet added more debug stuff, other minor changes and fixes --- .../src/audio/AudioMixerClientData.cpp | 5 +- .../src/audio/AvatarAudioRingBuffer.cpp | 2 +- interface/src/Audio.cpp | 12 ++-- libraries/audio/src/AudioRingBuffer.cpp | 1 + .../audio/src/InjectedAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 63 +++++++++++++------ .../audio/src/PositionalAudioRingBuffer.h | 10 ++- 7 files changed, 63 insertions(+), 32 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 3ee571fc46..0c41cc70f9 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -119,15 +119,14 @@ void AudioMixerClientData::checkBuffersBeforeFrameSend(AABox* checkSourceZone, A } void AudioMixerClientData::pushBuffersAfterFrameSend() { + QList::iterator i = _ringBuffers.begin(); while (i != _ringBuffers.end()) { // this was a used buffer, push the output pointer forwards PositionalAudioRingBuffer* audioBuffer = *i; if (audioBuffer->willBeAddedToMix()) { - audioBuffer->shiftReadPosition(audioBuffer->isStereo() - ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - + audioBuffer->shiftReadPosition(audioBuffer->getSamplesPerFrame()); audioBuffer->setWillBeAddedToMix(false); } else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector && audioBuffer->hasStarted() && audioBuffer->isStarved()) { diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 6658e446cf..0df50f99db 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -20,7 +20,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { _interframeTimeGapHistory.frameReceived(); - updateDesiredJitterBufferNumSamples(); + updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); return PositionalAudioRingBuffer::parseData(packet); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 271bcd5279..47e55ddc90 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -461,8 +461,8 @@ void Audio::handleAudioInput() { int16_t* inputAudioSamples = new int16_t[inputSamplesRequired]; _inputRingBuffer.readSamples(inputAudioSamples, inputSamplesRequired); - int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; - int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + const int numNetworkBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + const int numNetworkSamples = _isStereoInput ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // zero out the monoAudioSamples array and the locally injected audio memset(networkAudioSamples, 0, numNetworkBytes); @@ -622,6 +622,7 @@ void Audio::handleAudioInput() { SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { + MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); @@ -634,12 +635,11 @@ void Audio::handleAudioInput() { packetType = PacketTypeSilentAudioFrame; // we need to indicate how many silent samples this is to the audio mixer - audioDataPacket[0] = _isStereoInput - ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO - : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + networkAudioSamples[0] = numNetworkSamples; numAudioBytes = sizeof(int16_t); } else { - numAudioBytes = _isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; + numAudioBytes = numNetworkBytes; + //_isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { packetType = PacketTypeMicrophoneAudioWithEcho; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 2101fcb9cd..7b1e68ffbd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -158,6 +158,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); + printf("\n mixed. %d samples remaining\n", samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index ffe5876bfd..80bcda5acb 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -32,7 +32,7 @@ const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { _interframeTimeGapHistory.frameReceived(); - updateDesiredJitterBufferNumSamples(); + updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet QDataStream packetStream(packet); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index b537e5c6d6..813323ecb1 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,6 +33,10 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { + + static QQueue gaps; + static quint64 gapsSum = 0; + quint64 now = usecTimestampNow(); // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. @@ -87,9 +91,9 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _shouldOutputStarveDebug(true), _isStereo(isStereo), _listenerUnattenuatedZone(NULL), - _desiredJitterBufferNumSamples(getNumSamplesPerFrame()) + _desiredJitterBufferFrames(1), + _currentJitterBufferFrames(0) { - } int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { @@ -114,10 +118,15 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { if (numSilentSamples > 0) { addSilentFrame(numSilentSamples); } + printf("\nparsed silent packet of %d samples\n", numSilentSamples); } else { // there is audio data to read - readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); + int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); + readBytes += dataBytes; + + printf("\nparsed packet of %d data bytes\n", dataBytes); } + printf("%d samples available\n", samplesAvailable()); return readBytes; } @@ -166,40 +175,58 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int samplesPerFrame = getNumSamplesPerFrame(); + int samplesPerFrame = getSamplesPerFrame(); + int currentJitterBufferSamples = 3 * samplesPerFrame; //_currentJitterBufferFrames * samplesPerFrame; + +//printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + currentJitterBufferSamples)) { - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferNumSamples)) { +//printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); +//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + // if the buffer was starved and hasn't filled back up all the way, don't mix yet if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } - - return false; + + return false; + } else if (samplesAvailable() < samplesPerFrame) { + +//printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); +//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + // if the buffer doesn't have a full frame of samples for mixing, it is starved _isStarved = true; // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; + + // if buffer was starved, we've effectively increased the jitter buffer by one frame + // by "holding back" this ring buffer's contents until the next client frame is prepared. + _currentJitterBufferFrames++; +//printf("jbuffer size increased: new size: %d\n", _currentJitterBufferFrames); return false; - } else { - // good buffer, add this to the mix - _isStarved = false; - // since we've read data from ring buffer at least once - we've started - _hasStarted = true; - - return true; } +//printf("WILL MIX\n"); - return false; + // good buffer, add this to the mix + _isStarved = false; + + // since we've read data from ring buffer at least once - we've started + _hasStarted = true; + + return true; } -void PositionalAudioRingBuffer::updateDesiredJitterBufferNumSamples() { +void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { - int desiredJitterBufferNumFrames = (int)((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME + 0.5f); - _desiredJitterBufferNumSamples = desiredJitterBufferNumFrames * getNumSamplesPerFrame(); + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME); + if (_desiredJitterBufferFrames < 1) { + _desiredJitterBufferFrames = 1; + } } } \ No newline at end of file diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index de731c6136..87a53038a3 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -29,6 +29,7 @@ public: void frameReceived(); bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } + quint64 peekWindowMaxGap() const { return _windowMaxGap; } quint64 getWindowMaxGap(); private: @@ -75,14 +76,14 @@ public: AABox* getListenerUnattenuatedZone() const { return _listenerUnattenuatedZone; } void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } - int getNumSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; } + int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); PositionalAudioRingBuffer& operator= (const PositionalAudioRingBuffer&); - void updateDesiredJitterBufferNumSamples(); + void updateDesiredJitterBufferFrames(); PositionalAudioRingBuffer::Type _type; glm::vec3 _position; @@ -96,7 +97,10 @@ protected: AABox* _listenerUnattenuatedZone; InterframeTimeGapHistory _interframeTimeGapHistory; - int _desiredJitterBufferNumSamples; + int _desiredJitterBufferFrames; + int _currentJitterBufferFrames; + +quint64 _lastMixTime; }; #endif // hifi_PositionalAudioRingBuffer_h From 0e6abe33597dd0fc39a63d6f044d1374fdb0795d Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 20 Jun 2014 16:31:22 -0700 Subject: [PATCH 15/38] Update inactive chat window opacity to 0.8 --- interface/src/ui/ChatWindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/ui/ChatWindow.cpp b/interface/src/ui/ChatWindow.cpp index b101b50b2f..a7d2c212d2 100644 --- a/interface/src/ui/ChatWindow.cpp +++ b/interface/src/ui/ChatWindow.cpp @@ -31,7 +31,7 @@ const int NUM_MESSAGES_TO_TIME_STAMP = 20; const float OPACITY_ACTIVE = 1.0; -const float OPACITY_INACTIVE = 0.6; +const float OPACITY_INACTIVE = 0.8; const QRegularExpression regexLinks("((?:(?:ftp)|(?:https?)|(?:hifi))://\\S+)"); const QRegularExpression regexHifiLinks("([#@]\\S+)"); From cb6681f6f88e21196e97d3f6c5dd65a046cdd71d Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:13:42 -0700 Subject: [PATCH 16/38] completed code for dynamicly resizing jitter buffer --- libraries/audio/src/AudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 81 ++++++++++++++----- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 7b1e68ffbd..ae100dcdf1 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -158,7 +158,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); - printf("\n mixed. %d samples remaining\n", samplesAvailable()); +//printf("\nmixed. %d samples remaining\n", samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 813323ecb1..04559fc910 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,8 +33,7 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { - - static QQueue gaps; + static quint64 gapsSum = 0; quint64 now = usecTimestampNow(); @@ -46,6 +45,14 @@ void InterframeTimeGapHistory::frameReceived() { // update the current interval max if (gap > _currentIntervalMaxGap) { _currentIntervalMaxGap = gap; + + // keep the window max gap at least as large as the current interval max + // this allows the window max gap to respond immediately to a sudden spike in gap times + // also, this prevents the window max gap from staying at 0 until the first interval of samples filled up + if (_currentIntervalMaxGap > _windowMaxGap) { + _windowMaxGap = _currentIntervalMaxGap; + _newWindowMaxGapAvailable = true; + } } _numSamplesInCurrentInterval++; @@ -116,17 +123,35 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); if (numSilentSamples > 0) { - addSilentFrame(numSilentSamples); + + if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { + // our current jitter buffer size exceeds its desired value, so ignore some silent + // frames to get that size as close to desired as possible + int samplesPerFrame = getSamplesPerFrame(); + int numSilentFrames = numSilentSamples / samplesPerFrame; + int numFramesToDropDesired = _currentJitterBufferFrames - _desiredJitterBufferFrames; + + if (numSilentFrames > numFramesToDropDesired) { + // we have more than enough frames to drop to get the jitter buffer to its desired length + int numSilentFramesToAdd = numSilentFrames - numFramesToDropDesired; + addSilentFrame(numSilentFramesToAdd * samplesPerFrame); + _currentJitterBufferFrames = _desiredJitterBufferFrames; + + } else { + // we need to drop all frames to get the jitter buffer close as possible to its desired length + _currentJitterBufferFrames -= numSilentFrames; + printf("dropped silent packet! jbuffer size is now %d ###############", _currentJitterBufferFrames); + } + } else { + addSilentFrame(numSilentSamples); + } } - printf("\nparsed silent packet of %d samples\n", numSilentSamples); } else { // there is audio data to read int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); readBytes += dataBytes; - - printf("\nparsed packet of %d data bytes\n", dataBytes); } - printf("%d samples available\n", samplesAvailable()); +//printf("parsed packet. %d samples available\n", samplesAvailable()); return readBytes; } @@ -174,16 +199,18 @@ void PositionalAudioRingBuffer::updateNextOutputTrailingLoudness() { } bool PositionalAudioRingBuffer::shouldBeAddedToMix() { - int samplesPerFrame = getSamplesPerFrame(); - int currentJitterBufferSamples = 3 * samplesPerFrame; //_currentJitterBufferFrames * samplesPerFrame; + int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; //printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + currentJitterBufferSamples)) { + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferFrames)) { + +printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); +printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + + // if the buffer was starved, allow it to accrue at least the desired number of + // jitter buffer frames before we start taking frames from it for mixing -//printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); -//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer was starved and hasn't filled back up all the way, don't mix yet if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } @@ -192,26 +219,34 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } else if (samplesAvailable() < samplesPerFrame) { -//printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); -//printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer doesn't have a full frame of samples for mixing, it is starved +printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); +printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); + + // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; + + // set to 0 to indicate the jitter buffer is starved + _currentJitterBufferFrames = 0; // reset our _shouldOutputStarveDebug to true so the next is printed _shouldOutputStarveDebug = true; - // if buffer was starved, we've effectively increased the jitter buffer by one frame - // by "holding back" this ring buffer's contents until the next client frame is prepared. - _currentJitterBufferFrames++; -//printf("jbuffer size increased: new size: %d\n", _currentJitterBufferFrames); - return false; } //printf("WILL MIX\n"); // good buffer, add this to the mix - _isStarved = false; + + if (_isStarved) { + + // if this buffer was just finished replenishing after being starved, the number of frames in it now + // is the length of the jitter buffer + _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; + printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); + + _isStarved = false; + } // since we've read data from ring buffer at least once - we've started _hasStarted = true; @@ -228,5 +263,7 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } + + printf(" desired jbuffer size is now %d --------------------------------\n", _desiredJitterBufferFrames); } } \ No newline at end of file From 124fba35db2e4d2330234d35e6f144c6469cfb37 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:39:05 -0700 Subject: [PATCH 17/38] fixed minor comment typo --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 04559fc910..adadf5af91 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -240,7 +240,7 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp if (_isStarved) { - // if this buffer was just finished replenishing after being starved, the number of frames in it now + // if this buffer has just finished replenishing after being starved, the number of frames in it now // is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); From 6b72b161d3f7ac33838369f51e3a9756bffe860b Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 20 Jun 2014 17:52:46 -0700 Subject: [PATCH 18/38] fixed typo with frames/samples confusion. --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index adadf5af91..93f68b4dff 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -140,7 +140,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } else { // we need to drop all frames to get the jitter buffer close as possible to its desired length _currentJitterBufferFrames -= numSilentFrames; - printf("dropped silent packet! jbuffer size is now %d ###############", _currentJitterBufferFrames); + printf("dropped silent packet! jbuffer size is now %d ###############\n", _currentJitterBufferFrames); } } else { addSilentFrame(numSilentSamples); @@ -203,7 +203,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; //printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + _desiredJitterBufferFrames)) { + if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); @@ -241,8 +241,8 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp if (_isStarved) { // if this buffer has just finished replenishing after being starved, the number of frames in it now - // is the length of the jitter buffer - _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame; + // minus one (since a frame will be read immediately after this) is the length of the jitter buffer + _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); _isStarved = false; From de0c45a919b96b55a670069c469469e813d29fe1 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Jun 2014 18:09:51 -0700 Subject: [PATCH 19/38] Spanner mutation in test. --- .../metavoxels/src/AttributeRegistry.cpp | 49 +++++++++++++++++++ libraries/metavoxels/src/AttributeRegistry.h | 13 +++++ libraries/metavoxels/src/Bitstream.cpp | 24 +++++++++ libraries/metavoxels/src/Bitstream.h | 6 +++ libraries/metavoxels/src/MetavoxelData.cpp | 17 ++++++- libraries/metavoxels/src/MetavoxelData.h | 4 ++ libraries/metavoxels/src/SharedObject.cpp | 14 ++++-- libraries/shared/src/SharedUtil.h | 2 +- tests/metavoxels/src/MetavoxelTests.cpp | 42 +++++++++++++--- tests/metavoxels/src/MetavoxelTests.h | 2 + 10 files changed, 161 insertions(+), 12 deletions(-) diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp index e7a7f41850..33ce298859 100644 --- a/libraries/metavoxels/src/AttributeRegistry.cpp +++ b/libraries/metavoxels/src/AttributeRegistry.cpp @@ -211,6 +211,11 @@ void Attribute::writeMetavoxelSubdivision(const MetavoxelNode& root, MetavoxelSt root.writeSubdivision(state); } +bool Attribute::metavoxelRootsEqual(const MetavoxelNode& firstRoot, const MetavoxelNode& secondRoot, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod) { + return firstRoot.deepEquals(this, secondRoot, minimum, size, lod); +} + FloatAttribute::FloatAttribute(const QString& name, float defaultValue) : SimpleInlineAttribute(name, defaultValue) { } @@ -449,6 +454,12 @@ void SharedObjectAttribute::write(Bitstream& out, void* value, bool isLeaf) cons } } +bool SharedObjectAttribute::deepEqual(void* first, void* second) const { + SharedObjectPointer firstObject = decodeInline(first); + SharedObjectPointer secondObject = decodeInline(second); + return firstObject ? firstObject->equals(secondObject) : !secondObject; +} + bool SharedObjectAttribute::merge(void*& parent, void* children[], bool postRead) const { SharedObjectPointer firstChild = decodeInline(children[0]); for (int i = 1; i < MERGE_COUNT; i++) { @@ -489,6 +500,35 @@ MetavoxelNode* SharedObjectSetAttribute::createMetavoxelNode( return new MetavoxelNode(value, original); } +static bool setsEqual(const SharedObjectSet& firstSet, const SharedObjectSet& secondSet) { + if (firstSet.size() != secondSet.size()) { + return false; + } + // some hackiness here: we assume that the local ids of the first set correspond to the remote ids of the second, + // so that this will work with the tests + foreach (const SharedObjectPointer& firstObject, firstSet) { + int id = firstObject->getID(); + bool found = false; + foreach (const SharedObjectPointer& secondObject, secondSet) { + if (secondObject->getRemoteID() == id) { + if (!firstObject->equals(secondObject)) { + return false; + } + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; +} + +bool SharedObjectSetAttribute::deepEqual(void* first, void* second) const { + return setsEqual(decodeInline(first), decodeInline(second)); +} + bool SharedObjectSetAttribute::merge(void*& parent, void* children[], bool postRead) const { for (int i = 0; i < MERGE_COUNT; i++) { if (!decodeInline(children[i]).isEmpty()) { @@ -563,3 +603,12 @@ void SpannerSetAttribute::writeMetavoxelSubdivision(const MetavoxelNode& root, M state.stream << SharedObjectPointer(); } +bool SpannerSetAttribute::metavoxelRootsEqual(const MetavoxelNode& firstRoot, const MetavoxelNode& secondRoot, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod) { + + SharedObjectSet firstSet; + firstRoot.getSpanners(this, minimum, size, lod, firstSet); + SharedObjectSet secondSet; + secondRoot.getSpanners(this, minimum, size, lod, secondSet); + return setsEqual(firstSet, secondSet); +} diff --git a/libraries/metavoxels/src/AttributeRegistry.h b/libraries/metavoxels/src/AttributeRegistry.h index 00d974b8b6..7dc2e110b8 100644 --- a/libraries/metavoxels/src/AttributeRegistry.h +++ b/libraries/metavoxels/src/AttributeRegistry.h @@ -27,6 +27,7 @@ class QScriptValue; class Attribute; class MetavoxelData; +class MetavoxelLOD; class MetavoxelNode; class MetavoxelStreamState; @@ -213,6 +214,11 @@ public: virtual bool equal(void* first, void* second) const = 0; + virtual bool deepEqual(void* first, void* second) const { return equal(first, second); } + + virtual bool metavoxelRootsEqual(const MetavoxelNode& firstRoot, const MetavoxelNode& secondRoot, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod); + /// Merges the value of a parent and its children. /// \param postRead whether or not the merge is happening after a read /// \return whether or not the children and parent values are all equal @@ -406,6 +412,8 @@ public: virtual void read(Bitstream& in, void*& value, bool isLeaf) const; virtual void write(Bitstream& out, void* value, bool isLeaf) const; + virtual bool deepEqual(void* first, void* second) const; + virtual bool merge(void*& parent, void* children[], bool postRead = false) const; virtual void* createFromVariant(const QVariant& value) const; @@ -434,6 +442,8 @@ public: virtual MetavoxelNode* createMetavoxelNode(const AttributeValue& value, const MetavoxelNode* original) const; + virtual bool deepEqual(void* first, void* second) const; + virtual bool merge(void*& parent, void* children[], bool postRead = false) const; virtual AttributeValue inherit(const AttributeValue& parentValue) const; @@ -462,6 +472,9 @@ public: virtual void readMetavoxelSubdivision(MetavoxelData& data, MetavoxelStreamState& state); virtual void writeMetavoxelSubdivision(const MetavoxelNode& root, MetavoxelStreamState& state); + + virtual bool metavoxelRootsEqual(const MetavoxelNode& firstRoot, const MetavoxelNode& secondRoot, + const glm::vec3& minimum, float size, const MetavoxelLOD& lod); }; #endif // hifi_AttributeRegistry_h diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 44342abe33..0ef86e85c0 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -110,6 +110,10 @@ const TypeStreamer* Bitstream::getTypeStreamer(int type) { return getTypeStreamers().value(type); } +const ObjectStreamer* Bitstream::getObjectStreamer(const QMetaObject* metaObject) { + return getObjectStreamers().value(metaObject); +} + const QMetaObject* Bitstream::getMetaObject(const QByteArray& className) { return getMetaObjects().value(className); } @@ -2316,6 +2320,15 @@ QObject* MappedObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject return object; } +bool MappedObjectStreamer::equal(const QObject* first, const QObject* second) const { + foreach (const StreamerPropertyPair& property, _properties) { + if (!property.first->equal(property.second.read(first), property.second.read(second))) { + return false; + } + } + return true; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -2433,6 +2446,17 @@ QObject* GenericObjectStreamer::putJSONData(JSONReader& reader, const QJsonObjec return object; } +bool GenericObjectStreamer::equal(const QObject* first, const QObject* second) const { + const QVariantList& firstValues = static_cast(first)->getValues(); + const QVariantList& secondValues = static_cast(second)->getValues(); + for (int i = 0; i < _properties.size(); i++) { + if (!_properties.at(i).first->equal(firstValues.at(i), secondValues.at(i))) { + return false; + } + } + return true; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index b116bd4b49..e32f93dbe2 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -278,6 +278,9 @@ public: /// Returns the streamer registered for the supplied type, if any. static const TypeStreamer* getTypeStreamer(int type); + /// Returns the streamer registered for the supplied object, if any. + static const ObjectStreamer* getObjectStreamer(const QMetaObject* metaObject); + /// Returns the meta-object registered under the supplied class name, if any. static const QMetaObject* getMetaObject(const QByteArray& className); @@ -1022,6 +1025,7 @@ public: virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const = 0; + virtual bool equal(const QObject* first, const QObject* second) const = 0; virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -1047,6 +1051,7 @@ public: virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; + virtual bool equal(const QObject* first, const QObject* second) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -1070,6 +1075,7 @@ public: virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; + virtual bool equal(const QObject* first, const QObject* second) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 6d1031d3cf..7caa336cd2 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -697,7 +697,7 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l glm::vec3 minimum = getMinimum(); for (QHash::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) { MetavoxelNode* otherNode = other._roots.value(it.key()); - if (!(otherNode && it.value()->deepEquals(it.key(), *otherNode, minimum, _size, lod))) { + if (!(otherNode && it.key()->metavoxelRootsEqual(*it.value(), *otherNode, minimum, _size, lod))) { return false; } } @@ -1102,7 +1102,7 @@ void MetavoxelNode::clearChildren(const AttributePointer& attribute) { bool MetavoxelNode::deepEquals(const AttributePointer& attribute, const MetavoxelNode& other, const glm::vec3& minimum, float size, const MetavoxelLOD& lod) const { - if (!attribute->equal(_attributeValue, other._attributeValue)) { + if (!attribute->deepEqual(_attributeValue, other._attributeValue)) { return false; } if (!lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) { @@ -1125,6 +1125,19 @@ bool MetavoxelNode::deepEquals(const AttributePointer& attribute, const Metavoxe return true; } +void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::vec3& minimum, + float size, const MetavoxelLOD& lod, SharedObjectSet& results) const { + results.unite(decodeInline(_attributeValue)); + if (isLeaf() || !lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) { + return; + } + float nextSize = size * 0.5f; + for (int i = 0; i < CHILD_COUNT; i++) { + glm::vec3 nextMinimum = getNextMinimum(minimum, nextSize, i); + _children[i]->getSpanners(attribute, nextMinimum, nextSize, lod, results); + } +} + int MetavoxelVisitor::encodeOrder(int first, int second, int third, int fourth, int fifth, int sixth, int seventh, int eighth) { return first | (second << 3) | (third << 6) | (fourth << 9) | diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 0df97a7a4c..7496bf567d 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -221,6 +221,10 @@ public: /// Performs a deep comparison between this and the specified other node. bool deepEquals(const AttributePointer& attribute, const MetavoxelNode& other, const glm::vec3& minimum, float size, const MetavoxelLOD& lod) const; + + /// Retrieves all spanners satisfying the LOD constraint, placing them in the provided set. + void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum, + float size, const MetavoxelLOD& lod, SharedObjectSet& results) const; private: Q_DISABLE_COPY(MetavoxelNode) diff --git a/libraries/metavoxels/src/SharedObject.cpp b/libraries/metavoxels/src/SharedObject.cpp index 05af5f1bf8..d0a1842d31 100644 --- a/libraries/metavoxels/src/SharedObject.cpp +++ b/libraries/metavoxels/src/SharedObject.cpp @@ -84,11 +84,19 @@ bool SharedObject::equals(const SharedObject* other, bool sharedAncestry) const if (metaObject != other->metaObject() && !sharedAncestry) { return false; } - for (int i = 0; i < metaObject->propertyCount(); i++) { - QMetaProperty property = metaObject->property(i); - if (property.isStored() && property.read(this) != property.read(other)) { + // use the streamer, if we have one + const ObjectStreamer* streamer = Bitstream::getObjectStreamer(metaObject); + if (streamer) { + if (!streamer->equal(this, other)) { return false; } + } else { + for (int i = 0; i < metaObject->propertyCount(); i++) { + QMetaProperty property = metaObject->property(i); + if (property.isStored() && property.read(this) != property.read(other)) { + return false; + } + } } QList dynamicPropertyNames = this->dynamicPropertyNames(); if (dynamicPropertyNames.size() != other->dynamicPropertyNames().size()) { diff --git a/libraries/shared/src/SharedUtil.h b/libraries/shared/src/SharedUtil.h index dbbfb02365..e5c2a0afc9 100644 --- a/libraries/shared/src/SharedUtil.h +++ b/libraries/shared/src/SharedUtil.h @@ -68,7 +68,7 @@ float randFloat(); int randIntInRange (int min, int max); float randFloatInRange (float min,float max); float randomSign(); /// \return -1.0 or 1.0 -unsigned char randomColorValue(int minimum); +unsigned char randomColorValue(int minimum = 0); bool randomBoolean(); glm::quat safeMix(const glm::quat& q1, const glm::quat& q2, float alpha); diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 6aef40eab1..f22a7ef189 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -47,6 +47,8 @@ static int sharedObjectsDestroyed = 0; static int objectMutationsPerformed = 0; static int scriptObjectsCreated = 0; static int scriptMutationsPerformed = 0; +static int metavoxelMutationsPerformed = 0; +static int spannerMutationsPerformed = 0; static QByteArray createRandomBytes(int minimumSize, int maximumSize) { QByteArray bytes(randIntInRange(minimumSize, maximumSize), 0); @@ -395,6 +397,8 @@ bool MetavoxelTests::run() { qDebug() << "Sent" << datagramsSent << "datagrams with" << bytesSent << "bytes, received" << datagramsReceived << "with" << bytesReceived << "bytes"; qDebug() << "Max" << maxDatagramsPerPacket << "datagrams," << maxBytesPerPacket << "bytes per packet"; + qDebug() << "Performed" << metavoxelMutationsPerformed << "metavoxel mutations," << spannerMutationsPerformed << + "spanner mutations"; } qDebug() << "All tests passed!"; @@ -472,6 +476,12 @@ Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) : RandomVisitor visitor; _data.guide(visitor); qDebug() << "Created" << visitor.leafCount << "base leaves"; + + _data.insert(AttributeRegistry::getInstance()->getSpannersAttribute(), new Sphere()); + + _sphere = new Sphere(); + static_cast(_sphere.data())->setScale(0.01f); + _data.insert(AttributeRegistry::getInstance()->getSpannersAttribute(), _sphere); return; } // create the object that represents out delta-encoded state @@ -614,9 +624,10 @@ int MutateVisitor::visit(MetavoxelInfo& info) { if (info.size > MAXIMUM_LEAF_SIZE || (info.size > MINIMUM_LEAF_SIZE && randomBoolean())) { return encodeRandomOrder(); } - info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randIntInRange(0, 255), - randIntInRange(0, 255), randIntInRange(0, 255)))); + info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randomColorValue(), + randomColorValue(), randomColorValue()))); _mutationsRemaining--; + metavoxelMutationsPerformed++; return STOP_RECURSION; } @@ -651,6 +662,21 @@ bool Endpoint::simulate(int iterationNumber) { MutateVisitor visitor; _data.guide(visitor); + // perhaps mutate the spanner + if (randomBoolean()) { + SharedObjectPointer oldSphere = _sphere; + _sphere = _sphere->clone(true); + Sphere* newSphere = static_cast(_sphere.data()); + if (randomBoolean()) { + newSphere->setColor(QColor(randomColorValue(), randomColorValue(), randomColorValue())); + } else { + newSphere->setTranslation(newSphere->getTranslation() + glm::vec3(randFloatInRange(-0.01f, 0.01f), + randFloatInRange(-0.01f, 0.01f), randFloatInRange(-0.01f, 0.01f))); + } + _data.replace(AttributeRegistry::getInstance()->getSpannersAttribute(), oldSphere, _sphere); + spannerMutationsPerformed++; + } + // wait until we have a valid lod before sending if (!_lod.isValid()) { return false; @@ -772,7 +798,8 @@ void Endpoint::readMessage(Bitstream& in) { foreach (const SendRecord& sendRecord, _other->_sendRecords) { if (sendRecord.packetNumber == packetNumber) { if (!sendRecord.data.deepEquals(_data, _sendRecords.first().lod)) { - throw QString("Sent/received metavoxel data mismatch."); + qDebug() << "Sent/received metavoxel data mismatch."; + exit(true); } break; } @@ -807,17 +834,20 @@ void Endpoint::readMessage(Bitstream& in) { it != _other->_unreliableMessagesSent.end(); it++) { if (it->sequenceNumber == message.sequenceNumber) { if (!messagesEqual(it->submessage, message.submessage)) { - throw QString("Sent/received unreliable message mismatch."); + qDebug() << "Sent/received unreliable message mismatch."; + exit(true); } if (!it->state->equals(message.state)) { - throw QString("Delta-encoded object mismatch."); + qDebug() << "Delta-encoded object mismatch."; + exit(true); } _other->_unreliableMessagesSent.erase(_other->_unreliableMessagesSent.begin(), it + 1); unreliableMessagesReceived++; return; } } - throw QString("Received unsent/already sent unreliable message."); + qDebug() << "Received unsent/already sent unreliable message."; + exit(true); } void Endpoint::handleReliableMessage(const QVariant& message) { diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index 1de355661d..c340d78963 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -93,6 +93,8 @@ private: MetavoxelData _data; MetavoxelLOD _lod; + SharedObjectPointer _sphere; + Endpoint* _other; QList > _delayedDatagrams; float _highPriorityMessagesToSend; From 49f1d427eca90cbc3fff15fae45fe8f64b0cbe5b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Jun 2014 18:12:08 -0700 Subject: [PATCH 20/38] Removed incremental streaming stubs. --- libraries/metavoxels/src/MetavoxelData.cpp | 77 ---------------------- libraries/metavoxels/src/MetavoxelData.h | 5 -- 2 files changed, 82 deletions(-) diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp index 7caa336cd2..3e70e0f09b 100644 --- a/libraries/metavoxels/src/MetavoxelData.cpp +++ b/libraries/metavoxels/src/MetavoxelData.cpp @@ -602,83 +602,6 @@ void MetavoxelData::writeDelta(const MetavoxelData& reference, const MetavoxelLO } } -void MetavoxelData::readIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, - Bitstream& in, const MetavoxelLOD& lod) { - - // shallow copy the reference - *this = reference; - - // if nothing changed, return - bool changed; - in >> changed; - if (!changed) { - return; - } - - // expand if the size has changed - bool sizeChanged; - in >> sizeChanged; - if (sizeChanged) { - float size; - in >> size; - while (_size < size) { - expand(); - } - } - - // read the nodes removed - forever { - AttributePointer attribute; - in >> attribute; - if (!attribute) { - break; - } - _roots.take(attribute)->decrementReferenceCount(attribute); - } -} - -void MetavoxelData::writeIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, - Bitstream& out, const MetavoxelLOD& lod) const { - - // first things first: there might be no change whatsoever - glm::vec3 minimum = getMinimum(); - bool becameSubdivided = lod.becameSubdivided(minimum, _size, referenceLOD); - if (_size == reference._size && _roots == reference._roots && !becameSubdivided) { - out << false; - return; - } - out << true; - - // compare the size; if changed (rare), we must compare to the expanded reference - const MetavoxelData* expandedReference = &reference; - if (_size == reference._size) { - out << false; - } else { - out << true; - out << _size; - - MetavoxelData* expanded = new MetavoxelData(reference); - while (expanded->_size < _size) { - expanded->expand(); - } - expandedReference = expanded; - } - - // write the nodes removed - for (QHash::const_iterator it = expandedReference->_roots.constBegin(); - it != expandedReference->_roots.constEnd(); it++) { - if (!_roots.contains(it.key())) { - out << it.key(); - } - } - out << AttributePointer(); - - // delete the expanded reference if we had to expand - if (expandedReference != &reference) { - delete expandedReference; - } -} - MetavoxelNode* MetavoxelData::createRoot(const AttributePointer& attribute) { MetavoxelNode*& root = _roots[attribute]; if (root) { diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h index 7496bf567d..6a7ba33eb5 100644 --- a/libraries/metavoxels/src/MetavoxelData.h +++ b/libraries/metavoxels/src/MetavoxelData.h @@ -118,11 +118,6 @@ public: void writeDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, Bitstream& out, const MetavoxelLOD& lod) const; - void readIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, - Bitstream& in, const MetavoxelLOD& lod); - void writeIncrementalDelta(const MetavoxelData& reference, const MetavoxelLOD& referenceLOD, - Bitstream& out, const MetavoxelLOD& lod) const; - MetavoxelNode* getRoot(const AttributePointer& attribute) const { return _roots.value(attribute); } MetavoxelNode* createRoot(const AttributePointer& attribute); From 626bd42b7b60a29e1a838aa19dc6e6a62e525b9f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 20 Jun 2014 18:22:36 -0700 Subject: [PATCH 21/38] Use randomColorValue, since we have it. --- tests/metavoxels/src/MetavoxelTests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index f22a7ef189..287d3a648c 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -437,8 +437,8 @@ int RandomVisitor::visit(MetavoxelInfo& info) { if (info.size > MAXIMUM_LEAF_SIZE || (info.size > MINIMUM_LEAF_SIZE && randomBoolean())) { return DEFAULT_ORDER; } - info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randIntInRange(0, 255), - randIntInRange(0, 255), randIntInRange(0, 255)))); + info.outputValues[0] = OwnedAttributeValue(_outputs.at(0), encodeInline(qRgb(randomColorValue(), + randomColorValue(), randomColorValue()))); leafCount++; return STOP_RECURSION; } From 7196814ec3b2a087f966816becd7803475f5c810 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 10:22:16 -0700 Subject: [PATCH 22/38] added debug prints to look at injected ring buffers --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 ++ libraries/audio/src/AudioRingBuffer.cpp | 10 +++++++--- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 -- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0c41cc70f9..0080af4716 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,6 +90,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // we don't have a matching injected audio ring buffer, so add it matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier); _ringBuffers.push_back(matchingInjectedRingBuffer); +printf("created InjectedRingBuffer %p\n", (void*)matchingInjectedRingBuffer); } matchingInjectedRingBuffer->parseData(packet); @@ -132,6 +133,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; +printf("deleted injector %p\n", (void*)audioBuffer); i = _ringBuffers.erase(i); continue; } diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ae100dcdf1..8ce48a39e3 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -125,16 +125,19 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { std::less less; std::less_equal lessEqual; + // TODO: why is less(_endOfLastWrite, _nextOutput) a condition here? if (_hasStarted + /* && (less(_endOfLastWrite, _nextOutput) - && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))) { + && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))*/ + && samplesToCopy > _sampleCapacity - samplesAvailable()) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting."; _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; } - + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { @@ -145,6 +148,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); +printf("%p: writeData. %d samples available\n", (void*)this, samplesAvailable()); return samplesToCopy * sizeof(int16_t); } @@ -158,7 +162,7 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); -//printf("\nmixed. %d samples remaining\n", samplesAvailable()); +printf("%p: mixed. %d samples remaining\n", (void*)this, samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 93f68b4dff..03ac36d09e 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -151,8 +151,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); readBytes += dataBytes; } -//printf("parsed packet. %d samples available\n", samplesAvailable()); - return readBytes; } From e5d7d471e34b7037b9726c381f00d36247d880ff Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 10:55:30 -0700 Subject: [PATCH 23/38] removed debug code --- .../src/audio/AudioMixerClientData.cpp | 2 -- interface/src/Audio.cpp | 2 -- libraries/audio/src/AudioRingBuffer.cpp | 13 ++----- libraries/audio/src/AudioRingBuffer.h | 3 +- .../audio/src/PositionalAudioRingBuffer.cpp | 36 ++++--------------- .../audio/src/PositionalAudioRingBuffer.h | 3 -- 6 files changed, 10 insertions(+), 49 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0080af4716..0c41cc70f9 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -90,7 +90,6 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // we don't have a matching injected audio ring buffer, so add it matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier); _ringBuffers.push_back(matchingInjectedRingBuffer); -printf("created InjectedRingBuffer %p\n", (void*)matchingInjectedRingBuffer); } matchingInjectedRingBuffer->parseData(packet); @@ -133,7 +132,6 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete delete audioBuffer; -printf("deleted injector %p\n", (void*)audioBuffer); i = _ringBuffers.erase(i); continue; } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 47e55ddc90..a0a85f8888 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -622,7 +622,6 @@ void Audio::handleAudioInput() { SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); if (audioMixer && audioMixer->getActiveSocket()) { - MyAvatar* interfaceAvatar = Application::getInstance()->getAvatar(); glm::vec3 headPosition = interfaceAvatar->getHead()->getPosition(); glm::quat headOrientation = interfaceAvatar->getHead()->getFinalOrientationInWorldFrame(); @@ -639,7 +638,6 @@ void Audio::handleAudioInput() { numAudioBytes = sizeof(int16_t); } else { numAudioBytes = numNetworkBytes; - //_isStereoInput ? NETWORK_BUFFER_LENGTH_BYTES_STEREO : NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL; if (Menu::getInstance()->isOptionChecked(MenuOption::EchoServerAudio)) { packetType = PacketTypeMicrophoneAudioWithEcho; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 8ce48a39e3..1b6bdaa5d8 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -124,13 +124,8 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { std::less less; std::less_equal lessEqual; - - // TODO: why is less(_endOfLastWrite, _nextOutput) a condition here? - if (_hasStarted - /* - && (less(_endOfLastWrite, _nextOutput) - && lessEqual(_nextOutput, shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy)))*/ - && samplesToCopy > _sampleCapacity - samplesAvailable()) { + + if (_hasStarted && samplesToCopy > _sampleCapacity - samplesAvailable()) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting."; _endOfLastWrite = _buffer; @@ -147,8 +142,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - -printf("%p: writeData. %d samples available\n", (void*)this, samplesAvailable()); + return samplesToCopy * sizeof(int16_t); } @@ -162,7 +156,6 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); -printf("%p: mixed. %d samples remaining\n", (void*)this, samplesAvailable()); } unsigned int AudioRingBuffer::samplesAvailable() const { diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 3d2864f373..04cc67c8ac 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -74,14 +74,13 @@ public: bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); - protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); AudioRingBuffer& operator= (const AudioRingBuffer&); int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - + int _sampleCapacity; int _numFrameSamples; int16_t* _nextOutput; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 03ac36d09e..7c2369d7e5 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -33,12 +33,9 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() } void InterframeTimeGapHistory::frameReceived() { - - static quint64 gapsSum = 0; - quint64 now = usecTimestampNow(); - // make sure this isn't the first time frameReceived() is called, meaning there's actually a gap to calculate. + // make sure this isn't the first time frameReceived() is called so can actually calculate a gap. if (_lastFrameReceivedTime != 0) { quint64 gap = now - _lastFrameReceivedTime; @@ -123,7 +120,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); if (numSilentSamples > 0) { - if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { // our current jitter buffer size exceeds its desired value, so ignore some silent // frames to get that size as close to desired as possible @@ -140,7 +136,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } else { // we need to drop all frames to get the jitter buffer close as possible to its desired length _currentJitterBufferFrames -= numSilentFrames; - printf("dropped silent packet! jbuffer size is now %d ###############\n", _currentJitterBufferFrames); } } else { addSilentFrame(numSilentSamples); @@ -148,8 +143,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { } } else { // there is audio data to read - int dataBytes = writeData(packet.data() + readBytes, packet.size() - readBytes); - readBytes += dataBytes; + readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); } return readBytes; } @@ -200,12 +194,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { int samplesPerFrame = getSamplesPerFrame(); int desiredJitterBufferSamples = _desiredJitterBufferFrames * samplesPerFrame; -//printf("\nsamples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { - -printf("\nMIXING DELAYED! waiting for jitter buffer to fill after being starved\n"); -printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing @@ -214,15 +203,10 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp } return false; - - } else if (samplesAvailable() < samplesPerFrame) { - -printf("\nMIXING DELAYED! jitter buffer is starved!!!\n"); -printf("samples available: %d frames available: %d\n", samplesAvailable(), samplesAvailable() / samplesPerFrame); - + } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; - + // set to 0 to indicate the jitter buffer is starved _currentJitterBufferFrames = 0; @@ -230,19 +214,13 @@ printf("samples available: %d frames available: %d\n", samplesAvailable(), samp _shouldOutputStarveDebug = true; return false; - } -//printf("WILL MIX\n"); - + // good buffer, add this to the mix - if (_isStarved) { - // if this buffer has just finished replenishing after being starved, the number of frames in it now // minus one (since a frame will be read immediately after this) is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; - printf("jitter buffer length is now %d; desired: %d --------\n", _currentJitterBufferFrames, _desiredJitterBufferFrames); - _isStarved = false; } @@ -261,7 +239,5 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } - - printf(" desired jbuffer size is now %d --------------------------------\n", _desiredJitterBufferFrames); } -} \ No newline at end of file +} diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 87a53038a3..5fa24ff782 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -43,7 +43,6 @@ private: bool _newWindowMaxGapAvailable; }; - class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { @@ -99,8 +98,6 @@ protected: InterframeTimeGapHistory _interframeTimeGapHistory; int _desiredJitterBufferFrames; int _currentJitterBufferFrames; - -quint64 _lastMixTime; }; #endif // hifi_PositionalAudioRingBuffer_h From ffc20b8876e86fea8364a7535870ad9e8b4f8c6d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 11:22:04 -0700 Subject: [PATCH 24/38] changed InterframeTimeGapHistory to InterframeTimeGapStats --- assignment-client/src/audio/AvatarAudioRingBuffer.cpp | 2 +- libraries/audio/src/InjectedAudioRingBuffer.cpp | 2 +- libraries/audio/src/PositionalAudioRingBuffer.cpp | 10 +++++----- libraries/audio/src/PositionalAudioRingBuffer.h | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 0df50f99db..79ad7ae411 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -19,7 +19,7 @@ AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : } int AvatarAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapHistory.frameReceived(); + _interframeTimeGapStats.frameReceived(); updateDesiredJitterBufferFrames(); _shouldLoopbackForNode = (packetTypeForPacket(packet) == PacketTypeMicrophoneAudioWithEcho); diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 80bcda5acb..ed0476f6bd 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -31,7 +31,7 @@ InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) const uchar MAX_INJECTOR_VOLUME = 255; int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { - _interframeTimeGapHistory.frameReceived(); + _interframeTimeGapStats.frameReceived(); updateDesiredJitterBufferFrames(); // setup a data stream to read from this packet diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 7c2369d7e5..a22f57f3d7 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -21,7 +21,7 @@ #include "PositionalAudioRingBuffer.h" #include "SharedUtil.h" -InterframeTimeGapHistory::InterframeTimeGapHistory() +InterframeTimeGapStats::InterframeTimeGapStats() : _lastFrameReceivedTime(0), _numSamplesInCurrentInterval(0), _currentIntervalMaxGap(0), @@ -32,7 +32,7 @@ InterframeTimeGapHistory::InterframeTimeGapHistory() memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); } -void InterframeTimeGapHistory::frameReceived() { +void InterframeTimeGapStats::frameReceived() { quint64 now = usecTimestampNow(); // make sure this isn't the first time frameReceived() is called so can actually calculate a gap. @@ -79,7 +79,7 @@ void InterframeTimeGapHistory::frameReceived() { _lastFrameReceivedTime = now; } -quint64 InterframeTimeGapHistory::getWindowMaxGap() { +quint64 InterframeTimeGapStats::getWindowMaxGap() { _newWindowMaxGapAvailable = false; return _windowMaxGap; } @@ -234,8 +234,8 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_interframeTimeGapHistory.hasNewWindowMaxGapAvailable()) { - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapHistory.getWindowMaxGap() / USECS_PER_FRAME); + if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 5fa24ff782..8c2122f29e 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -23,9 +23,9 @@ const int TIME_GAP_NUM_SAMPLES_IN_INTERVAL = 500; const int TIME_GAP_NUM_INTERVALS_IN_WINDOW = 10; // class used to track time between incoming frames for the purpose of varying the jitter buffer length -class InterframeTimeGapHistory { +class InterframeTimeGapStats { public: - InterframeTimeGapHistory(); + InterframeTimeGapStats(); void frameReceived(); bool hasNewWindowMaxGapAvailable() const { return _newWindowMaxGapAvailable; } @@ -95,7 +95,7 @@ protected: float _nextOutputTrailingLoudness; AABox* _listenerUnattenuatedZone; - InterframeTimeGapHistory _interframeTimeGapHistory; + InterframeTimeGapStats _interframeTimeGapStats; int _desiredJitterBufferFrames; int _currentJitterBufferFrames; }; From fbdca59d3777a9b9561cd212e8644f14fd5bce43 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 11:56:14 -0700 Subject: [PATCH 25/38] fixed spacing issue --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index a22f57f3d7..0e2e785973 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -29,7 +29,7 @@ InterframeTimeGapStats::InterframeTimeGapStats() _windowMaxGap(0), _newWindowMaxGapAvailable(false) { - memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW*sizeof(quint64)); + memset(_intervalMaxGaps, 0, TIME_GAP_NUM_INTERVALS_IN_WINDOW * sizeof(quint64)); } void InterframeTimeGapStats::frameReceived() { From ed5c05a61b9536d98bedd85d0d2b080667bfa4e0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 23 Jun 2014 13:25:32 -0700 Subject: [PATCH 26/38] fix rounding of credit balance in application title --- interface/src/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 75c3f442ca..d7e3b7453d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3189,7 +3189,7 @@ void Application::updateWindowTitle(){ float creditBalance = accountManager.getAccountInfo().getBalance() / SATOSHIS_PER_CREDIT; QString creditBalanceString; - creditBalanceString.sprintf("%.8f", creditBalance); + creditBalanceString.sprintf("%.8f", floor(creditBalance + 0.5)); title += " - ₵" + creditBalanceString; } From 076bc115456ba43e585ea8cf2a81dd14386318ba Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 15:19:13 -0700 Subject: [PATCH 27/38] added server side audio mixer jitter stats --- assignment-client/src/audio/AudioMixer.cpp | 13 +++++++++++ .../src/audio/AudioMixerClientData.cpp | 23 +++++++++++++++++++ .../src/audio/AudioMixerClientData.h | 3 +++ .../audio/src/PositionalAudioRingBuffer.h | 3 +++ 4 files changed, 42 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 42be1aea5a..15b51ae7d5 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -424,6 +424,19 @@ void AudioMixer::sendStatsPacket() { } else { statsObject["average_mixes_per_listener"] = 0.0; } + + // add stats for each listerner + NodeList* nodeList = NodeList::getInstance(); + int clientNumber = 0; + foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { + clientNumber++; + AudioMixerClientData* clientData = static_cast(node->getLinkedData()); + if (clientData) { + QString property = "jitterStats." + QString::number(clientNumber); + statsObject[qPrintable(property)] = clientData->getJitterBufferStats(); + } + } + ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0c41cc70f9..1c0f68d5a3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -138,3 +138,26 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { i++; } } + +QString AudioMixerClientData::getJitterBufferStats() const { + QString result; + AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); + if (avatarRingBuffer) { + int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); + int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); + result += "mic.desired:" + QString::number(desiredJitterBuffer) + " current:" + QString::number(currentJitterBuffer); + } else { + result = "mic unknown"; + } + + for (int i = 0; i < _ringBuffers.size(); i++) { + if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { + int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); + int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); + result += "| injected["+QString::number(i)+"].desired:" + + QString::number(desiredJitterBuffer) + " current:" + QString::number(currentJitterBuffer); + } + } + + return result; +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 3c4ddd3459..1760b96f5d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -29,6 +29,9 @@ public: int parseData(const QByteArray& packet); void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); + + QString getJitterBufferStats() const; + private: QList _ringBuffers; }; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 8c2122f29e..4c7148abbe 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -77,6 +77,9 @@ public: int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } + int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } + protected: // disallow copying of PositionalAudioRingBuffer objects PositionalAudioRingBuffer(const PositionalAudioRingBuffer&); From b2c982976e58652480c363f5ab12a096dd7a947b Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 15:46:58 -0700 Subject: [PATCH 28/38] break up audio mixer stats across multiple packets --- assignment-client/src/audio/AudioMixer.cpp | 38 ++++++++++++++----- .../audio/src/PositionalAudioRingBuffer.cpp | 1 + 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 15b51ae7d5..ccf7a3248d 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -425,7 +425,20 @@ void AudioMixer::sendStatsPacket() { statsObject["average_mixes_per_listener"] = 0.0; } - // add stats for each listerner + ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); + _sumListeners = 0; + _sumMixes = 0; + _numStatFrames = 0; + + + // NOTE: These stats can be too large to fit in an MTU, so we break it up into multiple packts... + QJsonObject statsObject2; + + // add stats for each listerner + bool somethingToSend = false; + int sizeOfStats = 0; + int TOO_BIG_FOR_MTU = 1200; // some extra space for JSONification + NodeList* nodeList = NodeList::getInstance(); int clientNumber = 0; foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { @@ -433,16 +446,23 @@ void AudioMixer::sendStatsPacket() { AudioMixerClientData* clientData = static_cast(node->getLinkedData()); if (clientData) { QString property = "jitterStats." + QString::number(clientNumber); - statsObject[qPrintable(property)] = clientData->getJitterBufferStats(); + QString value = clientData->getJitterBufferStats(); + statsObject2[qPrintable(property)] = value; + somethingToSend = true; + sizeOfStats += property.size() + value.size(); + } + + // if we're too large, send the packet + if (sizeOfStats > TOO_BIG_FOR_MTU) { + nodeList->sendStatsToDomainServer(statsObject2); + sizeOfStats = 0; + statsObject2 = QJsonObject(); // clear it } } - - - ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); - - _sumListeners = 0; - _sumMixes = 0; - _numStatFrames = 0; + + if (somethingToSend) { + nodeList->sendStatsToDomainServer(statsObject2); + } } void AudioMixer::run() { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 0e2e785973..258150a1c5 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -235,6 +235,7 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; From 4becb66e2a307b0ac8c555f61ea38cf197270033 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 16:10:51 -0700 Subject: [PATCH 29/38] use node uuid in stats --- assignment-client/src/audio/AudioMixer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index ccf7a3248d..1d0b51a437 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -445,7 +445,7 @@ void AudioMixer::sendStatsPacket() { clientNumber++; AudioMixerClientData* clientData = static_cast(node->getLinkedData()); if (clientData) { - QString property = "jitterStats." + QString::number(clientNumber); + QString property = "jitterStats." + node->getUUID().toString(); QString value = clientData->getJitterBufferStats(); statsObject2[qPrintable(property)] = value; somethingToSend = true; @@ -457,6 +457,7 @@ void AudioMixer::sendStatsPacket() { nodeList->sendStatsToDomainServer(statsObject2); sizeOfStats = 0; statsObject2 = QJsonObject(); // clear it + somethingToSend = false; } } From 8a633743b3be83d5eb6320246d33a6fb72bd2e5e Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 16:40:38 -0700 Subject: [PATCH 30/38] ignore the numSilentSamples from client, always assume one frame --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 258150a1c5..115a99b453 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -119,6 +119,9 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); + // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples + numSilentSamples = getNumSamplesPerFrame(); + if (numSilentSamples > 0) { if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { // our current jitter buffer size exceeds its desired value, so ignore some silent From 77f262561c0087e34a15a418b9ae2c2e5c5425e4 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 16:47:10 -0700 Subject: [PATCH 31/38] typo --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 115a99b453..a2e0d08b91 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -120,7 +120,7 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { readBytes += sizeof(int16_t); // NOTE: fixes a bug in old clients that would send garbage for their number of silentSamples - numSilentSamples = getNumSamplesPerFrame(); + numSilentSamples = getSamplesPerFrame(); if (numSilentSamples > 0) { if (_currentJitterBufferFrames > _desiredJitterBufferFrames) { From 6434c0a2a49177e159d1df92dae7fe8069fcdf57 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 17:28:39 -0700 Subject: [PATCH 32/38] add more stats --- .../src/audio/AudioMixerClientData.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 1c0f68d5a3..cde1377053 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -145,7 +145,12 @@ QString AudioMixerClientData::getJitterBufferStats() const { if (avatarRingBuffer) { int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); - result += "mic.desired:" + QString::number(desiredJitterBuffer) + " current:" + QString::number(currentJitterBuffer); + int samplesAvailable = avatarRingBuffer->samplesAvailable(); + int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); + result += "mic.desired:" + QString::number(desiredJitterBuffer) + + " current:" + QString::number(currentJitterBuffer) + + " available:" + QString::number(framesAvailable) + + " samples:" + QString::number(samplesAvailable); } else { result = "mic unknown"; } @@ -154,8 +159,12 @@ QString AudioMixerClientData::getJitterBufferStats() const { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); - result += "| injected["+QString::number(i)+"].desired:" - + QString::number(desiredJitterBuffer) + " current:" + QString::number(currentJitterBuffer); + int samplesAvailable = _ringBuffers[i]->samplesAvailable(); + int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); + result += "| injected["+QString::number(i)+"].desired:" + QString::number(desiredJitterBuffer) + + " current:" + QString::number(currentJitterBuffer) + + " available:" + QString::number(framesAvailable) + + " samples:" + QString::number(samplesAvailable); } } From 355ab2cb957bd628de25139ced9e1a5fe9feb2e7 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 17:58:55 -0700 Subject: [PATCH 33/38] max desired must be one less than the max frames in our ringbuffer --- libraries/audio/src/AudioRingBuffer.cpp | 3 --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 1b6bdaa5d8..71fd2fe19d 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -122,9 +122,6 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { int samplesToCopy = std::min((quint64)(maxSize / sizeof(int16_t)), (quint64)_sampleCapacity); - std::less less; - std::less_equal lessEqual; - if (_hasStarted && samplesToCopy > _sampleCapacity - samplesAvailable()) { // this read will cross the next output, so call us starved and reset the buffer qDebug() << "Filled the ring buffer. Resetting."; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index a2e0d08b91..2ba407242c 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -243,5 +243,9 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; } + const int maxDesired = RING_BUFFER_LENGTH_FRAMES - 1; + if (_desiredJitterBufferFrames > maxDesired) { + _desiredJitterBufferFrames = maxDesired; + } } } From fda60cc8d5ea957d91d4bbca28aa37945fc8e75e Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Mon, 23 Jun 2014 18:45:24 -0700 Subject: [PATCH 34/38] remove resizing of jitter buffers for now, only ask for 1 frame --- libraries/audio/src/PositionalAudioRingBuffer.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 2ba407242c..5b6d5138d1 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -234,11 +234,12 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { - - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence + /* + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; @@ -247,5 +248,6 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames > maxDesired) { _desiredJitterBufferFrames = maxDesired; } + */ } } From c40c5d78c8db6e6d7e5ef0312ece8e830e89814e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 24 Jun 2014 08:50:39 -0700 Subject: [PATCH 35/38] Fix default audio devices on Windows Windows 8 provides the full friendly device name, not just the first 31 characters. --- interface/src/Audio.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index a0a85f8888..65912f83e8 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -218,9 +218,14 @@ QAudioDeviceInfo defaultAudioDeviceForMode(QAudio::Mode mode) { hr = pPropertyStore->GetValue(PKEY_Device_FriendlyName, &pv); pPropertyStore->Release(); pPropertyStore = NULL; - //QAudio devices seems to only take the 31 first characters of the Friendly Device Name. - const DWORD QT_WIN_MAX_AUDIO_DEVICENAME_LEN = 31; - deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal).left(QT_WIN_MAX_AUDIO_DEVICENAME_LEN); + deviceName = QString::fromWCharArray((wchar_t*)pv.pwszVal); + const DWORD WINDOWS7_MAJOR_VERSION = 6; + const DWORD WINDOWS7_MINOR_VERSION = 1; + if (osvi.dwMajorVersion <= WINDOWS7_MAJOR_VERSION && osvi.dwMinorVersion <= WINDOWS7_MINOR_VERSION) { + // Windows 7 provides only the 31 first characters of the device name. + const DWORD QT_WIN7_MAX_AUDIO_DEVICENAME_LEN = 31; + deviceName = deviceName.left(QT_WIN7_MAX_AUDIO_DEVICENAME_LEN); + } qDebug() << (mode == QAudio::AudioOutput ? "output" : "input") << " device:" << deviceName; PropVariantClear(&pv); } From 5c188ea81e7c14507282bce8ef2931a55c3415c0 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 24 Jun 2014 09:41:29 -0700 Subject: [PATCH 36/38] adding more stats --- assignment-client/src/audio/AudioMixerClientData.cpp | 4 ++++ libraries/audio/src/PositionalAudioRingBuffer.cpp | 11 +++++++++++ libraries/audio/src/PositionalAudioRingBuffer.h | 1 + 3 files changed, 16 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index cde1377053..def9b8c953 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -144,10 +144,12 @@ QString AudioMixerClientData::getJitterBufferStats() const { AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); + int calculatedJitterBuffer = avatarRingBuffer->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); int samplesAvailable = avatarRingBuffer->samplesAvailable(); int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); result += "mic.desired:" + QString::number(desiredJitterBuffer) + + " calculated:" + QString::number(calculatedJitterBuffer) + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) + " samples:" + QString::number(samplesAvailable); @@ -158,10 +160,12 @@ QString AudioMixerClientData::getJitterBufferStats() const { for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); + int calculatedJitterBuffer = _ringBuffers[i]->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); int samplesAvailable = _ringBuffers[i]->samplesAvailable(); int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); result += "| injected["+QString::number(i)+"].desired:" + QString::number(desiredJitterBuffer) + + " calculated:" + QString::number(calculatedJitterBuffer) + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) + " samples:" + QString::number(samplesAvailable); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 5b6d5138d1..e3c123e592 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -233,6 +233,17 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { return true; } +int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { + int calculatedDesiredJitterBufferFrames = 1; + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + + calculatedDesiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.peekWindowMaxGap() / USECS_PER_FRAME); + if (calculatedDesiredJitterBufferFrames < 1) { + calculatedDesiredJitterBufferFrames = 1; + } + return calculatedDesiredJitterBufferFrames; +} + void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 4c7148abbe..b9caeff605 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -77,6 +77,7 @@ public: int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } From 1d390faad877e1b5be734e3980100f2306bc0a6a Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 24 Jun 2014 10:29:38 -0700 Subject: [PATCH 37/38] add support for commandline/config value for dynamic jitter buffers --- assignment-client/src/audio/AudioMixer.cpp | 12 +++++++ assignment-client/src/audio/AudioMixer.h | 4 +++ .../src/audio/AudioMixerClientData.cpp | 14 ++++++--- .../src/audio/AvatarAudioRingBuffer.cpp | 4 +-- .../src/audio/AvatarAudioRingBuffer.h | 2 +- libraries/audio/src/AudioRingBuffer.cpp | 2 ++ libraries/audio/src/AudioRingBuffer.h | 3 ++ .../audio/src/InjectedAudioRingBuffer.cpp | 4 +-- libraries/audio/src/InjectedAudioRingBuffer.h | 2 +- .../audio/src/PositionalAudioRingBuffer.cpp | 31 ++++++++++--------- .../audio/src/PositionalAudioRingBuffer.h | 3 +- 11 files changed, 56 insertions(+), 25 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1d0b51a437..8e4ce04f0b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -64,6 +64,8 @@ void attachNewBufferToNode(Node *newNode) { } } +bool AudioMixer::_useDynamicJitterBuffers = false; + AudioMixer::AudioMixer(const QByteArray& packet) : ThreadedAssignment(packet), _trailingSleepRatio(1.0f), @@ -502,6 +504,16 @@ void AudioMixer::run() { << QString("%1, %2, %3").arg(destinationCenter.x).arg(destinationCenter.y).arg(destinationCenter.z); } + // check the payload to see if we have asked for dynamicJitterBuffer support + const QString DYNAMIC_JITTER_BUFFER_REGEX_STRING = "--dynamicJitterBuffer"; + QRegExp dynamicJitterBufferMatch(DYNAMIC_JITTER_BUFFER_REGEX_STRING); + if (dynamicJitterBufferMatch.indexIn(_payload) != -1) { + qDebug() << "Enable dynamic jitter buffers."; + _useDynamicJitterBuffers = true; + } else { + qDebug() << "Dynamic jitter buffers disabled, using old behavior."; + } + int nextFrame = 0; QElapsedTimer timer; timer.start(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 39f8cf63ae..8560978689 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -34,6 +34,9 @@ public slots: void readPendingDatagrams(); void sendStatsPacket(); + + static bool getUseDynamicJitterBuffers() { return _useDynamicJitterBuffers; } + private: /// adds one buffer to the mix for a listening node void addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuffer* bufferToAdd, @@ -54,6 +57,7 @@ private: int _sumMixes; AABox* _sourceUnattenuatedZone; AABox* _listenerUnattenuatedZone; + static bool _useDynamicJitterBuffers; }; #endif // hifi_AudioMixer_h diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index def9b8c953..395cebf9c2 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -16,6 +16,7 @@ #include "InjectedAudioRingBuffer.h" +#include "AudioMixer.h" #include "AudioMixerClientData.h" AudioMixerClientData::AudioMixerClientData() : @@ -65,7 +66,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { if (!avatarRingBuffer) { // we don't have an AvatarAudioRingBuffer yet, so add it - avatarRingBuffer = new AvatarAudioRingBuffer(isStereo); + avatarRingBuffer = new AvatarAudioRingBuffer(isStereo, AudioMixer::getUseDynamicJitterBuffers()); _ringBuffers.push_back(avatarRingBuffer); } @@ -88,7 +89,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { if (!matchingInjectedRingBuffer) { // we don't have a matching injected audio ring buffer, so add it - matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier); + matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, + AudioMixer::getUseDynamicJitterBuffers()); _ringBuffers.push_back(matchingInjectedRingBuffer); } @@ -146,13 +148,15 @@ QString AudioMixerClientData::getJitterBufferStats() const { int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); int calculatedJitterBuffer = avatarRingBuffer->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); + int resetCount = avatarRingBuffer->getResetCount(); int samplesAvailable = avatarRingBuffer->samplesAvailable(); int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); result += "mic.desired:" + QString::number(desiredJitterBuffer) + " calculated:" + QString::number(calculatedJitterBuffer) + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable); + + " samples:" + QString::number(samplesAvailable) + + " resets:" + QString::number(resetCount); } else { result = "mic unknown"; } @@ -162,13 +166,15 @@ QString AudioMixerClientData::getJitterBufferStats() const { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); int calculatedJitterBuffer = _ringBuffers[i]->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); + int resetCount = _ringBuffers[i]->getResetCount(); int samplesAvailable = _ringBuffers[i]->samplesAvailable(); int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); result += "| injected["+QString::number(i)+"].desired:" + QString::number(desiredJitterBuffer) + " calculated:" + QString::number(calculatedJitterBuffer) + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable); + + " samples:" + QString::number(samplesAvailable) + + " resets:" + QString::number(resetCount); } } diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp index 79ad7ae411..9c6cc32f57 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.cpp +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.cpp @@ -13,8 +13,8 @@ #include "AvatarAudioRingBuffer.h" -AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo) : - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo) { +AvatarAudioRingBuffer::AvatarAudioRingBuffer(bool isStereo, bool dynamicJitterBuffer) : + PositionalAudioRingBuffer(PositionalAudioRingBuffer::Microphone, isStereo, dynamicJitterBuffer) { } diff --git a/assignment-client/src/audio/AvatarAudioRingBuffer.h b/assignment-client/src/audio/AvatarAudioRingBuffer.h index f842c2aa33..e227e70958 100644 --- a/assignment-client/src/audio/AvatarAudioRingBuffer.h +++ b/assignment-client/src/audio/AvatarAudioRingBuffer.h @@ -18,7 +18,7 @@ class AvatarAudioRingBuffer : public PositionalAudioRingBuffer { public: - AvatarAudioRingBuffer(bool isStereo = false); + AvatarAudioRingBuffer(bool isStereo = false, bool dynamicJitterBuffer = false); int parseData(const QByteArray& packet); private: diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 71fd2fe19d..ee4027841b 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -21,6 +21,7 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), + _resetCount(0), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), _numFrameSamples(numFrameSamples), _isStarved(true), @@ -128,6 +129,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; + _resetCount++; } if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 04cc67c8ac..1ddcadeceb 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -71,6 +71,7 @@ public: bool isStarved() const { return _isStarved; } void setIsStarved(bool isStarved) { _isStarved = isStarved; } + int getResetCount() const { return _resetCount; } /// how many times has the ring buffer written past the end and reset bool hasStarted() const { return _hasStarted; } void addSilentFrame(int numSilentSamples); @@ -80,6 +81,8 @@ protected: AudioRingBuffer& operator= (const AudioRingBuffer&); int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; + + int _resetCount; /// how many times has the ring buffer written past the end and done a reset int _sampleCapacity; int _numFrameSamples; diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index ed0476f6bd..9b6529b49f 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -19,8 +19,8 @@ #include "InjectedAudioRingBuffer.h" -InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier) : - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector), +InjectedAudioRingBuffer::InjectedAudioRingBuffer(const QUuid& streamIdentifier, bool dynamicJitterBuffer) : + PositionalAudioRingBuffer(PositionalAudioRingBuffer::Injector, /* isStereo=*/ false , dynamicJitterBuffer), _streamIdentifier(streamIdentifier), _radius(0.0f), _attenuationRatio(0) diff --git a/libraries/audio/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h index fd766e2848..4e3fea672b 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -18,7 +18,7 @@ class InjectedAudioRingBuffer : public PositionalAudioRingBuffer { public: - InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid()); + InjectedAudioRingBuffer(const QUuid& streamIdentifier = QUuid(), bool dynamicJitterBuffer = false); int parseData(const QByteArray& packet); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index e3c123e592..23e258fe87 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -85,7 +85,9 @@ quint64 InterframeTimeGapStats::getWindowMaxGap() { } -PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo) : +PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, + bool isStereo, bool dynamicJitterBuffers) : + AudioRingBuffer(isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL), _type(type), _position(0.0f, 0.0f, 0.0f), @@ -96,7 +98,8 @@ PositionalAudioRingBuffer::PositionalAudioRingBuffer(PositionalAudioRingBuffer:: _isStereo(isStereo), _listenerUnattenuatedZone(NULL), _desiredJitterBufferFrames(1), - _currentJitterBufferFrames(0) + _currentJitterBufferFrames(0), + _dynamicJitterBuffers(dynamicJitterBuffers) { } @@ -246,19 +249,19 @@ int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { - - _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence - /* - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + if (!_dynamicJitterBuffers) { + _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence + } else { + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); - if (_desiredJitterBufferFrames < 1) { - _desiredJitterBufferFrames = 1; + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); + if (_desiredJitterBufferFrames < 1) { + _desiredJitterBufferFrames = 1; + } + const int maxDesired = RING_BUFFER_LENGTH_FRAMES - 1; + if (_desiredJitterBufferFrames > maxDesired) { + _desiredJitterBufferFrames = maxDesired; + } } - const int maxDesired = RING_BUFFER_LENGTH_FRAMES - 1; - if (_desiredJitterBufferFrames > maxDesired) { - _desiredJitterBufferFrames = maxDesired; - } - */ } } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index b9caeff605..b204dc766b 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -50,7 +50,7 @@ public: Injector }; - PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false); + PositionalAudioRingBuffer(PositionalAudioRingBuffer::Type type, bool isStereo = false, bool dynamicJitterBuffers = false); int parseData(const QByteArray& packet); int parsePositionalData(const QByteArray& positionalByteArray); @@ -102,6 +102,7 @@ protected: InterframeTimeGapStats _interframeTimeGapStats; int _desiredJitterBufferFrames; int _currentJitterBufferFrames; + bool _dynamicJitterBuffers; }; #endif // hifi_PositionalAudioRingBuffer_h From cdbda02765f90b4f7bf5a2b6e97428379106b5d8 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 24 Jun 2014 12:09:58 -0700 Subject: [PATCH 38/38] make default ring buffer much larger --- libraries/audio/src/AudioRingBuffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 1ddcadeceb..33fb0d238a 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -31,7 +31,7 @@ const int NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL = NETWORK_BUFFER_LENGTH_BYTE const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float) SAMPLE_RATE) * 1000 * 1000); -const short RING_BUFFER_LENGTH_FRAMES = 10; +const short RING_BUFFER_LENGTH_FRAMES = 100; const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); const int MIN_SAMPLE_VALUE = std::numeric_limits::min();