From e0a33f129c9a3c62673c879b66ab04ac46591f3d Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 23 Jun 2014 15:55:54 -0700 Subject: [PATCH 01/67] Updated oculus initialization --- interface/src/devices/OculusManager.cpp | 33 ++++++++++--------------- interface/src/devices/OculusManager.h | 7 ++---- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index b2ee4e8c18..19c6a2b193 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -31,31 +31,25 @@ bool OculusManager::_isConnected = false; using namespace OVR; using namespace OVR::Util::Render; -Ptr OculusManager::_deviceManager; -Ptr OculusManager::_hmdDevice; -Ptr OculusManager::_sensorDevice; -SensorFusion* OculusManager::_sensorFusion; -StereoConfig OculusManager::_stereoConfig; +ovrHmd OculusManager::_ovrHmd; +ovrHmdDesc OculusManager::_ovrHmdDesc; + #endif void OculusManager::connect() { #ifdef HAVE_LIBOVR - System::Init(); - _deviceManager = *DeviceManager::Create(); - _hmdDevice = *_deviceManager->EnumerateDevices().CreateDevice(); + ovr_Initialize(); - if (_hmdDevice) { + _ovrHmd = ovrHmd_Create(0); + if (_ovrHmd) { _isConnected = true; - - _sensorDevice = *_hmdDevice->GetSensor(); - _sensorFusion = new SensorFusion; - _sensorFusion->AttachToSensor(_sensorDevice); - _sensorFusion->SetPredictionEnabled(true); + + ovrHmd_GetDesc(_ovrHmd, &_ovrHmdDesc); + + ovrHmd_StartSensor(_ovrHmd, ovrSensorCap_Orientation | ovrSensorCap_YawCorrection | + ovrSensorCap_Position | ovrHmdCap_LowPersistence, + ovrSensorCap_Orientation); - HMDInfo info; - _hmdDevice->GetDeviceInfo(&info); - _stereoConfig.SetHMDInfo(info); - _program.addShaderFromSourceFile(QGLShader::Fragment, Application::resourcesPath() + "shaders/oculus.frag"); _program.link(); @@ -66,8 +60,7 @@ void OculusManager::connect() { _scaleInLocation = _program.uniformLocation("scaleIn"); _hmdWarpParamLocation = _program.uniformLocation("hmdWarpParam"); } else { - _deviceManager.Clear(); - System::Destroy(); + ovr_Shutdown(); } #endif } diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 21b9d67f4d..795fdde7ff 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -55,11 +55,8 @@ private: static bool _isConnected; #ifdef HAVE_LIBOVR - static OVR::Ptr _deviceManager; - static OVR::Ptr _hmdDevice; - static OVR::Ptr _sensorDevice; - static OVR::SensorFusion* _sensorFusion; - static OVR::Util::Render::StereoConfig _stereoConfig; + static ovrHmd _ovrHmd; + static ovrHmdDesc _ovrHmdDesc; #endif }; From a4f5ce2215116e433f321b7bd36650de939e3d13 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 23 Jun 2014 17:48:57 -0700 Subject: [PATCH 02/67] added stats for audiomixer jitter buffers lengths in Application --- assignment-client/src/audio/AudioMixer.cpp | 15 ++++++++--- .../src/audio/AudioMixerClientData.cpp | 25 +++++++++++++++++++ .../src/audio/AudioMixerClientData.h | 15 ++++++++++- interface/src/Audio.cpp | 12 ++++++++- interface/src/Audio.h | 6 +++++ interface/src/ui/Stats.cpp | 20 +++++++++++++-- libraries/audio/src/AudioRingBuffer.cpp | 8 ++++-- .../audio/src/PositionalAudioRingBuffer.cpp | 2 +- .../audio/src/PositionalAudioRingBuffer.h | 1 + libraries/networking/src/PacketHeaders.cpp | 2 ++ 10 files changed, 96 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 42be1aea5a..4c2d7198c6 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -472,7 +472,7 @@ void AudioMixer::run() { QElapsedTimer timer; timer.start(); - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(AudioMixerJitterBuffersStats) + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -546,10 +546,19 @@ void AudioMixer::run() { && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); + // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); + char* dataAt = clientMixBuffer + numBytesPacketHeader; - memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + // calculate and pack the jitter buffer size stats for this node + AudioMixerJitterBuffersStats jitterBuffersStats; + ((AudioMixerClientData*)node->getLinkedData())->calculateJitterBuffersStats(jitterBuffersStats); + memcpy(dataAt, &jitterBuffersStats, sizeof(AudioMixerJitterBuffersStats)); + dataAt += sizeof(AudioMixerJitterBuffersStats); + + // pack mixed audio + memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); ++_sumListeners; } diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0c41cc70f9..ebe12ead65 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -15,6 +15,7 @@ #include #include "InjectedAudioRingBuffer.h" +#include "SharedUtil.h" #include "AudioMixerClientData.h" @@ -138,3 +139,27 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { i++; } } + +void AudioMixerClientData::calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const { + int avatarJitterBufferFrames = 0; + int maxJitterBufferFrames = 0; + int sumJitterBufferFrames = 0; + + for (int i = 0; i < _ringBuffers.size(); i++) { + + int bufferJitterFrames = _ringBuffers[i]->getCurrentJitterBufferFrames(); + if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) { + avatarJitterBufferFrames = bufferJitterFrames; + } + + if (bufferJitterFrames > maxJitterBufferFrames) { + maxJitterBufferFrames = bufferJitterFrames; + } + + sumJitterBufferFrames += bufferJitterFrames; + } + + stats.avatarJitterBufferFrames = avatarJitterBufferFrames; + stats.maxJitterBufferFrames = maxJitterBufferFrames; + stats.avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 3c4ddd3459..928f635643 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -14,10 +14,21 @@ #include #include -#include +#include "PositionalAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h" +class AudioMixerJitterBuffersStats { +public: + AudioMixerJitterBuffersStats() + : avatarJitterBufferFrames(0), maxJitterBufferFrames(0), avgJitterBufferFrames(0) + {} + + int avatarJitterBufferFrames; + int maxJitterBufferFrames; + float avgJitterBufferFrames; +}; + class AudioMixerClientData : public NodeData { public: AudioMixerClientData(); @@ -29,6 +40,8 @@ public: int parseData(const QByteArray& packet); void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); + + void calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const; private: QList _ringBuffers; }; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index a0a85f8888..234ec31407 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -102,7 +102,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _samplesPerScope(NETWORK_SAMPLES_PER_FRAME * _framesPerScope), _scopeInput(0), _scopeOutputLeft(0), - _scopeOutputRight(0) + _scopeOutputRight(0), + _audioMixerJitterBufferStats() { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -800,7 +801,16 @@ void Audio::toggleStereoInput() { } } +void Audio::parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray) { + + int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); + const char* dataAt = reinterpret_cast(audioByteArray.data() + numBytesPacketHeader); + + memcpy(&_audioMixerJitterBufferStats, dataAt, sizeof(AudioMixerJitterBuffersStats)); +} + void Audio::processReceivedAudio(const QByteArray& audioByteArray) { + parseAudioMixerJitterBuffersStats(audioByteArray); _ringBuffer.parseData(audioByteArray); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 74fc373cb0..8ba42b0c07 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -16,6 +16,7 @@ #include #include "InterfaceConfig.h" +#include "../../assignment-client/src/audio/AudioMixerClientData.h" #include #include @@ -102,6 +103,8 @@ public slots: float getInputVolume() const { return (_audioInput) ? _audioInput->volume() : 0.0f; } void setInputVolume(float volume) { if (_audioInput) _audioInput->setVolume(volume); } + const AudioMixerJitterBuffersStats& getAudioMixerJitterBuffersStats() const { return _audioMixerJitterBufferStats; } + signals: bool muteToggled(); void preProcessOriginalInboundAudio(unsigned int sampleTime, QByteArray& samples, const QAudioFormat& format); @@ -216,6 +219,8 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); + void parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray); + // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; @@ -233,6 +238,7 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; + AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; }; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index fa62ecdb9b..17938a0f56 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -286,7 +286,7 @@ void Stats::display( pingVoxel = totalPingVoxel/voxelServerCount; } - lines = _expanded ? 4 : 3; + lines = _expanded ? 6 : 5; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -302,7 +302,6 @@ void Stats::display( char audioPing[30]; sprintf(audioPing, "Audio ping: %d", pingAudio); - char avatarPing[30]; sprintf(avatarPing, "Avatar ping: %d", pingAvatar); @@ -324,10 +323,27 @@ void Stats::display( drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); } + static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; + + const AudioMixerJitterBuffersStats& audioMixerJitterBufferStats = + Application::getInstance()->getAudio()->getAudioMixerJitterBuffersStats(); + + char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; + char audioMixerJitterBuffersStats[30]; + sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats.avatarJitterBufferFrames * MSECS_PER_FRAME, + audioMixerJitterBufferStats.maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats.avgJitterBufferFrames * MSECS_PER_FRAME); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + + verticalOffset = 0; horizontalOffset = _lastHorizontalOffset + _generalStatsWidth + _pingStatsWidth + 2; } + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); glm::vec3 avatarPos = myAvatar->getPosition(); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 1b6bdaa5d8..8f7d3c5663 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -16,6 +16,7 @@ #include #include "PacketHeaders.h" +#include "../../../assignment-client/src/audio/AudioMixerClientData.h" #include "AudioRingBuffer.h" @@ -63,8 +64,11 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { } int AudioRingBuffer::parseData(const QByteArray& packet) { - int numBytesPacketHeader = numBytesForPacketHeader(packet); - return writeData(packet.data() + numBytesPacketHeader, packet.size() - numBytesPacketHeader); + int numBytesBeforeAudioData = numBytesForPacketHeader(packet); + if (packetTypeForPacket(packet) == PacketTypeMixedAudio) { + numBytesBeforeAudioData += sizeof(AudioMixerJitterBuffersStats); + } + return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } qint64 AudioRingBuffer::readSamples(int16_t* destination, qint64 maxSamples) { diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 0e2e785973..264af75753 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -232,7 +232,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + static const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 8c2122f29e..92f8ca9576 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -76,6 +76,7 @@ public: void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } + int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } protected: // disallow copying of PositionalAudioRingBuffer objects diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index e2bc46b3be..293b5b0dfb 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -51,6 +51,8 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeMicrophoneAudioWithEcho: case PacketTypeSilentAudioFrame: return 1; + case PacketTypeMixedAudio: + return 1; case PacketTypeAvatarData: return 3; case PacketTypeAvatarIdentity: From 254946be9e207923b34ee2be16313ff5b0edd50d Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 24 Jun 2014 09:27:41 -0700 Subject: [PATCH 03/67] removed duplicate method from merge --- libraries/audio/src/PositionalAudioRingBuffer.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 8020d9b338..be3390805c 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -79,7 +79,6 @@ public: int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } - int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } protected: // disallow copying of PositionalAudioRingBuffer objects From 8e2bb130f7303b4d095a38bb29ce3f9203928d56 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 24 Jun 2014 12:18:40 -0700 Subject: [PATCH 04/67] added extra sample in AudioRingBuffer... changed buffer overflow behavior to overwrite old data in writeData; added overflow behavior to drop silent samples in addSilentFrame --- libraries/audio/src/AudioRingBuffer.cpp | 57 +++++++++++++------ libraries/audio/src/AudioRingBuffer.h | 1 + .../audio/src/PositionalAudioRingBuffer.cpp | 15 ++--- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d0a19a592e..304fc6b2cf 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -28,10 +28,12 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : _hasStarted(false), _randomAccessMode(randomAccessMode) { + _arrayLength = _sampleCapacity + 1; + if (numFrameSamples) { - _buffer = new int16_t[_sampleCapacity]; + _buffer = new int16_t[_arrayLength]; if (_randomAccessMode) { - memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); + memset(_buffer, 0, _arrayLength * sizeof(int16_t)); } _nextOutput = _buffer; _endOfLastWrite = _buffer; @@ -55,9 +57,10 @@ void AudioRingBuffer::reset() { void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { delete[] _buffer; _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; - _buffer = new int16_t[_sampleCapacity]; + _arrayLength = _sampleCapacity + 1; + _buffer = new int16_t[_arrayLength]; if (_randomAccessMode) { - memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); + memset(_buffer, 0, _arrayLength * sizeof(int16_t)); } _nextOutput = _buffer; _endOfLastWrite = _buffer; @@ -87,11 +90,11 @@ qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { numReadSamples = _endOfLastWrite ? (maxSize / sizeof(int16_t)) : 0; } - if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { + if (_nextOutput + numReadSamples > _buffer + _arrayLength) { // we're going to need to do two reads to get this data, it wraps around the edge // read to the end of the buffer - int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; + int numSamplesToEnd = (_buffer + _arrayLength) - _nextOutput; memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); if (_randomAccessMode) { memset(_nextOutput, 0, numSamplesToEnd * sizeof(int16_t)); // clear it @@ -125,19 +128,28 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // otherwise we should not copy that data, and leave the buffer pointers where they are int samplesToCopy = std::min((quint64)(maxSize / sizeof(int16_t)), (quint64)_sampleCapacity); - + /* if (_hasStarted && samplesToCopy > _sampleCapacity - samplesAvailable()) { - // this read will cross the next output, so call us starved and reset the buffer - qDebug() << "Filled the ring buffer. Resetting."; + // this write would overflow the buffer, so call us starved and reset the buffer + qDebug() << "Overflowed the ring buffer. Resetting."; _endOfLastWrite = _buffer; _nextOutput = _buffer; _isStarved = true; + }*/ + + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (samplesToCopy > samplesRoomFor) { + // there's not enough room for this write. erase old data to make room for this new data + int samplesToDelete = samplesToCopy - samplesRoomFor; + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + qDebug() << "Overflowed ring buffer! Overwriting old data"; +printf("_nextOutput at index %d\n", _nextOutput - _buffer); } - if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { + if (_endOfLastWrite + samplesToCopy <= _buffer + _arrayLength) { memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _arrayLength) - _endOfLastWrite; memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } @@ -157,6 +169,8 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); +printf("\n<<< mixed! %d samples remaining\n", samplesAvailable()); +printf("_nextOutput at index %d\n", _nextOutput - _buffer); } unsigned int AudioRingBuffer::samplesAvailable() const { @@ -166,7 +180,7 @@ unsigned int AudioRingBuffer::samplesAvailable() const { int sampleDifference = _endOfLastWrite - _nextOutput; if (sampleDifference < 0) { - sampleDifference += _sampleCapacity; + sampleDifference += _arrayLength; } return sampleDifference; @@ -174,13 +188,22 @@ unsigned int AudioRingBuffer::samplesAvailable() const { } void AudioRingBuffer::addSilentFrame(int numSilentSamples) { + + int samplesRoomFor = _sampleCapacity - samplesAvailable(); + if (numSilentSamples > samplesRoomFor) { + // there's not enough room for this write. write as many silent samples as we have room for + numSilentSamples = samplesRoomFor; + qDebug() << "Dropping some silent samples to prevent ring buffer overflow"; +printf("_nextOutput at index %d\n", _nextOutput - _buffer); + } + // memset zeroes into the buffer, accomodate a wrap around the end // push the _endOfLastWrite to the correct spot - if (_endOfLastWrite + numSilentSamples <= _buffer + _sampleCapacity) { + if (_endOfLastWrite + numSilentSamples <= _buffer + _arrayLength) { memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); _endOfLastWrite += numSilentSamples; } else { - int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _arrayLength) - _endOfLastWrite; memset(_endOfLastWrite, 0, numSamplesToEnd * sizeof(int16_t)); memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); @@ -198,12 +221,12 @@ bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSa int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _arrayLength) { // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - _sampleCapacity; + return position + numSamplesShift - _arrayLength; } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { // this shift will go around to the end of the ring - return position + numSamplesShift + _sampleCapacity; + return position + numSamplesShift + _arrayLength; } else { return position + numSamplesShift; } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 04cc67c8ac..33f9ada0bd 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -82,6 +82,7 @@ protected: int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; int _sampleCapacity; + int _arrayLength; int _numFrameSamples; int16_t* _nextOutput; int16_t* _endOfLastWrite; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 764624cd39..acc1bc0ee4 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -148,6 +148,8 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { // there is audio data to read readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); } +printf("\n >>> parse data. %d samples available\n", samplesAvailable()); +printf("_endOfLastWrite at index %d\n", _endOfLastWrite - _buffer); return readBytes; } @@ -200,7 +202,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing - +printf("NOT MIXED! waiting to refill after starve\n"); if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } @@ -209,7 +211,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; - +printf("NOT MIXED! buffer is now starved\n"); // set to 0 to indicate the jitter buffer is starved _currentJitterBufferFrames = 0; @@ -225,6 +227,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { // minus one (since a frame will be read immediately after this) is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; _isStarved = false; +printf("buffer has been refilled. current jbuffer frames: %d\n", _currentJitterBufferFrames); } // since we've read data from ring buffer at least once - we've started @@ -237,12 +240,7 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { static const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { - - _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence - /* - const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - + if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; @@ -251,6 +249,5 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { if (_desiredJitterBufferFrames > maxDesired) { _desiredJitterBufferFrames = maxDesired; } - */ } } From 5fd1d0a87c4f42f38aa0cc37d4466033b2d799da Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 24 Jun 2014 16:51:51 -0700 Subject: [PATCH 05/67] added AudioRingBuffer unit test fixed bug with _endOfLastWrite not rolling over in addSilentFrame --- libraries/audio/src/AudioRingBuffer.cpp | 8 +- libraries/audio/src/AudioRingBuffer.h | 2 +- tests/audio/CMakeLists.txt | 40 +++++++ tests/audio/src/AudioRingBufferTests.cpp | 146 +++++++++++++++++++++++ tests/audio/src/AudioRingBufferTests.h | 25 ++++ tests/audio/src/main.cpp | 18 +++ 6 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 tests/audio/CMakeLists.txt create mode 100644 tests/audio/src/AudioRingBufferTests.cpp create mode 100644 tests/audio/src/AudioRingBufferTests.h create mode 100644 tests/audio/src/main.cpp diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 304fc6b2cf..ef21f208d0 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -187,7 +187,7 @@ unsigned int AudioRingBuffer::samplesAvailable() const { } } -void AudioRingBuffer::addSilentFrame(int numSilentSamples) { +int AudioRingBuffer::addSilentFrame(int numSilentSamples) { int samplesRoomFor = _sampleCapacity - samplesAvailable(); if (numSilentSamples > samplesRoomFor) { @@ -201,14 +201,14 @@ printf("_nextOutput at index %d\n", _nextOutput - _buffer); // push the _endOfLastWrite to the correct spot if (_endOfLastWrite + numSilentSamples <= _buffer + _arrayLength) { memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); - _endOfLastWrite += numSilentSamples; } else { int numSamplesToEnd = (_buffer + _arrayLength) - _endOfLastWrite; memset(_endOfLastWrite, 0, numSamplesToEnd * sizeof(int16_t)); memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); - - _endOfLastWrite = _buffer + (numSilentSamples - numSamplesToEnd); } + _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numSilentSamples); + + return numSilentSamples * sizeof(int16_t); } bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSamples) const { diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index 33f9ada0bd..abe9cb044c 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -73,7 +73,7 @@ public: bool hasStarted() const { return _hasStarted; } - void addSilentFrame(int numSilentSamples); + int addSilentFrame(int numSilentSamples); protected: // disallow copying of AudioRingBuffer objects AudioRingBuffer(const AudioRingBuffer&); diff --git a/tests/audio/CMakeLists.txt b/tests/audio/CMakeLists.txt new file mode 100644 index 0000000000..5c5178cb71 --- /dev/null +++ b/tests/audio/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME audio-tests) + +set(ROOT_DIR ../..) +set(MACRO_DIR ${ROOT_DIR}/cmake/macros) + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +#find_package(Qt5Network REQUIRED) +#find_package(Qt5Script REQUIRED) +#find_package(Qt5Widgets REQUIRED) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +include(${MACRO_DIR}/AutoMTC.cmake) +auto_mtc(${TARGET_NAME} ${ROOT_DIR}) + +#qt5_use_modules(${TARGET_NAME} Network Script Widgets) + +#include glm +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} ${ROOT_DIR}) + +# link in the shared libraries +include(${MACRO_DIR}/LinkHifiLibrary.cmake) +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) +link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) +link_hifi_library(networking ${TARGET_NAME} ${ROOT_DIR}) + +IF (WIN32) + target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) + diff --git a/tests/audio/src/AudioRingBufferTests.cpp b/tests/audio/src/AudioRingBufferTests.cpp new file mode 100644 index 0000000000..506e81e13e --- /dev/null +++ b/tests/audio/src/AudioRingBufferTests.cpp @@ -0,0 +1,146 @@ +// +// AudioRingBufferTests.cpp +// tests/audio/src +// +// Created by Yixin Wang on 6/24/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioRingBufferTests.h" + +#include "SharedUtil.h" + +void AudioRingBufferTests::assertBufferSize(const AudioRingBuffer& buffer, int samples) { + if (buffer.samplesAvailable() != samples) { + qDebug("Unexpected num samples available! Exptected: %d Actual: %d\n", samples, buffer.samplesAvailable()); + } +} + +void AudioRingBufferTests::runAllTests() { + + int16_t writeData[10000]; + for (int i = 0; i < 10000; i++) { writeData[i] = i; } + int writeIndexAt; + + int16_t readData[10000]; + int readIndexAt; + + + AudioRingBuffer ringBuffer(10); // makes buffer of 100 int16_t samples + for (int T = 0; T < 300; T++) { + + writeIndexAt = 0; + readIndexAt = 0; + + // write 73 samples, 73 samples in buffer + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 73) / sizeof(int16_t); + assertBufferSize(ringBuffer, 73); + + // read 43 samples, 30 samples in buffer + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 43) / sizeof(int16_t); + assertBufferSize(ringBuffer, 30); + + // write 70 samples, 100 samples in buffer (full) + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 70) / sizeof(int16_t); + assertBufferSize(ringBuffer, 100); + + // read 100 samples, 0 samples in buffer (empty) + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100) / sizeof(int16_t); + assertBufferSize(ringBuffer, 0); + + + // verify 143 samples of read data + for (int i = 0; i < 143; i++) { + if (readData[i] != i) { + qDebug("first readData[%d] incorrect! Expcted: %d Actual: %d", i, i, readData[i]); + return; + } + } + + + writeIndexAt = 0; + readIndexAt = 0; + + // write 59 samples, 59 samples in buffer + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 59) / sizeof(int16_t); + assertBufferSize(ringBuffer, 59); + + // write 99 samples, 100 samples in buffer + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 99) / sizeof(int16_t); + assertBufferSize(ringBuffer, 100); + + // read 100 samples, 0 samples in buffer + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 100) / sizeof(int16_t); + assertBufferSize(ringBuffer, 0); + + // verify 100 samples of read data + for (int i = 0; i < 100; i++) { + readData[i] = writeIndexAt - 100 + i; + } + + + + + writeIndexAt = 0; + readIndexAt = 0; + + // write 77 samples, 77 samples in buffer + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 77) / sizeof(int16_t); + assertBufferSize(ringBuffer, 77); + + // write 24 samples, 100 samples in buffer (overwrote one sample: "0") + writeIndexAt += ringBuffer.writeSamples(&writeData[writeIndexAt], 24) / sizeof(int16_t); + assertBufferSize(ringBuffer, 100); + + // write 29 silent samples, 100 samples in buffer, make sure non were added + int samplesWritten; + if ((samplesWritten = ringBuffer.addSilentFrame(29)) != 0) { + qDebug("addSilentFrame(29) incorrect! Expected: 0 Actual: %d", samplesWritten); + return; + } + assertBufferSize(ringBuffer, 100); + + // read 3 samples, 97 samples in buffer (expect to read "1", "2", "3") + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3) / sizeof(int16_t); + for (int i = 0; i < 3; i++) { + if (readData[i] != i + 1) { + qDebug("Second readData[%d] incorrect! Expcted: %d Actual: %d", i, i + 1, readData[i]); + return; + } + } + assertBufferSize(ringBuffer, 97); + + // write 4 silent samples, 100 samples in buffer + if ((samplesWritten = ringBuffer.addSilentFrame(4) / sizeof(int16_t)) != 3) { + qDebug("addSilentFrame(4) incorrect! Exptected: 3 Actual: %d", samplesWritten); + return; + } + assertBufferSize(ringBuffer, 100); + + // read back 97 samples (the non-silent samples), 3 samples in buffer (expect to read "4" thru "100") + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 97) / sizeof(int16_t); + for (int i = 3; i < 100; i++) { + if (readData[i] != i + 1) { + qDebug("third readData[%d] incorrect! Expcted: %d Actual: %d", i, i + 1, readData[i]); + return; + } + } + assertBufferSize(ringBuffer, 3); + + // read back 3 silent samples, 0 samples in buffer + readIndexAt += ringBuffer.readSamples(&readData[readIndexAt], 3) / sizeof(int16_t); + for (int i = 100; i < 103; i++) { + if (readData[i] != 0) { + qDebug("Fourth readData[%d] incorrect! Expcted: %d Actual: %d", i, 0, readData[i]); + return; + } + } + assertBufferSize(ringBuffer, 0); + } + + qDebug() << "PASSED"; +} + diff --git a/tests/audio/src/AudioRingBufferTests.h b/tests/audio/src/AudioRingBufferTests.h new file mode 100644 index 0000000000..20cbe74699 --- /dev/null +++ b/tests/audio/src/AudioRingBufferTests.h @@ -0,0 +1,25 @@ +// +// AudioRingBufferTests.h +// tests/audio/src +// +// Created by Yixin Wang on 6/24/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioRingBufferTests_h +#define hifi_AudioRingBufferTests_h + +#include "AudioRingBuffer.h" + + +namespace AudioRingBufferTests { + + void runAllTests(); + + void assertBufferSize(const AudioRingBuffer& buffer, int samples); +}; + +#endif // hifi_AudioRingBufferTests_h diff --git a/tests/audio/src/main.cpp b/tests/audio/src/main.cpp new file mode 100644 index 0000000000..bf798982a9 --- /dev/null +++ b/tests/audio/src/main.cpp @@ -0,0 +1,18 @@ +// +// main.cpp +// tests/audio/src +// +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioRingBufferTests.h" +#include + +int main(int argc, char** argv) { + AudioRingBufferTests::runAllTests(); + getchar(); + return 0; +} From e1d28dfe735aae5d329ec11c47e9449e036c15d8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 24 Jun 2014 17:14:20 -0700 Subject: [PATCH 06/67] removed debug code --- interface/src/ui/Stats.cpp | 3 +-- libraries/audio/src/AudioRingBuffer.cpp | 12 ------------ libraries/audio/src/PositionalAudioRingBuffer.cpp | 6 +----- libraries/audio/src/PositionalAudioRingBuffer.h | 4 ++-- 4 files changed, 4 insertions(+), 21 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 17938a0f56..f00d78ea0f 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -342,8 +342,7 @@ void Stats::display( verticalOffset = 0; horizontalOffset = _lastHorizontalOffset + _generalStatsWidth + _pingStatsWidth + 2; } - - + MyAvatar* myAvatar = Application::getInstance()->getAvatar(); glm::vec3 avatarPos = myAvatar->getPosition(); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index ef21f208d0..382c63cb4a 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -128,14 +128,6 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // otherwise we should not copy that data, and leave the buffer pointers where they are int samplesToCopy = std::min((quint64)(maxSize / sizeof(int16_t)), (quint64)_sampleCapacity); - /* - if (_hasStarted && samplesToCopy > _sampleCapacity - samplesAvailable()) { - // this write would overflow the buffer, so call us starved and reset the buffer - qDebug() << "Overflowed the ring buffer. Resetting."; - _endOfLastWrite = _buffer; - _nextOutput = _buffer; - _isStarved = true; - }*/ int samplesRoomFor = _sampleCapacity - samplesAvailable(); if (samplesToCopy > samplesRoomFor) { @@ -143,7 +135,6 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { int samplesToDelete = samplesToCopy - samplesRoomFor; _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); qDebug() << "Overflowed ring buffer! Overwriting old data"; -printf("_nextOutput at index %d\n", _nextOutput - _buffer); } if (_endOfLastWrite + samplesToCopy <= _buffer + _arrayLength) { @@ -169,8 +160,6 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); -printf("\n<<< mixed! %d samples remaining\n", samplesAvailable()); -printf("_nextOutput at index %d\n", _nextOutput - _buffer); } unsigned int AudioRingBuffer::samplesAvailable() const { @@ -194,7 +183,6 @@ int AudioRingBuffer::addSilentFrame(int numSilentSamples) { // there's not enough room for this write. write as many silent samples as we have room for numSilentSamples = samplesRoomFor; qDebug() << "Dropping some silent samples to prevent ring buffer overflow"; -printf("_nextOutput at index %d\n", _nextOutput - _buffer); } // memset zeroes into the buffer, accomodate a wrap around the end diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index acc1bc0ee4..5466c4ff02 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -148,8 +148,6 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { // there is audio data to read readBytes += writeData(packet.data() + readBytes, packet.size() - readBytes); } -printf("\n >>> parse data. %d samples available\n", samplesAvailable()); -printf("_endOfLastWrite at index %d\n", _endOfLastWrite - _buffer); return readBytes; } @@ -202,7 +200,6 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing -printf("NOT MIXED! waiting to refill after starve\n"); if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } @@ -211,7 +208,7 @@ printf("NOT MIXED! waiting to refill after starve\n"); } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; -printf("NOT MIXED! buffer is now starved\n"); + // set to 0 to indicate the jitter buffer is starved _currentJitterBufferFrames = 0; @@ -227,7 +224,6 @@ printf("NOT MIXED! buffer is now starved\n"); // minus one (since a frame will be read immediately after this) is the length of the jitter buffer _currentJitterBufferFrames = samplesAvailable() / samplesPerFrame - 1; _isStarved = false; -printf("buffer has been refilled. current jbuffer frames: %d\n", _currentJitterBufferFrames); } // since we've read data from ring buffer at least once - we've started diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index be3390805c..19e6a2bfc5 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -76,9 +76,9 @@ public: void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; } int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } - int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } - + int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } + int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } protected: // disallow copying of PositionalAudioRingBuffer objects From 99bc7f8c9dddc534a7ece0b3a3a49787cf550c3e Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 09:20:15 -0700 Subject: [PATCH 07/67] #include "" to #include <> --- assignment-client/src/audio/AudioMixerClientData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 40576c67b9..b135ade2b3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -14,8 +14,8 @@ #include #include +#include -#include "PositionalAudioRingBuffer.h" #include "AvatarAudioRingBuffer.h" class AudioMixerJitterBuffersStats { From 1257e0595a6ac93ab65814a28a37ce12b5a69603 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 09:52:02 -0700 Subject: [PATCH 08/67] moved jbufferstats class to PositionalAudioRingBuffers --- assignment-client/src/audio/AudioMixerClientData.cpp | 6 +++--- assignment-client/src/audio/AudioMixerClientData.h | 11 ----------- interface/src/Audio.h | 2 +- interface/src/ui/Stats.cpp | 4 ++-- libraries/audio/src/AudioRingBuffer.cpp | 2 +- libraries/audio/src/PositionalAudioRingBuffer.h | 11 +++++++++++ 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 9b95dfca28..d5bd9531bf 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -161,9 +161,9 @@ void AudioMixerClientData::calculateJitterBuffersStats(AudioMixerJitterBuffersSt sumJitterBufferFrames += bufferJitterFrames; } - stats.avatarJitterBufferFrames = avatarJitterBufferFrames; - stats.maxJitterBufferFrames = maxJitterBufferFrames; - stats.avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); + stats._avatarJitterBufferFrames = avatarJitterBufferFrames; + stats._maxJitterBufferFrames = maxJitterBufferFrames; + stats._avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); } QString AudioMixerClientData::getJitterBufferStats() const { diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index b135ade2b3..80f02a37b2 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -18,17 +18,6 @@ #include "AvatarAudioRingBuffer.h" -class AudioMixerJitterBuffersStats { -public: - AudioMixerJitterBuffersStats() - : avatarJitterBufferFrames(0), maxJitterBufferFrames(0), avgJitterBufferFrames(0) - {} - - int avatarJitterBufferFrames; - int maxJitterBufferFrames; - float avgJitterBufferFrames; -}; - class AudioMixerClientData : public NodeData { public: AudioMixerClientData(); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 8ba42b0c07..4ea64d8aaa 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -16,7 +16,7 @@ #include #include "InterfaceConfig.h" -#include "../../assignment-client/src/audio/AudioMixerClientData.h" +#include "PositionalAudioRingBuffer.h" #include #include diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index f00d78ea0f..569dec9d44 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -330,8 +330,8 @@ void Stats::display( char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; char audioMixerJitterBuffersStats[30]; - sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats.avatarJitterBufferFrames * MSECS_PER_FRAME, - audioMixerJitterBufferStats.maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats.avgJitterBufferFrames * MSECS_PER_FRAME); + sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats._avatarJitterBufferFrames * MSECS_PER_FRAME, + audioMixerJitterBufferStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats._avgJitterBufferFrames * MSECS_PER_FRAME); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 8033ac753f..9fc0609396 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -16,9 +16,9 @@ #include #include "PacketHeaders.h" -#include "../../../assignment-client/src/audio/AudioMixerClientData.h" #include "AudioRingBuffer.h" +#include "PositionalAudioRingBuffer.h" AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 003ed219ab..5a1cfe146d 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -43,6 +43,17 @@ private: bool _newWindowMaxGapAvailable; }; +class AudioMixerJitterBuffersStats { +public: + AudioMixerJitterBuffersStats() + : _avatarJitterBufferFrames(0), _maxJitterBufferFrames(0), _avgJitterBufferFrames(0) + {} + + int _avatarJitterBufferFrames; + int _maxJitterBufferFrames; + float _avgJitterBufferFrames; +}; + class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { From 591bfa14b99a60057b9751003aef0e33e0d3866d Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 11:38:56 -0700 Subject: [PATCH 09/67] added PacketTypeAudioStreamStats for j-buffer stats ...instead of packing that info into audio packets. --- assignment-client/src/audio/AudioMixer.cpp | 38 +++++++++++++------ assignment-client/src/audio/AudioMixer.h | 2 + .../src/audio/AudioMixerClientData.cpp | 17 ++++++++- .../src/audio/AudioMixerClientData.h | 8 +++- interface/src/Audio.cpp | 19 +++++----- interface/src/Audio.h | 5 +-- interface/src/DatagramProcessor.cpp | 5 ++- libraries/audio/src/AudioRingBuffer.cpp | 6 +-- libraries/audio/src/AudioStreamStats.h | 26 +++++++++++++ .../audio/src/PositionalAudioRingBuffer.h | 11 ------ libraries/networking/src/PacketHeaders.h | 2 +- 11 files changed, 93 insertions(+), 46 deletions(-) create mode 100644 libraries/audio/src/AudioStreamStats.h diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 9b84670fb8..e8f6c05a3e 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -75,7 +75,8 @@ AudioMixer::AudioMixer(const QByteArray& packet) : _sumListeners(0), _sumMixes(0), _sourceUnattenuatedZone(NULL), - _listenerUnattenuatedZone(NULL) + _listenerUnattenuatedZone(NULL), + _lastSendAudioStreamStatsTime(usecTimestampNow()) { } @@ -448,7 +449,7 @@ void AudioMixer::sendStatsPacket() { AudioMixerClientData* clientData = static_cast(node->getLinkedData()); if (clientData) { QString property = "jitterStats." + node->getUUID().toString(); - QString value = clientData->getJitterBufferStats(); + QString value = clientData->getJitterBufferStatsString(); statsObject2[qPrintable(property)] = value; somethingToSend = true; sizeOfStats += property.size() + value.size(); @@ -587,6 +588,17 @@ void AudioMixer::run() { ++framesSinceCutoffEvent; } + + const quint64 TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS = 1 * USECS_PER_SECOND; + + char audioStreamStatsPacket[MAX_PACKET_SIZE]; + bool sendAudioStreamStats = false; + quint64 now = usecTimestampNow(); + if (now - _lastSendAudioStreamStatsTime > TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS) { + _lastSendAudioStreamStatsTime = now; + sendAudioStreamStats = true; + } + foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { @@ -596,20 +608,22 @@ void AudioMixer::run() { int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); char* dataAt = clientMixBuffer + numBytesPacketHeader; - // calculate and pack the jitter buffer size stats for this node - AudioMixerJitterBuffersStats jitterBuffersStats; - ((AudioMixerClientData*)node->getLinkedData())->calculateJitterBuffersStats(jitterBuffersStats); - memcpy(dataAt, &jitterBuffersStats, sizeof(AudioMixerJitterBuffersStats)); - dataAt += sizeof(AudioMixerJitterBuffersStats); - - // pack mixed audio - memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + // pack mixed audio, send mixed audio packet + memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + + // send an audio stream stats packet if it's time + if (sendAudioStreamStats) { + int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData()) + ->encodeAudioStreamStatsPacket(audioStreamStatsPacket); + nodeList->writeDatagram(audioStreamStatsPacket, numBytesWritten, node); + } + ++_sumListeners; } } - + // push forward the next output pointers for any audio buffers we used foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getLinkedData()) { diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 8560978689..2c94f32edc 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -58,6 +58,8 @@ private: AABox* _sourceUnattenuatedZone; AABox* _listenerUnattenuatedZone; static bool _useDynamicJitterBuffers; + + quint64 _lastSendAudioStreamStatsTime; }; #endif // hifi_AudioMixer_h diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d5bd9531bf..5cb945f8d1 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -142,7 +142,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } } -void AudioMixerClientData::calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const { +void AudioMixerClientData::getJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const { int avatarJitterBufferFrames = 0; int maxJitterBufferFrames = 0; int sumJitterBufferFrames = 0; @@ -166,7 +166,20 @@ void AudioMixerClientData::calculateJitterBuffersStats(AudioMixerJitterBuffersSt stats._avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); } -QString AudioMixerClientData::getJitterBufferStats() const { +int AudioMixerClientData::encodeAudioStreamStatsPacket(char* packet) const { + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeAudioStreamStats); + char* dataAt = packet + numBytesPacketHeader; + + // pack jitter buffer stats + AudioMixerJitterBuffersStats jitterBufferStats; + getJitterBuffersStats(jitterBufferStats); + memcpy(dataAt, &jitterBufferStats, sizeof(AudioMixerJitterBuffersStats)); + dataAt += sizeof(AudioMixerJitterBuffersStats); + + return dataAt - packet; +} + +QString AudioMixerClientData::getJitterBufferStatsString() const { QString result; AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 80f02a37b2..cbf10f8fdc 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -17,6 +17,7 @@ #include #include "AvatarAudioRingBuffer.h" +#include "AudioStreamStats.h" class AudioMixerClientData : public NodeData { public: @@ -30,8 +31,11 @@ public: void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); - void calculateJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const; - QString getJitterBufferStats() const; + void getJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const; + + int encodeAudioStreamStatsPacket(char* packet) const; + + QString getJitterBufferStatsString() const; private: QList _ringBuffers; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 2cebc5b8b9..f66d11085e 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -708,6 +708,16 @@ void Audio::addReceivedAudioToBuffer(const QByteArray& audioByteArray) { Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::AUDIO).updateValue(audioByteArray.size()); } +void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { + + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const char* dataAt = packet.constData() + numBytesPacketHeader; + + // parse audio mixer jitter buffer stats + memcpy(&_audioMixerJitterBufferStats, dataAt, sizeof(AudioMixerJitterBuffersStats)); + dataAt += sizeof(AudioMixerJitterBuffersStats); +} + // NOTE: numSamples is the total number of single channel samples, since callers will always call this with stereo // data we know that we will have 2x samples for each stereo time sample at the format's sample rate void Audio::addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples) { @@ -806,16 +816,7 @@ void Audio::toggleStereoInput() { } } -void Audio::parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray) { - - int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); - const char* dataAt = reinterpret_cast(audioByteArray.data() + numBytesPacketHeader); - - memcpy(&_audioMixerJitterBufferStats, dataAt, sizeof(AudioMixerJitterBuffersStats)); -} - void Audio::processReceivedAudio(const QByteArray& audioByteArray) { - parseAudioMixerJitterBuffersStats(audioByteArray); _ringBuffer.parseData(audioByteArray); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 4ea64d8aaa..a69423fdfd 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -16,7 +16,7 @@ #include #include "InterfaceConfig.h" -#include "PositionalAudioRingBuffer.h" +#include "AudioStreamStats.h" #include #include @@ -77,6 +77,7 @@ public slots: void start(); void stop(); void addReceivedAudioToBuffer(const QByteArray& audioByteArray); + void parseAudioStreamStatsPacket(const QByteArray& packet); void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); @@ -219,8 +220,6 @@ private: void renderGrid(const float* color, int x, int y, int width, int height, int rows, int cols); void renderLineStrip(const float* color, int x, int y, int n, int offset, const QByteArray* byteArray); - void parseAudioMixerJitterBuffersStats(const QByteArray& audioByteArray); - // Audio scope data static const unsigned int NETWORK_SAMPLES_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; static const unsigned int DEFAULT_FRAMES_PER_SCOPE = 5; diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index 29528da126..a159af7be5 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -51,7 +51,10 @@ void DatagramProcessor::processDatagrams() { QMetaObject::invokeMethod(&application->_audio, "addReceivedAudioToBuffer", Qt::QueuedConnection, Q_ARG(QByteArray, incomingPacket)); break; - + case PacketTypeAudioStreamStats: + QMetaObject::invokeMethod(&application->_audio, "parseAudioStreamStatsPacket", Qt::QueuedConnection, + Q_ARG(QByteArray, incomingPacket)); + break; case PacketTypeParticleAddResponse: // this will keep creatorTokenIDs to IDs mapped correctly Particle::handleAddParticleResponse(incomingPacket); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 9fc0609396..fd9ba0150b 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -16,9 +16,8 @@ #include #include "PacketHeaders.h" - #include "AudioRingBuffer.h" -#include "PositionalAudioRingBuffer.h" + AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), @@ -69,9 +68,6 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { int AudioRingBuffer::parseData(const QByteArray& packet) { int numBytesBeforeAudioData = numBytesForPacketHeader(packet); - if (packetTypeForPacket(packet) == PacketTypeMixedAudio) { - numBytesBeforeAudioData += sizeof(AudioMixerJitterBuffersStats); - } return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h new file mode 100644 index 0000000000..067323bde4 --- /dev/null +++ b/libraries/audio/src/AudioStreamStats.h @@ -0,0 +1,26 @@ +// +// AudioStreamStats.h +// libraries/audio/src +// +// Created by Yixin Wang on 6/25/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioStreamStats_h +#define hifi_AudioStreamStats_h + +class AudioMixerJitterBuffersStats { +public: + AudioMixerJitterBuffersStats() + : _avatarJitterBufferFrames(0), _maxJitterBufferFrames(0), _avgJitterBufferFrames(0) + {} + + quint16 _avatarJitterBufferFrames; + quint16 _maxJitterBufferFrames; + float _avgJitterBufferFrames; +}; + +#endif // hifi_AudioStreamStats_h \ No newline at end of file diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 5a1cfe146d..003ed219ab 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -43,17 +43,6 @@ private: bool _newWindowMaxGapAvailable; }; -class AudioMixerJitterBuffersStats { -public: - AudioMixerJitterBuffersStats() - : _avatarJitterBufferFrames(0), _maxJitterBufferFrames(0), _avgJitterBufferFrames(0) - {} - - int _avatarJitterBufferFrames; - int _maxJitterBufferFrames; - float _avgJitterBufferFrames; -}; - class PositionalAudioRingBuffer : public AudioRingBuffer { public: enum Type { diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 0f87b0e607..83350a32d1 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -40,7 +40,7 @@ enum PacketType { PacketTypeCreateAssignment, PacketTypeDomainOAuthRequest, PacketTypeMuteEnvironment, - PacketTypeDataServerSend, // reusable + PacketTypeAudioStreamStats, PacketTypeDataServerConfirm, PacketTypeVoxelQuery, PacketTypeVoxelData, From dd1c157f3a299da5a5f4a5573afe7b599531bde6 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 11:42:03 -0700 Subject: [PATCH 10/67] reverted updateDesiredJitterBufferFrames to what Brad wrote --- .../audio/src/PositionalAudioRingBuffer.cpp | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 35fdc4772c..5899656657 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -247,16 +247,20 @@ int PositionalAudioRingBuffer::getCalculatedDesiredJitterBufferFrames() const { } void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { - static const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; + if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { + if (!_dynamicJitterBuffers) { + _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence + } else { + const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - if (_interframeTimeGapStats.hasNewWindowMaxGapAvailable()) { - _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); - if (_desiredJitterBufferFrames < 1) { - _desiredJitterBufferFrames = 1; - } - const int maxDesired = RING_BUFFER_LENGTH_FRAMES - 1; - if (_desiredJitterBufferFrames > maxDesired) { - _desiredJitterBufferFrames = maxDesired; + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); + if (_desiredJitterBufferFrames < 1) { + _desiredJitterBufferFrames = 1; + } + const int maxDesired = RING_BUFFER_LENGTH_FRAMES - 1; + if (_desiredJitterBufferFrames > maxDesired) { + _desiredJitterBufferFrames = maxDesired; + } } } } From c583f8b0237d16a6c330550adfa2329604423cd9 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 11:48:32 -0700 Subject: [PATCH 11/67] removed unnecessary changes --- assignment-client/src/audio/AudioMixer.cpp | 4 ---- assignment-client/src/audio/AudioMixerClientData.cpp | 1 - libraries/networking/src/PacketHeaders.cpp | 2 -- 3 files changed, 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index e8f6c05a3e..df904e7f60 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -604,15 +604,11 @@ void AudioMixer::run() { && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { prepareMixForListeningNode(node.data()); - // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); - char* dataAt = clientMixBuffer + numBytesPacketHeader; - // pack mixed audio, send mixed audio packet memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); - // send an audio stream stats packet if it's time if (sendAudioStreamStats) { int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData()) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 5cb945f8d1..25b2295822 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -15,7 +15,6 @@ #include #include "InjectedAudioRingBuffer.h" -#include "SharedUtil.h" #include "AudioMixer.h" #include "AudioMixerClientData.h" diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index 293b5b0dfb..e2bc46b3be 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -51,8 +51,6 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeMicrophoneAudioWithEcho: case PacketTypeSilentAudioFrame: return 1; - case PacketTypeMixedAudio: - return 1; case PacketTypeAvatarData: return 3; case PacketTypeAvatarIdentity: From 7dd7422cbf70347dc137e599ae4d319d3a73c5b6 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 11:57:45 -0700 Subject: [PATCH 12/67] removed more unneccesary changes --- assignment-client/src/audio/AudioMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index df904e7f60..f41c3392c7 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -519,7 +519,7 @@ void AudioMixer::run() { QElapsedTimer timer; timer.start(); - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(AudioMixerJitterBuffersStats) + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; From 9b8f8dbf706392a2acf1750d29b24cc910a1a3e1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 12:05:59 -0700 Subject: [PATCH 13/67] removed more more unnecessary changes --- libraries/audio/src/AudioStreamStats.h | 2 +- libraries/audio/src/PositionalAudioRingBuffer.cpp | 5 +++-- libraries/audio/src/PositionalAudioRingBuffer.h | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 067323bde4..38d74cbd1c 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -23,4 +23,4 @@ public: float _avgJitterBufferFrames; }; -#endif // hifi_AudioStreamStats_h \ No newline at end of file +#endif // hifi_AudioStreamStats_h diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 5899656657..1e26c708ca 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -203,6 +203,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { if (!isNotStarvedOrHasMinimumSamples(samplesPerFrame + desiredJitterBufferSamples)) { // if the buffer was starved, allow it to accrue at least the desired number of // jitter buffer frames before we start taking frames from it for mixing + if (_shouldOutputStarveDebug) { _shouldOutputStarveDebug = false; } @@ -211,7 +212,7 @@ bool PositionalAudioRingBuffer::shouldBeAddedToMix() { } else if (samplesAvailable() < samplesPerFrame) { // if the buffer doesn't have a full frame of samples to take for mixing, it is starved _isStarved = true; - + // set to 0 to indicate the jitter buffer is starved _currentJitterBufferFrames = 0; @@ -252,7 +253,7 @@ void PositionalAudioRingBuffer::updateDesiredJitterBufferFrames() { _desiredJitterBufferFrames = 1; // HACK to see if this fixes the audio silence } else { const float USECS_PER_FRAME = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * USECS_PER_SECOND / (float)SAMPLE_RATE; - + _desiredJitterBufferFrames = ceilf((float)_interframeTimeGapStats.getWindowMaxGap() / USECS_PER_FRAME); if (_desiredJitterBufferFrames < 1) { _desiredJitterBufferFrames = 1; diff --git a/libraries/audio/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h index 003ed219ab..b204dc766b 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -78,7 +78,6 @@ public: int getSamplesPerFrame() const { return _isStereo ? NETWORK_BUFFER_LENGTH_SAMPLES_STEREO : NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; } int getCalculatedDesiredJitterBufferFrames() const; /// returns what we would calculate our desired as if asked - int getDesiredJitterBufferFrames() const { return _desiredJitterBufferFrames; } int getCurrentJitterBufferFrames() const { return _currentJitterBufferFrames; } From b220b092f4eb5a1cd25175412bd3e55216a673dc Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 16:35:08 -0700 Subject: [PATCH 14/67] added SequenceNumberStats and unit test --- .../networking/src/SequenceNumbersStats.cpp | 156 +++++++++++ .../networking/src/SequenceNumbersStats.h | 43 +++ tests/networking/CMakeLists.txt | 39 +++ .../src/SequenceNumberStatsTests.cpp | 258 ++++++++++++++++++ .../networking/src/SequenceNumberStatsTests.h | 28 ++ tests/networking/src/main.cpp | 19 ++ 6 files changed, 543 insertions(+) create mode 100644 libraries/networking/src/SequenceNumbersStats.cpp create mode 100644 libraries/networking/src/SequenceNumbersStats.h create mode 100644 tests/networking/CMakeLists.txt create mode 100644 tests/networking/src/SequenceNumberStatsTests.cpp create mode 100644 tests/networking/src/SequenceNumberStatsTests.h create mode 100644 tests/networking/src/main.cpp diff --git a/libraries/networking/src/SequenceNumbersStats.cpp b/libraries/networking/src/SequenceNumbersStats.cpp new file mode 100644 index 0000000000..97a9ed6b03 --- /dev/null +++ b/libraries/networking/src/SequenceNumbersStats.cpp @@ -0,0 +1,156 @@ +// +// SequenceNumberStats.cpp +// libraries/networking/src +// +// Created by Yixin Wang on 6/25/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SequenceNumbersStats.h" + +#include + +SequenceNumberStats::SequenceNumberStats() + : _lastReceived(std::numeric_limits::max()), + _missingSet(), + _numReceived(0), + _numEarly(0), + _numLate(0), + _numLost(0), + _numRecovered(0), + _numDuplicate(0) +{ +} + +void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging) { + + static const int UINT16_RANGE = std::numeric_limits::max() + 1; + static const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work + + // determine our expected sequence number... handle rollover appropriately + quint16 expected = _numReceived > 0 ? _lastReceived + (quint16)1 : incoming; + + if (incoming == expected) { // on time + _lastReceived = incoming; + } else { // out of order + + if (wantExtraDebugging) { + qDebug() << "out of order... got:" << incoming << "expected:" << expected; + } + + int incomingInt = (int)incoming; + int expectedInt = (int)expected; + + // check if the gap between incoming and expected is reasonable, taking possible rollover into consideration + int absGap = std::abs(incomingInt - expectedInt); + if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) { + // rollover likely occurred between incoming and expected. + // correct the larger of the two so that it's within [-UINT16_RANGE, -1] while the other remains within [0, UINT16_RANGE-1] + if (incomingInt > expectedInt) { + incomingInt -= UINT16_RANGE; + } else { + expectedInt -= UINT16_RANGE; + } + } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { + // ignore packet if gap is unreasonable + qDebug() << "ignoring unreasonable packet... sequence:" << incoming + << "previous:" << _lastReceived; + return; + } + + // now that rollover has been corrected for (if it occurred), incoming and expected can be + // compared to each other directly, though one of them might be negative + if (incomingInt > expectedInt) { // early + if (wantExtraDebugging) { + qDebug() << "this packet is earlier than expected..."; + qDebug() << ">>>>>>>> missing gap=" << (incomingInt - expectedInt); + } + + _numEarly++; + _numLost += (incomingInt - expectedInt); + + // add all sequence numbers that were skipped to the missing sequence numbers list + for (int missingInt = expectedInt; missingInt < incomingInt; missingInt++) { + _missingSet.insert((quint16)(missingInt < 0 ? missingInt + UINT16_RANGE : missingInt)); + } + _lastReceived = incoming; + } else { // late + if (wantExtraDebugging) { + qDebug() << "this packet is later than expected..."; + } + _numLate++; + + // remove this from missing sequence number if it's in there + if (_missingSet.remove(incoming)) { + if (wantExtraDebugging) { + qDebug() << "found it in _missingSet"; + } + _numLost--; + _numRecovered++; + } else { + if (wantExtraDebugging) { + qDebug() << "sequence:" << incoming << "was NOT found in _missingSet and is probably a duplicate"; + } + _numDuplicate++; + } + + // do not update _incomingLastSequence; it shouldn't become smaller + } + } + _numReceived++; + + // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP + // will be removed. + if (_missingSet.size() > MAX_REASONABLE_SEQUENCE_GAP) { + if (wantExtraDebugging) { + qDebug() << "_missingSet is too large! size:" << _missingSet.size(); + } + + // some older sequence numbers may be from before a rollover point; this must be handled. + // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received + // before the most recent rollover. + int cutoff = (int)_lastReceived - MAX_REASONABLE_SEQUENCE_GAP; + if (cutoff >= 0) { + quint16 nonRolloverCutoff = (quint16)cutoff; + QSet::iterator i = _missingSet.begin(); + while (i != _missingSet.end()) { + quint16 missing = *i; + if (wantExtraDebugging) { + qDebug() << "checking item:" << missing << "is it in need of pruning?"; + qDebug() << "old age cutoff:" << nonRolloverCutoff; + } + + if (missing > _lastReceived || missing < nonRolloverCutoff) { + i = _missingSet.erase(i); + if (wantExtraDebugging) { + qDebug() << "pruning really old missing sequence:" << missing; + } + } else { + i++; + } + } + } else { + quint16 rolloverCutoff = (quint16)(cutoff + UINT16_RANGE); + QSet::iterator i = _missingSet.begin(); + while (i != _missingSet.end()) { + quint16 missing = *i; + if (wantExtraDebugging) { + qDebug() << "checking item:" << missing << "is it in need of pruning?"; + qDebug() << "old age cutoff:" << rolloverCutoff; + } + + if (missing > _lastReceived && missing < rolloverCutoff) { + i = _missingSet.erase(i); + if (wantExtraDebugging) { + qDebug() << "pruning really old missing sequence:" << missing; + } + } else { + i++; + } + } + } + } +} \ No newline at end of file diff --git a/libraries/networking/src/SequenceNumbersStats.h b/libraries/networking/src/SequenceNumbersStats.h new file mode 100644 index 0000000000..3e9ef6f0bc --- /dev/null +++ b/libraries/networking/src/SequenceNumbersStats.h @@ -0,0 +1,43 @@ +// +// SequenceNumberStats.h +// libraries/networking/src +// +// Created by Yixin Wang on 6/25/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SequenceNumberStats_h +#define hifi_SequenceNumberStats_h + +#include "SharedUtil.h" + +class SequenceNumberStats { +public: + SequenceNumberStats(); + + void sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging = false); + + quint32 getNumReceived() const { return _numReceived; } + quint32 getNumEarly() const { return _numEarly; } + quint32 getNumLate() const { return _numLate; } + quint32 getNumLost() const { return _numLost; } + quint32 getNumRecovered() const { return _numRecovered; } + quint32 getNumDuplicate() const { return _numDuplicate; } + const QSet& getMissingSet() const { return _missingSet; } + +private: + quint16 _lastReceived; + QSet _missingSet; + + quint32 _numReceived; + quint32 _numEarly; + quint32 _numLate; + quint32 _numLost; + quint32 _numRecovered; + quint32 _numDuplicate; +}; + +#endif // hifi_SequenceNumberStats_h diff --git a/tests/networking/CMakeLists.txt b/tests/networking/CMakeLists.txt new file mode 100644 index 0000000000..2e094d2ce7 --- /dev/null +++ b/tests/networking/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME networking-tests) + +set(ROOT_DIR ../..) +set(MACRO_DIR ${ROOT_DIR}/cmake/macros) + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +#find_package(Qt5Network REQUIRED) +#find_package(Qt5Script REQUIRED) +#find_package(Qt5Widgets REQUIRED) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +include(${MACRO_DIR}/AutoMTC.cmake) +auto_mtc(${TARGET_NAME} ${ROOT_DIR}) + +#qt5_use_modules(${TARGET_NAME} Network Script Widgets) + +#include glm +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} ${ROOT_DIR}) + +# link in the shared libraries +include(${MACRO_DIR}/LinkHifiLibrary.cmake) +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) +link_hifi_library(networking ${TARGET_NAME} ${ROOT_DIR}) + +IF (WIN32) + target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) + diff --git a/tests/networking/src/SequenceNumberStatsTests.cpp b/tests/networking/src/SequenceNumberStatsTests.cpp new file mode 100644 index 0000000000..56da8fbaf9 --- /dev/null +++ b/tests/networking/src/SequenceNumberStatsTests.cpp @@ -0,0 +1,258 @@ +// +// AudioRingBufferTests.cpp +// tests/networking/src +// +// Created by Yixin Wang on 6/24/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SequenceNumberStatsTests.h" + +#include "SharedUtil.h" +#include + + +void SequenceNumberStatsTests::runAllTests() { + + rolloverTest(); + earlyLateTest(); + duplicateTest(); + pruneTest(); +} + +const int UINT16_RANGE = std::numeric_limits::max() + 1; + + +void SequenceNumberStatsTests::rolloverTest() { + + SequenceNumberStats stats; + + // insert enough samples to cause 3 rollovers + quint16 seq = 79; // start on some random number + for (int i = 0; i < 3 * UINT16_RANGE; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == 0); + assert(stats.getNumLate() == 0); + assert(stats.getNumLost() == 0); + assert(stats.getNumReceived() == i+1); + assert(stats.getNumRecovered() == 0); + } +} + +void SequenceNumberStatsTests::earlyLateTest() { + + SequenceNumberStats stats; + quint16 seq = 65530; + int numSent = 0; + + int numEarly = 0; + int numLate = 0; + int numLost = 0; + int numRecovered = 0; + + + for (int T = 0; T < 10000; T++) { + + // insert 7 consecutive + for (int i = 0; i < 7; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } + + // skip 10 + quint16 skipped = seq; + seq = seq + (quint16)10; + + // insert 36 consecutive + numEarly++; + numLost += 10; + for (int i = 0; i < 36; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } + + // send ones we skipped + for (int i = 0; i < 10; i++) { + stats.sequenceNumberReceived(skipped); + skipped = skipped + (quint16)1; + numSent++; + numLate++; + numLost--; + numRecovered++; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } + } +} + +void SequenceNumberStatsTests::duplicateTest() { + + SequenceNumberStats stats; + quint16 seq = 12345; + int numSent = 0; + + int numDuplicate = 0; + int numEarly = 0; + int numLate = 0; + int numLost = 0; + + + for (int T = 0; T < 10000; T++) { + + quint16 duplicate = seq; + + // insert 7 consecutive + for (int i = 0; i < 7; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + + // skip 10 + quint16 skipped = seq; + seq = seq + (quint16)10; + + + quint16 duplicate2 = seq; + + numEarly++; + numLost += 10; + // insert 36 consecutive + for (int i = 0; i < 36; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + + // send 5 duplicates from before skip + for (int i = 0; i < 5; i++) { + stats.sequenceNumberReceived(duplicate); + duplicate = duplicate + (quint16)1; + numSent++; + numDuplicate++; + numLate++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + + // send 5 duplicates from after skip + for (int i = 0; i < 5; i++) { + stats.sequenceNumberReceived(duplicate2); + duplicate2 = duplicate2 + (quint16)1; + numSent++; + numDuplicate++; + numLate++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + } +} + +void SequenceNumberStatsTests::pruneTest() { + + SequenceNumberStats stats; + quint16 seq = 54321; + int numSent = 0; + + int numEarly = 0; + int numLate = 0; + int numLost = 0; + + for (int T = 0; T < 1000; T++) { + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + // skip 1000 seq + seq = seq + (quint16)1000; + quint16 highestSkipped = seq - (quint16)1; + + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + numEarly++; + numLost += 1000; + + // skip 10 seq + seq = seq + (quint16)10; + quint16 highestSkipped2 = seq - (quint16)1; + + // insert 1 seq + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + numEarly++; + numLost += 10; + + const QSet& missingSet = stats.getMissingSet(); + assert(missingSet.size() <= 1000); + + for (int i = 0; i < 10; i++) { + assert(missingSet.contains(highestSkipped2)); + highestSkipped2 = highestSkipped2 - (quint16)1; + } + + for (int i = 0; i < 989; i++) { + assert(missingSet.contains(highestSkipped)); + highestSkipped = highestSkipped - (quint16)1; + } + for (int i = 0; i < 11; i++) { + assert(!missingSet.contains(highestSkipped)); + highestSkipped = highestSkipped - (quint16)1; + } + } +} \ No newline at end of file diff --git a/tests/networking/src/SequenceNumberStatsTests.h b/tests/networking/src/SequenceNumberStatsTests.h new file mode 100644 index 0000000000..b57bf69342 --- /dev/null +++ b/tests/networking/src/SequenceNumberStatsTests.h @@ -0,0 +1,28 @@ +// +// AudioRingBufferTests.h +// tests/networking/src +// +// Created by Yixin Wang on 6/24/2014 +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SequenceNumberStatsTests_h +#define hifi_SequenceNumberStatsTests_h + +#include "SequenceNumberStatsTests.h" +#include "SequenceNumbersStats.h" + +namespace SequenceNumberStatsTests { + + void runAllTests(); + + void rolloverTest(); + void earlyLateTest(); + void duplicateTest(); + void pruneTest(); +}; + +#endif // hifi_SequenceNumberStatsTests_h diff --git a/tests/networking/src/main.cpp b/tests/networking/src/main.cpp new file mode 100644 index 0000000000..91a59a0e41 --- /dev/null +++ b/tests/networking/src/main.cpp @@ -0,0 +1,19 @@ +// +// main.cpp +// tests/networking/src +// +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SequenceNumberStatsTests.h" +#include + +int main(int argc, char** argv) { + SequenceNumberStatsTests::runAllTests(); + printf("tests passed! press enter to exit"); + getchar(); + return 0; +} From 2ec04fb7563273adfd853e5fe6a421ae44c08126 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 17:20:57 -0700 Subject: [PATCH 15/67] replaced old seqnum tracking code with SequenceNumberStats --- .../octree/OctreeInboundPacketProcessor.cpp | 75 +------- .../src/octree/OctreeInboundPacketProcessor.h | 12 +- interface/src/Application.cpp | 2 +- interface/src/ui/OctreeStatsDialog.cpp | 17 +- .../networking/src/SequenceNumbersStats.cpp | 5 +- .../networking/src/SequenceNumbersStats.h | 3 + libraries/octree/src/OctreeSceneStats.cpp | 171 +----------------- libraries/octree/src/OctreeSceneStats.h | 20 +- tests/audio/src/main.cpp | 1 + 9 files changed, 37 insertions(+), 269 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 76a6845342..eaea2650b8 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -206,7 +206,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { } const SharedNodePointer& destinationNode = NodeList::getInstance()->getNodeHash().value(nodeUUID); - const QSet& missingSequenceNumbers = nodeStats.getMissingSequenceNumbers(); + const QSet& missingSequenceNumbers = nodeStats.getSequenceNumberStats().getMissingSet(); // construct nack packet(s) for this node int numSequenceNumbersAvailable = missingSequenceNumbers.size(); @@ -254,8 +254,7 @@ SingleSenderStats::SingleSenderStats() _totalLockWaitTime(0), _totalElementsInPacket(0), _totalPackets(0), - _incomingLastSequence(0), - _missingSequenceNumbers() + _sequenceNumberStats() { } @@ -263,74 +262,8 @@ SingleSenderStats::SingleSenderStats() void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { - const int UINT16_RANGE = std::numeric_limits::max() + 1; - const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work - const int MAX_MISSING_SEQUENCE_SIZE = 100; - - unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + (unsigned short int)1; - - if (incomingSequence == expectedSequence) { // on time - _incomingLastSequence = incomingSequence; - } else { // out of order - int incoming = (int)incomingSequence; - int expected = (int)expectedSequence; - - // check if the gap between incoming and expected is reasonable, taking possible rollover into consideration - int absGap = std::abs(incoming - expected); - if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) { - // rollover likely occurred between incoming and expected. - // correct the larger of the two so that it's within [-UINT16_RANGE, -1] while the other remains within [0, UINT16_RANGE-1] - if (incoming > expected) { - incoming -= UINT16_RANGE; - } else { - expected -= UINT16_RANGE; - } - } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { - // ignore packet if gap is unreasonable - qDebug() << "ignoring unreasonable packet... sequence:" << incomingSequence - << "_incomingLastSequence:" << _incomingLastSequence; - return; - } - - // now that rollover has been corrected for (if it occurred), incoming and expected can be - // compared to each other directly, though one of them might be negative - if (incoming > expected) { // early - // add all sequence numbers that were skipped to the missing sequence numbers list - for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { - _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); - } - _incomingLastSequence = incomingSequence; - } else { // late - // remove this from missing sequence number if it's in there - _missingSequenceNumbers.remove(incomingSequence); - - // do not update _incomingLastSequence; it shouldn't become smaller - } - } - - // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP - // will be removed. - if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { - // some older sequence numbers may be from before a rollover point; this must be handled. - // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received - // before the most recent rollover. - int cutoff = (int)_incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP; - if (cutoff >= 0) { - foreach(unsigned short int missingSequence, _missingSequenceNumbers) { - unsigned short int nonRolloverCutoff = (unsigned short int)cutoff; - if (missingSequence > _incomingLastSequence || missingSequence <= nonRolloverCutoff) { - _missingSequenceNumbers.remove(missingSequence); - } - } - } else { - unsigned short int rolloverCutoff = (unsigned short int)(cutoff + UINT16_RANGE); - foreach(unsigned short int missingSequence, _missingSequenceNumbers) { - if (missingSequence > _incomingLastSequence && missingSequence <= rolloverCutoff) { - _missingSequenceNumbers.remove(missingSequence); - } - } - } - } + // track sequence number + _sequenceNumberStats.sequenceNumberReceived(incomingSequence); // update other stats _totalTransitTime += transitTime; diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 46a57205cb..1c4f00c0a9 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -14,9 +14,10 @@ #ifndef hifi_OctreeInboundPacketProcessor_h #define hifi_OctreeInboundPacketProcessor_h -#include - #include + +#include "SequenceNumbersStats.h" + class OctreeServer; class SingleSenderStats { @@ -32,7 +33,8 @@ public: { return _totalElementsInPacket == 0 ? 0 : _totalProcessTime / _totalElementsInPacket; } quint64 getAverageLockWaitTimePerElement() const { return _totalElementsInPacket == 0 ? 0 : _totalLockWaitTime / _totalElementsInPacket; } - const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } + + const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); @@ -42,9 +44,7 @@ public: quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; - - unsigned short int _incomingLastSequence; - QSet _missingSequenceNumbers; + SequenceNumberStats _sequenceNumberStats; }; typedef QHash NodeToSenderStatsMap; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 75c3f442ca..f40185f34f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2164,7 +2164,7 @@ int Application::sendNackPackets() { OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; // make copy of missing sequence numbers from stats - const QSet missingSequenceNumbers = stats.getMissingSequenceNumbers(); + const QSet missingSequenceNumbers = stats.getSequenceNumberStats().getMissingSet(); _octreeSceneStatsLock.unlock(); diff --git a/interface/src/ui/OctreeStatsDialog.cpp b/interface/src/ui/OctreeStatsDialog.cpp index 3296d8ccb2..6303ff25d1 100644 --- a/interface/src/ui/OctreeStatsDialog.cpp +++ b/interface/src/ui/OctreeStatsDialog.cpp @@ -365,13 +365,14 @@ void OctreeStatsDialog::showOctreeServersOfType(int& serverCount, NodeType_t ser QString incomingPacketsString = locale.toString((uint)stats.getIncomingPackets()); QString incomingBytesString = locale.toString((uint)stats.getIncomingBytes()); QString incomingWastedBytesString = locale.toString((uint)stats.getIncomingWastedBytes()); - QString incomingOutOfOrderString = locale.toString((uint)stats.getIncomingOutOfOrder()); - QString incomingLateString = locale.toString((uint)stats.getIncomingLate()); - QString incomingReallyLateString = locale.toString((uint)stats.getIncomingReallyLate()); - QString incomingEarlyString = locale.toString((uint)stats.getIncomingEarly()); - QString incomingLikelyLostString = locale.toString((uint)stats.getIncomingLikelyLost()); - QString incomingRecovered = locale.toString((uint)stats.getIncomingRecovered()); - QString incomingDuplicateString = locale.toString((uint)stats.getIncomingPossibleDuplicate()); + const SequenceNumberStats& seqStats = stats.getSequenceNumberStats(); + QString incomingOutOfOrderString = locale.toString((uint)seqStats.getNumOutOfOrder()); + QString incomingLateString = locale.toString((uint)seqStats.getNumLate()); + QString incomingUnreasonableString = locale.toString((uint)seqStats.getNumUnreasonable()); + QString incomingEarlyString = locale.toString((uint)seqStats.getNumEarly()); + QString incomingLikelyLostString = locale.toString((uint)seqStats.getNumLost()); + QString incomingRecovered = locale.toString((uint)seqStats.getNumRecovered()); + QString incomingDuplicateString = locale.toString((uint)seqStats.getNumDuplicate()); int clockSkewInMS = node->getClockSkewUsec() / (int)USECS_PER_MSEC; QString incomingFlightTimeString = locale.toString((int)stats.getIncomingFlightTimeAverage()); @@ -385,7 +386,7 @@ void OctreeStatsDialog::showOctreeServersOfType(int& serverCount, NodeType_t ser serverDetails << "
" << " Out of Order: " << qPrintable(incomingOutOfOrderString) << "/ Early: " << qPrintable(incomingEarlyString) << "/ Late: " << qPrintable(incomingLateString) << - "/ Really Late: " << qPrintable(incomingReallyLateString) << + "/ Unreasonable: " << qPrintable(incomingUnreasonableString) << "/ Duplicate: " << qPrintable(incomingDuplicateString); serverDetails << "
" << diff --git a/libraries/networking/src/SequenceNumbersStats.cpp b/libraries/networking/src/SequenceNumbersStats.cpp index 97a9ed6b03..b1b6adc9ac 100644 --- a/libraries/networking/src/SequenceNumbersStats.cpp +++ b/libraries/networking/src/SequenceNumbersStats.cpp @@ -17,6 +17,7 @@ SequenceNumberStats::SequenceNumberStats() : _lastReceived(std::numeric_limits::max()), _missingSet(), _numReceived(0), + _numUnreasonable(0), _numEarly(0), _numLate(0), _numLost(0), @@ -30,6 +31,8 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa static const int UINT16_RANGE = std::numeric_limits::max() + 1; static const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work + _numReceived++; + // determine our expected sequence number... handle rollover appropriately quint16 expected = _numReceived > 0 ? _lastReceived + (quint16)1 : incoming; @@ -58,6 +61,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa // ignore packet if gap is unreasonable qDebug() << "ignoring unreasonable packet... sequence:" << incoming << "previous:" << _lastReceived; + _numUnreasonable++; return; } @@ -100,7 +104,6 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa // do not update _incomingLastSequence; it shouldn't become smaller } } - _numReceived++; // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP // will be removed. diff --git a/libraries/networking/src/SequenceNumbersStats.h b/libraries/networking/src/SequenceNumbersStats.h index 3e9ef6f0bc..8dbb876a5b 100644 --- a/libraries/networking/src/SequenceNumbersStats.h +++ b/libraries/networking/src/SequenceNumbersStats.h @@ -21,6 +21,8 @@ public: void sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging = false); quint32 getNumReceived() const { return _numReceived; } + quint32 getNumUnreasonable() const { return _numUnreasonable; } + quint32 getNumOutOfOrder() const { return _numEarly + _numLate; } quint32 getNumEarly() const { return _numEarly; } quint32 getNumLate() const { return _numLate; } quint32 getNumLost() const { return _numLost; } @@ -33,6 +35,7 @@ private: QSet _missingSet; quint32 _numReceived; + quint32 _numUnreasonable; quint32 _numEarly; quint32 _numLate; quint32 _numLost; diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 28445ec327..5fbc4b0945 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -39,14 +39,7 @@ OctreeSceneStats::OctreeSceneStats() : _incomingPacket(0), _incomingBytes(0), _incomingWastedBytes(0), - _incomingLastSequence(0), - _incomingLikelyLost(0), - _incomingRecovered(0), - _incomingEarly(0), - _incomingLate(0), - _incomingReallyLate(0), - _incomingPossibleDuplicate(0), - _missingSequenceNumbers(), + _sequenceNumberStats(), _incomingFlightTimeAverage(samples), _jurisdictionRoot(NULL) { @@ -150,15 +143,8 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingPacket = other._incomingPacket; _incomingBytes = other._incomingBytes; _incomingWastedBytes = other._incomingWastedBytes; - _incomingLastSequence = other._incomingLastSequence; - _incomingLikelyLost = other._incomingLikelyLost; - _incomingRecovered = other._incomingRecovered; - _incomingEarly = other._incomingEarly; - _incomingLate = other._incomingLate; - _incomingReallyLate = other._incomingReallyLate; - _incomingPossibleDuplicate = other._incomingPossibleDuplicate; - - _missingSequenceNumbers = other._missingSequenceNumbers; + + _sequenceNumberStats = other._sequenceNumberStats; } @@ -875,155 +861,8 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "ignoring unreasonable packet... flightTime:" << flightTime; return; // ignore any packets that are unreasonable } - - const int UINT16_RANGE = std::numeric_limits::max() + 1; - - // determine our expected sequence number... handle rollover appropriately - OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + (quint16)1 : sequence; - - const int USECS_PER_MSEC = 1000; - float flightTimeMsecs = flightTime / USECS_PER_MSEC; - _incomingFlightTimeAverage.updateAverage(flightTimeMsecs); - - // track out of order and possibly lost packets... - if (sequence == _incomingLastSequence) { - if (wantExtraDebugging) { - qDebug() << "last packet duplicate got:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; - } - } else { - if (sequence != expected) { - if (wantExtraDebugging) { - qDebug() << "out of order... got:" << sequence << "expected:" << expected; - } - - int sequenceInt = (int)sequence; - int expectedInt = (int)expected; - - // if distance between sequence and expected are more than half of the total range of possible seq numbers, - // assume that a rollover occurred between the two. - // correct the larger one so it's in the range [-UINT16_RANGE, -1] while the other remains in [0, UINT16_RANGE-1] - // after doing so, sequenceInt and expectedInt can be correctly compared to each other, though one may be negative - if (std::abs(sequenceInt - expectedInt) > UINT16_RANGE / 2) { - if (sequenceInt > expectedInt) { - sequenceInt -= UINT16_RANGE; - } - else { - expectedInt -= UINT16_RANGE; - } - } - - // Guard against possible corrupted packets... with bad sequence numbers - const int MAX_RESONABLE_SEQUENCE_OFFSET = 2000; - const int MIN_RESONABLE_SEQUENCE_OFFSET = -2000; - - int sequenceOffset = (sequenceInt - expectedInt); - if (sequenceOffset > MAX_RESONABLE_SEQUENCE_OFFSET || sequenceOffset < MIN_RESONABLE_SEQUENCE_OFFSET) { - qDebug() << "ignoring unreasonable packet... sequence:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; - return; // ignore any packets that are unreasonable - } - - // if the sequence is less than our expected, then this might be a packet - // that was delayed and so we should find it in our lostSequence list - if (sequenceInt < expectedInt) { - - // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] - // if rollover between them: sequenceInt in [-UINT16_RANGE, -1], expectedInt in [0, UINT16_RANGE-1] - - if (wantExtraDebugging) { - qDebug() << "this packet is later than expected..."; - } - if (sequenceInt < expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { - _incomingReallyLate++; - } - else { - _incomingLate++; - } - - if (_missingSequenceNumbers.contains(sequence)) { - if (wantExtraDebugging) { - qDebug() << "found it in _missingSequenceNumbers"; - } - _missingSequenceNumbers.remove(sequence); - _incomingLikelyLost--; - _incomingRecovered++; - } - else { - // if we're still in our pruning window, and we didn't find it in our missing list, - // than this is really unexpected and can probably only happen if the packet was a - // duplicate - if (sequenceInt >= expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { - if (wantExtraDebugging) { - qDebug() << "sequence:" << sequence << "WAS NOT found in _missingSequenceNumbers, and not that old... (expected - MAX_MISSING_SEQUENCE_OLD_AGE):" - << (uint16_t)(expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE); - } - _incomingPossibleDuplicate++; - } - } - - // don't update _incomingLastSequence in this case. - // only bump the last sequence if it was greater than our expected sequence, this will keep us from - // accidentally going backwards when an out of order (recovered) packet comes in - - } else { // sequenceInt > expectedInt - - // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] - // if rollover between them: sequenceInt in [0, UINT16_RANGE-1], expectedInt in [-UINT16_RANGE, -1] - - if (wantExtraDebugging) { - qDebug() << "this packet is earlier than expected..."; - } - _incomingEarly++; - - // hmm... so, we either didn't get some packets, or this guy came early... - int missing = sequenceInt - expectedInt; - if (wantExtraDebugging) { - qDebug() << ">>>>>>>> missing gap=" << missing; - } - _incomingLikelyLost += missing; - for (int missingSequenceInt = expectedInt; missingSequenceInt < sequenceInt; missingSequenceInt++) { - OCTREE_PACKET_SEQUENCE missingSequence = missingSequenceInt >= 0 ? missingSequenceInt : missingSequenceInt + UINT16_RANGE; - _missingSequenceNumbers << missingSequence; - } - - _incomingLastSequence = sequence; - } - } else { // sequence = expected - - _incomingLastSequence = sequence; - } - } - - // do some garbage collecting on our _missingSequenceNumbers - if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE) { - if (wantExtraDebugging) { - qDebug() << "too many _missingSequenceNumbers:" << _missingSequenceNumbers.size(); - } - - int oldAgeCutoff = (int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE; - - foreach(uint16_t missingItem, _missingSequenceNumbers) { - if (wantExtraDebugging) { - qDebug() << "checking item:" << missingItem << "is it in need of pruning?"; - qDebug() << "(_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE):" - << (uint16_t)((int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE); - } - - bool prune; - if (oldAgeCutoff >= 0) { - prune = (missingItem <= oldAgeCutoff || missingItem > _incomingLastSequence); - } - else { - prune = (missingItem <= oldAgeCutoff + UINT16_RANGE && missingItem > _incomingLastSequence); - } - - if (prune) { - if (wantExtraDebugging) { - qDebug() << "pruning really old missing sequence:" << missingItem; - } - _missingSequenceNumbers.remove(missingItem); - } - } - } + + _sequenceNumberStats.sequenceNumberReceived(sequence); // track packets here... _incomingPacket++; diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index 1c468a8dc6..173f4c225f 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -17,6 +17,7 @@ #include #include "JurisdictionMap.h" #include "OctreePacketData.h" +#include "SequenceNumbersStats.h" #define GREENISH 0x40ff40d0 #define YELLOWISH 0xffef40c0 @@ -164,16 +165,9 @@ public: quint32 getIncomingPackets() const { return _incomingPacket; } quint64 getIncomingBytes() const { return _incomingBytes; } quint64 getIncomingWastedBytes() const { return _incomingWastedBytes; } - quint32 getIncomingOutOfOrder() const { return _incomingLate + _incomingEarly; } - quint32 getIncomingLikelyLost() const { return _incomingLikelyLost; } - quint32 getIncomingRecovered() const { return _incomingRecovered; } - quint32 getIncomingEarly() const { return _incomingEarly; } - quint32 getIncomingLate() const { return _incomingLate; } - quint32 getIncomingReallyLate() const { return _incomingReallyLate; } - quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } - const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } + const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } private: @@ -268,14 +262,8 @@ private: quint64 _incomingBytes; quint64 _incomingWastedBytes; - quint16 _incomingLastSequence; /// last incoming sequence number - quint32 _incomingLikelyLost; /// count of packets likely lost, may be off by _incomingReallyLate count - quint32 _incomingRecovered; /// packets that were late, and we had in our missing list, we consider recovered - quint32 _incomingEarly; /// out of order earlier than expected - quint32 _incomingLate; /// out of order later than expected - quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late - quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate - QSet _missingSequenceNumbers; + SequenceNumberStats _sequenceNumberStats; + SimpleMovingAverage _incomingFlightTimeAverage; // features related items diff --git a/tests/audio/src/main.cpp b/tests/audio/src/main.cpp index bf798982a9..10f1a2e522 100644 --- a/tests/audio/src/main.cpp +++ b/tests/audio/src/main.cpp @@ -13,6 +13,7 @@ int main(int argc, char** argv) { AudioRingBufferTests::runAllTests(); + printf("all tests passed. press enter to exit\n"); getchar(); return 0; } From 37ce6931532f5c40e0851587559f7b36e7f927c3 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 18:01:34 -0700 Subject: [PATCH 16/67] added seqnum stats to AudioMixerClientData; added seqnums to Audio.cpp --- .../src/audio/AudioMixerClientData.cpp | 7 +++++++ assignment-client/src/audio/AudioMixerClientData.h | 4 ++++ interface/src/Audio.cpp | 13 +++++++++++-- interface/src/Audio.h | 2 ++ libraries/audio/src/AudioRingBuffer.cpp | 2 +- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 25b2295822..f35034eab3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -44,6 +44,13 @@ AvatarAudioRingBuffer* AudioMixerClientData::getAvatarAudioRingBuffer() const { } int AudioMixerClientData::parseData(const QByteArray& packet) { + + // parse sequence number for this packet + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const char* sequenceAt = packet.constData() + numBytesPacketHeader; + quint16 sequence = *(reinterpret_cast(sequenceAt)); + _sequenceNumberStats.sequenceNumberReceived(sequence); + PacketType packetType = packetTypeForPacket(packet); if (packetType == PacketTypeMicrophoneAudioWithEcho || packetType == PacketTypeMicrophoneAudioNoEcho diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index cbf10f8fdc..508572db43 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -18,6 +18,7 @@ #include "AvatarAudioRingBuffer.h" #include "AudioStreamStats.h" +#include "SequenceNumbersStats.h" class AudioMixerClientData : public NodeData { public: @@ -37,8 +38,11 @@ public: QString getJitterBufferStatsString() const; + const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } + private: QList _ringBuffers; + SequenceNumberStats _sequenceNumberStats; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index f66d11085e..c4f0e27a04 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -103,7 +103,8 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), - _audioMixerJitterBufferStats() + _audioMixerJitterBufferStats(), + _sequenceNumber(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -119,6 +120,7 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); + _sequenceNumber = 0; } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -422,7 +424,7 @@ void Audio::handleAudioInput() { static char audioDataPacket[MAX_PACKET_SIZE]; static int numBytesPacketHeader = numBytesForPacketHeaderGivenPacketType(PacketTypeMicrophoneAudioNoEcho); - static int leadingBytes = numBytesPacketHeader + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); + static int leadingBytes = numBytesPacketHeader + sizeof(quint16) + sizeof(glm::vec3) + sizeof(glm::quat) + sizeof(quint8); static int16_t* networkAudioSamples = (int16_t*) (audioDataPacket + leadingBytes); @@ -654,6 +656,10 @@ void Audio::handleAudioInput() { char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); + // pack seq number + memcpy(currentPacketPtr, &_sequenceNumber, sizeof(quint16)); + currentPacketPtr += sizeof(quint16); + // set the mono/stereo byte *currentPacketPtr++ = isStereo; @@ -669,6 +675,9 @@ void Audio::handleAudioInput() { Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); + } else { + // reset seq numbers if there's no connection with an audiomixer + _sequenceNumber = 0; } delete[] inputAudioSamples; } diff --git a/interface/src/Audio.h b/interface/src/Audio.h index a69423fdfd..c6fde709b8 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -238,6 +238,8 @@ private: QByteArray* _scopeOutputRight; AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; + + quint16 _sequenceNumber; }; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index fd9ba0150b..947c49905c 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -67,7 +67,7 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { } int AudioRingBuffer::parseData(const QByteArray& packet) { - int numBytesBeforeAudioData = numBytesForPacketHeader(packet); + int numBytesBeforeAudioData = numBytesForPacketHeader(packet) + sizeof(quint16); return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } From fe2eaa8e4a6bf9ba43fe7cf333fbd52aea660534 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 25 Jun 2014 18:02:38 -0700 Subject: [PATCH 17/67] forgot to increment seqnum in Audio.cpp --- interface/src/Audio.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c4f0e27a04..ac7598908c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -672,6 +672,7 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); + _sequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); From ad8fe38a5973456fdbd9a24fdb741981bff72038 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 09:27:06 -0700 Subject: [PATCH 18/67] removed extra sample in AudioRingBuffer for testing --- libraries/audio/src/AudioRingBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index fd9ba0150b..6345d36a81 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -28,7 +28,7 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : _hasStarted(false), _randomAccessMode(randomAccessMode) { - _arrayLength = _sampleCapacity + 1; + _arrayLength = _sampleCapacity;// +1; if (numFrameSamples) { _buffer = new int16_t[_arrayLength]; From 61b20b2be0880e9607bb8e888fe1444866e2cb64 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 09:56:19 -0700 Subject: [PATCH 19/67] Working 3.2 Oculus SDK --- interface/resources/shaders/oculus.frag | 36 +-- interface/resources/shaders/oculus.vert | 64 +++++ interface/src/devices/OculusManager.cpp | 328 +++++++++++++++++------- interface/src/devices/OculusManager.h | 52 +++- 4 files changed, 356 insertions(+), 124 deletions(-) create mode 100644 interface/resources/shaders/oculus.vert diff --git a/interface/resources/shaders/oculus.frag b/interface/resources/shaders/oculus.frag index f2b066a974..8e96428e17 100644 --- a/interface/resources/shaders/oculus.frag +++ b/interface/resources/shaders/oculus.frag @@ -4,12 +4,11 @@ // oculus.frag // fragment shader // -// Created by Andrzej Kapolka on 11/26/13. -// Copyright 2013 High Fidelity, Inc. +// Created by Ben Arnold on 6/24/14. +// Copyright 2014 High Fidelity, Inc. // -// this shader is an adaptation (HLSL -> GLSL, removed conditional) of the one in the Oculus sample -// code (Samples/OculusRoomTiny/RenderTiny_D3D1X_Device.cpp), which is under the Apache license -// (http://www.apache.org/licenses/LICENSE-2.0) +// this shader is an adaptation (HLSL -> GLSL) of the one in the +// Oculus_SDK_Overview.pdf for the 3.2 SDK. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -17,23 +16,16 @@ uniform sampler2D texture; -uniform vec2 lensCenter; -uniform vec2 screenCenter; -uniform vec2 scale; -uniform vec2 scaleIn; -uniform vec4 hmdWarpParam; - -vec2 hmdWarp(vec2 in01) { - vec2 theta = (in01 - lensCenter) * scaleIn; - float rSq = theta.x * theta.x + theta.y * theta.y; - vec2 theta1 = theta * (hmdWarpParam.x + hmdWarpParam.y * rSq + - hmdWarpParam.z * rSq * rSq + hmdWarpParam.w * rSq * rSq * rSq); - return lensCenter + scale * theta1; -} +varying float vFade; +varying vec2 oTexCoord0; +varying vec2 oTexCoord1; +varying vec2 oTexCoord2; void main(void) { - vec2 tc = hmdWarp(gl_TexCoord[0].st); - vec2 below = step(screenCenter.st + vec2(-0.25, -0.5), tc.st); - vec2 above = vec2(1.0, 1.0) - step(screenCenter.st + vec2(0.25, 0.5), tc.st); - gl_FragColor = mix(vec4(0.0, 0.0, 0.0, 1.0), texture2D(texture, tc), above.s * above.t * below.s * below.t); + // 3 samples for fixing chromatic aberrations + float r = texture2D(texture, oTexCoord0.xy).r; + float g = texture2D(texture, oTexCoord1.xy).g; + float b = texture2D(texture, oTexCoord2.xy).b; + + gl_FragColor = vec4(r * vFade, g * vFade, b * vFade, 1.0); } diff --git a/interface/resources/shaders/oculus.vert b/interface/resources/shaders/oculus.vert new file mode 100644 index 0000000000..bc9eba4cf3 --- /dev/null +++ b/interface/resources/shaders/oculus.vert @@ -0,0 +1,64 @@ +#version 120 + +// +// oculus.vert +// vertex shader +// +// Created by Ben Arnold on 6/24/14. +// Copyright 2014 High Fidelity, Inc. +// +// this shader is an adaptation (HLSL -> GLSL) of the one in the +// Oculus_SDK_Overview.pdf for the 3.2 SDK. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +uniform vec2 EyeToSourceUVScale; +uniform vec2 EyeToSourceUVOffset; +uniform mat4 EyeRotationStart; +uniform mat4 EyeRotationEnd; + +attribute vec2 position; +attribute vec4 color; +attribute vec2 texCoord0; +attribute vec2 texCoord1; +attribute vec2 texCoord2; + +varying float vFade; +varying vec2 oTexCoord0; +varying vec2 oTexCoord1; +varying vec2 oTexCoord2; + +vec2 TimewarpTexCoord(vec2 texCoord, mat4 rotMat) +{ + // Vertex inputs are in TanEyeAngle space for the R,G,B channels (i.e. after chromatic + // aberration and distortion). These are now "real world" vectors in direction (x,y,1) + // relative to the eye of the HMD. Apply the 3x3 timewarp rotation to these vectors. + vec3 transformed = vec3( rotMat * vec4(texCoord.xy, 1, 1) ).xyz; + + // Project them back onto the Z=1 plane of the rendered images. + vec2 flattened = (transformed.xy / transformed.z); + + // Scale them into ([0,0.5],[0,1]) or ([0.5,0],[0,1]) UV lookup space (depending on eye) + return (EyeToSourceUVScale * flattened + EyeToSourceUVOffset); +} + +void main() +{ + float timewarpMixFactor = color.a; + mat4 mixedEyeRot = EyeRotationStart * mat4(1.0 - timewarpMixFactor) + EyeRotationEnd * mat4(timewarpMixFactor); + + + oTexCoord0 = TimewarpTexCoord(texCoord0, mixedEyeRot); + oTexCoord1 = TimewarpTexCoord(texCoord1, mixedEyeRot); + oTexCoord2 = TimewarpTexCoord(texCoord2, mixedEyeRot); + + //Flip y texture coordinates + oTexCoord0.y = 1.0 - oTexCoord0.y; + oTexCoord1.y = 1.0 - oTexCoord1.y; + oTexCoord2.y = 1.0 - oTexCoord2.y; + + gl_Position = vec4(position.xy, 0.5, 1.0); + vFade = color.r; // For vignette fade +} \ No newline at end of file diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 19c6a2b193..8ef0945b55 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -11,28 +11,39 @@ #include "InterfaceConfig.h" +#include "OculusManager.h" + #include #include #include "Application.h" -#include "OculusManager.h" + +using namespace OVR; ProgramObject OculusManager::_program; int OculusManager::_textureLocation; -int OculusManager::_lensCenterLocation; -int OculusManager::_screenCenterLocation; -int OculusManager::_scaleLocation; -int OculusManager::_scaleInLocation; -int OculusManager::_hmdWarpParamLocation; +int OculusManager::_eyeToSourceUVScaleLocation; +int OculusManager::_eyeToSourceUVOffsetLocation; +int OculusManager::_eyeRotationStartLocation; +int OculusManager::_eyeRotationEndLocation; +int OculusManager::_positionAttributeLocation; +int OculusManager::_colorAttributeLocation; +int OculusManager::_texCoord0AttributeLocation; +int OculusManager::_texCoord1AttributeLocation; +int OculusManager::_texCoord2AttributeLocation; bool OculusManager::_isConnected = false; #ifdef HAVE_LIBOVR -using namespace OVR; -using namespace OVR::Util::Render; ovrHmd OculusManager::_ovrHmd; ovrHmdDesc OculusManager::_ovrHmdDesc; +ovrFovPort OculusManager::_eyeFov[ovrEye_Count]; +ovrSizei OculusManager::_renderTargetSize; +ovrVector2f OculusManager::_UVScaleOffset[ovrEye_Count][2]; +GLuint OculusManager::_vbo[ovrEye_Count]; +GLuint OculusManager::_indicesVbo[ovrEye_Count]; +GLsizei OculusManager::_meshSize[ovrEye_Count]; #endif @@ -46,25 +57,109 @@ void OculusManager::connect() { ovrHmd_GetDesc(_ovrHmd, &_ovrHmdDesc); + _eyeFov[0] = _ovrHmdDesc.DefaultEyeFov[0]; + _eyeFov[1] = _ovrHmdDesc.DefaultEyeFov[1]; + + //Get texture size + ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, + _ovrHmdDesc.DefaultEyeFov[0], 1.0f); + ovrSizei recommendedTex1Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Right, + _ovrHmdDesc.DefaultEyeFov[1], 1.0f); + _renderTargetSize.w = recommendedTex0Size.w + recommendedTex1Size.w; + _renderTargetSize.h = recommendedTex0Size.h; + if (_renderTargetSize.h < recommendedTex1Size.h) { + _renderTargetSize.h = recommendedTex1Size.h; + } + ovrHmd_StartSensor(_ovrHmd, ovrSensorCap_Orientation | ovrSensorCap_YawCorrection | - ovrSensorCap_Position | ovrHmdCap_LowPersistence, + ovrSensorCap_Position, ovrSensorCap_Orientation); + _program.addShaderFromSourceFile(QGLShader::Vertex, Application::resourcesPath() + "shaders/oculus.vert"); _program.addShaderFromSourceFile(QGLShader::Fragment, Application::resourcesPath() + "shaders/oculus.frag"); _program.link(); - + + generateDistortionMesh(); + + // Uniforms _textureLocation = _program.uniformLocation("texture"); - _lensCenterLocation = _program.uniformLocation("lensCenter"); - _screenCenterLocation = _program.uniformLocation("screenCenter"); - _scaleLocation = _program.uniformLocation("scale"); - _scaleInLocation = _program.uniformLocation("scaleIn"); - _hmdWarpParamLocation = _program.uniformLocation("hmdWarpParam"); + _eyeToSourceUVScaleLocation = _program.uniformLocation("EyeToSourceUVScale"); + _eyeToSourceUVOffsetLocation = _program.uniformLocation("EyeToSourceUVOffset"); + _eyeRotationStartLocation = _program.uniformLocation("EyeRotationStart"); + _eyeRotationEndLocation = _program.uniformLocation("EyeRotationEnd"); + + // Attributes + _positionAttributeLocation = _program.attributeLocation("position"); + _colorAttributeLocation = _program.attributeLocation("color"); + _texCoord0AttributeLocation = _program.attributeLocation("texCoord0"); + _texCoord1AttributeLocation = _program.attributeLocation("texCoord1"); + _texCoord2AttributeLocation = _program.attributeLocation("texCoord2"); + } else { ovr_Shutdown(); } #endif } +void OculusManager::generateDistortionMesh() { +#ifdef HAVE_LIBOVR + ovrEyeRenderDesc eyeDesc[2]; + eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); + eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); + + ovrRecti eyeRenderViewport[2]; + eyeRenderViewport[0].Pos = Vector2i(0, 0); + eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); + eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); + eyeRenderViewport[1].Size = eyeRenderViewport[0].Size; + + for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { + // Allocate and generate distortion mesh vertices + ovrDistortionMesh meshData; + ovrHmd_CreateDistortionMesh(_ovrHmd, eyeDesc[eyeNum].Eye, eyeDesc[eyeNum].Fov, _ovrHmdDesc.DistortionCaps, &meshData); + + ovrHmd_GetRenderScaleAndOffset(eyeDesc[eyeNum].Fov, _renderTargetSize, eyeRenderViewport[eyeNum], + _UVScaleOffset[eyeNum]); + + // Parse the vertex data and create a render ready vertex buffer + DistortionVertex* pVBVerts = (DistortionVertex*)OVR_ALLOC(sizeof(DistortionVertex) * meshData.VertexCount); + _meshSize[eyeNum] = meshData.IndexCount; + + DistortionVertex* v = pVBVerts; + ovrDistortionVertex* ov = meshData.pVertexData; + for (unsigned int vertNum = 0; vertNum < meshData.VertexCount; vertNum++) { + v->pos.x = ov->Pos.x; + v->pos.y = ov->Pos.y; + v->texR.x = ov->TexR.x; + v->texR.y = ov->TexR.y; + v->texG.x = ov->TexG.x; + v->texG.y = ov->TexG.y; + v->texB.x = ov->TexB.x; + v->texB.y = ov->TexB.y; + v->color.r = v->color.g = v->color.b = (GLubyte)(ov->VignetteFactor * 255.99f); + v->color.a = (GLubyte)(ov->TimeWarpFactor * 255.99f); + v++; + ov++; + } + + //vertices + glGenBuffers(1, &(_vbo[eyeNum])); + glBindBuffer(GL_ARRAY_BUFFER, _vbo[eyeNum]); + glBufferData(GL_ARRAY_BUFFER, sizeof(DistortionVertex) * meshData.VertexCount, pVBVerts, GL_STATIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + OVR_FREE(pVBVerts); + + //indices + glGenBuffers(1, &(_indicesVbo[eyeNum])); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indicesVbo[eyeNum]); + glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(unsigned short) * meshData.IndexCount, meshData.pIndexData, GL_STATIC_DRAW); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + ovrHmd_DestroyDistortionMesh(&meshData); + } +#endif +} + bool OculusManager::isConnected() { #ifdef HAVE_LIBOVR return _isConnected && Menu::getInstance()->isOptionChecked(MenuOption::EnableVRMode); @@ -75,14 +170,30 @@ bool OculusManager::isConnected() { void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenHeight) { #ifdef HAVE_LIBOVR - _stereoConfig.SetFullViewport(Viewport(0, 0, screenWidth, screenHeight)); - camera.setAspectRatio(_stereoConfig.GetAspect()); - camera.setFieldOfView(_stereoConfig.GetYFOVDegrees()); + ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, + _ovrHmdDesc.DefaultEyeFov[0], 1.0f); + ovrSizei recommendedTex1Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Right, + _ovrHmdDesc.DefaultEyeFov[1], 1.0f); + + float width = recommendedTex0Size.w + recommendedTex1Size.w; + float height = recommendedTex0Size.h; + if (height < recommendedTex1Size.h) { + height = recommendedTex1Size.h; + } + + camera.setAspectRatio(width / height); + camera.setFieldOfView(110); #endif } void OculusManager::display(Camera& whichCamera) { #ifdef HAVE_LIBOVR + static unsigned int frameIndex = 0; + ovrFrameTiming hmdFrameTiming = ovrHmd_BeginFrameTiming(_ovrHmd, frameIndex); + + ovrEyeRenderDesc eyeDesc[2]; + eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); + eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); ApplicationOverlay& applicationOverlay = Application::getInstance()->getApplicationOverlay(); // We only need to render the overlays to a texture once, then we just render the texture as a quad // PrioVR will only work if renderOverlay is called, calibration is connected to Application::renderingOverlay() @@ -91,119 +202,146 @@ void OculusManager::display(Camera& whichCamera) { Application::getInstance()->getGlowEffect()->prepare(); + ovrPosef eyeRenderPose[ovrEye_Count]; + + glEnable(GL_DEPTH_TEST); + glDepthMask(GL_TRUE); + // render the left eye view to the left side of the screen - const StereoEyeParams& leftEyeParams = _stereoConfig.GetEyeRenderParams(StereoEye_Left); + glMatrixMode(GL_PROJECTION); glPushMatrix(); - glLoadIdentity(); - glTranslatef(_stereoConfig.GetProjectionCenterOffset(), 0, 0); - gluPerspective(whichCamera.getFieldOfView(), whichCamera.getAspectRatio(), - whichCamera.getNearClip(), whichCamera.getFarClip()); - - glViewport(leftEyeParams.VP.x, leftEyeParams.VP.y, leftEyeParams.VP.w, leftEyeParams.VP.h); + glMatrixMode(GL_MODELVIEW); glPushMatrix(); - glLoadIdentity(); - glTranslatef(_stereoConfig.GetIPD() * 0.5f, 0, 0); - - Application::getInstance()->displaySide(whichCamera); - if (displayOverlays) { - applicationOverlay.displayOverlayTextureOculus(whichCamera); - } - - // and the right eye to the right side - const StereoEyeParams& rightEyeParams = _stereoConfig.GetEyeRenderParams(StereoEye_Right); - glMatrixMode(GL_PROJECTION); - glLoadIdentity(); - glTranslatef(-_stereoConfig.GetProjectionCenterOffset(), 0, 0); - gluPerspective(whichCamera.getFieldOfView(), whichCamera.getAspectRatio(), - whichCamera.getNearClip(), whichCamera.getFarClip()); - - glViewport(rightEyeParams.VP.x, rightEyeParams.VP.y, rightEyeParams.VP.w, rightEyeParams.VP.h); - glMatrixMode(GL_MODELVIEW); - glLoadIdentity(); - glTranslatef(_stereoConfig.GetIPD() * -0.5f, 0, 0); - - Application::getInstance()->displaySide(whichCamera); + eyeRenderPose[0] = ovrHmd_GetEyePose(_ovrHmd, ovrEye_Left); + eyeRenderPose[1] = ovrHmd_GetEyePose(_ovrHmd, ovrEye_Right); - if (displayOverlays) { - applicationOverlay.displayOverlayTextureOculus(whichCamera); + for (int eyeIndex = 0; eyeIndex < ovrEye_Count; eyeIndex++) { + + ovrEyeType eye = _ovrHmdDesc.EyeRenderOrder[eyeIndex]; + + Matrix4f proj = ovrMatrix4f_Projection(eyeDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); + proj.Transpose(); + + glMatrixMode(GL_PROJECTION); + glLoadIdentity(); + glLoadMatrixf((GLfloat *)proj.M); + glTranslatef(0.0f, 0, 0); + + // gluPerspective(whichCamera.getFieldOfView(), whichCamera.getAspectRatio(), + // whichCamera.getNearClip(), whichCamera.getFarClip()); + + + glViewport(eyeDesc[eye].DistortedViewport.Pos.x, eyeDesc[eye].DistortedViewport.Pos.y, + eyeDesc[eye].DistortedViewport.Size.w, eyeDesc[eye].DistortedViewport.Size.h); + glMatrixMode(GL_MODELVIEW); + glLoadIdentity(); + glTranslatef(eyeDesc[eye].ViewAdjust.x, eyeDesc[eye].ViewAdjust.y, eyeDesc[eye].ViewAdjust.z); + + Application::getInstance()->displaySide(whichCamera); + + if (displayOverlays) { + applicationOverlay.displayOverlayTextureOculus(whichCamera); + } } - + + //Wait till time-warp to reduce latency + ovr_WaitTillTime(hmdFrameTiming.TimewarpPointSeconds); + glPopMatrix(); - + // restore our normal viewport - const Viewport& fullViewport = _stereoConfig.GetFullViewport(); - glViewport(fullViewport.x, fullViewport.y, fullViewport.w, fullViewport.h); + glViewport(0, 0, Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); glBindTexture(GL_TEXTURE_2D, fbo->texture()); glMatrixMode(GL_PROJECTION); + glPopMatrix(); + glLoadIdentity(); - gluOrtho2D(fullViewport.x, fullViewport.x + fullViewport.w, fullViewport.y, fullViewport.y + fullViewport.h); + gluOrtho2D(0, Application::getInstance()->getGLWidget()->width(), 0, Application::getInstance()->getGLWidget()->height()); glDisable(GL_DEPTH_TEST); - - // for reference on setting these values, see SDK file Samples/OculusRoomTiny/RenderTiny_Device.cpp - - float scaleFactor = 1.0 / _stereoConfig.GetDistortionScale(); - float aspectRatio = _stereoConfig.GetAspect(); - + glDisable(GL_BLEND); _program.bind(); _program.setUniformValue(_textureLocation, 0); - const DistortionConfig& distortionConfig = _stereoConfig.GetDistortionConfig(); - _program.setUniformValue(_lensCenterLocation, (0.5 + distortionConfig.XCenterOffset * 0.5) * 0.5, 0.5); - _program.setUniformValue(_screenCenterLocation, 0.25, 0.5); - _program.setUniformValue(_scaleLocation, 0.25 * scaleFactor, 0.5 * scaleFactor * aspectRatio); - _program.setUniformValue(_scaleInLocation, 4, 2 / aspectRatio); - _program.setUniformValue(_hmdWarpParamLocation, distortionConfig.K[0], distortionConfig.K[1], - distortionConfig.K[2], distortionConfig.K[3]); + + _program.enableAttributeArray(_positionAttributeLocation); + _program.enableAttributeArray(_colorAttributeLocation); + _program.enableAttributeArray(_texCoord0AttributeLocation); + _program.enableAttributeArray(_texCoord1AttributeLocation); + _program.enableAttributeArray(_texCoord2AttributeLocation); + + for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { + GLfloat uvScale[2] = { _UVScaleOffset[eyeNum][0].x, _UVScaleOffset[eyeNum][0].y }; + _program.setUniformValueArray(_eyeToSourceUVScaleLocation, uvScale, 1, 2); + GLfloat uvOffset[2] = { _UVScaleOffset[eyeNum][1].x, _UVScaleOffset[eyeNum][1].y }; + _program.setUniformValueArray(_eyeToSourceUVOffsetLocation, uvOffset, 1, 2); - glColor3f(1, 0, 1); - glBegin(GL_QUADS); - glTexCoord2f(0, 0); - glVertex2f(0, 0); - glTexCoord2f(0.5, 0); - glVertex2f(leftEyeParams.VP.w, 0); - glTexCoord2f(0.5, 1); - glVertex2f(leftEyeParams.VP.w, leftEyeParams.VP.h); - glTexCoord2f(0, 1); - glVertex2f(0, leftEyeParams.VP.h); - glEnd(); + ovrMatrix4f timeWarpMatrices[2]; + Matrix4f transposeMatrices[2]; + ovrHmd_GetEyeTimewarpMatrices(_ovrHmd, (ovrEyeType)eyeNum, eyeRenderPose[eyeNum], timeWarpMatrices); + transposeMatrices[0] = Matrix4f(timeWarpMatrices[0]); + transposeMatrices[1] = Matrix4f(timeWarpMatrices[1]); + + transposeMatrices[0].Transpose(); + transposeMatrices[1].Transpose(); - _program.setUniformValue(_lensCenterLocation, 0.5 + (0.5 - distortionConfig.XCenterOffset * 0.5) * 0.5, 0.5); - _program.setUniformValue(_screenCenterLocation, 0.75, 0.5); - - glBegin(GL_QUADS); - glTexCoord2f(0.5, 0); - glVertex2f(leftEyeParams.VP.w, 0); - glTexCoord2f(1, 0); - glVertex2f(fullViewport.w, 0); - glTexCoord2f(1, 1); - glVertex2f(fullViewport.w, leftEyeParams.VP.h); - glTexCoord2f(0.5, 1); - glVertex2f(leftEyeParams.VP.w, leftEyeParams.VP.h); - glEnd(); + glUniformMatrix4fv(_eyeRotationStartLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[0].M); + + glUniformMatrix4fv(_eyeRotationEndLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[1].M); + + glBindBuffer(GL_ARRAY_BUFFER, _vbo[eyeNum]); + //pos + glVertexAttribPointer(_positionAttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)0); + glVertexAttribPointer(_texCoord0AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)8); + glVertexAttribPointer(_texCoord1AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)16); + glVertexAttribPointer(_texCoord2AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)24); + glVertexAttribPointer(_colorAttributeLocation, 4, GL_UNSIGNED_BYTE, GL_TRUE, sizeof(DistortionVertex), (void *)32); + + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indicesVbo[eyeNum]); + glDrawElements(GL_TRIANGLES, _meshSize[eyeNum], GL_UNSIGNED_SHORT, 0); + int error = glGetError(); + if (error != GL_NO_ERROR) { + printf("ERROR %d\n", error); + } + } + + _program.disableAttributeArray(_positionAttributeLocation); + _program.disableAttributeArray(_colorAttributeLocation); + _program.disableAttributeArray(_texCoord0AttributeLocation); + _program.disableAttributeArray(_texCoord1AttributeLocation); + _program.disableAttributeArray(_texCoord2AttributeLocation); glEnable(GL_BLEND); + glEnable(GL_DEPTH_TEST); + glBindBuffer(GL_ARRAY_BUFFER, 0); glBindTexture(GL_TEXTURE_2D, 0); _program.release(); - - glPopMatrix(); + + ovrHmd_EndFrameTiming(_ovrHmd); + frameIndex++; #endif } void OculusManager::reset() { #ifdef HAVE_LIBOVR - _sensorFusion->Reset(); + //_sensorFusion->Reset(); #endif } void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { #ifdef HAVE_LIBOVR - _sensorFusion->GetPredictedOrientation().GetEulerAngles(&yaw, &pitch, &roll); + ovrSensorState ss = ovrHmd_GetSensorState(_ovrHmd, 0.0); + + // if (ss.StatusFlags & (ovrStatus_OrientationTracked | ovrStatus_PositionTracked)) { + ovrPosef pose = ss.Recorded.Pose; + Quatf orientation = Quatf(pose.Orientation); + orientation.GetEulerAngles(&yaw, &pitch, &roll); + // } #endif } diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 795fdde7ff..2a131a5f55 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -12,14 +12,21 @@ #ifndef hifi_OculusManager_h #define hifi_OculusManager_h -#include - #ifdef HAVE_LIBOVR #include +//#include <../src/OVR_CAPI.h> #endif +#include "../src/Util/Util_Render_Stereo.h" +using namespace OVR::Util::Render; + +#include <../src/Kernel/OVR_SysFile.h> +#include <../src/Kernel/OVR_Log.h> +#include <../src/Kernel/OVR_Timer.h> + #include "renderer/ProgramObject.h" + const float DEFAULT_OCULUS_UI_ANGULAR_SIZE = 72.0f; class Camera; @@ -29,6 +36,8 @@ class OculusManager { public: static void connect(); + static void generateDistortionMesh(); + static bool isConnected(); static void configureCamera(Camera& camera, int screenWidth, int screenHeight); @@ -45,18 +54,47 @@ public: static void updateYawOffset(); private: + + struct DistortionVertex { + glm::vec2 pos; + glm::vec2 texR; + glm::vec2 texG; + glm::vec2 texB; + struct { + GLubyte r; + GLubyte g; + GLubyte b; + GLubyte a; + } color; + }; + static ProgramObject _program; + //Uniforms static int _textureLocation; - static int _lensCenterLocation; - static int _screenCenterLocation; - static int _scaleLocation; - static int _scaleInLocation; - static int _hmdWarpParamLocation; + static int _eyeToSourceUVScaleLocation; + static int _eyeToSourceUVOffsetLocation; + static int _eyeRotationStartLocation; + static int _eyeRotationEndLocation; + //Attributes + static int _positionAttributeLocation; + static int _colorAttributeLocation; + static int _texCoord0AttributeLocation; + static int _texCoord1AttributeLocation; + static int _texCoord2AttributeLocation; + static bool _isConnected; + + #ifdef HAVE_LIBOVR static ovrHmd _ovrHmd; static ovrHmdDesc _ovrHmdDesc; + static ovrFovPort _eyeFov[ovrEye_Count]; + static ovrSizei _renderTargetSize; + static ovrVector2f _UVScaleOffset[ovrEye_Count][2]; + static GLuint _vbo[ovrEye_Count]; + static GLuint _indicesVbo[ovrEye_Count]; + static GLsizei _meshSize[ovrEye_Count]; #endif }; From adb6c445a268c5178855bf5cc79794469d48a120 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 10:13:15 -0700 Subject: [PATCH 20/67] Fixed camera FoV, code cleanup --- interface/src/devices/OculusManager.cpp | 39 ++++++++++--------------- interface/src/devices/OculusManager.h | 1 + 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 8ef0945b55..7fb098ab87 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -44,6 +44,7 @@ ovrVector2f OculusManager::_UVScaleOffset[ovrEye_Count][2]; GLuint OculusManager::_vbo[ovrEye_Count]; GLuint OculusManager::_indicesVbo[ovrEye_Count]; GLsizei OculusManager::_meshSize[ovrEye_Count]; +ovrFrameTiming OculusManager::_hmdFrameTiming; #endif @@ -103,11 +104,11 @@ void OculusManager::connect() { void OculusManager::generateDistortionMesh() { #ifdef HAVE_LIBOVR - ovrEyeRenderDesc eyeDesc[2]; + ovrEyeRenderDesc eyeDesc[ovrEye_Count]; eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); - ovrRecti eyeRenderViewport[2]; + ovrRecti eyeRenderViewport[ovrEye_Count]; eyeRenderViewport[0].Pos = Vector2i(0, 0); eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); @@ -182,19 +183,20 @@ void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenH } camera.setAspectRatio(width / height); - camera.setFieldOfView(110); + camera.setFieldOfView(atan(_eyeFov[0].UpTan) * DEGREES_PER_RADIAN * 2.0f); #endif } void OculusManager::display(Camera& whichCamera) { #ifdef HAVE_LIBOVR static unsigned int frameIndex = 0; - ovrFrameTiming hmdFrameTiming = ovrHmd_BeginFrameTiming(_ovrHmd, frameIndex); + _hmdFrameTiming = ovrHmd_BeginFrameTiming(_ovrHmd, frameIndex); - ovrEyeRenderDesc eyeDesc[2]; + ovrEyeRenderDesc eyeDesc[ovrEye_Count]; eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); ApplicationOverlay& applicationOverlay = Application::getInstance()->getApplicationOverlay(); + // We only need to render the overlays to a texture once, then we just render the texture as a quad // PrioVR will only work if renderOverlay is called, calibration is connected to Application::renderingOverlay() applicationOverlay.renderOverlay(true); @@ -204,11 +206,6 @@ void OculusManager::display(Camera& whichCamera) { ovrPosef eyeRenderPose[ovrEye_Count]; - glEnable(GL_DEPTH_TEST); - glDepthMask(GL_TRUE); - - // render the left eye view to the left side of the screen - glMatrixMode(GL_PROJECTION); glPushMatrix(); @@ -230,12 +227,9 @@ void OculusManager::display(Camera& whichCamera) { glLoadMatrixf((GLfloat *)proj.M); glTranslatef(0.0f, 0, 0); - // gluPerspective(whichCamera.getFieldOfView(), whichCamera.getAspectRatio(), - // whichCamera.getNearClip(), whichCamera.getFarClip()); - - glViewport(eyeDesc[eye].DistortedViewport.Pos.x, eyeDesc[eye].DistortedViewport.Pos.y, eyeDesc[eye].DistortedViewport.Size.w, eyeDesc[eye].DistortedViewport.Size.h); + glMatrixMode(GL_MODELVIEW); glLoadIdentity(); glTranslatef(eyeDesc[eye].ViewAdjust.x, eyeDesc[eye].ViewAdjust.y, eyeDesc[eye].ViewAdjust.z); @@ -248,13 +242,14 @@ void OculusManager::display(Camera& whichCamera) { } //Wait till time-warp to reduce latency - ovr_WaitTillTime(hmdFrameTiming.TimewarpPointSeconds); + ovr_WaitTillTime(_hmdFrameTiming.TimewarpPointSeconds); glPopMatrix(); // restore our normal viewport glViewport(0, 0, Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); + //Bind the output texture from the glow shader QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); glBindTexture(GL_TEXTURE_2D, fbo->texture()); @@ -295,7 +290,7 @@ void OculusManager::display(Camera& whichCamera) { glUniformMatrix4fv(_eyeRotationEndLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[1].M); glBindBuffer(GL_ARRAY_BUFFER, _vbo[eyeNum]); - //pos + glVertexAttribPointer(_positionAttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)0); glVertexAttribPointer(_texCoord0AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)8); glVertexAttribPointer(_texCoord1AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)16); @@ -304,10 +299,6 @@ void OculusManager::display(Camera& whichCamera) { glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indicesVbo[eyeNum]); glDrawElements(GL_TRIANGLES, _meshSize[eyeNum], GL_UNSIGNED_SHORT, 0); - int error = glGetError(); - if (error != GL_NO_ERROR) { - printf("ERROR %d\n", error); - } } _program.disableAttributeArray(_positionAttributeLocation); @@ -335,13 +326,13 @@ void OculusManager::reset() { void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { #ifdef HAVE_LIBOVR - ovrSensorState ss = ovrHmd_GetSensorState(_ovrHmd, 0.0); + ovrSensorState ss = ovrHmd_GetSensorState(_ovrHmd, _hmdFrameTiming.ScanoutMidpointSeconds); - // if (ss.StatusFlags & (ovrStatus_OrientationTracked | ovrStatus_PositionTracked)) { - ovrPosef pose = ss.Recorded.Pose; + if (ss.StatusFlags & (ovrStatus_OrientationTracked | ovrStatus_PositionTracked)) { + ovrPosef pose = ss.Predicted.Pose; Quatf orientation = Quatf(pose.Orientation); orientation.GetEulerAngles(&yaw, &pitch, &roll); - // } + } #endif } diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 2a131a5f55..3d18f762a9 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -95,6 +95,7 @@ private: static GLuint _vbo[ovrEye_Count]; static GLuint _indicesVbo[ovrEye_Count]; static GLsizei _meshSize[ovrEye_Count]; + static ovrFrameTiming _hmdFrameTiming; #endif }; From 7f23a7fbde24bac37d44090eb7ee6c84e3da5f2c Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 11:26:10 -0700 Subject: [PATCH 21/67] switched AudioRingBuffer to use _isFull flag instead --- libraries/audio/src/AudioRingBuffer.cpp | 66 +++++++++++++++---------- libraries/audio/src/AudioRingBuffer.h | 2 +- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 6345d36a81..943bd1fb40 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -26,14 +26,13 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : _numFrameSamples(numFrameSamples), _isStarved(true), _hasStarted(false), - _randomAccessMode(randomAccessMode) + _randomAccessMode(randomAccessMode), + _isFull(false) { - _arrayLength = _sampleCapacity;// +1; - if (numFrameSamples) { - _buffer = new int16_t[_arrayLength]; + _buffer = new int16_t[_sampleCapacity]; if (_randomAccessMode) { - memset(_buffer, 0, _arrayLength * sizeof(int16_t)); + memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); } _nextOutput = _buffer; _endOfLastWrite = _buffer; @@ -57,10 +56,10 @@ void AudioRingBuffer::reset() { void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { delete[] _buffer; _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; - _arrayLength = _sampleCapacity + 1; - _buffer = new int16_t[_arrayLength]; + _sampleCapacity = _sampleCapacity + 1; + _buffer = new int16_t[_sampleCapacity]; if (_randomAccessMode) { - memset(_buffer, 0, _arrayLength * sizeof(int16_t)); + memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); } _nextOutput = _buffer; _endOfLastWrite = _buffer; @@ -87,11 +86,11 @@ qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { numReadSamples = _endOfLastWrite ? (maxSize / sizeof(int16_t)) : 0; } - if (_nextOutput + numReadSamples > _buffer + _arrayLength) { + if (_nextOutput + numReadSamples > _buffer + _sampleCapacity) { // we're going to need to do two reads to get this data, it wraps around the edge // read to the end of the buffer - int numSamplesToEnd = (_buffer + _arrayLength) - _nextOutput; + int numSamplesToEnd = (_buffer + _sampleCapacity) - _nextOutput; memcpy(data, _nextOutput, numSamplesToEnd * sizeof(int16_t)); if (_randomAccessMode) { memset(_nextOutput, 0, numSamplesToEnd * sizeof(int16_t)); // clear it @@ -112,6 +111,9 @@ qint64 AudioRingBuffer::readData(char *data, qint64 maxSize) { // push the position of _nextOutput by the number of samples read _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numReadSamples); + if (numReadSamples > 0) { + _isFull = false; + } return numReadSamples * sizeof(int16_t); } @@ -134,16 +136,19 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { qDebug() << "Overflowed ring buffer! Overwriting old data"; } - if (_endOfLastWrite + samplesToCopy <= _buffer + _arrayLength) { + if (_endOfLastWrite + samplesToCopy <= _buffer + _sampleCapacity) { memcpy(_endOfLastWrite, data, samplesToCopy * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + _arrayLength) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; memcpy(_endOfLastWrite, data, numSamplesToEnd * sizeof(int16_t)); memcpy(_buffer, data + (numSamplesToEnd * sizeof(int16_t)), (samplesToCopy - numSamplesToEnd) * sizeof(int16_t)); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, samplesToCopy); - + if (samplesToCopy > 0 && _endOfLastWrite == _nextOutput) { + _isFull = true; + } + return samplesToCopy * sizeof(int16_t); } @@ -156,21 +161,25 @@ const int16_t& AudioRingBuffer::operator[] (const int index) const { } void AudioRingBuffer::shiftReadPosition(unsigned int numSamples) { - _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); + if (numSamples > 0) { + _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, numSamples); + _isFull = false; + } } unsigned int AudioRingBuffer::samplesAvailable() const { if (!_endOfLastWrite) { return 0; - } else { - int sampleDifference = _endOfLastWrite - _nextOutput; - - if (sampleDifference < 0) { - sampleDifference += _arrayLength; - } - - return sampleDifference; } + if (_isFull) { + return _sampleCapacity; + } + + int sampleDifference = _endOfLastWrite - _nextOutput; + if (sampleDifference < 0) { + sampleDifference += _sampleCapacity; + } + return sampleDifference; } int AudioRingBuffer::addSilentFrame(int numSilentSamples) { @@ -184,14 +193,17 @@ int AudioRingBuffer::addSilentFrame(int numSilentSamples) { // memset zeroes into the buffer, accomodate a wrap around the end // push the _endOfLastWrite to the correct spot - if (_endOfLastWrite + numSilentSamples <= _buffer + _arrayLength) { + if (_endOfLastWrite + numSilentSamples <= _buffer + _sampleCapacity) { memset(_endOfLastWrite, 0, numSilentSamples * sizeof(int16_t)); } else { - int numSamplesToEnd = (_buffer + _arrayLength) - _endOfLastWrite; + int numSamplesToEnd = (_buffer + _sampleCapacity) - _endOfLastWrite; memset(_endOfLastWrite, 0, numSamplesToEnd * sizeof(int16_t)); memset(_buffer, 0, (numSilentSamples - numSamplesToEnd) * sizeof(int16_t)); } _endOfLastWrite = shiftedPositionAccomodatingWrap(_endOfLastWrite, numSilentSamples); + if (numSilentSamples > 0 && _nextOutput == _endOfLastWrite) { + _isFull = true; + } return numSilentSamples * sizeof(int16_t); } @@ -206,12 +218,12 @@ bool AudioRingBuffer::isNotStarvedOrHasMinimumSamples(unsigned int numRequiredSa int16_t* AudioRingBuffer::shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const { - if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _arrayLength) { + if (numSamplesShift > 0 && position + numSamplesShift >= _buffer + _sampleCapacity) { // this shift will wrap the position around to the beginning of the ring - return position + numSamplesShift - _arrayLength; + return position + numSamplesShift - _sampleCapacity; } else if (numSamplesShift < 0 && position + numSamplesShift < _buffer) { // this shift will go around to the end of the ring - return position + numSamplesShift + _arrayLength; + return position + numSamplesShift + _sampleCapacity; } else { return position + numSamplesShift; } diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index ae46af2776..b290c96d79 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -85,7 +85,7 @@ protected: int _resetCount; /// how many times has the ring buffer written past the end and done a reset int _sampleCapacity; - int _arrayLength; + bool _isFull; int _numFrameSamples; int16_t* _nextOutput; int16_t* _endOfLastWrite; From 634e0f38e80eeb6819d0bdaee11728e46979c15b Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 11:27:17 -0700 Subject: [PATCH 22/67] Accurate Oculus frame timing --- interface/src/GLCanvas.cpp | 19 ++++++++++++ interface/src/devices/OculusManager.cpp | 41 ++++++++++++++++++++----- interface/src/devices/OculusManager.h | 11 ++++--- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/interface/src/GLCanvas.cpp b/interface/src/GLCanvas.cpp index 49085e63df..17026b5d5c 100644 --- a/interface/src/GLCanvas.cpp +++ b/interface/src/GLCanvas.cpp @@ -12,6 +12,7 @@ #include "Application.h" #include "GLCanvas.h" +#include "devices/OculusManager.h" #include #include #include @@ -41,8 +42,17 @@ void GLCanvas::initializeGL() { void GLCanvas::paintGL() { if (!_throttleRendering && !Application::getInstance()->getWindow()->isMinimized()) { + //Need accurate frame timing for the oculus rift + if (OculusManager::isConnected()) { + OculusManager::beginFrameTiming(); + } + Application::getInstance()->paintGL(); swapBuffers(); + + if (OculusManager::isConnected()) { + OculusManager::endFrameTiming(); + } } } @@ -102,8 +112,17 @@ void GLCanvas::activeChanged(Qt::ApplicationState state) { void GLCanvas::throttleRender() { _frameTimer.start(_idleRenderInterval); if (!Application::getInstance()->getWindow()->isMinimized()) { + //Need accurate frame timing for the oculus rift + if (OculusManager::isConnected()) { + OculusManager::beginFrameTiming(); + } + Application::getInstance()->paintGL(); swapBuffers(); + + if (OculusManager::isConnected()) { + OculusManager::endFrameTiming(); + } } } diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 7fb098ab87..e23801754a 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -45,6 +45,8 @@ GLuint OculusManager::_vbo[ovrEye_Count]; GLuint OculusManager::_indicesVbo[ovrEye_Count]; GLsizei OculusManager::_meshSize[ovrEye_Count]; ovrFrameTiming OculusManager::_hmdFrameTiming; +unsigned int OculusManager::_frameIndex = 0; +bool OculusManager::_frameTimingActive = false; #endif @@ -169,6 +171,29 @@ bool OculusManager::isConnected() { #endif } +//Begins the frame timing for oculus prediction purposes +void OculusManager::beginFrameTiming() { +#ifdef HAVE_LIBOVR + + if (_frameTimingActive) { + printf("WARNING: Called OculusManager::beginFrameTiming() twice in a row, need to call OculusManager::endFrameTiming()."); + } + + _hmdFrameTiming = ovrHmd_BeginFrameTiming(_ovrHmd, _frameIndex); + _frameTimingActive = true; +#endif +} + +//Ends frame timing +void OculusManager::endFrameTiming() { +#ifdef HAVE_LIBOVR + ovrHmd_EndFrameTiming(_ovrHmd); + _frameIndex++; + _frameTimingActive = false; +#endif +} + +//Sets the camera FoV and aspect ratio void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenHeight) { #ifdef HAVE_LIBOVR ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, @@ -187,11 +212,15 @@ void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenH #endif } +//Displays everything for the oculus, frame timing must be active void OculusManager::display(Camera& whichCamera) { #ifdef HAVE_LIBOVR - static unsigned int frameIndex = 0; - _hmdFrameTiming = ovrHmd_BeginFrameTiming(_ovrHmd, frameIndex); - + //beginFrameTiming must be called before display + if (!_frameTimingActive) { + printf("WARNING: Called OculusManager::display() without calling OculusManager::beginFrameTiming() first."); + return; + } + ovrEyeRenderDesc eyeDesc[ovrEye_Count]; eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); @@ -225,7 +254,6 @@ void OculusManager::display(Camera& whichCamera) { glMatrixMode(GL_PROJECTION); glLoadIdentity(); glLoadMatrixf((GLfloat *)proj.M); - glTranslatef(0.0f, 0, 0); glViewport(eyeDesc[eye].DistortedViewport.Pos.x, eyeDesc[eye].DistortedViewport.Pos.y, eyeDesc[eye].DistortedViewport.Size.w, eyeDesc[eye].DistortedViewport.Size.h); @@ -270,6 +298,7 @@ void OculusManager::display(Camera& whichCamera) { _program.enableAttributeArray(_texCoord1AttributeLocation); _program.enableAttributeArray(_texCoord2AttributeLocation); + //Render the distortion meshes for each eye for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { GLfloat uvScale[2] = { _UVScaleOffset[eyeNum][0].x, _UVScaleOffset[eyeNum][0].y }; _program.setUniformValueArray(_eyeToSourceUVScaleLocation, uvScale, 1, 2); @@ -312,9 +341,6 @@ void OculusManager::display(Camera& whichCamera) { glBindBuffer(GL_ARRAY_BUFFER, 0); glBindTexture(GL_TEXTURE_2D, 0); _program.release(); - - ovrHmd_EndFrameTiming(_ovrHmd); - frameIndex++; #endif } @@ -324,6 +350,7 @@ void OculusManager::reset() { #endif } +//Gets the current predicted angles from the oculus sensors void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { #ifdef HAVE_LIBOVR ovrSensorState ss = ovrHmd_GetSensorState(_ovrHmd, _hmdFrameTiming.ScanoutMidpointSeconds); diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 3d18f762a9..8a34caf84f 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -14,15 +14,10 @@ #ifdef HAVE_LIBOVR #include -//#include <../src/OVR_CAPI.h> #endif #include "../src/Util/Util_Render_Stereo.h" -using namespace OVR::Util::Render; -#include <../src/Kernel/OVR_SysFile.h> -#include <../src/Kernel/OVR_Log.h> -#include <../src/Kernel/OVR_Timer.h> #include "renderer/ProgramObject.h" @@ -40,6 +35,10 @@ public: static bool isConnected(); + static void beginFrameTiming(); + + static void endFrameTiming(); + static void configureCamera(Camera& camera, int screenWidth, int screenHeight); static void display(Camera& whichCamera); @@ -96,6 +95,8 @@ private: static GLuint _indicesVbo[ovrEye_Count]; static GLsizei _meshSize[ovrEye_Count]; static ovrFrameTiming _hmdFrameTiming; + static unsigned int _frameIndex; + static bool _frameTimingActive; #endif }; From ad05c5382389000bbaedd6154dc2bffe6eda7913 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 11:33:54 -0700 Subject: [PATCH 23/67] moved audiomixer jbuffer stats to expanded section --- interface/src/ui/Stats.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 569dec9d44..ca77d6887c 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -286,7 +286,7 @@ void Stats::display( pingVoxel = totalPingVoxel/voxelServerCount; } - lines = _expanded ? 6 : 5; + lines = _expanded ? 6 : 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -321,24 +321,24 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); + + + static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; + + const AudioMixerJitterBuffersStats& audioMixerJitterBufferStats = + Application::getInstance()->getAudio()->getAudioMixerJitterBuffersStats(); + + char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; + char audioMixerJitterBuffersStats[30]; + sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats._avatarJitterBufferFrames * MSECS_PER_FRAME, + audioMixerJitterBufferStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats._avgJitterBufferFrames * MSECS_PER_FRAME); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); } - static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; - - const AudioMixerJitterBuffersStats& audioMixerJitterBufferStats = - Application::getInstance()->getAudio()->getAudioMixerJitterBuffersStats(); - - char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; - char audioMixerJitterBuffersStats[30]; - sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats._avatarJitterBufferFrames * MSECS_PER_FRAME, - audioMixerJitterBufferStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats._avgJitterBufferFrames * MSECS_PER_FRAME); - - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); - verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); - - verticalOffset = 0; horizontalOffset = _lastHorizontalOffset + _generalStatsWidth + _pingStatsWidth + 2; } From 07f7cc42ad5b3c624e5d7c4e2283888edd250a3b Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 12:58:23 -0700 Subject: [PATCH 24/67] Made spacebar reconnect the oculus. Code cleanup. --- interface/src/devices/OculusManager.cpp | 79 ++++++++++++++++++------- interface/src/devices/OculusManager.h | 15 +---- 2 files changed, 61 insertions(+), 33 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index e23801754a..9e4c2c1149 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -41,12 +41,13 @@ ovrHmdDesc OculusManager::_ovrHmdDesc; ovrFovPort OculusManager::_eyeFov[ovrEye_Count]; ovrSizei OculusManager::_renderTargetSize; ovrVector2f OculusManager::_UVScaleOffset[ovrEye_Count][2]; -GLuint OculusManager::_vbo[ovrEye_Count]; -GLuint OculusManager::_indicesVbo[ovrEye_Count]; -GLsizei OculusManager::_meshSize[ovrEye_Count]; +GLuint OculusManager::_vbo[ovrEye_Count] = { 0, 0 }; +GLuint OculusManager::_indicesVbo[ovrEye_Count] = { 0, 0 }; +GLsizei OculusManager::_meshSize[ovrEye_Count] = { 0, 0 }; ovrFrameTiming OculusManager::_hmdFrameTiming; unsigned int OculusManager::_frameIndex = 0; bool OculusManager::_frameTimingActive = false; +bool OculusManager::_programInitialized = false; #endif @@ -78,34 +79,69 @@ void OculusManager::connect() { ovrSensorCap_Position, ovrSensorCap_Orientation); - _program.addShaderFromSourceFile(QGLShader::Vertex, Application::resourcesPath() + "shaders/oculus.vert"); - _program.addShaderFromSourceFile(QGLShader::Fragment, Application::resourcesPath() + "shaders/oculus.frag"); - _program.link(); + if (!_programInitialized) { + // Shader program + _programInitialized = true; + _program.addShaderFromSourceFile(QGLShader::Vertex, Application::resourcesPath() + "shaders/oculus.vert"); + _program.addShaderFromSourceFile(QGLShader::Fragment, Application::resourcesPath() + "shaders/oculus.frag"); + _program.link(); + + // Uniforms + _textureLocation = _program.uniformLocation("texture"); + _eyeToSourceUVScaleLocation = _program.uniformLocation("EyeToSourceUVScale"); + _eyeToSourceUVOffsetLocation = _program.uniformLocation("EyeToSourceUVOffset"); + _eyeRotationStartLocation = _program.uniformLocation("EyeRotationStart"); + _eyeRotationEndLocation = _program.uniformLocation("EyeRotationEnd"); + + // Attributes + _positionAttributeLocation = _program.attributeLocation("position"); + _colorAttributeLocation = _program.attributeLocation("color"); + _texCoord0AttributeLocation = _program.attributeLocation("texCoord0"); + _texCoord1AttributeLocation = _program.attributeLocation("texCoord1"); + _texCoord2AttributeLocation = _program.attributeLocation("texCoord2"); + } + + //Generate the distortion VBOs generateDistortionMesh(); - // Uniforms - _textureLocation = _program.uniformLocation("texture"); - _eyeToSourceUVScaleLocation = _program.uniformLocation("EyeToSourceUVScale"); - _eyeToSourceUVOffsetLocation = _program.uniformLocation("EyeToSourceUVOffset"); - _eyeRotationStartLocation = _program.uniformLocation("EyeRotationStart"); - _eyeRotationEndLocation = _program.uniformLocation("EyeRotationEnd"); - - // Attributes - _positionAttributeLocation = _program.attributeLocation("position"); - _colorAttributeLocation = _program.attributeLocation("color"); - _texCoord0AttributeLocation = _program.attributeLocation("texCoord0"); - _texCoord1AttributeLocation = _program.attributeLocation("texCoord1"); - _texCoord2AttributeLocation = _program.attributeLocation("texCoord2"); - } else { + _isConnected = false; + ovrHmd_Destroy(_ovrHmd); ovr_Shutdown(); } #endif } +//Disconnects and deallocates the OR +void OculusManager::disconnect() { + if (_isConnected) { + _isConnected = false; + ovrHmd_Destroy(_ovrHmd); + ovr_Shutdown(); + + //Free the distortion mesh data + for (int i = 0; i < ovrEye_Count; i++) { + if (_vbo[i] != 0) { + glDeleteBuffers(1, &(_vbo[i])); + _vbo[i] = 0; + } + if (_indicesVbo[i] != 0) { + glDeleteBuffers(1, &(_indicesVbo[i])); + _indicesVbo[i] = 0; + } + } + } +} + void OculusManager::generateDistortionMesh() { #ifdef HAVE_LIBOVR + //Already have the distortion mesh + if (_vbo[0] != 0) { + printf("WARNING: Tried to generate Oculus distortion mesh twice without freeing the VBOs."); + return; + } + ovrEyeRenderDesc eyeDesc[ovrEye_Count]; eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); @@ -346,7 +382,8 @@ void OculusManager::display(Camera& whichCamera) { void OculusManager::reset() { #ifdef HAVE_LIBOVR - //_sensorFusion->Reset(); + disconnect(); + connect(); #endif } diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 8a34caf84f..ab17c94486 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -18,10 +18,8 @@ #include "../src/Util/Util_Render_Stereo.h" - #include "renderer/ProgramObject.h" - const float DEFAULT_OCULUS_UI_ANGULAR_SIZE = 72.0f; class Camera; @@ -30,19 +28,12 @@ class Camera; class OculusManager { public: static void connect(); - - static void generateDistortionMesh(); - + static void disconnect(); static bool isConnected(); - static void beginFrameTiming(); - static void endFrameTiming(); - static void configureCamera(Camera& camera, int screenWidth, int screenHeight); - static void display(Camera& whichCamera); - static void reset(); /// param \yaw[out] yaw in radians @@ -50,9 +41,8 @@ public: /// param \roll[out] roll in radians static void getEulerAngles(float& yaw, float& pitch, float& roll); - static void updateYawOffset(); - private: + static void generateDistortionMesh(); struct DistortionVertex { glm::vec2 pos; @@ -97,6 +87,7 @@ private: static ovrFrameTiming _hmdFrameTiming; static unsigned int _frameIndex; static bool _frameTimingActive; + static bool _programInitialized; #endif }; From ea2d75addcbb13d2a0c083e405c28f2a3260dc6d Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 13:05:10 -0700 Subject: [PATCH 25/67] Code cleanup --- interface/src/devices/OculusManager.cpp | 37 +++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 9e4c2c1149..80704a4356 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -41,8 +41,8 @@ ovrHmdDesc OculusManager::_ovrHmdDesc; ovrFovPort OculusManager::_eyeFov[ovrEye_Count]; ovrSizei OculusManager::_renderTargetSize; ovrVector2f OculusManager::_UVScaleOffset[ovrEye_Count][2]; -GLuint OculusManager::_vbo[ovrEye_Count] = { 0, 0 }; -GLuint OculusManager::_indicesVbo[ovrEye_Count] = { 0, 0 }; +GLuint OculusManager::_vertices[ovrEye_Count] = { 0, 0 }; +GLuint OculusManager::_indices[ovrEye_Count] = { 0, 0 }; GLsizei OculusManager::_meshSize[ovrEye_Count] = { 0, 0 }; ovrFrameTiming OculusManager::_hmdFrameTiming; unsigned int OculusManager::_frameIndex = 0; @@ -122,13 +122,13 @@ void OculusManager::disconnect() { //Free the distortion mesh data for (int i = 0; i < ovrEye_Count; i++) { - if (_vbo[i] != 0) { - glDeleteBuffers(1, &(_vbo[i])); - _vbo[i] = 0; + if (_vertices[i] != 0) { + glDeleteBuffers(1, &(_vertices[i])); + _vertices[i] = 0; } - if (_indicesVbo[i] != 0) { - glDeleteBuffers(1, &(_indicesVbo[i])); - _indicesVbo[i] = 0; + if (_indices[i] != 0) { + glDeleteBuffers(1, &(_indices[i])); + _indices[i] = 0; } } } @@ -136,8 +136,8 @@ void OculusManager::disconnect() { void OculusManager::generateDistortionMesh() { #ifdef HAVE_LIBOVR - //Already have the distortion mesh - if (_vbo[0] != 0) { + //Check if we already have the distortion mesh + if (_vertices[0] != 0) { printf("WARNING: Tried to generate Oculus distortion mesh twice without freeing the VBOs."); return; } @@ -164,6 +164,7 @@ void OculusManager::generateDistortionMesh() { DistortionVertex* pVBVerts = (DistortionVertex*)OVR_ALLOC(sizeof(DistortionVertex) * meshData.VertexCount); _meshSize[eyeNum] = meshData.IndexCount; + // Convert the oculus vertex data to the DistortionVertex format. DistortionVertex* v = pVBVerts; ovrDistortionVertex* ov = meshData.pVertexData; for (unsigned int vertNum = 0; vertNum < meshData.VertexCount; vertNum++) { @@ -182,18 +183,19 @@ void OculusManager::generateDistortionMesh() { } //vertices - glGenBuffers(1, &(_vbo[eyeNum])); - glBindBuffer(GL_ARRAY_BUFFER, _vbo[eyeNum]); + glGenBuffers(1, &(_vertices[eyeNum])); + glBindBuffer(GL_ARRAY_BUFFER, _vertices[eyeNum]); glBufferData(GL_ARRAY_BUFFER, sizeof(DistortionVertex) * meshData.VertexCount, pVBVerts, GL_STATIC_DRAW); glBindBuffer(GL_ARRAY_BUFFER, 0); - OVR_FREE(pVBVerts); //indices - glGenBuffers(1, &(_indicesVbo[eyeNum])); - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indicesVbo[eyeNum]); + glGenBuffers(1, &(_indices[eyeNum])); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indices[eyeNum]); glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(unsigned short) * meshData.IndexCount, meshData.pIndexData, GL_STATIC_DRAW); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + //Now that we have the VBOs we can get rid of the mesh data + OVR_FREE(pVBVerts); ovrHmd_DestroyDistortionMesh(&meshData); } #endif @@ -354,7 +356,7 @@ void OculusManager::display(Camera& whichCamera) { glUniformMatrix4fv(_eyeRotationEndLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[1].M); - glBindBuffer(GL_ARRAY_BUFFER, _vbo[eyeNum]); + glBindBuffer(GL_ARRAY_BUFFER, _vertices[eyeNum]); glVertexAttribPointer(_positionAttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)0); glVertexAttribPointer(_texCoord0AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)8); @@ -362,7 +364,7 @@ void OculusManager::display(Camera& whichCamera) { glVertexAttribPointer(_texCoord2AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)24); glVertexAttribPointer(_colorAttributeLocation, 4, GL_UNSIGNED_BYTE, GL_TRUE, sizeof(DistortionVertex), (void *)32); - glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indicesVbo[eyeNum]); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indices[eyeNum]); glDrawElements(GL_TRIANGLES, _meshSize[eyeNum], GL_UNSIGNED_SHORT, 0); } @@ -380,6 +382,7 @@ void OculusManager::display(Camera& whichCamera) { #endif } +//Tries to reconnect to the sensors void OculusManager::reset() { #ifdef HAVE_LIBOVR disconnect(); From 4b2d122f3d9bb3505b903aeada5456787840a8d9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 26 Jun 2014 16:12:11 -0700 Subject: [PATCH 26/67] Working on factoring out the common metavoxel endpoint/client code. --- interface/src/MetavoxelSystem.cpp | 32 ++-- interface/src/MetavoxelSystem.h | 6 +- .../metavoxels/src/DatagramSequencer.cpp | 6 + libraries/metavoxels/src/DatagramSequencer.h | 3 + libraries/metavoxels/src/Endpoint.cpp | 108 +++++++++++++ libraries/metavoxels/src/Endpoint.h | 75 +++++++++ .../metavoxels/src/MetavoxelClientManager.cpp | 151 ++++++++++++++++++ .../metavoxels/src/MetavoxelClientManager.h | 69 ++++++++ tests/metavoxels/CMakeLists.txt | 1 + tests/metavoxels/src/MetavoxelTests.cpp | 30 ++-- tests/metavoxels/src/MetavoxelTests.h | 8 +- tools/bitstream2json/CMakeLists.txt | 1 + tools/json2bitstream/CMakeLists.txt | 1 + 13 files changed, 453 insertions(+), 38 deletions(-) create mode 100644 libraries/metavoxels/src/Endpoint.cpp create mode 100644 libraries/metavoxels/src/Endpoint.h create mode 100644 libraries/metavoxels/src/MetavoxelClientManager.cpp create mode 100644 libraries/metavoxels/src/MetavoxelClientManager.h diff --git a/interface/src/MetavoxelSystem.cpp b/interface/src/MetavoxelSystem.cpp index 78c97c1703..23e32f9d2c 100644 --- a/interface/src/MetavoxelSystem.cpp +++ b/interface/src/MetavoxelSystem.cpp @@ -54,7 +54,7 @@ SharedObjectPointer MetavoxelSystem::findFirstRaySpannerIntersection( foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - MetavoxelClient* client = static_cast(node->getLinkedData()); + MetavoxelSystemClient* client = static_cast(node->getLinkedData()); if (client) { float clientDistance; SharedObjectPointer clientSpanner = client->getData().findFirstRaySpannerIntersection( @@ -76,7 +76,7 @@ void MetavoxelSystem::applyEdit(const MetavoxelEditMessage& edit, bool reliable) foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - MetavoxelClient* client = static_cast(node->getLinkedData()); + MetavoxelSystemClient* client = static_cast(node->getLinkedData()); if (client) { client->applyEdit(edit, reliable); } @@ -92,7 +92,7 @@ void MetavoxelSystem::simulate(float deltaTime) { foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - MetavoxelClient* client = static_cast(node->getLinkedData()); + MetavoxelSystemClient* client = static_cast(node->getLinkedData()); if (client) { client->simulate(deltaTime); client->guide(_simulateVisitor); @@ -153,7 +153,7 @@ void MetavoxelSystem::render() { foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - MetavoxelClient* client = static_cast(node->getLinkedData()); + MetavoxelSystemClient* client = static_cast(node->getLinkedData()); if (client) { client->guide(_renderVisitor); } @@ -164,7 +164,7 @@ void MetavoxelSystem::render() { void MetavoxelSystem::maybeAttachClient(const SharedNodePointer& node) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - node->setLinkedData(new MetavoxelClient(NodeList::getInstance()->nodeWithUUID(node->getUUID()))); + node->setLinkedData(new MetavoxelSystemClient(NodeList::getInstance()->nodeWithUUID(node->getUUID()))); } } @@ -235,7 +235,7 @@ bool MetavoxelSystem::RenderVisitor::visit(Spanner* spanner, const glm::vec3& cl return true; } -MetavoxelClient::MetavoxelClient(const SharedNodePointer& node) : +MetavoxelSystemClient::MetavoxelSystemClient(const SharedNodePointer& node) : _node(node), _sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)) { @@ -253,7 +253,7 @@ MetavoxelClient::MetavoxelClient(const SharedNodePointer& node) : _receiveRecords.append(receiveRecord); } -MetavoxelClient::~MetavoxelClient() { +MetavoxelSystemClient::~MetavoxelSystemClient() { // close the session Bitstream& out = _sequencer.startPacket(); out << QVariant::fromValue(CloseSessionMessage()); @@ -265,12 +265,12 @@ static MetavoxelLOD getLOD() { return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD); } -void MetavoxelClient::guide(MetavoxelVisitor& visitor) { +void MetavoxelSystemClient::guide(MetavoxelVisitor& visitor) { visitor.setLOD(getLOD()); _data.guide(visitor); } -void MetavoxelClient::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { +void MetavoxelSystemClient::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { if (reliable) { _sequencer.getReliableOutputChannel()->sendMessage(QVariant::fromValue(edit)); @@ -283,7 +283,7 @@ void MetavoxelClient::applyEdit(const MetavoxelEditMessage& edit, bool reliable) } } -void MetavoxelClient::simulate(float deltaTime) { +void MetavoxelSystemClient::simulate(float deltaTime) { Bitstream& out = _sequencer.startPacket(); ClientStateMessage state = { getLOD() }; @@ -295,19 +295,19 @@ void MetavoxelClient::simulate(float deltaTime) { _sendRecords.append(record); } -int MetavoxelClient::parseData(const QByteArray& packet) { +int MetavoxelSystemClient::parseData(const QByteArray& packet) { // process through sequencer QMetaObject::invokeMethod(&_sequencer, "receivedDatagram", Q_ARG(const QByteArray&, packet)); Application::getInstance()->getBandwidthMeter()->inputStream(BandwidthMeter::METAVOXELS).updateValue(packet.size()); return packet.size(); } -void MetavoxelClient::sendData(const QByteArray& data) { +void MetavoxelSystemClient::sendData(const QByteArray& data) { NodeList::getInstance()->writeDatagram(data, _node); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::METAVOXELS).updateValue(data.size()); } -void MetavoxelClient::readPacket(Bitstream& in) { +void MetavoxelSystemClient::readPacket(Bitstream& in) { QVariant message; in >> message; handleMessage(message, in); @@ -324,15 +324,15 @@ void MetavoxelClient::readPacket(Bitstream& in) { } } -void MetavoxelClient::clearSendRecordsBefore(int index) { +void MetavoxelSystemClient::clearSendRecordsBefore(int index) { _sendRecords.erase(_sendRecords.begin(), _sendRecords.begin() + index + 1); } -void MetavoxelClient::clearReceiveRecordsBefore(int index) { +void MetavoxelSystemClient::clearReceiveRecordsBefore(int index) { _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1); } -void MetavoxelClient::handleMessage(const QVariant& message, Bitstream& in) { +void MetavoxelSystemClient::handleMessage(const QVariant& message, Bitstream& in) { int userType = message.userType(); if (userType == MetavoxelDeltaMessage::Type) { _data.readDelta(_receiveRecords.first().data, _receiveRecords.first().lod, in, _sendRecords.first().lod); diff --git a/interface/src/MetavoxelSystem.h b/interface/src/MetavoxelSystem.h index f98a260ab1..f73f609732 100644 --- a/interface/src/MetavoxelSystem.h +++ b/interface/src/MetavoxelSystem.h @@ -89,13 +89,13 @@ private: }; /// A client session associated with a single server. -class MetavoxelClient : public NodeData { +class MetavoxelSystemClient : public NodeData { Q_OBJECT public: - MetavoxelClient(const SharedNodePointer& node); - virtual ~MetavoxelClient(); + MetavoxelSystemClient(const SharedNodePointer& node); + virtual ~MetavoxelSystemClient(); MetavoxelData& getData() { return _data; } diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp index a75a4bf95b..eb02497321 100644 --- a/libraries/metavoxels/src/DatagramSequencer.cpp +++ b/libraries/metavoxels/src/DatagramSequencer.cpp @@ -140,6 +140,12 @@ void DatagramSequencer::endPacket() { _outgoingPacketStream.device()->seek(0); } +void DatagramSequencer::cancelPacket() { + _outputStream.reset(); + _outputStream.getAndResetWriteMappings(); + _outgoingPacketStream.device()->seek(0); +} + /// Simple RAII-style object to keep a device open when in scope. class QIODeviceOpener { public: diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index 9a4cd1334b..aa8b6907ff 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -111,6 +111,9 @@ public: /// Sends the packet currently being written. void endPacket(); + /// Cancels the packet currently being written. + void cancelPacket(); + /// Processes a datagram received from the other party, emitting readyToRead when the entire packet /// has been successfully assembled. Q_INVOKABLE void receivedDatagram(const QByteArray& datagram); diff --git a/libraries/metavoxels/src/Endpoint.cpp b/libraries/metavoxels/src/Endpoint.cpp new file mode 100644 index 0000000000..a13351e671 --- /dev/null +++ b/libraries/metavoxels/src/Endpoint.cpp @@ -0,0 +1,108 @@ +// +// Endpoint.cpp +// libraries/metavoxels/src +// +// Created by Andrzej Kapolka on 6/26/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include "Endpoint.h" + +Endpoint::Endpoint(const SharedNodePointer& node) : + _node(node), + _sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)) { + + connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendDatagram(const QByteArray&))); + connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readMessage(Bitstream&))); + connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int))); + connect(&_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int))); + + // insert the baseline send and receive records + _sendRecords.append(maybeCreateSendRecord(true)); + _receiveRecords.append(maybeCreateReceiveRecord(true)); +} + +Endpoint::~Endpoint() { + foreach (PacketRecord* record, _sendRecords) { + delete record; + } + foreach (PacketRecord* record, _receiveRecords) { + delete record; + } +} + +void Endpoint::update() { + Bitstream& out = _sequencer.startPacket(); + out << QVariant(); + _sequencer.endPacket(); + + // record the send + _sendRecords.append(maybeCreateSendRecord()); +} + +int Endpoint::parseData(const QByteArray& packet) { + // process through sequencer + _sequencer.receivedDatagram(packet); + return packet.size(); +} + +void Endpoint::sendDatagram(const QByteArray& data) { + NodeList::getInstance()->writeDatagram(data, _node); +} + +void Endpoint::readMessage(Bitstream& in) { + QVariant message; + in >> message; + handleMessage(message, in); + + // record the receipt + _receiveRecords.append(maybeCreateReceiveRecord()); +} + +void Endpoint::clearSendRecordsBefore(int index) { + QList::iterator end = _sendRecords.begin() + index + 1; + for (QList::const_iterator it = _sendRecords.begin(); it != end; it++) { + delete *it; + } + _sendRecords.erase(_sendRecords.begin(), end); +} + +void Endpoint::clearReceiveRecordsBefore(int index) { + QList::iterator end = _receiveRecords.begin() + index + 1; + for (QList::const_iterator it = _receiveRecords.begin(); it != end; it++) { + delete *it; + } + _receiveRecords.erase(_receiveRecords.begin(), end); +} + +void Endpoint::writeUpdateMessage(Bitstream& out) { + out << QVariant(); +} + +void Endpoint::handleMessage(const QVariant& message, Bitstream& in) { + if (message.userType() == QMetaType::QVariantList) { + foreach (const QVariant& element, message.toList()) { + handleMessage(element, in); + } + } +} + +PacketRecord* Endpoint::maybeCreateSendRecord(bool baseline) const { + return NULL; +} + +PacketRecord* Endpoint::maybeCreateReceiveRecord(bool baseline) const { + return NULL; +} + +PacketRecord::PacketRecord(const MetavoxelLOD& lod) : + _lod(lod) { +} + +PacketRecord::~PacketRecord() { +} diff --git a/libraries/metavoxels/src/Endpoint.h b/libraries/metavoxels/src/Endpoint.h new file mode 100644 index 0000000000..ac88376844 --- /dev/null +++ b/libraries/metavoxels/src/Endpoint.h @@ -0,0 +1,75 @@ +// +// Endpoint.h +// libraries/metavoxels/src +// +// Created by Andrzej Kapolka on 6/26/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_Endpoint_h +#define hifi_Endpoint_h + +#include + +#include "DatagramSequencer.h" +#include "MetavoxelData.h" + +class PacketRecord; + +/// Base class for communication endpoints: clients or servers. +class Endpoint : public NodeData { + Q_OBJECT + +public: + + Endpoint(const SharedNodePointer& node); + virtual ~Endpoint(); + + void update(); + + virtual int parseData(const QByteArray& packet); + +protected slots: + + virtual void sendDatagram(const QByteArray& data); + virtual void readMessage(Bitstream& in); + + void clearSendRecordsBefore(int index); + void clearReceiveRecordsBefore(int index); + +protected: + + virtual void writeUpdateMessage(Bitstream& out); + virtual void handleMessage(const QVariant& message, Bitstream& in); + + virtual PacketRecord* maybeCreateSendRecord(bool baseline = false) const; + virtual PacketRecord* maybeCreateReceiveRecord(bool baseline = false) const; + + PacketRecord* getLastAcknowledgedSendRecord() const { return _sendRecords.first(); } + PacketRecord* getLastAcknowledgedReceiveRecord() const { return _receiveRecords.first(); } + + SharedNodePointer _node; + DatagramSequencer _sequencer; + + QList _sendRecords; + QList _receiveRecords; +}; + +/// Base class for packet records. +class PacketRecord { +public: + + PacketRecord(const MetavoxelLOD& lod = MetavoxelLOD()); + virtual ~PacketRecord(); + + const MetavoxelLOD& getLOD() const { return _lod; } + +private: + + MetavoxelLOD _lod; +}; + +#endif // hifi_Endpoint_h diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp new file mode 100644 index 0000000000..206e15becd --- /dev/null +++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp @@ -0,0 +1,151 @@ +// +// MetavoxelClientManager.cpp +// libraries/metavoxels/src +// +// Created by Andrzej Kapolka on 6/26/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MetavoxelClientManager.h" +#include "MetavoxelMessages.h" + +void MetavoxelClientManager::init() { + connect(NodeList::getInstance(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(maybeAttachClient(const SharedNodePointer&))); +} + +void MetavoxelClientManager::update() { + foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + if (node->getType() == NodeType::MetavoxelServer) { + QMutexLocker locker(&node->getMutex()); + MetavoxelClient* client = static_cast(node->getLinkedData()); + if (client) { + client->update(); + } + } + } +} + +SharedObjectPointer MetavoxelClientManager::findFirstRaySpannerIntersection(const glm::vec3& origin, + const glm::vec3& direction, const AttributePointer& attribute, float& distance) { + SharedObjectPointer closestSpanner; + float closestDistance = FLT_MAX; + foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + if (node->getType() == NodeType::MetavoxelServer) { + QMutexLocker locker(&node->getMutex()); + MetavoxelClient* client = static_cast(node->getLinkedData()); + if (client) { + float clientDistance; + SharedObjectPointer clientSpanner = client->getData().findFirstRaySpannerIntersection( + origin, direction, attribute, clientDistance); + if (clientSpanner && clientDistance < closestDistance) { + closestSpanner = clientSpanner; + closestDistance = clientDistance; + } + } + } + } + if (closestSpanner) { + distance = closestDistance; + } + return closestSpanner; +} + +void MetavoxelClientManager::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { + foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + if (node->getType() == NodeType::MetavoxelServer) { + QMutexLocker locker(&node->getMutex()); + MetavoxelClient* client = static_cast(node->getLinkedData()); + if (client) { + client->applyEdit(edit, reliable); + } + } + } +} + +MetavoxelLOD MetavoxelClientManager::getLOD() const { + return MetavoxelLOD(); +} + +void MetavoxelClientManager::maybeAttachClient(const SharedNodePointer& node) { + if (node->getType() == NodeType::MetavoxelServer) { + QMutexLocker locker(&node->getMutex()); + node->setLinkedData(new MetavoxelClient(node, this)); + } +} + +MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientManager* manager) : + Endpoint(node), + _manager(manager) { +} + +void MetavoxelClient::guide(MetavoxelVisitor& visitor) { + visitor.setLOD(_manager->getLOD()); + _data.guide(visitor); +} + +void MetavoxelClient::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { + if (reliable) { + _sequencer.getReliableOutputChannel()->sendMessage(QVariant::fromValue(edit)); + + } else { + // apply immediately to local tree + edit.apply(_data, _sequencer.getWeakSharedObjectHash()); + + // start sending it out + _sequencer.sendHighPriorityMessage(QVariant::fromValue(edit)); + } +} + +void MetavoxelClient::writeUpdateMessage(Bitstream& out) { + ClientStateMessage state = { _manager->getLOD() }; + out << QVariant::fromValue(state); +} + +void MetavoxelClient::readMessage(Bitstream& in) { + Endpoint::readMessage(in); + + // reapply local edits + foreach (const DatagramSequencer::HighPriorityMessage& message, _sequencer.getHighPriorityMessages()) { + if (message.data.userType() == MetavoxelEditMessage::Type) { + message.data.value().apply(_data, _sequencer.getWeakSharedObjectHash()); + } + } +} + +class ReceiveRecord : public PacketRecord { +public: + + ReceiveRecord(const MetavoxelLOD& lod = MetavoxelLOD(), const MetavoxelData& data = MetavoxelData()); + + const MetavoxelData& getData() const { return _data; } + +private: + + MetavoxelData _data; +}; + +ReceiveRecord::ReceiveRecord(const MetavoxelLOD& lod, const MetavoxelData& data) : + PacketRecord(lod), + _data(data) { +} + +void MetavoxelClient::handleMessage(const QVariant& message, Bitstream& in) { + if (message.userType() == MetavoxelDeltaMessage::Type) { + ReceiveRecord* receiveRecord = static_cast(getLastAcknowledgedReceiveRecord()); + _data.readDelta(receiveRecord->getData(), receiveRecord->getLOD(), in, getLastAcknowledgedSendRecord()->getLOD()); + + } else { + Endpoint::handleMessage(message, in); + } +} + +PacketRecord* MetavoxelClient::maybeCreateSendRecord(bool baseline) const { + return baseline ? new PacketRecord() : new PacketRecord(_manager->getLOD()); +} + +PacketRecord* MetavoxelClient::maybeCreateReceiveRecord(bool baseline) const { + return baseline ? new ReceiveRecord() : new ReceiveRecord(getLastAcknowledgedSendRecord()->getLOD(), _data); +} diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h new file mode 100644 index 0000000000..cd34cdeab2 --- /dev/null +++ b/libraries/metavoxels/src/MetavoxelClientManager.h @@ -0,0 +1,69 @@ +// +// MetavoxelClientManager.h +// libraries/metavoxels/src +// +// Created by Andrzej Kapolka on 6/26/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MetavoxelClientManager_h +#define hifi_MetavoxelClientManager_h + +#include "Endpoint.h" + +class MetavoxelEditMessage; + +/// Manages the set of connected metavoxel clients. +class MetavoxelClientManager : public QObject { + Q_OBJECT + +public: + + virtual void init(); + virtual void update(); + + SharedObjectPointer findFirstRaySpannerIntersection(const glm::vec3& origin, const glm::vec3& direction, + const AttributePointer& attribute, float& distance); + + Q_INVOKABLE void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); + + virtual MetavoxelLOD getLOD() const; + +private slots: + + void maybeAttachClient(const SharedNodePointer& node); +}; + +/// Base class for metavoxel clients. +class MetavoxelClient : public Endpoint { + Q_OBJECT + +public: + + MetavoxelClient(const SharedNodePointer& node, MetavoxelClientManager* manager); + + MetavoxelData& getData() { return _data; } + + void guide(MetavoxelVisitor& visitor); + + void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); + +protected: + + virtual void writeUpdateMessage(Bitstream& out); + virtual void readMessage(Bitstream& in); + virtual void handleMessage(const QVariant& message, Bitstream& in); + + virtual PacketRecord* maybeCreateSendRecord(bool baseline) const; + virtual PacketRecord* maybeCreateReceiveRecord(bool baseline) const; + +private: + + MetavoxelClientManager* _manager; + MetavoxelData _data; +}; + +#endif // hifi_MetavoxelClientManager_h diff --git a/tests/metavoxels/CMakeLists.txt b/tests/metavoxels/CMakeLists.txt index 5f11c7290e..ca141c5137 100644 --- a/tests/metavoxels/CMakeLists.txt +++ b/tests/metavoxels/CMakeLists.txt @@ -27,6 +27,7 @@ include_glm(${TARGET_NAME} "${ROOT_DIR}") # link in the shared libraries include(${MACRO_DIR}/LinkHifiLibrary.cmake) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(networking ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") IF (WIN32) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 68aaf7ec70..dc44134600 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -463,7 +463,7 @@ bool MetavoxelTests::run() { qDebug(); // create two endpoints with the same header - Endpoint alice(datagramHeader), bob(datagramHeader); + TestEndpoint alice(datagramHeader), bob(datagramHeader); alice.setOther(&bob); bob.setOther(&alice); @@ -497,7 +497,7 @@ bool MetavoxelTests::run() { datagramsReceived = bytesReceived = maxDatagramsPerPacket = maxBytesPerPacket = 0; // create two endpoints with the same header - Endpoint alice(datagramHeader, Endpoint::CONGESTION_MODE), bob(datagramHeader, Endpoint::CONGESTION_MODE); + TestEndpoint alice(datagramHeader, TestEndpoint::CONGESTION_MODE), bob(datagramHeader, TestEndpoint::CONGESTION_MODE); alice.setOther(&bob); bob.setOther(&alice); @@ -537,8 +537,8 @@ bool MetavoxelTests::run() { datagramsSent = bytesSent = datagramsReceived = bytesReceived = maxDatagramsPerPacket = maxBytesPerPacket = 0; // create client and server endpoints - Endpoint client(datagramHeader, Endpoint::METAVOXEL_CLIENT_MODE); - Endpoint server(datagramHeader, Endpoint::METAVOXEL_SERVER_MODE); + TestEndpoint client(datagramHeader, TestEndpoint::METAVOXEL_CLIENT_MODE); + TestEndpoint server(datagramHeader, TestEndpoint::METAVOXEL_SERVER_MODE); client.setOther(&server); server.setOther(&client); @@ -599,7 +599,7 @@ int RandomVisitor::visit(MetavoxelInfo& info) { return STOP_RECURSION; } -Endpoint::Endpoint(const QByteArray& datagramHeader, Mode mode) : +TestEndpoint::TestEndpoint(const QByteArray& datagramHeader, Mode mode) : _mode(mode), _sequencer(new DatagramSequencer(datagramHeader, this)), _highPriorityMessagesToSend(0.0f), @@ -800,7 +800,7 @@ int MutateVisitor::visit(MetavoxelInfo& info) { return STOP_RECURSION; } -bool Endpoint::simulate(int iterationNumber) { +bool TestEndpoint::simulate(int iterationNumber) { // update/send our delayed datagrams for (QList::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) { if (it->second-- == 1) { @@ -940,7 +940,7 @@ bool Endpoint::simulate(int iterationNumber) { return false; } -void Endpoint::sendDatagram(const QByteArray& datagram) { +void TestEndpoint::sendDatagram(const QByteArray& datagram) { datagramsSent++; bytesSent += datagram.size(); @@ -970,7 +970,7 @@ void Endpoint::sendDatagram(const QByteArray& datagram) { _other->receiveDatagram(datagram); } -void Endpoint::handleHighPriorityMessage(const QVariant& message) { +void TestEndpoint::handleHighPriorityMessage(const QVariant& message) { if (message.userType() == ClearSharedObjectMessage::Type) { return; } @@ -984,7 +984,7 @@ void Endpoint::handleHighPriorityMessage(const QVariant& message) { highPriorityMessagesReceived++; } -void Endpoint::readMessage(Bitstream& in) { +void TestEndpoint::readMessage(Bitstream& in) { if (_mode == CONGESTION_MODE) { QVariant message; in >> message; @@ -1056,7 +1056,7 @@ void Endpoint::readMessage(Bitstream& in) { exit(true); } -void Endpoint::handleReliableMessage(const QVariant& message) { +void TestEndpoint::handleReliableMessage(const QVariant& message) { if (message.userType() == ClearSharedObjectMessage::Type || message.userType() == ClearMainChannelSharedObjectMessage::Type) { return; @@ -1071,7 +1071,7 @@ void Endpoint::handleReliableMessage(const QVariant& message) { reliableMessagesReceived++; } -void Endpoint::readReliableChannel() { +void TestEndpoint::readReliableChannel() { CircularBuffer& buffer = _sequencer->getReliableInputChannel(1)->getBuffer(); QByteArray bytes = buffer.read(buffer.bytesAvailable()); if (_other->_dataStreamed.size() < bytes.size()) { @@ -1085,15 +1085,15 @@ void Endpoint::readReliableChannel() { streamedBytesReceived += bytes.size(); } -void Endpoint::clearSendRecordsBefore(int index) { +void TestEndpoint::clearSendRecordsBefore(int index) { _sendRecords.erase(_sendRecords.begin(), _sendRecords.begin() + index + 1); } -void Endpoint::clearReceiveRecordsBefore(int index) { +void TestEndpoint::clearReceiveRecordsBefore(int index) { _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1); } -void Endpoint::receiveDatagram(const QByteArray& datagram) { +void TestEndpoint::receiveDatagram(const QByteArray& datagram) { if (_mode == CONGESTION_MODE) { if (datagram.size() <= _remainingPipelineCapacity) { // have to copy the datagram; the one we're passed is a reference to a shared buffer @@ -1107,7 +1107,7 @@ void Endpoint::receiveDatagram(const QByteArray& datagram) { } } -void Endpoint::handleMessage(const QVariant& message, Bitstream& in) { +void TestEndpoint::handleMessage(const QVariant& message, Bitstream& in) { int userType = message.userType(); if (userType == ClientStateMessage::Type) { ClientStateMessage state = message.value(); diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index f9a314dcd7..96d719a875 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -35,16 +35,16 @@ public: }; /// Represents a simulated endpoint. -class Endpoint : public QObject { +class TestEndpoint : public QObject { Q_OBJECT public: enum Mode { BASIC_PEER_MODE, CONGESTION_MODE, METAVOXEL_SERVER_MODE, METAVOXEL_CLIENT_MODE }; - Endpoint(const QByteArray& datagramHeader, Mode mode = BASIC_PEER_MODE); + TestEndpoint(const QByteArray& datagramHeader, Mode mode = BASIC_PEER_MODE); - void setOther(Endpoint* other) { _other = other; } + void setOther(TestEndpoint* other) { _other = other; } /// Perform a simulation step. /// \return true if failure was detected @@ -97,7 +97,7 @@ private: SharedObjectPointer _sphere; - Endpoint* _other; + TestEndpoint* _other; typedef QPair ByteArrayIntPair; QList _delayedDatagrams; diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt index fde80b4d33..73fbe92204 100644 --- a/tools/bitstream2json/CMakeLists.txt +++ b/tools/bitstream2json/CMakeLists.txt @@ -18,6 +18,7 @@ include(${MACRO_DIR}/SetupHifiProject.cmake) setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(networking ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") include(${MACRO_DIR}/IncludeGLM.cmake) diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt index 51382d5858..1dc01c4c7f 100644 --- a/tools/json2bitstream/CMakeLists.txt +++ b/tools/json2bitstream/CMakeLists.txt @@ -18,6 +18,7 @@ include(${MACRO_DIR}/SetupHifiProject.cmake) setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(networking ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") include(${MACRO_DIR}/IncludeGLM.cmake) From cb48825561c20f462a91f0916ffb7d5f83e9e7e1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 16:52:23 -0700 Subject: [PATCH 27/67] added seq number to all Audio types, untested --- assignment-client/src/Agent.cpp | 2 + assignment-client/src/audio/AudioMixer.cpp | 26 +++++++--- .../src/audio/AudioMixerClientData.cpp | 12 ++++- .../src/audio/AudioMixerClientData.h | 10 +++- interface/src/Application.cpp | 3 ++ interface/src/Audio.cpp | 20 +++++--- interface/src/Audio.h | 3 +- libraries/audio/src/AudioInjector.cpp | 10 ++++ libraries/audio/src/AudioRingBuffer.cpp | 1 + .../audio/src/PositionalAudioRingBuffer.cpp | 3 ++ libraries/networking/src/PacketHeaders.cpp | 2 + .../networking/src/SequenceNumbersStats.cpp | 2 +- libraries/script-engine/src/ScriptEngine.cpp | 49 ++++++++++++++----- libraries/script-engine/src/ScriptEngine.h | 3 ++ 14 files changed, 117 insertions(+), 29 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index fcc2288356..561acc0eb0 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -147,6 +147,8 @@ void Agent::readPendingDatagrams() { } } else if (datagramPacketType == PacketTypeMixedAudio) { + // TODO: track sequence numbers for mixed audio??? + // parse the data and grab the average loudness _receivedAudioBuffer.parseData(receivedPacket); diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f41c3392c7..6b43fb37e0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -519,7 +519,7 @@ void AudioMixer::run() { QElapsedTimer timer; timer.start(); - char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(quint16) + numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)]; int usecToSleep = BUFFER_SEND_INTERVAL_USECS; @@ -602,18 +602,32 @@ void AudioMixer::run() { foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData() && ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) { + + AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); + prepareMixForListeningNode(node.data()); + // pack header int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio); + char* dataAt = clientMixBuffer + numBytesPacketHeader; - memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); - nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node); + // pack sequence number + quint16 sequence = nodeData->getOutgoingSequenceNumber(); + memcpy(dataAt, &sequence, sizeof(quint16)); + dataAt += sizeof(quint16); + + // pack mixed audio samples + memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO); + dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; + + // send mixed audio packet + nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + nodeData->incrementOutgoingSequenceNumber(); // send an audio stream stats packet if it's time if (sendAudioStreamStats) { - int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData()) - ->encodeAudioStreamStatsPacket(audioStreamStatsPacket); - nodeList->writeDatagram(audioStreamStatsPacket, numBytesWritten, node); + int numBytesAudioStreamStatsPacket = nodeData->encodeAudioStreamStatsPacket(audioStreamStatsPacket); + nodeList->writeDatagram(audioStreamStatsPacket, numBytesAudioStreamStatsPacket, node); } ++_sumListeners; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index f35034eab3..8f50647463 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -20,7 +20,9 @@ #include "AudioMixerClientData.h" AudioMixerClientData::AudioMixerClientData() : - _ringBuffers() + _ringBuffers(), + _outgoingSequenceNumber(0), + _incomingAvatarSequenceNumberStats() { } @@ -49,13 +51,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { int numBytesPacketHeader = numBytesForPacketHeader(packet); const char* sequenceAt = packet.constData() + numBytesPacketHeader; quint16 sequence = *(reinterpret_cast(sequenceAt)); - _sequenceNumberStats.sequenceNumberReceived(sequence); PacketType packetType = packetTypeForPacket(packet); if (packetType == PacketTypeMicrophoneAudioWithEcho || packetType == PacketTypeMicrophoneAudioNoEcho || packetType == PacketTypeSilentAudioFrame) { + _incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence); + // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); @@ -85,6 +88,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // grab the stream identifier for this injected audio QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID)); + _incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); + InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; for (int i = 0; i < _ringBuffers.size(); i++) { @@ -140,6 +145,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector && audioBuffer->hasStarted() && audioBuffer->isStarved()) { // this is an empty audio buffer that has starved, safe to delete + // also delete its sequence number stats + QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier(); + _incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier); delete audioBuffer; i = _ringBuffers.erase(i); continue; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 508572db43..c418df7905 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -38,11 +38,17 @@ public: QString getJitterBufferStatsString() const; - const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } + void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; } + + quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; } + //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } private: QList _ringBuffers; - SequenceNumberStats _sequenceNumberStats; + + quint16 _outgoingSequenceNumber; + SequenceNumberStats _incomingAvatarSequenceNumberStats; + QHash _incomingInjectedSequenceNumberStatsMap; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f40185f34f..80ef1ff6a1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3608,6 +3608,9 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript // when the application is about to quit, stop our script engine so it unwinds properly connect(this, SIGNAL(aboutToQuit()), scriptEngine, SLOT(stop())); + NodeList* nodeList = NodeList::getInstance(); + connect(nodeList, &NodeList::nodeKilled, scriptEngine, &ScriptEngine::nodeKilled); + scriptEngine->moveToThread(workerThread); // Starts an event loop, and emits workerThread->started() diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ac7598908c..ff019712d7 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeOutputLeft(0), _scopeOutputRight(0), _audioMixerJitterBufferStats(), - _sequenceNumber(0) + _outgoingSequenceNumber(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _sequenceNumber = 0; + _outgoingSequenceNumber = 0; } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -656,8 +656,8 @@ void Audio::handleAudioInput() { char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); - // pack seq number - memcpy(currentPacketPtr, &_sequenceNumber, sizeof(quint16)); + // pack sequence number + memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16)); currentPacketPtr += sizeof(quint16); // set the mono/stereo byte @@ -672,13 +672,13 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); - _sequenceNumber++; + _outgoingSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); } else { // reset seq numbers if there's no connection with an audiomixer - _sequenceNumber = 0; + _outgoingSequenceNumber = 0; } delete[] inputAudioSamples; } @@ -827,6 +827,14 @@ void Audio::toggleStereoInput() { } void Audio::processReceivedAudio(const QByteArray& audioByteArray) { + + // parse sequence number for this packet + int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); + const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; + quint16 sequence = *((quint16*)sequenceAt); + _incomingSequenceNumberStats.sequenceNumberReceived(sequence); + + // parse audio data _ringBuffer.parseData(audioByteArray); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index c6fde709b8..cb04418eaf 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -239,7 +239,8 @@ private: AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; - quint16 _sequenceNumber; + quint16 _outgoingSequenceNumber; + SequenceNumberStats _incomingSequenceNumberStats; }; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 129dc47bd0..d9d4dc6d3a 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -61,6 +61,11 @@ void AudioInjector::injectAudio() { QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); QDataStream packetStream(&injectAudioPacket, QIODevice::Append); + // skip sequence number for now + int numPreSequenceNumberBytes = injectAudioPacket.size(); + packetStream.skipRawData(sizeof(quint16)); + + // pack stream identifier (a generated UUID) packetStream << QUuid::createUuid(); // pack the flag for loopback @@ -91,6 +96,7 @@ void AudioInjector::injectAudio() { bool shouldLoop = _options.getLoop(); // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks + quint16 outgoingSequenceNumber = 0; while (currentSendPosition < soundByteArray.size() && !_shouldStop) { int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, @@ -98,6 +104,9 @@ void AudioInjector::injectAudio() { // resize the QByteArray to the right size injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy); + + // pack the sequence number + memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16)); // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy); @@ -107,6 +116,7 @@ void AudioInjector::injectAudio() { // send off this audio packet nodeList->writeDatagram(injectAudioPacket, audioMixer); + outgoingSequenceNumber++; currentSendPosition += bytesToCopy; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index d6b9624695..10aa6f47cf 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -66,6 +66,7 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { } int AudioRingBuffer::parseData(const QByteArray& packet) { + // skip packet header and sequence number int numBytesBeforeAudioData = numBytesForPacketHeader(packet) + sizeof(quint16); return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData); } diff --git a/libraries/audio/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp index 1e26c708ca..e11d73358c 100644 --- a/libraries/audio/src/PositionalAudioRingBuffer.cpp +++ b/libraries/audio/src/PositionalAudioRingBuffer.cpp @@ -107,6 +107,9 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) { // skip the packet header (includes the source UUID) int readBytes = numBytesForPacketHeader(packet); + + // skip the sequence number + readBytes += sizeof(quint16); // hop over the channel flag that has already been read in AudioMixerClientData readBytes += sizeof(quint8); diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index e2bc46b3be..a5c05a6ae9 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -50,6 +50,8 @@ PacketVersion versionForPacketType(PacketType type) { case PacketTypeMicrophoneAudioNoEcho: case PacketTypeMicrophoneAudioWithEcho: case PacketTypeSilentAudioFrame: + return 2; + case PacketTypeMixedAudio: return 1; case PacketTypeAvatarData: return 3; diff --git a/libraries/networking/src/SequenceNumbersStats.cpp b/libraries/networking/src/SequenceNumbersStats.cpp index b1b6adc9ac..eadf08e378 100644 --- a/libraries/networking/src/SequenceNumbersStats.cpp +++ b/libraries/networking/src/SequenceNumbersStats.cpp @@ -59,7 +59,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa } } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { // ignore packet if gap is unreasonable - qDebug() << "ignoring unreasonable packet... sequence:" << incoming + qDebug() << "ignoring unreasonable sequence number:" << incoming << "previous:" << _lastReceived; _numUnreasonable++; return; diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index b0cce114a9..b142819260 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -459,32 +459,55 @@ void ScriptEngine::run() { } } - QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame - ? PacketTypeSilentAudioFrame - : PacketTypeMicrophoneAudioNoEcho); + char audioPacket[MAX_PACKET_SIZE]; - QDataStream packetStream(&audioPacket, QIODevice::Append); + // pack header + int numBytesPacketHeader = populatePacketHeader(audioPacket, silentFrame + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); + char* dataAt = audioPacket + numBytesPacketHeader; + + // skip over sequence number for now; will be packed when destination node is known + char* sequenceAt = dataAt; + dataAt += sizeof(quint16); // use the orientation and position of this avatar for the source of this audio - packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); + memcpy(dataAt, &_avatarData->getPosition(), sizeof(glm::vec3)); + dataAt += sizeof(glm::vec3); + glm::quat headOrientation = _avatarData->getHeadOrientation(); - packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); + memcpy(dataAt, &headOrientation, sizeof(glm::quat)); + dataAt += sizeof(glm::quat); if (silentFrame) { if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here break; } - // write the number of silent samples so the audio-mixer can uphold timing - packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); + memcpy(dataAt, &SCRIPT_AUDIO_BUFFER_SAMPLES, sizeof(int16_t)); + dataAt += sizeof(int16_t); } else if (nextSoundOutput) { // write the raw audio data - packetStream.writeRawData(reinterpret_cast(nextSoundOutput), - numAvailableSamples * sizeof(int16_t)); + int numAvailableBytes = numAvailableSamples * sizeof(int16_t); + memcpy(dataAt, nextSoundOutput, numAvailableBytes); + dataAt += numAvailableBytes; } - nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); + // write audio packet to AudioMixer nodes + int audioPacketSize = dataAt - audioPacket; + NodeList* nodeList = NodeList::getInstance(); + foreach(const SharedNodePointer& node, nodeList->getNodeHash()) { + // only send to nodes of type AudioMixer + if (node->getType() == NodeType::AudioMixer) { + // pack sequence number + quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++; + memcpy(sequenceAt, &sequence, sizeof(quint16)); + + // send audio packet + nodeList->writeDatagram(audioPacket, audioPacketSize, node); + } + } } } @@ -658,3 +681,7 @@ void ScriptEngine::include(const QString& includeFile) { emit errorMessage("Uncaught exception at (" + includeFile + ") line" + QString::number(line) + ":" + result.toString()); } } + +void ScriptEngine::nodeKilled(SharedNodePointer node) { + _outgoingSequenceNumbers.remove(node->getUUID()); +} diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index bf2ac40568..f1aaf6f0bb 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -100,6 +100,8 @@ public slots: void include(const QString& includeFile); void print(const QString& message); + void nodeKilled(SharedNodePointer node); + signals: void update(float deltaTime); void scriptEnding(); @@ -146,6 +148,7 @@ private: ScriptUUID _uuidLibrary; AnimationCache _animationCache; + QHash _outgoingSequenceNumbers; }; #endif // hifi_ScriptEngine_h From c5474f95bdf6b3a779aab1f41dcc2af43da799a5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 26 Jun 2014 16:55:29 -0700 Subject: [PATCH 28/67] forgot to skip seq number in InjectedAudioRingBuffer::parseData() --- libraries/audio/src/InjectedAudioRingBuffer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/audio/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp index 9b6529b49f..0d7cea356b 100644 --- a/libraries/audio/src/InjectedAudioRingBuffer.cpp +++ b/libraries/audio/src/InjectedAudioRingBuffer.cpp @@ -38,6 +38,9 @@ int InjectedAudioRingBuffer::parseData(const QByteArray& packet) { QDataStream packetStream(packet); packetStream.skipRawData(numBytesForPacketHeader(packet)); + // push past the sequence number + packetStream.skipRawData(sizeof(quint16)); + // push past the stream identifier packetStream.skipRawData(NUM_BYTES_RFC4122_UUID); From 67c5281205c330393c65642a4343947156509272 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Thu, 26 Jun 2014 17:10:53 -0700 Subject: [PATCH 29/67] Fixed CMake for oculus. Gave Oculus its own camera for minimum latency. --- cmake/modules/FindLibOVR.cmake | 11 ++--- interface/CMakeLists.txt | 6 ++- interface/resources/shaders/oculus.vert | 1 - interface/src/Application.cpp | 4 +- interface/src/devices/OculusManager.cpp | 60 ++++++++++++++++++------- interface/src/devices/OculusManager.h | 8 ++-- 6 files changed, 57 insertions(+), 33 deletions(-) diff --git a/cmake/modules/FindLibOVR.cmake b/cmake/modules/FindLibOVR.cmake index f65088e817..91714e8a59 100644 --- a/cmake/modules/FindLibOVR.cmake +++ b/cmake/modules/FindLibOVR.cmake @@ -42,14 +42,11 @@ else (LIBOVR_LIBRARIES AND LIBOVR_INCLUDE_DIRS) if (UDEV_LIBRARY AND XINERAMA_LIBRARY AND OVR_LIBRARY) set(LIBOVR_LIBRARIES "${OVR_LIBRARY};${UDEV_LIBRARY};${XINERAMA_LIBRARY}" CACHE INTERNAL "Oculus libraries") endif (UDEV_LIBRARY AND XINERAMA_LIBRARY AND OVR_LIBRARY) - elseif (WIN32) - if (CMAKE_BUILD_TYPE MATCHES DEBUG) - set(WINDOWS_LIBOVR_NAME "libovrd.lib") - else() - set(WINDOWS_LIBOVR_NAME "libovr.lib") - endif() + elseif (WIN32) + find_library(LIBOVR_RELEASE_LIBRARIES "Lib/Win32/libovr.lib" HINTS ${LIBOVR_SEARCH_DIRS}) + find_library(LIBOVR_DEBUG_LIBRARIES "Lib/Win32/libovrd.lib" HINTS ${LIBOVR_SEARCH_DIRS}) - find_library(LIBOVR_LIBRARIES "Lib/Win32/${WINDOWS_LIBOVR_NAME}" HINTS ${LIBOVR_SEARCH_DIRS}) + set(LIBOVR_LIBRARIES "${LIBOVR_RELEASE_LIBRARIES} ${LIBOVR_DEBUG_LIBRARIES}") endif () if (LIBOVR_INCLUDE_DIRS AND LIBOVR_LIBRARIES) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index cf203c41d9..855a97374f 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -183,7 +183,11 @@ if (LIBOVR_FOUND AND NOT DISABLE_LIBOVR) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${LIBOVR_INCLUDE_DIRS}") endif () - target_link_libraries(${TARGET_NAME} "${LIBOVR_LIBRARIES}") + if (WIN32) + target_link_libraries(${TARGET_NAME} optimized "${LIBOVR_RELEASE_LIBRARIES}" debug "${LIBOVR_DEBUG_LIBRARIES}") + else() + target_link_libraries(${TARGET_NAME} "${LIBOVR_LIBRARIES}") + endif() endif (LIBOVR_FOUND AND NOT DISABLE_LIBOVR) # and with PrioVR library diff --git a/interface/resources/shaders/oculus.vert b/interface/resources/shaders/oculus.vert index bc9eba4cf3..389615dde1 100644 --- a/interface/resources/shaders/oculus.vert +++ b/interface/resources/shaders/oculus.vert @@ -49,7 +49,6 @@ void main() float timewarpMixFactor = color.a; mat4 mixedEyeRot = EyeRotationStart * mat4(1.0 - timewarpMixFactor) + EyeRotationEnd * mat4(timewarpMixFactor); - oTexCoord0 = TimewarpTexCoord(texCoord0, mixedEyeRot); oTexCoord1 = TimewarpTexCoord(texCoord1, mixedEyeRot); oTexCoord2 = TimewarpTexCoord(texCoord2, mixedEyeRot); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 9bc08aa188..89f9445498 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3136,9 +3136,7 @@ void Application::resetSensors() { _faceshift.reset(); _visage.reset(); - if (OculusManager::isConnected()) { - OculusManager::reset(); - } + OculusManager::reset(); _prioVR.reset(); diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 80704a4356..42aec65d82 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -45,9 +45,11 @@ GLuint OculusManager::_vertices[ovrEye_Count] = { 0, 0 }; GLuint OculusManager::_indices[ovrEye_Count] = { 0, 0 }; GLsizei OculusManager::_meshSize[ovrEye_Count] = { 0, 0 }; ovrFrameTiming OculusManager::_hmdFrameTiming; +ovrRecti OculusManager::_eyeRenderViewport[ovrEye_Count]; unsigned int OculusManager::_frameIndex = 0; bool OculusManager::_frameTimingActive = false; bool OculusManager::_programInitialized = false; +Camera* OculusManager::_camera = NULL; #endif @@ -66,19 +68,25 @@ void OculusManager::connect() { //Get texture size ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, - _ovrHmdDesc.DefaultEyeFov[0], 1.0f); + _eyeFov[0], 1.0f); ovrSizei recommendedTex1Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Right, - _ovrHmdDesc.DefaultEyeFov[1], 1.0f); + _eyeFov[1], 1.0f); _renderTargetSize.w = recommendedTex0Size.w + recommendedTex1Size.w; _renderTargetSize.h = recommendedTex0Size.h; if (_renderTargetSize.h < recommendedTex1Size.h) { _renderTargetSize.h = recommendedTex1Size.h; } + _renderTargetSize.w = Application::getInstance()->getGLWidget()->width(); + _renderTargetSize.h = Application::getInstance()->getGLWidget()->height(); + ovrHmd_StartSensor(_ovrHmd, ovrSensorCap_Orientation | ovrSensorCap_YawCorrection | ovrSensorCap_Position, ovrSensorCap_Orientation); + if (!_camera) { + _camera = new Camera; + } if (!_programInitialized) { // Shader program @@ -146,18 +154,17 @@ void OculusManager::generateDistortionMesh() { eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); - ovrRecti eyeRenderViewport[ovrEye_Count]; - eyeRenderViewport[0].Pos = Vector2i(0, 0); - eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); - eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); - eyeRenderViewport[1].Size = eyeRenderViewport[0].Size; + _eyeRenderViewport[0].Pos = Vector2i(0, 0); + _eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); + _eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); + _eyeRenderViewport[1].Size = _eyeRenderViewport[0].Size; for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { // Allocate and generate distortion mesh vertices ovrDistortionMesh meshData; ovrHmd_CreateDistortionMesh(_ovrHmd, eyeDesc[eyeNum].Eye, eyeDesc[eyeNum].Fov, _ovrHmdDesc.DistortionCaps, &meshData); - ovrHmd_GetRenderScaleAndOffset(eyeDesc[eyeNum].Fov, _renderTargetSize, eyeRenderViewport[eyeNum], + ovrHmd_GetRenderScaleAndOffset(eyeDesc[eyeNum].Fov, _renderTargetSize, _eyeRenderViewport[eyeNum], _UVScaleOffset[eyeNum]); // Parse the vertex data and create a render ready vertex buffer @@ -235,9 +242,9 @@ void OculusManager::endFrameTiming() { void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenHeight) { #ifdef HAVE_LIBOVR ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, - _ovrHmdDesc.DefaultEyeFov[0], 1.0f); + _eyeFov[0], 1.0f); ovrSizei recommendedTex1Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Right, - _ovrHmdDesc.DefaultEyeFov[1], 1.0f); + _eyeFov[1], 1.0f); float width = recommendedTex0Size.w + recommendedTex1Size.w; float height = recommendedTex0Size.h; @@ -273,37 +280,55 @@ void OculusManager::display(Camera& whichCamera) { ovrPosef eyeRenderPose[ovrEye_Count]; + _camera->setTightness(0.0f); // In first person, camera follows (untweaked) head exactly without delay + _camera->setTargetPosition(whichCamera.getPosition()); + if (!Menu::getInstance()->isOptionChecked(MenuOption::AllowOculusCameraModeChange) || whichCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { + _camera->setDistance(0.0f); + } + _camera->setUpShift(0.0f); + _camera->setTightness(0.0f); // Cam + glMatrixMode(GL_PROJECTION); glPushMatrix(); glMatrixMode(GL_MODELVIEW); glPushMatrix(); - - eyeRenderPose[0] = ovrHmd_GetEyePose(_ovrHmd, ovrEye_Left); - eyeRenderPose[1] = ovrHmd_GetEyePose(_ovrHmd, ovrEye_Right); + + glm::quat orientation; for (int eyeIndex = 0; eyeIndex < ovrEye_Count; eyeIndex++) { ovrEyeType eye = _ovrHmdDesc.EyeRenderOrder[eyeIndex]; + //Set the camera rotation for this eye + eyeRenderPose[eye] = ovrHmd_GetEyePose(_ovrHmd, eye); + orientation.x = eyeRenderPose[eye].Orientation.x; + orientation.y = eyeRenderPose[eye].Orientation.y; + orientation.z = eyeRenderPose[eye].Orientation.z; + orientation.w = eyeRenderPose[eye].Orientation.w; + _camera->setTargetRotation(orientation); + _camera->update(1.0f / Application::getInstance()->getFps()); + Matrix4f proj = ovrMatrix4f_Projection(eyeDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); proj.Transpose(); glMatrixMode(GL_PROJECTION); glLoadIdentity(); glLoadMatrixf((GLfloat *)proj.M); + + printf("%d %d\n", _renderTargetSize.w, _renderTargetSize.h); - glViewport(eyeDesc[eye].DistortedViewport.Pos.x, eyeDesc[eye].DistortedViewport.Pos.y, - eyeDesc[eye].DistortedViewport.Size.w, eyeDesc[eye].DistortedViewport.Size.h); + glViewport(_eyeRenderViewport[eye].Pos.x, _eyeRenderViewport[eye].Pos.y, + _eyeRenderViewport[eye].Size.w, _eyeRenderViewport[eye].Size.h); glMatrixMode(GL_MODELVIEW); glLoadIdentity(); glTranslatef(eyeDesc[eye].ViewAdjust.x, eyeDesc[eye].ViewAdjust.y, eyeDesc[eye].ViewAdjust.z); - Application::getInstance()->displaySide(whichCamera); + Application::getInstance()->displaySide(*_camera); if (displayOverlays) { - applicationOverlay.displayOverlayTextureOculus(whichCamera); + applicationOverlay.displayOverlayTextureOculus(*_camera); } } @@ -399,6 +424,7 @@ void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { ovrPosef pose = ss.Predicted.Pose; Quatf orientation = Quatf(pose.Orientation); orientation.GetEulerAngles(&yaw, &pitch, &roll); + printf("%f %f %f\n", yaw, pitch, roll); } #endif } diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index ab17c94486..607484b98d 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -73,21 +73,21 @@ private: static bool _isConnected; - - #ifdef HAVE_LIBOVR static ovrHmd _ovrHmd; static ovrHmdDesc _ovrHmdDesc; static ovrFovPort _eyeFov[ovrEye_Count]; static ovrSizei _renderTargetSize; static ovrVector2f _UVScaleOffset[ovrEye_Count][2]; - static GLuint _vbo[ovrEye_Count]; - static GLuint _indicesVbo[ovrEye_Count]; + static GLuint _vertices[ovrEye_Count]; + static GLuint _indices[ovrEye_Count]; static GLsizei _meshSize[ovrEye_Count]; static ovrFrameTiming _hmdFrameTiming; + static ovrRecti _eyeRenderViewport[ovrEye_Count]; static unsigned int _frameIndex; static bool _frameTimingActive; static bool _programInitialized; + static Camera* _camera; #endif }; From f16098daef57bcbd98459aaa995eef9545cd0038 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Thu, 26 Jun 2014 19:11:54 -0700 Subject: [PATCH 30/67] Use the refactored endpoint/client classes in the actual client/server. --- .../src/metavoxels/MetavoxelServer.cpp | 57 ++---- .../src/metavoxels/MetavoxelServer.h | 36 ++-- interface/src/MetavoxelSystem.cpp | 166 ++---------------- interface/src/MetavoxelSystem.h | 75 ++------ interface/src/ui/MetavoxelEditor.cpp | 1 + libraries/metavoxels/src/Endpoint.cpp | 17 +- libraries/metavoxels/src/Endpoint.h | 15 +- .../metavoxels/src/MetavoxelClientManager.cpp | 41 ++--- .../metavoxels/src/MetavoxelClientManager.h | 12 +- 9 files changed, 101 insertions(+), 319 deletions(-) diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp index 3bf1632074..d0c0d4c781 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.cpp +++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp @@ -69,7 +69,7 @@ void MetavoxelServer::readPendingDatagrams() { void MetavoxelServer::maybeAttachSession(const SharedNodePointer& node) { if (node->getType() == NodeType::Agent) { QMutexLocker locker(&node->getMutex()); - node->setLinkedData(new MetavoxelSession(this, NodeList::getInstance()->nodeWithUUID(node->getUUID()))); + node->setLinkedData(new MetavoxelSession(node, this)); } } @@ -77,7 +77,7 @@ void MetavoxelServer::sendDeltas() { // send deltas for all sessions foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { if (node->getType() == NodeType::Agent) { - static_cast(node->getLinkedData())->sendDelta(); + static_cast(node->getLinkedData())->update(); } } @@ -89,59 +89,34 @@ void MetavoxelServer::sendDeltas() { _sendTimer.start(qMax(0, 2 * SEND_INTERVAL - elapsed)); } -MetavoxelSession::MetavoxelSession(MetavoxelServer* server, const SharedNodePointer& node) : - _server(server), - _sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)), - _node(node) { +MetavoxelSession::MetavoxelSession(const SharedNodePointer& node, MetavoxelServer* server) : + Endpoint(node, new PacketRecord(), NULL), + _server(server) { - connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendData(const QByteArray&))); - connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readPacket(Bitstream&))); - connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int))); connect(&_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), SLOT(handleMessage(const QVariant&))); connect(_sequencer.getReliableInputChannel(), SIGNAL(receivedMessage(const QVariant&)), SLOT(handleMessage(const QVariant&))); - - // insert the baseline send record - SendRecord record = { 0 }; - _sendRecords.append(record); } -MetavoxelSession::~MetavoxelSession() { -} - -int MetavoxelSession::parseData(const QByteArray& packet) { - // process through sequencer - _sequencer.receivedDatagram(packet); - return packet.size(); -} - -void MetavoxelSession::sendDelta() { +void MetavoxelSession::update() { // wait until we have a valid lod - if (!_lod.isValid()) { - return; + if (_lod.isValid()) { + Endpoint::update(); } - Bitstream& out = _sequencer.startPacket(); +} + +void MetavoxelSession::writeUpdateMessage(Bitstream& out) { out << QVariant::fromValue(MetavoxelDeltaMessage()); - _server->getData().writeDelta(_sendRecords.first().data, _sendRecords.first().lod, out, _lod); - _sequencer.endPacket(); - - // record the send - SendRecord record = { _sequencer.getOutgoingPacketNumber(), _server->getData(), _lod }; - _sendRecords.append(record); + PacketRecord* sendRecord = getLastAcknowledgedSendRecord(); + _server->getData().writeDelta(sendRecord->getData(), sendRecord->getLOD(), out, _lod); } -void MetavoxelSession::sendData(const QByteArray& data) { - NodeList::getInstance()->writeDatagram(data, _node); -} - -void MetavoxelSession::readPacket(Bitstream& in) { - QVariant message; - in >> message; +void MetavoxelSession::handleMessage(const QVariant& message, Bitstream& in) { handleMessage(message); } -void MetavoxelSession::clearSendRecordsBefore(int index) { - _sendRecords.erase(_sendRecords.begin(), _sendRecords.begin() + index + 1); +PacketRecord* MetavoxelSession::maybeCreateSendRecord() const { + return new PacketRecord(_lod, _server->getData()); } void MetavoxelSession::handleMessage(const QVariant& message) { diff --git a/assignment-client/src/metavoxels/MetavoxelServer.h b/assignment-client/src/metavoxels/MetavoxelServer.h index b01bb9b412..d9b010e282 100644 --- a/assignment-client/src/metavoxels/MetavoxelServer.h +++ b/assignment-client/src/metavoxels/MetavoxelServer.h @@ -17,8 +17,7 @@ #include -#include -#include +#include class MetavoxelEditMessage; class MetavoxelSession; @@ -53,46 +52,31 @@ private: }; /// Contains the state of a single client session. -class MetavoxelSession : public NodeData { +class MetavoxelSession : public Endpoint { Q_OBJECT public: - MetavoxelSession(MetavoxelServer* server, const SharedNodePointer& node); - virtual ~MetavoxelSession(); + MetavoxelSession(const SharedNodePointer& node, MetavoxelServer* server); - virtual int parseData(const QByteArray& packet); + virtual void update(); - void sendDelta(); +protected: + + virtual void writeUpdateMessage(Bitstream& out); + virtual void handleMessage(const QVariant& message, Bitstream& in); + + virtual PacketRecord* maybeCreateSendRecord() const; private slots: - void sendData(const QByteArray& data); - - void readPacket(Bitstream& in); - - void clearSendRecordsBefore(int index); - void handleMessage(const QVariant& message); private: - class SendRecord { - public: - int packetNumber; - MetavoxelData data; - MetavoxelLOD lod; - }; - MetavoxelServer* _server; - DatagramSequencer _sequencer; - - SharedNodePointer _node; - MetavoxelLOD _lod; - - QList _sendRecords; }; #endif // hifi_MetavoxelServer_h diff --git a/interface/src/MetavoxelSystem.cpp b/interface/src/MetavoxelSystem.cpp index 23e32f9d2c..7a5119a62d 100644 --- a/interface/src/MetavoxelSystem.cpp +++ b/interface/src/MetavoxelSystem.cpp @@ -35,6 +35,8 @@ MetavoxelSystem::MetavoxelSystem() : } void MetavoxelSystem::init() { + MetavoxelClientManager::init(); + if (!_program.isLinked()) { _program.addShaderFromSourceFile(QGLShader::Vertex, Application::resourcesPath() + "shaders/metavoxel_point.vert"); _program.link(); @@ -43,62 +45,19 @@ void MetavoxelSystem::init() { } _buffer.setUsagePattern(QOpenGLBuffer::DynamicDraw); _buffer.create(); - - connect(NodeList::getInstance(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(maybeAttachClient(const SharedNodePointer&))); } -SharedObjectPointer MetavoxelSystem::findFirstRaySpannerIntersection( - const glm::vec3& origin, const glm::vec3& direction, const AttributePointer& attribute, float& distance) { - SharedObjectPointer closestSpanner; - float closestDistance = FLT_MAX; - foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { - if (node->getType() == NodeType::MetavoxelServer) { - QMutexLocker locker(&node->getMutex()); - MetavoxelSystemClient* client = static_cast(node->getLinkedData()); - if (client) { - float clientDistance; - SharedObjectPointer clientSpanner = client->getData().findFirstRaySpannerIntersection( - origin, direction, attribute, clientDistance); - if (clientSpanner && clientDistance < closestDistance) { - closestSpanner = clientSpanner; - closestDistance = clientDistance; - } - } - } - } - if (closestSpanner) { - distance = closestDistance; - } - return closestSpanner; -} - -void MetavoxelSystem::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { - foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { - if (node->getType() == NodeType::MetavoxelServer) { - QMutexLocker locker(&node->getMutex()); - MetavoxelSystemClient* client = static_cast(node->getLinkedData()); - if (client) { - client->applyEdit(edit, reliable); - } - } - } +MetavoxelLOD MetavoxelSystem::getLOD() const { + const float FIXED_LOD_THRESHOLD = 0.01f; + return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD); } void MetavoxelSystem::simulate(float deltaTime) { - // simulate the clients + // update the clients _points.clear(); _simulateVisitor.setDeltaTime(deltaTime); _simulateVisitor.setOrder(-Application::getInstance()->getViewFrustum()->getDirection()); - foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { - if (node->getType() == NodeType::MetavoxelServer) { - QMutexLocker locker(&node->getMutex()); - MetavoxelSystemClient* client = static_cast(node->getLinkedData()); - if (client) { - client->simulate(deltaTime); - client->guide(_simulateVisitor); - } - } - } + update(); _buffer.bind(); int bytes = _points.size() * sizeof(Point); @@ -161,11 +120,13 @@ void MetavoxelSystem::render() { } } -void MetavoxelSystem::maybeAttachClient(const SharedNodePointer& node) { - if (node->getType() == NodeType::MetavoxelServer) { - QMutexLocker locker(&node->getMutex()); - node->setLinkedData(new MetavoxelSystemClient(NodeList::getInstance()->nodeWithUUID(node->getUUID()))); - } +MetavoxelClient* MetavoxelSystem::createClient(const SharedNodePointer& node) { + return new MetavoxelSystemClient(node, this); +} + +void MetavoxelSystem::updateClient(MetavoxelClient* client) { + MetavoxelClientManager::updateClient(client); + client->guide(_simulateVisitor); } MetavoxelSystem::SimulateVisitor::SimulateVisitor(QVector& points) : @@ -235,64 +196,8 @@ bool MetavoxelSystem::RenderVisitor::visit(Spanner* spanner, const glm::vec3& cl return true; } -MetavoxelSystemClient::MetavoxelSystemClient(const SharedNodePointer& node) : - _node(node), - _sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)) { - - connect(&_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendData(const QByteArray&))); - connect(&_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readPacket(Bitstream&))); - connect(&_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int))); - connect(&_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int))); - - // insert the baseline send record - SendRecord sendRecord = { 0 }; - _sendRecords.append(sendRecord); - - // insert the baseline receive record - ReceiveRecord receiveRecord = { 0, _data }; - _receiveRecords.append(receiveRecord); -} - -MetavoxelSystemClient::~MetavoxelSystemClient() { - // close the session - Bitstream& out = _sequencer.startPacket(); - out << QVariant::fromValue(CloseSessionMessage()); - _sequencer.endPacket(); -} - -static MetavoxelLOD getLOD() { - const float FIXED_LOD_THRESHOLD = 0.01f; - return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD); -} - -void MetavoxelSystemClient::guide(MetavoxelVisitor& visitor) { - visitor.setLOD(getLOD()); - _data.guide(visitor); -} - -void MetavoxelSystemClient::applyEdit(const MetavoxelEditMessage& edit, bool reliable) { - if (reliable) { - _sequencer.getReliableOutputChannel()->sendMessage(QVariant::fromValue(edit)); - - } else { - // apply immediately to local tree - edit.apply(_data, _sequencer.getWeakSharedObjectHash()); - - // start sending it out - _sequencer.sendHighPriorityMessage(QVariant::fromValue(edit)); - } -} - -void MetavoxelSystemClient::simulate(float deltaTime) { - Bitstream& out = _sequencer.startPacket(); - - ClientStateMessage state = { getLOD() }; - out << QVariant::fromValue(state); - _sequencer.endPacket(); - - // record the send - SendRecord record = { _sequencer.getOutgoingPacketNumber(), state.lod }; - _sendRecords.append(record); +MetavoxelSystemClient::MetavoxelSystemClient(const SharedNodePointer& node, MetavoxelSystem* system) : + MetavoxelClient(node, system) { } int MetavoxelSystemClient::parseData(const QByteArray& packet) { @@ -302,48 +207,11 @@ int MetavoxelSystemClient::parseData(const QByteArray& packet) { return packet.size(); } -void MetavoxelSystemClient::sendData(const QByteArray& data) { +void MetavoxelSystemClient::sendDatagram(const QByteArray& data) { NodeList::getInstance()->writeDatagram(data, _node); Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::METAVOXELS).updateValue(data.size()); } -void MetavoxelSystemClient::readPacket(Bitstream& in) { - QVariant message; - in >> message; - handleMessage(message, in); - - // record the receipt - ReceiveRecord record = { _sequencer.getIncomingPacketNumber(), _data, _sendRecords.first().lod }; - _receiveRecords.append(record); - - // reapply local edits - foreach (const DatagramSequencer::HighPriorityMessage& message, _sequencer.getHighPriorityMessages()) { - if (message.data.userType() == MetavoxelEditMessage::Type) { - message.data.value().apply(_data, _sequencer.getWeakSharedObjectHash()); - } - } -} - -void MetavoxelSystemClient::clearSendRecordsBefore(int index) { - _sendRecords.erase(_sendRecords.begin(), _sendRecords.begin() + index + 1); -} - -void MetavoxelSystemClient::clearReceiveRecordsBefore(int index) { - _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1); -} - -void MetavoxelSystemClient::handleMessage(const QVariant& message, Bitstream& in) { - int userType = message.userType(); - if (userType == MetavoxelDeltaMessage::Type) { - _data.readDelta(_receiveRecords.first().data, _receiveRecords.first().lod, in, _sendRecords.first().lod); - - } else if (userType == QMetaType::QVariantList) { - foreach (const QVariant& element, message.toList()) { - handleMessage(element, in); - } - } -} - static void enableClipPlane(GLenum plane, float x, float y, float z, float w) { GLdouble coefficients[] = { x, y, z, w }; glClipPlane(plane, coefficients); diff --git a/interface/src/MetavoxelSystem.h b/interface/src/MetavoxelSystem.h index f73f609732..f3a4fd4412 100644 --- a/interface/src/MetavoxelSystem.h +++ b/interface/src/MetavoxelSystem.h @@ -18,37 +18,31 @@ #include -#include - -#include -#include -#include +#include #include "renderer/ProgramObject.h" class Model; /// Renders a metavoxel tree. -class MetavoxelSystem : public QObject { +class MetavoxelSystem : public MetavoxelClientManager { Q_OBJECT public: MetavoxelSystem(); - void init(); - - SharedObjectPointer findFirstRaySpannerIntersection(const glm::vec3& origin, const glm::vec3& direction, - const AttributePointer& attribute, float& distance); - - Q_INVOKABLE void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); + virtual void init(); + + virtual MetavoxelLOD getLOD() const; void simulate(float deltaTime); void render(); - -private slots: - void maybeAttachClient(const SharedNodePointer& node); +protected: + + virtual MetavoxelClient* createClient(const SharedNodePointer& node); + virtual void updateClient(MetavoxelClient* client); private: @@ -89,59 +83,18 @@ private: }; /// A client session associated with a single server. -class MetavoxelSystemClient : public NodeData { +class MetavoxelSystemClient : public MetavoxelClient { Q_OBJECT public: - MetavoxelSystemClient(const SharedNodePointer& node); - virtual ~MetavoxelSystemClient(); - - MetavoxelData& getData() { return _data; } - - void guide(MetavoxelVisitor& visitor); - - void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false); - - void simulate(float deltaTime); - + MetavoxelSystemClient(const SharedNodePointer& node, MetavoxelSystem* system); + virtual int parseData(const QByteArray& packet); -private slots: +protected: - void sendData(const QByteArray& data); - - void readPacket(Bitstream& in); - - void clearSendRecordsBefore(int index); - - void clearReceiveRecordsBefore(int index); - -private: - - void handleMessage(const QVariant& message, Bitstream& in); - - class SendRecord { - public: - int packetNumber; - MetavoxelLOD lod; - }; - - class ReceiveRecord { - public: - int packetNumber; - MetavoxelData data; - MetavoxelLOD lod; - }; - - SharedNodePointer _node; - - DatagramSequencer _sequencer; - - MetavoxelData _data; - - QList _sendRecords; - QList _receiveRecords; + virtual void sendDatagram(const QByteArray& data); }; /// Base class for spanner renderers; provides clipping. diff --git a/interface/src/ui/MetavoxelEditor.cpp b/interface/src/ui/MetavoxelEditor.cpp index c1deb20111..1f0c2498c5 100644 --- a/interface/src/ui/MetavoxelEditor.cpp +++ b/interface/src/ui/MetavoxelEditor.cpp @@ -29,6 +29,7 @@ #include #include +#include #include "Application.h" #include "MetavoxelEditor.h" diff --git a/libraries/metavoxels/src/Endpoint.cpp b/libraries/metavoxels/src/Endpoint.cpp index a13351e671..c656054504 100644 --- a/libraries/metavoxels/src/Endpoint.cpp +++ b/libraries/metavoxels/src/Endpoint.cpp @@ -13,7 +13,7 @@ #include "Endpoint.h" -Endpoint::Endpoint(const SharedNodePointer& node) : +Endpoint::Endpoint(const SharedNodePointer& node, PacketRecord* baselineSendRecord, PacketRecord* baselineReceiveRecord) : _node(node), _sequencer(byteArrayWithPopulatedHeader(PacketTypeMetavoxelData)) { @@ -23,8 +23,8 @@ Endpoint::Endpoint(const SharedNodePointer& node) : connect(&_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int))); // insert the baseline send and receive records - _sendRecords.append(maybeCreateSendRecord(true)); - _receiveRecords.append(maybeCreateReceiveRecord(true)); + _sendRecords.append(baselineSendRecord); + _receiveRecords.append(baselineReceiveRecord); } Endpoint::~Endpoint() { @@ -38,7 +38,7 @@ Endpoint::~Endpoint() { void Endpoint::update() { Bitstream& out = _sequencer.startPacket(); - out << QVariant(); + writeUpdateMessage(out); _sequencer.endPacket(); // record the send @@ -92,16 +92,17 @@ void Endpoint::handleMessage(const QVariant& message, Bitstream& in) { } } -PacketRecord* Endpoint::maybeCreateSendRecord(bool baseline) const { +PacketRecord* Endpoint::maybeCreateSendRecord() const { return NULL; } -PacketRecord* Endpoint::maybeCreateReceiveRecord(bool baseline) const { +PacketRecord* Endpoint::maybeCreateReceiveRecord() const { return NULL; } -PacketRecord::PacketRecord(const MetavoxelLOD& lod) : - _lod(lod) { +PacketRecord::PacketRecord(const MetavoxelLOD& lod, const MetavoxelData& data) : + _lod(lod), + _data(data) { } PacketRecord::~PacketRecord() { diff --git a/libraries/metavoxels/src/Endpoint.h b/libraries/metavoxels/src/Endpoint.h index ac88376844..d253a69ded 100644 --- a/libraries/metavoxels/src/Endpoint.h +++ b/libraries/metavoxels/src/Endpoint.h @@ -19,16 +19,17 @@ class PacketRecord; -/// Base class for communication endpoints: clients or servers. +/// Base class for communication endpoints: clients and server sessions. class Endpoint : public NodeData { Q_OBJECT public: - Endpoint(const SharedNodePointer& node); + Endpoint(const SharedNodePointer& node, PacketRecord* baselineSendRecord = NULL, + PacketRecord* baselineReceiveRecord = NULL); virtual ~Endpoint(); - void update(); + virtual void update(); virtual int parseData(const QByteArray& packet); @@ -45,8 +46,8 @@ protected: virtual void writeUpdateMessage(Bitstream& out); virtual void handleMessage(const QVariant& message, Bitstream& in); - virtual PacketRecord* maybeCreateSendRecord(bool baseline = false) const; - virtual PacketRecord* maybeCreateReceiveRecord(bool baseline = false) const; + virtual PacketRecord* maybeCreateSendRecord() const; + virtual PacketRecord* maybeCreateReceiveRecord() const; PacketRecord* getLastAcknowledgedSendRecord() const { return _sendRecords.first(); } PacketRecord* getLastAcknowledgedReceiveRecord() const { return _receiveRecords.first(); } @@ -62,14 +63,16 @@ protected: class PacketRecord { public: - PacketRecord(const MetavoxelLOD& lod = MetavoxelLOD()); + PacketRecord(const MetavoxelLOD& lod = MetavoxelLOD(), const MetavoxelData& data = MetavoxelData()); virtual ~PacketRecord(); const MetavoxelLOD& getLOD() const { return _lod; } + const MetavoxelData& getData() const { return _data; } private: MetavoxelLOD _lod; + MetavoxelData _data; }; #endif // hifi_Endpoint_h diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp index 206e15becd..008a477187 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.cpp +++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp @@ -22,7 +22,7 @@ void MetavoxelClientManager::update() { QMutexLocker locker(&node->getMutex()); MetavoxelClient* client = static_cast(node->getLinkedData()); if (client) { - client->update(); + updateClient(client); } } } @@ -72,12 +72,20 @@ MetavoxelLOD MetavoxelClientManager::getLOD() const { void MetavoxelClientManager::maybeAttachClient(const SharedNodePointer& node) { if (node->getType() == NodeType::MetavoxelServer) { QMutexLocker locker(&node->getMutex()); - node->setLinkedData(new MetavoxelClient(node, this)); + node->setLinkedData(createClient(node)); } } +MetavoxelClient* MetavoxelClientManager::createClient(const SharedNodePointer& node) { + return new MetavoxelClient(node, this); +} + +void MetavoxelClientManager::updateClient(MetavoxelClient* client) { + client->update(); +} + MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientManager* manager) : - Endpoint(node), + Endpoint(node, new PacketRecord(), new PacketRecord()), _manager(manager) { } @@ -115,26 +123,9 @@ void MetavoxelClient::readMessage(Bitstream& in) { } } -class ReceiveRecord : public PacketRecord { -public: - - ReceiveRecord(const MetavoxelLOD& lod = MetavoxelLOD(), const MetavoxelData& data = MetavoxelData()); - - const MetavoxelData& getData() const { return _data; } - -private: - - MetavoxelData _data; -}; - -ReceiveRecord::ReceiveRecord(const MetavoxelLOD& lod, const MetavoxelData& data) : - PacketRecord(lod), - _data(data) { -} - void MetavoxelClient::handleMessage(const QVariant& message, Bitstream& in) { if (message.userType() == MetavoxelDeltaMessage::Type) { - ReceiveRecord* receiveRecord = static_cast(getLastAcknowledgedReceiveRecord()); + PacketRecord* receiveRecord = getLastAcknowledgedReceiveRecord(); _data.readDelta(receiveRecord->getData(), receiveRecord->getLOD(), in, getLastAcknowledgedSendRecord()->getLOD()); } else { @@ -142,10 +133,10 @@ void MetavoxelClient::handleMessage(const QVariant& message, Bitstream& in) { } } -PacketRecord* MetavoxelClient::maybeCreateSendRecord(bool baseline) const { - return baseline ? new PacketRecord() : new PacketRecord(_manager->getLOD()); +PacketRecord* MetavoxelClient::maybeCreateSendRecord() const { + return new PacketRecord(_manager->getLOD()); } -PacketRecord* MetavoxelClient::maybeCreateReceiveRecord(bool baseline) const { - return baseline ? new ReceiveRecord() : new ReceiveRecord(getLastAcknowledgedSendRecord()->getLOD(), _data); +PacketRecord* MetavoxelClient::maybeCreateReceiveRecord() const { + return new PacketRecord(getLastAcknowledgedSendRecord()->getLOD(), _data); } diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h index cd34cdeab2..dd11e871ec 100644 --- a/libraries/metavoxels/src/MetavoxelClientManager.h +++ b/libraries/metavoxels/src/MetavoxelClientManager.h @@ -14,6 +14,7 @@ #include "Endpoint.h" +class MetavoxelClient; class MetavoxelEditMessage; /// Manages the set of connected metavoxel clients. @@ -23,7 +24,7 @@ class MetavoxelClientManager : public QObject { public: virtual void init(); - virtual void update(); + void update(); SharedObjectPointer findFirstRaySpannerIntersection(const glm::vec3& origin, const glm::vec3& direction, const AttributePointer& attribute, float& distance); @@ -35,6 +36,11 @@ public: private slots: void maybeAttachClient(const SharedNodePointer& node); + +protected: + + virtual MetavoxelClient* createClient(const SharedNodePointer& node); + virtual void updateClient(MetavoxelClient* client); }; /// Base class for metavoxel clients. @@ -57,8 +63,8 @@ protected: virtual void readMessage(Bitstream& in); virtual void handleMessage(const QVariant& message, Bitstream& in); - virtual PacketRecord* maybeCreateSendRecord(bool baseline) const; - virtual PacketRecord* maybeCreateReceiveRecord(bool baseline) const; + virtual PacketRecord* maybeCreateSendRecord() const; + virtual PacketRecord* maybeCreateReceiveRecord() const; private: From 1af2996cf2923522c19af27729b82941558ccd01 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Fri, 27 Jun 2014 09:53:31 -0700 Subject: [PATCH 31/67] Properly use body orientation for Oculus camera --- interface/resources/shaders/oculus.vert | 4 ++-- interface/src/devices/OculusManager.cpp | 4 ++-- interface/src/devices/OculusManager.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/interface/resources/shaders/oculus.vert b/interface/resources/shaders/oculus.vert index 389615dde1..800fbd9317 100644 --- a/interface/resources/shaders/oculus.vert +++ b/interface/resources/shaders/oculus.vert @@ -35,7 +35,7 @@ vec2 TimewarpTexCoord(vec2 texCoord, mat4 rotMat) // Vertex inputs are in TanEyeAngle space for the R,G,B channels (i.e. after chromatic // aberration and distortion). These are now "real world" vectors in direction (x,y,1) // relative to the eye of the HMD. Apply the 3x3 timewarp rotation to these vectors. - vec3 transformed = vec3( rotMat * vec4(texCoord.xy, 1, 1) ).xyz; + vec3 transformed = vec3( rotMat * vec4(texCoord.xy, 1, 1) ); // Project them back onto the Z=1 plane of the rendered images. vec2 flattened = (transformed.xy / transformed.z); @@ -47,7 +47,7 @@ vec2 TimewarpTexCoord(vec2 texCoord, mat4 rotMat) void main() { float timewarpMixFactor = color.a; - mat4 mixedEyeRot = EyeRotationStart * mat4(1.0 - timewarpMixFactor) + EyeRotationEnd * mat4(timewarpMixFactor); + mat4 mixedEyeRot = EyeRotationStart * (1.0 - timewarpMixFactor) + EyeRotationEnd * (timewarpMixFactor); oTexCoord0 = TimewarpTexCoord(texCoord0, mixedEyeRot); oTexCoord1 = TimewarpTexCoord(texCoord1, mixedEyeRot); diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 42aec65d82..dbcc1daaf0 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -258,7 +258,7 @@ void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenH } //Displays everything for the oculus, frame timing must be active -void OculusManager::display(Camera& whichCamera) { +void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamera) { #ifdef HAVE_LIBOVR //beginFrameTiming must be called before display if (!_frameTimingActive) { @@ -306,7 +306,7 @@ void OculusManager::display(Camera& whichCamera) { orientation.y = eyeRenderPose[eye].Orientation.y; orientation.z = eyeRenderPose[eye].Orientation.z; orientation.w = eyeRenderPose[eye].Orientation.w; - _camera->setTargetRotation(orientation); + _camera->setTargetRotation(orientation * bodyOrientation); _camera->update(1.0f / Application::getInstance()->getFps()); Matrix4f proj = ovrMatrix4f_Projection(eyeDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 607484b98d..34c93fd8f1 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -33,7 +33,7 @@ public: static void beginFrameTiming(); static void endFrameTiming(); static void configureCamera(Camera& camera, int screenWidth, int screenHeight); - static void display(Camera& whichCamera); + static void display(const glm::quat &bodyOrientation, Camera& whichCamera); static void reset(); /// param \yaw[out] yaw in radians From ff438a24344c63befa510a66d5f2bee168c8945a Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 11:23:50 -0700 Subject: [PATCH 32/67] added seq stats for Agent, fixed AudioInjector to do << (quint16)0 instead of skipRawData(sizeof(quint16) to correctly allocate space for seq number; added debug prints; --- assignment-client/src/Agent.cpp | 8 +++++++- assignment-client/src/Agent.h | 2 ++ assignment-client/src/audio/AudioMixer.cpp | 5 ++++- .../src/audio/AudioMixerClientData.cpp | 20 +++++++++++-------- .../src/audio/AudioMixerClientData.h | 10 +++++----- interface/src/Audio.cpp | 14 +++++++------ interface/src/Audio.h | 4 ++-- libraries/audio/src/AudioInjector.cpp | 15 ++++++++------ libraries/script-engine/src/ScriptEngine.cpp | 6 +++--- libraries/script-engine/src/ScriptEngine.h | 2 +- 10 files changed, 53 insertions(+), 33 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 561acc0eb0..a2c9b30ac1 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -147,7 +147,13 @@ void Agent::readPendingDatagrams() { } } else if (datagramPacketType == PacketTypeMixedAudio) { - // TODO: track sequence numbers for mixed audio??? + + // parse sequence number for this packet + int numBytesPacketHeader = numBytesForPacketHeader(receivedPacket); + const char* sequenceAt = receivedPacket.constData() + numBytesPacketHeader; + quint16 sequence = *(reinterpret_cast(sequenceAt)); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("mixed audio received %d\n", sequence); // parse the data and grab the average loudness _receivedAudioBuffer.parseData(receivedPacket); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 9af95e757c..ec8f7c88cb 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -71,6 +71,8 @@ private: ModelTreeHeadlessViewer _modelViewer; MixedAudioRingBuffer _receivedAudioBuffer; + SequenceNumberStats _incomingMixedAudioSequenceNumberStats; + AvatarHashMap _avatarHashMap; }; diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6b43fb37e0..094d6da371 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -457,6 +457,7 @@ void AudioMixer::sendStatsPacket() { // if we're too large, send the packet if (sizeOfStats > TOO_BIG_FOR_MTU) { +printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); sizeOfStats = 0; statsObject2 = QJsonObject(); // clear it @@ -465,6 +466,7 @@ void AudioMixer::sendStatsPacket() { } if (somethingToSend) { +printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); } } @@ -621,7 +623,8 @@ void AudioMixer::run() { dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; // send mixed audio packet - nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + //nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); +printf("mixed audio sent %d\n", sequence); nodeData->incrementOutgoingSequenceNumber(); // send an audio stream stats packet if it's time diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 8f50647463..6eec020991 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -21,8 +21,8 @@ AudioMixerClientData::AudioMixerClientData() : _ringBuffers(), - _outgoingSequenceNumber(0), - _incomingAvatarSequenceNumberStats() + _outgoingMixedAudioSequenceNumber(0), + _incomingAvatarAudioSequenceNumberStats() { } @@ -57,13 +57,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { || packetType == PacketTypeMicrophoneAudioNoEcho || packetType == PacketTypeSilentAudioFrame) { - _incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("avatar audio received %d\n", sequence); // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); // read the first byte after the header to see if this is a stereo or mono buffer - quint8 channelFlag = packet.at(numBytesForPacketHeader(packet)); + quint8 channelFlag = packet.at(numBytesForPacketHeader(packet) + sizeof(quint16)); bool isStereo = channelFlag == 1; if (avatarRingBuffer && avatarRingBuffer->isStereo() != isStereo) { @@ -86,9 +87,10 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { // this is injected audio // grab the stream identifier for this injected audio - QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID)); + QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID)); - _incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); + _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); +printf("injected stream %s received seq %d\n", streamIdentifier.toString().toLatin1().data(), sequence); InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; @@ -96,6 +98,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector && ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) { matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i]; +printf("\t matching ring buffer found.\n"); } } @@ -104,6 +107,7 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()); _ringBuffers.push_back(matchingInjectedRingBuffer); +printf("\t no matching ring buffer, creating new one. _ringBuffer size now %d\n", _ringBuffers.size()); } matchingInjectedRingBuffer->parseData(packet); @@ -147,7 +151,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { // this is an empty audio buffer that has starved, safe to delete // also delete its sequence number stats QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier(); - _incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier); + _incomingInjectedAudioSequenceNumberStatsMap.remove(streamIdentifier); delete audioBuffer; i = _ringBuffers.erase(i); continue; @@ -212,7 +216,7 @@ QString AudioMixerClientData::getJitterBufferStatsString() const { } else { result = "mic unknown"; } - + printf("\tget jitter buffer stats string. _ringBuffer.size = %d\n", _ringBuffers.size()); for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index c418df7905..eb4e6f5dae 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -38,17 +38,17 @@ public: QString getJitterBufferStatsString() const; - void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; } + void incrementOutgoingSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } - quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; } + quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } private: QList _ringBuffers; - quint16 _outgoingSequenceNumber; - SequenceNumberStats _incomingAvatarSequenceNumberStats; - QHash _incomingInjectedSequenceNumberStatsMap; + quint16 _outgoingMixedAudioSequenceNumber; + SequenceNumberStats _incomingAvatarAudioSequenceNumberStats; + QHash _incomingInjectedAudioSequenceNumberStatsMap; }; #endif // hifi_AudioMixerClientData_h diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index ff019712d7..024fb66cc1 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeOutputLeft(0), _scopeOutputRight(0), _audioMixerJitterBufferStats(), - _outgoingSequenceNumber(0) + _outgoingAvatarAudioSequenceNumber(0) { // clear the array of locally injected samples memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL); @@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); - _outgoingSequenceNumber = 0; + _outgoingAvatarAudioSequenceNumber = 0; } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { @@ -657,7 +657,7 @@ void Audio::handleAudioInput() { char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType); // pack sequence number - memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16)); + memcpy(currentPacketPtr, &_outgoingAvatarAudioSequenceNumber, sizeof(quint16)); currentPacketPtr += sizeof(quint16); // set the mono/stereo byte @@ -672,13 +672,14 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); - _outgoingSequenceNumber++; +printf("avatar audio sent %d\n", _outgoingAvatarAudioSequenceNumber); + _outgoingAvatarAudioSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); } else { // reset seq numbers if there's no connection with an audiomixer - _outgoingSequenceNumber = 0; + _outgoingAvatarAudioSequenceNumber = 0; } delete[] inputAudioSamples; } @@ -832,7 +833,8 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; quint16 sequence = *((quint16*)sequenceAt); - _incomingSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); +printf("mixed audio received %d\n", sequence); // parse audio data _ringBuffer.parseData(audioByteArray); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index cb04418eaf..f08cce9467 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -239,8 +239,8 @@ private: AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; - quint16 _outgoingSequenceNumber; - SequenceNumberStats _incomingSequenceNumberStats; + quint16 _outgoingAvatarAudioSequenceNumber; + SequenceNumberStats _incomingMixedAudioSequenceNumberStats; }; diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index d9d4dc6d3a..01dfc8696a 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -61,12 +61,13 @@ void AudioInjector::injectAudio() { QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); QDataStream packetStream(&injectAudioPacket, QIODevice::Append); - // skip sequence number for now + // pack some placeholder sequence number for now int numPreSequenceNumberBytes = injectAudioPacket.size(); - packetStream.skipRawData(sizeof(quint16)); + packetStream << (quint16)0; // pack stream identifier (a generated UUID) - packetStream << QUuid::createUuid(); +QUuid streamID; + packetStream << (streamID = QUuid::createUuid()); // pack the flag for loopback uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface()); @@ -86,6 +87,7 @@ void AudioInjector::injectAudio() { quint8 volume = MAX_INJECTOR_VOLUME * _options.getVolume(); packetStream << volume; + QElapsedTimer timer; timer.start(); int nextFrame = 0; @@ -96,7 +98,7 @@ void AudioInjector::injectAudio() { bool shouldLoop = _options.getLoop(); // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks - quint16 outgoingSequenceNumber = 0; + quint16 outgoingInjectedAudioSequenceNumber = 0; while (currentSendPosition < soundByteArray.size() && !_shouldStop) { int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, @@ -106,7 +108,7 @@ void AudioInjector::injectAudio() { injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy); // pack the sequence number - memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16)); + memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingInjectedAudioSequenceNumber, sizeof(quint16)); // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy); @@ -116,7 +118,8 @@ void AudioInjector::injectAudio() { // send off this audio packet nodeList->writeDatagram(injectAudioPacket, audioMixer); - outgoingSequenceNumber++; +printf("injector stream %s sent seq %d\n", streamID.toString().toLatin1().data(), outgoingInjectedAudioSequenceNumber); + outgoingInjectedAudioSequenceNumber++; currentSendPosition += bytesToCopy; diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index b142819260..09b60d8e0d 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -501,9 +501,9 @@ void ScriptEngine::run() { // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { // pack sequence number - quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++; + quint16 sequence = _outgoingInjectedAudioSequenceNumbers[node->getUUID()]++; memcpy(sequenceAt, &sequence, sizeof(quint16)); - +printf("script engine audio sent %d\n", sequence); // send audio packet nodeList->writeDatagram(audioPacket, audioPacketSize, node); } @@ -683,5 +683,5 @@ void ScriptEngine::include(const QString& includeFile) { } void ScriptEngine::nodeKilled(SharedNodePointer node) { - _outgoingSequenceNumbers.remove(node->getUUID()); + _outgoingInjectedAudioSequenceNumbers.remove(node->getUUID()); } diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index f1aaf6f0bb..cd969e49cd 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -148,7 +148,7 @@ private: ScriptUUID _uuidLibrary; AnimationCache _animationCache; - QHash _outgoingSequenceNumbers; + QHash _outgoingInjectedAudioSequenceNumbers; }; #endif // hifi_ScriptEngine_h From 508fed9146030ff64b53fc244912272501203f18 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Fri, 27 Jun 2014 11:44:05 -0700 Subject: [PATCH 33/67] Add ability for FBO to have a different res than glWidget --- interface/src/Application.cpp | 10 ++++- interface/src/devices/OculusManager.cpp | 38 +++++++++++------- interface/src/devices/OculusManager.h | 2 + interface/src/renderer/TextureCache.cpp | 51 +++++++++++++++++++++---- interface/src/renderer/TextureCache.h | 5 +++ 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 89f9445498..117842dee4 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -565,6 +565,14 @@ void Application::paintGL() { bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); PerformanceWarning warn(showWarnings, "Application::paintGL()"); + // Set the desired FBO texture size. If it hasn't changed, this does nothing. + // Otherwise, it must rebuild the FBOs + if (OculusManager::isConnected()) { + _textureCache.setFrameBufferSize(OculusManager::getRenderTargetSize()); + } else { + _textureCache.setFrameBufferSize(_glWidget->size()); + } + glEnable(GL_LINE_SMOOTH); if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { @@ -630,7 +638,7 @@ void Application::paintGL() { } if (OculusManager::isConnected()) { - OculusManager::display(whichCamera); + OculusManager::display(_myAvatar->getOrientation(), whichCamera); } else if (TV3DManager::isConnected()) { _glowEffect.prepare(); diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index dbcc1daaf0..35122ba0f5 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -39,6 +39,7 @@ bool OculusManager::_isConnected = false; ovrHmd OculusManager::_ovrHmd; ovrHmdDesc OculusManager::_ovrHmdDesc; ovrFovPort OculusManager::_eyeFov[ovrEye_Count]; +ovrEyeRenderDesc OculusManager::_eyeRenderDesc[ovrEye_Count]; ovrSizei OculusManager::_renderTargetSize; ovrVector2f OculusManager::_UVScaleOffset[ovrEye_Count][2]; GLuint OculusManager::_vertices[ovrEye_Count] = { 0, 0 }; @@ -77,8 +78,13 @@ void OculusManager::connect() { _renderTargetSize.h = recommendedTex1Size.h; } - _renderTargetSize.w = Application::getInstance()->getGLWidget()->width(); - _renderTargetSize.h = Application::getInstance()->getGLWidget()->height(); + _eyeRenderDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); + _eyeRenderDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); + + //_renderTargetSize.w = Application::getInstance()->getGLWidget()->width(); + //_renderTargetSize.h = Application::getInstance()->getGLWidget()->height(); + + ovrHmd_SetEnabledCaps(_ovrHmd, ovrHmdCap_LowPersistence | ovrHmdCap_LatencyTest); ovrHmd_StartSensor(_ovrHmd, ovrSensorCap_Orientation | ovrSensorCap_YawCorrection | ovrSensorCap_Position, @@ -150,10 +156,6 @@ void OculusManager::generateDistortionMesh() { return; } - ovrEyeRenderDesc eyeDesc[ovrEye_Count]; - eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); - eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); - _eyeRenderViewport[0].Pos = Vector2i(0, 0); _eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); _eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); @@ -162,9 +164,9 @@ void OculusManager::generateDistortionMesh() { for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { // Allocate and generate distortion mesh vertices ovrDistortionMesh meshData; - ovrHmd_CreateDistortionMesh(_ovrHmd, eyeDesc[eyeNum].Eye, eyeDesc[eyeNum].Fov, _ovrHmdDesc.DistortionCaps, &meshData); + ovrHmd_CreateDistortionMesh(_ovrHmd, _eyeRenderDesc[eyeNum].Eye, _eyeRenderDesc[eyeNum].Fov, _ovrHmdDesc.DistortionCaps, &meshData); - ovrHmd_GetRenderScaleAndOffset(eyeDesc[eyeNum].Fov, _renderTargetSize, _eyeRenderViewport[eyeNum], + ovrHmd_GetRenderScaleAndOffset(_eyeRenderDesc[eyeNum].Fov, _renderTargetSize, _eyeRenderViewport[eyeNum], _UVScaleOffset[eyeNum]); // Parse the vertex data and create a render ready vertex buffer @@ -266,9 +268,6 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer return; } - ovrEyeRenderDesc eyeDesc[ovrEye_Count]; - eyeDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); - eyeDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); ApplicationOverlay& applicationOverlay = Application::getInstance()->getApplicationOverlay(); // We only need to render the overlays to a texture once, then we just render the texture as a quad @@ -296,6 +295,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer glm::quat orientation; + //Render each eye into an fbo for (int eyeIndex = 0; eyeIndex < ovrEye_Count; eyeIndex++) { ovrEyeType eye = _ovrHmdDesc.EyeRenderOrder[eyeIndex]; @@ -309,7 +309,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer _camera->setTargetRotation(orientation * bodyOrientation); _camera->update(1.0f / Application::getInstance()->getFps()); - Matrix4f proj = ovrMatrix4f_Projection(eyeDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); + Matrix4f proj = ovrMatrix4f_Projection(_eyeRenderDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); proj.Transpose(); glMatrixMode(GL_PROJECTION); @@ -323,7 +323,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer glMatrixMode(GL_MODELVIEW); glLoadIdentity(); - glTranslatef(eyeDesc[eye].ViewAdjust.x, eyeDesc[eye].ViewAdjust.y, eyeDesc[eye].ViewAdjust.z); + glTranslatef(_eyeRenderDesc[eye].ViewAdjust.x, _eyeRenderDesc[eye].ViewAdjust.y, _eyeRenderDesc[eye].ViewAdjust.z); Application::getInstance()->displaySide(*_camera); @@ -424,8 +424,18 @@ void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { ovrPosef pose = ss.Predicted.Pose; Quatf orientation = Quatf(pose.Orientation); orientation.GetEulerAngles(&yaw, &pitch, &roll); - printf("%f %f %f\n", yaw, pitch, roll); } #endif } +QSize OculusManager::getRenderTargetSize() { +#ifdef HAVE_LIBOVR + QSize rv; + rv.setWidth(_renderTargetSize.w); + rv.setHeight(_renderTargetSize.h); + return rv; +#else + return QSize(100, 100); +#endif +} + diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 34c93fd8f1..bd4f401efb 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -40,6 +40,7 @@ public: /// param \pitch[out] pitch in radians /// param \roll[out] roll in radians static void getEulerAngles(float& yaw, float& pitch, float& roll); + static QSize getRenderTargetSize(); private: static void generateDistortionMesh(); @@ -77,6 +78,7 @@ private: static ovrHmd _ovrHmd; static ovrHmdDesc _ovrHmdDesc; static ovrFovPort _eyeFov[ovrEye_Count]; + static ovrEyeRenderDesc _eyeRenderDesc[ovrEye_Count]; static ovrSizei _renderTargetSize; static ovrVector2f _UVScaleOffset[ovrEye_Count][2]; static GLuint _vertices[ovrEye_Count]; diff --git a/interface/src/renderer/TextureCache.cpp b/interface/src/renderer/TextureCache.cpp index 0588ca70d2..55a67ce854 100644 --- a/interface/src/renderer/TextureCache.cpp +++ b/interface/src/renderer/TextureCache.cpp @@ -28,10 +28,12 @@ TextureCache::TextureCache() : _permutationNormalTextureID(0), _whiteTextureID(0), _blueTextureID(0), + _primaryDepthTextureID(0), _primaryFramebufferObject(NULL), _secondaryFramebufferObject(NULL), _tertiaryFramebufferObject(NULL), - _shadowFramebufferObject(NULL) + _shadowFramebufferObject(NULL), + _frameBufferSize(100, 100) { } @@ -46,9 +48,41 @@ TextureCache::~TextureCache() { glDeleteTextures(1, &_primaryDepthTextureID); } - delete _primaryFramebufferObject; - delete _secondaryFramebufferObject; - delete _tertiaryFramebufferObject; + if (_primaryFramebufferObject) { + delete _primaryFramebufferObject; + } + + if (_secondaryFramebufferObject) { + delete _secondaryFramebufferObject; + } + + if (_tertiaryFramebufferObject) { + delete _tertiaryFramebufferObject; + } +} + +void TextureCache::setFrameBufferSize(QSize frameBufferSize) { + //If the size changed, we need to delete our FBOs + if (_frameBufferSize != frameBufferSize) { + _frameBufferSize = frameBufferSize; + + if (_primaryFramebufferObject) { + delete _primaryFramebufferObject; + _primaryFramebufferObject = NULL; + glDeleteTextures(1, &_primaryDepthTextureID); + _primaryDepthTextureID = 0; + } + + if (_secondaryFramebufferObject) { + delete _secondaryFramebufferObject; + _secondaryFramebufferObject = NULL; + } + + if (_tertiaryFramebufferObject) { + delete _tertiaryFramebufferObject; + _tertiaryFramebufferObject = NULL; + } + } } GLuint TextureCache::getPermutationNormalTextureID() { @@ -131,13 +165,14 @@ QSharedPointer TextureCache::getTexture(const QUrl& url, bool no } QOpenGLFramebufferObject* TextureCache::getPrimaryFramebufferObject() { + if (!_primaryFramebufferObject) { _primaryFramebufferObject = createFramebufferObject(); - + glGenTextures(1, &_primaryDepthTextureID); glBindTexture(GL_TEXTURE_2D, _primaryDepthTextureID); - QSize size = Application::getInstance()->getGLWidget()->size(); - glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, size.width(), size.height(), + + glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, _frameBufferSize.width(), _frameBufferSize.height(), 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); @@ -230,7 +265,7 @@ QSharedPointer TextureCache::createResource(const QUrl& url, } QOpenGLFramebufferObject* TextureCache::createFramebufferObject() { - QOpenGLFramebufferObject* fbo = new QOpenGLFramebufferObject(Application::getInstance()->getGLWidget()->size()); + QOpenGLFramebufferObject* fbo = new QOpenGLFramebufferObject(_frameBufferSize); Application::getInstance()->getGLWidget()->installEventFilter(this); glBindTexture(GL_TEXTURE_2D, fbo->texture()); diff --git a/interface/src/renderer/TextureCache.h b/interface/src/renderer/TextureCache.h index f4444b6dfc..248a451e3a 100644 --- a/interface/src/renderer/TextureCache.h +++ b/interface/src/renderer/TextureCache.h @@ -32,6 +32,9 @@ public: TextureCache(); virtual ~TextureCache(); + /// Sets the desired texture resolution for the framebuffer objects. + void setFrameBufferSize(QSize frameBufferSize); + /// Returns the ID of the permutation/normal texture used for Perlin noise shader programs. This texture /// has two lines: the first, a set of random numbers in [0, 255] to be used as permutation offsets, and /// the second, a set of random unit vectors to be used as noise gradients. @@ -94,6 +97,8 @@ private: QOpenGLFramebufferObject* _shadowFramebufferObject; GLuint _shadowDepthTextureID; + + QSize _frameBufferSize; }; /// A simple object wrapper for an OpenGL texture. From d4470c04d615c0b0bf5b453020e8510a8c39feb0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 27 Jun 2014 12:34:56 -0700 Subject: [PATCH 34/67] Modify tests to use Endpoint base class. --- tests/metavoxels/src/MetavoxelTests.cpp | 255 +++++++++++++----------- tests/metavoxels/src/MetavoxelTests.h | 50 ++--- 2 files changed, 156 insertions(+), 149 deletions(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index dc44134600..48d1d4429d 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -440,6 +440,8 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { } bool MetavoxelTests::run() { + LimitedNodeList::createInstance(); + // seed the random number generator so that our tests are reproducible srand(0xBAAAAABE); @@ -456,14 +458,13 @@ bool MetavoxelTests::run() { } } - QByteArray datagramHeader("testheader"); const int SIMULATION_ITERATIONS = 10000; if (test == 0 || test == 2) { qDebug() << "Running transmission test..."; qDebug(); // create two endpoints with the same header - TestEndpoint alice(datagramHeader), bob(datagramHeader); + TestEndpoint alice, bob; alice.setOther(&bob); bob.setOther(&alice); @@ -497,7 +498,7 @@ bool MetavoxelTests::run() { datagramsReceived = bytesReceived = maxDatagramsPerPacket = maxBytesPerPacket = 0; // create two endpoints with the same header - TestEndpoint alice(datagramHeader, TestEndpoint::CONGESTION_MODE), bob(datagramHeader, TestEndpoint::CONGESTION_MODE); + TestEndpoint alice(TestEndpoint::CONGESTION_MODE), bob(TestEndpoint::CONGESTION_MODE); alice.setOther(&bob); bob.setOther(&alice); @@ -537,8 +538,8 @@ bool MetavoxelTests::run() { datagramsSent = bytesSent = datagramsReceived = bytesReceived = maxDatagramsPerPacket = maxBytesPerPacket = 0; // create client and server endpoints - TestEndpoint client(datagramHeader, TestEndpoint::METAVOXEL_CLIENT_MODE); - TestEndpoint server(datagramHeader, TestEndpoint::METAVOXEL_SERVER_MODE); + TestEndpoint client(TestEndpoint::METAVOXEL_CLIENT_MODE); + TestEndpoint server(TestEndpoint::METAVOXEL_SERVER_MODE); client.setOther(&server); server.setOther(&client); @@ -599,28 +600,57 @@ int RandomVisitor::visit(MetavoxelInfo& info) { return STOP_RECURSION; } -TestEndpoint::TestEndpoint(const QByteArray& datagramHeader, Mode mode) : +class TestSendRecord : public PacketRecord { +public: + + TestSendRecord(const MetavoxelLOD& lod = MetavoxelLOD(), const MetavoxelData& data = MetavoxelData(), + const SharedObjectPointer& localState = SharedObjectPointer(), int packetNumber = 0); + + const SharedObjectPointer& getLocalState() const { return _localState; } + int getPacketNumber() const { return _packetNumber; } + +private: + + SharedObjectPointer _localState; + int _packetNumber; + +}; + +TestSendRecord::TestSendRecord(const MetavoxelLOD& lod, const MetavoxelData& data, + const SharedObjectPointer& localState, int packetNumber) : + PacketRecord(lod, data), + _localState(localState), + _packetNumber(packetNumber) { +} + +class TestReceiveRecord : public PacketRecord { +public: + + TestReceiveRecord(const MetavoxelLOD& lod = MetavoxelLOD(), const MetavoxelData& data = MetavoxelData(), + const SharedObjectPointer& remoteState = SharedObjectPointer()); + + const SharedObjectPointer& getRemoteState() const { return _remoteState; } + +private: + + SharedObjectPointer _remoteState; +}; + +TestReceiveRecord::TestReceiveRecord(const MetavoxelLOD& lod, + const MetavoxelData& data, const SharedObjectPointer& remoteState) : + PacketRecord(lod, data), + _remoteState(remoteState) { +} + +TestEndpoint::TestEndpoint(Mode mode) : + Endpoint(SharedNodePointer(), new TestSendRecord(), new TestReceiveRecord()), _mode(mode), - _sequencer(new DatagramSequencer(datagramHeader, this)), _highPriorityMessagesToSend(0.0f), _reliableMessagesToSend(0.0f) { - connect(_sequencer, SIGNAL(readyToWrite(const QByteArray&)), SLOT(sendDatagram(const QByteArray&))); - connect(_sequencer, SIGNAL(readyToRead(Bitstream&)), SLOT(readMessage(Bitstream&))); - connect(_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), + connect(&_sequencer, SIGNAL(receivedHighPriorityMessage(const QVariant&)), SLOT(handleHighPriorityMessage(const QVariant&))); - connect(_sequencer, SIGNAL(sendAcknowledged(int)), SLOT(clearSendRecordsBefore(int))); - connect(_sequencer, SIGNAL(receiveAcknowledged(int)), SLOT(clearReceiveRecordsBefore(int))); - - // insert the baseline send record - SendRecord sendRecord = { 0 }; - _sendRecords.append(sendRecord); - - // insert the baseline receive record - ReceiveRecord receiveRecord = { 0 }; - _receiveRecords.append(receiveRecord); - if (mode == METAVOXEL_CLIENT_MODE) { _lod = MetavoxelLOD(glm::vec3(), 0.01f); return; @@ -643,15 +673,15 @@ TestEndpoint::TestEndpoint(const QByteArray& datagramHeader, Mode mode) : // create the object that represents out delta-encoded state _localState = new TestSharedObjectA(); - connect(_sequencer->getReliableInputChannel(), SIGNAL(receivedMessage(const QVariant&)), + connect(_sequencer.getReliableInputChannel(), SIGNAL(receivedMessage(const QVariant&)), SLOT(handleReliableMessage(const QVariant&))); - ReliableChannel* secondInput = _sequencer->getReliableInputChannel(1); + ReliableChannel* secondInput = _sequencer.getReliableInputChannel(1); secondInput->setMessagesEnabled(false); connect(&secondInput->getBuffer(), SIGNAL(readyRead()), SLOT(readReliableChannel())); // enqueue a large amount of data in a low-priority channel - ReliableChannel* output = _sequencer->getReliableOutputChannel(1); + ReliableChannel* output = _sequencer.getReliableOutputChannel(1); output->setPriority(0.25f); output->setMessagesEnabled(false); QByteArray bytes; @@ -804,7 +834,7 @@ bool TestEndpoint::simulate(int iterationNumber) { // update/send our delayed datagrams for (QList::iterator it = _delayedDatagrams.begin(); it != _delayedDatagrams.end(); ) { if (it->second-- == 1) { - _other->receiveDatagram(it->first); + _other->parseData(it->first); it = _delayedDatagrams.erase(it); } else { @@ -819,41 +849,39 @@ bool TestEndpoint::simulate(int iterationNumber) { ByteArrayVector datagrams = _pipeline.takeLast(); _pipeline.prepend(ByteArrayVector()); foreach (const QByteArray& datagram, datagrams) { - _sequencer->receivedDatagram(datagram); + _sequencer.receivedDatagram(datagram); datagramsReceived++; bytesReceived += datagram.size(); _remainingPipelineCapacity += datagram.size(); } - int packetCount = _sequencer->startPacketGroup(); + int packetCount = _sequencer.startPacketGroup(); groupsSent++; maxPacketsPerGroup = qMax(maxPacketsPerGroup, packetCount); for (int i = 0; i < packetCount; i++) { oldDatagramsSent = datagramsSent; oldBytesSent = bytesSent; - Bitstream& out = _sequencer->startPacket(); + Bitstream& out = _sequencer.startPacket(); out << QVariant(); - _sequencer->endPacket(); + _sequencer.endPacket(); maxDatagramsPerPacket = qMax(maxDatagramsPerPacket, datagramsSent - oldDatagramsSent); maxBytesPerPacket = qMax(maxBytesPerPacket, bytesSent - oldBytesSent); // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber() }; - _sendRecords.append(record); + _sendRecords.append(maybeCreateSendRecord()); } return false; } else if (_mode == METAVOXEL_CLIENT_MODE) { - Bitstream& out = _sequencer->startPacket(); + Bitstream& out = _sequencer.startPacket(); ClientStateMessage state = { _lod }; out << QVariant::fromValue(state); - _sequencer->endPacket(); + _sequencer.endPacket(); // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber(), SharedObjectPointer(), MetavoxelData(), _lod }; - _sendRecords.append(record); + _sendRecords.append(maybeCreateSendRecord()); } else if (_mode == METAVOXEL_SERVER_MODE) { // make a random change @@ -879,16 +907,15 @@ bool TestEndpoint::simulate(int iterationNumber) { if (!_lod.isValid()) { return false; } - Bitstream& out = _sequencer->startPacket(); + Bitstream& out = _sequencer.startPacket(); out << QVariant::fromValue(MetavoxelDeltaMessage()); - _data.writeDelta(_sendRecords.first().data, _sendRecords.first().lod, out, _lod); - + PacketRecord* sendRecord = getLastAcknowledgedSendRecord(); + _data.writeDelta(sendRecord->getData(), sendRecord->getLOD(), out, _lod); + _sequencer.endPacket(); + // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber() + 1, SharedObjectPointer(), _data, _lod }; - _sendRecords.append(record); - - _sequencer->endPacket(); - + _sendRecords.append(maybeCreateSendRecord()); + } else { // enqueue some number of high priority messages const float MIN_HIGH_PRIORITY_MESSAGES = 0.0f; @@ -897,7 +924,7 @@ bool TestEndpoint::simulate(int iterationNumber) { while (_highPriorityMessagesToSend >= 1.0f) { QVariant message = createRandomMessage(); _highPriorityMessagesSent.append(message); - _sequencer->sendHighPriorityMessage(message); + _sequencer.sendHighPriorityMessage(message); highPriorityMessagesSent++; _highPriorityMessagesToSend -= 1.0f; } @@ -909,7 +936,7 @@ bool TestEndpoint::simulate(int iterationNumber) { while (_reliableMessagesToSend >= 1.0f) { QVariant message = createRandomMessage(); _reliableMessagesSent.append(message); - _sequencer->getReliableOutputChannel()->sendMessage(message); + _sequencer.getReliableOutputChannel()->sendMessage(message); reliableMessagesSent++; _reliableMessagesToSend -= 1.0f; } @@ -919,12 +946,12 @@ bool TestEndpoint::simulate(int iterationNumber) { // send a packet try { - Bitstream& out = _sequencer->startPacket(); + Bitstream& out = _sequencer.startPacket(); SequencedTestMessage message = { iterationNumber, createRandomMessage(), _localState }; _unreliableMessagesSent.append(message); unreliableMessagesSent++; out << message; - _sequencer->endPacket(); + _sequencer.endPacket(); } catch (const QString& message) { qDebug() << message; @@ -932,14 +959,28 @@ bool TestEndpoint::simulate(int iterationNumber) { } // record the send - SendRecord record = { _sequencer->getOutgoingPacketNumber(), _localState }; - _sendRecords.append(record); + _sendRecords.append(maybeCreateSendRecord()); } maxDatagramsPerPacket = qMax(maxDatagramsPerPacket, datagramsSent - oldDatagramsSent); maxBytesPerPacket = qMax(maxBytesPerPacket, bytesSent - oldBytesSent); return false; } +int TestEndpoint::parseData(const QByteArray& packet) { + if (_mode == CONGESTION_MODE) { + if (packet.size() <= _remainingPipelineCapacity) { + // have to copy the datagram; the one we're passed is a reference to a shared buffer + _pipeline[0].append(QByteArray(packet.constData(), packet.size())); + _remainingPipelineCapacity -= packet.size(); + } + } else { + _sequencer.receivedDatagram(packet); + datagramsReceived++; + bytesReceived += packet.size(); + } + return packet.size(); +} + void TestEndpoint::sendDatagram(const QByteArray& datagram) { datagramsSent++; bytesSent += datagram.size(); @@ -967,21 +1008,7 @@ void TestEndpoint::sendDatagram(const QByteArray& datagram) { } } - _other->receiveDatagram(datagram); -} - -void TestEndpoint::handleHighPriorityMessage(const QVariant& message) { - if (message.userType() == ClearSharedObjectMessage::Type) { - return; - } - if (_other->_highPriorityMessagesSent.isEmpty()) { - throw QString("Received unsent/already sent high priority message."); - } - QVariant sentMessage = _other->_highPriorityMessagesSent.takeFirst(); - if (!messagesEqual(message, sentMessage)) { - throw QString("Sent/received high priority message mismatch."); - } - highPriorityMessagesReceived++; + _other->parseData(datagram); } void TestEndpoint::readMessage(Bitstream& in) { @@ -990,8 +1017,7 @@ void TestEndpoint::readMessage(Bitstream& in) { in >> message; // record the receipt - ReceiveRecord record = { _sequencer->getIncomingPacketNumber() }; - _receiveRecords.append(record); + _receiveRecords.append(maybeCreateReceiveRecord()); return; } if (_mode == METAVOXEL_CLIENT_MODE) { @@ -1000,10 +1026,11 @@ void TestEndpoint::readMessage(Bitstream& in) { handleMessage(message, in); // deep-compare data to sent version - int packetNumber = _sequencer->getIncomingPacketNumber(); - foreach (const SendRecord& sendRecord, _other->_sendRecords) { - if (sendRecord.packetNumber == packetNumber) { - if (!sendRecord.data.deepEquals(_data, _sendRecords.first().lod)) { + int packetNumber = _sequencer.getIncomingPacketNumber(); + foreach (PacketRecord* record, _other->_sendRecords) { + TestSendRecord* sendRecord = static_cast(record); + if (sendRecord->getPacketNumber() == packetNumber) { + if (!sendRecord->getData().deepEquals(_data, getLastAcknowledgedSendRecord()->getLOD())) { qDebug() << "Sent/received metavoxel data mismatch."; exit(true); } @@ -1012,8 +1039,7 @@ void TestEndpoint::readMessage(Bitstream& in) { } // record the receipt - ReceiveRecord record = { packetNumber, SharedObjectPointer(), _data, _sendRecords.first().lod }; - _receiveRecords.append(record); + _receiveRecords.append(maybeCreateReceiveRecord()); return; } if (_mode == METAVOXEL_SERVER_MODE) { @@ -1022,8 +1048,7 @@ void TestEndpoint::readMessage(Bitstream& in) { handleMessage(message, in); // record the receipt - ReceiveRecord record = { _sequencer->getIncomingPacketNumber() }; - _receiveRecords.append(record); + _receiveRecords.append(maybeCreateReceiveRecord()); return; } @@ -1033,8 +1058,7 @@ void TestEndpoint::readMessage(Bitstream& in) { _remoteState = message.state; // record the receipt - ReceiveRecord record = { _sequencer->getIncomingPacketNumber(), message.state }; - _receiveRecords.append(record); + _receiveRecords.append(maybeCreateReceiveRecord()); for (QList::iterator it = _other->_unreliableMessagesSent.begin(); it != _other->_unreliableMessagesSent.end(); it++) { @@ -1056,6 +1080,45 @@ void TestEndpoint::readMessage(Bitstream& in) { exit(true); } +void TestEndpoint::handleMessage(const QVariant& message, Bitstream& in) { + int userType = message.userType(); + if (userType == ClientStateMessage::Type) { + ClientStateMessage state = message.value(); + _lod = state.lod; + + } else if (userType == MetavoxelDeltaMessage::Type) { + PacketRecord* receiveRecord = getLastAcknowledgedReceiveRecord(); + _data.readDelta(receiveRecord->getData(), receiveRecord->getLOD(), in, getLastAcknowledgedSendRecord()->getLOD()); + + } else if (userType == QMetaType::QVariantList) { + foreach (const QVariant& element, message.toList()) { + handleMessage(element, in); + } + } +} + +PacketRecord* TestEndpoint::maybeCreateSendRecord() const { + return new TestSendRecord(_lod, _data, _localState, _sequencer.getOutgoingPacketNumber()); +} + +PacketRecord* TestEndpoint::maybeCreateReceiveRecord() const { + return new TestReceiveRecord(getLastAcknowledgedSendRecord()->getLOD(), _data, _remoteState); +} + +void TestEndpoint::handleHighPriorityMessage(const QVariant& message) { + if (message.userType() == ClearSharedObjectMessage::Type) { + return; + } + if (_other->_highPriorityMessagesSent.isEmpty()) { + throw QString("Received unsent/already sent high priority message."); + } + QVariant sentMessage = _other->_highPriorityMessagesSent.takeFirst(); + if (!messagesEqual(message, sentMessage)) { + throw QString("Sent/received high priority message mismatch."); + } + highPriorityMessagesReceived++; +} + void TestEndpoint::handleReliableMessage(const QVariant& message) { if (message.userType() == ClearSharedObjectMessage::Type || message.userType() == ClearMainChannelSharedObjectMessage::Type) { @@ -1072,7 +1135,7 @@ void TestEndpoint::handleReliableMessage(const QVariant& message) { } void TestEndpoint::readReliableChannel() { - CircularBuffer& buffer = _sequencer->getReliableInputChannel(1)->getBuffer(); + CircularBuffer& buffer = _sequencer.getReliableInputChannel(1)->getBuffer(); QByteArray bytes = buffer.read(buffer.bytesAvailable()); if (_other->_dataStreamed.size() < bytes.size()) { throw QString("Received unsent/already sent streamed data."); @@ -1085,44 +1148,6 @@ void TestEndpoint::readReliableChannel() { streamedBytesReceived += bytes.size(); } -void TestEndpoint::clearSendRecordsBefore(int index) { - _sendRecords.erase(_sendRecords.begin(), _sendRecords.begin() + index + 1); -} - -void TestEndpoint::clearReceiveRecordsBefore(int index) { - _receiveRecords.erase(_receiveRecords.begin(), _receiveRecords.begin() + index + 1); -} - -void TestEndpoint::receiveDatagram(const QByteArray& datagram) { - if (_mode == CONGESTION_MODE) { - if (datagram.size() <= _remainingPipelineCapacity) { - // have to copy the datagram; the one we're passed is a reference to a shared buffer - _pipeline[0].append(QByteArray(datagram.constData(), datagram.size())); - _remainingPipelineCapacity -= datagram.size(); - } - } else { - _sequencer->receivedDatagram(datagram); - datagramsReceived++; - bytesReceived += datagram.size(); - } -} - -void TestEndpoint::handleMessage(const QVariant& message, Bitstream& in) { - int userType = message.userType(); - if (userType == ClientStateMessage::Type) { - ClientStateMessage state = message.value(); - _lod = state.lod; - - } else if (userType == MetavoxelDeltaMessage::Type) { - _data.readDelta(_receiveRecords.first().data, _receiveRecords.first().lod, in, _sendRecords.first().lod); - - } else if (userType == QMetaType::QVariantList) { - foreach (const QVariant& element, message.toList()) { - handleMessage(element, in); - } - } -} - TestSharedObjectA::TestSharedObjectA(float foo, TestEnum baz, TestFlags bong) : _foo(foo), _baz(baz), diff --git a/tests/metavoxels/src/MetavoxelTests.h b/tests/metavoxels/src/MetavoxelTests.h index 96d719a875..476a8c6295 100644 --- a/tests/metavoxels/src/MetavoxelTests.h +++ b/tests/metavoxels/src/MetavoxelTests.h @@ -15,8 +15,7 @@ #include #include -#include -#include +#include #include class SequencedTestMessage; @@ -35,14 +34,14 @@ public: }; /// Represents a simulated endpoint. -class TestEndpoint : public QObject { +class TestEndpoint : public Endpoint { Q_OBJECT public: enum Mode { BASIC_PEER_MODE, CONGESTION_MODE, METAVOXEL_SERVER_MODE, METAVOXEL_CLIENT_MODE }; - TestEndpoint(const QByteArray& datagramHeader, Mode mode = BASIC_PEER_MODE); + TestEndpoint(Mode mode = BASIC_PEER_MODE); void setOther(TestEndpoint* other) { _other = other; } @@ -50,45 +49,28 @@ public: /// \return true if failure was detected bool simulate(int iterationNumber); -private slots: + virtual int parseData(const QByteArray& packet); + +protected: - void sendDatagram(const QByteArray& datagram); + virtual void sendDatagram(const QByteArray& data); + virtual void readMessage(Bitstream& in); + + virtual void handleMessage(const QVariant& message, Bitstream& in); + + virtual PacketRecord* maybeCreateSendRecord() const; + virtual PacketRecord* maybeCreateReceiveRecord() const; + +private slots: + void handleHighPriorityMessage(const QVariant& message); - void readMessage(Bitstream& in); void handleReliableMessage(const QVariant& message); void readReliableChannel(); - void clearSendRecordsBefore(int index); - void clearReceiveRecordsBefore(int index); - private: - void receiveDatagram(const QByteArray& datagram); - - void handleMessage(const QVariant& message, Bitstream& in); - - class SendRecord { - public: - int packetNumber; - SharedObjectPointer localState; - MetavoxelData data; - MetavoxelLOD lod; - }; - - class ReceiveRecord { - public: - int packetNumber; - SharedObjectPointer remoteState; - MetavoxelData data; - MetavoxelLOD lod; - }; - Mode _mode; - DatagramSequencer* _sequencer; - QList _sendRecords; - QList _receiveRecords; - SharedObjectPointer _localState; SharedObjectPointer _remoteState; From 0ae58bd7f95ed37f92412c97fa04b1e64f9bdb90 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 15:39:11 -0700 Subject: [PATCH 35/67] added senderUUID arg to seq stats --- assignment-client/src/Agent.cpp | 4 +- assignment-client/src/audio/AudioMixer.cpp | 2 +- .../src/audio/AudioMixerClientData.cpp | 2 - .../src/audio/AudioMixerClientData.h | 2 +- .../octree/OctreeInboundPacketProcessor.cpp | 6 +- .../src/octree/OctreeInboundPacketProcessor.h | 6 +- interface/src/Application.cpp | 3 +- interface/src/Audio.cpp | 4 +- interface/src/ui/OctreeStatsDialog.cpp | 2 +- ...mbersStats.cpp => SequenceNumberStats.cpp} | 127 ++++--- ...ceNumbersStats.h => SequenceNumberStats.h} | 9 +- libraries/octree/src/OctreeSceneStats.cpp | 6 +- libraries/octree/src/OctreeSceneStats.h | 6 +- .../src/SequenceNumberStatsTests.cpp | 341 +++++++++--------- .../networking/src/SequenceNumberStatsTests.h | 2 +- 15 files changed, 284 insertions(+), 238 deletions(-) rename libraries/networking/src/{SequenceNumbersStats.cpp => SequenceNumberStats.cpp} (56%) rename libraries/networking/src/{SequenceNumbersStats.h => SequenceNumberStats.h} (84%) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index a2c9b30ac1..81ac9be04b 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -148,11 +148,13 @@ void Agent::readPendingDatagrams() { } else if (datagramPacketType == PacketTypeMixedAudio) { + QUuid senderUUID = uuidFromPacketHeader(receivedPacket); + // parse sequence number for this packet int numBytesPacketHeader = numBytesForPacketHeader(receivedPacket); const char* sequenceAt = receivedPacket.constData() + numBytesPacketHeader; quint16 sequence = *(reinterpret_cast(sequenceAt)); - _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); printf("mixed audio received %d\n", sequence); // parse the data and grab the average loudness diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 094d6da371..24539cb043 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -623,7 +623,7 @@ void AudioMixer::run() { dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO; // send mixed audio packet - //nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); + nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); printf("mixed audio sent %d\n", sequence); nodeData->incrementOutgoingSequenceNumber(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 6eec020991..795f9d4037 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -98,7 +98,6 @@ printf("injected stream %s received seq %d\n", streamIdentifier.toString().toLat if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector && ((InjectedAudioRingBuffer*) _ringBuffers[i])->getStreamIdentifier() == streamIdentifier) { matchingInjectedRingBuffer = (InjectedAudioRingBuffer*) _ringBuffers[i]; -printf("\t matching ring buffer found.\n"); } } @@ -107,7 +106,6 @@ printf("\t matching ring buffer found.\n"); matchingInjectedRingBuffer = new InjectedAudioRingBuffer(streamIdentifier, AudioMixer::getUseDynamicJitterBuffers()); _ringBuffers.push_back(matchingInjectedRingBuffer); -printf("\t no matching ring buffer, creating new one. _ringBuffer size now %d\n", _ringBuffers.size()); } matchingInjectedRingBuffer->parseData(packet); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index eb4e6f5dae..7cb5aa750e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -18,7 +18,7 @@ #include "AvatarAudioRingBuffer.h" #include "AudioStreamStats.h" -#include "SequenceNumbersStats.h" +#include "SequenceNumberStats.h" class AudioMixerClientData : public NodeData { public: diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index eaea2650b8..02a659eb96 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -206,7 +206,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { } const SharedNodePointer& destinationNode = NodeList::getInstance()->getNodeHash().value(nodeUUID); - const QSet& missingSequenceNumbers = nodeStats.getSequenceNumberStats().getMissingSet(); + const QSet& missingSequenceNumbers = nodeStats.getIncomingEditSequenceNumberStats().getMissingSet(); // construct nack packet(s) for this node int numSequenceNumbersAvailable = missingSequenceNumbers.size(); @@ -254,7 +254,7 @@ SingleSenderStats::SingleSenderStats() _totalLockWaitTime(0), _totalElementsInPacket(0), _totalPackets(0), - _sequenceNumberStats() + _incomingEditSequenceNumberStats() { } @@ -263,7 +263,7 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { // track sequence number - _sequenceNumberStats.sequenceNumberReceived(incomingSequence); + _incomingEditSequenceNumberStats.sequenceNumberReceived(incomingSequence); // update other stats _totalTransitTime += transitTime; diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 1c4f00c0a9..67135a8f27 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -16,7 +16,7 @@ #include -#include "SequenceNumbersStats.h" +#include "SequenceNumberStats.h" class OctreeServer; @@ -34,7 +34,7 @@ public: quint64 getAverageLockWaitTimePerElement() const { return _totalElementsInPacket == 0 ? 0 : _totalLockWaitTime / _totalElementsInPacket; } - const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } + const SequenceNumberStats& getIncomingEditSequenceNumberStats() const { return _incomingEditSequenceNumberStats; } void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); @@ -44,7 +44,7 @@ public: quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; - SequenceNumberStats _sequenceNumberStats; + SequenceNumberStats _incomingEditSequenceNumberStats; }; typedef QHash NodeToSenderStatsMap; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 80ef1ff6a1..cfebc97c05 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2164,7 +2164,8 @@ int Application::sendNackPackets() { OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; // make copy of missing sequence numbers from stats - const QSet missingSequenceNumbers = stats.getSequenceNumberStats().getMissingSet(); + const QSet missingSequenceNumbers = + stats.getIncomingOctreeSequenceNumberStats().getMissingSet(); _octreeSceneStatsLock.unlock(); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 024fb66cc1..9e129f9cb9 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -829,11 +829,13 @@ void Audio::toggleStereoInput() { void Audio::processReceivedAudio(const QByteArray& audioByteArray) { + QUuid senderUUID = uuidFromPacketHeader(audioByteArray); + // parse sequence number for this packet int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray); const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; quint16 sequence = *((quint16*)sequenceAt); - _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence); + _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); printf("mixed audio received %d\n", sequence); // parse audio data diff --git a/interface/src/ui/OctreeStatsDialog.cpp b/interface/src/ui/OctreeStatsDialog.cpp index 6303ff25d1..afa799815f 100644 --- a/interface/src/ui/OctreeStatsDialog.cpp +++ b/interface/src/ui/OctreeStatsDialog.cpp @@ -365,7 +365,7 @@ void OctreeStatsDialog::showOctreeServersOfType(int& serverCount, NodeType_t ser QString incomingPacketsString = locale.toString((uint)stats.getIncomingPackets()); QString incomingBytesString = locale.toString((uint)stats.getIncomingBytes()); QString incomingWastedBytesString = locale.toString((uint)stats.getIncomingWastedBytes()); - const SequenceNumberStats& seqStats = stats.getSequenceNumberStats(); + const SequenceNumberStats& seqStats = stats.getIncomingOctreeSequenceNumberStats(); QString incomingOutOfOrderString = locale.toString((uint)seqStats.getNumOutOfOrder()); QString incomingLateString = locale.toString((uint)seqStats.getNumLate()); QString incomingUnreasonableString = locale.toString((uint)seqStats.getNumUnreasonable()); diff --git a/libraries/networking/src/SequenceNumbersStats.cpp b/libraries/networking/src/SequenceNumberStats.cpp similarity index 56% rename from libraries/networking/src/SequenceNumbersStats.cpp rename to libraries/networking/src/SequenceNumberStats.cpp index eadf08e378..15d3c0542e 100644 --- a/libraries/networking/src/SequenceNumbersStats.cpp +++ b/libraries/networking/src/SequenceNumberStats.cpp @@ -9,7 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include "SequenceNumbersStats.h" +#include "SequenceNumberStats.h" #include @@ -22,20 +22,40 @@ SequenceNumberStats::SequenceNumberStats() _numLate(0), _numLost(0), _numRecovered(0), - _numDuplicate(0) + _numDuplicate(0), + _lastSenderUUID() { } -void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging) { +void SequenceNumberStats::reset() { + _missingSet.clear(); + _numReceived = 0; + _numUnreasonable = 0; + _numEarly = 0; + _numLate = 0; + _numLost = 0; + _numRecovered = 0; + _numDuplicate = 0; +} - static const int UINT16_RANGE = std::numeric_limits::max() + 1; - static const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work +static const int UINT16_RANGE = std::numeric_limits::max() + 1; +static const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work - _numReceived++; +void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, QUuid senderUUID, const bool wantExtraDebugging) { + + // if the sender node has changed, reset all stats + if (senderUUID != _lastSenderUUID) { + qDebug() << "sequence number stats was reset due to new sender node"; + qDebug() << "previous:" << _lastSenderUUID << "current:" << senderUUID; + reset(); + _lastSenderUUID = senderUUID; + } // determine our expected sequence number... handle rollover appropriately quint16 expected = _numReceived > 0 ? _lastReceived + (quint16)1 : incoming; + _numReceived++; + if (incoming == expected) { // on time _lastReceived = incoming; } else { // out of order @@ -80,6 +100,13 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa for (int missingInt = expectedInt; missingInt < incomingInt; missingInt++) { _missingSet.insert((quint16)(missingInt < 0 ? missingInt + UINT16_RANGE : missingInt)); } + + // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP + // will be removed. + if (_missingSet.size() > MAX_REASONABLE_SEQUENCE_GAP) { + pruneMissingSet(wantExtraDebugging); + } + _lastReceived = incoming; } else { // late if (wantExtraDebugging) { @@ -104,56 +131,54 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa // do not update _incomingLastSequence; it shouldn't become smaller } } +} - // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP - // will be removed. - if (_missingSet.size() > MAX_REASONABLE_SEQUENCE_GAP) { - if (wantExtraDebugging) { - qDebug() << "_missingSet is too large! size:" << _missingSet.size(); - } +void SequenceNumberStats::pruneMissingSet(const bool wantExtraDebugging) { + if (wantExtraDebugging) { + qDebug() << "pruning _missingSet! size:" << _missingSet.size(); + } - // some older sequence numbers may be from before a rollover point; this must be handled. - // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received - // before the most recent rollover. - int cutoff = (int)_lastReceived - MAX_REASONABLE_SEQUENCE_GAP; - if (cutoff >= 0) { - quint16 nonRolloverCutoff = (quint16)cutoff; - QSet::iterator i = _missingSet.begin(); - while (i != _missingSet.end()) { - quint16 missing = *i; - if (wantExtraDebugging) { - qDebug() << "checking item:" << missing << "is it in need of pruning?"; - qDebug() << "old age cutoff:" << nonRolloverCutoff; - } - - if (missing > _lastReceived || missing < nonRolloverCutoff) { - i = _missingSet.erase(i); - if (wantExtraDebugging) { - qDebug() << "pruning really old missing sequence:" << missing; - } - } else { - i++; - } + // some older sequence numbers may be from before a rollover point; this must be handled. + // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received + // before the most recent rollover. + int cutoff = (int)_lastReceived - MAX_REASONABLE_SEQUENCE_GAP; + if (cutoff >= 0) { + quint16 nonRolloverCutoff = (quint16)cutoff; + QSet::iterator i = _missingSet.begin(); + while (i != _missingSet.end()) { + quint16 missing = *i; + if (wantExtraDebugging) { + qDebug() << "checking item:" << missing << "is it in need of pruning?"; + qDebug() << "old age cutoff:" << nonRolloverCutoff; } - } else { - quint16 rolloverCutoff = (quint16)(cutoff + UINT16_RANGE); - QSet::iterator i = _missingSet.begin(); - while (i != _missingSet.end()) { - quint16 missing = *i; - if (wantExtraDebugging) { - qDebug() << "checking item:" << missing << "is it in need of pruning?"; - qDebug() << "old age cutoff:" << rolloverCutoff; - } - if (missing > _lastReceived && missing < rolloverCutoff) { - i = _missingSet.erase(i); - if (wantExtraDebugging) { - qDebug() << "pruning really old missing sequence:" << missing; - } - } else { - i++; + if (missing > _lastReceived || missing < nonRolloverCutoff) { + i = _missingSet.erase(i); + if (wantExtraDebugging) { + qDebug() << "pruning really old missing sequence:" << missing; } + } else { + i++; + } + } + } else { + quint16 rolloverCutoff = (quint16)(cutoff + UINT16_RANGE); + QSet::iterator i = _missingSet.begin(); + while (i != _missingSet.end()) { + quint16 missing = *i; + if (wantExtraDebugging) { + qDebug() << "checking item:" << missing << "is it in need of pruning?"; + qDebug() << "old age cutoff:" << rolloverCutoff; + } + + if (missing > _lastReceived && missing < rolloverCutoff) { + i = _missingSet.erase(i); + if (wantExtraDebugging) { + qDebug() << "pruning really old missing sequence:" << missing; + } + } else { + i++; } } } -} \ No newline at end of file +} diff --git a/libraries/networking/src/SequenceNumbersStats.h b/libraries/networking/src/SequenceNumberStats.h similarity index 84% rename from libraries/networking/src/SequenceNumbersStats.h rename to libraries/networking/src/SequenceNumberStats.h index 8dbb876a5b..b2561552ef 100644 --- a/libraries/networking/src/SequenceNumbersStats.h +++ b/libraries/networking/src/SequenceNumberStats.h @@ -13,12 +13,15 @@ #define hifi_SequenceNumberStats_h #include "SharedUtil.h" +#include class SequenceNumberStats { public: SequenceNumberStats(); - void sequenceNumberReceived(quint16 incoming, const bool wantExtraDebugging = false); + void reset(); + + void sequenceNumberReceived(quint16 incoming, QUuid senderUUID = QUuid(), const bool wantExtraDebugging = false); quint32 getNumReceived() const { return _numReceived; } quint32 getNumUnreasonable() const { return _numUnreasonable; } @@ -31,6 +34,8 @@ public: const QSet& getMissingSet() const { return _missingSet; } private: + void pruneMissingSet(const bool wantExtraDebugging); + quint16 _lastReceived; QSet _missingSet; @@ -41,6 +46,8 @@ private: quint32 _numLost; quint32 _numRecovered; quint32 _numDuplicate; + + QUuid _lastSenderUUID; }; #endif // hifi_SequenceNumberStats_h diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 5fbc4b0945..713a5f78f0 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -39,7 +39,7 @@ OctreeSceneStats::OctreeSceneStats() : _incomingPacket(0), _incomingBytes(0), _incomingWastedBytes(0), - _sequenceNumberStats(), + _incomingOctreeSequenceNumberStats(), _incomingFlightTimeAverage(samples), _jurisdictionRoot(NULL) { @@ -144,7 +144,7 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingBytes = other._incomingBytes; _incomingWastedBytes = other._incomingWastedBytes; - _sequenceNumberStats = other._sequenceNumberStats; + _incomingOctreeSequenceNumberStats = other._incomingOctreeSequenceNumberStats; } @@ -862,7 +862,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, return; // ignore any packets that are unreasonable } - _sequenceNumberStats.sequenceNumberReceived(sequence); + _incomingOctreeSequenceNumberStats.sequenceNumberReceived(sequence); // track packets here... _incomingPacket++; diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index 173f4c225f..d7b65c63be 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -17,7 +17,7 @@ #include #include "JurisdictionMap.h" #include "OctreePacketData.h" -#include "SequenceNumbersStats.h" +#include "SequenceNumberStats.h" #define GREENISH 0x40ff40d0 #define YELLOWISH 0xffef40c0 @@ -167,7 +167,7 @@ public: quint64 getIncomingWastedBytes() const { return _incomingWastedBytes; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } - const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; } + const SequenceNumberStats& getIncomingOctreeSequenceNumberStats() const { return _incomingOctreeSequenceNumberStats; } private: @@ -262,7 +262,7 @@ private: quint64 _incomingBytes; quint64 _incomingWastedBytes; - SequenceNumberStats _sequenceNumberStats; + SequenceNumberStats _incomingOctreeSequenceNumberStats; SimpleMovingAverage _incomingFlightTimeAverage; diff --git a/tests/networking/src/SequenceNumberStatsTests.cpp b/tests/networking/src/SequenceNumberStatsTests.cpp index 56da8fbaf9..5fd742fd65 100644 --- a/tests/networking/src/SequenceNumberStatsTests.cpp +++ b/tests/networking/src/SequenceNumberStatsTests.cpp @@ -32,16 +32,20 @@ void SequenceNumberStatsTests::rolloverTest() { // insert enough samples to cause 3 rollovers quint16 seq = 79; // start on some random number - for (int i = 0; i < 3 * UINT16_RANGE; i++) { - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - assert(stats.getNumDuplicate() == 0); - assert(stats.getNumEarly() == 0); - assert(stats.getNumLate() == 0); - assert(stats.getNumLost() == 0); - assert(stats.getNumReceived() == i+1); - assert(stats.getNumRecovered() == 0); + for (int R = 0; R < 2; R++) { + for (int i = 0; i < 3 * UINT16_RANGE; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == 0); + assert(stats.getNumLate() == 0); + assert(stats.getNumLost() == 0); + assert(stats.getNumReceived() == i + 1); + assert(stats.getNumRecovered() == 0); + } + stats.reset(); } } @@ -56,59 +60,61 @@ void SequenceNumberStatsTests::earlyLateTest() { int numLost = 0; int numRecovered = 0; + for (int R = 0; R < 2; R++) { + for (int T = 0; T < 10000; T++) { - for (int T = 0; T < 10000; T++) { + // insert 7 consecutive + for (int i = 0; i < 7; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; - // insert 7 consecutive - for (int i = 0; i < 7; i++) { - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } - assert(stats.getNumDuplicate() == 0); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == numRecovered); - } - - // skip 10 - quint16 skipped = seq; - seq = seq + (quint16)10; - - // insert 36 consecutive - numEarly++; - numLost += 10; - for (int i = 0; i < 36; i++) { - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; - - assert(stats.getNumDuplicate() == 0); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == numRecovered); - } - - // send ones we skipped - for (int i = 0; i < 10; i++) { - stats.sequenceNumberReceived(skipped); - skipped = skipped + (quint16)1; - numSent++; - numLate++; - numLost--; - numRecovered++; - - assert(stats.getNumDuplicate() == 0); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == numRecovered); + // skip 10 + quint16 skipped = seq; + seq = seq + (quint16)10; + + // insert 36 consecutive + numEarly++; + numLost += 10; + for (int i = 0; i < 36; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } + + // send ones we skipped + for (int i = 0; i < 10; i++) { + stats.sequenceNumberReceived(skipped); + skipped = skipped + (quint16)1; + numSent++; + numLate++; + numLost--; + numRecovered++; + + assert(stats.getNumDuplicate() == 0); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == numRecovered); + } } + stats.reset(); } } @@ -123,79 +129,81 @@ void SequenceNumberStatsTests::duplicateTest() { int numLate = 0; int numLost = 0; + for (int R = 0; R < 2; R++) { + for (int T = 0; T < 10000; T++) { - for (int T = 0; T < 10000; T++) { + quint16 duplicate = seq; - quint16 duplicate = seq; + // insert 7 consecutive + for (int i = 0; i < 7; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; - // insert 7 consecutive - for (int i = 0; i < 7; i++) { - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } - assert(stats.getNumDuplicate() == numDuplicate); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == 0); - } - - // skip 10 - quint16 skipped = seq; - seq = seq + (quint16)10; - - - quint16 duplicate2 = seq; - - numEarly++; - numLost += 10; - // insert 36 consecutive - for (int i = 0; i < 36; i++) { - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; - - assert(stats.getNumDuplicate() == numDuplicate); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == 0); - } - - // send 5 duplicates from before skip - for (int i = 0; i < 5; i++) { - stats.sequenceNumberReceived(duplicate); - duplicate = duplicate + (quint16)1; - numSent++; - numDuplicate++; - numLate++; - - assert(stats.getNumDuplicate() == numDuplicate); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == 0); - } - - // send 5 duplicates from after skip - for (int i = 0; i < 5; i++) { - stats.sequenceNumberReceived(duplicate2); - duplicate2 = duplicate2 + (quint16)1; - numSent++; - numDuplicate++; - numLate++; - - assert(stats.getNumDuplicate() == numDuplicate); - assert(stats.getNumEarly() == numEarly); - assert(stats.getNumLate() == numLate); - assert(stats.getNumLost() == numLost); - assert(stats.getNumReceived() == numSent); - assert(stats.getNumRecovered() == 0); + // skip 10 + quint16 skipped = seq; + seq = seq + (quint16)10; + + + quint16 duplicate2 = seq; + + numEarly++; + numLost += 10; + // insert 36 consecutive + for (int i = 0; i < 36; i++) { + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + + // send 5 duplicates from before skip + for (int i = 0; i < 5; i++) { + stats.sequenceNumberReceived(duplicate); + duplicate = duplicate + (quint16)1; + numSent++; + numDuplicate++; + numLate++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } + + // send 5 duplicates from after skip + for (int i = 0; i < 5; i++) { + stats.sequenceNumberReceived(duplicate2); + duplicate2 = duplicate2 + (quint16)1; + numSent++; + numDuplicate++; + numLate++; + + assert(stats.getNumDuplicate() == numDuplicate); + assert(stats.getNumEarly() == numEarly); + assert(stats.getNumLate() == numLate); + assert(stats.getNumLost() == numLost); + assert(stats.getNumReceived() == numSent); + assert(stats.getNumRecovered() == 0); + } } + stats.reset(); } } @@ -209,50 +217,53 @@ void SequenceNumberStatsTests::pruneTest() { int numLate = 0; int numLost = 0; - for (int T = 0; T < 1000; T++) { - // insert 1 seq - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; + for (int R = 0; R < 2; R++) { + for (int T = 0; T < 1000; T++) { + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; - // skip 1000 seq - seq = seq + (quint16)1000; - quint16 highestSkipped = seq - (quint16)1; + // skip 1000 seq + seq = seq + (quint16)1000; + quint16 highestSkipped = seq - (quint16)1; - // insert 1 seq - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; - numEarly++; - numLost += 1000; + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + numEarly++; + numLost += 1000; - // skip 10 seq - seq = seq + (quint16)10; - quint16 highestSkipped2 = seq - (quint16)1; + // skip 10 seq + seq = seq + (quint16)10; + quint16 highestSkipped2 = seq - (quint16)1; - // insert 1 seq - // insert 1 seq - stats.sequenceNumberReceived(seq); - seq = seq + (quint16)1; - numSent++; - numEarly++; - numLost += 10; + // insert 1 seq + // insert 1 seq + stats.sequenceNumberReceived(seq); + seq = seq + (quint16)1; + numSent++; + numEarly++; + numLost += 10; - const QSet& missingSet = stats.getMissingSet(); - assert(missingSet.size() <= 1000); + const QSet& missingSet = stats.getMissingSet(); + assert(missingSet.size() <= 1000); - for (int i = 0; i < 10; i++) { - assert(missingSet.contains(highestSkipped2)); - highestSkipped2 = highestSkipped2 - (quint16)1; - } - - for (int i = 0; i < 989; i++) { - assert(missingSet.contains(highestSkipped)); - highestSkipped = highestSkipped - (quint16)1; - } - for (int i = 0; i < 11; i++) { - assert(!missingSet.contains(highestSkipped)); - highestSkipped = highestSkipped - (quint16)1; + for (int i = 0; i < 10; i++) { + assert(missingSet.contains(highestSkipped2)); + highestSkipped2 = highestSkipped2 - (quint16)1; + } + + for (int i = 0; i < 989; i++) { + assert(missingSet.contains(highestSkipped)); + highestSkipped = highestSkipped - (quint16)1; + } + for (int i = 0; i < 11; i++) { + assert(!missingSet.contains(highestSkipped)); + highestSkipped = highestSkipped - (quint16)1; + } } + stats.reset(); } -} \ No newline at end of file +} diff --git a/tests/networking/src/SequenceNumberStatsTests.h b/tests/networking/src/SequenceNumberStatsTests.h index b57bf69342..53a0b66480 100644 --- a/tests/networking/src/SequenceNumberStatsTests.h +++ b/tests/networking/src/SequenceNumberStatsTests.h @@ -13,7 +13,7 @@ #define hifi_SequenceNumberStatsTests_h #include "SequenceNumberStatsTests.h" -#include "SequenceNumbersStats.h" +#include "SequenceNumberStats.h" namespace SequenceNumberStatsTests { From f2bdacd5a8c425eca66420ee22afc6678e10fdf5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 27 Jun 2014 16:19:18 -0700 Subject: [PATCH 36/67] Fix for shared object reference tracking. --- tests/metavoxels/src/MetavoxelTests.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 48d1d4429d..61ab664310 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -1098,11 +1098,13 @@ void TestEndpoint::handleMessage(const QVariant& message, Bitstream& in) { } PacketRecord* TestEndpoint::maybeCreateSendRecord() const { - return new TestSendRecord(_lod, _data, _localState, _sequencer.getOutgoingPacketNumber()); + return new TestSendRecord(_lod, (_mode == METAVOXEL_CLIENT_MODE) ? MetavoxelData() : _data, + _localState, _sequencer.getOutgoingPacketNumber()); } PacketRecord* TestEndpoint::maybeCreateReceiveRecord() const { - return new TestReceiveRecord(getLastAcknowledgedSendRecord()->getLOD(), _data, _remoteState); + return new TestReceiveRecord(getLastAcknowledgedSendRecord()->getLOD(), + (_mode == METAVOXEL_SERVER_MODE) ? MetavoxelData() : _data, _remoteState); } void TestEndpoint::handleHighPriorityMessage(const QVariant& message) { From 9ceac2c631b13d5929378874ef7a3f10ffdffa0a Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 16:34:30 -0700 Subject: [PATCH 37/67] fixed Audio.cpp jitter buffer size clamp bug --- interface/src/Audio.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 9e129f9cb9..5d42234be1 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -672,7 +672,7 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); -printf("avatar audio sent %d\n", _outgoingAvatarAudioSequenceNumber); +//printf("avatar audio sent %d\n", _outgoingAvatarAudioSequenceNumber); _outgoingAvatarAudioSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) @@ -836,7 +836,7 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; quint16 sequence = *((quint16*)sequenceAt); _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); -printf("mixed audio received %d\n", sequence); +printf("\nmixed audio received %d\n", sequence); // parse audio data _ringBuffer.parseData(audioByteArray); @@ -844,16 +844,22 @@ printf("mixed audio received %d\n", sequence); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); +printf("parse data: ring buffer now has %d samples\n", _ringBuffer.samplesAvailable()); + if (!_ringBuffer.isStarved() && _audioOutput && _audioOutput->bytesFree() == _audioOutput->bufferSize()) { // we don't have any audio data left in the output buffer // we just starved //qDebug() << "Audio output just starved."; _ringBuffer.setIsStarved(true); _numFramesDisplayStarve = 10; + +printf("ring buffer just starved!\n"); } // if there is anything in the ring buffer, decide what to do if (_ringBuffer.samplesAvailable() > 0) { + + int numNetworkOutputSamples = _ringBuffer.samplesAvailable(); int numDeviceOutputSamples = numNetworkOutputSamples / networkOutputToOutputRatio; @@ -861,11 +867,14 @@ printf("mixed audio received %d\n", sequence); QByteArray outputBuffer; outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t)); - int numSamplesNeededToStartPlayback = NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (_jitterBufferSamples * 2); + int numSamplesNeededToStartPlayback = (_jitterBufferSamples * 2);// + NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numSamplesNeededToStartPlayback)) { // We are still waiting for enough samples to begin playback // qDebug() << numNetworkOutputSamples << " samples so far, waiting for " << numSamplesNeededToStartPlayback; + +printf("MIX DELAYED: waiting for ring buffer to refill to %d samples after starve\n", numSamplesNeededToStartPlayback); + } else { // We are either already playing back, or we have enough audio to start playing back. //qDebug() << "pushing " << numNetworkOutputSamples; @@ -878,6 +887,8 @@ printf("mixed audio received %d\n", sequence); buffer.resize(numNetworkOutputSamples * sizeof(int16_t)); _ringBuffer.readSamples((int16_t*)buffer.data(), numNetworkOutputSamples); +printf("mixed. %d samples remaining\n", _ringBuffer.samplesAvailable()); + // Accumulate direct transmission of audio from sender to receiver if (Menu::getInstance()->isOptionChecked(MenuOption::AudioSpatialProcessingIncludeOriginal)) { emit preProcessOriginalInboundAudio(sampleTime, buffer, _desiredOutputFormat); @@ -897,6 +908,7 @@ printf("mixed audio received %d\n", sequence); // copy the samples we'll resample from the ring buffer - this also // pushes the read pointer of the ring buffer forwards _ringBuffer.readSamples(ringBufferSamples, numNetworkOutputSamples); +printf("mixed. %d samples remaining\n", _ringBuffer.samplesAvailable()); } // copy the packet from the RB to the output From 4587828a3a00e98cd4b70b12cb933c3e1f04a7d8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 16:48:21 -0700 Subject: [PATCH 38/67] removed debug prints --- assignment-client/src/Agent.cpp | 1 - assignment-client/src/audio/AudioMixer.cpp | 5 +---- assignment-client/src/audio/AudioMixerClientData.cpp | 2 -- assignment-client/src/audio/AudioMixerClientData.h | 2 +- interface/src/Audio.cpp | 11 ----------- libraries/script-engine/src/ScriptEngine.cpp | 6 +++--- libraries/script-engine/src/ScriptEngine.h | 2 +- 7 files changed, 6 insertions(+), 23 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 81ac9be04b..bbb367fdcb 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -155,7 +155,6 @@ void Agent::readPendingDatagrams() { const char* sequenceAt = receivedPacket.constData() + numBytesPacketHeader; quint16 sequence = *(reinterpret_cast(sequenceAt)); _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); -printf("mixed audio received %d\n", sequence); // parse the data and grab the average loudness _receivedAudioBuffer.parseData(receivedPacket); diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 24539cb043..bc5c2f1a06 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -457,7 +457,6 @@ void AudioMixer::sendStatsPacket() { // if we're too large, send the packet if (sizeOfStats > TOO_BIG_FOR_MTU) { -printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); sizeOfStats = 0; statsObject2 = QJsonObject(); // clear it @@ -466,7 +465,6 @@ printf("sending stats to domain server: size: %d\n", sizeOfStats); } if (somethingToSend) { -printf("sending stats to domain server: size: %d\n", sizeOfStats); nodeList->sendStatsToDomainServer(statsObject2); } } @@ -624,8 +622,7 @@ void AudioMixer::run() { // send mixed audio packet nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node); -printf("mixed audio sent %d\n", sequence); - nodeData->incrementOutgoingSequenceNumber(); + nodeData->incrementOutgoingMixedAudioSequenceNumber(); // send an audio stream stats packet if it's time if (sendAudioStreamStats) { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 795f9d4037..9b1bba5226 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -58,7 +58,6 @@ int AudioMixerClientData::parseData(const QByteArray& packet) { || packetType == PacketTypeSilentAudioFrame) { _incomingAvatarAudioSequenceNumberStats.sequenceNumberReceived(sequence); -printf("avatar audio received %d\n", sequence); // grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist) AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); @@ -90,7 +89,6 @@ printf("avatar audio received %d\n", sequence); QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet) + sizeof(quint16), NUM_BYTES_RFC4122_UUID)); _incomingInjectedAudioSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence); -printf("injected stream %s received seq %d\n", streamIdentifier.toString().toLatin1().data(), sequence); InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 7cb5aa750e..71b1bff711 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -38,7 +38,7 @@ public: QString getJitterBufferStatsString() const; - void incrementOutgoingSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } + void incrementOutgoingMixedAudioSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 5d42234be1..35ae1342bb 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -672,7 +672,6 @@ void Audio::handleAudioInput() { currentPacketPtr += sizeof(headOrientation); nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer); -//printf("avatar audio sent %d\n", _outgoingAvatarAudioSequenceNumber); _outgoingAvatarAudioSequenceNumber++; Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) @@ -836,7 +835,6 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader; quint16 sequence = *((quint16*)sequenceAt); _incomingMixedAudioSequenceNumberStats.sequenceNumberReceived(sequence, senderUUID); -printf("\nmixed audio received %d\n", sequence); // parse audio data _ringBuffer.parseData(audioByteArray); @@ -844,16 +842,12 @@ printf("\nmixed audio received %d\n", sequence); float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate()) * (_desiredOutputFormat.channelCount() / (float) _outputFormat.channelCount()); -printf("parse data: ring buffer now has %d samples\n", _ringBuffer.samplesAvailable()); - if (!_ringBuffer.isStarved() && _audioOutput && _audioOutput->bytesFree() == _audioOutput->bufferSize()) { // we don't have any audio data left in the output buffer // we just starved //qDebug() << "Audio output just starved."; _ringBuffer.setIsStarved(true); _numFramesDisplayStarve = 10; - -printf("ring buffer just starved!\n"); } // if there is anything in the ring buffer, decide what to do @@ -872,9 +866,6 @@ printf("ring buffer just starved!\n"); if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numSamplesNeededToStartPlayback)) { // We are still waiting for enough samples to begin playback // qDebug() << numNetworkOutputSamples << " samples so far, waiting for " << numSamplesNeededToStartPlayback; - -printf("MIX DELAYED: waiting for ring buffer to refill to %d samples after starve\n", numSamplesNeededToStartPlayback); - } else { // We are either already playing back, or we have enough audio to start playing back. //qDebug() << "pushing " << numNetworkOutputSamples; @@ -887,7 +878,6 @@ printf("MIX DELAYED: waiting for ring buffer to refill to %d samples after starv buffer.resize(numNetworkOutputSamples * sizeof(int16_t)); _ringBuffer.readSamples((int16_t*)buffer.data(), numNetworkOutputSamples); -printf("mixed. %d samples remaining\n", _ringBuffer.samplesAvailable()); // Accumulate direct transmission of audio from sender to receiver if (Menu::getInstance()->isOptionChecked(MenuOption::AudioSpatialProcessingIncludeOriginal)) { @@ -908,7 +898,6 @@ printf("mixed. %d samples remaining\n", _ringBuffer.samplesAvailable()); // copy the samples we'll resample from the ring buffer - this also // pushes the read pointer of the ring buffer forwards _ringBuffer.readSamples(ringBufferSamples, numNetworkOutputSamples); -printf("mixed. %d samples remaining\n", _ringBuffer.samplesAvailable()); } // copy the packet from the RB to the output diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 09b60d8e0d..d78069bb62 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -501,9 +501,9 @@ void ScriptEngine::run() { // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { // pack sequence number - quint16 sequence = _outgoingInjectedAudioSequenceNumbers[node->getUUID()]++; + quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++; memcpy(sequenceAt, &sequence, sizeof(quint16)); -printf("script engine audio sent %d\n", sequence); + // send audio packet nodeList->writeDatagram(audioPacket, audioPacketSize, node); } @@ -683,5 +683,5 @@ void ScriptEngine::include(const QString& includeFile) { } void ScriptEngine::nodeKilled(SharedNodePointer node) { - _outgoingInjectedAudioSequenceNumbers.remove(node->getUUID()); + _outgoingScriptAudioSequenceNumbers.remove(node->getUUID()); } diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index cd969e49cd..5b01b8124a 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -148,7 +148,7 @@ private: ScriptUUID _uuidLibrary; AnimationCache _animationCache; - QHash _outgoingInjectedAudioSequenceNumbers; + QHash _outgoingScriptAudioSequenceNumbers; }; #endif // hifi_ScriptEngine_h From a0795fd6e6fd33783c53f9cd99515f4a03bcd371 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 16:52:15 -0700 Subject: [PATCH 39/67] removed extra print --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 9b1bba5226..c6333ff645 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -212,7 +212,7 @@ QString AudioMixerClientData::getJitterBufferStatsString() const { } else { result = "mic unknown"; } - printf("\tget jitter buffer stats string. _ringBuffer.size = %d\n", _ringBuffers.size()); + for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); From b623276fdef5086c2c413f352cbdccfff0cb9c67 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 27 Jun 2014 17:39:51 -0700 Subject: [PATCH 40/67] Increase the LOD threshold for now to get around spanner issue. --- tests/metavoxels/src/MetavoxelTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 61ab664310..7b7d0430d0 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -652,7 +652,7 @@ TestEndpoint::TestEndpoint(Mode mode) : SLOT(handleHighPriorityMessage(const QVariant&))); if (mode == METAVOXEL_CLIENT_MODE) { - _lod = MetavoxelLOD(glm::vec3(), 0.01f); + _lod = MetavoxelLOD(glm::vec3(), 0.001f); return; } if (mode == METAVOXEL_SERVER_MODE) { From c3f571745c6e9dfc27f5c00060582ac084db32b8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 27 Jun 2014 17:42:22 -0700 Subject: [PATCH 41/67] removed more debug stuff --- interface/src/Audio.cpp | 5 ----- interface/src/ui/Stats.cpp | 2 +- libraries/audio/src/AudioInjector.cpp | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 35ae1342bb..6173605f05 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -676,9 +676,6 @@ void Audio::handleAudioInput() { Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO) .updateValue(numAudioBytes + leadingBytes); - } else { - // reset seq numbers if there's no connection with an audiomixer - _outgoingAvatarAudioSequenceNumber = 0; } delete[] inputAudioSamples; } @@ -852,8 +849,6 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { // if there is anything in the ring buffer, decide what to do if (_ringBuffer.samplesAvailable() > 0) { - - int numNetworkOutputSamples = _ringBuffer.samplesAvailable(); int numDeviceOutputSamples = numNetworkOutputSamples / networkOutputToOutputRatio; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index ca77d6887c..bdd1d63a0b 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -299,7 +299,7 @@ void Stats::display( (float) audio->getNetworkSampleRate() * 1000.f); drawText(30, glWidget->height() - 22, scale, rotation, font, audioJitter, color); - + char audioPing[30]; sprintf(audioPing, "Audio ping: %d", pingAudio); diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 01dfc8696a..2dd8680697 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -66,8 +66,7 @@ void AudioInjector::injectAudio() { packetStream << (quint16)0; // pack stream identifier (a generated UUID) -QUuid streamID; - packetStream << (streamID = QUuid::createUuid()); + packetStream << QUuid::createUuid(); // pack the flag for loopback uchar loopbackFlag = (uchar) (!_options.getLoopbackAudioInterface()); @@ -118,7 +117,6 @@ QUuid streamID; // send off this audio packet nodeList->writeDatagram(injectAudioPacket, audioMixer); -printf("injector stream %s sent seq %d\n", streamID.toString().toLatin1().data(), outgoingInjectedAudioSequenceNumber); outgoingInjectedAudioSequenceNumber++; currentSendPosition += bytesToCopy; From de3c1ebf0f10158d94b0b7bf5a9785f68f73adda Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 09:42:22 -0700 Subject: [PATCH 42/67] changed class name to AudioStreamStats, reverted ScriptEngine packing back to QDataStream --- .../src/audio/AudioMixerClientData.cpp | 10 ++--- .../src/audio/AudioMixerClientData.h | 2 +- interface/src/Audio.cpp | 6 +-- interface/src/Audio.h | 4 +- interface/src/ui/Stats.cpp | 8 ++-- libraries/audio/src/AudioStreamStats.h | 4 +- libraries/script-engine/src/ScriptEngine.cpp | 41 ++++++++----------- 7 files changed, 34 insertions(+), 41 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index c6333ff645..ea13289186 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -156,7 +156,7 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } } -void AudioMixerClientData::getJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const { +void AudioMixerClientData::getAudioStreamStats(AudioStreamStats& stats) const { int avatarJitterBufferFrames = 0; int maxJitterBufferFrames = 0; int sumJitterBufferFrames = 0; @@ -185,10 +185,10 @@ int AudioMixerClientData::encodeAudioStreamStatsPacket(char* packet) const { char* dataAt = packet + numBytesPacketHeader; // pack jitter buffer stats - AudioMixerJitterBuffersStats jitterBufferStats; - getJitterBuffersStats(jitterBufferStats); - memcpy(dataAt, &jitterBufferStats, sizeof(AudioMixerJitterBuffersStats)); - dataAt += sizeof(AudioMixerJitterBuffersStats); + AudioStreamStats stats; + getAudioStreamStats(stats); + memcpy(dataAt, &stats, sizeof(AudioStreamStats)); + dataAt += sizeof(AudioStreamStats); return dataAt - packet; } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 71b1bff711..d9d6f45a0e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -32,7 +32,7 @@ public: void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); - void getJitterBuffersStats(AudioMixerJitterBuffersStats& stats) const; + void getAudioStreamStats(AudioStreamStats& stats) const; int encodeAudioStreamStatsPacket(char* packet) const; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 6173605f05..5c1bc96961 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -103,7 +103,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), - _audioMixerJitterBufferStats(), + _audioMixerStreamStats(), _outgoingAvatarAudioSequenceNumber(0) { // clear the array of locally injected samples @@ -721,8 +721,8 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { const char* dataAt = packet.constData() + numBytesPacketHeader; // parse audio mixer jitter buffer stats - memcpy(&_audioMixerJitterBufferStats, dataAt, sizeof(AudioMixerJitterBuffersStats)); - dataAt += sizeof(AudioMixerJitterBuffersStats); + memcpy(&_audioMixerStreamStats, dataAt, sizeof(AudioStreamStats)); + dataAt += sizeof(AudioStreamStats); } // NOTE: numSamples is the total number of single channel samples, since callers will always call this with stereo diff --git a/interface/src/Audio.h b/interface/src/Audio.h index f08cce9467..f3b01161d1 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -104,7 +104,7 @@ public slots: float getInputVolume() const { return (_audioInput) ? _audioInput->volume() : 0.0f; } void setInputVolume(float volume) { if (_audioInput) _audioInput->setVolume(volume); } - const AudioMixerJitterBuffersStats& getAudioMixerJitterBuffersStats() const { return _audioMixerJitterBufferStats; } + const AudioStreamStats& getAudioMixerStreamStats() const { return _audioMixerStreamStats; } signals: bool muteToggled(); @@ -237,7 +237,7 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - AudioMixerJitterBuffersStats _audioMixerJitterBufferStats; + AudioStreamStats _audioMixerStreamStats; quint16 _outgoingAvatarAudioSequenceNumber; SequenceNumberStats _incomingMixedAudioSequenceNumberStats; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index bdd1d63a0b..9697f5af03 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -325,13 +325,13 @@ void Stats::display( static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; - const AudioMixerJitterBuffersStats& audioMixerJitterBufferStats = - Application::getInstance()->getAudio()->getAudioMixerJitterBuffersStats(); + const AudioStreamStats& audioMixerStreamStats = + Application::getInstance()->getAudio()->getAudioMixerStreamStats(); char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; char audioMixerJitterBuffersStats[30]; - sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerJitterBufferStats._avatarJitterBufferFrames * MSECS_PER_FRAME, - audioMixerJitterBufferStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerJitterBufferStats._avgJitterBufferFrames * MSECS_PER_FRAME); + sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerStreamStats._avatarJitterBufferFrames * MSECS_PER_FRAME, + audioMixerStreamStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerStreamStats._avgJitterBufferFrames * MSECS_PER_FRAME); verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 38d74cbd1c..5746462bcf 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -12,9 +12,9 @@ #ifndef hifi_AudioStreamStats_h #define hifi_AudioStreamStats_h -class AudioMixerJitterBuffersStats { +class AudioStreamStats { public: - AudioMixerJitterBuffersStats() + AudioStreamStats() : _avatarJitterBufferFrames(0), _maxJitterBufferFrames(0), _avgJitterBufferFrames(0) {} diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index d78069bb62..1b225b3699 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -458,54 +458,47 @@ void ScriptEngine::run() { _numAvatarSoundSentBytes = 0; } } + + QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); - char audioPacket[MAX_PACKET_SIZE]; + QDataStream packetStream(&audioPacket, QIODevice::Append); - // pack header - int numBytesPacketHeader = populatePacketHeader(audioPacket, silentFrame - ? PacketTypeSilentAudioFrame - : PacketTypeMicrophoneAudioNoEcho); - char* dataAt = audioPacket + numBytesPacketHeader; - - // skip over sequence number for now; will be packed when destination node is known - char* sequenceAt = dataAt; - dataAt += sizeof(quint16); + // pack a placeholder value for sequence number for now, will be packed when destination node is known + int numPreSequenceNumberBytes = audioPacket.size(); + packetStream << (quint16)0; // use the orientation and position of this avatar for the source of this audio - memcpy(dataAt, &_avatarData->getPosition(), sizeof(glm::vec3)); - dataAt += sizeof(glm::vec3); - + packetStream.writeRawData(reinterpret_cast(&_avatarData->getPosition()), sizeof(glm::vec3)); glm::quat headOrientation = _avatarData->getHeadOrientation(); - memcpy(dataAt, &headOrientation, sizeof(glm::quat)); - dataAt += sizeof(glm::quat); + packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); if (silentFrame) { if (!_isListeningToAudioStream) { // if we have a silent frame and we're not listening then just send nothing and break out of here break; } + // write the number of silent samples so the audio-mixer can uphold timing - memcpy(dataAt, &SCRIPT_AUDIO_BUFFER_SAMPLES, sizeof(int16_t)); - dataAt += sizeof(int16_t); + packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); } else if (nextSoundOutput) { // write the raw audio data - int numAvailableBytes = numAvailableSamples * sizeof(int16_t); - memcpy(dataAt, nextSoundOutput, numAvailableBytes); - dataAt += numAvailableBytes; + packetStream.writeRawData(reinterpret_cast(nextSoundOutput), + numAvailableSamples * sizeof(int16_t)); } // write audio packet to AudioMixer nodes - int audioPacketSize = dataAt - audioPacket; NodeList* nodeList = NodeList::getInstance(); foreach(const SharedNodePointer& node, nodeList->getNodeHash()) { // only send to nodes of type AudioMixer if (node->getType() == NodeType::AudioMixer) { // pack sequence number quint16 sequence = _outgoingScriptAudioSequenceNumbers[node->getUUID()]++; - memcpy(sequenceAt, &sequence, sizeof(quint16)); - + memcpy(audioPacket.data() + numPreSequenceNumberBytes, &sequence, sizeof(quint16)); + // send audio packet - nodeList->writeDatagram(audioPacket, audioPacketSize, node); + nodeList->writeDatagram(audioPacket, node); } } } From 9e61b5e45f2ff46da0d2eb1ce610c100886e67f6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Jun 2014 11:16:35 -0700 Subject: [PATCH 43/67] Fix Windows link error (hopefully). --- tools/bitstream2json/CMakeLists.txt | 4 ++++ tools/json2bitstream/CMakeLists.txt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt index 73fbe92204..d5b82adbd9 100644 --- a/tools/bitstream2json/CMakeLists.txt +++ b/tools/bitstream2json/CMakeLists.txt @@ -24,4 +24,8 @@ link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} "${ROOT_DIR}") +IF (WIN32) + target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt index 1dc01c4c7f..b93c57b582 100644 --- a/tools/json2bitstream/CMakeLists.txt +++ b/tools/json2bitstream/CMakeLists.txt @@ -24,4 +24,8 @@ link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} "${ROOT_DIR}") +IF (WIN32) + target_link_libraries(${TARGET_NAME} Winmm Ws2_32) +ENDIF(WIN32) + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) From d109ef6fe1dbd39eba3793960e352e51950effb9 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 11:33:22 -0700 Subject: [PATCH 44/67] Proper mirror and third person mode in Oculus --- interface/src/Application.cpp | 22 +++---- interface/src/avatar/MyAvatar.cpp | 17 ++++-- interface/src/devices/OculusManager.cpp | 79 ++++++++++++++++--------- interface/src/devices/OculusManager.h | 3 +- interface/src/renderer/GlowEffect.cpp | 2 +- 5 files changed, 74 insertions(+), 49 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 117842dee4..906098ebe3 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -582,8 +582,8 @@ void Application::paintGL() { } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { _myCamera.setTightness(0.0f); // Camera is directly connected to head without smoothing - _myCamera.setTargetPosition(_myAvatar->getUprightHeadPosition()); - _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); + _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); + _myCamera.setTargetRotation(_myAvatar->getWorldAlignedOrientation()); } else if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { _myCamera.setTightness(0.0f); @@ -594,17 +594,6 @@ void Application::paintGL() { _myCamera.setTargetRotation(_myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f))); } - if (OculusManager::isConnected()) { - // Oculus in third person causes nausea, so only allow it if option is checked in dev menu - if (!Menu::getInstance()->isOptionChecked(MenuOption::AllowOculusCameraModeChange) || _myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { - _myCamera.setDistance(0.0f); - _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); - _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); - } - _myCamera.setUpShift(0.0f); - _myCamera.setTightness(0.0f); // Camera is directly connected to head without smoothing - } - // Update camera position _myCamera.update( 1.f/_fps ); @@ -638,7 +627,12 @@ void Application::paintGL() { } if (OculusManager::isConnected()) { - OculusManager::display(_myAvatar->getOrientation(), whichCamera); + //When in mirror mode, use camera rotation. Otherwise, use body rotation + if (whichCamera.getMode() == CAMERA_MODE_MIRROR) { + OculusManager::display(whichCamera.getRotation(), whichCamera.getPosition(), whichCamera); + } else { + OculusManager::display(_myAvatar->getWorldAlignedOrientation(), whichCamera.getPosition(), whichCamera); + } } else if (TV3DManager::isConnected()) { _glowEffect.prepare(); diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 7367f64d73..7599baf7da 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -856,7 +856,9 @@ void MyAvatar::renderBody(RenderMode renderMode, float glowLevel) { renderAttachments(renderMode); // Render head so long as the camera isn't inside it - if (shouldRenderHead(Application::getInstance()->getCamera()->getPosition(), renderMode)) { + const Camera *camera = Application::getInstance()->getCamera(); + const glm::vec3 cameraPos = camera->getPosition() + (camera->getRotation() * glm::vec3(0.0f, 0.0f, 1.0f)) * camera->getDistance(); + if (shouldRenderHead(cameraPos, renderMode)) { getHead()->render(1.0f, modelRenderMode); if (Menu::getInstance()->isOptionChecked(MenuOption::StringHair)) { renderHair(); @@ -920,9 +922,16 @@ void MyAvatar::updateOrientation(float deltaTime) { glm::vec3 angularVelocity(yaw - head->getBaseYaw(), pitch - head->getBasePitch(), roll - head->getBaseRoll()); head->setAngularVelocity(angularVelocity); - head->setBaseYaw(yaw); - head->setBasePitch(pitch); - head->setBaseRoll(roll); + //Invert yaw and roll when in mirror mode + if (Application::getInstance()->getCamera()->getMode() == CAMERA_MODE_MIRROR) { + head->setBaseYaw(-yaw); + head->setBasePitch(pitch); + head->setBaseRoll(-roll); + } else { + head->setBaseYaw(yaw); + head->setBasePitch(pitch); + head->setBaseRoll(roll); + } } diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 35122ba0f5..b505f16c8b 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -81,9 +81,6 @@ void OculusManager::connect() { _eyeRenderDesc[0] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Left, _eyeFov[0]); _eyeRenderDesc[1] = ovrHmd_GetRenderDesc(_ovrHmd, ovrEye_Right, _eyeFov[1]); - //_renderTargetSize.w = Application::getInstance()->getGLWidget()->width(); - //_renderTargetSize.h = Application::getInstance()->getGLWidget()->height(); - ovrHmd_SetEnabledCaps(_ovrHmd, ovrHmdCap_LowPersistence | ovrHmdCap_LatencyTest); ovrHmd_StartSensor(_ovrHmd, ovrSensorCap_Orientation | ovrSensorCap_YawCorrection | @@ -260,7 +257,7 @@ void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenH } //Displays everything for the oculus, frame timing must be active -void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamera) { +void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &position, Camera& whichCamera) { #ifdef HAVE_LIBOVR //beginFrameTiming must be called before display if (!_frameTimingActive) { @@ -280,12 +277,19 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer ovrPosef eyeRenderPose[ovrEye_Count]; _camera->setTightness(0.0f); // In first person, camera follows (untweaked) head exactly without delay - _camera->setTargetPosition(whichCamera.getPosition()); - if (!Menu::getInstance()->isOptionChecked(MenuOption::AllowOculusCameraModeChange) || whichCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { - _camera->setDistance(0.0f); - } + _camera->setDistance(0.0f); _camera->setUpShift(0.0f); - _camera->setTightness(0.0f); // Cam + + glm::quat additionalRotation; + + // if (_camera->getMode() == CAMERA_MODE_THIRD_PERSON) { + //// _camera->setTargetPosition(_myAvatar->getUprightHeadPosition()); + //// _camera->setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); + + // } else if (_camera->getMode() == CAMERA_MODE_MIRROR) { + // _camera->setDistance(MIRROR_FULLSCREEN_DISTANCE); + // additionalRotation = glm::quat(glm::vec3(0.0f, PI, 0.0f)); + // } glMatrixMode(GL_PROJECTION); glPushMatrix(); @@ -294,6 +298,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer glPushMatrix(); glm::quat orientation; + glm::vec3 positionOffset; //Render each eye into an fbo for (int eyeIndex = 0; eyeIndex < ovrEye_Count; eyeIndex++) { @@ -306,7 +311,12 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer orientation.y = eyeRenderPose[eye].Orientation.y; orientation.z = eyeRenderPose[eye].Orientation.z; orientation.w = eyeRenderPose[eye].Orientation.w; - _camera->setTargetRotation(orientation * bodyOrientation); + + const float OCULUS_POSITION_SCALE = 0.2; + positionOffset = bodyOrientation * orientation * glm::vec3(eyeRenderPose[eye].Position.x, eyeRenderPose[eye].Position.y, eyeRenderPose[eye].Position.z) * OCULUS_POSITION_SCALE; + + _camera->setTargetRotation(additionalRotation * bodyOrientation * orientation); + _camera->setTargetPosition(position); _camera->update(1.0f / Application::getInstance()->getFps()); Matrix4f proj = ovrMatrix4f_Projection(_eyeRenderDesc[eye].Fov, whichCamera.getNearClip(), whichCamera.getFarClip(), true); @@ -315,8 +325,6 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer glMatrixMode(GL_PROJECTION); glLoadIdentity(); glLoadMatrixf((GLfloat *)proj.M); - - printf("%d %d\n", _renderTargetSize.w, _renderTargetSize.h); glViewport(_eyeRenderViewport[eye].Pos.x, _eyeRenderViewport[eye].Pos.y, _eyeRenderViewport[eye].Size.w, _eyeRenderViewport[eye].Size.h); @@ -337,36 +345,51 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer glPopMatrix(); + //Full texture viewport for glow effect + glViewport(0, 0, _renderTargetSize.w, _renderTargetSize.h); + + //Bind the output texture from the glow shader + // QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); + Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->release(); + glBindTexture(GL_TEXTURE_2D, Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->texture()); + // restore our normal viewport glViewport(0, 0, Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); - //Bind the output texture from the glow shader - QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); - glBindTexture(GL_TEXTURE_2D, fbo->texture()); - glMatrixMode(GL_PROJECTION); glPopMatrix(); + //Renders the distorted mesh onto the screen + renderDistortionMesh(eyeRenderPose); + + glBindTexture(GL_TEXTURE_2D, 0); + +#endif +} + +void OculusManager::renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]) { + glLoadIdentity(); gluOrtho2D(0, Application::getInstance()->getGLWidget()->width(), 0, Application::getInstance()->getGLWidget()->height()); + glDisable(GL_DEPTH_TEST); glDisable(GL_BLEND); _program.bind(); _program.setUniformValue(_textureLocation, 0); - + _program.enableAttributeArray(_positionAttributeLocation); _program.enableAttributeArray(_colorAttributeLocation); _program.enableAttributeArray(_texCoord0AttributeLocation); _program.enableAttributeArray(_texCoord1AttributeLocation); _program.enableAttributeArray(_texCoord2AttributeLocation); - + //Render the distortion meshes for each eye for (int eyeNum = 0; eyeNum < ovrEye_Count; eyeNum++) { GLfloat uvScale[2] = { _UVScaleOffset[eyeNum][0].x, _UVScaleOffset[eyeNum][0].y }; _program.setUniformValueArray(_eyeToSourceUVScaleLocation, uvScale, 1, 2); GLfloat uvOffset[2] = { _UVScaleOffset[eyeNum][1].x, _UVScaleOffset[eyeNum][1].y }; - _program.setUniformValueArray(_eyeToSourceUVOffsetLocation, uvOffset, 1, 2); + _program.setUniformValueArray(_eyeToSourceUVOffsetLocation, uvOffset, 1, 2); ovrMatrix4f timeWarpMatrices[2]; Matrix4f transposeMatrices[2]; @@ -376,19 +399,19 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer transposeMatrices[0].Transpose(); transposeMatrices[1].Transpose(); - + glUniformMatrix4fv(_eyeRotationStartLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[0].M); - + glUniformMatrix4fv(_eyeRotationEndLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[1].M); - + glBindBuffer(GL_ARRAY_BUFFER, _vertices[eyeNum]); - + glVertexAttribPointer(_positionAttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)0); glVertexAttribPointer(_texCoord0AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)8); glVertexAttribPointer(_texCoord1AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)16); glVertexAttribPointer(_texCoord2AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)24); glVertexAttribPointer(_colorAttributeLocation, 4, GL_UNSIGNED_BYTE, GL_TRUE, sizeof(DistortionVertex), (void *)32); - + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indices[eyeNum]); glDrawElements(GL_TRIANGLES, _meshSize[eyeNum], GL_UNSIGNED_SHORT, 0); } @@ -398,13 +421,11 @@ void OculusManager::display(const glm::quat &bodyOrientation, Camera& whichCamer _program.disableAttributeArray(_texCoord0AttributeLocation); _program.disableAttributeArray(_texCoord1AttributeLocation); _program.disableAttributeArray(_texCoord2AttributeLocation); - - glEnable(GL_BLEND); + + glEnable(GL_BLEND); glEnable(GL_DEPTH_TEST); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindTexture(GL_TEXTURE_2D, 0); _program.release(); -#endif + glBindBuffer(GL_ARRAY_BUFFER, 0); } //Tries to reconnect to the sensors diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index bd4f401efb..aa736c2eac 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -33,7 +33,7 @@ public: static void beginFrameTiming(); static void endFrameTiming(); static void configureCamera(Camera& camera, int screenWidth, int screenHeight); - static void display(const glm::quat &bodyOrientation, Camera& whichCamera); + static void display(const glm::quat &bodyOrientation, const glm::vec3 &position, Camera& whichCamera); static void reset(); /// param \yaw[out] yaw in radians @@ -44,6 +44,7 @@ public: private: static void generateDistortionMesh(); + static void renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]); struct DistortionVertex { glm::vec2 pos; diff --git a/interface/src/renderer/GlowEffect.cpp b/interface/src/renderer/GlowEffect.cpp index 262a632df0..c163136956 100644 --- a/interface/src/renderer/GlowEffect.cpp +++ b/interface/src/renderer/GlowEffect.cpp @@ -180,7 +180,7 @@ QOpenGLFramebufferObject* GlowEffect::render(bool toTexture) { glBindTexture(GL_TEXTURE_2D, oldDiffusedFBO->texture()); _diffuseProgram->bind(); - QSize size = Application::getInstance()->getGLWidget()->size(); + QSize size = primaryFBO->size(); _diffuseProgram->setUniformValue(_diffusionScaleLocation, 1.0f / size.width(), 1.0f / size.height()); renderFullscreenQuad(); From ffab329af486ee1e62ade19a124b03a19ba21399 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 11:39:33 -0700 Subject: [PATCH 45/67] Made camera line up with eyes in mirror mode --- interface/src/Application.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 906098ebe3..260abb5836 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -587,11 +587,9 @@ void Application::paintGL() { } else if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { _myCamera.setTightness(0.0f); - glm::vec3 eyePosition = _myAvatar->getHead()->calculateAverageEyePosition(); - float headHeight = eyePosition.y - _myAvatar->getPosition().y; _myCamera.setDistance(MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); - _myCamera.setTargetPosition(_myAvatar->getPosition() + glm::vec3(0, headHeight + (_raiseMirror * _myAvatar->getScale()), 0)); _myCamera.setTargetRotation(_myAvatar->getWorldAlignedOrientation() * glm::quat(glm::vec3(0.0f, PI + _rotateMirror, 0.0f))); + _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); } // Update camera position From ef57970510bc949dc6c431e4c51d2a4ea03376be Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 11:50:42 -0700 Subject: [PATCH 46/67] Comments and code cleanup --- interface/src/devices/OculusManager.cpp | 33 ++++++------------------- interface/src/devices/OculusManager.h | 1 + 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index b505f16c8b..19111f62c9 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -3,6 +3,7 @@ // interface/src/devices // // Created by Stephen Birarda on 5/9/13. +// Refactored by Ben Arnold on 6/30/2014 // Copyright 2012 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. @@ -153,6 +154,7 @@ void OculusManager::generateDistortionMesh() { return; } + //Viewport for the render target for each eye _eyeRenderViewport[0].Pos = Vector2i(0, 0); _eyeRenderViewport[0].Size = Sizei(_renderTargetSize.w / 2, _renderTargetSize.h); _eyeRenderViewport[1].Pos = Vector2i((_renderTargetSize.w + 1) / 2, 0); @@ -240,18 +242,7 @@ void OculusManager::endFrameTiming() { //Sets the camera FoV and aspect ratio void OculusManager::configureCamera(Camera& camera, int screenWidth, int screenHeight) { #ifdef HAVE_LIBOVR - ovrSizei recommendedTex0Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Left, - _eyeFov[0], 1.0f); - ovrSizei recommendedTex1Size = ovrHmd_GetFovTextureSize(_ovrHmd, ovrEye_Right, - _eyeFov[1], 1.0f); - - float width = recommendedTex0Size.w + recommendedTex1Size.w; - float height = recommendedTex0Size.h; - if (height < recommendedTex1Size.h) { - height = recommendedTex1Size.h; - } - - camera.setAspectRatio(width / height); + camera.setAspectRatio(_renderTargetSize.w / _renderTargetSize.h); camera.setFieldOfView(atan(_eyeFov[0].UpTan) * DEGREES_PER_RADIAN * 2.0f); #endif } @@ -280,17 +271,6 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p _camera->setDistance(0.0f); _camera->setUpShift(0.0f); - glm::quat additionalRotation; - - // if (_camera->getMode() == CAMERA_MODE_THIRD_PERSON) { - //// _camera->setTargetPosition(_myAvatar->getUprightHeadPosition()); - //// _camera->setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); - - // } else if (_camera->getMode() == CAMERA_MODE_MIRROR) { - // _camera->setDistance(MIRROR_FULLSCREEN_DISTANCE); - // additionalRotation = glm::quat(glm::vec3(0.0f, PI, 0.0f)); - // } - glMatrixMode(GL_PROJECTION); glPushMatrix(); @@ -315,7 +295,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p const float OCULUS_POSITION_SCALE = 0.2; positionOffset = bodyOrientation * orientation * glm::vec3(eyeRenderPose[eye].Position.x, eyeRenderPose[eye].Position.y, eyeRenderPose[eye].Position.z) * OCULUS_POSITION_SCALE; - _camera->setTargetRotation(additionalRotation * bodyOrientation * orientation); + _camera->setTargetRotation(bodyOrientation * orientation); _camera->setTargetPosition(position); _camera->update(1.0f / Application::getInstance()->getFps()); @@ -393,19 +373,21 @@ void OculusManager::renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]) { ovrMatrix4f timeWarpMatrices[2]; Matrix4f transposeMatrices[2]; + //Grabs the timewarp matrices to be used in the shader ovrHmd_GetEyeTimewarpMatrices(_ovrHmd, (ovrEyeType)eyeNum, eyeRenderPose[eyeNum], timeWarpMatrices); transposeMatrices[0] = Matrix4f(timeWarpMatrices[0]); transposeMatrices[1] = Matrix4f(timeWarpMatrices[1]); + //Have to transpose the matrices before using them transposeMatrices[0].Transpose(); transposeMatrices[1].Transpose(); glUniformMatrix4fv(_eyeRotationStartLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[0].M); - glUniformMatrix4fv(_eyeRotationEndLocation, 1, GL_FALSE, (GLfloat *)transposeMatrices[1].M); glBindBuffer(GL_ARRAY_BUFFER, _vertices[eyeNum]); + //Set vertex attribute pointers glVertexAttribPointer(_positionAttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)0); glVertexAttribPointer(_texCoord0AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)8); glVertexAttribPointer(_texCoord1AttributeLocation, 2, GL_FLOAT, GL_FALSE, sizeof(DistortionVertex), (void *)16); @@ -449,6 +431,7 @@ void OculusManager::getEulerAngles(float& yaw, float& pitch, float& roll) { #endif } +//Used to set the size of the glow framebuffers QSize OculusManager::getRenderTargetSize() { #ifdef HAVE_LIBOVR QSize rv; diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index aa736c2eac..506d1c3e92 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -3,6 +3,7 @@ // interface/src/devices // // Created by Stephen Birarda on 5/9/13. +// Refactored by Ben Arnold on 6/30/2014 // Copyright 2012 High Fidelity, Inc. // // Distributed under the Apache License, Version 2.0. From d2f86278b2fb3a8457d4372c0a2cf30350d4ecfb Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 12:31:58 -0700 Subject: [PATCH 47/67] modified AudioStreamStats to per-stream with seq stats added AudioStreamStats info to interface overlay stats --- assignment-client/src/audio/AudioMixer.cpp | 4 +- .../src/audio/AudioMixerClientData.cpp | 91 +++++++++++++------ .../src/audio/AudioMixerClientData.h | 4 +- interface/src/Audio.cpp | 28 +++++- interface/src/Audio.h | 6 +- interface/src/ui/Stats.cpp | 39 +++++--- libraries/audio/src/AudioStreamStats.h | 28 +++++- 7 files changed, 144 insertions(+), 56 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bc5c2f1a06..6095bd1712 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -591,7 +591,6 @@ void AudioMixer::run() { const quint64 TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS = 1 * USECS_PER_SECOND; - char audioStreamStatsPacket[MAX_PACKET_SIZE]; bool sendAudioStreamStats = false; quint64 now = usecTimestampNow(); if (now - _lastSendAudioStreamStatsTime > TOO_LONG_SINCE_LAST_SEND_AUDIO_STREAM_STATS) { @@ -626,8 +625,7 @@ void AudioMixer::run() { // send an audio stream stats packet if it's time if (sendAudioStreamStats) { - int numBytesAudioStreamStatsPacket = nodeData->encodeAudioStreamStatsPacket(audioStreamStatsPacket); - nodeList->writeDatagram(audioStreamStatsPacket, numBytesAudioStreamStatsPacket, node); + nodeData->sendAudioStreamStatsPackets(node); } ++_sumListeners; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index ea13289186..b58c5f90b5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -156,41 +156,76 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } } -void AudioMixerClientData::getAudioStreamStats(AudioStreamStats& stats) const { - int avatarJitterBufferFrames = 0; - int maxJitterBufferFrames = 0; - int sumJitterBufferFrames = 0; +void AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const { + const SequenceNumberStats* streamSequenceNumberStats = &_incomingAvatarAudioSequenceNumberStats; - for (int i = 0; i < _ringBuffers.size(); i++) { - - int bufferJitterFrames = _ringBuffers[i]->getCurrentJitterBufferFrames(); - if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Microphone) { - avatarJitterBufferFrames = bufferJitterFrames; - } - - if (bufferJitterFrames > maxJitterBufferFrames) { - maxJitterBufferFrames = bufferJitterFrames; - } - - sumJitterBufferFrames += bufferJitterFrames; + stats._streamType = ringBuffer->getType(); + if (stats._streamType == PositionalAudioRingBuffer::Injector) { + stats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); + streamSequenceNumberStats = &_incomingInjectedAudioSequenceNumberStatsMap.value(stats._streamIdentifier); } - - stats._avatarJitterBufferFrames = avatarJitterBufferFrames; - stats._maxJitterBufferFrames = maxJitterBufferFrames; - stats._avgJitterBufferFrames = (float)sumJitterBufferFrames / (float)_ringBuffers.size(); + stats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); + + stats._packetsReceived = streamSequenceNumberStats->getNumReceived(); + stats._packetsUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); + stats._packetsEarly = streamSequenceNumberStats->getNumEarly(); + stats._packetsLate = streamSequenceNumberStats->getNumLate(); + stats._packetsLost = streamSequenceNumberStats->getNumLost(); + stats._packetsRecovered = streamSequenceNumberStats->getNumRecovered(); + stats._packetsDuplicate = streamSequenceNumberStats->getNumDuplicate(); } -int AudioMixerClientData::encodeAudioStreamStatsPacket(char* packet) const { +void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const { + + char packet[MAX_PACKET_SIZE]; + AudioStreamStats streamStats; + + NodeList* nodeList = NodeList::getInstance(); + + // The append flag is a boolean value that will be packed right after the header. The first packet sent + // inside this method will have 0 for this flag, while every subsequent packet will have 1 for this flag. + // The sole purpose of this flag is so the client can clear its map of injected audio stream stats when + // it receives a packet with an appendFlag of 0. This prevents the buildup of dead audio stream stats in the client. + quint8 appendFlag = 0; + + // pack header int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeAudioStreamStats); - char* dataAt = packet + numBytesPacketHeader; + char* headerEndAt = packet + numBytesPacketHeader; - // pack jitter buffer stats - AudioStreamStats stats; - getAudioStreamStats(stats); - memcpy(dataAt, &stats, sizeof(AudioStreamStats)); - dataAt += sizeof(AudioStreamStats); + // calculate how many stream stat structs we can fit in each packet + const int numStreamStatsRoomFor = (MAX_PACKET_SIZE - numBytesPacketHeader - sizeof(quint8) - sizeof(quint16)) / sizeof(AudioStreamStats); - return dataAt - packet; + // pack and send stream stats packets until all ring buffers' stats are sent + int numStreamStatsRemaining = _ringBuffers.size(); + QList::ConstIterator ringBuffersIterator = _ringBuffers.constBegin(); + while (numStreamStatsRemaining > 0) { + + char* dataAt = headerEndAt; + + // pack the append flag + memcpy(dataAt, &appendFlag, sizeof(quint8)); + appendFlag = 1; + dataAt += sizeof(quint8); + + // calculate and pack the number of stream stats to follow + quint16 numStreamStatsToPack = std::min(numStreamStatsRemaining, numStreamStatsRoomFor); + memcpy(dataAt, &numStreamStatsToPack, sizeof(quint16)); + dataAt += sizeof(quint16); + + // pack the calculated number of stream stats + for (int i = 0; i < numStreamStatsToPack; i++) { + getAudioStreamStatsOfStream(*ringBuffersIterator, streamStats); + + memcpy(dataAt, &streamStats, sizeof(AudioStreamStats)); + dataAt += sizeof(AudioStreamStats); + + ringBuffersIterator++; + } + numStreamStatsRemaining -= numStreamStatsToPack; + + // send the current packet + nodeList->writeDatagram(packet, dataAt - packet, destinationNode); + } } QString AudioMixerClientData::getJitterBufferStatsString() const { diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index d9d6f45a0e..4e4bf41318 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -32,9 +32,9 @@ public: void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); - void getAudioStreamStats(AudioStreamStats& stats) const; + void getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const; - int encodeAudioStreamStatsPacket(char* packet) const; + void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const; QString getJitterBufferStatsString() const; diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 5c1bc96961..c52e2128b0 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -103,7 +103,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) : _scopeInput(0), _scopeOutputLeft(0), _scopeOutputRight(0), - _audioMixerStreamStats(), + _audioMixerAvatarStreamStats(), _outgoingAvatarAudioSequenceNumber(0) { // clear the array of locally injected samples @@ -720,9 +720,29 @@ void Audio::parseAudioStreamStatsPacket(const QByteArray& packet) { int numBytesPacketHeader = numBytesForPacketHeader(packet); const char* dataAt = packet.constData() + numBytesPacketHeader; - // parse audio mixer jitter buffer stats - memcpy(&_audioMixerStreamStats, dataAt, sizeof(AudioStreamStats)); - dataAt += sizeof(AudioStreamStats); + // parse the appendFlag, clear injected audio stream stats if 0 + quint8 appendFlag = *(reinterpret_cast(dataAt)); + dataAt += sizeof(quint8); + if (!appendFlag) { + _audioMixerInjectedStreamStatsMap.clear(); + } + + // parse the number of stream stats structs to follow + quint16 numStreamStats = *(reinterpret_cast(dataAt)); + dataAt += sizeof(quint16); + + // parse the stream stats + AudioStreamStats streamStats; + for (quint16 i = 0; i < numStreamStats; i++) { + memcpy(&streamStats, dataAt, sizeof(AudioStreamStats)); + dataAt += sizeof(AudioStreamStats); + + if (streamStats._streamType == PositionalAudioRingBuffer::Microphone) { + _audioMixerAvatarStreamStats = streamStats; + } else { + _audioMixerInjectedStreamStatsMap[streamStats._streamIdentifier] = streamStats; + } + } } // NOTE: numSamples is the total number of single channel samples, since callers will always call this with stereo diff --git a/interface/src/Audio.h b/interface/src/Audio.h index f3b01161d1..62c524ba2c 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -104,7 +104,8 @@ public slots: float getInputVolume() const { return (_audioInput) ? _audioInput->volume() : 0.0f; } void setInputVolume(float volume) { if (_audioInput) _audioInput->setVolume(volume); } - const AudioStreamStats& getAudioMixerStreamStats() const { return _audioMixerStreamStats; } + const AudioStreamStats& getAudioMixerAvatarStreamStats() const { return _audioMixerAvatarStreamStats; } + const QHash& getAudioMixerInjectedStreamStatsMap() const { return _audioMixerInjectedStreamStatsMap; } signals: bool muteToggled(); @@ -237,7 +238,8 @@ private: QByteArray* _scopeOutputLeft; QByteArray* _scopeOutputRight; - AudioStreamStats _audioMixerStreamStats; + AudioStreamStats _audioMixerAvatarStreamStats; + QHash _audioMixerInjectedStreamStatsMap; quint16 _outgoingAvatarAudioSequenceNumber; SequenceNumberStats _incomingMixedAudioSequenceNumberStats; diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 9697f5af03..c97a8ceca1 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -286,11 +286,16 @@ void Stats::display( pingVoxel = totalPingVoxel/voxelServerCount; } - lines = _expanded ? 6 : 3; + + Audio* audio = Application::getInstance()->getAudio(); + const AudioStreamStats& audioMixerAvatarStreamStats = audio->getAudioMixerAvatarStreamStats(); + const QHash& audioMixerInjectedStreamStatsMap = audio->getAudioMixerInjectedStreamStatsMap(); + + lines = _expanded ? 7 + audioMixerInjectedStreamStatsMap.size(): 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; - Audio* audio = Application::getInstance()->getAudio(); + char audioJitter[30]; sprintf(audioJitter, @@ -322,21 +327,29 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); - - static const float MSECS_PER_FRAME = (float)NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL * (float)MSECS_PER_SECOND / (float)SAMPLE_RATE; - - const AudioStreamStats& audioMixerStreamStats = - Application::getInstance()->getAudio()->getAudioMixerStreamStats(); - - char* audioMixerJitterBuffersStatsLabel = "AudioMixer j-buffers msecs:"; - char audioMixerJitterBuffersStats[30]; - sprintf(audioMixerJitterBuffersStats, "mic/max/avg: %.1f / %.1f / %.1f", audioMixerStreamStats._avatarJitterBufferFrames * MSECS_PER_FRAME, - audioMixerStreamStats._maxJitterBufferFrames * MSECS_PER_FRAME, audioMixerStreamStats._avgJitterBufferFrames * MSECS_PER_FRAME); - + char* audioMixerJitterBuffersStatsLabel = "AudioMixer stream stats:"; + char* audioMixerJitterBuffersStatsLabel2 = "early/late/lost, jframes"; + verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel2, color); + + char audioMixerJitterBuffersStats[30]; + sprintf(audioMixerJitterBuffersStats, "mic: %d/%d/%d, %d", audioMixerAvatarStreamStats._packetsEarly, + audioMixerAvatarStreamStats._packetsLate, audioMixerAvatarStreamStats._packetsLost, + audioMixerAvatarStreamStats._jitterBufferFrames); + verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + + foreach(AudioStreamStats injectedStreamStats, audioMixerInjectedStreamStatsMap) { + sprintf(audioMixerJitterBuffersStats, "inj: %d/%d/%d, %d", injectedStreamStats._packetsEarly, + injectedStreamStats._packetsLate, injectedStreamStats._packetsLost, injectedStreamStats._jitterBufferFrames); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + } } verticalOffset = 0; diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 5746462bcf..004d697fcf 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -12,15 +12,35 @@ #ifndef hifi_AudioStreamStats_h #define hifi_AudioStreamStats_h +#include "PositionalAudioRingBuffer.h" + class AudioStreamStats { public: AudioStreamStats() - : _avatarJitterBufferFrames(0), _maxJitterBufferFrames(0), _avgJitterBufferFrames(0) + : _streamType(PositionalAudioRingBuffer::Microphone), + _streamIdentifier(), + _jitterBufferFrames(0), + _packetsReceived(0), + _packetsUnreasonable(0), + _packetsEarly(0), + _packetsLate(0), + _packetsLost(0), + _packetsRecovered(0), + _packetsDuplicate(0) {} - quint16 _avatarJitterBufferFrames; - quint16 _maxJitterBufferFrames; - float _avgJitterBufferFrames; + PositionalAudioRingBuffer::Type _streamType; + QUuid _streamIdentifier; + + quint16 _jitterBufferFrames; + + quint32 _packetsReceived; + quint32 _packetsUnreasonable; + quint32 _packetsEarly; + quint32 _packetsLate; + quint32 _packetsLost; + quint32 _packetsRecovered; + quint32 _packetsDuplicate; }; #endif // hifi_AudioStreamStats_h From 73a1e759a4983bf714c40cacbe74d727ddd03cf0 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 30 Jun 2014 12:35:34 -0700 Subject: [PATCH 48/67] Include an additional bit with the objects that says whether what follows is a delta. The clients may be retaining references to the objects, so we can't rely on their having been deleted. --- libraries/metavoxels/src/Bitstream.cpp | 14 ++++++++++++-- tests/metavoxels/src/MetavoxelTests.cpp | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 0ef86e85c0..d18903f923 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1414,8 +1414,10 @@ Bitstream& Bitstream::operator<(const SharedObjectPointer& object) { *this << object->getOriginID(); QPointer reference = _sharedObjectReferences.value(object->getOriginID()); if (reference) { + *this << true; writeRawDelta((const QObject*)object.data(), (const QObject*)reference.data()); } else { + *this << false; *this << (QObject*)object.data(); } return *this; @@ -1430,19 +1432,27 @@ Bitstream& Bitstream::operator>(SharedObjectPointer& object) { } int originID; *this >> originID; + bool delta; + *this >> delta; QPointer reference = _sharedObjectReferences.value(originID); QPointer& pointer = _weakSharedObjectHash[id]; if (pointer) { ObjectStreamerPointer objectStreamer; _objectStreamerStreamer >> objectStreamer; - if (reference) { + if (delta) { + if (!reference) { + qWarning() << "Delta without reference" << id << originID; + } objectStreamer->readRawDelta(*this, reference.data(), pointer.data()); } else { objectStreamer->read(*this, pointer.data()); } } else { QObject* rawObject; - if (reference) { + if (delta) { + if (!reference) { + qWarning() << "Delta without reference" << id << originID; + } readRawDelta(rawObject, (const QObject*)reference.data()); } else { *this >> rawObject; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 7b7d0430d0..61ab664310 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -652,7 +652,7 @@ TestEndpoint::TestEndpoint(Mode mode) : SLOT(handleHighPriorityMessage(const QVariant&))); if (mode == METAVOXEL_CLIENT_MODE) { - _lod = MetavoxelLOD(glm::vec3(), 0.001f); + _lod = MetavoxelLOD(glm::vec3(), 0.01f); return; } if (mode == METAVOXEL_SERVER_MODE) { From 5a2d2bba8ff7ec4412dc74043ff93825aa0fefd7 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 12:42:43 -0700 Subject: [PATCH 49/67] Updated readme.txt --- interface/external/oculus/readme.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/interface/external/oculus/readme.txt b/interface/external/oculus/readme.txt index 002047b73f..f689f81478 100644 --- a/interface/external/oculus/readme.txt +++ b/interface/external/oculus/readme.txt @@ -2,18 +2,12 @@ Instructions for adding the Oculus library (LibOVR) to Interface Stephen Birarda, March 6, 2014 -You can download the Oculus SDK from https://developer.oculusvr.com/ (account creation required). Interface has been tested with SDK version 0.2.5. +You can download the Oculus SDK from https://developer.oculusvr.com/ (account creation required). Interface has been tested with SDK version 0.3.2. 1. Copy the Oculus SDK folders from the LibOVR directory (Lib, Include, Src) into the interface/externals/oculus folder. This readme.txt should be there as well. You may optionally choose to copy the SDK folders to a location outside the repository (so you can re-use with different checkouts and different projects). If so our CMake find module expects you to set the ENV variable 'HIFI_LIB_DIR' to a directory containing a subfolder 'oculus' that contains the three folders mentioned above. - - NOTE: On OS X there is a linker error with version 0.2.5c of the Oculus SDK. - It must be re-built (from the included LibOVR_With_Samples.xcodeproj) with RRTI support. - In XCode Build Settings for the ovr target, set "Enable C++ Runtime Types" to yes. - Then, Archive and use the organizer to save a copy of the built products. - In the exported directory you will have a new libovr.a to copy into the oculus directory from above. 2. Clear your build directory, run cmake and build, and you should be all set. \ No newline at end of file From dbb370aecfc68a5f5c1fb246befa56657fdd3444 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 12:49:54 -0700 Subject: [PATCH 50/67] added early/late/lost to domain page stats --- assignment-client/src/audio/AudioMixer.cpp | 2 +- .../src/audio/AudioMixerClientData.cpp | 20 +++++++++++-------- .../src/audio/AudioMixerClientData.h | 5 +---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 6095bd1712..6746526d9f 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -449,7 +449,7 @@ void AudioMixer::sendStatsPacket() { AudioMixerClientData* clientData = static_cast(node->getLinkedData()); if (clientData) { QString property = "jitterStats." + node->getUUID().toString(); - QString value = clientData->getJitterBufferStatsString(); + QString value = clientData->getAudioStreamStatsString(); statsObject2[qPrintable(property)] = value; somethingToSend = true; sizeOfStats += property.size() + value.size(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b58c5f90b5..be2d359bdc 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -228,7 +228,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& } } -QString AudioMixerClientData::getJitterBufferStatsString() const { +QString AudioMixerClientData::getAudioStreamStatsString() const { QString result; AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer(); if (avatarRingBuffer) { @@ -247,7 +247,7 @@ QString AudioMixerClientData::getJitterBufferStatsString() const { } else { result = "mic unknown"; } - + AudioStreamStats streamStats; for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); @@ -256,12 +256,16 @@ QString AudioMixerClientData::getJitterBufferStatsString() const { int resetCount = _ringBuffers[i]->getResetCount(); int samplesAvailable = _ringBuffers[i]->samplesAvailable(); int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); - result += "| injected["+QString::number(i)+"].desired:" + QString::number(desiredJitterBuffer) - + " calculated:" + QString::number(calculatedJitterBuffer) - + " current:" + QString::number(currentJitterBuffer) - + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable) - + " resets:" + QString::number(resetCount); + getAudioStreamStatsOfStream(_ringBuffers[i], streamStats); + result += "| injected[" + QString::number(i) + "].desired:" + QString::number(desiredJitterBuffer) + + " calculated:" + QString::number(calculatedJitterBuffer) + + " current:" + QString::number(currentJitterBuffer) + + " available:" + QString::number(framesAvailable) + + " samples:" + QString::number(samplesAvailable) + + " resets:" + QString::number(resetCount) + + " early:" + QString::number(streamStats._packetsEarly) + + " late:" + QString::number(streamStats._packetsLate) + + " lost:" + QString::number(streamStats._packetsLost); } } return result; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 4e4bf41318..6d4c9074bd 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -33,15 +33,12 @@ public: void pushBuffersAfterFrameSend(); void getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const; + QString getAudioStreamStatsString() const; void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const; - QString getJitterBufferStatsString() const; - void incrementOutgoingMixedAudioSequenceNumber() { _outgoingMixedAudioSequenceNumber++; } - quint16 getOutgoingSequenceNumber() const { return _outgoingMixedAudioSequenceNumber; } - //const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; } private: QList _ringBuffers; From 0711eb063eb4e7cbe7667744b8e136488fab6c23 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 13:07:46 -0700 Subject: [PATCH 51/67] fix reference to temp object error? --- .../src/audio/AudioMixerClientData.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index be2d359bdc..b2a92efb25 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -157,22 +157,23 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } void AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const { - const SequenceNumberStats* streamSequenceNumberStats = &_incomingAvatarAudioSequenceNumberStats; + const SequenceNumberStats& streamSequenceNumberStats = stats._streamType == PositionalAudioRingBuffer::Injector + ? _incomingAvatarAudioSequenceNumberStats + : _incomingInjectedAudioSequenceNumberStatsMap[stats._streamIdentifier]; stats._streamType = ringBuffer->getType(); if (stats._streamType == PositionalAudioRingBuffer::Injector) { stats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); - streamSequenceNumberStats = &_incomingInjectedAudioSequenceNumberStatsMap.value(stats._streamIdentifier); } stats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); - stats._packetsReceived = streamSequenceNumberStats->getNumReceived(); - stats._packetsUnreasonable = streamSequenceNumberStats->getNumUnreasonable(); - stats._packetsEarly = streamSequenceNumberStats->getNumEarly(); - stats._packetsLate = streamSequenceNumberStats->getNumLate(); - stats._packetsLost = streamSequenceNumberStats->getNumLost(); - stats._packetsRecovered = streamSequenceNumberStats->getNumRecovered(); - stats._packetsDuplicate = streamSequenceNumberStats->getNumDuplicate(); + stats._packetsReceived = streamSequenceNumberStats.getNumReceived(); + stats._packetsUnreasonable = streamSequenceNumberStats.getNumUnreasonable(); + stats._packetsEarly = streamSequenceNumberStats.getNumEarly(); + stats._packetsLate = streamSequenceNumberStats.getNumLate(); + stats._packetsLost = streamSequenceNumberStats.getNumLost(); + stats._packetsRecovered = streamSequenceNumberStats.getNumRecovered(); + stats._packetsDuplicate = streamSequenceNumberStats.getNumDuplicate(); } void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const { From c4a85736e7998c41852b6a029671a330d017aa69 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 13:51:43 -0700 Subject: [PATCH 52/67] removed unnecessary changes from ScriptEngine --- libraries/script-engine/src/ScriptEngine.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 1b225b3699..34f22af234 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -460,8 +460,8 @@ void ScriptEngine::run() { } QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame - ? PacketTypeSilentAudioFrame - : PacketTypeMicrophoneAudioNoEcho); + ? PacketTypeSilentAudioFrame + : PacketTypeMicrophoneAudioNoEcho); QDataStream packetStream(&audioPacket, QIODevice::Append); @@ -485,7 +485,7 @@ void ScriptEngine::run() { } else if (nextSoundOutput) { // write the raw audio data packetStream.writeRawData(reinterpret_cast(nextSoundOutput), - numAvailableSamples * sizeof(int16_t)); + numAvailableSamples * sizeof(int16_t)); } // write audio packet to AudioMixer nodes From 6b6dccda2f0771037c6f73e0f6a8c527b965e4b8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 14:21:38 -0700 Subject: [PATCH 53/67] _resetCount to _overflowCount in AudioRingBuffer --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 ++++---- libraries/audio/src/AudioRingBuffer.cpp | 2 +- libraries/audio/src/AudioRingBuffer.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b2a92efb25..2f377b3d0f 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -236,7 +236,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { int desiredJitterBuffer = avatarRingBuffer->getDesiredJitterBufferFrames(); int calculatedJitterBuffer = avatarRingBuffer->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = avatarRingBuffer->getCurrentJitterBufferFrames(); - int resetCount = avatarRingBuffer->getResetCount(); + int overflowCount = avatarRingBuffer->getOverflowCount(); int samplesAvailable = avatarRingBuffer->samplesAvailable(); int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); result += "mic.desired:" + QString::number(desiredJitterBuffer) @@ -244,7 +244,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) + " samples:" + QString::number(samplesAvailable) - + " resets:" + QString::number(resetCount); + + " overflows:" + QString::number(overflowCount); } else { result = "mic unknown"; } @@ -254,7 +254,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); int calculatedJitterBuffer = _ringBuffers[i]->getCalculatedDesiredJitterBufferFrames(); int currentJitterBuffer = _ringBuffers[i]->getCurrentJitterBufferFrames(); - int resetCount = _ringBuffers[i]->getResetCount(); + int overflowCount = _ringBuffers[i]->getOverflowCount(); int samplesAvailable = _ringBuffers[i]->samplesAvailable(); int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); getAudioStreamStatsOfStream(_ringBuffers[i], streamStats); @@ -263,7 +263,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) + " samples:" + QString::number(samplesAvailable) - + " resets:" + QString::number(resetCount) + + " overflows:" + QString::number(overflowCount) + " early:" + QString::number(streamStats._packetsEarly) + " late:" + QString::number(streamStats._packetsLate) + " lost:" + QString::number(streamStats._packetsLost); diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 07b48fe88d..11219771cf 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -21,7 +21,7 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), - _resetCount(0), + _overflowCount(0), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), _numFrameSamples(numFrameSamples), _isStarved(true), diff --git a/libraries/audio/src/AudioRingBuffer.h b/libraries/audio/src/AudioRingBuffer.h index b290c96d79..8d19f9c0bb 100644 --- a/libraries/audio/src/AudioRingBuffer.h +++ b/libraries/audio/src/AudioRingBuffer.h @@ -71,7 +71,7 @@ public: bool isStarved() const { return _isStarved; } void setIsStarved(bool isStarved) { _isStarved = isStarved; } - int getResetCount() const { return _resetCount; } /// how many times has the ring buffer written past the end and reset + int getOverflowCount() const { return _overflowCount; } /// how many times has the ring buffer has overwritten old data bool hasStarted() const { return _hasStarted; } int addSilentFrame(int numSilentSamples); @@ -82,7 +82,7 @@ protected: int16_t* shiftedPositionAccomodatingWrap(int16_t* position, int numSamplesShift) const; - int _resetCount; /// how many times has the ring buffer written past the end and done a reset + int _overflowCount; /// how many times has the ring buffer has overwritten old data int _sampleCapacity; bool _isFull; From 1bfe3670204a0435022fb74e6eeb75a910c65197 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 15:06:04 -0700 Subject: [PATCH 54/67] fixed error with retrieving stream stats in AudioMixerClientData --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 2f377b3d0f..134096735d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -157,13 +157,15 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } void AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const { - const SequenceNumberStats& streamSequenceNumberStats = stats._streamType == PositionalAudioRingBuffer::Injector - ? _incomingAvatarAudioSequenceNumberStats - : _incomingInjectedAudioSequenceNumberStatsMap[stats._streamIdentifier]; + + SequenceNumberStats streamSequenceNumberStats; stats._streamType = ringBuffer->getType(); if (stats._streamType == PositionalAudioRingBuffer::Injector) { stats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); + streamSequenceNumberStats = _incomingInjectedAudioSequenceNumberStatsMap.value(stats._streamIdentifier); + } else { + streamSequenceNumberStats = _incomingAvatarAudioSequenceNumberStats; } stats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); From e51eb34f05e735b035aa35edc10f7f5420d678fe Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 15:14:21 -0700 Subject: [PATCH 55/67] Fixed build errors when libovr is not present --- interface/src/devices/OculusManager.cpp | 14 ++++++++++---- interface/src/devices/OculusManager.h | 5 ++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 19111f62c9..038b7ea8f5 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -20,6 +20,8 @@ #include "Application.h" +#ifdef HAVE_LIBOVR + using namespace OVR; ProgramObject OculusManager::_program; @@ -35,8 +37,6 @@ int OculusManager::_texCoord1AttributeLocation; int OculusManager::_texCoord2AttributeLocation; bool OculusManager::_isConnected = false; -#ifdef HAVE_LIBOVR - ovrHmd OculusManager::_ovrHmd; ovrHmdDesc OculusManager::_ovrHmdDesc; ovrFovPort OculusManager::_eyeFov[ovrEye_Count]; @@ -127,6 +127,7 @@ void OculusManager::connect() { //Disconnects and deallocates the OR void OculusManager::disconnect() { +#ifdef HAVE_LIBOVR if (_isConnected) { _isConnected = false; ovrHmd_Destroy(_ovrHmd); @@ -144,10 +145,12 @@ void OculusManager::disconnect() { } } } +#endif } -void OculusManager::generateDistortionMesh() { #ifdef HAVE_LIBOVR +void OculusManager::generateDistortionMesh() { + //Check if we already have the distortion mesh if (_vertices[0] != 0) { printf("WARNING: Tried to generate Oculus distortion mesh twice without freeing the VBOs."); @@ -206,8 +209,9 @@ void OculusManager::generateDistortionMesh() { OVR_FREE(pVBVerts); ovrHmd_DestroyDistortionMesh(&meshData); } -#endif + } +#endif bool OculusManager::isConnected() { #ifdef HAVE_LIBOVR @@ -347,6 +351,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p #endif } +#ifdef HAVE_LIBOVR void OculusManager::renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]) { glLoadIdentity(); @@ -409,6 +414,7 @@ void OculusManager::renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]) { _program.release(); glBindBuffer(GL_ARRAY_BUFFER, 0); } +#endif //Tries to reconnect to the sensors void OculusManager::reset() { diff --git a/interface/src/devices/OculusManager.h b/interface/src/devices/OculusManager.h index 506d1c3e92..7798875c2c 100644 --- a/interface/src/devices/OculusManager.h +++ b/interface/src/devices/OculusManager.h @@ -15,9 +15,8 @@ #ifdef HAVE_LIBOVR #include -#endif - #include "../src/Util/Util_Render_Stereo.h" +#endif #include "renderer/ProgramObject.h" @@ -44,6 +43,7 @@ public: static QSize getRenderTargetSize(); private: +#ifdef HAVE_LIBOVR static void generateDistortionMesh(); static void renderDistortionMesh(ovrPosef eyeRenderPose[ovrEye_Count]); @@ -76,7 +76,6 @@ private: static bool _isConnected; -#ifdef HAVE_LIBOVR static ovrHmd _ovrHmd; static ovrHmdDesc _ovrHmdDesc; static ovrFovPort _eyeFov[ovrEye_Count]; From ae2780dcabab5202fbd2bd74e72356c70a7f0f55 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 15:20:42 -0700 Subject: [PATCH 56/67] changed getAudioStreamStatsOfStream to return stats instead of taking pointer --- .../src/audio/AudioMixerClientData.cpp | 39 +++++++++---------- .../src/audio/AudioMixerClientData.h | 2 +- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 134096735d..0065bff959 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -156,33 +156,33 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() { } } -void AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const { - +AudioStreamStats AudioMixerClientData::getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const { + AudioStreamStats streamStats; SequenceNumberStats streamSequenceNumberStats; - stats._streamType = ringBuffer->getType(); - if (stats._streamType == PositionalAudioRingBuffer::Injector) { - stats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); - streamSequenceNumberStats = _incomingInjectedAudioSequenceNumberStatsMap.value(stats._streamIdentifier); + streamStats._streamType = ringBuffer->getType(); + if (streamStats._streamType == PositionalAudioRingBuffer::Injector) { + streamStats._streamIdentifier = ((InjectedAudioRingBuffer*)ringBuffer)->getStreamIdentifier(); + streamSequenceNumberStats = _incomingInjectedAudioSequenceNumberStatsMap.value(streamStats._streamIdentifier); } else { streamSequenceNumberStats = _incomingAvatarAudioSequenceNumberStats; } - stats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); + streamStats._jitterBufferFrames = ringBuffer->getCurrentJitterBufferFrames(); - stats._packetsReceived = streamSequenceNumberStats.getNumReceived(); - stats._packetsUnreasonable = streamSequenceNumberStats.getNumUnreasonable(); - stats._packetsEarly = streamSequenceNumberStats.getNumEarly(); - stats._packetsLate = streamSequenceNumberStats.getNumLate(); - stats._packetsLost = streamSequenceNumberStats.getNumLost(); - stats._packetsRecovered = streamSequenceNumberStats.getNumRecovered(); - stats._packetsDuplicate = streamSequenceNumberStats.getNumDuplicate(); + streamStats._packetsReceived = streamSequenceNumberStats.getNumReceived(); + streamStats._packetsUnreasonable = streamSequenceNumberStats.getNumUnreasonable(); + streamStats._packetsEarly = streamSequenceNumberStats.getNumEarly(); + streamStats._packetsLate = streamSequenceNumberStats.getNumLate(); + streamStats._packetsLost = streamSequenceNumberStats.getNumLost(); + streamStats._packetsRecovered = streamSequenceNumberStats.getNumRecovered(); + streamStats._packetsDuplicate = streamSequenceNumberStats.getNumDuplicate(); + + return streamStats; } void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const { char packet[MAX_PACKET_SIZE]; - AudioStreamStats streamStats; - NodeList* nodeList = NodeList::getInstance(); // The append flag is a boolean value that will be packed right after the header. The first packet sent @@ -217,8 +217,7 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // pack the calculated number of stream stats for (int i = 0; i < numStreamStatsToPack; i++) { - getAudioStreamStatsOfStream(*ringBuffersIterator, streamStats); - + AudioStreamStats streamStats = getAudioStreamStatsOfStream(*ringBuffersIterator); memcpy(dataAt, &streamStats, sizeof(AudioStreamStats)); dataAt += sizeof(AudioStreamStats); @@ -250,7 +249,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { } else { result = "mic unknown"; } - AudioStreamStats streamStats; + for (int i = 0; i < _ringBuffers.size(); i++) { if (_ringBuffers[i]->getType() == PositionalAudioRingBuffer::Injector) { int desiredJitterBuffer = _ringBuffers[i]->getDesiredJitterBufferFrames(); @@ -259,7 +258,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { int overflowCount = _ringBuffers[i]->getOverflowCount(); int samplesAvailable = _ringBuffers[i]->samplesAvailable(); int framesAvailable = (samplesAvailable / _ringBuffers[i]->getSamplesPerFrame()); - getAudioStreamStatsOfStream(_ringBuffers[i], streamStats); + AudioStreamStats streamStats = getAudioStreamStatsOfStream(_ringBuffers[i]); result += "| injected[" + QString::number(i) + "].desired:" + QString::number(desiredJitterBuffer) + " calculated:" + QString::number(calculatedJitterBuffer) + " current:" + QString::number(currentJitterBuffer) diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 6d4c9074bd..65fd4b3da3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -32,7 +32,7 @@ public: void checkBuffersBeforeFrameSend(AABox* checkSourceZone = NULL, AABox* listenerZone = NULL); void pushBuffersAfterFrameSend(); - void getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer, AudioStreamStats& stats) const; + AudioStreamStats getAudioStreamStatsOfStream(const PositionalAudioRingBuffer* ringBuffer) const; QString getAudioStreamStatsString() const; void sendAudioStreamStatsPackets(const SharedNodePointer& destinationNode) const; From 1b0ce510f614b1480a424d6e6cd08baecd7cb994 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 15:33:31 -0700 Subject: [PATCH 57/67] fixed a typo, tidied up more minor details --- libraries/audio/src/AudioInjector.cpp | 1 - libraries/audio/src/AudioRingBuffer.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 2dd8680697..e5c1230832 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -86,7 +86,6 @@ void AudioInjector::injectAudio() { quint8 volume = MAX_INJECTOR_VOLUME * _options.getVolume(); packetStream << volume; - QElapsedTimer timer; timer.start(); int nextFrame = 0; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 11219771cf..6ae3b19541 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -23,11 +23,11 @@ AudioRingBuffer::AudioRingBuffer(int numFrameSamples, bool randomAccessMode) : NodeData(), _overflowCount(0), _sampleCapacity(numFrameSamples * RING_BUFFER_LENGTH_FRAMES), + _isFull(false), _numFrameSamples(numFrameSamples), _isStarved(true), _hasStarted(false), - _randomAccessMode(randomAccessMode), - _isFull(false) + _randomAccessMode(randomAccessMode) { if (numFrameSamples) { _buffer = new int16_t[_sampleCapacity]; @@ -56,7 +56,6 @@ void AudioRingBuffer::reset() { void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) { delete[] _buffer; _sampleCapacity = numFrameSamples * RING_BUFFER_LENGTH_FRAMES; - _sampleCapacity = _sampleCapacity + 1; _buffer = new int16_t[_sampleCapacity]; if (_randomAccessMode) { memset(_buffer, 0, _sampleCapacity * sizeof(int16_t)); @@ -133,6 +132,7 @@ qint64 AudioRingBuffer::writeData(const char* data, qint64 maxSize) { // there's not enough room for this write. erase old data to make room for this new data int samplesToDelete = samplesToCopy - samplesRoomFor; _nextOutput = shiftedPositionAccomodatingWrap(_nextOutput, samplesToDelete); + _overflowCount++; qDebug() << "Overflowed ring buffer! Overwriting old data"; } From f2dfa6b978823fe9d83bf32b1e88a5f98fac42a1 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 16:10:58 -0700 Subject: [PATCH 58/67] fixed warnings --- interface/src/ui/Stats.cpp | 4 ++-- libraries/octree/src/OctreeSceneStats.cpp | 4 ---- tests/networking/src/SequenceNumberStatsTests.cpp | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index c97a8ceca1..8a2a3471cd 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -327,8 +327,8 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); - char* audioMixerJitterBuffersStatsLabel = "AudioMixer stream stats:"; - char* audioMixerJitterBuffersStatsLabel2 = "early/late/lost, jframes"; + char audioMixerJitterBuffersStatsLabel[] = "AudioMixer stream stats:"; + char audioMixerJitterBuffersStatsLabel2[] = "early/late/lost, jframes"; verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 713a5f78f0..e585c8dfe6 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -21,10 +21,6 @@ #include "OctreeSceneStats.h" -const uint16_t MAX_MISSING_SEQUENCE = 100; /// how many items in our _missingSequenceNumbers before we start to prune them -const uint16_t MAX_MISSING_SEQUENCE_OLD_AGE = 1000; /// age we allow items in _missingSequenceNumbers to be before pruning - - const int samples = 100; OctreeSceneStats::OctreeSceneStats() : _isReadyToSend(false), diff --git a/tests/networking/src/SequenceNumberStatsTests.cpp b/tests/networking/src/SequenceNumberStatsTests.cpp index 5fd742fd65..89a14deb20 100644 --- a/tests/networking/src/SequenceNumberStatsTests.cpp +++ b/tests/networking/src/SequenceNumberStatsTests.cpp @@ -149,7 +149,6 @@ void SequenceNumberStatsTests::duplicateTest() { } // skip 10 - quint16 skipped = seq; seq = seq + (quint16)10; @@ -214,7 +213,6 @@ void SequenceNumberStatsTests::pruneTest() { int numSent = 0; int numEarly = 0; - int numLate = 0; int numLost = 0; for (int R = 0; R < 2; R++) { From 02f12c27c50289073dfa43ed87a3abf8f00bb999 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 16:20:29 -0700 Subject: [PATCH 59/67] forgot to add early/late/lost domain page stats for mic stream --- .../src/audio/AudioMixerClientData.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0065bff959..0d2589ce5f 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -240,12 +240,16 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { int overflowCount = avatarRingBuffer->getOverflowCount(); int samplesAvailable = avatarRingBuffer->samplesAvailable(); int framesAvailable = (samplesAvailable / avatarRingBuffer->getSamplesPerFrame()); - result += "mic.desired:" + QString::number(desiredJitterBuffer) - + " calculated:" + QString::number(calculatedJitterBuffer) - + " current:" + QString::number(currentJitterBuffer) - + " available:" + QString::number(framesAvailable) - + " samples:" + QString::number(samplesAvailable) - + " overflows:" + QString::number(overflowCount); + AudioStreamStats streamStats = getAudioStreamStatsOfStream(avatarRingBuffer); + result += "mic.desired:" + QString::number(desiredJitterBuffer) + + " calculated:" + QString::number(calculatedJitterBuffer) + + " current:" + QString::number(currentJitterBuffer) + + " available:" + QString::number(framesAvailable) + + " samples:" + QString::number(samplesAvailable) + + " overflows:" + QString::number(overflowCount); + + " early:" + QString::number(streamStats._packetsEarly) + + " late:" + QString::number(streamStats._packetsLate) + + " lost:" + QString::number(streamStats._packetsLost); } else { result = "mic unknown"; } From de0e65eb030d80b8537e2f52a026e95b81af14f2 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 16:26:50 -0700 Subject: [PATCH 60/67] Restored glow effect --- interface/src/devices/OculusManager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index 038b7ea8f5..e9742b782f 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -333,9 +333,8 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p glViewport(0, 0, _renderTargetSize.w, _renderTargetSize.h); //Bind the output texture from the glow shader - // QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); - Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->release(); - glBindTexture(GL_TEXTURE_2D, Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->texture()); + QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); + glBindTexture(GL_TEXTURE_2D, fbo->texture()); // restore our normal viewport glViewport(0, 0, Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); From 272732872958b647181bd83f4587f998cc5ade75 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 16:46:25 -0700 Subject: [PATCH 61/67] Removed unneeded menu option --- interface/src/Menu.cpp | 1 - interface/src/Menu.h | 1 - 2 files changed, 2 deletions(-) diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 9357ba2004..f0decfba45 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -400,7 +400,6 @@ Menu::Menu() : addCheckableActionToQMenuAndActionHash(avatarOptionsMenu, MenuOption::ChatCircling, 0, false); QMenu* oculusOptionsMenu = developerMenu->addMenu("Oculus Options"); - addCheckableActionToQMenuAndActionHash(oculusOptionsMenu, MenuOption::AllowOculusCameraModeChange, 0, false); addCheckableActionToQMenuAndActionHash(oculusOptionsMenu, MenuOption::DisplayOculusOverlays, 0, true); QMenu* sixenseOptionsMenu = developerMenu->addMenu("Sixense Options"); diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 7417f953c6..267544e0d9 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -295,7 +295,6 @@ private: namespace MenuOption { const QString AboutApp = "About Interface"; const QString AlignForearmsWithWrists = "Align Forearms with Wrists"; - const QString AllowOculusCameraModeChange = "Allow Oculus Camera Mode Change (Nausea)"; const QString AlternateIK = "Alternate IK"; const QString AmbientOcclusion = "Ambient Occlusion"; const QString Animations = "Animations..."; From 7a6b20fd8450b73ca8efe1c758d87db6ba88cd43 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 17:01:22 -0700 Subject: [PATCH 62/67] added downstream audio stats to stats overlay --- interface/src/Audio.h | 2 ++ interface/src/ui/Stats.cpp | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 62c524ba2c..e6c72f587e 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -73,6 +73,8 @@ public: bool getProcessSpatialAudio() const { return _processSpatialAudio; } + const SequenceNumberStats& getIncomingMixedAudioSequenceNumberStats() const { return _incomingMixedAudioSequenceNumberStats; } + public slots: void start(); void stop(); diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp index 8a2a3471cd..379dd35df7 100644 --- a/interface/src/ui/Stats.cpp +++ b/interface/src/ui/Stats.cpp @@ -291,7 +291,7 @@ void Stats::display( const AudioStreamStats& audioMixerAvatarStreamStats = audio->getAudioMixerAvatarStreamStats(); const QHash& audioMixerInjectedStreamStatsMap = audio->getAudioMixerInjectedStreamStatsMap(); - lines = _expanded ? 7 + audioMixerInjectedStreamStatsMap.size(): 3; + lines = _expanded ? 10 + audioMixerInjectedStreamStatsMap.size(): 3; drawBackground(backgroundColor, horizontalOffset, 0, _pingStatsWidth, lines * STATS_PELS_PER_LINE + 10); horizontalOffset += 5; @@ -327,28 +327,46 @@ void Stats::display( verticalOffset += STATS_PELS_PER_LINE; drawText(horizontalOffset, verticalOffset, scale, rotation, font, voxelMaxPing, color); - char audioMixerJitterBuffersStatsLabel[] = "AudioMixer stream stats:"; - char audioMixerJitterBuffersStatsLabel2[] = "early/late/lost, jframes"; + char audioMixerStatsLabelString[] = "AudioMixer stats:"; + char streamStatsFormatLabelString[] = "early/late/lost, jframes"; verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerStatsLabelString, color); verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStatsLabel2, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, streamStatsFormatLabelString, color); - char audioMixerJitterBuffersStats[30]; - sprintf(audioMixerJitterBuffersStats, "mic: %d/%d/%d, %d", audioMixerAvatarStreamStats._packetsEarly, + + char downstreamLabelString[] = " Downstream:"; + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamLabelString, color); + + const SequenceNumberStats& downstreamAudioSequenceNumberStats = audio->getIncomingMixedAudioSequenceNumberStats(); + char downstreamAudioStatsString[30]; + sprintf(downstreamAudioStatsString, " mix: %d/%d/%d, %d", downstreamAudioSequenceNumberStats.getNumEarly(), + downstreamAudioSequenceNumberStats.getNumLate(), downstreamAudioSequenceNumberStats.getNumLost(), + audio->getJitterBufferSamples() / NETWORK_BUFFER_LENGTH_SAMPLES_STEREO); + + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, downstreamAudioStatsString, color); + + char upstreamLabelString[] = " Upstream:"; + verticalOffset += STATS_PELS_PER_LINE; + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamLabelString, color); + + char upstreamAudioStatsString[30]; + sprintf(upstreamAudioStatsString, " mic: %d/%d/%d, %d", audioMixerAvatarStreamStats._packetsEarly, audioMixerAvatarStreamStats._packetsLate, audioMixerAvatarStreamStats._packetsLost, audioMixerAvatarStreamStats._jitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); foreach(AudioStreamStats injectedStreamStats, audioMixerInjectedStreamStatsMap) { - sprintf(audioMixerJitterBuffersStats, "inj: %d/%d/%d, %d", injectedStreamStats._packetsEarly, + sprintf(upstreamAudioStatsString, " inj: %d/%d/%d, %d", injectedStreamStats._packetsEarly, injectedStreamStats._packetsLate, injectedStreamStats._packetsLost, injectedStreamStats._jitterBufferFrames); verticalOffset += STATS_PELS_PER_LINE; - drawText(horizontalOffset, verticalOffset, scale, rotation, font, audioMixerJitterBuffersStats, color); + drawText(horizontalOffset, verticalOffset, scale, rotation, font, upstreamAudioStatsString, color); } } From 1cd2066e794004aa65d98e6e90c5ef80fbf81329 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Mon, 30 Jun 2014 17:13:47 -0700 Subject: [PATCH 63/67] Added option to disable glow effect for 10fps boost for oculus --- interface/src/Application.cpp | 23 +++++++++++++++++++---- interface/src/Menu.cpp | 2 ++ interface/src/Menu.h | 1 + interface/src/devices/OculusManager.cpp | 21 ++++++++++++++++----- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index b29f7ed20a..fb137b6018 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -565,6 +565,8 @@ void Application::paintGL() { bool showWarnings = Menu::getInstance()->isOptionChecked(MenuOption::PipelineWarnings); PerformanceWarning warn(showWarnings, "Application::paintGL()"); + const bool glowEnabled = Menu::getInstance()->isOptionChecked(MenuOption::EnableGlowEffect); + // Set the desired FBO texture size. If it hasn't changed, this does nothing. // Otherwise, it must rebuild the FBOs if (OculusManager::isConnected()) { @@ -624,6 +626,11 @@ void Application::paintGL() { updateShadowMap(); } + //If we aren't using the glow shader, we have to clear the color and depth buffer + if (!glowEnabled) { + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + } + if (OculusManager::isConnected()) { //When in mirror mode, use camera rotation. Otherwise, use body rotation if (whichCamera.getMode() == CAMERA_MODE_MIRROR) { @@ -633,12 +640,18 @@ void Application::paintGL() { } } else if (TV3DManager::isConnected()) { - _glowEffect.prepare(); + if (glowEnabled) { + _glowEffect.prepare(); + } TV3DManager::display(whichCamera); - _glowEffect.render(); + if (glowEnabled) { + _glowEffect.render(); + } } else { - _glowEffect.prepare(); + if (glowEnabled) { + _glowEffect.prepare(); + } glMatrixMode(GL_MODELVIEW); glPushMatrix(); @@ -646,7 +659,9 @@ void Application::paintGL() { displaySide(whichCamera); glPopMatrix(); - _glowEffect.render(); + if (glowEnabled) { + _glowEffect.render(); + } if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { renderRearViewMirror(_mirrorViewRect); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index f0decfba45..402347c5d4 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -332,6 +332,8 @@ Menu::Menu() : addCheckableActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::Stars, Qt::Key_Asterisk, true); addCheckableActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::Atmosphere, Qt::SHIFT | Qt::Key_A, true); + + addCheckableActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::EnableGlowEffect, 0, true); addActionToQMenuAndActionHash(renderOptionsMenu, MenuOption::GlowMode, 0, diff --git a/interface/src/Menu.h b/interface/src/Menu.h index 267544e0d9..a15d3712f1 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -351,6 +351,7 @@ namespace MenuOption { const QString DontFadeOnVoxelServerChanges = "Don't Fade In/Out on Voxel Server Changes"; const QString EchoLocalAudio = "Echo Local Audio"; const QString EchoServerAudio = "Echo Server Audio"; + const QString EnableGlowEffect = "Enable Glow Effect (Warning: Poor Oculus Performance)"; const QString Enable3DTVMode = "Enable 3DTV Mode"; const QString EnableVRMode = "Enable VR Mode"; const QString ExpandMiscAvatarTiming = "Expand Misc MyAvatar Timing"; diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index e9742b782f..fed0259604 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -266,8 +266,14 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p // PrioVR will only work if renderOverlay is called, calibration is connected to Application::renderingOverlay() applicationOverlay.renderOverlay(true); const bool displayOverlays = Menu::getInstance()->isOptionChecked(MenuOption::DisplayOculusOverlays); - - Application::getInstance()->getGlowEffect()->prepare(); + + //Bind our framebuffer object. If we are rendering the glow effect, we let the glow effect shader take care of it + if (Menu::getInstance()->isOptionChecked(MenuOption::EnableGlowEffect)) { + Application::getInstance()->getGlowEffect()->prepare(); + } else { + Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->bind(); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + } ovrPosef eyeRenderPose[ovrEye_Count]; @@ -332,9 +338,14 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p //Full texture viewport for glow effect glViewport(0, 0, _renderTargetSize.w, _renderTargetSize.h); - //Bind the output texture from the glow shader - QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); - glBindTexture(GL_TEXTURE_2D, fbo->texture()); + //Bind the output texture from the glow shader. If glow effect is disabled, we just grab the texture + if (Menu::getInstance()->isOptionChecked(MenuOption::EnableGlowEffect)) { + QOpenGLFramebufferObject* fbo = Application::getInstance()->getGlowEffect()->render(true); + glBindTexture(GL_TEXTURE_2D, fbo->texture()); + } else { + Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->release(); + glBindTexture(GL_TEXTURE_2D, Application::getInstance()->getTextureCache()->getPrimaryFramebufferObject()->texture()); + } // restore our normal viewport glViewport(0, 0, Application::getInstance()->getGLWidget()->width(), Application::getInstance()->getGLWidget()->height()); From 4abfd2b6b730b11890577b625de611d850fa1187 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 17:16:28 -0700 Subject: [PATCH 64/67] removed forgotten semicolon in stats string --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 0d2589ce5f..9b14ecfd19 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -246,7 +246,7 @@ QString AudioMixerClientData::getAudioStreamStatsString() const { + " current:" + QString::number(currentJitterBuffer) + " available:" + QString::number(framesAvailable) + " samples:" + QString::number(samplesAvailable) - + " overflows:" + QString::number(overflowCount); + + " overflows:" + QString::number(overflowCount) + " early:" + QString::number(streamStats._packetsEarly) + " late:" + QString::number(streamStats._packetsLate) + " lost:" + QString::number(streamStats._packetsLost); From f51a77ce45a3a87e81b91a0c9053cf4364a8cab0 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 30 Jun 2014 18:00:29 -0700 Subject: [PATCH 65/67] reset the incoming audio seq stats in Application::nodeKilled when audiomixer is killed --- interface/src/Application.cpp | 4 ++++ interface/src/Audio.cpp | 2 ++ interface/src/Audio.h | 1 + 3 files changed, 7 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a032fa7ea0..a024b3f429 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3291,6 +3291,10 @@ void Application::nodeKilled(SharedNodePointer node) { _particleEditSender.nodeKilled(node); _modelEditSender.nodeKilled(node); + if (node->getType() == NodeType::AudioMixer) { + QMetaObject::invokeMethod(&_audio, "resetIncomingMixedAudioSequenceNumberStats"); + } + if (node->getType() == NodeType::VoxelServer) { QUuid nodeUUID = node->getUUID(); // see if this is the first we've heard of this node... diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index c52e2128b0..cda7e2b230 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -121,6 +121,8 @@ void Audio::init(QGLWidget *parent) { void Audio::reset() { _ringBuffer.reset(); _outgoingAvatarAudioSequenceNumber = 0; + _audioMixerInjectedStreamStatsMap.clear(); + _incomingMixedAudioSequenceNumberStats.reset(); } QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) { diff --git a/interface/src/Audio.h b/interface/src/Audio.h index e6c72f587e..9f04e5cb03 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -83,6 +83,7 @@ public slots: void addSpatialAudioToBuffer(unsigned int sampleTime, const QByteArray& spatialAudio, unsigned int numSamples); void handleAudioInput(); void reset(); + void resetIncomingMixedAudioSequenceNumberStats() { _incomingMixedAudioSequenceNumberStats.reset(); } void toggleMute(); void toggleAudioNoiseReduction(); void toggleToneInjection(); From 80bfb0766baea1d6069ec875c7f2f89d466bb849 Mon Sep 17 00:00:00 2001 From: barnold1953 Date: Tue, 1 Jul 2014 10:28:40 -0700 Subject: [PATCH 66/67] Fixed warnings --- interface/src/Application.cpp | 3 ++- interface/src/devices/OculusManager.cpp | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index fb137b6018..7fb3ded0d8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -583,8 +583,9 @@ void Application::paintGL() { _myCamera.setTargetRotation(_myAvatar->getHead()->getCameraOrientation()); } else if (_myCamera.getMode() == CAMERA_MODE_THIRD_PERSON) { + //Note, the camera distance is set in Camera::setMode() so we dont have to do it here. _myCamera.setTightness(0.0f); // Camera is directly connected to head without smoothing - _myCamera.setTargetPosition(_myAvatar->getHead()->calculateAverageEyePosition()); + _myCamera.setTargetPosition(_myAvatar->getUprightHeadPosition()); _myCamera.setTargetRotation(_myAvatar->getWorldAlignedOrientation()); } else if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { diff --git a/interface/src/devices/OculusManager.cpp b/interface/src/devices/OculusManager.cpp index fed0259604..d9549438b5 100644 --- a/interface/src/devices/OculusManager.cpp +++ b/interface/src/devices/OculusManager.cpp @@ -288,8 +288,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p glPushMatrix(); glm::quat orientation; - glm::vec3 positionOffset; - + //Render each eye into an fbo for (int eyeIndex = 0; eyeIndex < ovrEye_Count; eyeIndex++) { @@ -301,10 +300,7 @@ void OculusManager::display(const glm::quat &bodyOrientation, const glm::vec3 &p orientation.y = eyeRenderPose[eye].Orientation.y; orientation.z = eyeRenderPose[eye].Orientation.z; orientation.w = eyeRenderPose[eye].Orientation.w; - - const float OCULUS_POSITION_SCALE = 0.2; - positionOffset = bodyOrientation * orientation * glm::vec3(eyeRenderPose[eye].Position.x, eyeRenderPose[eye].Position.y, eyeRenderPose[eye].Position.z) * OCULUS_POSITION_SCALE; - + _camera->setTargetRotation(bodyOrientation * orientation); _camera->setTargetPosition(position); _camera->update(1.0f / Application::getInstance()->getFps()); From 99fd1c7106d5509a8709b93195a6c0d1d6e05eee Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 1 Jul 2014 10:44:07 -0700 Subject: [PATCH 67/67] changed numSamplesNeedToStartPlayback calculation in Audio.cpp --- interface/src/Audio.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index cda7e2b230..f5f148969c 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -878,7 +878,8 @@ void Audio::processReceivedAudio(const QByteArray& audioByteArray) { QByteArray outputBuffer; outputBuffer.resize(numDeviceOutputSamples * sizeof(int16_t)); - int numSamplesNeededToStartPlayback = (_jitterBufferSamples * 2);// + NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; + int numSamplesNeededToStartPlayback = std::min(NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (_jitterBufferSamples * 2), + _ringBuffer.getSampleCapacity()); if (!_ringBuffer.isNotStarvedOrHasMinimumSamples(numSamplesNeededToStartPlayback)) { // We are still waiting for enough samples to begin playback