From 277eefafd766ae00ac9d83a3d293e49fc57c0802 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 23 Sep 2016 14:39:07 -0700 Subject: [PATCH 01/25] remove the audio stats dialog --- interface/src/Application.cpp | 6 - interface/src/Application.h | 1 - interface/src/Menu.cpp | 4 - interface/src/Menu.h | 1 - interface/src/ui/AudioStatsDialog.cpp | 296 -------------------------- interface/src/ui/AudioStatsDialog.h | 112 ---------- interface/src/ui/DialogsManager.cpp | 14 -- interface/src/ui/DialogsManager.h | 4 - interface/src/ui/Stats.h | 5 - 9 files changed, 443 deletions(-) delete mode 100644 interface/src/ui/AudioStatsDialog.cpp delete mode 100644 interface/src/ui/AudioStatsDialog.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 97a10ea232..ce589bf52f 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3635,12 +3635,6 @@ void Application::updateDialogs(float deltaTime) const { PerformanceWarning warn(showWarnings, "Application::updateDialogs()"); auto dialogsManager = DependencyManager::get(); - // Update audio stats dialog, if any - AudioStatsDialog* audioStatsDialog = dialogsManager->getAudioStatsDialog(); - if(audioStatsDialog) { - audioStatsDialog->update(); - } - // Update bandwidth dialog, if any BandwidthDialog* bandwidthDialog = dialogsManager->getBandwidthDialog(); if (bandwidthDialog) { diff --git a/interface/src/Application.h b/interface/src/Application.h index 4c52ff8526..365997487d 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -57,7 +57,6 @@ #include "scripting/ControllerScriptingInterface.h" #include "scripting/DialogsManagerScriptingInterface.h" #include "ui/ApplicationOverlay.h" -#include "ui/AudioStatsDialog.h" #include "ui/BandwidthDialog.h" #include "ui/LodToolsDialog.h" #include "ui/LogDialog.h" diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index 08abbf63d2..8fb6d524da 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -617,10 +617,6 @@ Menu::Menu() { audioScopeFramesGroup->addAction(fiftyFrames); } - // Developer > Audio > Audio Network Stats... - addActionToQMenuAndActionHash(audioDebugMenu, MenuOption::AudioNetworkStats, 0, - dialogsManager.data(), SLOT(audioStatsDetails())); - // Developer > Physics >>> MenuWrapper* physicsOptionsMenu = developerMenu->addMenu("Physics"); { diff --git a/interface/src/Menu.h b/interface/src/Menu.h index b25603caeb..0dba30c921 100644 --- a/interface/src/Menu.h +++ b/interface/src/Menu.h @@ -37,7 +37,6 @@ namespace MenuOption { const QString AssetMigration = "ATP Asset Migration"; const QString AssetServer = "Asset Browser"; const QString Attachments = "Attachments..."; - const QString AudioNetworkStats = "Audio Network Stats"; const QString AudioNoiseReduction = "Audio Noise Reduction"; const QString AudioScope = "Show Scope"; const QString AudioScopeFiftyFrames = "Fifty"; diff --git a/interface/src/ui/AudioStatsDialog.cpp b/interface/src/ui/AudioStatsDialog.cpp deleted file mode 100644 index e3cca9f0fe..0000000000 --- a/interface/src/ui/AudioStatsDialog.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// -// AudioStatsDialog.cpp -// interface/src/ui -// -// Created by Bridget Went on 7/9/15. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "AudioStatsDialog.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - - - -const unsigned COLOR0 = 0x33cc99ff; -const unsigned COLOR1 = 0xffef40c0; -const unsigned COLOR2 = 0xd0d0d0a0; -const unsigned COLOR3 = 0x01DD7880; - - -AudioStatsDisplay::AudioStatsDisplay(QFormLayout* form, - QString text, unsigned colorRGBA) : -_text(text), -_colorRGBA(colorRGBA) -{ - _label = new QLabel(); - _label->setAlignment(Qt::AlignCenter); - - QPalette palette = _label->palette(); - unsigned rgb = colorRGBA >> 8; - rgb = ((rgb & 0xfefefeu) >> 1) + ((rgb & 0xf8f8f8) >> 3); - palette.setColor(QPalette::WindowText, QColor::fromRgb(rgb)); - _label->setPalette(palette); - - form->addRow(_label); -} - -void AudioStatsDisplay::paint() { - _label->setText(_strBuf); -} - -void AudioStatsDisplay::updatedDisplay(QString str) { - _strBuf = str; -} - - -AudioStatsDialog::AudioStatsDialog(QWidget* parent) : - QDialog(parent, Qt::Window | Qt::WindowCloseButtonHint | Qt::WindowStaysOnTopHint) { - - setWindowTitle("Audio Network Statistics"); - - // Get statistics from the Audio Client - _stats = &DependencyManager::get()->getStats(); - - // Create layout - _form = new QFormLayout(); - _form->setSizeConstraint(QLayout::SetFixedSize); - - // Initialize channels' content (needed to correctly size channels) - updateStats(); - - // Create channels - _audioDisplayChannels = QVector>(1); - - _audioMixerID = addChannel(_form, _audioMixerStats, COLOR0); - _upstreamClientID = addChannel(_form, _upstreamClientStats, COLOR1); - _upstreamMixerID = addChannel(_form, _upstreamMixerStats, COLOR2); - _downstreamID = addChannel(_form, _downstreamStats, COLOR3); - _upstreamInjectedID = addChannel(_form, _upstreamInjectedStats, COLOR0); - - // Initialize channels - updateChannels(); - - // Future renders - connect(averageUpdateTimer, SIGNAL(timeout()), this, SLOT(renderStats())); - averageUpdateTimer->start(200); - - // Initial render - QDialog::setLayout(_form); -} - -int AudioStatsDialog::addChannel(QFormLayout* form, QVector& stats, const unsigned color) { - - int channelID = _audioDisplayChannels.size() - 1; - - for (int i = 0; i < stats.size(); i++) - // Create new display label - _audioDisplayChannels[channelID].push_back(new AudioStatsDisplay(form, stats.at(i), color)); - - // Expand vector to fit next channel - _audioDisplayChannels.resize(_audioDisplayChannels.size() + 1); - - return channelID; -} - -void AudioStatsDialog::renderStats() { - updateStats(); - updateChannels(); -} - -void AudioStatsDialog::updateChannels() { - updateChannel(_audioMixerStats, _audioMixerID); - updateChannel(_upstreamClientStats, _upstreamClientID); - updateChannel(_upstreamMixerStats, _upstreamMixerID); - updateChannel(_downstreamStats, _downstreamID); - updateChannel(_upstreamInjectedStats, _upstreamInjectedID); -} - -void AudioStatsDialog::updateChannel(QVector& stats, int channelID) { - // Update all stat displays at specified channel - for (int i = 0; i < stats.size(); i++) - _audioDisplayChannels[channelID].at(i)->updatedDisplay(stats.at(i)); -} - -void AudioStatsDialog::updateStats() { - - // Clear current stats from all vectors - clearAllChannels(); - - double audioInputBufferLatency{ 0.0 }; - double inputRingBufferLatency{ 0.0 }; - double networkRoundtripLatency{ 0.0 }; - double mixerRingBufferLatency{ 0.0 }; - double outputRingBufferLatency{ 0.0 }; - double audioOutputBufferLatency{ 0.0 }; - - if (SharedNodePointer audioMixerNodePointer = DependencyManager::get()->soloNodeOfType(NodeType::AudioMixer)) { - audioInputBufferLatency = (double)_stats->getInputMsRead().getWindowMax(); - inputRingBufferLatency = (double)_stats->getInputMsUnplayed().getWindowMax(); - networkRoundtripLatency = (double)audioMixerNodePointer->getPingMs(); - mixerRingBufferLatency = (double)_stats->getMixerAvatarStreamStats()._unplayedMs; - outputRingBufferLatency = (double)_stats->getMixerDownstreamStats()._unplayedMs; - audioOutputBufferLatency = (double)_stats->getOutputMsUnplayed().getWindowMax(); - } - - double totalLatency = audioInputBufferLatency + inputRingBufferLatency + mixerRingBufferLatency - + outputRingBufferLatency + audioOutputBufferLatency + networkRoundtripLatency; - - QString stats; - _audioMixerStats.push_back("PIPELINE (averaged over the past 10s)"); - stats = "Input Read:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(audioInputBufferLatency, 'f', 0))); - stats = "Input Ring:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(inputRingBufferLatency, 'f', 0))); - stats = "Network (client->mixer):\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(networkRoundtripLatency / 2, 'f', 0))); - stats = "Mixer Ring:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(mixerRingBufferLatency, 'f', 0))); - stats = "Network (mixer->client):\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(networkRoundtripLatency / 2, 'f', 0))); - stats = "Output Ring:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(outputRingBufferLatency, 'f', 0))); - stats = "Output Read:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(audioOutputBufferLatency, 'f', 0))); - stats = "TOTAL:\t%1 ms"; - _audioMixerStats.push_back(stats.arg(QString::number(totalLatency, 'f', 0))); - - const MovingMinMaxAvg& packetSentTimeGaps = _stats->getPacketTimegaps(); - - _upstreamClientStats.push_back("\nUpstream Mic Audio Packets Sent Gaps (by client):"); - - stats = "Inter-packet timegaps"; - _upstreamClientStats.push_back(stats); - stats = "overall min:\t%1, max:\t%2, avg:\t%3"; - stats = stats.arg(formatUsecTime(packetSentTimeGaps.getMin()), - formatUsecTime(packetSentTimeGaps.getMax()), - formatUsecTime(packetSentTimeGaps.getAverage())); - _upstreamClientStats.push_back(stats); - - stats = "last window min:\t%1, max:\t%2, avg:\t%3"; - stats = stats.arg(formatUsecTime(packetSentTimeGaps.getWindowMin()), - formatUsecTime(packetSentTimeGaps.getWindowMax()), - formatUsecTime(packetSentTimeGaps.getWindowAverage())); - _upstreamClientStats.push_back(stats); - - _upstreamMixerStats.push_back("\nMIXER STREAM"); - _upstreamMixerStats.push_back("(this client's remote mixer stream performance)"); - - renderAudioStreamStats(&_stats->getMixerAvatarStreamStats(), &_upstreamMixerStats); - - _downstreamStats.push_back("\nCLIENT STREAM"); - - AudioStreamStats downstreamStats = _stats->getMixerDownstreamStats(); - - renderAudioStreamStats(&downstreamStats, &_downstreamStats); - - - if (_shouldShowInjectedStreams) { - - foreach(const AudioStreamStats& injectedStreamAudioStats, _stats->getMixerInjectedStreamStatsMap()) { - stats = "\nINJECTED STREAM (ID: %1)"; - stats = stats.arg(injectedStreamAudioStats._streamIdentifier.toString()); - _upstreamInjectedStats.push_back(stats); - - renderAudioStreamStats(&injectedStreamAudioStats, &_upstreamInjectedStats); - } - - } -} - - -void AudioStatsDialog::renderAudioStreamStats(const AudioStreamStats* streamStats, QVector* audioStreamStats) { - - QString stats = "Packet Loss"; - audioStreamStats->push_back(stats); - stats = "overall:\t%1%\t(%2 lost), window:\t%3%\t(%4 lost)"; - stats = stats.arg(QString::number((int)(streamStats->_packetStreamStats.getLostRate() * 100.0f)), - QString::number((int)(streamStats->_packetStreamStats._lost)), - QString::number((int)(streamStats->_packetStreamWindowStats.getLostRate() * 100.0f)), - QString::number((int)(streamStats->_packetStreamWindowStats._lost))); - audioStreamStats->push_back(stats); - - stats = "Ringbuffer"; - audioStreamStats->push_back(stats); - stats = "available frames (avg):\t%1\t(%2), desired:\t%3"; - stats = stats.arg(QString::number(streamStats->_framesAvailable), - QString::number(streamStats->_framesAvailableAverage), - QString::number(streamStats->_desiredJitterBufferFrames)); - audioStreamStats->push_back(stats); - stats = "starves:\t%1, last starve duration:\t%2, drops:\t%3, overflows:\t%4"; - stats = stats.arg(QString::number(streamStats->_starveCount), - QString::number(streamStats->_consecutiveNotMixedCount), - QString::number(streamStats->_framesDropped), - QString::number(streamStats->_overflowCount)); - audioStreamStats->push_back(stats); - - stats = "Inter-packet timegaps"; - audioStreamStats->push_back(stats); - - stats = "overall min:\t%1, max:\t%2, avg:\t%3"; - stats = stats.arg(formatUsecTime(streamStats->_timeGapMin), - formatUsecTime(streamStats->_timeGapMax), - formatUsecTime(streamStats->_timeGapAverage)); - audioStreamStats->push_back(stats); - - - stats = "last window min:\t%1, max:\t%2, avg:\t%3"; - stats = stats.arg(formatUsecTime(streamStats->_timeGapWindowMin), - formatUsecTime(streamStats->_timeGapWindowMax), - formatUsecTime(streamStats->_timeGapWindowAverage)); - audioStreamStats->push_back(stats); -} - -void AudioStatsDialog::clearAllChannels() { - _audioMixerStats.clear(); - _upstreamClientStats.clear(); - _upstreamMixerStats.clear(); - _downstreamStats.clear(); - _upstreamInjectedStats.clear(); -} - -void AudioStatsDialog::paintEvent(QPaintEvent* event) { - - // Repaint each stat in each channel - for (int i = 0; i < _audioDisplayChannels.size(); i++) { - for(int j = 0; j < _audioDisplayChannels[i].size(); j++) { - _audioDisplayChannels[i].at(j)->paint(); - } - } - - QDialog::paintEvent(event); -} - -void AudioStatsDialog::reject() { - // Just regularly close upon ESC - QDialog::close(); -} - -void AudioStatsDialog::closeEvent(QCloseEvent* event) { - QDialog::closeEvent(event); - emit closed(); -} - -AudioStatsDialog::~AudioStatsDialog() { - clearAllChannels(); - for (int i = 0; i < _audioDisplayChannels.size(); i++) { - _audioDisplayChannels[i].clear(); - for(int j = 0; j < _audioDisplayChannels[i].size(); j++) { - delete _audioDisplayChannels[i].at(j); - } - } - -} - - diff --git a/interface/src/ui/AudioStatsDialog.h b/interface/src/ui/AudioStatsDialog.h deleted file mode 100644 index 59da056de4..0000000000 --- a/interface/src/ui/AudioStatsDialog.h +++ /dev/null @@ -1,112 +0,0 @@ -// -// AudioStatsDialog.h -// hifi -// -// Created by Bridget Went on 7/9/15. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - - -#ifndef __hifi__AudioStatsDialog__ -#define __hifi__AudioStatsDialog__ - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include - -class AudioIOStats; -class AudioStreamStats; - -//display -class AudioStatsDisplay : public QObject, public Dependency { - Q_OBJECT - SINGLETON_DEPENDENCY -public: - AudioStatsDisplay(QFormLayout* form, QString text, unsigned colorRGBA); - void updatedDisplay(QString str); - void paint(); - -private: - QString _strBuf; - QLabel* _label; - QString _text; - unsigned _colorRGBA; - -}; - -//dialog -class AudioStatsDialog : public QDialog { - Q_OBJECT -public: - AudioStatsDialog(QWidget* parent); - ~AudioStatsDialog(); - - void paintEvent(QPaintEvent*) override; - -private: - // audio stats methods for rendering - QVector _audioMixerStats; - QVector _upstreamClientStats; - QVector _upstreamMixerStats; - QVector _downstreamStats; - QVector _upstreamInjectedStats; - - int _audioMixerID; - int _upstreamClientID; - int _upstreamMixerID; - int _downstreamID; - int _upstreamInjectedID; - - QVector> _audioDisplayChannels; - - void updateStats(); - int addChannel(QFormLayout* form, QVector& stats, const unsigned color); - void updateChannel(QVector& stats, const int channelID); - void updateChannels(); - void clearAllChannels(); - void renderAudioStreamStats(const AudioStreamStats* streamStats, QVector* audioStreamstats); - - - const AudioIOStats* _stats; - QFormLayout* _form; - - bool _shouldShowInjectedStreams{ false }; - - -signals: - - - void closed(); - - public slots: - - - void reject() override; - void renderStats(); - -protected: - - // Emits a 'closed' signal when this dialog is closed. - void closeEvent(QCloseEvent*) override; - -private: - QTimer* averageUpdateTimer = new QTimer(this); -}; - - - - - -#endif /* defined(__hifi__AudioStatsDialog__) */ - diff --git a/interface/src/ui/DialogsManager.cpp b/interface/src/ui/DialogsManager.cpp index 1b868f4154..1c9a60348a 100644 --- a/interface/src/ui/DialogsManager.cpp +++ b/interface/src/ui/DialogsManager.cpp @@ -99,20 +99,6 @@ void DialogsManager::cachesSizeDialog() { _cachesSizeDialog->raise(); } -void DialogsManager::audioStatsDetails() { - if (! _audioStatsDialog) { - _audioStatsDialog = new AudioStatsDialog(qApp->getWindow()); - connect(_audioStatsDialog, SIGNAL(closed()), _audioStatsDialog, SLOT(deleteLater())); - - if (_hmdToolsDialog) { - _hmdToolsDialog->watchWindow(_audioStatsDialog->windowHandle()); - } - - _audioStatsDialog->show(); - } - _audioStatsDialog->raise(); -} - void DialogsManager::bandwidthDetails() { if (! _bandwidthDialog) { _bandwidthDialog = new BandwidthDialog(qApp->getWindow()); diff --git a/interface/src/ui/DialogsManager.h b/interface/src/ui/DialogsManager.h index 5e25afd130..3b47c7a5db 100644 --- a/interface/src/ui/DialogsManager.h +++ b/interface/src/ui/DialogsManager.h @@ -20,7 +20,6 @@ class AnimationsDialog; class AttachmentsDialog; -class AudioStatsDialog; class BandwidthDialog; class CachesSizeDialog; class DiskCacheEditor; @@ -35,7 +34,6 @@ class DialogsManager : public QObject, public Dependency { SINGLETON_DEPENDENCY public: - QPointer getAudioStatsDialog() const { return _audioStatsDialog; } QPointer getBandwidthDialog() const { return _bandwidthDialog; } QPointer getHMDToolsDialog() const { return _hmdToolsDialog; } QPointer getLodToolsDialog() const { return _lodToolsDialog; } @@ -51,7 +49,6 @@ public slots: void showLoginDialog(); void octreeStatsDetails(); void cachesSizeDialog(); - void audioStatsDetails(); void bandwidthDetails(); void lodTools(); void hmdTools(bool showTools); @@ -77,7 +74,6 @@ private: QPointer _animationsDialog; QPointer _attachmentsDialog; - QPointer _audioStatsDialog; QPointer _bandwidthDialog; QPointer _cachesSizeDialog; QPointer _diskCacheEditor; diff --git a/interface/src/ui/Stats.h b/interface/src/ui/Stats.h index 138f24cf19..a7b099d543 100644 --- a/interface/src/ui/Stats.h +++ b/interface/src/ui/Stats.h @@ -28,8 +28,6 @@ class Stats : public QQuickItem { Q_PROPERTY(bool expanded READ isExpanded WRITE setExpanded NOTIFY expandedChanged) Q_PROPERTY(bool timingExpanded READ isTimingExpanded NOTIFY timingExpandedChanged) Q_PROPERTY(QString monospaceFont READ monospaceFont CONSTANT) - Q_PROPERTY(float audioPacketlossUpstream READ getAudioPacketLossUpstream) - Q_PROPERTY(float audioPacketlossDownstream READ getAudioPacketLossDownstream) STATS_PROPERTY(int, serverCount, 0) // How often the app is creating new gpu::Frames @@ -100,9 +98,6 @@ public: return _monospaceFont; } - float getAudioPacketLossUpstream() { return _audioStats->getMixerAvatarStreamStats()._packetStreamStats.getLostRate(); } - float getAudioPacketLossDownstream() { return _audioStats->getMixerDownstreamStats()._packetStreamStats.getLostRate(); } - void updateStats(bool force = false); bool isExpanded() { return _expanded; } From b9c4018b8eaa65e296149822a09f1183d026314b Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Fri, 23 Sep 2016 14:39:28 -0700 Subject: [PATCH 02/25] expose AudioStats to qml/js --- .../src/audio/AudioMixerClientData.cpp | 11 +- interface/src/Application.cpp | 2 + libraries/audio-client/src/AudioClient.h | 2 +- libraries/audio-client/src/AudioIOStats.cpp | 127 ++++++++++++------ libraries/audio-client/src/AudioIOStats.h | 122 ++++++++++++++--- .../networking/src/udt/PacketHeaders.cpp | 3 +- libraries/networking/src/udt/PacketHeaders.h | 5 +- 7 files changed, 201 insertions(+), 71 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 42d385c1f6..42928259f8 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -49,7 +49,7 @@ AudioMixerClientData::~AudioMixerClientData() { AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() { QReadLocker readLocker { &_streamsLock }; - + auto it = _audioStreams.find(QUuid()); if (it != _audioStreams.end()) { return dynamic_cast(it->second.get()); @@ -75,7 +75,7 @@ void AudioMixerClientData::removeHRTFForStream(const QUuid& nodeID, const QUuid& int AudioMixerClientData::parseData(ReceivedMessage& message) { PacketType packetType = message.getType(); - + if (packetType == PacketType::AudioStreamStats) { // skip over header, appendFlag, and num stats packed @@ -219,9 +219,10 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& auto nodeList = DependencyManager::get(); // The append flag is a boolean value that will be packed right after the header. The first packet sent - // inside this method will have 0 for this flag, while every subsequent packet will have 1 for this flag. - // The sole purpose of this flag is so the client can clear its map of injected audio stream stats when - // it receives a packet with an appendFlag of 0. This prevents the buildup of dead audio stream stats in the client. + // inside this method will have 0 for this flag, every subsequent packet but the last will have 1 for this flag, + // and the last packet will have 2 for this flag. + // This flag allows the client to know when it has received all stats packets, so it can group any downstream effects, + // and clear its cache of injector stream stats; it helps to prevent buildup of dead audio stream stats in the client. quint8 appendFlag = 0; auto streamsCopy = getAudioStreams(); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ce589bf52f..ca31456d4a 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1593,6 +1593,7 @@ void Application::initializeUi() { // though I can't find it. Hence, "ApplicationInterface" rootContext->setContextProperty("ApplicationInterface", this); rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance()); + rootContext->setContextProperty("AudioStats", DependencyManager::get()->getStats().data()); rootContext->setContextProperty("Controller", DependencyManager::get().data()); rootContext->setContextProperty("Entities", DependencyManager::get().data()); FileScriptingInterface* fileDownload = new FileScriptingInterface(engine); @@ -4874,6 +4875,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerGlobalObject("Stats", Stats::getInstance()); scriptEngine->registerGlobalObject("Settings", SettingsScriptingInterface::getInstance()); scriptEngine->registerGlobalObject("AudioDevice", AudioDeviceScriptingInterface::getInstance()); + scriptEngine->registerGlobalObject("AudioStats", DependencyManager::get()->getStats().data()); // Caches scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get().data()); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 926212cf47..d7574bfa76 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -162,7 +162,7 @@ public slots: void handleSelectedAudioFormat(QSharedPointer message); void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec); - void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); } + void sendDownstreamAudioStatsPacket() { _stats.publish(); } void handleAudioInput(); void handleRecordedAudioInput(const QByteArray& audio); void reset(); diff --git a/libraries/audio-client/src/AudioIOStats.cpp b/libraries/audio-client/src/AudioIOStats.cpp index 330854058f..4a2dd70bff 100644 --- a/libraries/audio-client/src/AudioIOStats.cpp +++ b/libraries/audio-client/src/AudioIOStats.cpp @@ -18,22 +18,24 @@ #include "AudioIOStats.h" -// This is called 5x/sec (see AudioStatsDialog), and we want it to log the last 5s -static const int INPUT_READS_WINDOW = 25; -static const int INPUT_UNPLAYED_WINDOW = 25; -static const int OUTPUT_UNPLAYED_WINDOW = 25; +// This is called 1x/sec (see AudioClient) and we want it to log the last 5s +static const int INPUT_READS_WINDOW = 5; +static const int INPUT_UNPLAYED_WINDOW = 5; +static const int OUTPUT_UNPLAYED_WINDOW = 5; static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000.0f / AudioConstants::NETWORK_FRAME_MSECS); AudioIOStats::AudioIOStats(MixedProcessedAudioStream* receivedAudioStream) : + _interface(new AudioStatsInterface(this)), _receivedAudioStream(receivedAudioStream), - _inputMsRead(0, INPUT_READS_WINDOW), - _inputMsUnplayed(0, INPUT_UNPLAYED_WINDOW), - _outputMsUnplayed(0, OUTPUT_UNPLAYED_WINDOW), + _inputMsRead(1, INPUT_READS_WINDOW), + _inputMsUnplayed(1, INPUT_UNPLAYED_WINDOW), + _outputMsUnplayed(1, OUTPUT_UNPLAYED_WINDOW), _lastSentPacketTime(0), - _packetTimegaps(0, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS) + _packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS) { + } void AudioIOStats::reset() { @@ -44,11 +46,13 @@ void AudioIOStats::reset() { _outputMsUnplayed.reset(); _packetTimegaps.reset(); - _mixerAvatarStreamStats = AudioStreamStats(); - _mixerInjectedStreamStatsMap.clear(); + _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps); + _interface->updateMixerStream(AudioStreamStats()); + _interface->updateClientStream(AudioStreamStats()); + _interface->updateInjectorStreams(QHash()); } -void AudioIOStats::sentPacket() { +void AudioIOStats::sentPacket() const { // first time this is 0 if (_lastSentPacketTime == 0) { _lastSentPacketTime = usecTimestampNow(); @@ -60,37 +64,13 @@ void AudioIOStats::sentPacket() { } } -const MovingMinMaxAvg& AudioIOStats::getInputMsRead() const { - _inputMsRead.currentIntervalComplete(); - return _inputMsRead; -} - -const MovingMinMaxAvg& AudioIOStats::getInputMsUnplayed() const { - _inputMsUnplayed.currentIntervalComplete(); - return _inputMsUnplayed; -} - -const MovingMinMaxAvg& AudioIOStats::getOutputMsUnplayed() const { - _outputMsUnplayed.currentIntervalComplete(); - return _outputMsUnplayed; -} - -const MovingMinMaxAvg& AudioIOStats::getPacketTimegaps() const { - _packetTimegaps.currentIntervalComplete(); - return _packetTimegaps; -} - -const AudioStreamStats AudioIOStats::getMixerDownstreamStats() const { - return _receivedAudioStream->getAudioStreamStats(); -} - void AudioIOStats::processStreamStatsPacket(QSharedPointer message, SharedNodePointer sendingNode) { // parse the appendFlag, clear injected audio stream stats if 0 quint8 appendFlag; message->readPrimitive(&appendFlag); - if (!appendFlag) { - _mixerInjectedStreamStatsMap.clear(); + if (appendFlag == 0) { + _injectorStreams.clear(); } // parse the number of stream stats structs to follow @@ -103,14 +83,18 @@ void AudioIOStats::processStreamStatsPacket(QSharedPointer mess message->readPrimitive(&streamStats); if (streamStats._streamType == PositionalAudioStream::Microphone) { - _mixerAvatarStreamStats = streamStats; + _interface->updateMixerStream(streamStats); } else { - _mixerInjectedStreamStatsMap[streamStats._streamIdentifier] = streamStats; + _injectorStreams[streamStats._streamIdentifier] = streamStats; } } + + if (appendFlag == 2) { + _interface->updateInjectorStreams(_injectorStreams); + } } -void AudioIOStats::sendDownstreamAudioStatsPacket() { +void AudioIOStats::publish() { auto audioIO = DependencyManager::get(); // call _receivedAudioStream's per-second callback @@ -126,6 +110,11 @@ void AudioIOStats::sendDownstreamAudioStatsPacket() { quint16 numStreamStatsToPack = 1; AudioStreamStats stats = _receivedAudioStream->getAudioStreamStats(); + // update the interface + _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps); + _interface->updateClientStream(stats); + + // prepare a packet to the mixer int statsPacketSize = sizeof(appendFlag) + sizeof(numStreamStatsToPack) + sizeof(stats); auto statsPacket = NLPacket::create(PacketType::AudioStreamStats, statsPacketSize); @@ -137,7 +126,63 @@ void AudioIOStats::sendDownstreamAudioStatsPacket() { // pack downstream audio stream stats statsPacket->writePrimitive(stats); - + // send packet nodeList->sendPacket(std::move(statsPacket), *audioMixer); } + +AudioStreamStatsInterface::AudioStreamStatsInterface(QObject* parent) : + QObject(parent) {} + +void AudioStreamStatsInterface::updateStream(const AudioStreamStats& stats) { + lossRate(stats._packetStreamStats.getLostRate()); + lossCount(stats._packetStreamStats._lost); + lossRateWindow(stats._packetStreamWindowStats.getLostRate()); + lossCountWindow(stats._packetStreamWindowStats._lost); + + framesDesired(stats._desiredJitterBufferFrames); + framesAvailable(stats._framesAvailable); + framesAvailableAvg(stats._framesAvailableAverage); + + unplayedMsMax(stats._unplayedMs); + + starveCount(stats._starveCount); + lastStarveDurationCount(stats._consecutiveNotMixedCount); + dropCount(stats._framesDropped); + overflowCount(stats._overflowCount); + + timegapMsMax(stats._timeGapMax / USECS_PER_MSEC); + timegapMsAvg(stats._timeGapAverage / USECS_PER_MSEC); + timegapMsMaxWindow(stats._timeGapWindowMax / USECS_PER_MSEC); + timegapMsAvgWindow(stats._timeGapWindowAverage / USECS_PER_MSEC); +} + +AudioStatsInterface::AudioStatsInterface(QObject* parent) : + QObject(parent), + _client(new AudioStreamStatsInterface(this)), + _mixer(new AudioStreamStatsInterface(this)), + _injectors(new QObject(this)) {} + + +void AudioStatsInterface::updateLocalBuffers(const MovingMinMaxAvg& inputMsRead, + const MovingMinMaxAvg& inputMsUnplayed, + const MovingMinMaxAvg& outputMsUnplayed, + const MovingMinMaxAvg& timegaps) { + if (SharedNodePointer audioNode = DependencyManager::get()->soloNodeOfType(NodeType::AudioMixer)) { + pingMs(audioNode->getPingMs()); + } + + inputReadMsMax(inputMsRead.getWindowMax()); + inputUnplayedMsMax(inputMsUnplayed.getWindowMax()); + outputUnplayedMsMax(outputMsUnplayed.getWindowMax()); + + sentTimegapMsMax(timegaps.getMax() / USECS_PER_MSEC); + sentTimegapMsAvg(timegaps.getAverage() / USECS_PER_MSEC); + sentTimegapMsMaxWindow(timegaps.getWindowMax() / USECS_PER_MSEC); + sentTimegapMsAvgWindow(timegaps.getWindowAverage() / USECS_PER_MSEC); +} + +void AudioStatsInterface::updateInjectorStreams(const QHash& stats) { + // TODO + emit injectorStreamsChanged(); +} diff --git a/libraries/audio-client/src/AudioIOStats.h b/libraries/audio-client/src/AudioIOStats.h index 45217c5af6..1914092b7a 100644 --- a/libraries/audio-client/src/AudioIOStats.h +++ b/libraries/audio-client/src/AudioIOStats.h @@ -22,44 +22,122 @@ class MixedProcessedAudioStream; +#define AUDIO_PROPERTY(TYPE, NAME) \ + Q_PROPERTY(TYPE NAME READ NAME NOTIFY NAME##Changed) \ + public: \ + TYPE NAME() const { return _##NAME; } \ + void NAME(TYPE value) { \ + if (_##NAME != value) { \ + _##NAME = value; \ + emit NAME##Changed(value); \ + } \ + } \ + Q_SIGNAL void NAME##Changed(TYPE value); \ + private: \ + TYPE _##NAME{ (TYPE)0 }; + +class AudioStreamStatsInterface : public QObject { + Q_OBJECT + AUDIO_PROPERTY(float, lossRate) + AUDIO_PROPERTY(float, lossCount) + AUDIO_PROPERTY(float, lossRateWindow) + AUDIO_PROPERTY(float, lossCountWindow) + + AUDIO_PROPERTY(int, framesDesired) + AUDIO_PROPERTY(int, framesAvailable) + AUDIO_PROPERTY(int, framesAvailableAvg) + AUDIO_PROPERTY(float, unplayedMsMax) + + AUDIO_PROPERTY(int, starveCount) + AUDIO_PROPERTY(int, lastStarveDurationCount) + AUDIO_PROPERTY(int, dropCount) + AUDIO_PROPERTY(int, overflowCount) + + AUDIO_PROPERTY(quint64, timegapMsMax) + AUDIO_PROPERTY(quint64, timegapMsAvg) + AUDIO_PROPERTY(quint64, timegapMsMaxWindow) + AUDIO_PROPERTY(quint64, timegapMsAvgWindow) + +public: + void updateStream(const AudioStreamStats& stats); + +private: + friend class AudioStatsInterface; + AudioStreamStatsInterface(QObject* parent); +}; + +class AudioStatsInterface : public QObject { + Q_OBJECT + AUDIO_PROPERTY(float, pingMs); + + AUDIO_PROPERTY(float, inputReadMsMax); + AUDIO_PROPERTY(float, inputUnplayedMsMax); + AUDIO_PROPERTY(float, outputUnplayedMsMax); + + AUDIO_PROPERTY(quint64, sentTimegapMsMax); + AUDIO_PROPERTY(quint64, sentTimegapMsAvg); + AUDIO_PROPERTY(quint64, sentTimegapMsMaxWindow); + AUDIO_PROPERTY(quint64, sentTimegapMsAvgWindow); + + Q_PROPERTY(AudioStreamStatsInterface* mixerStream READ getMixerStream); + Q_PROPERTY(AudioStreamStatsInterface* clientStream READ getClientStream); + Q_PROPERTY(QObject* injectorStreams READ getInjectorStreams NOTIFY injectorStreamsChanged); + +public: + AudioStreamStatsInterface* getMixerStream() const { return _mixer; } + AudioStreamStatsInterface* getClientStream() const { return _client; } + QObject* getInjectorStreams() const { return _injectors; } + + void updateLocalBuffers(const MovingMinMaxAvg& inputMsRead, + const MovingMinMaxAvg& inputMsUnplayed, + const MovingMinMaxAvg& outputMsUnplayed, + const MovingMinMaxAvg& timegaps); + void updateClientStream(const AudioStreamStats& stats) { _client->updateStream(stats); } + void updateMixerStream(const AudioStreamStats& stats) { _mixer->updateStream(stats); } + void updateInjectorStreams(const QHash& stats); + +signals: + void injectorStreamsChanged(); + +private: + friend class AudioIOStats; + AudioStatsInterface(QObject* parent); + AudioStreamStatsInterface* _client; + AudioStreamStatsInterface* _mixer; + QObject* _injectors; +}; + class AudioIOStats : public QObject { Q_OBJECT public: AudioIOStats(MixedProcessedAudioStream* receivedAudioStream); - - void reset(); - - void updateInputMsRead(float ms) { _inputMsRead.update(ms); } - void updateInputMsUnplayed(float ms) { _inputMsUnplayed.update(ms); } - void updateOutputMsUnplayed(float ms) { _outputMsUnplayed.update(ms); } - void sentPacket(); - - const MovingMinMaxAvg& getInputMsRead() const; - const MovingMinMaxAvg& getInputMsUnplayed() const; - const MovingMinMaxAvg& getOutputMsUnplayed() const; - const MovingMinMaxAvg& getPacketTimegaps() const; - const AudioStreamStats getMixerDownstreamStats() const; - const AudioStreamStats& getMixerAvatarStreamStats() const { return _mixerAvatarStreamStats; } - const QHash& getMixerInjectedStreamStatsMap() const { return _mixerInjectedStreamStatsMap; } - - void sendDownstreamAudioStatsPacket(); + void reset(); + + AudioStatsInterface* data() const { return _interface; } + + void updateInputMsRead(float ms) const { _inputMsRead.update(ms); } + void updateInputMsUnplayed(float ms) const { _inputMsUnplayed.update(ms); } + void updateOutputMsUnplayed(float ms) const { _outputMsUnplayed.update(ms); } + void sentPacket() const; + + void publish(); public slots: void processStreamStatsPacket(QSharedPointer message, SharedNodePointer sendingNode); private: - MixedProcessedAudioStream* _receivedAudioStream; + AudioStatsInterface* _interface; mutable MovingMinMaxAvg _inputMsRead; mutable MovingMinMaxAvg _inputMsUnplayed; mutable MovingMinMaxAvg _outputMsUnplayed; - quint64 _lastSentPacketTime; + mutable quint64 _lastSentPacketTime; mutable MovingMinMaxAvg _packetTimegaps; - - AudioStreamStats _mixerAvatarStreamStats; - QHash _mixerInjectedStreamStatsMap; + + MixedProcessedAudioStream* _receivedAudioStream; + QHash _injectorStreams; }; #endif // hifi_AudioIOStats_h diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ec4e724c1b..9b9f649603 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -77,7 +77,8 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::InjectAudio: case PacketType::MicrophoneAudioNoEcho: case PacketType::MicrophoneAudioWithEcho: - return static_cast(AudioVersion::Exactly10msAudioPackets); + case PacketType::AudioStreamStats: + return static_cast(AudioVersion::CurrentVersion); default: return 17; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index aa775b9f53..bb0fac5620 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -223,7 +223,10 @@ enum class DomainListVersion : PacketVersion { enum class AudioVersion : PacketVersion { HasCompressedAudio = 17, CodecNameInAudioPackets, - Exactly10msAudioPackets + Exactly10msAudioPackets, + TerminatingStreamStats, + // add new versions above this line + CurrentVersion }; #endif // hifi_PacketHeaders_h From e6c0baa1ff8a492a964c6f310cfd4e78e8e1328e Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 15:35:45 -0700 Subject: [PATCH 03/25] remake audio stats in qml --- scripts/developer/utilities/audio/Jitter.qml | 36 ++++++++++ .../developer/utilities/audio/MovingValue.qml | 40 ++++++++++++ .../utilities/audio/MovingValuePair.qml | 65 +++++++++++++++++++ scripts/developer/utilities/audio/Section.qml | 56 ++++++++++++++++ scripts/developer/utilities/audio/Stream.qml | 63 ++++++++++++++++++ scripts/developer/utilities/audio/Value.qml | 33 ++++++++++ scripts/developer/utilities/audio/stats.js | 25 +++++++ scripts/developer/utilities/audio/stats.qml | 64 ++++++++++++++++++ 8 files changed, 382 insertions(+) create mode 100644 scripts/developer/utilities/audio/Jitter.qml create mode 100644 scripts/developer/utilities/audio/MovingValue.qml create mode 100644 scripts/developer/utilities/audio/MovingValuePair.qml create mode 100644 scripts/developer/utilities/audio/Section.qml create mode 100644 scripts/developer/utilities/audio/Stream.qml create mode 100644 scripts/developer/utilities/audio/Value.qml create mode 100644 scripts/developer/utilities/audio/stats.js create mode 100644 scripts/developer/utilities/audio/stats.qml diff --git a/scripts/developer/utilities/audio/Jitter.qml b/scripts/developer/utilities/audio/Jitter.qml new file mode 100644 index 0000000000..73a2a05741 --- /dev/null +++ b/scripts/developer/utilities/audio/Jitter.qml @@ -0,0 +1,36 @@ +// +// Jitter.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +ColumnLayout { + property var max + property var avg + property var maxWindow + property var avgWindow + + MovingValuePair { + label: "Total" + label1: "Average" + label2: "Maximum" + source1: avg + source2: max + } + MovingValuePair { + label: "Window" + label1: "Average" + label2: "Maximum" + source1: avgWindow + source2: maxWindow + } +} + diff --git a/scripts/developer/utilities/audio/MovingValue.qml b/scripts/developer/utilities/audio/MovingValue.qml new file mode 100644 index 0000000000..770e585af5 --- /dev/null +++ b/scripts/developer/utilities/audio/MovingValue.qml @@ -0,0 +1,40 @@ +// +// MovingValue.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +RowLayout { + id: value + property string label + property var source + property string unit: "ms" + + width: parent.width + property int dataPixelWidth: 150 + + Label { + Layout.preferredWidth: dataPixelWidth + text: value.label + } + Label { + Layout.preferredWidth: 0 + horizontalAlignment: Text.AlignRight + text: value.source + ' ' + unit + } + Label { + Layout.fillWidth: true + horizontalAlignment: Text.AlignRight + Layout.alignment: Qt.AlignRight + text: "Placeholder" + } +} + diff --git a/scripts/developer/utilities/audio/MovingValuePair.qml b/scripts/developer/utilities/audio/MovingValuePair.qml new file mode 100644 index 0000000000..f043b6348d --- /dev/null +++ b/scripts/developer/utilities/audio/MovingValuePair.qml @@ -0,0 +1,65 @@ +// +// MovingValuePair.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +RowLayout { + id: value + property string label + property string label1 + property string label2 + property var source + property var source1 + property var source2 + property string unit: "ms" + + property int labelPixelWidth: 50 + property int dataPixelWidth: 100 + + Label { + Layout.preferredWidth: labelPixelWidth - value.spacing + text: value.label + } + + ColumnLayout { + RowLayout { + Label { + Layout.preferredWidth: dataPixelWidth + text: value.label1 + } + Label { + Layout.preferredWidth: 0 + horizontalAlignment: Text.AlignRight + text: value.source1 + ' ' + unit + } + } + RowLayout { + Label { + Layout.preferredWidth: dataPixelWidth + text: value.label2 + } + Label { + Layout.preferredWidth: 0 + horizontalAlignment: Text.AlignRight + text: value.source2 + ' ' + unit + } + } + } + Label { + Layout.fillWidth: true + horizontalAlignment: Text.AlignRight + Layout.alignment: Qt.AlignRight + text: "Placeholder" + } + +} + diff --git a/scripts/developer/utilities/audio/Section.qml b/scripts/developer/utilities/audio/Section.qml new file mode 100644 index 0000000000..100d05f6b2 --- /dev/null +++ b/scripts/developer/utilities/audio/Section.qml @@ -0,0 +1,56 @@ +// +// Section.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +Rectangle { + id: section + property string label: "Section" + property string description: "Description" + property alias control : loader.sourceComponent + + width: parent.width + height: content.height + border.width * 2 + content.spacing * 2 + border.color: "black" + border.width: 5 + radius: border.width * 2 + + ColumnLayout { + id: content + x: section.radius; y: section.radius + spacing: section.border.width + width: section.width - 2 * x + + // label + Label { + Layout.alignment: Qt.AlignCenter + text: hoverArea.containsMouse ? section.description : section.label + font.bold: true + + MouseArea { + id: hoverArea + anchors.fill: parent + hoverEnabled: true + } + } + + // spacer + Item { } + + // control + Loader { + id: loader + Layout.preferredWidth: parent.width + } + } +} + diff --git a/scripts/developer/utilities/audio/Stream.qml b/scripts/developer/utilities/audio/Stream.qml new file mode 100644 index 0000000000..c638850243 --- /dev/null +++ b/scripts/developer/utilities/audio/Stream.qml @@ -0,0 +1,63 @@ +// +// Stream.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +ColumnLayout { + property var stream + + Label { + Layout.alignment: Qt.AlignCenter + text: "Ring Buffer" + font.italic: true + } + MovingValue { + label: "Desired" + source: stream.framesDesired + unit: "frames" + } + MovingValue { + label: "Unplayed" + source: stream.unplayedMsMax + } + Value { + label: "Available (avg)" + source: stream.framesAvailable + " (" + stream.framesAvailableAvg + ") frames" + } + + Label { + Layout.alignment: Qt.AlignCenter + text: "Jitter" + font.italic: true + } + Jitter { + max: stream.timegapMsMax + avg: stream.timegapMsAvg + maxWindow: stream.timegapMsMaxWindow + avgWindow: stream.timegapMsAvgWindow + } + + Label { + Layout.alignment: Qt.AlignCenter + text: "Packet Loss" + font.italic: true + } + Value { + label: "Overall" + source: stream.lossRate + "% (" + stream.lossCount + " lost)" + } + Value { + label: "Window" + source: stream.lossRateWindow + "% (" + stream.lossCountWindow + " lost)" + } +} + diff --git a/scripts/developer/utilities/audio/Value.qml b/scripts/developer/utilities/audio/Value.qml new file mode 100644 index 0000000000..cb88ceb1e3 --- /dev/null +++ b/scripts/developer/utilities/audio/Value.qml @@ -0,0 +1,33 @@ +// +// Value.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +RowLayout { + id: value + property string label + property var source + + width: parent.width + property int dataPixelWidth: 150 + + Label { + Layout.preferredWidth: dataPixelWidth + text: value.label + } + Label { + Layout.preferredWidth: 0 + horizontalAlignment: Text.AlignRight + text: value.source + } +} + diff --git a/scripts/developer/utilities/audio/stats.js b/scripts/developer/utilities/audio/stats.js new file mode 100644 index 0000000000..af38092685 --- /dev/null +++ b/scripts/developer/utilities/audio/stats.js @@ -0,0 +1,25 @@ +// +// stats.js +// scripts/developer/utilities/audio +// +// Zach Pomerantz, created on 9/22/2016. +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +var INITIAL_WIDTH = 400; +var INITIAL_OFFSET = 50; + +// Set up the qml ui +var qml = Script.resolvePath('stats.qml'); +var window = new OverlayWindow({ + title: 'Audio Interface Statistics', + source: qml, + width: 400, height: 720 // stats.qml may be too large for some screens +}); +window.setPosition(INITIAL_OFFSET, INITIAL_OFFSET); + +window.closed.connect(function() { Script.stop(); }); + diff --git a/scripts/developer/utilities/audio/stats.qml b/scripts/developer/utilities/audio/stats.qml new file mode 100644 index 0000000000..4698d6b977 --- /dev/null +++ b/scripts/developer/utilities/audio/stats.qml @@ -0,0 +1,64 @@ +// +// stats.qml +// scripts/developer/utilities/audio +// +// Created by Zach Pomerantz on 9/22/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html +// +import QtQuick 2.5 +import QtQuick.Controls 1.4 +import QtQuick.Layouts 1.3 + +Column { + width: parent.width + height: parent.height + + Section { + label: "Latency" + description: "Audio pipeline latency, broken out and summed" + control: ColumnLayout { + MovingValue { label: "Input Read"; source: AudioStats.inputReadMsMax } + MovingValue { label: "Input Ring"; source: AudioStats.inputUnplayedMsMax } + MovingValue { label: "Network (client->mixer)"; source: AudioStats.pingMs / 2 } + MovingValue { label: "Mixer Ring"; source: AudioStats.mixerStream.unplayedMsMax } + MovingValue { label: "Network (mixer->client)"; source: AudioStats.pingMs / 2 } + MovingValue { label: "Output Ring"; source: AudioStats.clientStream.unplayedMsMax } + MovingValue { label: "Output Read"; source: AudioStats.outputUnplayedMsMax } + MovingValue { label: "TOTAL" + source: AudioStats.inputReadMsMax + + AudioStats.inputUnplayedMsMax + + AudioStats.outputUnplayedMsMax + + AudioStats.mixerStream.unplayedMsMax + + AudioStats.clientStream.unplayedMsMax + + AudioStats.pingMs + } + } + } + + Section { + label: "Upstream Jitter" + description: "Timegaps in packets sent to the mixer" + control: Jitter { + max: AudioStats.sentTimegapMsMax + avg: AudioStats.sentTimegapMsAvg + maxWindow: AudioStats.sentTimegapMsMaxWindow + avgWindow: AudioStats.sentTimegapMsAvgWindow + } + } + + Section { + label: "Mixer (upstream)" + description: "This client's remote audio stream, as seen by the server's mixer" + control: Stream { stream: AudioStats.mixerStream } + } + + Section { + label: "Client (downstream)" + description: "This client's received audio stream, between the network and the OS" + control: Stream { stream: AudioStats.clientStream } + } +} + From f5e1d4dd2bcb5b5212f85cfc6d2854f23a7e78d4 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 15:38:21 -0700 Subject: [PATCH 04/25] notify on audio stream changes --- libraries/audio-client/src/AudioIOStats.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/audio-client/src/AudioIOStats.h b/libraries/audio-client/src/AudioIOStats.h index 1914092b7a..da668da1ac 100644 --- a/libraries/audio-client/src/AudioIOStats.h +++ b/libraries/audio-client/src/AudioIOStats.h @@ -79,8 +79,8 @@ class AudioStatsInterface : public QObject { AUDIO_PROPERTY(quint64, sentTimegapMsMaxWindow); AUDIO_PROPERTY(quint64, sentTimegapMsAvgWindow); - Q_PROPERTY(AudioStreamStatsInterface* mixerStream READ getMixerStream); - Q_PROPERTY(AudioStreamStatsInterface* clientStream READ getClientStream); + Q_PROPERTY(AudioStreamStatsInterface* mixerStream READ getMixerStream NOTIFY mixerStreamChanged); + Q_PROPERTY(AudioStreamStatsInterface* clientStream READ getClientStream NOTIFY clientStreamChanged); Q_PROPERTY(QObject* injectorStreams READ getInjectorStreams NOTIFY injectorStreamsChanged); public: @@ -92,11 +92,13 @@ public: const MovingMinMaxAvg& inputMsUnplayed, const MovingMinMaxAvg& outputMsUnplayed, const MovingMinMaxAvg& timegaps); - void updateClientStream(const AudioStreamStats& stats) { _client->updateStream(stats); } - void updateMixerStream(const AudioStreamStats& stats) { _mixer->updateStream(stats); } + void updateMixerStream(const AudioStreamStats& stats) { _mixer->updateStream(stats); emit mixerStreamChanged(); } + void updateClientStream(const AudioStreamStats& stats) { _client->updateStream(stats); emit clientStreamChanged(); } void updateInjectorStreams(const QHash& stats); signals: + void mixerStreamChanged(); + void clientStreamChanged(); void injectorStreamsChanged(); private: From 6035374004794dc44d4dc5be052d219395e8dae2 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 16:33:56 -0700 Subject: [PATCH 05/25] add graphs to audio stats --- scripts/developer/utilities/audio/Jitter.qml | 20 +-- .../developer/utilities/audio/MovingValue.qml | 15 ++- .../utilities/audio/MovingValuePair.qml | 20 +-- scripts/developer/utilities/audio/Stream.qml | 5 + scripts/developer/utilities/audio/stats.js | 2 +- scripts/developer/utilities/audio/stats.qml | 115 +++++++++++------- .../utilities/lib/plotperf/PlotPerf.qml | 13 +- 7 files changed, 126 insertions(+), 64 deletions(-) diff --git a/scripts/developer/utilities/audio/Jitter.qml b/scripts/developer/utilities/audio/Jitter.qml index 73a2a05741..5908c89399 100644 --- a/scripts/developer/utilities/audio/Jitter.qml +++ b/scripts/developer/utilities/audio/Jitter.qml @@ -13,24 +13,28 @@ import QtQuick.Controls 1.4 import QtQuick.Layouts 1.3 ColumnLayout { + id: jitter property var max property var avg property var maxWindow property var avgWindow + property bool showGraphs: false MovingValuePair { label: "Total" - label1: "Average" - label2: "Maximum" - source1: avg - source2: max + label1: "Maximum" + label2: "Average" + source1: max + source2: avg + showGraphs: jitter.showGraphs } MovingValuePair { label: "Window" - label1: "Average" - label2: "Maximum" - source1: avgWindow - source2: maxWindow + label1: "Maximum" + label2: "Average" + source1: maxWindow + source2: avgWindow + showGraphs: jitter.showGraphs } } diff --git a/scripts/developer/utilities/audio/MovingValue.qml b/scripts/developer/utilities/audio/MovingValue.qml index 770e585af5..ebeda3ed73 100644 --- a/scripts/developer/utilities/audio/MovingValue.qml +++ b/scripts/developer/utilities/audio/MovingValue.qml @@ -11,12 +11,14 @@ import QtQuick 2.5 import QtQuick.Controls 1.4 import QtQuick.Layouts 1.3 +import "../lib/plotperf" RowLayout { id: value property string label property var source property string unit: "ms" + property bool showGraphs: false width: parent.width property int dataPixelWidth: 150 @@ -26,15 +28,20 @@ RowLayout { text: value.label } Label { + visible: !value.showGraphs Layout.preferredWidth: 0 horizontalAlignment: Text.AlignRight text: value.source + ' ' + unit } - Label { + PlotPerf { + visible: value.showGraphs Layout.fillWidth: true - horizontalAlignment: Text.AlignRight - Layout.alignment: Qt.AlignRight - text: "Placeholder" + height: 70 + + valueUnit: value.unit + valueNumDigits: 0 + + plots: [{ binding: "source" }] } } diff --git a/scripts/developer/utilities/audio/MovingValuePair.qml b/scripts/developer/utilities/audio/MovingValuePair.qml index f043b6348d..ad7913b34f 100644 --- a/scripts/developer/utilities/audio/MovingValuePair.qml +++ b/scripts/developer/utilities/audio/MovingValuePair.qml @@ -11,16 +11,17 @@ import QtQuick 2.5 import QtQuick.Controls 1.4 import QtQuick.Layouts 1.3 +import "../lib/plotperf" RowLayout { id: value property string label property string label1 property string label2 - property var source property var source1 property var source2 property string unit: "ms" + property bool showGraphs: false property int labelPixelWidth: 50 property int dataPixelWidth: 100 @@ -37,6 +38,7 @@ RowLayout { text: value.label1 } Label { + visible: !value.showGraphs Layout.preferredWidth: 0 horizontalAlignment: Text.AlignRight text: value.source1 + ' ' + unit @@ -48,18 +50,22 @@ RowLayout { text: value.label2 } Label { + visible: !value.showGraphs Layout.preferredWidth: 0 horizontalAlignment: Text.AlignRight text: value.source2 + ' ' + unit } } } - Label { + PlotPerf { + visible: value.showGraphs + Layout.fillWidth: true - horizontalAlignment: Text.AlignRight - Layout.alignment: Qt.AlignRight - text: "Placeholder" + height: 70 + + valueUnit: value.unit + valueNumDigits: 0 + + plots: [{ binding: "source1" }, { binding: "source2" }] } - } - diff --git a/scripts/developer/utilities/audio/Stream.qml b/scripts/developer/utilities/audio/Stream.qml index c638850243..71e4b9eedf 100644 --- a/scripts/developer/utilities/audio/Stream.qml +++ b/scripts/developer/utilities/audio/Stream.qml @@ -13,7 +13,9 @@ import QtQuick.Controls 1.4 import QtQuick.Layouts 1.3 ColumnLayout { + id: root property var stream + property bool showGraphs: false Label { Layout.alignment: Qt.AlignCenter @@ -24,10 +26,12 @@ ColumnLayout { label: "Desired" source: stream.framesDesired unit: "frames" + showGraphs: root.showGraphs } MovingValue { label: "Unplayed" source: stream.unplayedMsMax + showGraphs: root.showGraphs } Value { label: "Available (avg)" @@ -44,6 +48,7 @@ ColumnLayout { avg: stream.timegapMsAvg maxWindow: stream.timegapMsMaxWindow avgWindow: stream.timegapMsAvgWindow + showGraphs: root.showGraphs } Label { diff --git a/scripts/developer/utilities/audio/stats.js b/scripts/developer/utilities/audio/stats.js index af38092685..8d3bf8d637 100644 --- a/scripts/developer/utilities/audio/stats.js +++ b/scripts/developer/utilities/audio/stats.js @@ -17,7 +17,7 @@ var qml = Script.resolvePath('stats.qml'); var window = new OverlayWindow({ title: 'Audio Interface Statistics', source: qml, - width: 400, height: 720 // stats.qml may be too large for some screens + width: 800, height: 720 // stats.qml may be too large for some screens }); window.setPosition(INITIAL_OFFSET, INITIAL_OFFSET); diff --git a/scripts/developer/utilities/audio/stats.qml b/scripts/developer/utilities/audio/stats.qml index 4698d6b977..5faa1a0e60 100644 --- a/scripts/developer/utilities/audio/stats.qml +++ b/scripts/developer/utilities/audio/stats.qml @@ -13,52 +13,85 @@ import QtQuick.Controls 1.4 import QtQuick.Layouts 1.3 Column { + id: stats width: parent.width height: parent.height + property bool showGraphs: toggleGraphs.checked - Section { - label: "Latency" - description: "Audio pipeline latency, broken out and summed" - control: ColumnLayout { - MovingValue { label: "Input Read"; source: AudioStats.inputReadMsMax } - MovingValue { label: "Input Ring"; source: AudioStats.inputUnplayedMsMax } - MovingValue { label: "Network (client->mixer)"; source: AudioStats.pingMs / 2 } - MovingValue { label: "Mixer Ring"; source: AudioStats.mixerStream.unplayedMsMax } - MovingValue { label: "Network (mixer->client)"; source: AudioStats.pingMs / 2 } - MovingValue { label: "Output Ring"; source: AudioStats.clientStream.unplayedMsMax } - MovingValue { label: "Output Read"; source: AudioStats.outputUnplayedMsMax } - MovingValue { label: "TOTAL" - source: AudioStats.inputReadMsMax + - AudioStats.inputUnplayedMsMax + - AudioStats.outputUnplayedMsMax + - AudioStats.mixerStream.unplayedMsMax + - AudioStats.clientStream.unplayedMsMax + - AudioStats.pingMs + RowLayout { + width: parent.width + height: 30 + + Button { + id: toggleGraphs + property bool checked: false + + Layout.alignment: Qt.AlignCenter + + text: checked ? "Hide graphs" : "Show graphs" + onClicked: function() { checked = !checked; } + } + } + + Grid { + width: parent.width + height: parent.height - 30 + + Column { + width: parent.width / 2 + height: parent.height + + Section { + label: "Latency" + description: "Audio pipeline latency, broken out and summed" + control: ColumnLayout { + MovingValue { label: "Input Read"; source: AudioStats.inputReadMsMax; showGraphs: stats.showGraphs } + MovingValue { label: "Input Ring"; source: AudioStats.inputUnplayedMsMax; showGraphs: stats.showGraphs } + MovingValue { label: "Network (client->mixer)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Mixer Ring"; source: AudioStats.mixerStream.unplayedMsMax; showGraphs: stats.showGraphs } + MovingValue { label: "Network (mixer->client)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Output Ring"; source: AudioStats.clientStream.unplayedMsMax; showGraphs: stats.showGraphs } + MovingValue { label: "Output Read"; source: AudioStats.outputUnplayedMsMax; showGraphs: stats.showGraphs } + MovingValue { label: "TOTAL"; showGraphs: stats.showGraphs + source: AudioStats.inputReadMsMax + + AudioStats.inputUnplayedMsMax + + AudioStats.outputUnplayedMsMax + + AudioStats.mixerStream.unplayedMsMax + + AudioStats.clientStream.unplayedMsMax + + AudioStats.pingMs + } + } + } + + Section { + label: "Upstream Jitter" + description: "Timegaps in packets sent to the mixer" + control: Jitter { + max: AudioStats.sentTimegapMsMax + avg: AudioStats.sentTimegapMsAvg + maxWindow: AudioStats.sentTimegapMsMaxWindow + avgWindow: AudioStats.sentTimegapMsAvgWindow + showGraphs: stats.showGraphs + } + } + } + + Column { + width: parent.width / 2 + height: parent.height + + Section { + label: "Mixer (upstream)" + description: "This client's remote audio stream, as seen by the server's mixer" + control: Stream { stream: AudioStats.mixerStream; showGraphs: stats.showGraphs } + } + + Section { + label: "Client (downstream)" + description: "This client's received audio stream, between the network and the OS" + control: Stream { stream: AudioStats.clientStream; showGraphs: stats.showGraphs } } } } - - Section { - label: "Upstream Jitter" - description: "Timegaps in packets sent to the mixer" - control: Jitter { - max: AudioStats.sentTimegapMsMax - avg: AudioStats.sentTimegapMsAvg - maxWindow: AudioStats.sentTimegapMsMaxWindow - avgWindow: AudioStats.sentTimegapMsAvgWindow - } - } - - Section { - label: "Mixer (upstream)" - description: "This client's remote audio stream, as seen by the server's mixer" - control: Stream { stream: AudioStats.mixerStream } - } - - Section { - label: "Client (downstream)" - description: "This client's received audio stream, between the network and the OS" - control: Stream { stream: AudioStats.clientStream } - } } diff --git a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml index c6b95dca30..4edc71c343 100644 --- a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml +++ b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml @@ -55,9 +55,17 @@ Item { function createValues() { for (var i =0; i < plots.length; i++) { var plot = plots[i]; + var object = plot["object"] || root.object; + var value = plot["prop"]; + var isBinding = plot["binding"]; + if (isBinding) { + object = root.parent; + value = isBinding; + } _values.push( { - object: (plot["object"] !== undefined ? plot["object"] : root.object), - value: plot["prop"], + object: object, + value: value, + fromBinding: isBinding, valueMax: 1, numSamplesConstantMax: 0, valueHistory: new Array(), @@ -218,7 +226,6 @@ Item { anchors.rightMargin: -hitboxExtension onClicked: { - print("PerfPlot clicked!") resetMax(); } } From d3132d8746d83513c070ade1c18ef08cc3156453 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 16:50:51 -0700 Subject: [PATCH 06/25] denote terminal audio stats packet --- .../src/audio/AudioMixerClientData.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 42928259f8..a56c140326 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -234,14 +234,21 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& while (numStreamStatsRemaining > 0) { auto statsPacket = NLPacket::create(PacketType::AudioStreamStats); + int numStreamStatsRoomFor = (int)(statsPacket->size() - sizeof(quint8) - sizeof(quint16)) / sizeof(AudioStreamStats); + + // calculate the number of stream stats to follow + quint16 numStreamStatsToPack = std::min(numStreamStatsRemaining, numStreamStatsRoomFor); + + // is this the terminal packet? + if (numStreamStatsRemaining <= numStreamStatsToPack) { + appendFlag = 2; + } + // pack the append flag in this packet statsPacket->writePrimitive(appendFlag); appendFlag = 1; - int numStreamStatsRoomFor = (int)(statsPacket->size() - sizeof(quint8) - sizeof(quint16)) / sizeof(AudioStreamStats); - - // calculate and pack the number of stream stats to follow - quint16 numStreamStatsToPack = std::min(numStreamStatsRemaining, numStreamStatsRoomFor); + // pack the number of stream stats to follow statsPacket->writePrimitive(numStreamStatsToPack); // pack the calculated number of stream stats From dbe3104f09a653d7fcf77f8c069631ff89402629 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 17:23:18 -0700 Subject: [PATCH 07/25] fix -Wreorder --- libraries/audio-client/src/AudioIOStats.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/audio-client/src/AudioIOStats.cpp b/libraries/audio-client/src/AudioIOStats.cpp index 4a2dd70bff..77ce82e565 100644 --- a/libraries/audio-client/src/AudioIOStats.cpp +++ b/libraries/audio-client/src/AudioIOStats.cpp @@ -28,12 +28,12 @@ static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000. AudioIOStats::AudioIOStats(MixedProcessedAudioStream* receivedAudioStream) : _interface(new AudioStatsInterface(this)), - _receivedAudioStream(receivedAudioStream), _inputMsRead(1, INPUT_READS_WINDOW), _inputMsUnplayed(1, INPUT_UNPLAYED_WINDOW), _outputMsUnplayed(1, OUTPUT_UNPLAYED_WINDOW), _lastSentPacketTime(0), - _packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS) + _packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS), + _receivedAudioStream(receivedAudioStream) { } From 055fabf57b9c80b3344b6ff6bd3f7dcd9973bb5c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 17:48:33 -0700 Subject: [PATCH 08/25] fix audio AppendFlag semantics --- assignment-client/src/audio/AudioMixerClientData.cpp | 10 ++++------ libraries/audio-client/src/AudioIOStats.cpp | 6 +++--- libraries/audio/src/AudioStreamStats.h | 7 +++++++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index a56c140326..33fb90c3f4 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -218,12 +218,10 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& auto nodeList = DependencyManager::get(); - // The append flag is a boolean value that will be packed right after the header. The first packet sent - // inside this method will have 0 for this flag, every subsequent packet but the last will have 1 for this flag, - // and the last packet will have 2 for this flag. + // The append flag is a boolean value that will be packed right after the header. // This flag allows the client to know when it has received all stats packets, so it can group any downstream effects, // and clear its cache of injector stream stats; it helps to prevent buildup of dead audio stream stats in the client. - quint8 appendFlag = 0; + quint8 appendFlag = AudioStreamStats::START; auto streamsCopy = getAudioStreams(); @@ -241,12 +239,12 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer& // is this the terminal packet? if (numStreamStatsRemaining <= numStreamStatsToPack) { - appendFlag = 2; + appendFlag |= AudioStreamStats::END; } // pack the append flag in this packet statsPacket->writePrimitive(appendFlag); - appendFlag = 1; + appendFlag = 0; // pack the number of stream stats to follow statsPacket->writePrimitive(numStreamStatsToPack); diff --git a/libraries/audio-client/src/AudioIOStats.cpp b/libraries/audio-client/src/AudioIOStats.cpp index 77ce82e565..0d952fd22c 100644 --- a/libraries/audio-client/src/AudioIOStats.cpp +++ b/libraries/audio-client/src/AudioIOStats.cpp @@ -69,7 +69,7 @@ void AudioIOStats::processStreamStatsPacket(QSharedPointer mess quint8 appendFlag; message->readPrimitive(&appendFlag); - if (appendFlag == 0) { + if (appendFlag & AudioStreamStats::START) { _injectorStreams.clear(); } @@ -89,7 +89,7 @@ void AudioIOStats::processStreamStatsPacket(QSharedPointer mess } } - if (appendFlag == 2) { + if (appendFlag & AudioStreamStats::END) { _interface->updateInjectorStreams(_injectorStreams); } } @@ -106,7 +106,7 @@ void AudioIOStats::publish() { return; } - quint8 appendFlag = 0; + quint8 appendFlag = AudioStreamStats::START | AudioStreamStats::END; quint16 numStreamStatsToPack = 1; AudioStreamStats stats = _receivedAudioStream->getAudioStreamStats(); diff --git a/libraries/audio/src/AudioStreamStats.h b/libraries/audio/src/AudioStreamStats.h index 046c4e5a47..1230c14706 100644 --- a/libraries/audio/src/AudioStreamStats.h +++ b/libraries/audio/src/AudioStreamStats.h @@ -16,6 +16,13 @@ class AudioStreamStats { public: + // Intermediate packets should have no flag set + // Unique packets should have both flags set + enum AppendFlag : quint8 { + START = 1, + END = 2 + }; + AudioStreamStats() : _streamType(-1), _streamIdentifier(), From 96e9399b89c640dabfceb6d99d186256e420e88c Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 18:50:28 -0700 Subject: [PATCH 09/25] start MinMaxAvg at default, not numeric limit --- libraries/shared/src/MovingMinMaxAvg.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 580baf7317..3530709317 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -21,16 +21,16 @@ template class MinMaxAvg { public: MinMaxAvg() - : _min(std::numeric_limits::max()), - _max(std::numeric_limits::min()), + : _min(T()), + _max(T()), _average(0.0), _samples(0), _last(0) {} void reset() { - _min = std::numeric_limits::max(); - _max = std::numeric_limits::min(); + _min = T(); + _max = T(); _average = 0.0; _samples = 0; _last = 0; From efb71c18a161caa40d2a0df7f5181da308bd62fc Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 18:51:10 -0700 Subject: [PATCH 10/25] expose audio injector streams --- libraries/audio-client/src/AudioIOStats.cpp | 27 ++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/libraries/audio-client/src/AudioIOStats.cpp b/libraries/audio-client/src/AudioIOStats.cpp index 0d952fd22c..3bd3f4a47d 100644 --- a/libraries/audio-client/src/AudioIOStats.cpp +++ b/libraries/audio-client/src/AudioIOStats.cpp @@ -183,6 +183,31 @@ void AudioStatsInterface::updateLocalBuffers(const MovingMinMaxAvg& input } void AudioStatsInterface::updateInjectorStreams(const QHash& stats) { - // TODO + // Get existing injectors + auto injectorIds = _injectors->dynamicPropertyNames(); + + // Go over reported injectors + QHash::const_iterator injector = stats.constBegin(); + while (injector != stats.constEnd()) { + const auto id = injector.key().toByteArray(); + // Mark existing injector (those left will be removed) + injectorIds.removeOne(id); + auto injectorProperty = _injectors->property(id); + // Add new injector + if (!injectorProperty.isValid()) { + injectorProperty = QVariant::fromValue(new AudioStreamStatsInterface(this)); + _injectors->setProperty(id, injectorProperty); + } + // Update property with reported injector + injectorProperty.value()->updateStream(injector.value()); + ++injector; + } + + // Remove unreported injectors + for (auto& id : injectorIds) { + _injectors->property(id).value()->deleteLater(); + _injectors->setProperty(id, QVariant()); + } + emit injectorStreamsChanged(); } From f40e46ac9264ac654e40bd0cf12e0beb1c1d43c9 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 19:26:54 -0700 Subject: [PATCH 11/25] limit percentages to two decimals --- scripts/developer/utilities/audio/Stream.qml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/developer/utilities/audio/Stream.qml b/scripts/developer/utilities/audio/Stream.qml index 71e4b9eedf..602bccc222 100644 --- a/scripts/developer/utilities/audio/Stream.qml +++ b/scripts/developer/utilities/audio/Stream.qml @@ -58,11 +58,11 @@ ColumnLayout { } Value { label: "Overall" - source: stream.lossRate + "% (" + stream.lossCount + " lost)" + source: stream.lossRate.toFixed(2) + "% (" + stream.lossCount + " lost)" } Value { label: "Window" - source: stream.lossRateWindow + "% (" + stream.lossCountWindow + " lost)" + source: stream.lossRateWindow.toFixed(2) + "% (" + stream.lossCountWindow + " lost)" } } From 239b0c21810fa58c736b5f12a0f6dcfa41d54efc Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Mon, 26 Sep 2016 19:27:03 -0700 Subject: [PATCH 12/25] fix PlotPerf reset --- scripts/developer/utilities/lib/plotperf/PlotPerf.qml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml index 4edc71c343..e61467ffd7 100644 --- a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml +++ b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml @@ -16,8 +16,6 @@ Item { width: parent.width height: 100 - property int hitboxExtension : 20 - // The title of the graph property string title @@ -218,12 +216,7 @@ Item { MouseArea { id: hitbox - anchors.fill:mycanvas - - anchors.topMargin: -hitboxExtension - anchors.bottomMargin: -hitboxExtension - anchors.leftMargin: -hitboxExtension - anchors.rightMargin: -hitboxExtension + anchors.fill: mycanvas onClicked: { resetMax(); From 6cee47337627fd008d29276080996816f6833a47 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 27 Sep 2016 10:22:32 -0700 Subject: [PATCH 13/25] format and color audio stats --- scripts/developer/utilities/audio/Jitter.qml | 20 +++++++++++-------- .../developer/utilities/audio/MovingValue.qml | 10 ++++++---- .../utilities/audio/MovingValuePair.qml | 18 +++++++++++------ scripts/developer/utilities/audio/Stream.qml | 11 ++++++---- scripts/developer/utilities/audio/Value.qml | 3 +++ scripts/developer/utilities/audio/stats.js | 2 +- scripts/developer/utilities/audio/stats.qml | 6 +++--- 7 files changed, 44 insertions(+), 26 deletions(-) diff --git a/scripts/developer/utilities/audio/Jitter.qml b/scripts/developer/utilities/audio/Jitter.qml index 5908c89399..c513f6f92a 100644 --- a/scripts/developer/utilities/audio/Jitter.qml +++ b/scripts/developer/utilities/audio/Jitter.qml @@ -20,20 +20,24 @@ ColumnLayout { property var avgWindow property bool showGraphs: false - MovingValuePair { - label: "Total" - label1: "Maximum" - label2: "Average" - source1: max - source2: avg - showGraphs: jitter.showGraphs - } MovingValuePair { label: "Window" label1: "Maximum" label2: "Average" source1: maxWindow source2: avgWindow + color1: "red" + color2: "dimgrey" + showGraphs: jitter.showGraphs + } + MovingValuePair { + label: "Overall" + label1: "Maximum" + label2: "Average" + source1: max + source2: avg + color1: "firebrick" + color2: "dimgrey" showGraphs: jitter.showGraphs } } diff --git a/scripts/developer/utilities/audio/MovingValue.qml b/scripts/developer/utilities/audio/MovingValue.qml index ebeda3ed73..9acd95e3a6 100644 --- a/scripts/developer/utilities/audio/MovingValue.qml +++ b/scripts/developer/utilities/audio/MovingValue.qml @@ -19,18 +19,20 @@ RowLayout { property var source property string unit: "ms" property bool showGraphs: false + property color color: "darkslategrey" width: parent.width - property int dataPixelWidth: 150 Label { - Layout.preferredWidth: dataPixelWidth + Layout.preferredWidth: 100 + color: value.color text: value.label } Label { visible: !value.showGraphs - Layout.preferredWidth: 0 + Layout.preferredWidth: 50 horizontalAlignment: Text.AlignRight + color: value.color text: value.source + ' ' + unit } PlotPerf { @@ -41,7 +43,7 @@ RowLayout { valueUnit: value.unit valueNumDigits: 0 - plots: [{ binding: "source" }] + plots: [{ binding: "source", color: value.color }] } } diff --git a/scripts/developer/utilities/audio/MovingValuePair.qml b/scripts/developer/utilities/audio/MovingValuePair.qml index ad7913b34f..3229b9b6dc 100644 --- a/scripts/developer/utilities/audio/MovingValuePair.qml +++ b/scripts/developer/utilities/audio/MovingValuePair.qml @@ -20,6 +20,8 @@ RowLayout { property string label2 property var source1 property var source2 + property color color1 + property color color2 property string unit: "ms" property bool showGraphs: false @@ -27,32 +29,36 @@ RowLayout { property int dataPixelWidth: 100 Label { - Layout.preferredWidth: labelPixelWidth - value.spacing + Layout.preferredWidth: 50 - value.spacing text: value.label } ColumnLayout { RowLayout { Label { - Layout.preferredWidth: dataPixelWidth + Layout.preferredWidth: 50 + color: value.color1 text: value.label1 } Label { visible: !value.showGraphs - Layout.preferredWidth: 0 + Layout.preferredWidth: 50 horizontalAlignment: Text.AlignRight + color: value.color1 text: value.source1 + ' ' + unit } } RowLayout { Label { - Layout.preferredWidth: dataPixelWidth + Layout.preferredWidth: 50 + color: value.color2 text: value.label2 } Label { visible: !value.showGraphs - Layout.preferredWidth: 0 + Layout.preferredWidth: 50 horizontalAlignment: Text.AlignRight + color: value.color2 text: value.source2 + ' ' + unit } } @@ -66,6 +72,6 @@ RowLayout { valueUnit: value.unit valueNumDigits: 0 - plots: [{ binding: "source1" }, { binding: "source2" }] + plots: [{ binding: "source1", color: value.color1 }, { binding: "source2", color: value.color2 }] } } diff --git a/scripts/developer/utilities/audio/Stream.qml b/scripts/developer/utilities/audio/Stream.qml index 602bccc222..936fce220e 100644 --- a/scripts/developer/utilities/audio/Stream.qml +++ b/scripts/developer/utilities/audio/Stream.qml @@ -24,12 +24,14 @@ ColumnLayout { } MovingValue { label: "Desired" + color: "limegreen" source: stream.framesDesired unit: "frames" showGraphs: root.showGraphs } MovingValue { label: "Unplayed" + color: "darkblue" source: stream.unplayedMsMax showGraphs: root.showGraphs } @@ -56,13 +58,14 @@ ColumnLayout { text: "Packet Loss" font.italic: true } - Value { - label: "Overall" - source: stream.lossRate.toFixed(2) + "% (" + stream.lossCount + " lost)" - } Value { label: "Window" source: stream.lossRateWindow.toFixed(2) + "% (" + stream.lossCountWindow + " lost)" } + Value { + label: "Overall" + color: "dimgrey" + source: stream.lossRate.toFixed(2) + "% (" + stream.lossCount + " lost)" + } } diff --git a/scripts/developer/utilities/audio/Value.qml b/scripts/developer/utilities/audio/Value.qml index cb88ceb1e3..70df7695bc 100644 --- a/scripts/developer/utilities/audio/Value.qml +++ b/scripts/developer/utilities/audio/Value.qml @@ -16,17 +16,20 @@ RowLayout { id: value property string label property var source + property color color: "darkslategrey" width: parent.width property int dataPixelWidth: 150 Label { Layout.preferredWidth: dataPixelWidth + color: value.color text: value.label } Label { Layout.preferredWidth: 0 horizontalAlignment: Text.AlignRight + color: value.color text: value.source } } diff --git a/scripts/developer/utilities/audio/stats.js b/scripts/developer/utilities/audio/stats.js index 8d3bf8d637..493271ac99 100644 --- a/scripts/developer/utilities/audio/stats.js +++ b/scripts/developer/utilities/audio/stats.js @@ -17,7 +17,7 @@ var qml = Script.resolvePath('stats.qml'); var window = new OverlayWindow({ title: 'Audio Interface Statistics', source: qml, - width: 800, height: 720 // stats.qml may be too large for some screens + width: 500, height: 520 // stats.qml may be too large for some screens }); window.setPosition(INITIAL_OFFSET, INITIAL_OFFSET); diff --git a/scripts/developer/utilities/audio/stats.qml b/scripts/developer/utilities/audio/stats.qml index 5faa1a0e60..212025c73a 100644 --- a/scripts/developer/utilities/audio/stats.qml +++ b/scripts/developer/utilities/audio/stats.qml @@ -47,12 +47,12 @@ Column { control: ColumnLayout { MovingValue { label: "Input Read"; source: AudioStats.inputReadMsMax; showGraphs: stats.showGraphs } MovingValue { label: "Input Ring"; source: AudioStats.inputUnplayedMsMax; showGraphs: stats.showGraphs } - MovingValue { label: "Network (client->mixer)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Network (up)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } MovingValue { label: "Mixer Ring"; source: AudioStats.mixerStream.unplayedMsMax; showGraphs: stats.showGraphs } - MovingValue { label: "Network (mixer->client)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Network (down)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } MovingValue { label: "Output Ring"; source: AudioStats.clientStream.unplayedMsMax; showGraphs: stats.showGraphs } MovingValue { label: "Output Read"; source: AudioStats.outputUnplayedMsMax; showGraphs: stats.showGraphs } - MovingValue { label: "TOTAL"; showGraphs: stats.showGraphs + MovingValue { label: "TOTAL"; color: "black"; showGraphs: stats.showGraphs source: AudioStats.inputReadMsMax + AudioStats.inputUnplayedMsMax + AudioStats.outputUnplayedMsMax + From c391c26805e13a47c9c63600ab93f2c8312f5af8 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 27 Sep 2016 10:29:35 -0700 Subject: [PATCH 14/25] improve contrast by changing plotperf opacity --- scripts/developer/utilities/audio/Jitter.qml | 4 ++-- scripts/developer/utilities/audio/MovingValue.qml | 1 + scripts/developer/utilities/audio/MovingValuePair.qml | 1 + scripts/developer/utilities/lib/plotperf/PlotPerf.qml | 4 +++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/developer/utilities/audio/Jitter.qml b/scripts/developer/utilities/audio/Jitter.qml index c513f6f92a..04b126584b 100644 --- a/scripts/developer/utilities/audio/Jitter.qml +++ b/scripts/developer/utilities/audio/Jitter.qml @@ -27,7 +27,7 @@ ColumnLayout { source1: maxWindow source2: avgWindow color1: "red" - color2: "dimgrey" + color2: "darkslategrey" showGraphs: jitter.showGraphs } MovingValuePair { @@ -37,7 +37,7 @@ ColumnLayout { source1: max source2: avg color1: "firebrick" - color2: "dimgrey" + color2: "darkslategrey" showGraphs: jitter.showGraphs } } diff --git a/scripts/developer/utilities/audio/MovingValue.qml b/scripts/developer/utilities/audio/MovingValue.qml index 9acd95e3a6..26fed9319d 100644 --- a/scripts/developer/utilities/audio/MovingValue.qml +++ b/scripts/developer/utilities/audio/MovingValue.qml @@ -42,6 +42,7 @@ RowLayout { valueUnit: value.unit valueNumDigits: 0 + backgroundOpacity: 0.2 plots: [{ binding: "source", color: value.color }] } diff --git a/scripts/developer/utilities/audio/MovingValuePair.qml b/scripts/developer/utilities/audio/MovingValuePair.qml index 3229b9b6dc..b24ab658d2 100644 --- a/scripts/developer/utilities/audio/MovingValuePair.qml +++ b/scripts/developer/utilities/audio/MovingValuePair.qml @@ -71,6 +71,7 @@ RowLayout { valueUnit: value.unit valueNumDigits: 0 + backgroundOpacity: 0.2 plots: [{ binding: "source1", color: value.color1 }, { binding: "source2", color: value.color2 }] } diff --git a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml index e61467ffd7..13d9053adf 100644 --- a/scripts/developer/utilities/lib/plotperf/PlotPerf.qml +++ b/scripts/developer/utilities/lib/plotperf/PlotPerf.qml @@ -22,6 +22,8 @@ Item { // The object used as the default source object for the prop plots property var object + property var backgroundOpacity: 0.6 + // Plots is an array of plot descriptor // a default plot descriptor expects the following object: // prop: [ { @@ -185,7 +187,7 @@ Item { ctx.fillText(text, 0, lineHeight); } function displayBackground(ctx) { - ctx.fillStyle = Qt.rgba(0, 0, 0, 0.6); + ctx.fillStyle = Qt.rgba(0, 0, 0, root.backgroundOpacity); ctx.fillRect(0, 0, width, height); ctx.strokeStyle= "grey"; From aacfa7d6c56ec57d9183604c75caf4c67223f07d Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 27 Sep 2016 10:41:00 -0700 Subject: [PATCH 15/25] back out AudioVersion::CurrentVersion --- libraries/networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 9b9f649603..bb4257f5e4 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -78,7 +78,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::MicrophoneAudioNoEcho: case PacketType::MicrophoneAudioWithEcho: case PacketType::AudioStreamStats: - return static_cast(AudioVersion::CurrentVersion); + return static_cast(AudioVersion::TerminatingStreamStats); default: return 17; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index bb0fac5620..6a91b87d55 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -225,8 +225,6 @@ enum class AudioVersion : PacketVersion { CodecNameInAudioPackets, Exactly10msAudioPackets, TerminatingStreamStats, - // add new versions above this line - CurrentVersion }; #endif // hifi_PacketHeaders_h From 247fad5bfe31c0f9a07d0b8dd5d191c58b985925 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 27 Sep 2016 17:24:34 -0700 Subject: [PATCH 16/25] clean up jitter stats --- scripts/developer/utilities/audio/Jitter.qml | 26 ++----- .../developer/utilities/audio/MovingValue.qml | 3 +- .../utilities/audio/MovingValuePair.qml | 78 ------------------- scripts/developer/utilities/audio/Stream.qml | 10 +-- scripts/developer/utilities/audio/stats.qml | 10 +-- 5 files changed, 17 insertions(+), 110 deletions(-) delete mode 100644 scripts/developer/utilities/audio/MovingValuePair.qml diff --git a/scripts/developer/utilities/audio/Jitter.qml b/scripts/developer/utilities/audio/Jitter.qml index 04b126584b..91f197a919 100644 --- a/scripts/developer/utilities/audio/Jitter.qml +++ b/scripts/developer/utilities/audio/Jitter.qml @@ -16,29 +16,17 @@ ColumnLayout { id: jitter property var max property var avg - property var maxWindow - property var avgWindow property bool showGraphs: false - MovingValuePair { - label: "Window" - label1: "Maximum" - label2: "Average" - source1: maxWindow - source2: avgWindow - color1: "red" - color2: "darkslategrey" + MovingValue { + label: "Jitter" + color: "red" + source: max - avg showGraphs: jitter.showGraphs } - MovingValuePair { - label: "Overall" - label1: "Maximum" - label2: "Average" - source1: max - source2: avg - color1: "firebrick" - color2: "darkslategrey" - showGraphs: jitter.showGraphs + Value { + label: "Average" + source: avg } } diff --git a/scripts/developer/utilities/audio/MovingValue.qml b/scripts/developer/utilities/audio/MovingValue.qml index 26fed9319d..bbd9c31d6b 100644 --- a/scripts/developer/utilities/audio/MovingValue.qml +++ b/scripts/developer/utilities/audio/MovingValue.qml @@ -20,6 +20,7 @@ RowLayout { property string unit: "ms" property bool showGraphs: false property color color: "darkslategrey" + property int decimals: 0 width: parent.width @@ -33,7 +34,7 @@ RowLayout { Layout.preferredWidth: 50 horizontalAlignment: Text.AlignRight color: value.color - text: value.source + ' ' + unit + text: value.source.toFixed(decimals) + ' ' + unit } PlotPerf { visible: value.showGraphs diff --git a/scripts/developer/utilities/audio/MovingValuePair.qml b/scripts/developer/utilities/audio/MovingValuePair.qml deleted file mode 100644 index b24ab658d2..0000000000 --- a/scripts/developer/utilities/audio/MovingValuePair.qml +++ /dev/null @@ -1,78 +0,0 @@ -// -// MovingValuePair.qml -// scripts/developer/utilities/audio -// -// Created by Zach Pomerantz on 9/22/2016 -// Copyright 2016 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or https://www.apache.org/licenses/LICENSE-2.0.html -// -import QtQuick 2.5 -import QtQuick.Controls 1.4 -import QtQuick.Layouts 1.3 -import "../lib/plotperf" - -RowLayout { - id: value - property string label - property string label1 - property string label2 - property var source1 - property var source2 - property color color1 - property color color2 - property string unit: "ms" - property bool showGraphs: false - - property int labelPixelWidth: 50 - property int dataPixelWidth: 100 - - Label { - Layout.preferredWidth: 50 - value.spacing - text: value.label - } - - ColumnLayout { - RowLayout { - Label { - Layout.preferredWidth: 50 - color: value.color1 - text: value.label1 - } - Label { - visible: !value.showGraphs - Layout.preferredWidth: 50 - horizontalAlignment: Text.AlignRight - color: value.color1 - text: value.source1 + ' ' + unit - } - } - RowLayout { - Label { - Layout.preferredWidth: 50 - color: value.color2 - text: value.label2 - } - Label { - visible: !value.showGraphs - Layout.preferredWidth: 50 - horizontalAlignment: Text.AlignRight - color: value.color2 - text: value.source2 + ' ' + unit - } - } - } - PlotPerf { - visible: value.showGraphs - - Layout.fillWidth: true - height: 70 - - valueUnit: value.unit - valueNumDigits: 0 - backgroundOpacity: 0.2 - - plots: [{ binding: "source1", color: value.color1 }, { binding: "source2", color: value.color2 }] - } -} diff --git a/scripts/developer/utilities/audio/Stream.qml b/scripts/developer/utilities/audio/Stream.qml index 936fce220e..e9383b627a 100644 --- a/scripts/developer/utilities/audio/Stream.qml +++ b/scripts/developer/utilities/audio/Stream.qml @@ -23,14 +23,14 @@ ColumnLayout { font.italic: true } MovingValue { - label: "Desired" + label: "Minimum Depth" color: "limegreen" source: stream.framesDesired unit: "frames" showGraphs: root.showGraphs } MovingValue { - label: "Unplayed" + label: "Buffer Depth" color: "darkblue" source: stream.unplayedMsMax showGraphs: root.showGraphs @@ -46,10 +46,8 @@ ColumnLayout { font.italic: true } Jitter { - max: stream.timegapMsMax - avg: stream.timegapMsAvg - maxWindow: stream.timegapMsMaxWindow - avgWindow: stream.timegapMsAvgWindow + max: stream.timegapMsMaxWindow + avg: stream.timegapMsAvgWindow showGraphs: root.showGraphs } diff --git a/scripts/developer/utilities/audio/stats.qml b/scripts/developer/utilities/audio/stats.qml index 212025c73a..346e5e3544 100644 --- a/scripts/developer/utilities/audio/stats.qml +++ b/scripts/developer/utilities/audio/stats.qml @@ -47,9 +47,9 @@ Column { control: ColumnLayout { MovingValue { label: "Input Read"; source: AudioStats.inputReadMsMax; showGraphs: stats.showGraphs } MovingValue { label: "Input Ring"; source: AudioStats.inputUnplayedMsMax; showGraphs: stats.showGraphs } - MovingValue { label: "Network (up)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Network (up)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs; decimals: 1 } MovingValue { label: "Mixer Ring"; source: AudioStats.mixerStream.unplayedMsMax; showGraphs: stats.showGraphs } - MovingValue { label: "Network (down)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs } + MovingValue { label: "Network (down)"; source: AudioStats.pingMs / 2; showGraphs: stats.showGraphs; decimals: 1 } MovingValue { label: "Output Ring"; source: AudioStats.clientStream.unplayedMsMax; showGraphs: stats.showGraphs } MovingValue { label: "Output Read"; source: AudioStats.outputUnplayedMsMax; showGraphs: stats.showGraphs } MovingValue { label: "TOTAL"; color: "black"; showGraphs: stats.showGraphs @@ -67,10 +67,8 @@ Column { label: "Upstream Jitter" description: "Timegaps in packets sent to the mixer" control: Jitter { - max: AudioStats.sentTimegapMsMax - avg: AudioStats.sentTimegapMsAvg - maxWindow: AudioStats.sentTimegapMsMaxWindow - avgWindow: AudioStats.sentTimegapMsAvgWindow + max: AudioStats.sentTimegapMsMaxWindow + avg: AudioStats.sentTimegapMsAvgWindow showGraphs: stats.showGraphs } } From 3ad039cadc15e5a0a55f5581fda0663917cbc372 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 18 Oct 2016 13:57:03 -0700 Subject: [PATCH 17/25] Update default system toolbar position --- interface/resources/qml/hifi/Desktop.qml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/resources/qml/hifi/Desktop.qml b/interface/resources/qml/hifi/Desktop.qml index 7e523bbb70..8bb97a3938 100644 --- a/interface/resources/qml/hifi/Desktop.qml +++ b/interface/resources/qml/hifi/Desktop.qml @@ -53,9 +53,10 @@ OriginalDesktop.Desktop { Toolbar { id: sysToolbar; objectName: "com.highfidelity.interface.toolbar.system"; - // Magic: sysToolbar.x and y come from settings, and are bound before the properties specified here are applied. - x: sysToolbar.x; - y: sysToolbar.y; + // These values will be overridden by sysToolbar.x/y if there is a saved position in Settings + // On exit, the sysToolbar position is saved to settings + x: 30 + y: 50 } property var toolbars: (function (map) { // answer dictionary preloaded with sysToolbar map[sysToolbar.objectName] = sysToolbar; From fc7a1c31ec3c25b2707d2b3a9dbcab9542aaf2a9 Mon Sep 17 00:00:00 2001 From: Zach Pomerantz Date: Tue, 18 Oct 2016 14:11:53 -0700 Subject: [PATCH 18/25] Revert "start MinMaxAvg at default, not numeric limit" This reverts commit 96e9399b89c640dabfceb6d99d186256e420e88c. --- libraries/shared/src/MovingMinMaxAvg.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/shared/src/MovingMinMaxAvg.h b/libraries/shared/src/MovingMinMaxAvg.h index 3530709317..580baf7317 100644 --- a/libraries/shared/src/MovingMinMaxAvg.h +++ b/libraries/shared/src/MovingMinMaxAvg.h @@ -21,16 +21,16 @@ template class MinMaxAvg { public: MinMaxAvg() - : _min(T()), - _max(T()), + : _min(std::numeric_limits::max()), + _max(std::numeric_limits::min()), _average(0.0), _samples(0), _last(0) {} void reset() { - _min = T(); - _max = T(); + _min = std::numeric_limits::max(); + _max = std::numeric_limits::min(); _average = 0.0; _samples = 0; _last = 0; From b31acc421a7eb38607910f678a95d082ce81dd8c Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Thu, 20 Oct 2016 10:36:14 -0700 Subject: [PATCH 19/25] fix crash on startup --- libraries/physics/src/CharacterController.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/physics/src/CharacterController.cpp b/libraries/physics/src/CharacterController.cpp index 19207f13ed..5c85f8fc50 100644 --- a/libraries/physics/src/CharacterController.cpp +++ b/libraries/physics/src/CharacterController.cpp @@ -524,7 +524,7 @@ void CharacterController::computeNewVelocity(btScalar dt, glm::vec3& velocity) { } void CharacterController::preSimulation() { - if (_enabled && _dynamicsWorld) { + if (_enabled && _dynamicsWorld && _rigidBody) { quint64 now = usecTimestampNow(); // slam body to where it is supposed to be @@ -632,9 +632,10 @@ void CharacterController::preSimulation() { void CharacterController::postSimulation() { // postSimulation() exists for symmetry and just in case we need to do something here later - - btVector3 velocity = _rigidBody->getLinearVelocity(); - _velocityChange = velocity - _preSimulationVelocity; + if (_enabled && _dynamicsWorld && _rigidBody) { + btVector3 velocity = _rigidBody->getLinearVelocity(); + _velocityChange = velocity - _preSimulationVelocity; + } } From fad0607bf6c66219f80de893c13fde5497fdd764 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 20 Oct 2016 11:49:23 -0700 Subject: [PATCH 20/25] Remove showing of overlays in away.js --- scripts/system/away.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/system/away.js b/scripts/system/away.js index f4a1a22bf6..51853a1607 100644 --- a/scripts/system/away.js +++ b/scripts/system/away.js @@ -54,7 +54,6 @@ var AWAY_INTRO = { var isEnabled = true; var wasMuted; // unknonwn? var isAway = false; // we start in the un-away state -var wasOverlaysVisible = Menu.isOptionChecked("Overlays"); var eventMappingName = "io.highfidelity.away"; // goActive on hand controller button events, too. var eventMapping = Controller.newMapping(eventMappingName); var avatarPosition = MyAvatar.position; @@ -178,12 +177,6 @@ function goAway(fromStartup) { playAwayAnimation(); // animation is still seen by others showOverlay(); - // remember the View > Overlays state... - wasOverlaysVisible = Menu.isOptionChecked("Overlays"); - - // show overlays so that people can see the "Away" message - Menu.setIsOptionChecked("Overlays", true); - // tell the Reticle, we want to stop capturing the mouse until we come back Reticle.allowMouseCapture = false; // Allow users to find their way to other applications, our menus, etc. @@ -233,9 +226,6 @@ function goActive() { hideOverlay(); - // restore overlays state to what it was when we went "away" - Menu.setIsOptionChecked("Overlays", wasOverlaysVisible); - // tell the Reticle, we are ready to capture the mouse again and it should be visible Reticle.allowMouseCapture = true; Reticle.visible = true; From 95236600273be51e903a13496eb8c51e330e6eb3 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Thu, 20 Oct 2016 11:34:40 -0700 Subject: [PATCH 21/25] Fix offscreen QML texture leak, improve texture sharing for same size surfaces --- interface/src/ui/ApplicationOverlay.cpp | 4 +- interface/src/ui/overlays/Web3DOverlay.cpp | 6 +- .../src/RenderableWebEntityItem.cpp | 5 +- libraries/gl/src/gl/OffscreenQmlSurface.cpp | 255 +++++++++++++----- libraries/gl/src/gl/OffscreenQmlSurface.h | 24 +- libraries/gl/src/gl/TextureRecycler.cpp | 91 ------- libraries/gl/src/gl/TextureRecycler.h | 54 ---- libraries/gpu/src/gpu/Forward.h | 1 + libraries/gpu/src/gpu/Texture.cpp | 28 ++ libraries/gpu/src/gpu/Texture.h | 4 +- scripts/developer/tests/webSpawnTool.js | 105 ++++++++ 11 files changed, 344 insertions(+), 233 deletions(-) delete mode 100644 libraries/gl/src/gl/TextureRecycler.cpp delete mode 100644 libraries/gl/src/gl/TextureRecycler.h create mode 100644 scripts/developer/tests/webSpawnTool.js diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index ae27d5be1a..32336fe3be 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -98,9 +98,7 @@ void ApplicationOverlay::renderQmlUi(RenderArgs* renderArgs) { PROFILE_RANGE(__FUNCTION__); if (!_uiTexture) { - _uiTexture = gpu::TexturePointer(gpu::Texture::createExternal2D([](uint32_t recycleTexture, void* recycleFence){ - DependencyManager::get()->releaseTexture({ recycleTexture, recycleFence }); - })); + _uiTexture = gpu::TexturePointer(gpu::Texture::createExternal2D(OffscreenQmlSurface::getDiscardLambda())); _uiTexture->setSource(__FUNCTION__); } // Once we move UI rendering and screen rendering to different diff --git a/interface/src/ui/overlays/Web3DOverlay.cpp b/interface/src/ui/overlays/Web3DOverlay.cpp index b74d1b78ed..773f850e2f 100644 --- a/interface/src/ui/overlays/Web3DOverlay.cpp +++ b/interface/src/ui/overlays/Web3DOverlay.cpp @@ -49,6 +49,8 @@ Web3DOverlay::~Web3DOverlay() { if (_webSurface) { _webSurface->pause(); _webSurface->disconnect(_connection); + + // The lifetime of the QML surface MUST be managed by the main thread // Additionally, we MUST use local variables copied by value, rather than // member variables, since they would implicitly refer to a this that @@ -111,9 +113,7 @@ void Web3DOverlay::render(RenderArgs* args) { if (!_texture) { auto webSurface = _webSurface; - _texture = gpu::TexturePointer(gpu::Texture::createExternal2D([webSurface](uint32_t recycleTexture, void* recycleFence) { - webSurface->releaseTexture({ recycleTexture, recycleFence }); - })); + _texture = gpu::TexturePointer(gpu::Texture::createExternal2D(OffscreenQmlSurface::getDiscardLambda())); _texture->setSource(__FUNCTION__); } OffscreenQmlSurface::TextureAndFence newTextureAndFence; diff --git a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp index a27db728e3..b71fab9439 100644 --- a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp @@ -201,10 +201,7 @@ void RenderableWebEntityItem::render(RenderArgs* args) { if (!_texture) { auto webSurface = _webSurface; - auto recycler = [webSurface] (uint32_t recycleTexture, void* recycleFence) { - webSurface->releaseTexture({ recycleTexture, recycleFence }); - }; - _texture = gpu::TexturePointer(gpu::Texture::createExternal2D(recycler)); + _texture = gpu::TexturePointer(gpu::Texture::createExternal2D(OffscreenQmlSurface::getDiscardLambda())); _texture->setSource(__FUNCTION__); } OffscreenQmlSurface::TextureAndFence newTextureAndFence; diff --git a/libraries/gl/src/gl/OffscreenQmlSurface.cpp b/libraries/gl/src/gl/OffscreenQmlSurface.cpp index 39f46b91d7..4622646f22 100644 --- a/libraries/gl/src/gl/OffscreenQmlSurface.cpp +++ b/libraries/gl/src/gl/OffscreenQmlSurface.cpp @@ -8,9 +8,8 @@ #include "OffscreenQmlSurface.h" #include "Config.h" -#include -#include -#include +#include +#include #include #include @@ -37,9 +36,150 @@ #include "OffscreenGLCanvas.h" #include "GLHelpers.h" #include "GLLogging.h" -#include "TextureRecycler.h" #include "Context.h" +struct TextureSet { + // The number of surfaces with this size + size_t count { 0 }; + std::list returnedTextures; +}; + +uint64_t uvec2ToUint64(const uvec2& v) { + uint64_t result = v.x; + result <<= 32; + result |= v.y; + return result; +} + +class OffscreenTextures { +public: + GLuint getNextTexture(const uvec2& size) { + assert(QThread::currentThread() == qApp->thread()); + assert(textures.count(size)); + + recycle(); + + ++_activeTextureCount; + auto sizeKey = uvec2ToUint64(size); + auto& textureSet = _textures[sizeKey]; + if (!textureSet.returnedTextures.empty()) { + auto textureAndFence = textureSet.returnedTextures.front(); + textureSet.returnedTextures.pop_front(); + waitOnFence(static_cast(textureAndFence.second)); + return textureAndFence.first; + } + + return createTexture(size); + } + + void releaseSize(const uvec2& size) { + assert(QOpenGLContext::currentContext()); + assert(QThread::currentThread() == qApp->thread()); + assert(textures.count(size)); + auto sizeKey = uvec2ToUint64(size); + auto& textureSet = _textures[sizeKey]; + if (0 == --textureSet.count) { + for (const auto& textureAndFence : textureSet.returnedTextures) { + destroy(textureAndFence); + } + _textures.erase(sizeKey); + } + } + + void acquireSize(const uvec2& size) { + assert(QThread::currentThread() == qApp->thread()); + assert(textures.count(size)); + auto sizeKey = uvec2ToUint64(size); + auto& textureSet = _textures[sizeKey]; + ++textureSet.count; + } + + // May be called on any thread + void releaseTexture(const OffscreenQmlSurface::TextureAndFence & textureAndFence) { + --_activeTextureCount; + Lock lock(_mutex); + _returnedTextures.push_back(textureAndFence); + } + + void report() { + uint64_t now = usecTimestampNow(); + if ((now - _lastReport) > USECS_PER_SECOND * 5) { + _lastReport = now; + qCDebug(glLogging) << "Current offscreen texture count " << _allTextureCount; + qCDebug(glLogging) << "Current offscreen active texture count " << _activeTextureCount; + } + } + +private: + static void waitOnFence(GLsync fence) { + glWaitSync(fence, 0, GL_TIMEOUT_IGNORED); + glDeleteSync(fence); + } + + void destroyTexture(GLuint texture) { + --_allTextureCount; + _textureSizes.erase(texture); + glDeleteTextures(1, &texture); + } + + void destroy(const OffscreenQmlSurface::TextureAndFence& textureAndFence) { + waitOnFence(static_cast(textureAndFence.second)); + destroyTexture(textureAndFence.first); + } + + GLuint createTexture(const uvec2& size) { + // Need a new texture + uint32_t newTexture; + glGenTextures(1, &newTexture); + ++_allTextureCount; + _textureSizes[newTexture] = size; + glBindTexture(GL_TEXTURE_2D, newTexture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 8.0f); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_LOD_BIAS, -0.2f); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 8.0f); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, size.x, size.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0); + return newTexture; + } + + void recycle() { + assert(QThread::currentThread() == qApp->thread()); + // First handle any global returns + std::list returnedTextures; + { + Lock lock(_mutex); + returnedTextures.swap(_returnedTextures); + } + + for (auto textureAndFence : returnedTextures) { + GLuint texture = textureAndFence.first; + uvec2 size = _textureSizes[texture]; + auto sizeKey = uvec2ToUint64(size); + // Textures can be returned after all surfaces of the given size have been destroyed, + // in which case we just destroy the texture + if (!_textures.count(sizeKey)) { + destroy(textureAndFence); + continue; + } + _textures[sizeKey].returnedTextures.push_back(textureAndFence); + } + } + + using Mutex = std::mutex; + using Lock = std::unique_lock; + std::atomic _allTextureCount; + std::atomic _activeTextureCount; + std::unordered_map _textures; + std::unordered_map _textureSizes; + Mutex _mutex; + std::list _returnedTextures; + uint64_t _lastReport { 0 }; +} offscreenTextures; class UrlHandler : public QObject { Q_OBJECT @@ -98,28 +238,6 @@ QNetworkAccessManager* QmlNetworkAccessManagerFactory::create(QObject* parent) { Q_DECLARE_LOGGING_CATEGORY(offscreenFocus) Q_LOGGING_CATEGORY(offscreenFocus, "hifi.offscreen.focus") -void OffscreenQmlSurface::setupFbo() { - _canvas->makeCurrent(); - _textures.setSize(_size); - if (_depthStencil) { - glDeleteRenderbuffers(1, &_depthStencil); - _depthStencil = 0; - } - glGenRenderbuffers(1, &_depthStencil); - glBindRenderbuffer(GL_RENDERBUFFER, _depthStencil); - glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, _size.x, _size.y); - - if (_fbo) { - glDeleteFramebuffers(1, &_fbo); - _fbo = 0; - } - glGenFramebuffers(1, &_fbo); - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, _fbo); - glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, _depthStencil); - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); - _canvas->doneCurrent(); -} - void OffscreenQmlSurface::cleanup() { _canvas->makeCurrent(); @@ -134,7 +252,8 @@ void OffscreenQmlSurface::cleanup() { _fbo = 0; } - _textures.clear(); + offscreenTextures.releaseSize(_size); + _canvas->doneCurrent(); } @@ -148,26 +267,7 @@ void OffscreenQmlSurface::render() { _renderControl->sync(); _quickWindow->setRenderTarget(_fbo, QSize(_size.x, _size.y)); - // Clear out any pending textures to be returned - { - std::list returnedTextures; - { - std::unique_lock lock(_textureMutex); - returnedTextures.swap(_returnedTextures); - } - if (!returnedTextures.empty()) { - for (const auto& textureAndFence : returnedTextures) { - GLsync fence = static_cast(textureAndFence.second); - if (fence) { - glWaitSync(fence, 0, GL_TIMEOUT_IGNORED); - glDeleteSync(fence); - } - _textures.recycleTexture(textureAndFence.first); - } - } - } - - GLuint texture = _textures.getNextTexture(); + GLuint texture = offscreenTextures.getNextTexture(_size); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, _fbo); glFramebufferTexture(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, texture, 0); PROFILE_RANGE("qml_render->rendercontrol") @@ -177,12 +277,11 @@ void OffscreenQmlSurface::render() { glGenerateMipmap(GL_TEXTURE_2D); glBindTexture(GL_TEXTURE_2D, 0); + { - std::unique_lock lock(_textureMutex); // If the most recent texture was unused, we can directly recycle it if (_latestTextureAndFence.first) { - _textures.recycleTexture(_latestTextureAndFence.first); - glDeleteSync(static_cast(_latestTextureAndFence.second)); + offscreenTextures.releaseTexture(_latestTextureAndFence); _latestTextureAndFence = { 0, 0 }; } @@ -199,7 +298,6 @@ void OffscreenQmlSurface::render() { bool OffscreenQmlSurface::fetchTexture(TextureAndFence& textureAndFence) { textureAndFence = { 0, 0 }; - std::unique_lock lock(_textureMutex); if (0 == _latestTextureAndFence.first) { return false; } @@ -210,20 +308,18 @@ bool OffscreenQmlSurface::fetchTexture(TextureAndFence& textureAndFence) { return true; } -void OffscreenQmlSurface::releaseTexture(const TextureAndFence& textureAndFence) { - std::unique_lock lock(_textureMutex); - _returnedTextures.push_back(textureAndFence); +std::function OffscreenQmlSurface::getDiscardLambda() { + return [](uint32_t texture, void* fence) { + offscreenTextures.releaseTexture({ texture, static_cast(fence) }); + }; } bool OffscreenQmlSurface::allowNewFrame(uint8_t fps) { // If we already have a pending texture, don't render another one // i.e. don't render faster than the consumer context, since it wastes // GPU cycles on producing output that will never be seen - { - std::unique_lock lock(_textureMutex); - if (0 != _latestTextureAndFence.first) { - return false; - } + if (0 != _latestTextureAndFence.first) { + return false; } auto minRenderInterval = USECS_PER_SECOND / fps; @@ -307,7 +403,6 @@ void OffscreenQmlSurface::create(QOpenGLContext* shareContext) { } _glData = ::getGLContextData(); _renderControl->initialize(_canvas->getContext()); - setupFbo(); // When Quick says there is a need to render, we will not render immediately. Instead, // a timer with a small interval is used to get better performance. @@ -367,9 +462,40 @@ void OffscreenQmlSurface::resize(const QSize& newSize_, bool forceResize) { } qCDebug(glLogging) << "Offscreen UI resizing to " << newSize.width() << "x" << newSize.height() << " with pixel ratio " << pixelRatio; + + _canvas->makeCurrent(); + + // Release hold on the textures of the old size + if (uvec2() != _size) { + // If the most recent texture was unused, we can directly recycle it + if (_latestTextureAndFence.first) { + offscreenTextures.releaseTexture(_latestTextureAndFence); + _latestTextureAndFence = { 0, 0 }; + } + offscreenTextures.releaseSize(_size); + } + _size = newOffscreenSize; - _textures.setSize(_size); - setupFbo(); + + // Acquire the new texture size + if (uvec2() != _size) { + offscreenTextures.acquireSize(_size); + if (_depthStencil) { + glDeleteRenderbuffers(1, &_depthStencil); + _depthStencil = 0; + } + glGenRenderbuffers(1, &_depthStencil); + glBindRenderbuffer(GL_RENDERBUFFER, _depthStencil); + glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, _size.x, _size.y); + if (!_fbo) { + glGenFramebuffers(1, &_fbo); + } + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, _fbo); + glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, _depthStencil); + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); + } + + _canvas->doneCurrent(); } QQuickItem* OffscreenQmlSurface::getRootItem() { @@ -421,7 +547,7 @@ QObject* OffscreenQmlSurface::finishQmlLoad(std::functionisError()) { QList errorList = _qmlComponent->errors(); foreach(const QQmlError& error, errorList) - qWarning() << error.url() << error.line() << error; + qCWarning(glLogging) << error.url() << error.line() << error; if (!_rootItem) { qFatal("Unable to finish loading QML root"); } @@ -474,6 +600,7 @@ QObject* OffscreenQmlSurface::finishQmlLoad(std::function +#include +#include #include #include -#include -#include - +#include +#include #include #include -#include "TextureRecycler.h" class QWindow; class QMyQuickRenderControl; @@ -30,6 +30,11 @@ class QQmlContext; class QQmlComponent; class QQuickWindow; class QQuickItem; + +// GPU resources are typically buffered for one copy being used by the renderer, +// one copy in flight, and one copy being used by the receiver +#define GPU_RESOURCE_BUFFER_SIZE 3 + class OffscreenQmlSurface : public QObject { Q_OBJECT Q_PROPERTY(bool focusText READ isFocusText NOTIFY focusTextChanged) @@ -82,9 +87,8 @@ public: // when the texture is safe to read. // Returns false if no new texture is available bool fetchTexture(TextureAndFence& textureAndFence); - // Release a previously acquired texture, along with a fence which indicates when reads from the - // texture have completed. - void releaseTexture(const TextureAndFence& textureAndFence); + + static std::function getDiscardLambda(); signals: void focusObjectChanged(QObject* newFocus); @@ -133,14 +137,10 @@ private: uint32_t _fbo { 0 }; uint32_t _depthStencil { 0 }; uint64_t _lastRenderTime { 0 }; - uvec2 _size { 1920, 1080 }; - TextureRecycler _textures { true }; + uvec2 _size; // Texture management - std::mutex _textureMutex; TextureAndFence _latestTextureAndFence { 0, 0 }; - std::list _returnedTextures; - bool _render { false }; bool _polish { true }; diff --git a/libraries/gl/src/gl/TextureRecycler.cpp b/libraries/gl/src/gl/TextureRecycler.cpp deleted file mode 100644 index 04cd38f156..0000000000 --- a/libraries/gl/src/gl/TextureRecycler.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// -// Created by Bradley Austin Davis on 2016-10-05 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// -#include "TextureRecycler.h" -#include "Config.h" - -#include - - -void TextureRecycler::setSize(const uvec2& size) { - if (size == _size) { - return; - } - _size = size; - while (!_readyTextures.empty()) { - _readyTextures.pop(); - } - std::set toDelete; - std::for_each(_allTextures.begin(), _allTextures.end(), [&](Map::const_reference item) { - if (!item.second._active && item.second._size != _size) { - toDelete.insert(item.first); - } - }); - std::for_each(toDelete.begin(), toDelete.end(), [&](Map::key_type key) { - _allTextures.erase(key); - }); -} - -void TextureRecycler::clear() { - while (!_readyTextures.empty()) { - _readyTextures.pop(); - } - _allTextures.clear(); -} - -void TextureRecycler::addTexture() { - uint32_t newTexture; - glGenTextures(1, &newTexture); - glBindTexture(GL_TEXTURE_2D, newTexture); - if (_useMipmaps) { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); - } else { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - } - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 8.0f); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_LOD_BIAS, -0.2f); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 8.0f); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, _size.x, _size.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0); - _allTextures.emplace(std::piecewise_construct, std::forward_as_tuple(newTexture), std::forward_as_tuple(newTexture, _size)); - _readyTextures.push(newTexture); -} - -uint32_t TextureRecycler::getNextTexture() { - while (_allTextures.size() < _textureCount) { - addTexture(); - } - - if (_readyTextures.empty()) { - addTexture(); - } - - uint32_t result = _readyTextures.front(); - _readyTextures.pop(); - auto& item = _allTextures[result]; - item._active = true; - return result; -} - -void TextureRecycler::recycleTexture(GLuint texture) { - Q_ASSERT(_allTextures.count(texture)); - auto& item = _allTextures[texture]; - Q_ASSERT(item._active); - item._active = false; - if (item._size != _size) { - // Buh-bye - _allTextures.erase(texture); - return; - } - - _readyTextures.push(item._tex); -} - diff --git a/libraries/gl/src/gl/TextureRecycler.h b/libraries/gl/src/gl/TextureRecycler.h deleted file mode 100644 index 4175dfd201..0000000000 --- a/libraries/gl/src/gl/TextureRecycler.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by Bradley Austin Davis on 2015-04-04 -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// -#pragma once -#ifndef hifi_TextureRecycler_h -#define hifi_TextureRecycler_h - -#include -#include -#include - -#include - -// GPU resources are typically buffered for one copy being used by the renderer, -// one copy in flight, and one copy being used by the receiver -#define GPU_RESOURCE_BUFFER_SIZE 3 - -class TextureRecycler { -public: - TextureRecycler(bool useMipmaps) : _useMipmaps(useMipmaps) {} - void setSize(const uvec2& size); - void setTextureCount(uint8_t textureCount); - void clear(); - uint32_t getNextTexture(); - void recycleTexture(uint32_t texture); - -private: - void addTexture(); - - struct TexInfo { - const uint32_t _tex{ 0 }; - const uvec2 _size; - bool _active { false }; - - TexInfo() {} - TexInfo(uint32_t tex, const uvec2& size) : _tex(tex), _size(size) {} - TexInfo(const TexInfo& other) : _tex(other._tex), _size(other._size) {} - }; - - using Map = std::map; - using Queue = std::queue; - - Map _allTextures; - Queue _readyTextures; - uvec2 _size{ 1920, 1080 }; - bool _useMipmaps; - uint8_t _textureCount { GPU_RESOURCE_BUFFER_SIZE }; -}; - -#endif diff --git a/libraries/gpu/src/gpu/Forward.h b/libraries/gpu/src/gpu/Forward.h index b3e2d6f8a8..f28c18eb11 100644 --- a/libraries/gpu/src/gpu/Forward.h +++ b/libraries/gpu/src/gpu/Forward.h @@ -115,6 +115,7 @@ namespace gpu { GPUObject* getGPUObject() const { return _gpuObject.get(); } friend class Backend; + friend class Texture; }; namespace gl { diff --git a/libraries/gpu/src/gpu/Texture.cpp b/libraries/gpu/src/gpu/Texture.cpp index 924f5145b9..264b729331 100755 --- a/libraries/gpu/src/gpu/Texture.cpp +++ b/libraries/gpu/src/gpu/Texture.cpp @@ -287,6 +287,22 @@ Texture::Texture(): Texture::~Texture() { _textureCPUCount--; + if (getUsage().isExternal()) { + Texture::ExternalUpdates externalUpdates; + { + Lock lock(_externalMutex); + _externalUpdates.swap(externalUpdates); + } + for (const auto& update : externalUpdates) { + assert(_externalRecycler); + _externalRecycler(update.first, update.second); + } + // Force the GL object to be destroyed here + // If we let the normal destructor do it, then it will be + // cleared after the _externalRecycler has been destroyed, + // resulting in leaked texture memory + gpuObject.setGPUObject(nullptr); + } } Texture::Size Texture::resize(Type type, const Element& texelFormat, uint16 width, uint16 height, uint16 depth, uint16 numSamples, uint16 numSlices) { @@ -935,8 +951,20 @@ Vec3u Texture::evalMipDimensions(uint16 level) const { return glm::max(dimensions, Vec3u(1)); } +void Texture::setExternalRecycler(const ExternalRecycler& recycler) { + Lock lock(_externalMutex); + _externalRecycler = recycler; +} + +Texture::ExternalRecycler Texture::getExternalRecycler() const { + Lock lock(_externalMutex); + Texture::ExternalRecycler result = _externalRecycler; + return result; +} + void Texture::setExternalTexture(uint32 externalId, void* externalFence) { Lock lock(_externalMutex); + assert(_externalRecycler); _externalUpdates.push_back({ externalId, externalFence }); } diff --git a/libraries/gpu/src/gpu/Texture.h b/libraries/gpu/src/gpu/Texture.h index 9c3e88c67a..3f0d28b741 100755 --- a/libraries/gpu/src/gpu/Texture.h +++ b/libraries/gpu/src/gpu/Texture.h @@ -466,8 +466,8 @@ public: void notifyMipFaceGPULoaded(uint16 level, uint8 face = 0) const { return _storage->notifyMipFaceGPULoaded(level, face); } void setExternalTexture(uint32 externalId, void* externalFence); - void setExternalRecycler(const ExternalRecycler& recycler) { _externalRecycler = recycler; } - ExternalRecycler getExternalRecycler() const { return _externalRecycler; } + void setExternalRecycler(const ExternalRecycler& recycler); + ExternalRecycler getExternalRecycler() const; const GPUObjectPointer gpuObject {}; diff --git a/scripts/developer/tests/webSpawnTool.js b/scripts/developer/tests/webSpawnTool.js new file mode 100644 index 0000000000..596fb08bde --- /dev/null +++ b/scripts/developer/tests/webSpawnTool.js @@ -0,0 +1,105 @@ +// webSpawnTool.js +// +// Stress tests the rendering of web surfaces over time +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +ENTITY_SPAWNER = function (properties) { + properties = properties || {}; + var RADIUS = properties.radius || 5.0; // Spawn within this radius (square) + var TEST_ENTITY_NAME = properties.entityName || "WebEntitySpawnTest"; + var NUM_ENTITIES = properties.count || 10000; // number of entities to spawn + var ENTITY_SPAWN_LIMIT = properties.spawnLimit || 1; + var ENTITY_SPAWN_INTERVAL = properties.spawnInterval || properties.interval || 2; + var ENTITY_LIFETIME = properties.lifetime || 10; // Entity timeout (when/if we crash, we need the entities to delete themselves) + + function makeEntity(properties) { + var entity = Entities.addEntity(properties); + return { + destroy: function () { + Entities.deleteEntity(entity) + }, + getAge: function () { + return Entities.getEntityProperties(entity).age; + } + }; + } + + function randomPositionXZ(center, radius) { + return { + x: center.x + (Math.random() * radius * 2.0) - radius, + y: center.y, + z: center.z + (Math.random() * radius * 2.0) - radius + }; + } + + var entities = []; + var entitiesToCreate = 0; + var entitiesSpawned = 0; + var spawnTimer = 0.0; + var keepAliveTimer = 0.0; + + function clear () { + var ids = Entities.findEntities(MyAvatar.position, 50); + var that = this; + ids.forEach(function(id) { + var properties = Entities.getEntityProperties(id); + if (properties.name == TEST_ENTITY_NAME) { + Entities.deleteEntity(id); + } + }, this); + } + + function createEntities () { + entitiesToCreate = NUM_ENTITIES; + Script.update.connect(spawnEntities); + } + + function spawnEntities (dt) { + if (entitiesToCreate <= 0) { + Script.update.disconnect(spawnEntities); + print("Finished spawning entities"); + } + else if ((spawnTimer -= dt) < 0.0){ + spawnTimer = ENTITY_SPAWN_INTERVAL; + + var n = Math.min(entitiesToCreate, ENTITY_SPAWN_LIMIT); + print("Spawning " + n + " entities (" + (entitiesSpawned += n) + ")"); + + entitiesToCreate -= n; + + var center = Vec3.sum(MyAvatar.position, Vec3.multiplyQbyV(MyAvatar.orientation, { x: 0, y: 0, z: RADIUS * -1.5 })); + for (; n > 0; --n) { + entities.push(makeEntity({ + type: "Web", + sourceUrl: "https://www.reddit.com/r/random/", + name: TEST_ENTITY_NAME, + position: randomPositionXZ(center, RADIUS), + rotation: MyAvatar.orientation, + dimensions: { x: .8 + Math.random() * 0.8, y: 0.45 + Math.random() * 0.45, z: 0.01 }, + lifetime: ENTITY_LIFETIME + })); + } + } + } + + function despawnEntities () { + print("despawning entities"); + entities.forEach(function (entity) { + entity.destroy(); + }); + entities = []; + } + + function init () { + Script.update.disconnect(init); + clear(); + createEntities(); + Script.scriptEnding.connect(despawnEntities); + } + Script.update.connect(init); +}; + +ENTITY_SPAWNER(); From 53864abaf273b36c144c8b6451ebda1b91d97bd5 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Thu, 20 Oct 2016 13:37:31 -0700 Subject: [PATCH 22/25] edit.js: Fix for picking and manipulating objects via hand controllers. The ray picking was not using the proper grab sphere offset, so it became detached from the visual ray, making picking difficult. --- scripts/system/libraries/entitySelectionTool.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/system/libraries/entitySelectionTool.js b/scripts/system/libraries/entitySelectionTool.js index 9d98a19305..d31f4cb722 100644 --- a/scripts/system/libraries/entitySelectionTool.js +++ b/scripts/system/libraries/entitySelectionTool.js @@ -16,6 +16,8 @@ HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; SPACE_LOCAL = "local"; SPACE_WORLD = "world"; +Script.include("./controllers.js"); + function objectTranslationPlanePoint(position, dimensions) { var newPosition = { x: position.x, y: position.y, z: position.z }; newPosition.y -= dimensions.y / 2.0; @@ -1046,12 +1048,11 @@ SelectionDisplay = (function() { that.triggerMapping.from(Controller.Standard.RT).peek().to(makeTriggerHandler(Controller.Standard.RightHand)); that.triggerMapping.from(Controller.Standard.LT).peek().to(makeTriggerHandler(Controller.Standard.LeftHand)); function controllerComputePickRay() { - var controllerPose = Controller.getPoseValue(activeHand); + var controllerPose = getControllerWorldLocation(activeHand, true); if (controllerPose.valid && that.triggered) { - var controllerPosition = Vec3.sum(Vec3.multiplyQbyV(MyAvatar.orientation, controllerPose.translation), - MyAvatar.position); + var controllerPosition = controllerPose.translation; // This gets point direction right, but if you want general quaternion it would be more complicated: - var controllerDirection = Quat.getUp(Quat.multiply(MyAvatar.orientation, controllerPose.rotation)); + var controllerDirection = Quat.getUp(controllerPose.rotation); return {origin: controllerPosition, direction: controllerDirection}; } } From c27ee634ea4ebc866c912ef01459305395cfae22 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Thu, 20 Oct 2016 13:56:01 -0700 Subject: [PATCH 23/25] Fix debug compile issues --- libraries/gl/src/gl/OffscreenQmlSurface.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/gl/src/gl/OffscreenQmlSurface.cpp b/libraries/gl/src/gl/OffscreenQmlSurface.cpp index 4622646f22..c45a3323db 100644 --- a/libraries/gl/src/gl/OffscreenQmlSurface.cpp +++ b/libraries/gl/src/gl/OffscreenQmlSurface.cpp @@ -55,12 +55,12 @@ class OffscreenTextures { public: GLuint getNextTexture(const uvec2& size) { assert(QThread::currentThread() == qApp->thread()); - assert(textures.count(size)); recycle(); ++_activeTextureCount; auto sizeKey = uvec2ToUint64(size); + assert(_textures.count(sizeKey)); auto& textureSet = _textures[sizeKey]; if (!textureSet.returnedTextures.empty()) { auto textureAndFence = textureSet.returnedTextures.front(); @@ -73,10 +73,9 @@ public: } void releaseSize(const uvec2& size) { - assert(QOpenGLContext::currentContext()); assert(QThread::currentThread() == qApp->thread()); - assert(textures.count(size)); auto sizeKey = uvec2ToUint64(size); + assert(_textures.count(sizeKey)); auto& textureSet = _textures[sizeKey]; if (0 == --textureSet.count) { for (const auto& textureAndFence : textureSet.returnedTextures) { @@ -88,7 +87,6 @@ public: void acquireSize(const uvec2& size) { assert(QThread::currentThread() == qApp->thread()); - assert(textures.count(size)); auto sizeKey = uvec2ToUint64(size); auto& textureSet = _textures[sizeKey]; ++textureSet.count; @@ -241,6 +239,7 @@ Q_LOGGING_CATEGORY(offscreenFocus, "hifi.offscreen.focus") void OffscreenQmlSurface::cleanup() { _canvas->makeCurrent(); + _renderControl->invalidate(); delete _renderControl; // and invalidate if (_depthStencil) { From 014111080524a8f9f093018f2d059044728ae8a8 Mon Sep 17 00:00:00 2001 From: howard-stearns Date: Thu, 20 Oct 2016 15:42:19 -0700 Subject: [PATCH 24/25] fix flicker in system (inside the hud) hand-lasers. --- .../src/display-plugins/hmd/HmdDisplayPlugin.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp index 1c8617fded..a65efefa51 100644 --- a/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp +++ b/libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp @@ -379,10 +379,6 @@ void HmdDisplayPlugin::updateFrameData() { mat4 model = _presentHandPoses[i]; vec3 castStart = vec3(model[3]); vec3 castDirection = glm::quat_cast(model) * laserDirection; - if (glm::abs(glm::length2(castDirection) - 1.0f) > EPSILON) { - castDirection = glm::normalize(castDirection); - castDirection = glm::inverse(_presentUiModelTransform.getRotation()) * castDirection; - } // this offset needs to match GRAB_POINT_SPHERE_OFFSET in scripts/system/libraries/controllers.js:19 static const vec3 GRAB_POINT_SPHERE_OFFSET(0.04f, 0.13f, 0.039f); // x = upward, y = forward, z = lateral From 5886996bc357d8099fcf60aaf8284691dbbb71d3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 20 Oct 2016 16:29:40 -0700 Subject: [PATCH 25/25] always go to dev-welcome when not using stable services --- libraries/networking/src/AddressManager.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/networking/src/AddressManager.h b/libraries/networking/src/AddressManager.h index 1707132c08..06c5a8c554 100644 --- a/libraries/networking/src/AddressManager.h +++ b/libraries/networking/src/AddressManager.h @@ -23,7 +23,13 @@ #include "AccountManager.h" const QString HIFI_URL_SCHEME = "hifi"; + +#if USE_STABLE_GLOBAL_SERVICES const QString DEFAULT_HIFI_ADDRESS = "hifi://welcome"; +#else +const QString DEFAULT_HIFI_ADDRESS = "hifi://dev-welcome"; +#endif + const QString SANDBOX_HIFI_ADDRESS = "hifi://localhost"; const QString INDEX_PATH = "/";