Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-19 13:43:49 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

expose AudioStats to qml/js

Browse source
This commit is contained in:
Zach Pomerantz 2016-09-23 14:39:28 -07:00
parent 277eefafd7
commit b9c4018b8e
7 changed files with 201 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -49,7 +49,7 @@ AudioMixerClientData::~AudioMixerClientData() {
AvatarAudioStream* AudioMixerClientData::getAvatarAudioStream() {
QReadLocker readLocker { &_streamsLock };
auto it = _audioStreams.find(QUuid());
if (it != _audioStreams.end()) {
return dynamic_cast<AvatarAudioStream*>(it->second.get());
@ -75,7 +75,7 @@ void AudioMixerClientData::removeHRTFForStream(const QUuid& nodeID, const QUuid&
int AudioMixerClientData::parseData(ReceivedMessage& message) {
PacketType packetType = message.getType();
if (packetType == PacketType::AudioStreamStats) {
// skip over header, appendFlag, and num stats packed
@ -219,9 +219,10 @@ void AudioMixerClientData::sendAudioStreamStatsPackets(const SharedNodePointer&
auto nodeList = DependencyManager::get<NodeList>();
// The append flag is a boolean value that will be packed right after the header. The first packet sent
// inside this method will have 0 for this flag, while every subsequent packet will have 1 for this flag.
// The sole purpose of this flag is so the client can clear its map of injected audio stream stats when
// it receives a packet with an appendFlag of 0. This prevents the buildup of dead audio stream stats in the client.
// inside this method will have 0 for this flag, every subsequent packet but the last will have 1 for this flag,
// and the last packet will have 2 for this flag.
// This flag allows the client to know when it has received all stats packets, so it can group any downstream effects,
// and clear its cache of injector stream stats; it helps to prevent buildup of dead audio stream stats in the client.
quint8 appendFlag = 0;
auto streamsCopy = getAudioStreams();

2
interface/src/Application.cpp
View file

@ -1593,6 +1593,7 @@ void Application::initializeUi() {
// though I can't find it. Hence, "ApplicationInterface"
rootContext->setContextProperty("ApplicationInterface", this);
rootContext->setContextProperty("Audio", &AudioScriptingInterface::getInstance());
rootContext->setContextProperty("AudioStats", DependencyManager::get<AudioClient>()->getStats().data());
rootContext->setContextProperty("Controller", DependencyManager::get<controller::ScriptingInterface>().data());
rootContext->setContextProperty("Entities", DependencyManager::get<EntityScriptingInterface>().data());
FileScriptingInterface* fileDownload = new FileScriptingInterface(engine);
@ -4874,6 +4875,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
scriptEngine->registerGlobalObject("Stats", Stats::getInstance());
scriptEngine->registerGlobalObject("Settings", SettingsScriptingInterface::getInstance());
scriptEngine->registerGlobalObject("AudioDevice", AudioDeviceScriptingInterface::getInstance());
scriptEngine->registerGlobalObject("AudioStats", DependencyManager::get<AudioClient>()->getStats().data());
// Caches
scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get<AnimationCache>().data());

2
libraries/audio-client/src/AudioClient.h
View file

@ -162,7 +162,7 @@ public slots:
void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);
void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec);
void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); }
void sendDownstreamAudioStatsPacket() { _stats.publish(); }
void handleAudioInput();
void handleRecordedAudioInput(const QByteArray& audio);
void reset();

127
libraries/audio-client/src/AudioIOStats.cpp
View file

@ -18,22 +18,24 @@
#include "AudioIOStats.h"
// This is called 5x/sec (see AudioStatsDialog), and we want it to log the last 5s
static const int INPUT_READS_WINDOW = 25;
static const int INPUT_UNPLAYED_WINDOW = 25;
static const int OUTPUT_UNPLAYED_WINDOW = 25;
// This is called 1x/sec (see AudioClient) and we want it to log the last 5s
static const int INPUT_READS_WINDOW = 5;
static const int INPUT_UNPLAYED_WINDOW = 5;
static const int OUTPUT_UNPLAYED_WINDOW = 5;
static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000.0f / AudioConstants::NETWORK_FRAME_MSECS);
AudioIOStats::AudioIOStats(MixedProcessedAudioStream* receivedAudioStream) :
_interface(new AudioStatsInterface(this)),
_receivedAudioStream(receivedAudioStream),
_inputMsRead(0, INPUT_READS_WINDOW),
_inputMsUnplayed(0, INPUT_UNPLAYED_WINDOW),
_outputMsUnplayed(0, OUTPUT_UNPLAYED_WINDOW),
_inputMsRead(1, INPUT_READS_WINDOW),
_inputMsUnplayed(1, INPUT_UNPLAYED_WINDOW),
_outputMsUnplayed(1, OUTPUT_UNPLAYED_WINDOW),
_lastSentPacketTime(0),
_packetTimegaps(0, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS)
_packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS)
{
}
void AudioIOStats::reset() {
@ -44,11 +46,13 @@ void AudioIOStats::reset() {
_outputMsUnplayed.reset();
_packetTimegaps.reset();
_mixerAvatarStreamStats = AudioStreamStats();
_mixerInjectedStreamStatsMap.clear();
_interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps);
_interface->updateMixerStream(AudioStreamStats());
_interface->updateClientStream(AudioStreamStats());
_interface->updateInjectorStreams(QHash<QUuid, AudioStreamStats>());
}
void AudioIOStats::sentPacket() {
void AudioIOStats::sentPacket() const {
// first time this is 0
if (_lastSentPacketTime == 0) {
_lastSentPacketTime = usecTimestampNow();
@ -60,37 +64,13 @@ void AudioIOStats::sentPacket() {
}
}
const MovingMinMaxAvg<float>& AudioIOStats::getInputMsRead() const {
_inputMsRead.currentIntervalComplete();
return _inputMsRead;
}
const MovingMinMaxAvg<float>& AudioIOStats::getInputMsUnplayed() const {
_inputMsUnplayed.currentIntervalComplete();
return _inputMsUnplayed;
}
const MovingMinMaxAvg<float>& AudioIOStats::getOutputMsUnplayed() const {
_outputMsUnplayed.currentIntervalComplete();
return _outputMsUnplayed;
}
const MovingMinMaxAvg<quint64>& AudioIOStats::getPacketTimegaps() const {
_packetTimegaps.currentIntervalComplete();
return _packetTimegaps;
}
const AudioStreamStats AudioIOStats::getMixerDownstreamStats() const {
return _receivedAudioStream->getAudioStreamStats();
}
void AudioIOStats::processStreamStatsPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
// parse the appendFlag, clear injected audio stream stats if 0
quint8 appendFlag;
message->readPrimitive(&appendFlag);
if (!appendFlag) {
_mixerInjectedStreamStatsMap.clear();
if (appendFlag == 0) {
_injectorStreams.clear();
}
// parse the number of stream stats structs to follow
@ -103,14 +83,18 @@ void AudioIOStats::processStreamStatsPacket(QSharedPointer<ReceivedMessage> mess
message->readPrimitive(&streamStats);
if (streamStats._streamType == PositionalAudioStream::Microphone) {
_mixerAvatarStreamStats = streamStats;
_interface->updateMixerStream(streamStats);
} else {
_mixerInjectedStreamStatsMap[streamStats._streamIdentifier] = streamStats;
_injectorStreams[streamStats._streamIdentifier] = streamStats;
}
}
if (appendFlag == 2) {
_interface->updateInjectorStreams(_injectorStreams);
}
}
void AudioIOStats::sendDownstreamAudioStatsPacket() {
void AudioIOStats::publish() {
auto audioIO = DependencyManager::get<AudioClient>();
// call _receivedAudioStream's per-second callback
@ -126,6 +110,11 @@ void AudioIOStats::sendDownstreamAudioStatsPacket() {
quint16 numStreamStatsToPack = 1;
AudioStreamStats stats = _receivedAudioStream->getAudioStreamStats();
// update the interface
_interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps);
_interface->updateClientStream(stats);
// prepare a packet to the mixer
int statsPacketSize = sizeof(appendFlag) + sizeof(numStreamStatsToPack) + sizeof(stats);
auto statsPacket = NLPacket::create(PacketType::AudioStreamStats, statsPacketSize);
@ -137,7 +126,63 @@ void AudioIOStats::sendDownstreamAudioStatsPacket() {
// pack downstream audio stream stats
statsPacket->writePrimitive(stats);
// send packet
nodeList->sendPacket(std::move(statsPacket), *audioMixer);
}
AudioStreamStatsInterface::AudioStreamStatsInterface(QObject* parent) :
QObject(parent) {}
void AudioStreamStatsInterface::updateStream(const AudioStreamStats& stats) {
lossRate(stats._packetStreamStats.getLostRate());
lossCount(stats._packetStreamStats._lost);
lossRateWindow(stats._packetStreamWindowStats.getLostRate());
lossCountWindow(stats._packetStreamWindowStats._lost);
framesDesired(stats._desiredJitterBufferFrames);
framesAvailable(stats._framesAvailable);
framesAvailableAvg(stats._framesAvailableAverage);
unplayedMsMax(stats._unplayedMs);
starveCount(stats._starveCount);
lastStarveDurationCount(stats._consecutiveNotMixedCount);
dropCount(stats._framesDropped);
overflowCount(stats._overflowCount);
timegapMsMax(stats._timeGapMax / USECS_PER_MSEC);
timegapMsAvg(stats._timeGapAverage / USECS_PER_MSEC);
timegapMsMaxWindow(stats._timeGapWindowMax / USECS_PER_MSEC);
timegapMsAvgWindow(stats._timeGapWindowAverage / USECS_PER_MSEC);
}
AudioStatsInterface::AudioStatsInterface(QObject* parent) :
QObject(parent),
_client(new AudioStreamStatsInterface(this)),
_mixer(new AudioStreamStatsInterface(this)),
_injectors(new QObject(this)) {}
void AudioStatsInterface::updateLocalBuffers(const MovingMinMaxAvg<float>& inputMsRead,
const MovingMinMaxAvg<float>& inputMsUnplayed,
const MovingMinMaxAvg<float>& outputMsUnplayed,
const MovingMinMaxAvg<quint64>& timegaps) {
if (SharedNodePointer audioNode = DependencyManager::get<NodeList>()->soloNodeOfType(NodeType::AudioMixer)) {
pingMs(audioNode->getPingMs());
}
inputReadMsMax(inputMsRead.getWindowMax());
inputUnplayedMsMax(inputMsUnplayed.getWindowMax());
outputUnplayedMsMax(outputMsUnplayed.getWindowMax());
sentTimegapMsMax(timegaps.getMax() / USECS_PER_MSEC);
sentTimegapMsAvg(timegaps.getAverage() / USECS_PER_MSEC);
sentTimegapMsMaxWindow(timegaps.getWindowMax() / USECS_PER_MSEC);
sentTimegapMsAvgWindow(timegaps.getWindowAverage() / USECS_PER_MSEC);
}
void AudioStatsInterface::updateInjectorStreams(const QHash<QUuid, AudioStreamStats>& stats) {
// TODO
emit injectorStreamsChanged();
}

122
libraries/audio-client/src/AudioIOStats.h
View file

@ -22,44 +22,122 @@
class MixedProcessedAudioStream;
#define AUDIO_PROPERTY(TYPE, NAME) \
Q_PROPERTY(TYPE NAME READ NAME NOTIFY NAME##Changed) \
public: \
TYPE NAME() const { return _##NAME; } \
void NAME(TYPE value) { \
if (_##NAME != value) { \
_##NAME = value; \
emit NAME##Changed(value); \
} \
} \
Q_SIGNAL void NAME##Changed(TYPE value); \
private: \
TYPE _##NAME{ (TYPE)0 };
class AudioStreamStatsInterface : public QObject {
Q_OBJECT
AUDIO_PROPERTY(float, lossRate)
AUDIO_PROPERTY(float, lossCount)
AUDIO_PROPERTY(float, lossRateWindow)
AUDIO_PROPERTY(float, lossCountWindow)
AUDIO_PROPERTY(int, framesDesired)
AUDIO_PROPERTY(int, framesAvailable)
AUDIO_PROPERTY(int, framesAvailableAvg)
AUDIO_PROPERTY(float, unplayedMsMax)
AUDIO_PROPERTY(int, starveCount)
AUDIO_PROPERTY(int, lastStarveDurationCount)
AUDIO_PROPERTY(int, dropCount)
AUDIO_PROPERTY(int, overflowCount)
AUDIO_PROPERTY(quint64, timegapMsMax)
AUDIO_PROPERTY(quint64, timegapMsAvg)
AUDIO_PROPERTY(quint64, timegapMsMaxWindow)
AUDIO_PROPERTY(quint64, timegapMsAvgWindow)
public:
void updateStream(const AudioStreamStats& stats);
private:
friend class AudioStatsInterface;
AudioStreamStatsInterface(QObject* parent);
};
class AudioStatsInterface : public QObject {
Q_OBJECT
AUDIO_PROPERTY(float, pingMs);
AUDIO_PROPERTY(float, inputReadMsMax);
AUDIO_PROPERTY(float, inputUnplayedMsMax);
AUDIO_PROPERTY(float, outputUnplayedMsMax);
AUDIO_PROPERTY(quint64, sentTimegapMsMax);
AUDIO_PROPERTY(quint64, sentTimegapMsAvg);
AUDIO_PROPERTY(quint64, sentTimegapMsMaxWindow);
AUDIO_PROPERTY(quint64, sentTimegapMsAvgWindow);
Q_PROPERTY(AudioStreamStatsInterface* mixerStream READ getMixerStream);
Q_PROPERTY(AudioStreamStatsInterface* clientStream READ getClientStream);
Q_PROPERTY(QObject* injectorStreams READ getInjectorStreams NOTIFY injectorStreamsChanged);
public:
AudioStreamStatsInterface* getMixerStream() const { return _mixer; }
AudioStreamStatsInterface* getClientStream() const { return _client; }
QObject* getInjectorStreams() const { return _injectors; }
void updateLocalBuffers(const MovingMinMaxAvg<float>& inputMsRead,
const MovingMinMaxAvg<float>& inputMsUnplayed,
const MovingMinMaxAvg<float>& outputMsUnplayed,
const MovingMinMaxAvg<quint64>& timegaps);
void updateClientStream(const AudioStreamStats& stats) { _client->updateStream(stats); }
void updateMixerStream(const AudioStreamStats& stats) { _mixer->updateStream(stats); }
void updateInjectorStreams(const QHash<QUuid, AudioStreamStats>& stats);
signals:
void injectorStreamsChanged();
private:
friend class AudioIOStats;
AudioStatsInterface(QObject* parent);
AudioStreamStatsInterface* _client;
AudioStreamStatsInterface* _mixer;
QObject* _injectors;
};
class AudioIOStats : public QObject {
Q_OBJECT
public:
AudioIOStats(MixedProcessedAudioStream* receivedAudioStream);
void reset();
void updateInputMsRead(float ms) { _inputMsRead.update(ms); }
void updateInputMsUnplayed(float ms) { _inputMsUnplayed.update(ms); }
void updateOutputMsUnplayed(float ms) { _outputMsUnplayed.update(ms); }
void sentPacket();
const MovingMinMaxAvg<float>& getInputMsRead() const;
const MovingMinMaxAvg<float>& getInputMsUnplayed() const;
const MovingMinMaxAvg<float>& getOutputMsUnplayed() const;
const MovingMinMaxAvg<quint64>& getPacketTimegaps() const;
const AudioStreamStats getMixerDownstreamStats() const;
const AudioStreamStats& getMixerAvatarStreamStats() const { return _mixerAvatarStreamStats; }
const QHash<QUuid, AudioStreamStats>& getMixerInjectedStreamStatsMap() const { return _mixerInjectedStreamStatsMap; }
void sendDownstreamAudioStatsPacket();
void reset();
AudioStatsInterface* data() const { return _interface; }
void updateInputMsRead(float ms) const { _inputMsRead.update(ms); }
void updateInputMsUnplayed(float ms) const { _inputMsUnplayed.update(ms); }
void updateOutputMsUnplayed(float ms) const { _outputMsUnplayed.update(ms); }
void sentPacket() const;
void publish();
public slots:
void processStreamStatsPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode);
private:
MixedProcessedAudioStream* _receivedAudioStream;
AudioStatsInterface* _interface;
mutable MovingMinMaxAvg<float> _inputMsRead;
mutable MovingMinMaxAvg<float> _inputMsUnplayed;
mutable MovingMinMaxAvg<float> _outputMsUnplayed;
quint64 _lastSentPacketTime;
mutable quint64 _lastSentPacketTime;
mutable MovingMinMaxAvg<quint64> _packetTimegaps;
AudioStreamStats _mixerAvatarStreamStats;
QHash<QUuid, AudioStreamStats> _mixerInjectedStreamStatsMap;
MixedProcessedAudioStream* _receivedAudioStream;
QHash<QUuid, AudioStreamStats> _injectorStreams;
};
#endif // hifi_AudioIOStats_h

3
libraries/networking/src/udt/PacketHeaders.cpp
View file

@ -77,7 +77,8 @@ PacketVersion versionForPacketType(PacketType packetType) {
case PacketType::InjectAudio:
case PacketType::MicrophoneAudioNoEcho:
case PacketType::MicrophoneAudioWithEcho:
return static_cast<PacketVersion>(AudioVersion::Exactly10msAudioPackets);
case PacketType::AudioStreamStats:
return static_cast<PacketVersion>(AudioVersion::CurrentVersion);
default:
return 17;

5
libraries/networking/src/udt/PacketHeaders.h
View file

@ -223,7 +223,10 @@ enum class DomainListVersion : PacketVersion {
enum class AudioVersion : PacketVersion {
HasCompressedAudio = 17,
CodecNameInAudioPackets,
Exactly10msAudioPackets
Exactly10msAudioPackets,
TerminatingStreamStats,
// add new versions above this line
CurrentVersion
};
#endif // hifi_PacketHeaders_h