Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-23 01:04:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

added seq number to all Audio types, untested

Browse source
This commit is contained in:
wangyix 2014-06-26 16:52:23 -07:00
parent 97139b0bd0
commit cb48825561
14 changed files with 117 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
assignment-client/src/Agent.cpp
View file

@ -147,6 +147,8 @@ void Agent::readPendingDatagrams() {
}
} else if (datagramPacketType == PacketTypeMixedAudio) {
// TODO: track sequence numbers for mixed audio???
// parse the data and grab the average loudness
_receivedAudioBuffer.parseData(receivedPacket);

26
assignment-client/src/audio/AudioMixer.cpp
View file

@ -519,7 +519,7 @@ void AudioMixer::run() {
QElapsedTimer timer;
timer.start();
char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO
char* clientMixBuffer = new char[NETWORK_BUFFER_LENGTH_BYTES_STEREO + sizeof(quint16)
+ numBytesForPacketHeaderGivenPacketType(PacketTypeMixedAudio)];
int usecToSleep = BUFFER_SEND_INTERVAL_USECS;
@ -602,18 +602,32 @@ void AudioMixer::run() {
foreach (const SharedNodePointer& node, nodeList->getNodeHash()) {
if (node->getType() == NodeType::Agent && node->getActiveSocket() && node->getLinkedData()
&& ((AudioMixerClientData*) node->getLinkedData())->getAvatarAudioRingBuffer()) {
AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData();
prepareMixForListeningNode(node.data());
// pack header
int numBytesPacketHeader = populatePacketHeader(clientMixBuffer, PacketTypeMixedAudio);
char* dataAt = clientMixBuffer + numBytesPacketHeader;
memcpy(clientMixBuffer + numBytesPacketHeader, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
nodeList->writeDatagram(clientMixBuffer, NETWORK_BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader, node);
// pack sequence number
quint16 sequence = nodeData->getOutgoingSequenceNumber();
memcpy(dataAt, &sequence, sizeof(quint16));
dataAt += sizeof(quint16);
// pack mixed audio samples
memcpy(dataAt, _clientSamples, NETWORK_BUFFER_LENGTH_BYTES_STEREO);
dataAt += NETWORK_BUFFER_LENGTH_BYTES_STEREO;
// send mixed audio packet
nodeList->writeDatagram(clientMixBuffer, dataAt - clientMixBuffer, node);
nodeData->incrementOutgoingSequenceNumber();
// send an audio stream stats packet if it's time
if (sendAudioStreamStats) {
int numBytesWritten = ((AudioMixerClientData*)node->getLinkedData())
->encodeAudioStreamStatsPacket(audioStreamStatsPacket);
nodeList->writeDatagram(audioStreamStatsPacket, numBytesWritten, node);
int numBytesAudioStreamStatsPacket = nodeData->encodeAudioStreamStatsPacket(audioStreamStatsPacket);
nodeList->writeDatagram(audioStreamStatsPacket, numBytesAudioStreamStatsPacket, node);
}
++_sumListeners;

12
assignment-client/src/audio/AudioMixerClientData.cpp
View file

@ -20,7 +20,9 @@
#include "AudioMixerClientData.h"
AudioMixerClientData::AudioMixerClientData() :
_ringBuffers()
_ringBuffers(),
_outgoingSequenceNumber(0),
_incomingAvatarSequenceNumberStats()
{
}
@ -49,13 +51,14 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
int numBytesPacketHeader = numBytesForPacketHeader(packet);
const char* sequenceAt = packet.constData() + numBytesPacketHeader;
quint16 sequence = *(reinterpret_cast<const quint16*>(sequenceAt));
_sequenceNumberStats.sequenceNumberReceived(sequence);
PacketType packetType = packetTypeForPacket(packet);
if (packetType == PacketTypeMicrophoneAudioWithEcho
|| packetType == PacketTypeMicrophoneAudioNoEcho
|| packetType == PacketTypeSilentAudioFrame) {
_incomingAvatarSequenceNumberStats.sequenceNumberReceived(sequence);
// grab the AvatarAudioRingBuffer from the vector (or create it if it doesn't exist)
AvatarAudioRingBuffer* avatarRingBuffer = getAvatarAudioRingBuffer();
@ -85,6 +88,8 @@ int AudioMixerClientData::parseData(const QByteArray& packet) {
// grab the stream identifier for this injected audio
QUuid streamIdentifier = QUuid::fromRfc4122(packet.mid(numBytesForPacketHeader(packet), NUM_BYTES_RFC4122_UUID));
_incomingInjectedSequenceNumberStatsMap[streamIdentifier].sequenceNumberReceived(sequence);
InjectedAudioRingBuffer* matchingInjectedRingBuffer = NULL;
for (int i = 0; i < _ringBuffers.size(); i++) {
@ -140,6 +145,9 @@ void AudioMixerClientData::pushBuffersAfterFrameSend() {
} else if (audioBuffer->getType() == PositionalAudioRingBuffer::Injector
&& audioBuffer->hasStarted() && audioBuffer->isStarved()) {
// this is an empty audio buffer that has starved, safe to delete
// also delete its sequence number stats
QUuid streamIdentifier = ((InjectedAudioRingBuffer*)audioBuffer)->getStreamIdentifier();
_incomingInjectedSequenceNumberStatsMap.remove(streamIdentifier);
delete audioBuffer;
i = _ringBuffers.erase(i);
continue;

10
assignment-client/src/audio/AudioMixerClientData.h
View file

@ -38,11 +38,17 @@ public:
QString getJitterBufferStatsString() const;
const SequenceNumberStats& getSequenceNumberStats() const { return _sequenceNumberStats; }
void incrementOutgoingSequenceNumber() { _outgoingSequenceNumber++; }
quint16 getOutgoingSequenceNumber() const { return _outgoingSequenceNumber; }
//const SequenceNumberStats& getIncomingSequenceNumberStats() const { return _incomingSequenceNumberStats; }
private:
QList<PositionalAudioRingBuffer*> _ringBuffers;
SequenceNumberStats _sequenceNumberStats;
quint16 _outgoingSequenceNumber;
SequenceNumberStats _incomingAvatarSequenceNumberStats;
QHash<QUuid, SequenceNumberStats> _incomingInjectedSequenceNumberStatsMap;
};
#endif // hifi_AudioMixerClientData_h

3
interface/src/Application.cpp
View file

@ -3608,6 +3608,9 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript
// when the application is about to quit, stop our script engine so it unwinds properly
connect(this, SIGNAL(aboutToQuit()), scriptEngine, SLOT(stop()));
NodeList* nodeList = NodeList::getInstance();
connect(nodeList, &NodeList::nodeKilled, scriptEngine, &ScriptEngine::nodeKilled);
scriptEngine->moveToThread(workerThread);
// Starts an event loop, and emits workerThread->started()

20
interface/src/Audio.cpp
View file

@ -104,7 +104,7 @@ Audio::Audio(int16_t initialJitterBufferSamples, QObject* parent) :
_scopeOutputLeft(0),
_scopeOutputRight(0),
_audioMixerJitterBufferStats(),
_sequenceNumber(0)
_outgoingSequenceNumber(0)
{
// clear the array of locally injected samples
memset(_localProceduralSamples, 0, NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL);
@ -120,7 +120,7 @@ void Audio::init(QGLWidget *parent) {
void Audio::reset() {
_ringBuffer.reset();
_sequenceNumber = 0;
_outgoingSequenceNumber = 0;
}
QAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName) {
@ -656,8 +656,8 @@ void Audio::handleAudioInput() {
char* currentPacketPtr = audioDataPacket + populatePacketHeader(audioDataPacket, packetType);
// pack seq number
memcpy(currentPacketPtr, &_sequenceNumber, sizeof(quint16));
// pack sequence number
memcpy(currentPacketPtr, &_outgoingSequenceNumber, sizeof(quint16));
currentPacketPtr += sizeof(quint16);
// set the mono/stereo byte
@ -672,13 +672,13 @@ void Audio::handleAudioInput() {
currentPacketPtr += sizeof(headOrientation);
nodeList->writeDatagram(audioDataPacket, numAudioBytes + leadingBytes, audioMixer);
_sequenceNumber++;
_outgoingSequenceNumber++;
Application::getInstance()->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO)
.updateValue(numAudioBytes + leadingBytes);
} else {
// reset seq numbers if there's no connection with an audiomixer
_sequenceNumber = 0;
_outgoingSequenceNumber = 0;
}
delete[] inputAudioSamples;
}
@ -827,6 +827,14 @@ void Audio::toggleStereoInput() {
}
void Audio::processReceivedAudio(const QByteArray& audioByteArray) {
// parse sequence number for this packet
int numBytesPacketHeader = numBytesForPacketHeader(audioByteArray);
const char* sequenceAt = audioByteArray.constData() + numBytesPacketHeader;
quint16 sequence = *((quint16*)sequenceAt);
_incomingSequenceNumberStats.sequenceNumberReceived(sequence);
// parse audio data
_ringBuffer.parseData(audioByteArray);
float networkOutputToOutputRatio = (_desiredOutputFormat.sampleRate() / (float) _outputFormat.sampleRate())

3
interface/src/Audio.h
View file

@ -239,7 +239,8 @@ private:
AudioMixerJitterBuffersStats _audioMixerJitterBufferStats;
quint16 _sequenceNumber;
quint16 _outgoingSequenceNumber;
SequenceNumberStats _incomingSequenceNumberStats;
};

10
libraries/audio/src/AudioInjector.cpp
View file

@ -61,6 +61,11 @@ void AudioInjector::injectAudio() {
QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
// skip sequence number for now
int numPreSequenceNumberBytes = injectAudioPacket.size();
packetStream.skipRawData(sizeof(quint16));
// pack stream identifier (a generated UUID)
packetStream << QUuid::createUuid();
// pack the flag for loopback
@ -91,6 +96,7 @@ void AudioInjector::injectAudio() {
bool shouldLoop = _options.getLoop();
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
quint16 outgoingSequenceNumber = 0;
while (currentSendPosition < soundByteArray.size() && !_shouldStop) {
int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
@ -98,6 +104,9 @@ void AudioInjector::injectAudio() {
// resize the QByteArray to the right size
injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
// pack the sequence number
memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingSequenceNumber, sizeof(quint16));
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy);
@ -107,6 +116,7 @@ void AudioInjector::injectAudio() {
// send off this audio packet
nodeList->writeDatagram(injectAudioPacket, audioMixer);
outgoingSequenceNumber++;
currentSendPosition += bytesToCopy;

1
libraries/audio/src/AudioRingBuffer.cpp
View file

@ -66,6 +66,7 @@ void AudioRingBuffer::resizeForFrameSize(qint64 numFrameSamples) {
}
int AudioRingBuffer::parseData(const QByteArray& packet) {
// skip packet header and sequence number
int numBytesBeforeAudioData = numBytesForPacketHeader(packet) + sizeof(quint16);
return writeData(packet.data() + numBytesBeforeAudioData, packet.size() - numBytesBeforeAudioData);
}

3
libraries/audio/src/PositionalAudioRingBuffer.cpp
View file

@ -107,6 +107,9 @@ int PositionalAudioRingBuffer::parseData(const QByteArray& packet) {
// skip the packet header (includes the source UUID)
int readBytes = numBytesForPacketHeader(packet);
// skip the sequence number
readBytes += sizeof(quint16);
// hop over the channel flag that has already been read in AudioMixerClientData
readBytes += sizeof(quint8);

2
libraries/networking/src/PacketHeaders.cpp
View file

@ -50,6 +50,8 @@ PacketVersion versionForPacketType(PacketType type) {
case PacketTypeMicrophoneAudioNoEcho:
case PacketTypeMicrophoneAudioWithEcho:
case PacketTypeSilentAudioFrame:
return 2;
case PacketTypeMixedAudio:
return 1;
case PacketTypeAvatarData:
return 3;

2
libraries/networking/src/SequenceNumbersStats.cpp
View file

@ -59,7 +59,7 @@ void SequenceNumberStats::sequenceNumberReceived(quint16 incoming, const bool wa
}
} else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) {
// ignore packet if gap is unreasonable
qDebug() << "ignoring unreasonable packet... sequence:" << incoming
qDebug() << "ignoring unreasonable sequence number:" << incoming
<< "previous:" << _lastReceived;
_numUnreasonable++;
return;

49
libraries/script-engine/src/ScriptEngine.cpp
View file

@ -459,32 +459,55 @@ void ScriptEngine::run() {
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
char audioPacket[MAX_PACKET_SIZE];
QDataStream packetStream(&audioPacket, QIODevice::Append);
// pack header
int numBytesPacketHeader = populatePacketHeader(audioPacket, silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
char* dataAt = audioPacket + numBytesPacketHeader;
// skip over sequence number for now; will be packed when destination node is known
char* sequenceAt = dataAt;
dataAt += sizeof(quint16);
// use the orientation and position of this avatar for the source of this audio
packetStream.writeRawData(reinterpret_cast<const char*>(&_avatarData->getPosition()), sizeof(glm::vec3));
memcpy(dataAt, &_avatarData->getPosition(), sizeof(glm::vec3));
dataAt += sizeof(glm::vec3);
glm::quat headOrientation = _avatarData->getHeadOrientation();
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
memcpy(dataAt, &headOrientation, sizeof(glm::quat));
dataAt += sizeof(glm::quat);
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
memcpy(dataAt, &SCRIPT_AUDIO_BUFFER_SAMPLES, sizeof(int16_t));
dataAt += sizeof(int16_t);
} else if (nextSoundOutput) {
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
numAvailableSamples * sizeof(int16_t));
int numAvailableBytes = numAvailableSamples * sizeof(int16_t);
memcpy(dataAt, nextSoundOutput, numAvailableBytes);
dataAt += numAvailableBytes;
}
nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
// write audio packet to AudioMixer nodes
int audioPacketSize = dataAt - audioPacket;
NodeList* nodeList = NodeList::getInstance();
foreach(const SharedNodePointer& node, nodeList->getNodeHash()) {
// only send to nodes of type AudioMixer
if (node->getType() == NodeType::AudioMixer) {
// pack sequence number
quint16 sequence = _outgoingSequenceNumbers[node->getUUID()]++;
memcpy(sequenceAt, &sequence, sizeof(quint16));
// send audio packet
nodeList->writeDatagram(audioPacket, audioPacketSize, node);
}
}
}
}
@ -658,3 +681,7 @@ void ScriptEngine::include(const QString& includeFile) {
emit errorMessage("Uncaught exception at (" + includeFile + ") line" + QString::number(line) + ":" + result.toString());
}
}
void ScriptEngine::nodeKilled(SharedNodePointer node) {
_outgoingSequenceNumbers.remove(node->getUUID());
}

3
libraries/script-engine/src/ScriptEngine.h
View file

@ -100,6 +100,8 @@ public slots:
void include(const QString& includeFile);
void print(const QString& message);
void nodeKilled(SharedNodePointer node);
signals:
void update(float deltaTime);
void scriptEnding();
@ -146,6 +148,7 @@ private:
ScriptUUID _uuidLibrary;
AnimationCache _animationCache;
QHash<QUuid, quint16> _outgoingSequenceNumbers;
};
#endif // hifi_ScriptEngine_h