Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-16 15:30:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

allow Agents to microphone audio from Sound objects

Browse source
This commit is contained in:
Stephen Birarda 2014-03-19 16:14:58 -07:00
parent 08c5c49891
commit 8c4fad443f
4 changed files with 67 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
assignment-client/src/Agent.cpp
View file

@ -26,8 +26,7 @@
Agent::Agent(const QByteArray& packet) :
ThreadedAssignment(packet),
_voxelEditSender(),
_particleEditSender(),
_avatarAudioStream(NULL)
_particleEditSender()
{
// be the parent of the script engine so it gets moved when we do
_scriptEngine.setParent(this);
@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) :
_scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender);
}
Agent::~Agent() {
delete _avatarAudioStream;
}
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) {
if (sendAvatarAudioStream) {
// the agentAudioStream number of samples is related to the ScriptEngine callback rate
_avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES];
// fill the _audioStream with zeroes to start
memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t));
_scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES);
_scriptEngine.setAvatarAudioBuffer(_avatarAudioStream);
} else {
delete _avatarAudioStream;
_avatarAudioStream = NULL;
_scriptEngine.setAvatarAudioBuffer(NULL);
}
}
void Agent::readPendingDatagrams() {
QByteArray receivedPacket;
HifiSockAddr senderSockAddr;

14
assignment-client/src/Agent.h
View file

@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment {
Q_OBJECT
Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar)
Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream)
Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound)
Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream)
public:
Agent(const QByteArray& packet);
~Agent();
void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); }
bool isAvatar() const { return _scriptEngine.isAvatar(); }
void setSendAvatarAudioStream(bool sendAvatarAudioStream);
bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); }
bool isPlayingAvatarSound() const { return _scriptEngine.isPlayingAvatarSound(); }
bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); }
void setIsListeningToAudioStream(bool isListeningToAudioStream)
{ _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); }
public slots:
void run();
void readPendingDatagrams();
void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); }
private:
ScriptEngine _scriptEngine;
@ -50,8 +54,6 @@ private:
ParticleTreeHeadlessViewer _particleViewer;
VoxelTreeHeadlessViewer _voxelViewer;
int16_t* _avatarAudioStream;
};
#endif /* defined(__hifi__Agent__) */

65
libraries/script-engine/src/ScriptEngine.cpp
View file

@ -14,6 +14,7 @@
#include <QtNetwork/QNetworkRequest>
#include <QtNetwork/QNetworkReply>
#include <AudioRingBuffer.h>
#include <AvatarData.h>
#include <NodeList.h>
#include <PacketHeaders.h>
@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co
_avatarIdentityTimer(NULL),
_avatarBillboardTimer(NULL),
_timerFunctionMap(),
_avatarAudioBuffer(NULL),
_isListeningToAudioStream(false),
_avatarSound(NULL),
_numAvatarSoundSentBytes(0),
_controllerScriptingInterface(controllerScriptingInterface),
_avatarData(NULL),
_wantMenuItems(wantMenuItems),
@ -260,27 +263,55 @@ void ScriptEngine::run() {
}
if (_isAvatar && _avatarData) {
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
avatarPacket.append(_avatarData->toByteArray());
nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) {
// if have an avatar audio stream then send it out to our audio-mixer
if (_isListeningToAudioStream || _avatarSound) {
// if we have an avatar audio stream then send it out to our audio-mixer
bool silentFrame = true;
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) {
if (_avatarAudioBuffer[i] != 0) {
silentFrame = false;
break;
int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
const int16_t* nextSoundOutput = NULL;
if (_avatarSound) {
const QByteArray& soundByteArray = _avatarSound->getByteArray();
nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+ _numAvatarSoundSentBytes);
int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
? SCRIPT_AUDIO_BUFFER_BYTES
: soundByteArray.size() - _numAvatarSoundSentBytes;
numAvailableSamples = numAvailableBytes / sizeof(int16_t);
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < numAvailableSamples; ++i) {
if (nextSoundOutput[i] != 0) {
silentFrame = false;
break;
}
}
_numAvatarSoundSentBytes += numAvailableBytes;
if (_numAvatarSoundSentBytes == soundByteArray.size()) {
// we're done with this sound object - so set our pointer back to NULL
// and our sent bytes back to zero
_avatarSound = NULL;
_numAvatarSoundSentBytes = 0;
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
QDataStream packetStream(&audioPacket, QIODevice::Append);
// use the orientation and position of this avatar for the source of this audio
@ -289,13 +320,17 @@ void ScriptEngine::run() {
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
int16_t numSilentSamples = _numAvatarAudioBufferSamples;
packetStream.writeRawData(reinterpret_cast<const char*>(&numSilentSamples), sizeof(int16_t));
} else {
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
} else if (nextSoundOutput) {
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(_avatarAudioBuffer),
_numAvatarAudioBufferSamples * sizeof(int16_t));
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
numAvailableSamples * sizeof(int16_t));
}
nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
@ -303,7 +338,7 @@ void ScriptEngine::run() {
}
qint64 now = usecTimestampNow();
float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND;
float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
emit update(deltaTime);
lastUpdate = now;

14
libraries/script-engine/src/ScriptEngine.h
View file

@ -56,10 +56,11 @@ public:
void setAvatarData(AvatarData* avatarData, const QString& objectName);
void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; }
bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; }
void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples)
{ _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; }
bool isListeningToAudioStream() const { return _isListeningToAudioStream; }
void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; }
void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; }
bool isPlayingAvatarSound() const { return _avatarSound != NULL; }
void init();
void run(); /// runs continuously until Agent.stop() is called
@ -91,8 +92,9 @@ protected:
QTimer* _avatarIdentityTimer;
QTimer* _avatarBillboardTimer;
QHash<QTimer*, QScriptValue> _timerFunctionMap;
int16_t* _avatarAudioBuffer;
int _numAvatarAudioBufferSamples;
bool _isListeningToAudioStream;
Sound* _avatarSound;
int _numAvatarSoundSentBytes;
private:
void sendAvatarIdentityPacket();