diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 16674efcbf..4eb6b17260 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -26,8 +26,7 @@ Agent::Agent(const QByteArray& packet) : ThreadedAssignment(packet), _voxelEditSender(), - _particleEditSender(), - _avatarAudioStream(NULL) + _particleEditSender() { // be the parent of the script engine so it gets moved when we do _scriptEngine.setParent(this); @@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) : _scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender); } -Agent::~Agent() { - delete _avatarAudioStream; -} - -const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5); - -void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) { - if (sendAvatarAudioStream) { - // the agentAudioStream number of samples is related to the ScriptEngine callback rate - _avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES]; - - // fill the _audioStream with zeroes to start - memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t)); - - _scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES); - _scriptEngine.setAvatarAudioBuffer(_avatarAudioStream); - } else { - delete _avatarAudioStream; - _avatarAudioStream = NULL; - - _scriptEngine.setAvatarAudioBuffer(NULL); - } -} - void Agent::readPendingDatagrams() { QByteArray receivedPacket; HifiSockAddr senderSockAddr; diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index a051f42faf..b638c39356 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment { Q_OBJECT Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar) - Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream) + Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound) + Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream) public: Agent(const QByteArray& packet); - ~Agent(); void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); } bool isAvatar() const { return _scriptEngine.isAvatar(); } - void setSendAvatarAudioStream(bool sendAvatarAudioStream); - bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); } + bool isPlayingAvatarSound() const { return _scriptEngine.isPlayingAvatarSound(); } + + bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); } + void setIsListeningToAudioStream(bool isListeningToAudioStream) + { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); } public slots: void run(); void readPendingDatagrams(); + void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); } private: ScriptEngine _scriptEngine; @@ -50,8 +54,6 @@ private: ParticleTreeHeadlessViewer _particleViewer; VoxelTreeHeadlessViewer _voxelViewer; - - int16_t* _avatarAudioStream; }; #endif /* defined(__hifi__Agent__) */ diff --git a/examples/bot.js b/examples/bot.js index 5fd4785b76..ae337f5031 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -29,7 +29,6 @@ var CHANCE_OF_BIG_MOVE = 0.1; var isMoving = false; var isTurningHead = false; -var isPlayingAudio = false; var X_MIN = 0.0; var X_MAX = 5.0; @@ -60,20 +59,11 @@ function clamp(val, min, max){ } // Play a random sound from a list of conversational audio clips -function audioDone() { - isPlayingAudio = false; -} - var AVERAGE_AUDIO_LENGTH = 8000; -function playRandomSound(position) { - if (!isPlayingAudio) { +function playRandomSound() { + if (!Agent.isPlayingAvatarSound) { var whichSound = Math.floor((Math.random() * sounds.length) % sounds.length); - var audioOptions = new AudioInjectionOptions(); - audioOptions.volume = 0.25 + (Math.random() * 0.75); - audioOptions.position = position; - Audio.playSound(sounds[whichSound], audioOptions); - isPlayingAudio = true; - Script.setTimeout(audioDone, AVERAGE_AUDIO_LENGTH); + Audio.playSound(sounds[whichSound]); } } @@ -104,6 +94,7 @@ Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-publi Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png"; Agent.isAvatar = true; +Agent.isListeningToAudioStream = true; // change the avatar's position to the random one Avatar.position = firstPosition; @@ -111,10 +102,10 @@ printVector("New bot, position = ", Avatar.position); function updateBehavior(deltaTime) { if (Math.random() < CHANCE_OF_SOUND) { - playRandomSound(Avatar.position); + playRandomSound(); } - if (isPlayingAudio) { + if (Agent.isPlayingAvatarSound) { Avatar.handPosition = Vec3.sum(Avatar.position, Quat.getFront(Avatar.orientation)); } diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index cc61633e60..c820347cab 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co _avatarIdentityTimer(NULL), _avatarBillboardTimer(NULL), _timerFunctionMap(), - _avatarAudioBuffer(NULL), + _isListeningToAudioStream(false), + _avatarSound(NULL), + _numAvatarSoundSentBytes(0), _controllerScriptingInterface(controllerScriptingInterface), _avatarData(NULL), _wantMenuItems(wantMenuItems), @@ -260,27 +263,55 @@ void ScriptEngine::run() { } if (_isAvatar && _avatarData) { + + const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5); + const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t); + QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData); avatarPacket.append(_avatarData->toByteArray()); nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer); - if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) { - // if have an avatar audio stream then send it out to our audio-mixer - + if (_isListeningToAudioStream || _avatarSound) { + // if we have an avatar audio stream then send it out to our audio-mixer bool silentFrame = true; - // check if the all of the _numAvatarAudioBufferSamples to be sent are silence - for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) { - if (_avatarAudioBuffer[i] != 0) { - silentFrame = false; - break; + int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES; + const int16_t* nextSoundOutput = NULL; + + if (_avatarSound) { + + const QByteArray& soundByteArray = _avatarSound->getByteArray(); + nextSoundOutput = reinterpret_cast(soundByteArray.data() + + _numAvatarSoundSentBytes); + + int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES + ? SCRIPT_AUDIO_BUFFER_BYTES + : soundByteArray.size() - _numAvatarSoundSentBytes; + numAvailableSamples = numAvailableBytes / sizeof(int16_t); + + + // check if the all of the _numAvatarAudioBufferSamples to be sent are silence + for (int i = 0; i < numAvailableSamples; ++i) { + if (nextSoundOutput[i] != 0) { + silentFrame = false; + break; + } + } + + _numAvatarSoundSentBytes += numAvailableBytes; + if (_numAvatarSoundSentBytes == soundByteArray.size()) { + // we're done with this sound object - so set our pointer back to NULL + // and our sent bytes back to zero + _avatarSound = NULL; + _numAvatarSoundSentBytes = 0; } } QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame ? PacketTypeSilentAudioFrame : PacketTypeMicrophoneAudioNoEcho); + QDataStream packetStream(&audioPacket, QIODevice::Append); // use the orientation and position of this avatar for the source of this audio @@ -289,13 +320,17 @@ void ScriptEngine::run() { packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); if (silentFrame) { + if (!_isListeningToAudioStream) { + // if we have a silent frame and we're not listening then just send nothing and break out of here + break; + } + // write the number of silent samples so the audio-mixer can uphold timing - int16_t numSilentSamples = _numAvatarAudioBufferSamples; - packetStream.writeRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); - } else { + packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); + } else if (nextSoundOutput) { // write the raw audio data - packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), - _numAvatarAudioBufferSamples * sizeof(int16_t)); + packetStream.writeRawData(reinterpret_cast(nextSoundOutput), + numAvailableSamples * sizeof(int16_t)); } nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); @@ -303,7 +338,7 @@ void ScriptEngine::run() { } qint64 now = usecTimestampNow(); - float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND; + float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND; emit update(deltaTime); lastUpdate = now; diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index 606d0aabf4..755418b0c1 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -56,10 +56,11 @@ public: void setAvatarData(AvatarData* avatarData, const QString& objectName); - void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; } - bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; } - void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples) - { _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; } + bool isListeningToAudioStream() const { return _isListeningToAudioStream; } + void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; } + + void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; } + bool isPlayingAvatarSound() const { return _avatarSound != NULL; } void init(); void run(); /// runs continuously until Agent.stop() is called @@ -91,8 +92,9 @@ protected: QTimer* _avatarIdentityTimer; QTimer* _avatarBillboardTimer; QHash _timerFunctionMap; - int16_t* _avatarAudioBuffer; - int _numAvatarAudioBufferSamples; + bool _isListeningToAudioStream; + Sound* _avatarSound; + int _numAvatarSoundSentBytes; private: void sendAvatarIdentityPacket();