Merge pull request #2392 from birarda/audio-scaling

allow Agents to send microphone audio using Sound objects

assignment-client/src/Agent.cpp

@@ -26,8 +26,7 @@
 Agent::Agent(const QByteArray& packet) :
     ThreadedAssignment(packet),
     _voxelEditSender(),
-    _particleEditSender(),
+    _particleEditSender()
-    _avatarAudioStream(NULL)
 {
     // be the parent of the script engine so it gets moved when we do
     _scriptEngine.setParent(this);
@@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) :
     _scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender);
 }
 
-Agent::~Agent() {
-    delete _avatarAudioStream;
-}
-
-const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
-
-void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) {
-    if (sendAvatarAudioStream) {
-        // the agentAudioStream number of samples is related to the ScriptEngine callback rate
-        _avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES];
-        
-        // fill the _audioStream with zeroes to start
-        memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t));
-        
-        _scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES);
-        _scriptEngine.setAvatarAudioBuffer(_avatarAudioStream);
-    } else {
-        delete _avatarAudioStream;
-        _avatarAudioStream = NULL;
-        
-        _scriptEngine.setAvatarAudioBuffer(NULL);
-    }
-}
-
 void Agent::readPendingDatagrams() {
     QByteArray receivedPacket;
     HifiSockAddr senderSockAddr;

assignment-client/src/Agent.h

@@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment {
     Q_OBJECT
     
     Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar)
-    Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream)
+    Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound)
+    Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream)
 public:
     Agent(const QByteArray& packet);
-    ~Agent();
     
     void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); }
     bool isAvatar() const { return _scriptEngine.isAvatar(); }
     
-    void setSendAvatarAudioStream(bool sendAvatarAudioStream);
+    bool isPlayingAvatarSound() const  { return _scriptEngine.isPlayingAvatarSound(); }
-    bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); }
+    
+    bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); }
+    void setIsListeningToAudioStream(bool isListeningToAudioStream)
+        { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); }
     
 public slots:
     void run();
     void readPendingDatagrams();
+    void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); }
 
 private:
     ScriptEngine _scriptEngine;
@@ -50,8 +54,6 @@ private:
 
     ParticleTreeHeadlessViewer _particleViewer;
     VoxelTreeHeadlessViewer _voxelViewer;
-    
-    int16_t* _avatarAudioStream;
 };
 
 #endif /* defined(__hifi__Agent__) */

examples/bot.js

@@ -29,7 +29,6 @@ var CHANCE_OF_BIG_MOVE = 0.1;
 
 var isMoving = false;
 var isTurningHead = false;
-var isPlayingAudio = false; 
 
 var X_MIN = 0.0;
 var X_MAX = 5.0;
@@ -60,20 +59,11 @@ function clamp(val, min, max){
 }
 
 //  Play a random sound from a list of conversational audio clips
-function audioDone() {
-  isPlayingAudio = false;
-}
-
 var AVERAGE_AUDIO_LENGTH = 8000;
-function playRandomSound(position) {
+function playRandomSound() {
-  if (!isPlayingAudio) {
+  if (!Agent.isPlayingAvatarSound) {
     var whichSound = Math.floor((Math.random() * sounds.length) % sounds.length);
-    var audioOptions = new AudioInjectionOptions();
+    Audio.playSound(sounds[whichSound]);
-    audioOptions.volume = 0.25 + (Math.random() * 0.75);
-    audioOptions.position = position;
-    Audio.playSound(sounds[whichSound], audioOptions);
-    isPlayingAudio = true;
-    Script.setTimeout(audioDone, AVERAGE_AUDIO_LENGTH);
   }
 }
 
@@ -104,6 +94,7 @@ Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-publi
 Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png";
 
 Agent.isAvatar = true;
+Agent.isListeningToAudioStream = true;
 
 // change the avatar's position to the random one
 Avatar.position = firstPosition;  
@@ -111,10 +102,10 @@ printVector("New bot, position = ", Avatar.position);
 
 function updateBehavior(deltaTime) {
   if (Math.random() < CHANCE_OF_SOUND) {
-    playRandomSound(Avatar.position);
+    playRandomSound();
   }
 
-  if (isPlayingAudio) {
+  if (Agent.isPlayingAvatarSound) {
     Avatar.handPosition = Vec3.sum(Avatar.position, Quat.getFront(Avatar.orientation));
   }
 

libraries/script-engine/src/ScriptEngine.cpp

@@ -14,6 +14,7 @@
 #include <QtNetwork/QNetworkRequest>
 #include <QtNetwork/QNetworkReply>
 
+#include <AudioRingBuffer.h>
 #include <AvatarData.h>
 #include <NodeList.h>
 #include <PacketHeaders.h>
@@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co
     _avatarIdentityTimer(NULL),
     _avatarBillboardTimer(NULL),
     _timerFunctionMap(),
-    _avatarAudioBuffer(NULL),
+    _isListeningToAudioStream(false),
+    _avatarSound(NULL),
+    _numAvatarSoundSentBytes(0),
     _controllerScriptingInterface(controllerScriptingInterface),
     _avatarData(NULL),
     _wantMenuItems(wantMenuItems),
@@ -260,27 +263,55 @@ void ScriptEngine::run() {
         }
         
         if (_isAvatar && _avatarData) {
+            
+            const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
+            const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
+            
             QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
             avatarPacket.append(_avatarData->toByteArray());
             
             nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
             
-            if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) {
+            if (_isListeningToAudioStream || _avatarSound) {
-                // if have an avatar audio stream then send it out to our audio-mixer
+                // if we have an avatar audio stream then send it out to our audio-mixer
-                
                 bool silentFrame = true;
                 
-                // check if the all of the _numAvatarAudioBufferSamples to be sent are silence
+                int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
-                for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) {
+                const int16_t* nextSoundOutput = NULL;
-                    if (_avatarAudioBuffer[i] != 0) {
+                
-                        silentFrame = false;
+                if (_avatarSound) {
-                        break;
+                    
+                    const QByteArray& soundByteArray = _avatarSound->getByteArray();
+                    nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+                                                                       + _numAvatarSoundSentBytes);
+                    
+                    int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
+                        ? SCRIPT_AUDIO_BUFFER_BYTES
+                        : soundByteArray.size() - _numAvatarSoundSentBytes;
+                    numAvailableSamples = numAvailableBytes / sizeof(int16_t);
+                    
+                    
+                    // check if the all of the _numAvatarAudioBufferSamples to be sent are silence
+                    for (int i = 0; i < numAvailableSamples; ++i) {
+                        if (nextSoundOutput[i] != 0) {
+                            silentFrame = false;
+                            break;
+                        }
+                    }
+                    
+                    _numAvatarSoundSentBytes += numAvailableBytes;
+                    if (_numAvatarSoundSentBytes == soundByteArray.size()) {
+                        // we're done with this sound object - so set our pointer back to NULL
+                        // and our sent bytes back to zero
+                        _avatarSound = NULL;
+                        _numAvatarSoundSentBytes = 0;
                     }
                 }
                 
                 QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
                                                                       ? PacketTypeSilentAudioFrame
                                                                       : PacketTypeMicrophoneAudioNoEcho);
+                
                 QDataStream packetStream(&audioPacket, QIODevice::Append);
                 
                 // use the orientation and position of this avatar for the source of this audio
@@ -289,13 +320,17 @@ void ScriptEngine::run() {
                 packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
                 
                 if (silentFrame) {
+                    if (!_isListeningToAudioStream) {
+                        // if we have a silent frame and we're not listening then just send nothing and break out of here
+                        break;
+                    }
+                    
                     // write the number of silent samples so the audio-mixer can uphold timing
-                    int16_t numSilentSamples = _numAvatarAudioBufferSamples;
+                    packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
-                    packetStream.writeRawData(reinterpret_cast<const char*>(&numSilentSamples), sizeof(int16_t));
+                } else if (nextSoundOutput) {
-                } else {
                     // write the raw audio data
-                    packetStream.writeRawData(reinterpret_cast<const char*>(_avatarAudioBuffer),
+                    packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
-                                              _numAvatarAudioBufferSamples * sizeof(int16_t));
+                                              numAvailableSamples * sizeof(int16_t));
                 }
                 
                 nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
@@ -303,7 +338,7 @@ void ScriptEngine::run() {
         }
 
         qint64 now = usecTimestampNow();
-        float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND;
+        float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
         emit update(deltaTime);
         lastUpdate = now;
 

libraries/script-engine/src/ScriptEngine.h

@@ -56,10 +56,11 @@ public:
     
     void setAvatarData(AvatarData* avatarData, const QString& objectName);
     
-    void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; }
+    bool isListeningToAudioStream() const { return _isListeningToAudioStream; }
-    bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; }
+    void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; }
-    void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples)
+    
-        { _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; }
+    void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; }
+    bool isPlayingAvatarSound() const { return _avatarSound != NULL; }
     
     void init();
     void run(); /// runs continuously until Agent.stop() is called
@@ -91,8 +92,9 @@ protected:
     QTimer* _avatarIdentityTimer;
     QTimer* _avatarBillboardTimer;
     QHash<QTimer*, QScriptValue> _timerFunctionMap;
-    int16_t* _avatarAudioBuffer;
+    bool _isListeningToAudioStream;
-    int _numAvatarAudioBufferSamples;
+    Sound* _avatarSound;
+    int _numAvatarSoundSentBytes;
 
 private:
     void sendAvatarIdentityPacket();