merge fix

assignment-client/src/Agent.cpp

@@ -26,8 +26,7 @@
 Agent::Agent(const QByteArray& packet) :
     ThreadedAssignment(packet),
     _voxelEditSender(),
-    _particleEditSender(),
+    _particleEditSender()
-    _avatarAudioStream(NULL)
 {
     // be the parent of the script engine so it gets moved when we do
     _scriptEngine.setParent(this);
@@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) :
     _scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender);
 }
 
-Agent::~Agent() {
-    delete _avatarAudioStream;
-}
-
-const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
-
-void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) {
-    if (sendAvatarAudioStream) {
-        // the agentAudioStream number of samples is related to the ScriptEngine callback rate
-        _avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES];
-        
-        // fill the _audioStream with zeroes to start
-        memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t));
-        
-        _scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES);
-        _scriptEngine.setAvatarAudioBuffer(_avatarAudioStream);
-    } else {
-        delete _avatarAudioStream;
-        _avatarAudioStream = NULL;
-        
-        _scriptEngine.setAvatarAudioBuffer(NULL);
-    }
-}
-
 void Agent::readPendingDatagrams() {
     QByteArray receivedPacket;
     HifiSockAddr senderSockAddr;

assignment-client/src/Agent.h

@@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment {
     Q_OBJECT
     
     Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar)
-    Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream)
+    Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound)
+    Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream)
 public:
     Agent(const QByteArray& packet);
-    ~Agent();
     
     void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); }
     bool isAvatar() const { return _scriptEngine.isAvatar(); }
     
-    void setSendAvatarAudioStream(bool sendAvatarAudioStream);
+    bool isPlayingAvatarSound() const  { return _scriptEngine.isPlayingAvatarSound(); }
-    bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); }
+    
+    bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); }
+    void setIsListeningToAudioStream(bool isListeningToAudioStream)
+        { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); }
     
 public slots:
     void run();
     void readPendingDatagrams();
+    void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); }
 
 private:
     ScriptEngine _scriptEngine;
@@ -50,8 +54,6 @@ private:
 
     ParticleTreeHeadlessViewer _particleViewer;
     VoxelTreeHeadlessViewer _voxelViewer;
-    
-    int16_t* _avatarAudioStream;
 };
 
 #endif /* defined(__hifi__Agent__) */

examples/bot.js

@@ -70,20 +70,11 @@ function clamp(val, min, max){
 }
 
 //  Play a random sound from a list of conversational audio clips
-function audioDone() {
-  isPlayingAudio = false;
-}
-
 var AVERAGE_AUDIO_LENGTH = 8000;
-function playRandomSound(position) {
+function playRandomSound() {
-  if (!isPlayingAudio) {
+  if (!Agent.isPlayingAvatarSound) {
     var whichSound = Math.floor((Math.random() * sounds.length) % sounds.length);
-    var audioOptions = new AudioInjectionOptions();
+    Audio.playSound(sounds[whichSound]);
-    audioOptions.volume = 0.25 + (Math.random() * 0.75);
-    audioOptions.position = position;
-    Audio.playSound(sounds[whichSound], audioOptions);
-    isPlayingAudio = true;
-    Script.setTimeout(audioDone, AVERAGE_AUDIO_LENGTH);
   }
 }
 
@@ -114,6 +105,7 @@ Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-publi
 Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png";
 
 Agent.isAvatar = true;
+Agent.isListeningToAudioStream = true;
 
 print("Joint List:");
 var jointList = Avatar.getJointNames(); 
@@ -153,10 +145,10 @@ function updateBehavior(deltaTime) {
   }
 
   if (Math.random() < CHANCE_OF_SOUND) {
-    playRandomSound(Avatar.position);
+    playRandomSound();
   }
 
-  if (isPlayingAudio) {
+  if (Agent.isPlayingAvatarSound) {
     Avatar.handPosition = Vec3.sum(Avatar.position, Quat.getFront(Avatar.orientation));
   }
 

interface/src/renderer/TextureCache.cpp

@@ -38,9 +38,6 @@ TextureCache::~TextureCache() {
     if (_whiteTextureID != 0) {
         glDeleteTextures(1, &_whiteTextureID);
     }
-    foreach (GLuint id, _fileTextureIDs) {
-        glDeleteTextures(1, &id);
-    }
     if (_primaryFramebufferObject) {
         glDeleteTextures(1, &_primaryDepthTextureID);
     }
@@ -104,23 +101,6 @@ GLuint TextureCache::getBlueTextureID() {
     return _blueTextureID;
 }
 
-GLuint TextureCache::getFileTextureID(const QString& filename) {
-    GLuint id = _fileTextureIDs.value(filename);
-    if (id == 0) {
-        QImage image = QImage(filename).convertToFormat(QImage::Format_ARGB32);
-    
-        glGenTextures(1, &id);
-        glBindTexture(GL_TEXTURE_2D, id);
-        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1,
-            GL_BGRA, GL_UNSIGNED_BYTE, image.constBits());
-        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-        glBindTexture(GL_TEXTURE_2D, 0);
-        
-        _fileTextureIDs.insert(filename, id);
-    }
-    return id;
-}
-
 QSharedPointer<NetworkTexture> TextureCache::getTexture(const QUrl& url, bool normalMap, bool dilatable) {
     if (!dilatable) {
         return ResourceCache::getResource(url, QUrl(), false, &normalMap).staticCast<NetworkTexture>();
@@ -293,27 +273,50 @@ void ImageReader::run() {
         _reply->deleteLater();
         return;
     }
+    QUrl url = _reply->url();
     QImage image = QImage::fromData(_reply->readAll());
+    _reply->deleteLater();
+    
+    // enforce a fixed maximum
+    const int MAXIMUM_SIZE = 1024;
+    if (image.width() > MAXIMUM_SIZE || image.height() > MAXIMUM_SIZE) {
+        qDebug() << "Image greater than maximum size:" << url << image.width() << image.height();
+        image = image.scaled(MAXIMUM_SIZE, MAXIMUM_SIZE, Qt::KeepAspectRatio);
+    }
+    
+    if (!image.hasAlphaChannel()) {
+        if (image.format() != QImage::Format_RGB888) {
+            image = image.convertToFormat(QImage::Format_RGB888);
+        }
+        QMetaObject::invokeMethod(texture.data(), "setImage", Q_ARG(const QImage&, image), Q_ARG(bool, false));
+        return;
+    }
     if (image.format() != QImage::Format_ARGB32) {
         image = image.convertToFormat(QImage::Format_ARGB32);
     }
     
-    // check for translucency
+    // check for translucency/false transparency
+    int opaquePixels = 0;
     int translucentPixels = 0;
     const int EIGHT_BIT_MAXIMUM = 255;
     const int RGB_BITS = 24;
     for (int y = 0; y < image.height(); y++) {
         for (int x = 0; x < image.width(); x++) {
             int alpha = image.pixel(x, y) >> RGB_BITS;
-            if (alpha != 0 && alpha != EIGHT_BIT_MAXIMUM) {
+            if (alpha == EIGHT_BIT_MAXIMUM) {
+                opaquePixels++;
+            } else if (alpha != 0) {
                 translucentPixels++;
             }
         }
     }
     int imageArea = image.width() * image.height();
+    if (opaquePixels == imageArea) {
+        qDebug() << "Image with alpha channel is completely opaque:" << url;
+        image.convertToFormat(QImage::Format_RGB888);
+    }
     QMetaObject::invokeMethod(texture.data(), "setImage", Q_ARG(const QImage&, image),
         Q_ARG(bool, translucentPixels >= imageArea / 2));
-    _reply->deleteLater();
 }
 
 void NetworkTexture::downloadFinished(QNetworkReply* reply) {
@@ -327,8 +330,13 @@ void NetworkTexture::setImage(const QImage& image, bool translucent) {
     finishedLoading(true);
     imageLoaded(image);
     glBindTexture(GL_TEXTURE_2D, getID());
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1,
+    if (image.hasAlphaChannel()) {
-        GL_BGRA, GL_UNSIGNED_BYTE, image.constBits());
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1,
+            GL_BGRA, GL_UNSIGNED_BYTE, image.constBits());
+    } else {
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, image.width(), image.height(), 1,
+            GL_RGB, GL_UNSIGNED_BYTE, image.constBits());
+    }
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     glBindTexture(GL_TEXTURE_2D, 0);
 }
@@ -360,8 +368,13 @@ QSharedPointer<Texture> DilatableNetworkTexture::getDilatedTexture(float dilatio
             painter.end();
             
             glBindTexture(GL_TEXTURE_2D, texture->getID());
-            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, dilatedImage.width(), dilatedImage.height(), 1,
+            if (dilatedImage.hasAlphaChannel()) {
-                GL_BGRA, GL_UNSIGNED_BYTE, dilatedImage.constBits());
+                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, dilatedImage.width(), dilatedImage.height(), 1,
+                    GL_BGRA, GL_UNSIGNED_BYTE, dilatedImage.constBits());
+            } else {
+                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, dilatedImage.width(), dilatedImage.height(), 1,
+                    GL_RGB, GL_UNSIGNED_BYTE, dilatedImage.constBits());
+            }
             glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
             glBindTexture(GL_TEXTURE_2D, 0);
         }

interface/src/renderer/TextureCache.h

@@ -39,9 +39,6 @@ public:
 
     /// Returns the ID of a pale blue texture (useful for a normal map).
     GLuint getBlueTextureID();
-    
-    /// Returns the ID of a texture containing the contents of the specified file, loading it if necessary. 
-    GLuint getFileTextureID(const QString& filename);
 
     /// Loads a texture from the specified URL.
     QSharedPointer<NetworkTexture> getTexture(const QUrl& url, bool normalMap = false, bool dilatable = false);
@@ -84,8 +81,6 @@ private:
     GLuint _whiteTextureID;
     GLuint _blueTextureID;
     
-    QHash<QString, GLuint> _fileTextureIDs;
-
     QHash<QUrl, QWeakPointer<NetworkTexture> > _dilatableNetworkTextures;
     
     GLuint _primaryDepthTextureID;

libraries/script-engine/src/ScriptEngine.cpp

@@ -14,6 +14,7 @@
 #include <QtNetwork/QNetworkRequest>
 #include <QtNetwork/QNetworkReply>
 
+#include <AudioRingBuffer.h>
 #include <AvatarData.h>
 #include <NodeList.h>
 #include <PacketHeaders.h>
@@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co
     _avatarIdentityTimer(NULL),
     _avatarBillboardTimer(NULL),
     _timerFunctionMap(),
-    _avatarAudioBuffer(NULL),
+    _isListeningToAudioStream(false),
+    _avatarSound(NULL),
+    _numAvatarSoundSentBytes(0),
     _controllerScriptingInterface(controllerScriptingInterface),
     _avatarData(NULL),
     _wantMenuItems(wantMenuItems),
@@ -260,27 +263,55 @@ void ScriptEngine::run() {
         }
         
         if (_isAvatar && _avatarData) {
+            
+            const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
+            const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
+            
             QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
             avatarPacket.append(_avatarData->toByteArray());
             
             nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
             
-            if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) {
+            if (_isListeningToAudioStream || _avatarSound) {
-                // if have an avatar audio stream then send it out to our audio-mixer
+                // if we have an avatar audio stream then send it out to our audio-mixer
-                
                 bool silentFrame = true;
                 
-                // check if the all of the _numAvatarAudioBufferSamples to be sent are silence
+                int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
-                for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) {
+                const int16_t* nextSoundOutput = NULL;
-                    if (_avatarAudioBuffer[i] != 0) {
+                
-                        silentFrame = false;
+                if (_avatarSound) {
-                        break;
+                    
+                    const QByteArray& soundByteArray = _avatarSound->getByteArray();
+                    nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+                                                                       + _numAvatarSoundSentBytes);
+                    
+                    int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
+                        ? SCRIPT_AUDIO_BUFFER_BYTES
+                        : soundByteArray.size() - _numAvatarSoundSentBytes;
+                    numAvailableSamples = numAvailableBytes / sizeof(int16_t);
+                    
+                    
+                    // check if the all of the _numAvatarAudioBufferSamples to be sent are silence
+                    for (int i = 0; i < numAvailableSamples; ++i) {
+                        if (nextSoundOutput[i] != 0) {
+                            silentFrame = false;
+                            break;
+                        }
+                    }
+                    
+                    _numAvatarSoundSentBytes += numAvailableBytes;
+                    if (_numAvatarSoundSentBytes == soundByteArray.size()) {
+                        // we're done with this sound object - so set our pointer back to NULL
+                        // and our sent bytes back to zero
+                        _avatarSound = NULL;
+                        _numAvatarSoundSentBytes = 0;
                     }
                 }
                 
                 QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
                                                                       ? PacketTypeSilentAudioFrame
                                                                       : PacketTypeMicrophoneAudioNoEcho);
+                
                 QDataStream packetStream(&audioPacket, QIODevice::Append);
                 
                 // use the orientation and position of this avatar for the source of this audio
@@ -289,13 +320,17 @@ void ScriptEngine::run() {
                 packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
                 
                 if (silentFrame) {
+                    if (!_isListeningToAudioStream) {
+                        // if we have a silent frame and we're not listening then just send nothing and break out of here
+                        break;
+                    }
+                    
                     // write the number of silent samples so the audio-mixer can uphold timing
-                    int16_t numSilentSamples = _numAvatarAudioBufferSamples;
+                    packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
-                    packetStream.writeRawData(reinterpret_cast<const char*>(&numSilentSamples), sizeof(int16_t));
+                } else if (nextSoundOutput) {
-                } else {
                     // write the raw audio data
-                    packetStream.writeRawData(reinterpret_cast<const char*>(_avatarAudioBuffer),
+                    packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
-                                              _numAvatarAudioBufferSamples * sizeof(int16_t));
+                                              numAvailableSamples * sizeof(int16_t));
                 }
                 
                 nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
@@ -303,7 +338,7 @@ void ScriptEngine::run() {
         }
 
         qint64 now = usecTimestampNow();
-        float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND;
+        float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
         emit update(deltaTime);
         lastUpdate = now;
 

libraries/script-engine/src/ScriptEngine.h

@@ -56,10 +56,11 @@ public:
     
     void setAvatarData(AvatarData* avatarData, const QString& objectName);
     
-    void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; }
+    bool isListeningToAudioStream() const { return _isListeningToAudioStream; }
-    bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; }
+    void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; }
-    void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples)
+    
-        { _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; }
+    void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; }
+    bool isPlayingAvatarSound() const { return _avatarSound != NULL; }
     
     void init();
     void run(); /// runs continuously until Agent.stop() is called
@@ -91,8 +92,9 @@ protected:
     QTimer* _avatarIdentityTimer;
     QTimer* _avatarBillboardTimer;
     QHash<QTimer*, QScriptValue> _timerFunctionMap;
-    int16_t* _avatarAudioBuffer;
+    bool _isListeningToAudioStream;
-    int _numAvatarAudioBufferSamples;
+    Sound* _avatarSound;
+    int _numAvatarSoundSentBytes;
 
 private:
     void sendAvatarIdentityPacket();