From e34f2ca69b4f8835f1d3d597ee18c7222e6af7d8 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 19 Mar 2014 16:11:15 -0700 Subject: [PATCH 1/3] Remove file texture loading (since we can just use file URLs), don't add alpha channels for opaque textures (and note when textures with alpha channels are entirely opaque), enforce a maximum texture size of 1024 by 1024. --- interface/src/renderer/TextureCache.cpp | 67 +++++++++++++++---------- interface/src/renderer/TextureCache.h | 5 -- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/interface/src/renderer/TextureCache.cpp b/interface/src/renderer/TextureCache.cpp index 91cf1c8b6e..2b43c89998 100644 --- a/interface/src/renderer/TextureCache.cpp +++ b/interface/src/renderer/TextureCache.cpp @@ -38,9 +38,6 @@ TextureCache::~TextureCache() { if (_whiteTextureID != 0) { glDeleteTextures(1, &_whiteTextureID); } - foreach (GLuint id, _fileTextureIDs) { - glDeleteTextures(1, &id); - } if (_primaryFramebufferObject) { glDeleteTextures(1, &_primaryDepthTextureID); } @@ -104,23 +101,6 @@ GLuint TextureCache::getBlueTextureID() { return _blueTextureID; } -GLuint TextureCache::getFileTextureID(const QString& filename) { - GLuint id = _fileTextureIDs.value(filename); - if (id == 0) { - QImage image = QImage(filename).convertToFormat(QImage::Format_ARGB32); - - glGenTextures(1, &id); - glBindTexture(GL_TEXTURE_2D, id); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1, - GL_BGRA, GL_UNSIGNED_BYTE, image.constBits()); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glBindTexture(GL_TEXTURE_2D, 0); - - _fileTextureIDs.insert(filename, id); - } - return id; -} - QSharedPointer TextureCache::getTexture(const QUrl& url, bool normalMap, bool dilatable) { if (!dilatable) { return ResourceCache::getResource(url, QUrl(), false, &normalMap).staticCast(); @@ -293,27 +273,50 @@ void ImageReader::run() { _reply->deleteLater(); return; } + QUrl url = _reply->url(); QImage image = QImage::fromData(_reply->readAll()); + _reply->deleteLater(); + + // enforce a fixed maximum + const int MAXIMUM_SIZE = 1024; + if (image.width() > MAXIMUM_SIZE || image.height() > MAXIMUM_SIZE) { + qDebug() << "Image greater than maximum size:" << url << image.width() << image.height(); + image = image.scaled(MAXIMUM_SIZE, MAXIMUM_SIZE, Qt::KeepAspectRatio); + } + + if (!image.hasAlphaChannel()) { + if (image.format() != QImage::Format_RGB888) { + image = image.convertToFormat(QImage::Format_RGB888); + } + QMetaObject::invokeMethod(texture.data(), "setImage", Q_ARG(const QImage&, image), Q_ARG(bool, false)); + return; + } if (image.format() != QImage::Format_ARGB32) { image = image.convertToFormat(QImage::Format_ARGB32); } - // check for translucency + // check for translucency/false transparency + int opaquePixels = 0; int translucentPixels = 0; const int EIGHT_BIT_MAXIMUM = 255; const int RGB_BITS = 24; for (int y = 0; y < image.height(); y++) { for (int x = 0; x < image.width(); x++) { int alpha = image.pixel(x, y) >> RGB_BITS; - if (alpha != 0 && alpha != EIGHT_BIT_MAXIMUM) { + if (alpha == EIGHT_BIT_MAXIMUM) { + opaquePixels++; + } else if (alpha != 0) { translucentPixels++; } } } int imageArea = image.width() * image.height(); + if (opaquePixels == imageArea) { + qDebug() << "Image with alpha channel is completely opaque:" << url; + image.convertToFormat(QImage::Format_RGB888); + } QMetaObject::invokeMethod(texture.data(), "setImage", Q_ARG(const QImage&, image), Q_ARG(bool, translucentPixels >= imageArea / 2)); - _reply->deleteLater(); } void NetworkTexture::downloadFinished(QNetworkReply* reply) { @@ -327,8 +330,13 @@ void NetworkTexture::setImage(const QImage& image, bool translucent) { finishedLoading(true); imageLoaded(image); glBindTexture(GL_TEXTURE_2D, getID()); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1, - GL_BGRA, GL_UNSIGNED_BYTE, image.constBits()); + if (image.hasAlphaChannel()) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, image.width(), image.height(), 1, + GL_BGRA, GL_UNSIGNED_BYTE, image.constBits()); + } else { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, image.width(), image.height(), 1, + GL_RGB, GL_UNSIGNED_BYTE, image.constBits()); + } glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, 0); } @@ -360,8 +368,13 @@ QSharedPointer DilatableNetworkTexture::getDilatedTexture(float dilatio painter.end(); glBindTexture(GL_TEXTURE_2D, texture->getID()); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, dilatedImage.width(), dilatedImage.height(), 1, - GL_BGRA, GL_UNSIGNED_BYTE, dilatedImage.constBits()); + if (dilatedImage.hasAlphaChannel()) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, dilatedImage.width(), dilatedImage.height(), 1, + GL_BGRA, GL_UNSIGNED_BYTE, dilatedImage.constBits()); + } else { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, dilatedImage.width(), dilatedImage.height(), 1, + GL_RGB, GL_UNSIGNED_BYTE, dilatedImage.constBits()); + } glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, 0); } diff --git a/interface/src/renderer/TextureCache.h b/interface/src/renderer/TextureCache.h index ebeb35119c..dc874ab7b0 100644 --- a/interface/src/renderer/TextureCache.h +++ b/interface/src/renderer/TextureCache.h @@ -39,9 +39,6 @@ public: /// Returns the ID of a pale blue texture (useful for a normal map). GLuint getBlueTextureID(); - - /// Returns the ID of a texture containing the contents of the specified file, loading it if necessary. - GLuint getFileTextureID(const QString& filename); /// Loads a texture from the specified URL. QSharedPointer getTexture(const QUrl& url, bool normalMap = false, bool dilatable = false); @@ -84,8 +81,6 @@ private: GLuint _whiteTextureID; GLuint _blueTextureID; - QHash _fileTextureIDs; - QHash > _dilatableNetworkTextures; GLuint _primaryDepthTextureID; From 8c4fad443f0ab8de6f0a28cce26a9cfa53567a4b Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 19 Mar 2014 16:14:58 -0700 Subject: [PATCH 2/3] allow Agents to microphone audio from Sound objects --- assignment-client/src/Agent.cpp | 27 +------- assignment-client/src/Agent.h | 14 +++-- libraries/script-engine/src/ScriptEngine.cpp | 65 +++++++++++++++----- libraries/script-engine/src/ScriptEngine.h | 14 +++-- 4 files changed, 67 insertions(+), 53 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 46f4d233c2..cb8edaab47 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -26,8 +26,7 @@ Agent::Agent(const QByteArray& packet) : ThreadedAssignment(packet), _voxelEditSender(), - _particleEditSender(), - _avatarAudioStream(NULL) + _particleEditSender() { // be the parent of the script engine so it gets moved when we do _scriptEngine.setParent(this); @@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) : _scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender); } -Agent::~Agent() { - delete _avatarAudioStream; -} - -const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5); - -void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) { - if (sendAvatarAudioStream) { - // the agentAudioStream number of samples is related to the ScriptEngine callback rate - _avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES]; - - // fill the _audioStream with zeroes to start - memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t)); - - _scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES); - _scriptEngine.setAvatarAudioBuffer(_avatarAudioStream); - } else { - delete _avatarAudioStream; - _avatarAudioStream = NULL; - - _scriptEngine.setAvatarAudioBuffer(NULL); - } -} - void Agent::readPendingDatagrams() { QByteArray receivedPacket; HifiSockAddr senderSockAddr; diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index a051f42faf..b638c39356 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment { Q_OBJECT Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar) - Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream) + Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound) + Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream) public: Agent(const QByteArray& packet); - ~Agent(); void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); } bool isAvatar() const { return _scriptEngine.isAvatar(); } - void setSendAvatarAudioStream(bool sendAvatarAudioStream); - bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); } + bool isPlayingAvatarSound() const { return _scriptEngine.isPlayingAvatarSound(); } + + bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); } + void setIsListeningToAudioStream(bool isListeningToAudioStream) + { _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); } public slots: void run(); void readPendingDatagrams(); + void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); } private: ScriptEngine _scriptEngine; @@ -50,8 +54,6 @@ private: ParticleTreeHeadlessViewer _particleViewer; VoxelTreeHeadlessViewer _voxelViewer; - - int16_t* _avatarAudioStream; }; #endif /* defined(__hifi__Agent__) */ diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index cc61633e60..c820347cab 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co _avatarIdentityTimer(NULL), _avatarBillboardTimer(NULL), _timerFunctionMap(), - _avatarAudioBuffer(NULL), + _isListeningToAudioStream(false), + _avatarSound(NULL), + _numAvatarSoundSentBytes(0), _controllerScriptingInterface(controllerScriptingInterface), _avatarData(NULL), _wantMenuItems(wantMenuItems), @@ -260,27 +263,55 @@ void ScriptEngine::run() { } if (_isAvatar && _avatarData) { + + const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5); + const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t); + QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData); avatarPacket.append(_avatarData->toByteArray()); nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer); - if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) { - // if have an avatar audio stream then send it out to our audio-mixer - + if (_isListeningToAudioStream || _avatarSound) { + // if we have an avatar audio stream then send it out to our audio-mixer bool silentFrame = true; - // check if the all of the _numAvatarAudioBufferSamples to be sent are silence - for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) { - if (_avatarAudioBuffer[i] != 0) { - silentFrame = false; - break; + int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES; + const int16_t* nextSoundOutput = NULL; + + if (_avatarSound) { + + const QByteArray& soundByteArray = _avatarSound->getByteArray(); + nextSoundOutput = reinterpret_cast(soundByteArray.data() + + _numAvatarSoundSentBytes); + + int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES + ? SCRIPT_AUDIO_BUFFER_BYTES + : soundByteArray.size() - _numAvatarSoundSentBytes; + numAvailableSamples = numAvailableBytes / sizeof(int16_t); + + + // check if the all of the _numAvatarAudioBufferSamples to be sent are silence + for (int i = 0; i < numAvailableSamples; ++i) { + if (nextSoundOutput[i] != 0) { + silentFrame = false; + break; + } + } + + _numAvatarSoundSentBytes += numAvailableBytes; + if (_numAvatarSoundSentBytes == soundByteArray.size()) { + // we're done with this sound object - so set our pointer back to NULL + // and our sent bytes back to zero + _avatarSound = NULL; + _numAvatarSoundSentBytes = 0; } } QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame ? PacketTypeSilentAudioFrame : PacketTypeMicrophoneAudioNoEcho); + QDataStream packetStream(&audioPacket, QIODevice::Append); // use the orientation and position of this avatar for the source of this audio @@ -289,13 +320,17 @@ void ScriptEngine::run() { packetStream.writeRawData(reinterpret_cast(&headOrientation), sizeof(glm::quat)); if (silentFrame) { + if (!_isListeningToAudioStream) { + // if we have a silent frame and we're not listening then just send nothing and break out of here + break; + } + // write the number of silent samples so the audio-mixer can uphold timing - int16_t numSilentSamples = _numAvatarAudioBufferSamples; - packetStream.writeRawData(reinterpret_cast(&numSilentSamples), sizeof(int16_t)); - } else { + packetStream.writeRawData(reinterpret_cast(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t)); + } else if (nextSoundOutput) { // write the raw audio data - packetStream.writeRawData(reinterpret_cast(_avatarAudioBuffer), - _numAvatarAudioBufferSamples * sizeof(int16_t)); + packetStream.writeRawData(reinterpret_cast(nextSoundOutput), + numAvailableSamples * sizeof(int16_t)); } nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer); @@ -303,7 +338,7 @@ void ScriptEngine::run() { } qint64 now = usecTimestampNow(); - float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND; + float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND; emit update(deltaTime); lastUpdate = now; diff --git a/libraries/script-engine/src/ScriptEngine.h b/libraries/script-engine/src/ScriptEngine.h index 606d0aabf4..755418b0c1 100644 --- a/libraries/script-engine/src/ScriptEngine.h +++ b/libraries/script-engine/src/ScriptEngine.h @@ -56,10 +56,11 @@ public: void setAvatarData(AvatarData* avatarData, const QString& objectName); - void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; } - bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; } - void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples) - { _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; } + bool isListeningToAudioStream() const { return _isListeningToAudioStream; } + void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; } + + void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; } + bool isPlayingAvatarSound() const { return _avatarSound != NULL; } void init(); void run(); /// runs continuously until Agent.stop() is called @@ -91,8 +92,9 @@ protected: QTimer* _avatarIdentityTimer; QTimer* _avatarBillboardTimer; QHash _timerFunctionMap; - int16_t* _avatarAudioBuffer; - int _numAvatarAudioBufferSamples; + bool _isListeningToAudioStream; + Sound* _avatarSound; + int _numAvatarSoundSentBytes; private: void sendAvatarIdentityPacket(); From dd9c4aa8541d87d44860ed66eebd79bd55eb878e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 19 Mar 2014 16:18:44 -0700 Subject: [PATCH 3/3] changes to bot.js for new Agent microphone audio --- examples/bot.js | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/examples/bot.js b/examples/bot.js index 5fd4785b76..ae337f5031 100644 --- a/examples/bot.js +++ b/examples/bot.js @@ -29,7 +29,6 @@ var CHANCE_OF_BIG_MOVE = 0.1; var isMoving = false; var isTurningHead = false; -var isPlayingAudio = false; var X_MIN = 0.0; var X_MAX = 5.0; @@ -60,20 +59,11 @@ function clamp(val, min, max){ } // Play a random sound from a list of conversational audio clips -function audioDone() { - isPlayingAudio = false; -} - var AVERAGE_AUDIO_LENGTH = 8000; -function playRandomSound(position) { - if (!isPlayingAudio) { +function playRandomSound() { + if (!Agent.isPlayingAvatarSound) { var whichSound = Math.floor((Math.random() * sounds.length) % sounds.length); - var audioOptions = new AudioInjectionOptions(); - audioOptions.volume = 0.25 + (Math.random() * 0.75); - audioOptions.position = position; - Audio.playSound(sounds[whichSound], audioOptions); - isPlayingAudio = true; - Script.setTimeout(audioDone, AVERAGE_AUDIO_LENGTH); + Audio.playSound(sounds[whichSound]); } } @@ -104,6 +94,7 @@ Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-publi Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png"; Agent.isAvatar = true; +Agent.isListeningToAudioStream = true; // change the avatar's position to the random one Avatar.position = firstPosition; @@ -111,10 +102,10 @@ printVector("New bot, position = ", Avatar.position); function updateBehavior(deltaTime) { if (Math.random() < CHANCE_OF_SOUND) { - playRandomSound(Avatar.position); + playRandomSound(); } - if (isPlayingAudio) { + if (Agent.isPlayingAvatarSound) { Avatar.handPosition = Vec3.sum(Avatar.position, Quat.getFront(Avatar.orientation)); }