From 58e773340abc7645105fa368ffc9d1df9193ff0d Mon Sep 17 00:00:00 2001 From: gaitat Date: Fri, 14 Feb 2014 09:40:59 -0500 Subject: [PATCH] Worklist Job #19503 Exposing the average audio intensity to javascript through the MyAvatar object. --- examples/audioBall.js | 78 ++++++++++++++++++++++++++++++ examples/audioBallLifetime.js | 62 ++++++++++++++++++++++++ interface/src/Audio.h | 2 + interface/src/avatar/MyAvatar.cpp | 6 ++- libraries/avatars/src/AvatarData.h | 8 ++- libraries/avatars/src/HeadData.h | 11 +++-- 6 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 examples/audioBall.js create mode 100644 examples/audioBallLifetime.js diff --git a/examples/audioBall.js b/examples/audioBall.js new file mode 100644 index 0000000000..b2761a3dd3 --- /dev/null +++ b/examples/audioBall.js @@ -0,0 +1,78 @@ +// +// audioBall.js +// hifi +// +// Created by Athanasios Gaitatzes on 2/10/14. +// Copyright (c) 2014 HighFidelity, Inc. All rights reserved. +// +// This script creates a particle in front of the user that stays in front of +// the user's avatar as they move, and animates it's radius and color +// in response to the audio intensity. +// + +// add two vectors +function vPlus(a, b) { + var rval = { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z }; + return rval; +} + +// multiply scalar with vector +function vsMult(s, v) { + var rval = { x: s * v.x, y: s * v.y, z: s * v.z }; + return rval; +} + +var sound = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Animals/mexicanWhipoorwill.raw"); +var FACTOR = 0.20; + +var countParticles = 0; // the first time around we want to create the particle and thereafter to modify it. +var particleID; + +function updateParticle() +{ + // the particle should be placed in front of the user's avatar + var avatarFront = Quat.getFront(MyAvatar.orientation); + + // move particle three units in front of the avatar + var particlePosition = vPlus(MyAvatar.position, vsMult (3, avatarFront)); + + // play a sound at the location of the particle + var options = new AudioInjectionOptions(); + options.position = particlePosition; + options.volume = 0.75; + Audio.playSound(sound, options); + + var audioCardAverageLoudness = MyAvatar.audioCardAverageLoudness * FACTOR; + + if (countParticles < 1) + { + var particleProperies = { + position: particlePosition // the particle should stay in front of the user's avatar as he moves + , color: { red: 0, green: 255, blue: 0 } + , radius: audioCardAverageLoudness + , velocity: { x: 0.0, y: 0.0, z: 0.0 } + , gravity: { x: 0.0, y: 0.0, z: 0.0 } + , damping: 0.0 + } + + particleID = Particles.addParticle (particleProperies); + countParticles++; + } + else + { + // animates the particles radius and color in response to the changing audio intensity + var newProperties = { + position: particlePosition // the particle should stay in front of the user's avatar as he moves + , color: { red: 0, green: 255 * audioCardAverageLoudness, blue: 0 } + , radius: audioCardAverageLoudness + }; + + Particles.editParticle (particleID, newProperties); + } +} + +// register the call back so it fires before each data send +Script.willSendVisualDataCallback.connect(updateParticle); + +// register our scriptEnding callback +Script.scriptEnding.connect(function scriptEnding() {}); diff --git a/examples/audioBallLifetime.js b/examples/audioBallLifetime.js new file mode 100644 index 0000000000..7df4a0cdad --- /dev/null +++ b/examples/audioBallLifetime.js @@ -0,0 +1,62 @@ +// +// audioBall.js +// hifi +// +// Created by Athanasios Gaitatzes on 2/10/14. +// Copyright (c) 2014 HighFidelity, Inc. All rights reserved. +// +// This script creates a particle in front of the user that stays in front of +// the user's avatar as they move, and animates it's radius and color +// in response to the audio intensity. +// + +// add two vectors +function vPlus(a, b) { + var rval = { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z }; + return rval; +} + +// multiply scalar with vector +function vsMult(s, v) { + var rval = { x: s * v.x, y: s * v.y, z: s * v.z }; + return rval; +} + +var sound = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Animals/mexicanWhipoorwill.raw"); +var FACTOR = 0.20; + +function addParticle() +{ + // the particle should be placed in front of the user's avatar + var avatarFront = Quat.getFront(MyAvatar.orientation); + + // move particle three units in front of the avatar + var particlePosition = vPlus(MyAvatar.position, vsMult (3, avatarFront)); + + // play a sound at the location of the particle + var options = new AudioInjectionOptions(); + options.position = particlePosition; + options.volume = 0.25; + Audio.playSound(sound, options); + + var audioCardAverageLoudness = MyAvatar.audioCardAverageLoudness * FACTOR; + + // animates the particles radius and color in response to the changing audio intensity + var particleProperies = { + position: particlePosition // the particle should stay in front of the user's avatar as he moves + , color: { red: 0, green: 255 * audioCardAverageLoudness, blue: 0 } + , radius: audioCardAverageLoudness + , velocity: { x: 0.0, y: 0.0, z: 0.0 } + , gravity: { x: 0.0, y: 0.0, z: 0.0 } + , damping: 0.0 + , lifetime: 0.05 + } + + Particles.addParticle (particleProperies); +} + +// register the call back so it fires before each data send +Script.willSendVisualDataCallback.connect(addParticle); + +// register our scriptEnding callback +Script.scriptEnding.connect(function scriptEnding() {}); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 88e2731006..a356d93425 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -47,6 +47,8 @@ public: float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _averageInputLoudness, 0.f); } + float getAudioCardAverageInputLoudness() const { return _averageInputLoudness; } // saki + void setNoiseGateEnabled(bool noiseGateEnabled) { _noiseGateEnabled = noiseGateEnabled; } void setLastAcceleration(const glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; } diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 5628740770..f4570dca45 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -157,7 +157,9 @@ void MyAvatar::update(float deltaTime) { } // Get audio loudness data from audio input device - _head.setAudioLoudness(Application::getInstance()->getAudio()->getLastInputLoudness()); + Audio *audio = Application::getInstance()->getAudio(); + _head.setAudioLoudness(audio->getLastInputLoudness()); + _head.setAudioCardAverageLoudness(audio->getAudioCardAverageInputLoudness()); // saki if (Menu::getInstance()->isOptionChecked(MenuOption::Gravity)) { setGravity(Application::getInstance()->getEnvironment()->getGravity(getPosition())); @@ -639,7 +641,7 @@ void MyAvatar::loadData(QSettings* settings) { _position.y = loadSetting(settings, "position_y", 0.0f); _position.z = loadSetting(settings, "position_z", 0.0f); - _head.setPupilDilation(settings->value("pupilDilation", 0.0f).toFloat()); + _head.setPupilDilation(loadSetting(settings, "pupilDilation", 0.0f)); _leanScale = loadSetting(settings, "leanScale", 0.05f); _targetScale = loadSetting(settings, "scale", 1.0f); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 46d92c0f2e..65f2a318e7 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -75,7 +75,9 @@ class AvatarData : public NodeData { Q_PROPERTY(glm::quat orientation READ getOrientation WRITE setOrientation) Q_PROPERTY(float headPitch READ getHeadPitch WRITE setHeadPitch) - + + Q_PROPERTY(float audioCardAverageLoudness READ getAudioCardAverageLoudness WRITE setAudioCardAverageLoudness) // saki + Q_PROPERTY(QUrl faceModelURL READ getFaceModelURL WRITE setFaceModelURL) Q_PROPERTY(QUrl skeletonModelURL READ getSkeletonModelURL WRITE setSkeletonModelURL) public: @@ -106,6 +108,10 @@ public: float getHeadPitch() const { return _headData->getPitch(); } void setHeadPitch(float value) { _headData->setPitch(value); }; + // access to Head().set/getAverageLoudness + float getAudioCardAverageLoudness() const { return _headData->getAudioCardAverageLoudness(); } // saki + void setAudioCardAverageLoudness(float value) { _headData->setAudioCardAverageLoudness(value); }; // saki + // Scale float getTargetScale() const { return _targetScale; } void setTargetScale(float targetScale) { _targetScale = targetScale; } diff --git a/libraries/avatars/src/HeadData.h b/libraries/avatars/src/HeadData.h index fde684bbf1..477a790f93 100644 --- a/libraries/avatars/src/HeadData.h +++ b/libraries/avatars/src/HeadData.h @@ -41,9 +41,13 @@ public: float getRoll() const { return _roll; } void setRoll(float roll) { _roll = glm::clamp(roll, MIN_HEAD_ROLL, MAX_HEAD_ROLL); } - - void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; } - + + float getAudioLoudness() const { return _audioLoudness; } + void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; } + + float getAudioCardAverageLoudness() const { return _audioCardAverageLoudness; } // saki + void setAudioCardAverageLoudness(float audioCardAverageLoudness) { _audioCardAverageLoudness = audioCardAverageLoudness; } // saki + const std::vector& getBlendshapeCoefficients() const { return _blendshapeCoefficients; } float getPupilDilation() const { return _pupilDilation; } @@ -79,6 +83,7 @@ protected: float _rightEyeBlink; float _averageLoudness; float _browAudioLift; + float _audioCardAverageLoudness; // saki std::vector _blendshapeCoefficients; float _pupilDilation; AvatarData* _owningAvatar;