Worklist Job #19503

Exposing the average audio intensity to javascript through the MyAvatar
object.

examples/audioBall.js

@@ -0,0 +1,78 @@
+//
+//  audioBall.js
+//  hifi
+//
+//  Created by Athanasios Gaitatzes on 2/10/14.
+//  Copyright (c) 2014 HighFidelity, Inc. All rights reserved.
+//
+//  This script creates a particle in front of the user that stays in front of
+//  the user's avatar as they move, and animates it's radius and color
+//  in response to the audio intensity.
+//
+
+// add two vectors
+function vPlus(a, b) { 
+    var rval = { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z };
+    return rval;
+}
+
+// multiply scalar with vector
+function vsMult(s, v) {
+    var rval = { x: s * v.x, y: s * v.y, z: s * v.z };
+    return rval;
+}
+
+var sound = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Animals/mexicanWhipoorwill.raw");
+var FACTOR = 0.20;
+
+var countParticles = 0;    // the first time around we want to create the particle and thereafter to modify it.
+var particleID;
+
+function updateParticle()
+{
+    // the particle should be placed in front of the user's avatar
+	var avatarFront = Quat.getFront(MyAvatar.orientation);
+
+    // move particle three units in front of the avatar
+    var particlePosition = vPlus(MyAvatar.position, vsMult (3, avatarFront));
+
+    // play a sound at the location of the particle
+    var options = new AudioInjectionOptions();
+    options.position = particlePosition;
+    options.volume = 0.75;
+    Audio.playSound(sound, options);
+
+	var audioCardAverageLoudness = MyAvatar.audioCardAverageLoudness * FACTOR;
+
+    if (countParticles < 1)
+    {
+        var particleProperies = {
+            position: particlePosition // the particle should stay in front of the user's avatar as he moves
+        ,   color: { red: 0, green: 255, blue: 0 }
+        ,   radius: audioCardAverageLoudness
+        ,   velocity: { x: 0.0, y: 0.0, z: 0.0 }
+        ,   gravity: { x: 0.0, y: 0.0, z: 0.0 }
+        ,   damping: 0.0
+        }
+
+        particleID = Particles.addParticle (particleProperies);
+        countParticles++;
+    }
+    else
+    {
+        // animates the particles radius and color in response to the changing audio intensity
+        var newProperties = {
+            position: particlePosition // the particle should stay in front of the user's avatar as he moves
+        ,   color: { red: 0, green: 255 * audioCardAverageLoudness, blue: 0 }
+        ,   radius: audioCardAverageLoudness
+        };
+
+        Particles.editParticle (particleID, newProperties);
+    }
+}
+
+// register the call back so it fires before each data send
+Script.willSendVisualDataCallback.connect(updateParticle);
+
+// register our scriptEnding callback
+Script.scriptEnding.connect(function scriptEnding() {});

examples/audioBallLifetime.js

@@ -0,0 +1,62 @@
+//
+//  audioBall.js
+//  hifi
+//
+//  Created by Athanasios Gaitatzes on 2/10/14.
+//  Copyright (c) 2014 HighFidelity, Inc. All rights reserved.
+//
+//  This script creates a particle in front of the user that stays in front of
+//  the user's avatar as they move, and animates it's radius and color
+//  in response to the audio intensity.
+//
+
+// add two vectors
+function vPlus(a, b) { 
+    var rval = { x: a.x + b.x, y: a.y + b.y, z: a.z + b.z };
+    return rval;
+}
+
+// multiply scalar with vector
+function vsMult(s, v) {
+    var rval = { x: s * v.x, y: s * v.y, z: s * v.z };
+    return rval;
+}
+
+var sound = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Animals/mexicanWhipoorwill.raw");
+var FACTOR = 0.20;
+
+function addParticle()
+{
+    // the particle should be placed in front of the user's avatar
+    var avatarFront = Quat.getFront(MyAvatar.orientation);
+
+    // move particle three units in front of the avatar
+    var particlePosition = vPlus(MyAvatar.position, vsMult (3, avatarFront));
+
+    // play a sound at the location of the particle
+    var options = new AudioInjectionOptions();
+    options.position = particlePosition;
+    options.volume = 0.25;
+    Audio.playSound(sound, options);
+
+    var audioCardAverageLoudness = MyAvatar.audioCardAverageLoudness * FACTOR;
+
+    // animates the particles radius and color in response to the changing audio intensity
+    var particleProperies = {
+        position: particlePosition // the particle should stay in front of the user's avatar as he moves
+    ,   color: { red: 0, green: 255 * audioCardAverageLoudness, blue: 0 }
+    ,   radius: audioCardAverageLoudness
+    ,   velocity: { x: 0.0, y: 0.0, z: 0.0 }
+    ,   gravity: { x: 0.0, y: 0.0, z: 0.0 }
+    ,   damping: 0.0
+    ,   lifetime: 0.05
+    }
+
+    Particles.addParticle (particleProperies);
+}
+
+// register the call back so it fires before each data send
+Script.willSendVisualDataCallback.connect(addParticle);
+
+// register our scriptEnding callback
+Script.scriptEnding.connect(function scriptEnding() {});

interface/src/Audio.h

@@ -47,6 +47,8 @@ public:
     
     float getLastInputLoudness() const { return glm::max(_lastInputLoudness - _averageInputLoudness, 0.f); }
     
+    float getAudioCardAverageInputLoudness() const { return _averageInputLoudness; }	// saki
+
     void setNoiseGateEnabled(bool noiseGateEnabled) { _noiseGateEnabled = noiseGateEnabled; }
     
     void setLastAcceleration(const glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }

interface/src/avatar/MyAvatar.cpp

@@ -157,7 +157,9 @@ void MyAvatar::update(float deltaTime) {
     }
 
     //  Get audio loudness data from audio input device
-    _head.setAudioLoudness(Application::getInstance()->getAudio()->getLastInputLoudness());
+	Audio *audio = Application::getInstance()->getAudio();
+    _head.setAudioLoudness(audio->getLastInputLoudness());
+    _head.setAudioCardAverageLoudness(audio->getAudioCardAverageInputLoudness());	// saki
 
     if (Menu::getInstance()->isOptionChecked(MenuOption::Gravity)) {
         setGravity(Application::getInstance()->getEnvironment()->getGravity(getPosition()));
@@ -639,7 +641,7 @@ void MyAvatar::loadData(QSettings* settings) {
     _position.y = loadSetting(settings, "position_y", 0.0f);
     _position.z = loadSetting(settings, "position_z", 0.0f);
 
-    _head.setPupilDilation(settings->value("pupilDilation", 0.0f).toFloat());
+    _head.setPupilDilation(loadSetting(settings, "pupilDilation", 0.0f));
 
     _leanScale = loadSetting(settings, "leanScale", 0.05f);
     _targetScale = loadSetting(settings, "scale", 1.0f);

libraries/avatars/src/AvatarData.h

@@ -75,7 +75,9 @@ class AvatarData : public NodeData {
 
     Q_PROPERTY(glm::quat orientation READ getOrientation WRITE setOrientation)
     Q_PROPERTY(float headPitch READ getHeadPitch WRITE setHeadPitch)
-    
+
+    Q_PROPERTY(float audioCardAverageLoudness READ getAudioCardAverageLoudness WRITE setAudioCardAverageLoudness)	// saki
+
     Q_PROPERTY(QUrl faceModelURL READ getFaceModelURL WRITE setFaceModelURL)
     Q_PROPERTY(QUrl skeletonModelURL READ getSkeletonModelURL WRITE setSkeletonModelURL)
 public:
@@ -106,6 +108,10 @@ public:
     float getHeadPitch() const { return _headData->getPitch(); }
     void setHeadPitch(float value) { _headData->setPitch(value); };
 
+    // access to Head().set/getAverageLoudness
+    float getAudioCardAverageLoudness() const { return _headData->getAudioCardAverageLoudness(); }	// saki
+    void setAudioCardAverageLoudness(float value) { _headData->setAudioCardAverageLoudness(value); };	// saki
+
     //  Scale
     float getTargetScale() const { return _targetScale; }
     void setTargetScale(float targetScale) { _targetScale = targetScale; }

libraries/avatars/src/HeadData.h

@@ -41,9 +41,13 @@ public:
     
     float getRoll() const { return _roll; }
     void setRoll(float roll) { _roll = glm::clamp(roll, MIN_HEAD_ROLL, MAX_HEAD_ROLL); }
-    
-    void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; }
-    
+
+    float getAudioLoudness() const { return _audioLoudness; }
+	void setAudioLoudness(float audioLoudness) { _audioLoudness = audioLoudness; }
+
+    float getAudioCardAverageLoudness() const { return _audioCardAverageLoudness; }	// saki
+	void setAudioCardAverageLoudness(float audioCardAverageLoudness) { _audioCardAverageLoudness = audioCardAverageLoudness; }	// saki
+
     const std::vector<float>& getBlendshapeCoefficients() const { return _blendshapeCoefficients; }
     
     float getPupilDilation() const { return _pupilDilation; }
@@ -79,6 +83,7 @@ protected:
     float _rightEyeBlink;
     float _averageLoudness;
     float _browAudioLift;
+	float _audioCardAverageLoudness;	// saki
     std::vector<float> _blendshapeCoefficients;
     float _pupilDilation;
     AvatarData* _owningAvatar;