Merge pull request #5880 from thoys/20668

Worklist job #20668 - JS: Add option to listen audio from camera

examples/utilities/tools/developerMenuItems.js

@@ -13,6 +13,7 @@
 
 var createdRenderMenu = false;
 var createdGeneratedAudioMenu = false;
+var createdAudioListenerModeMenu = false;
 var createdStereoInputMenuItem = false;
 
 var DEVELOPER_MENU = "Developer";
@@ -29,6 +30,20 @@ var AUDIO_SOURCE_INJECT = "Generated Audio";
 var AUDIO_SOURCE_MENU = AUDIO_MENU + " > Generated Audio Source";
 var AUDIO_SOURCE_PINK_NOISE = "Pink Noise";
 var AUDIO_SOURCE_SINE_440 = "Sine 440hz";
+var AUDIO_LISTENER_MODE_MENU =  AUDIO_MENU + " > Audio Listener Mode"
+var AUDIO_LISTENER_MODE_FROM_HEAD = "Audio from head";
+var AUDIO_LISTENER_MODE_FROM_CAMERA = "Audio from camera";
+var AUDIO_LISTENER_MODE_CUSTOM = "Audio from custom position";
+
+// be sure that the audio listener options are in the right order (same as the enumerator)
+var AUDIO_LISTENER_OPTIONS = [
+    // MyAvatar.FROM_HEAD (0)
+    AUDIO_LISTENER_MODE_FROM_HEAD,
+    // MyAvatar.FROM_CAMERA (1)
+    AUDIO_LISTENER_MODE_FROM_CAMERA,
+    // MyAvatar.CUSTOM (2)
+    AUDIO_LISTENER_MODE_CUSTOM
+];
 var AUDIO_STEREO_INPUT = "Stereo Input";
 
 
@@ -67,7 +82,6 @@ function setupMenus() {
         Menu.addMenuItem({ menuName: RENDER_MENU, menuItemName: AVATARS_ITEM, isCheckable: true, isChecked: Scene.shouldRenderAvatars })
     }
     
-    
     if (!Menu.menuExists(AUDIO_MENU)) {
         Menu.addMenu(AUDIO_MENU);
     }
@@ -80,6 +94,15 @@ function setupMenus() {
         Audio.selectPinkNoise();
         createdGeneratedAudioMenu = true;
     }
+
+    if (!Menu.menuExists(AUDIO_LISTENER_MODE_MENU)) {
+        Menu.addMenu(AUDIO_LISTENER_MODE_MENU);
+        for (var i = 0; i < AUDIO_LISTENER_OPTIONS.length; i++) {
+            Menu.addMenuItem({ menuName: AUDIO_LISTENER_MODE_MENU, menuItemName: AUDIO_LISTENER_OPTIONS[i], isCheckable: true, isChecked: (MyAvatar.audioListenerMode === i) });
+        }
+        createdAudioListenerModeMenu = true;
+    }
+
     if (!Menu.menuItemExists(AUDIO_MENU, AUDIO_STEREO_INPUT)) {
         Menu.addMenuItem({ menuName: AUDIO_MENU, menuItemName: AUDIO_STEREO_INPUT, isCheckable: true, isChecked: false });
         createdStereoInputMenuItem = true;
@@ -107,6 +130,14 @@ Menu.menuItemEvent.connect(function (menuItem) {
         Menu.setIsOptionChecked(AUDIO_SOURCE_PINK_NOISE, false);
     } else if (menuItem == AUDIO_STEREO_INPUT) {
         Audio.setStereoInput(Menu.isOptionChecked(AUDIO_STEREO_INPUT))
+    } else if (AUDIO_LISTENER_OPTIONS.indexOf(menuItem) !== -1) {
+        MyAvatar.audioListenerMode = AUDIO_LISTENER_OPTIONS.indexOf(menuItem);
+    }
+});
+
+MyAvatar.audioListenerModeChanged.connect(function() {
+    for (var i = 0; i < AUDIO_LISTENER_OPTIONS.length; i++) {
+        Menu.setIsOptionChecked(AUDIO_LISTENER_OPTIONS[i], (MyAvatar.audioListenerMode === i));
     }
 });
 
@@ -134,6 +165,10 @@ function scriptEnding() {
         Menu.removeMenu(AUDIO_SOURCE_MENU);
     }
 
+    if (createdAudioListenerModeMenu) {
+        Menu.removeMenu(AUDIO_LISTENER_MODE_MENU);
+    }
+
     if (createdStereoInputMenuItem) {
         Menu.removeMenuItem(AUDIO_MENU, AUDIO_STEREO_INPUT);
     }

interface/src/Application.cpp

@@ -4024,6 +4024,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
 
     // hook our avatar and avatar hash map object into this script engine
     scriptEngine->registerGlobalObject("MyAvatar", _myAvatar);
+    qScriptRegisterMetaType(scriptEngine, audioListenModeToScriptValue, audioListenModeFromScriptValue);
+
     scriptEngine->registerGlobalObject("AvatarList", DependencyManager::get<AvatarManager>().data());
 
     scriptEngine->registerGlobalObject("Camera", &_myCamera);

interface/src/Application.h

@@ -142,8 +142,8 @@ public:
     static Application* getInstance() { return qApp; } // TODO: replace fully by qApp
     static const glm::vec3& getPositionForPath() { return getInstance()->_myAvatar->getPosition(); }
     static glm::quat getOrientationForPath() { return getInstance()->_myAvatar->getOrientation(); }
-    static glm::vec3 getPositionForAudio() { return getInstance()->_myAvatar->getHead()->getPosition(); }
+    static glm::vec3 getPositionForAudio() { return getInstance()->_myAvatar->getPositionForAudio(); }
-    static glm::quat getOrientationForAudio() { return getInstance()->_myAvatar->getHead()->getFinalOrientationInWorldFrame(); }
+    static glm::quat getOrientationForAudio() { return getInstance()->_myAvatar->getOrientationForAudio(); }
     static void initPlugins();
     static void shutdownPlugins();
 

interface/src/avatar/MyAvatar.cpp

@@ -109,7 +109,8 @@ MyAvatar::MyAvatar(RigPointer rig) :
     _goToPosition(),
     _goToOrientation(),
     _rig(rig),
-    _prevShouldDrawHead(true)
+    _prevShouldDrawHead(true),
+    _audioListenerMode(FROM_HEAD)
 {
     for (int i = 0; i < MAX_DRIVE_KEYS; i++) {
         _driveKeys[i] = 0.0f;
@@ -1814,3 +1815,42 @@ glm::mat4 MyAvatar::deriveBodyFromHMDSensor() const {
     // avatar facing is determined solely by hmd orientation.
     return createMatFromQuatAndPos(hmdOrientationYawOnly, bodyPos);
 }
+
+glm::vec3 MyAvatar::getPositionForAudio() {
+    switch (_audioListenerMode) {
+        case AudioListenerMode::FROM_HEAD:
+            return getHead()->getPosition();
+        case AudioListenerMode::FROM_CAMERA:
+            return Application::getInstance()->getCamera()->getPosition();
+        case AudioListenerMode::CUSTOM:
+            return _customListenPosition;
+    }
+    return vec3();
+}
+
+glm::quat MyAvatar::getOrientationForAudio() {
+    switch (_audioListenerMode) {
+        case AudioListenerMode::FROM_HEAD:
+            return getHead()->getFinalOrientationInWorldFrame();
+        case AudioListenerMode::FROM_CAMERA:
+            return Application::getInstance()->getCamera()->getOrientation();
+        case AudioListenerMode::CUSTOM:
+            return _customListenOrientation;
+    }
+    return quat();
+}
+
+void MyAvatar::setAudioListenerMode(AudioListenerMode audioListenerMode) {
+    if (_audioListenerMode != audioListenerMode) {
+        _audioListenerMode = audioListenerMode;
+        emit audioListenerModeChanged();
+    }
+}
+
+QScriptValue audioListenModeToScriptValue(QScriptEngine* engine, const AudioListenerMode& audioListenerMode) {
+    return audioListenerMode;
+}
+
+void audioListenModeFromScriptValue(const QScriptValue& object, AudioListenerMode& audioListenerMode) {
+    audioListenerMode = (AudioListenerMode)object.toUInt16();
+}

interface/src/avatar/MyAvatar.h

@@ -26,6 +26,14 @@ enum eyeContactTarget {
     MOUTH
 };
 
+enum AudioListenerMode {
+    FROM_HEAD = 0,
+    FROM_CAMERA,
+    CUSTOM
+};
+Q_DECLARE_METATYPE(AudioListenerMode);
+
+
 class MyAvatar : public Avatar {
     Q_OBJECT
     Q_PROPERTY(bool shouldRenderLocally READ getShouldRenderLocally WRITE setShouldRenderLocally)
@@ -33,12 +41,21 @@ class MyAvatar : public Avatar {
     Q_PROPERTY(float motorTimescale READ getScriptedMotorTimescale WRITE setScriptedMotorTimescale)
     Q_PROPERTY(QString motorReferenceFrame READ getScriptedMotorFrame WRITE setScriptedMotorFrame)
     Q_PROPERTY(QString collisionSoundURL READ getCollisionSoundURL WRITE setCollisionSoundURL)
+    Q_PROPERTY(AudioListenerMode audioListenerMode READ getAudioListenerMode WRITE setAudioListenerMode)
+    Q_PROPERTY(glm::vec3 customListenPosition READ getCustomListenPosition WRITE setCustomListenPosition)
+    Q_PROPERTY(glm::quat customListenOrientation READ getCustomListenOrientation WRITE setCustomListenOrientation)
+    Q_PROPERTY(AudioListenerMode FROM_HEAD READ getAudioListenerModeHead)
+    Q_PROPERTY(AudioListenerMode FROM_CAMERA READ getAudioListenerModeCamera)
+    Q_PROPERTY(AudioListenerMode CUSTOM READ getAudioListenerModeCustom)
     //TODO: make gravity feature work Q_PROPERTY(glm::vec3 gravity READ getGravity WRITE setGravity)
 
 public:
     MyAvatar(RigPointer rig);
     ~MyAvatar();
 
+    AudioListenerMode getAudioListenerModeHead() const { return FROM_HEAD; }
+    AudioListenerMode getAudioListenerModeCamera() const { return FROM_CAMERA; }
+    AudioListenerMode getAudioListenerModeCustom() const { return CUSTOM; }
 
     void reset();
     void update(float deltaTime);
@@ -155,6 +172,13 @@ public:
     void doUpdateBillboard();
     void destroyAnimGraph();
 
+    AudioListenerMode getAudioListenerMode() { return _audioListenerMode; }
+    void setAudioListenerMode(AudioListenerMode audioListenerMode);
+    glm::vec3 getCustomListenPosition() { return _customListenPosition; }
+    void setCustomListenPosition(glm::vec3 customListenPosition) { _customListenPosition = customListenPosition; }
+    glm::quat getCustomListenOrientation() { return _customListenOrientation; }
+    void setCustomListenOrientation(glm::quat customListenOrientation) { _customListenOrientation = customListenOrientation; }
+
 public slots:
     void increaseSize();
     void decreaseSize();
@@ -204,7 +228,11 @@ public slots:
     void setEnableMeshVisible(bool isEnabled);
     void setAnimGraphUrl(const QString& url) { _animGraphUrl = url; }
 
+    glm::vec3 getPositionForAudio();
+    glm::quat getOrientationForAudio();
+
 signals:
+    void audioListenerModeChanged();
     void transformChanged();
     void newCollisionSoundURL(const QUrl& url);
     void collisionWithEntity(const Collision& collision);
@@ -330,6 +358,13 @@ private:
     bool _enableDebugDrawBindPose = false;
     bool _enableDebugDrawAnimPose = false;
     AnimSkeleton::ConstPointer _debugDrawSkeleton = nullptr;
+
+    AudioListenerMode _audioListenerMode;
+    glm::vec3 _customListenPosition;
+    glm::quat _customListenOrientation;
 };
 
+QScriptValue audioListenModeToScriptValue(QScriptEngine* engine, const AudioListenerMode& audioListenerMode);
+void audioListenModeFromScriptValue(const QScriptValue& object, AudioListenerMode& audioListenerMode);
+
 #endif // hifi_MyAvatar_h

libraries/shared/src/RegisteredMetaTypes.h

@@ -59,7 +59,7 @@ void qURLFromScriptValue(const QScriptValue& object, QUrl& url);
 
 QScriptValue qVectorVec3ToScriptValue(QScriptEngine* engine, const QVector<glm::vec3>& vector);
 void qVectorVec3FromScriptValue(const QScriptValue& array, QVector<glm::vec3>& vector);
-QVector<glm::vec3> qVectorVec3FromScriptValue( const QScriptValue& array);
+QVector<glm::vec3> qVectorVec3FromScriptValue(const QScriptValue& array);
 
 QScriptValue qVectorFloatToScriptValue(QScriptEngine* engine, const QVector<float>& vector);
 void qVectorFloatFromScriptValue(const QScriptValue& array, QVector<float>& vector);