Merge pull request #9316 from hyperlogic/tablet-ui

Added local only spatialized button click sounds to tablet

interface/resources/qml/hifi/tablet/Tablet.qml

@@ -40,6 +40,9 @@ Item {
             button[key] = properties[key];
         });
 
+        // pass a reference to the tabletRoot object to the button.
+        button.tabletRoot = parent.parent;
+
         return button;
     }
 

interface/resources/qml/hifi/tablet/TabletButton.qml

@@ -11,6 +11,7 @@ Item {
     property bool isActive: false
     property bool inDebugMode: false
     property bool isEntered: false
+    property var tabletRoot;
     width: 129
     height: 129
 
@@ -118,6 +119,9 @@ Item {
                 }
             }
             tabletButton.clicked();
+            if (tabletRoot) {
+                tabletRoot.playButtonClickSound();
+            }
         }
         onEntered: {
             console.log("Tablet Button Hovered!");

interface/resources/qml/hifi/tablet/TabletRoot.qml

@@ -1,4 +1,5 @@
 import QtQuick 2.0
+import Hifi 1.0
 
 Item {
     id: tabletRoot
@@ -13,6 +14,15 @@ Item {
         loader.item.url = url;
     }
 
+    SoundEffect {
+        id: buttonClickSound
+        source: "../../../sounds/button-click.wav"
+    }
+
+    function playButtonClickSound() {
+        buttonClickSound.play();
+    }
+
     Loader {
         id: loader
         objectName: "loader"

libraries/script-engine/src/TabletScriptingInterface.cpp

@@ -10,8 +10,18 @@
 
 #include <QtCore/QThread>
 
+#include <AudioInjector.h>
+#include <PathUtils.h>
 #include "ScriptEngineLogging.h"
 
+TabletScriptingInterface::TabletScriptingInterface() {
+    qmlRegisterType<SoundEffect>("Hifi", 1, 0, "SoundEffect");
+}
+
+TabletScriptingInterface::~TabletScriptingInterface() {
+    qDebug() << "Shutting down TabletScriptingInterface";
+}
+
 QObject* TabletScriptingInterface::getTablet(const QString& tabletId) {
 
     std::lock_guard<std::mutex> guard(_mutex);
@@ -31,7 +41,7 @@ QObject* TabletScriptingInterface::getTablet(const QString& tabletId) {
 
 void TabletScriptingInterface::setQmlTabletRoot(QString tabletId, QQuickItem* qmlTabletRoot, QObject* qmlOffscreenSurface) {
     TabletProxy* tablet = qobject_cast<TabletProxy*>(getTablet(tabletId));
-    if (tablet && qmlOffscreenSurface) {
+    if (tablet) {
         tablet->setQmlTabletRoot(qmlTabletRoot, qmlOffscreenSurface);
     } else {
         qCWarning(scriptengine) << "TabletScriptingInterface::setupTablet() bad tablet object";
@@ -155,6 +165,10 @@ void TabletProxy::updateAudioBar(const double micLevel) {
     }
 }
 
+void TabletProxy::updateTabletPosition(glm::vec3 tabletPosition) {
+    _position.set(tabletPosition);
+}
+
 void TabletProxy::emitScriptEvent(QVariant msg) {
     if (_qmlOffscreenSurface) {
         QMetaObject::invokeMethod(_qmlOffscreenSurface, "emitScriptEvent", Qt::AutoConnection, Q_ARG(QVariant, msg));
@@ -233,4 +247,46 @@ void TabletButtonProxy::editProperties(QVariantMap properties) {
     }
 }
 
+//
+// SoundEffect
+//
+
+SoundEffect::~SoundEffect() {
+    if (_sound) {
+        _sound->deleteLater();
+    }
+    if (_injector) {
+        _injector->deleteLater();
+    }
+}
+
+QUrl SoundEffect::getSource() const {
+    return _url;
+}
+
+void SoundEffect::setSource(QUrl url) {
+    _url = url;
+    _sound = DependencyManager::get<SoundCache>()->getSound(_url);
+}
+
+void SoundEffect::play() {
+    auto tsi = DependencyManager::get<TabletScriptingInterface>();
+    if (tsi) {
+        TabletProxy* tablet = qobject_cast<TabletProxy*>(tsi->getTablet("com.highfidelity.interface.tablet.system"));
+        if (tablet) {
+            AudioInjectorOptions options;
+            options.position = tablet->getPosition();
+            options.localOnly = true;
+            if (_injector) {
+                _injector->setOptions(options);
+                _injector->restart();
+            } else {
+                QByteArray samples = _sound->getByteArray();
+                _injector = AudioInjector::playSound(samples, options);
+            }
+        }
+    }
+}
+
 #include "TabletScriptingInterface.moc"
+

libraries/script-engine/src/TabletScriptingInterface.h

@@ -10,6 +10,7 @@
 #define hifi_TabletScriptingInterface_h
 
 #include <mutex>
+#include <atomic>
 
 #include <QObject>
 #include <QVariant>
@@ -17,11 +18,17 @@
 #include <QQuickItem>
 #include <QUuid>
 
-#include <DependencyManager.h>
+#include <glm/glm.hpp>
+#include <glm/gtc/quaternion.hpp>
+#include <glm/gtx/quaternion.hpp>
 
+#include <DependencyManager.h>
+#include <SoundCache.h>
+#include <ThreadSafeValueCache.h>
 
 class TabletProxy;
 class TabletButtonProxy;
+class AudioInjector;
 
 /**jsdoc
  * @namespace Tablet
@@ -29,6 +36,9 @@ class TabletButtonProxy;
 class TabletScriptingInterface : public QObject, public Dependency {
     Q_OBJECT
 public:
+    TabletScriptingInterface();
+    virtual ~TabletScriptingInterface();
+
     /**jsdoc
      * Creates or retruns a new TabletProxy and returns it.
      * @function Tablet.getTablet
@@ -85,12 +95,23 @@ public:
     Q_INVOKABLE void removeButton(QObject* tabletButtonProxy);
 
     /**jsdoc
-     * @function TabletProxy#updateAudioBar
      * Updates the audio bar in tablet to reflect latest mic level
+     * @function TabletProxy#updateAudioBar
      * @param micLevel {double} mic level value between 0 and 1
      */
     Q_INVOKABLE void updateAudioBar(const double micLevel);
 
+
+    /**jsdoc
+     * Updates the tablet's position in world space.  This is necessary for the tablet
+     * to emit audio with the correct spatialization.
+     * @function TabletProxy#updateTabletPosition
+     * @param position {vec3} tablet position in world space.
+     */
+    Q_INVOKABLE void updateTabletPosition(glm::vec3 tabletPosition);
+
+    glm::vec3 getPosition() const { return _position.get(); }
+
     QString getName() const { return _name; }
 
     /**jsdoc
@@ -120,6 +141,7 @@ protected:
     std::vector<QSharedPointer<TabletButtonProxy>> _tabletButtonProxies;
     QQuickItem* _qmlTabletRoot { nullptr };
     QObject* _qmlOffscreenSurface { nullptr };
+    ThreadSafeValueCache<glm::vec3> _position;
 };
 
 /**jsdoc
@@ -168,4 +190,24 @@ protected:
     QVariantMap _properties;
 };
 
+
+// Exposed to qml only, not java script
+class SoundEffect : public QQuickItem {
+    Q_OBJECT
+    Q_PROPERTY(QUrl source READ getSource WRITE setSource)
+public:
+
+    virtual ~SoundEffect();
+
+    QUrl getSource() const;
+    void setSource(QUrl url);
+
+    Q_INVOKABLE void play();
+protected:
+    QUrl _url;
+    SharedSoundPointer _sound;
+    AudioInjector* _injector { nullptr };
+};
+
+
 #endif // hifi_TabletScriptingInterface_h

scripts/system/libraries/WebTablet.js

@@ -171,3 +171,8 @@ WebTablet.unpickle = function (string) {
     tablet.__proto__ = WebTablet.prototype;
     return tablet;
 };
+
+WebTablet.prototype.getPosition = function () {
+    return Overlays.getProperty(this.webOverlayID, "position");
+}
+

scripts/system/tablet-ui/tabletUI.js

@@ -48,8 +48,11 @@
     function updateShowTablet() {
         if (tabletShown) {
             var currentMicLevel = getMicLevel();
-            var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");      
+            var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
             tablet.updateAudioBar(currentMicLevel);
+            if (UIWebTablet) {
+               tablet.updateTabletPosition(UIWebTablet.getPosition());
+            }
         }
 
         if (HMD.showTablet && !tabletShown) {
@@ -83,7 +86,7 @@
         accumulatedLevel = AVERAGING_RATIO * accumulatedLevel + (1 - AVERAGING_RATIO) * (MyAvatar.audioLoudness);
         // Convert to log base 2
         var logLevel = Math.log(accumulatedLevel + 1) / LOG2;
-        
+
         if (logLevel <= LOUDNESS_FLOOR) {
             micLevel = logLevel / LOUDNESS_FLOOR * LOUDNESS_SCALE;
         } else {
@@ -94,6 +97,4 @@
         }
         return micLevel;
     }
-
-
 }()); // END LOCAL_SCOPE