Merge pull request #3301 from huffman/19864

Code Review for Job #19864

examples/speechControl.js

@@ -0,0 +1,201 @@
+//
+//  speechControl.js
+//  examples
+//
+//  Created by Ryan Huffman on 07/31/14.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+var ACCELERATION = 80;
+var STEP_DURATION = 1.0; // Duration of a step command in seconds
+var TURN_DEGREES = 90;
+var SLIGHT_TURN_DEGREES = 45;
+var TURN_AROUND_DEGREES = 180;
+var TURN_RATE = 90; // Turn rate in degrees per second
+
+
+/*****************************************************************************/
+/** COMMANDS *****************************************************************/
+var CMD_MOVE_FORWARD = "Move forward";
+var CMD_MOVE_BACKWARD = "Move backward";
+var CMD_MOVE_UP = "Move up";
+var CMD_MOVE_DOWN = "Move down";
+var CMD_MOVE_LEFT = "Move left";
+var CMD_MOVE_RIGHT = "Move right";
+
+var CMD_STEP_FORWARD = "Step forward";
+var CMD_STEP_BACKWARD = "Step backward";
+var CMD_STEP_LEFT = "Step left";
+var CMD_STEP_RIGHT = "Step right";
+var CMD_STEP_UP = "Step up";
+var CMD_STEP_DOWN = "Step down";
+
+var CMD_TURN_LEFT = "Turn left";
+var CMD_TURN_SLIGHT_LEFT = "Turn slight left";
+var CMD_TURN_RIGHT = "Turn right";
+var CMD_TURN_SLIGHT_RIGHT = "Turn slight right";
+var CMD_TURN_AROUND = "Turn around";
+
+var CMD_STOP = "Stop";
+
+var CMD_SHOW_COMMANDS = "Show commands";
+
+var MOVE_COMMANDS = [
+    CMD_MOVE_FORWARD,
+    CMD_MOVE_BACKWARD,
+    CMD_MOVE_UP,
+    CMD_MOVE_DOWN,
+    CMD_MOVE_LEFT,
+    CMD_MOVE_RIGHT,
+];
+
+var STEP_COMMANDS = [
+    CMD_STEP_FORWARD,
+    CMD_STEP_BACKWARD,
+    CMD_STEP_UP,
+    CMD_STEP_DOWN,
+    CMD_STEP_LEFT,
+    CMD_STEP_RIGHT,
+];
+
+var TURN_COMMANDS = [
+    CMD_TURN_LEFT,
+    CMD_TURN_SLIGHT_LEFT,
+    CMD_TURN_RIGHT,
+    CMD_TURN_SLIGHT_RIGHT,
+    CMD_TURN_AROUND,
+];
+
+var OTHER_COMMANDS = [
+    CMD_STOP,
+    CMD_SHOW_COMMANDS,
+];
+
+var ALL_COMMANDS = []
+    .concat(MOVE_COMMANDS)
+    .concat(STEP_COMMANDS)
+    .concat(TURN_COMMANDS)
+    .concat(OTHER_COMMANDS);
+
+/** END OF COMMANDS **********************************************************/
+/*****************************************************************************/
+
+
+var currentCommandFunc = null;
+
+function handleCommandRecognized(command) {
+    if (MOVE_COMMANDS.indexOf(command) > -1 || STEP_COMMANDS.indexOf(command) > -1) {
+        // If this is a STEP_* command, we will want to countdown the duration
+        // of time to move.  MOVE_* commands don't stop.
+        var timeRemaining = MOVE_COMMANDS.indexOf(command) > -1 ? 0 : STEP_DURATION;
+        var accel = { x: 0, y: 0, z: 0 };
+
+        if (command == CMD_MOVE_FORWARD || command == CMD_STEP_FORWARD) {
+            accel = { x: 0, y: 0, z: 1 };
+        } else if (command == CMD_MOVE_BACKWARD || command == CMD_STEP_BACKWARD) {
+            accel = { x: 0, y: 0, z: -1 };
+        } else if (command === CMD_MOVE_UP || command == CMD_STEP_UP) {
+            accel = { x: 0, y: 1, z: 0 };
+        } else if (command == CMD_MOVE_DOWN || command == CMD_STEP_DOWN) {
+            accel = { x: 0, y: -1, z: 0 };
+        } else if (command == CMD_MOVE_LEFT || command == CMD_STEP_LEFT) {
+            accel = { x: -1, y: 0, z: 0 };
+        } else if (command == CMD_MOVE_RIGHT || command == CMD_STEP_RIGHT) {
+            accel = { x: 1, y: 0, z: 0 };
+        }
+
+        currentCommandFunc = function(dt) {
+            if (timeRemaining > 0 && dt >= timeRemaining) {
+                dt = timeRemaining;
+            }
+
+            var headOrientation = MyAvatar.headOrientation;
+            var front = Quat.getFront(headOrientation);
+            var right = Quat.getRight(headOrientation);
+            var up = Quat.getUp(headOrientation);
+
+            var thrust = Vec3.multiply(front, accel.z * ACCELERATION);
+            thrust = Vec3.sum(thrust, Vec3.multiply(right, accel.x * ACCELERATION));
+            thrust = Vec3.sum(thrust, Vec3.multiply(up, accel.y * ACCELERATION));
+            MyAvatar.addThrust(thrust);
+
+            if (timeRemaining > 0) {
+                timeRemaining -= dt;
+                return timeRemaining > 0;
+            }
+
+            return true;
+        };
+    } else if (TURN_COMMANDS.indexOf(command) > -1) {
+        var degreesRemaining;
+        var sign;
+        if (command == CMD_TURN_LEFT) {
+            sign = 1;
+            degreesRemaining = TURN_DEGREES;
+        } else if (command == CMD_TURN_RIGHT) {
+            sign = -1;
+            degreesRemaining = TURN_DEGREES;
+        } else if (command == CMD_TURN_SLIGHT_LEFT) {
+            sign = 1;
+            degreesRemaining = SLIGHT_TURN_DEGREES;
+        } else if (command == CMD_TURN_SLIGHT_RIGHT) {
+            sign = -1;
+            degreesRemaining = SLIGHT_TURN_DEGREES;
+        } else if (command == CMD_TURN_AROUND) {
+            sign = 1;
+            degreesRemaining = TURN_AROUND_DEGREES;
+        }
+        currentCommandFunc = function(dt) {
+            // Determine how much to turn by
+            var turnAmount = TURN_RATE * dt;
+            if (turnAmount > degreesRemaining) {
+                turnAmount = degreesRemaining;
+            }
+
+            // Apply turn
+            var orientation = MyAvatar.orientation;
+            var deltaOrientation = Quat.fromPitchYawRollDegrees(0, sign * turnAmount, 0);
+            MyAvatar.orientation = Quat.multiply(orientation, deltaOrientation);
+
+            degreesRemaining -= turnAmount;
+            return turnAmount > 0;
+        }
+    } else if (command == CMD_STOP) {
+        currentCommandFunc = null;
+    } else if (command == CMD_SHOW_COMMANDS) {
+        var msg = "";
+        for (var i = 0; i < ALL_COMMANDS.length; i++) {
+            msg += ALL_COMMANDS[i] + "\n";
+        }
+        Window.alert(msg);
+    }
+}
+
+function update(dt) {
+    if (currentCommandFunc) {
+        if (currentCommandFunc(dt) === false) {
+            currentCommandFunc = null;
+        }
+    }
+}
+
+function setup() {
+    for (var i = 0; i < ALL_COMMANDS.length; i++) {
+        SpeechRecognizer.addCommand(ALL_COMMANDS[i]);
+    }
+}
+
+function scriptEnding() {
+    for (var i = 0; i < ALL_COMMANDS.length; i++) {
+        SpeechRecognizer.removeCommand(ALL_COMMANDS[i]);
+    }
+}
+
+Script.scriptEnding.connect(scriptEnding);
+Script.update.connect(update);
+SpeechRecognizer.commandRecognized.connect(handleCommandRecognized);
+
+setup();

interface/CMakeLists.txt

@@ -46,6 +46,15 @@ foreach(SUBDIR avatar devices renderer ui starfield location scripting voxels pa
   set(INTERFACE_SRCS ${INTERFACE_SRCS} "${SUBDIR_SRCS}")
 endforeach(SUBDIR)
 
+# Add SpeechRecognizer if on OS X, otherwise remove
+if (APPLE)
+    file(GLOB INTERFACE_OBJCPP_SRCS "src/SpeechRecognizer.mm")
+    set(INTERFACE_SRCS ${INTERFACE_SRCS} ${INTERFACE_OBJCPP_SRCS})
+else ()
+    get_filename_component(SPEECHRECOGNIZER_H "src/SpeechRecognizer.h" ABSOLUTE)
+    list(REMOVE_ITEM INTERFACE_SRCS ${SPEECHRECOGNIZER_H})
+endif ()
+
 find_package(Qt5 COMPONENTS Gui Multimedia Network OpenGL Script Svg WebKitWidgets)
 
 # grab the ui files in resources/ui

interface/src/Application.cpp

@@ -3749,6 +3749,10 @@ ScriptEngine* Application::loadScript(const QString& scriptName, bool loadScript
     scriptEngine->registerGlobalObject("Camera", cameraScriptable);
     connect(scriptEngine, SIGNAL(finished(const QString&)), cameraScriptable, SLOT(deleteLater()));
 
+#ifdef Q_OS_MAC
+    scriptEngine->registerGlobalObject("SpeechRecognizer", Menu::getInstance()->getSpeechRecognizer());
+#endif
+
     ClipboardScriptingInterface* clipboardScriptable = new ClipboardScriptingInterface();
     scriptEngine->registerGlobalObject("Clipboard", clipboardScriptable);
     connect(scriptEngine, SIGNAL(finished(const QString&)), clipboardScriptable, SLOT(deleteLater()));

interface/src/Menu.cpp

@@ -94,6 +94,9 @@ Menu::Menu() :
     _octreeStatsDialog(NULL),
     _lodToolsDialog(NULL),
     _userLocationsDialog(NULL),
+#ifdef Q_OS_MAC
+    _speechRecognizer(),
+#endif
     _maxVoxels(DEFAULT_MAX_VOXELS_PER_SYSTEM),
     _voxelSizeScale(DEFAULT_OCTREE_SIZE_SCALE),
     _oculusUIAngularSize(DEFAULT_OCULUS_UI_ANGULAR_SIZE),
@@ -225,6 +228,12 @@ Menu::Menu() :
     addActionToQMenuAndActionHash(toolsMenu, MenuOption::MetavoxelEditor, 0, this, SLOT(showMetavoxelEditor()));
     addActionToQMenuAndActionHash(toolsMenu, MenuOption::ScriptEditor,  Qt::ALT | Qt::Key_S, this, SLOT(showScriptEditor()));
 
+#ifdef Q_OS_MAC
+    QAction* speechRecognizerAction = addCheckableActionToQMenuAndActionHash(toolsMenu, MenuOption::ControlWithSpeech,
+            Qt::CTRL | Qt::SHIFT | Qt::Key_C, _speechRecognizer.getEnabled(), &_speechRecognizer, SLOT(setEnabled(bool)));
+    connect(&_speechRecognizer, SIGNAL(enabledUpdated(bool)), speechRecognizerAction, SLOT(setChecked(bool)));
+#endif
+
 #ifdef HAVE_QXMPP
     _chatAction = addActionToQMenuAndActionHash(toolsMenu,
                                                 MenuOption::Chat,
@@ -688,6 +697,10 @@ void Menu::loadSettings(QSettings* settings) {
                                          QStandardPaths::writableLocation(QStandardPaths::DesktopLocation)).toString();
     setScriptsLocation(settings->value("scriptsLocation", QString()).toString());
 
+#ifdef Q_OS_MAC
+    _speechRecognizer.setEnabled(settings->value("speechRecognitionEnabled", false).toBool());
+#endif
+
     settings->beginGroup("View Frustum Offset Camera");
     // in case settings is corrupt or missing loadSetting() will check for NaN
     _viewFrustumOffset.yaw = loadSetting(settings, "viewFrustumOffsetYaw", 0.0f);
@@ -735,6 +748,9 @@ void Menu::saveSettings(QSettings* settings) {
     settings->setValue("boundaryLevelAdjust", _boundaryLevelAdjust);
     settings->setValue("snapshotsLocation", _snapshotsLocation);
     settings->setValue("scriptsLocation", _scriptsLocation);
+#ifdef Q_OS_MAC
+    settings->setValue("speechRecognitionEnabled", _speechRecognizer.getEnabled());
+#endif
     settings->beginGroup("View Frustum Offset Camera");
     settings->setValue("viewFrustumOffsetYaw", _viewFrustumOffset.yaw);
     settings->setValue("viewFrustumOffsetPitch", _viewFrustumOffset.pitch);

interface/src/Menu.h

@@ -23,6 +23,10 @@
 #include <MenuItemProperties.h>
 #include <OctreeConstants.h>
 
+#ifdef Q_OS_MAC
+#include "SpeechRecognizer.h"
+#endif
+
 #include "location/LocationManager.h"
 #include "ui/PreferencesDialog.h"
 #include "ui/ChatWindow.h"
@@ -137,6 +141,10 @@ public:
     void setBoundaryLevelAdjust(int boundaryLevelAdjust);
     int getBoundaryLevelAdjust() const { return _boundaryLevelAdjust; }
 
+#ifdef Q_OS_MAC
+    SpeechRecognizer* getSpeechRecognizer() { return &_speechRecognizer; }
+#endif
+
     // User Tweakable PPS from Voxel Server
     int getMaxVoxelPacketsPerSecond() const { return _maxVoxelPacketsPerSecond; }
     void setMaxVoxelPacketsPerSecond(int maxVoxelPacketsPerSecond) { _maxVoxelPacketsPerSecond = maxVoxelPacketsPerSecond; }
@@ -272,6 +280,9 @@ private:
     OctreeStatsDialog* _octreeStatsDialog;
     LodToolsDialog* _lodToolsDialog;
     UserLocationsDialog* _userLocationsDialog;
+#ifdef Q_OS_MAC
+    SpeechRecognizer _speechRecognizer;
+#endif
     int _maxVoxels;
     float _voxelSizeScale;
     float _oculusUIAngularSize;
@@ -351,6 +362,7 @@ namespace MenuOption {
     const QString CollideWithVoxels = "Collide With Voxels";
     const QString Collisions = "Collisions";
     const QString Console = "Console...";
+    const QString ControlWithSpeech = "Control With Speech";
     const QString DecreaseAvatarSize = "Decrease Avatar Size";
     const QString DecreaseVoxelSize = "Decrease Voxel Size";
     const QString DisableActivityLogger = "Disable Activity Logger";

interface/src/SpeechRecognizer.h

@@ -0,0 +1,47 @@
+//
+//  SpeechRecognizer.h
+//  interface/src
+//
+//  Created by Ryan Huffman on 07/31/14.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SpeechRecognizer_h
+#define hifi_SpeechRecognizer_h
+
+#include <QObject>
+#include <QSet>
+#include <QString>
+
+class SpeechRecognizer : public QObject {
+    Q_OBJECT
+public:
+    SpeechRecognizer();
+    ~SpeechRecognizer();
+
+    void handleCommandRecognized(const char* command);
+    bool getEnabled() const { return _enabled; }
+
+public slots:
+    void setEnabled(bool enabled);
+    void addCommand(const QString& command);
+    void removeCommand(const QString& command);
+
+signals:
+    void commandRecognized(const QString& command);
+    void enabledUpdated(bool enabled);
+
+protected:
+    void reloadCommands();
+
+private:
+    bool _enabled;
+    QSet<QString> _commands;
+    void* _speechRecognizerDelegate;
+    void* _speechRecognizer;
+};
+
+#endif // hifi_SpeechRecognizer_h

interface/src/SpeechRecognizer.mm

@@ -0,0 +1,109 @@
+//
+//  SpeechRecognizer.mm
+//  interface/src
+//
+//  Created by Ryan Huffman on 07/31/14.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <QtGlobal>
+#ifdef Q_OS_MAC
+
+#import <Foundation/Foundation.h>
+#import <AppKit/NSSpeechRecognizer.h>
+#import <AppKit/NSWorkspace.h>
+
+#include <QDebug>
+
+#include "SpeechRecognizer.h"
+
+@interface SpeechRecognizerDelegate : NSObject <NSSpeechRecognizerDelegate> {
+    SpeechRecognizer* _listener;
+}
+
+- (void)setListener:(SpeechRecognizer*)listener;
+- (void)speechRecognizer:(NSSpeechRecognizer*)sender didRecognizeCommand:(id)command;
+
+@end
+
+@implementation SpeechRecognizerDelegate
+
+- (void)setListener:(SpeechRecognizer*)listener {
+    _listener = listener;
+}
+
+- (void)speechRecognizer:(NSSpeechRecognizer*)sender didRecognizeCommand:(id)command {
+    _listener->handleCommandRecognized(((NSString*)command).UTF8String);
+}
+
+@end
+
+SpeechRecognizer::SpeechRecognizer() :
+    QObject(),
+    _enabled(false),
+    _commands(),
+    _speechRecognizerDelegate([[SpeechRecognizerDelegate alloc] init]),
+    _speechRecognizer(NULL) {
+
+    [(id)_speechRecognizerDelegate setListener:this];
+}
+
+SpeechRecognizer::~SpeechRecognizer() {
+    if (_speechRecognizer) {
+        [(id)_speechRecognizer dealloc];
+    }
+    if (_speechRecognizerDelegate) {
+        [(id)_speechRecognizerDelegate dealloc];
+    }
+}
+
+void SpeechRecognizer::handleCommandRecognized(const char* command) {
+    emit commandRecognized(QString(command));
+}
+
+void SpeechRecognizer::setEnabled(bool enabled) {
+    if (enabled == _enabled) {
+        return;
+    }
+
+    _enabled = enabled;
+    if (_enabled) {
+        _speechRecognizer = [[NSSpeechRecognizer alloc] init];
+
+        reloadCommands();
+
+        [(id)_speechRecognizer setDelegate:(id)_speechRecognizerDelegate];
+        [(id)_speechRecognizer startListening];
+    } else {
+        [(id)_speechRecognizer stopListening];
+        [(id)_speechRecognizer dealloc];
+        _speechRecognizer = NULL;
+    }
+
+    emit enabledUpdated(_enabled);
+}
+
+void SpeechRecognizer::reloadCommands() {
+    if (_speechRecognizer) {
+        NSMutableArray* cmds = [NSMutableArray array];
+        for (QSet<QString>::const_iterator iter = _commands.constBegin(); iter != _commands.constEnd(); iter++) {
+            [cmds addObject:[NSString stringWithUTF8String:(*iter).toLocal8Bit().data()]];
+        }
+        [(id)_speechRecognizer setCommands:cmds];
+    }
+}
+
+void SpeechRecognizer::addCommand(const QString& command) {
+    _commands.insert(command);
+    reloadCommands();
+}
+
+void SpeechRecognizer::removeCommand(const QString& command) {
+    _commands.remove(command);
+    reloadCommands();
+}
+
+#endif // Q_OS_MAC