Merge branch 'master' of https://github.com/worklist/hifi into 19368

audio-mixer/src/main.cpp

@@ -137,15 +137,8 @@ int main(int argc, const char* argv[]) {
                 // if we should be sending stats to Logstash send the appropriate average now
                 const char MIXER_LOGSTASH_METRIC_NAME[] = "audio-mixer-frame-time-usage";
                 
-                // we're sending a floating point percentage with two mandatory numbers after decimal point
-                // that could be up to 6 bytes
-                const int MIXER_LOGSTASH_PACKET_BYTES = strlen(MIXER_LOGSTASH_METRIC_NAME) + 7;
-                char logstashPacket[MIXER_LOGSTASH_PACKET_BYTES];
-                
                 float averageFrameTimePercentage = sumFrameTimePercentages / numStatCollections;
-                int packetBytes = sprintf(logstashPacket, "%s %.2f", MIXER_LOGSTASH_METRIC_NAME, averageFrameTimePercentage);
+                Logstash::stashValue(MIXER_LOGSTASH_METRIC_NAME, averageFrameTimePercentage);
-                
-                agentList->getAgentSocket()->send(Logstash::socket(), logstashPacket, packetBytes);
                 
                 sumFrameTimePercentages = 0.0f;
                 numStatCollections = 0;

interface/src/Application.cpp

@@ -45,14 +45,15 @@
 #include <QtDebug>
 #include <QFileDialog>
 #include <QDesktopServices>
-#include <PairingHandler.h>
 
 #include <AgentTypes.h>
-#include <PacketHeaders.h>
-#include <PerfStat.h>
 #include <AudioInjectionManager.h>
 #include <AudioInjector.h>
+#include <Logstash.h>
 #include <OctalCode.h>
+#include <PacketHeaders.h>
+#include <PairingHandler.h>
+#include <PerfStat.h>
 
 #include "Application.h"
 #include "InterfaceConfig.h"
@@ -75,6 +76,10 @@ const glm::vec3 START_LOCATION(4.f, 0.f, 5.f);   //  Where one's own agent begin
 const int IDLE_SIMULATE_MSECS = 16;              //  How often should call simulate and other stuff
                                                  //  in the idle loop?  (60 FPS is default)
 
+const int STARTUP_JITTER_SAMPLES = PACKET_LENGTH_SAMPLES_PER_CHANNEL / 2;
+                                                 //  Startup optimistically with small jitter buffer that 
+                                                 //  will start playback on the second received audio packet.
+
 // customized canvas that simply forwards requests/events to the singleton application
 class GLCanvas : public QGLWidget {
 protected:
@@ -90,11 +95,14 @@ protected:
     virtual void mousePressEvent(QMouseEvent* event);
     virtual void mouseReleaseEvent(QMouseEvent* event);
     
+    virtual bool event(QEvent* event);
+    
     virtual void wheelEvent(QWheelEvent* event);
 };
 
 void GLCanvas::initializeGL() {
     Application::getInstance()->initializeGL();
+    setAttribute(Qt::WA_AcceptTouchEvents);
 }
 
 void GLCanvas::paintGL() {
@@ -125,6 +133,25 @@ void GLCanvas::mouseReleaseEvent(QMouseEvent* event) {
     Application::getInstance()->mouseReleaseEvent(event);
 }
 
+int updateTime = 0;
+bool GLCanvas::event(QEvent* event) {
+    switch (event->type()) {
+        case QEvent::TouchBegin:
+            Application::getInstance()->touchBeginEvent(static_cast<QTouchEvent*>(event));
+            event->accept();
+            return true;
+        case QEvent::TouchEnd:
+            Application::getInstance()->touchEndEvent(static_cast<QTouchEvent*>(event));
+            return true;
+        case QEvent::TouchUpdate:
+            Application::getInstance()->touchUpdateEvent(static_cast<QTouchEvent*>(event));
+            return true;
+        default:
+            break;
+    }
+    return QGLWidget::event(event);
+}
+
 void GLCanvas::wheelEvent(QWheelEvent* event) {
     Application::getInstance()->wheelEvent(event);
 }
@@ -147,6 +174,9 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
         _audioScope(256, 200, true),
         _mouseX(0),
         _mouseY(0),
+        _touchAvgX(0.0f),
+        _touchAvgY(0.0f),
+        _isTouchPressed(false),
         _mousePressed(false),
         _mouseVoxelScale(1.0f / 1024.0f),
         _justEditedVoxel(false),
@@ -158,7 +188,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
         _oculusProgram(0),
         _oculusDistortionScale(1.25),
 #ifndef _WIN32
-        _audio(&_audioScope),
+        _audio(&_audioScope, STARTUP_JITTER_SAMPLES),
 #endif
         _stopNetworkReceiveThread(false),  
         _packetCount(0),
@@ -288,12 +318,17 @@ void Application::initializeGL() {
     idleTimer->start(0);
     
     if (_justStarted) {
-        float startupTime = (usecTimestampNow() - usecTimestamp(&_applicationStartupTime))/1000000.0;
+        float startupTime = (usecTimestampNow() - usecTimestamp(&_applicationStartupTime)) / 1000000.0;
         _justStarted = false;
         char title[50];
         sprintf(title, "Interface: %4.2f seconds\n", startupTime);
         printLog("%s", title);
         _window->setWindowTitle(title);
+        
+        const char LOGSTASH_INTERFACE_START_TIME_KEY[] = "interface-start-time";
+        
+        // ask the Logstash class to record the startup time
+        Logstash::stashValue(LOGSTASH_INTERFACE_START_TIME_KEY, startupTime);
     }
     
     // update before the first render
@@ -306,7 +341,7 @@ void Application::paintGL() {
     glEnable(GL_LINE_SMOOTH);
     glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
     
-    float headCameraScale = _serialHeadSensor.active ? _headCameraPitchYawScale : 1.0f;
+    float headCameraScale = _serialHeadSensor.isActive() ? _headCameraPitchYawScale : 1.0f;
     
     if (_myCamera.getMode() == CAMERA_MODE_MIRROR) {
         _myCamera.setTightness     (100.0f); 
@@ -535,6 +570,7 @@ void Application::keyPressEvent(QKeyEvent* event) {
                 
             case Qt::Key_Space:
                 resetSensors();
+                _audio.reset();
                 break;
                 
             case Qt::Key_G:
@@ -759,6 +795,40 @@ void Application::mouseReleaseEvent(QMouseEvent* event) {
     }
 }
 
+void Application::touchUpdateEvent(QTouchEvent* event) {
+    bool validTouch = false;
+    if (activeWindow() == _window) {
+        const QList<QTouchEvent::TouchPoint>& tPoints = event->touchPoints();
+        _touchAvgX = 0.0f;
+        _touchAvgY = 0.0f;
+        int numTouches = tPoints.count();
+        if (numTouches > 1) {
+            for (int i = 0; i < numTouches; ++i) {
+                _touchAvgX += tPoints[i].pos().x();
+                _touchAvgY += tPoints[i].pos().y();
+            }
+            _touchAvgX /= (float)(numTouches);
+            _touchAvgY /= (float)(numTouches);
+            validTouch = true;
+        }
+    }
+    if (!_isTouchPressed) {
+        _touchDragStartedAvgX = _touchAvgX;
+        _touchDragStartedAvgY = _touchAvgY;
+    }
+    _isTouchPressed = validTouch;
+}
+
+void Application::touchBeginEvent(QTouchEvent* event) {
+    touchUpdateEvent(event);
+}
+
+void Application::touchEndEvent(QTouchEvent* event) {
+    _touchDragStartedAvgX = _touchAvgX;
+    _touchDragStartedAvgY = _touchAvgY;
+    _isTouchPressed = false;
+}
+
 void Application::wheelEvent(QWheelEvent* event) {
     if (activeWindow() == _window) {
         if (checkedVoxelModeAction() == 0) {
@@ -786,7 +856,7 @@ void Application::timer() {
     gettimeofday(&_timerStart, NULL);
     
     // if we haven't detected gyros, check for them now
-    if (!_serialHeadSensor.active) {
+    if (!_serialHeadSensor.isActive()) {
         _serialHeadSensor.pair();
     }
     
@@ -908,6 +978,12 @@ void Application::editPreferences() {
     leanScale->setValue(_myAvatar.getLeanScale());
     form->addRow("Lean Scale:", leanScale);
 
+    QSpinBox* audioJitterBufferSamples = new QSpinBox();
+    audioJitterBufferSamples->setMaximum(10000);
+    audioJitterBufferSamples->setMinimum(-10000);
+    audioJitterBufferSamples->setValue(_audioJitterBufferSamples);
+    form->addRow("Audio Jitter Buffer Samples (0 for automatic):", audioJitterBufferSamples);
+
     QDialogButtonBox* buttons = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel);
     dialog.connect(buttons, SIGNAL(accepted()), SLOT(accept()));
     dialog.connect(buttons, SIGNAL(rejected()), SLOT(reject()));
@@ -922,21 +998,20 @@ void Application::editPreferences() {
     _audio.setIsCancellingEcho( audioEchoCancellation->isChecked() );
     _headCameraPitchYawScale = headCameraPitchYawScale->value();
     _myAvatar.setLeanScale(leanScale->value());
+    _audioJitterBufferSamples = audioJitterBufferSamples->value();
+    if (!shouldDynamicallySetJitterBuffer()) {
+        _audio.setJitterBufferSamples(_audioJitterBufferSamples);
+    }
 }
 
 void Application::pair() {
     PairingHandler::sendPairRequest();
 }
 
-void Application::setHead(bool head) {    
+void Application::setRenderMirrored(bool mirrored) {
-    if (head) {
+    if (mirrored) {
-        _myCamera.setMode(CAMERA_MODE_MIRROR);
-        _myCamera.setModeShiftRate(100.0f);
         _manualFirstPerson->setChecked(false);
-        
+        _manualThirdPerson->setChecked(false);
-    } else {
-        _myCamera.setMode(CAMERA_MODE_THIRD_PERSON);
-        _myCamera.setModeShiftRate(1.0f);
     }
 }
 
@@ -951,8 +1026,16 @@ void Application::setFullscreen(bool fullscreen) {
 }
 
 void Application::setRenderFirstPerson(bool firstPerson) {
-    if (firstPerson && _lookingInMirror->isChecked()) {
+    if (firstPerson) {
-        _lookingInMirror->trigger();
+        _lookingInMirror->setChecked(false);
+        _manualThirdPerson->setChecked(false);
+    }
+}
+
+void Application::setRenderThirdPerson(bool thirdPerson) {
+    if (thirdPerson) {
+        _lookingInMirror->setChecked(false);
+        _manualFirstPerson->setChecked(false);
     }
 }
 
@@ -1253,7 +1336,7 @@ void Application::initMenu() {
     pairMenu->addAction("Pair", this, SLOT(pair()));
     
     QMenu* optionsMenu = menuBar->addMenu("Options");
-    (_lookingInMirror = optionsMenu->addAction("Mirror", this, SLOT(setHead(bool)), Qt::Key_H))->setCheckable(true);
+    (_lookingInMirror = optionsMenu->addAction("Mirror", this, SLOT(setRenderMirrored(bool)), Qt::Key_H))->setCheckable(true);
     (_echoAudioMode = optionsMenu->addAction("Echo Audio"))->setCheckable(true);
     
     optionsMenu->addAction("Noise", this, SLOT(setNoise(bool)), Qt::Key_N)->setCheckable(true);
@@ -1261,6 +1344,8 @@ void Application::initMenu() {
     _gyroLook->setChecked(false);
     (_mouseLook = optionsMenu->addAction("Mouse Look"))->setCheckable(true);
     _mouseLook->setChecked(true);
+    (_touchLook = optionsMenu->addAction("Touch Look"))->setCheckable(true);
+    _touchLook->setChecked(false);
     (_showHeadMouse = optionsMenu->addAction("Head Mouse"))->setCheckable(true);
     _showHeadMouse->setChecked(false);
     (_transmitterDrives = optionsMenu->addAction("Transmitter Drive"))->setCheckable(true);
@@ -1288,12 +1373,15 @@ void Application::initMenu() {
     _renderAvatarsOn->setChecked(true);
     (_renderAvatarBalls = renderMenu->addAction("Avatar as Balls"))->setCheckable(true);
     _renderAvatarBalls->setChecked(false);
+    renderMenu->addAction("Cycle Voxeltar Mode", _myAvatar.getVoxels(), SLOT(cycleMode()));
     (_renderFrameTimerOn = renderMenu->addAction("Show Timer"))->setCheckable(true);
     _renderFrameTimerOn->setChecked(false);
     (_renderLookatOn = renderMenu->addAction("Lookat Vectors"))->setCheckable(true);
     _renderLookatOn->setChecked(false);
     (_manualFirstPerson = renderMenu->addAction(
         "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true);
+    (_manualThirdPerson = renderMenu->addAction(
+        "Third Person", this, SLOT(setRenderThirdPerson(bool))))->setCheckable(true);
     
     QMenu* toolsMenu = menuBar->addMenu("Tools");
     (_renderStatsOn = toolsMenu->addAction("Stats"))->setCheckable(true);
@@ -1366,6 +1454,7 @@ void Application::initMenu() {
     debugMenu->addAction("Wants Res-In", this, SLOT(setWantsResIn(bool)))->setCheckable(true);
     debugMenu->addAction("Wants Monochrome", this, SLOT(setWantsMonochrome(bool)))->setCheckable(true);
     debugMenu->addAction("Wants View Delta Sending", this, SLOT(setWantsDelta(bool)))->setCheckable(true);
+    (_shouldLowPassFilter = debugMenu->addAction("Test: LowPass filter"))->setCheckable(true);
     debugMenu->addAction("Wants Occlusion Culling", this, SLOT(setWantsOcclusionCulling(bool)))->setCheckable(true);
 
     QMenu* settingsMenu = menuBar->addMenu("Settings");
@@ -1431,7 +1520,7 @@ void Application::init() {
   
     _myAvatar.init();
     _myAvatar.setPosition(START_LOCATION);
-    _myCamera.setMode(CAMERA_MODE_THIRD_PERSON );
+    _myCamera.setMode(CAMERA_MODE_THIRD_PERSON);
     _myCamera.setModeShiftRate(1.0f);
     _myAvatar.setDisplayingLookatVectors(false);  
     
@@ -1446,6 +1535,12 @@ void Application::init() {
     gettimeofday(&_lastTimeIdle, NULL);
 
     loadSettings();
+    if (!shouldDynamicallySetJitterBuffer()) {
+        _audio.setJitterBufferSamples(_audioJitterBufferSamples);
+    }
+    
+    printLog("Loaded settings.\n");
+
     
     sendAvatarVoxelURLMessage(_myAvatar.getVoxels()->getVoxelURL());
 
@@ -1595,8 +1690,14 @@ void Application::update(float deltaTime) {
                                   _glWidget->height());
     }
    
+    //  Update from Touch
+    if (_isTouchPressed && _touchLook->isChecked()) {
+        _myAvatar.updateFromTouch(_touchAvgX - _touchDragStartedAvgX,
+                                  _touchAvgY - _touchDragStartedAvgY);
+    }
+    
     //  Read serial port interface devices
-    if (_serialHeadSensor.active) {
+    if (_serialHeadSensor.isActive()) {
         _serialHeadSensor.readData(deltaTime);
     }
     
@@ -1639,25 +1740,35 @@ void Application::update(float deltaTime) {
         _myAvatar.simulate(deltaTime, NULL);
     }
     
-        if (_myCamera.getMode() != CAMERA_MODE_MIRROR && !OculusManager::isConnected()) {        
+    if (!OculusManager::isConnected()) {        
-            if (_manualFirstPerson->isChecked()) {
+        if (_lookingInMirror->isChecked()) {
-                if (_myCamera.getMode() != CAMERA_MODE_FIRST_PERSON ) {
+            if (_myCamera.getMode() != CAMERA_MODE_MIRROR) {
-                    _myCamera.setMode(CAMERA_MODE_FIRST_PERSON);
+                _myCamera.setMode(CAMERA_MODE_MIRROR);
-                    _myCamera.setModeShiftRate(1.0f);
+                _myCamera.setModeShiftRate(100.0f);
-                }
+            }
+        } else if (_manualFirstPerson->isChecked()) {
+            if (_myCamera.getMode() != CAMERA_MODE_FIRST_PERSON) {
+                _myCamera.setMode(CAMERA_MODE_FIRST_PERSON);
+                _myCamera.setModeShiftRate(1.0f);
+            }
+        } else if (_manualThirdPerson->isChecked()) {
+            if (_myCamera.getMode() != CAMERA_MODE_THIRD_PERSON) {
+                _myCamera.setMode(CAMERA_MODE_THIRD_PERSON);
+                _myCamera.setModeShiftRate(1.0f);
+            }
         } else {
             const float THIRD_PERSON_SHIFT_VELOCITY = 2.0f;
             const float TIME_BEFORE_SHIFT_INTO_FIRST_PERSON = 0.75f;
             const float TIME_BEFORE_SHIFT_INTO_THIRD_PERSON = 0.1f;
             
             if ((_myAvatar.getElapsedTimeStopped() > TIME_BEFORE_SHIFT_INTO_FIRST_PERSON)
-                && (_myCamera.getMode() != CAMERA_MODE_FIRST_PERSON)) {
+                    && (_myCamera.getMode() != CAMERA_MODE_FIRST_PERSON)) {
-                    _myCamera.setMode(CAMERA_MODE_FIRST_PERSON);
+                _myCamera.setMode(CAMERA_MODE_FIRST_PERSON);
-                    _myCamera.setModeShiftRate(1.0f);
+                _myCamera.setModeShiftRate(1.0f);
-                }
+            }
             if ((_myAvatar.getSpeed() > THIRD_PERSON_SHIFT_VELOCITY)
-                && (_myAvatar.getElapsedTimeMoving() > TIME_BEFORE_SHIFT_INTO_THIRD_PERSON)
+                    && (_myAvatar.getElapsedTimeMoving() > TIME_BEFORE_SHIFT_INTO_THIRD_PERSON)
-                && (_myCamera.getMode() != CAMERA_MODE_THIRD_PERSON)) {
+                    && (_myCamera.getMode() != CAMERA_MODE_THIRD_PERSON)) {
                 _myCamera.setMode(CAMERA_MODE_THIRD_PERSON);
                 _myCamera.setModeShiftRate(1000.0f);
             }
@@ -1674,8 +1785,10 @@ void Application::update(float deltaTime) {
 
 void Application::updateAvatar(float deltaTime) {
 
+    // Update my avatar's head position from gyros and/or webcam
+    _myAvatar.updateHeadFromGyrosAndOrWebcam();
         
-    if (_serialHeadSensor.active) {
+    if (_serialHeadSensor.isActive()) {
       
         // Update avatar head translation
         if (_gyroLook->isChecked()) {
@@ -1685,9 +1798,6 @@ void Application::updateAvatar(float deltaTime) {
             _myCamera.setEyeOffsetPosition(headPosition);
         }
 
-        // Update my avatar's head position from gyros
-        _myAvatar.updateHeadFromGyros(deltaTime, &_serialHeadSensor);
-        
         //  Grab latest readings from the gyros
         float measuredPitchRate = _serialHeadSensor.getLastPitchRate();
         float measuredYawRate = _serialHeadSensor.getLastYawRate();
@@ -2585,9 +2695,10 @@ void Application::resetSensors() {
     _headMouseX = _mouseX = _glWidget->width() / 2;
     _headMouseY = _mouseY = _glWidget->height() / 2;
     
-    if (_serialHeadSensor.active) {
+    if (_serialHeadSensor.isActive()) {
         _serialHeadSensor.resetAverages();
     }
+    _webcam.reset();
     QCursor::setPos(_headMouseX, _headMouseY);
     _myAvatar.reset();
     _myTransmitter.resetLevels();
@@ -2737,7 +2848,7 @@ void Application::loadSettings(QSettings* settings) {
     }
 
     _headCameraPitchYawScale = loadSetting(settings, "headCameraPitchYawScale", 0.0f);
-
+    _audioJitterBufferSamples = loadSetting(settings, "audioJitterBufferSamples", 0);
     settings->beginGroup("View Frustum Offset Camera");
     // in case settings is corrupt or missing loadSetting() will check for NaN
     _viewFrustumOffsetYaw      = loadSetting(settings, "viewFrustumOffsetYaw"     , 0.0f);
@@ -2761,6 +2872,7 @@ void Application::saveSettings(QSettings* settings) {
     }
 
     settings->setValue("headCameraPitchYawScale", _headCameraPitchYawScale);
+    settings->setValue("audioJitterBufferSamples", _audioJitterBufferSamples);
     settings->beginGroup("View Frustum Offset Camera");
     settings->setValue("viewFrustumOffsetYaw",      _viewFrustumOffsetYaw);
     settings->setValue("viewFrustumOffsetPitch",    _viewFrustumOffsetPitch);

interface/src/Application.h

@@ -16,6 +16,7 @@
 #include <QApplication>
 #include <QAction>
 #include <QSettings>
+#include <QTouchEvent>
 #include <QList>
 
 #include <AgentList.h>
@@ -68,6 +69,10 @@ public:
     void mousePressEvent(QMouseEvent* event);
     void mouseReleaseEvent(QMouseEvent* event);
 
+    void touchBeginEvent(QTouchEvent* event);
+    void touchEndEvent(QTouchEvent* event);
+    void touchUpdateEvent(QTouchEvent* event);
+    
     void wheelEvent(QWheelEvent* event);
     
     const glm::vec3 getMouseVoxelWorldCoordinates(const VoxelDetail _mouseVoxel);
@@ -78,8 +83,12 @@ public:
     VoxelSystem* getVoxels() { return &_voxels; }
     QSettings* getSettings() { return _settings; }
     Environment* getEnvironment() { return &_environment; }
+    SerialInterface* getSerialHeadSensor() { return &_serialHeadSensor; }
     Webcam* getWebcam() { return &_webcam; }
     bool shouldEchoAudio() { return _echoAudioMode->isChecked(); }
+    bool shouldLowPassFilter() { return _shouldLowPassFilter->isChecked(); }
+    
+    bool shouldDynamicallySetJitterBuffer() { return _audioJitterBufferSamples == 0; }
     
     QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; }
     
@@ -93,11 +102,12 @@ private slots:
     
     void pair();
     
-    void setHead(bool head);
+    void setRenderMirrored(bool mirrored);
     void setNoise(bool noise);
     void setFullscreen(bool fullscreen);
     
     void setRenderFirstPerson(bool firstPerson);
+    void setRenderThirdPerson(bool thirdPerson);
     
     void renderThrustAtVoxel(const glm::vec3& thrust);
     void renderLineToTouchedVoxel();
@@ -181,9 +191,11 @@ private:
     
     QAction* _lookingInMirror;       // Are we currently rendering one's own head as if in mirror?
     QAction* _echoAudioMode;         // Are we asking the mixer to echo back our audio?
+    QAction* _shouldLowPassFilter;   // Use test lowpass filter
     QAction* _gyroLook;              // Whether to allow the gyro data from head to move your view
     QAction* _renderAvatarBalls;     // Switch between voxels and joints/balls for avatar render
     QAction* _mouseLook;             // Whether the have the mouse near edge of screen move your view
+    QAction* _touchLook;             // Whether a 2-finger touch may be used to control look direction
     QAction* _showHeadMouse;         // Whether the have the mouse near edge of screen move your view
     QAction* _transmitterDrives;     // Whether to have Transmitter data move/steer the Avatar
     QAction* _gravityUse;            // Whether gravity is on or not
@@ -197,6 +209,7 @@ private:
     QAction* _renderFrameTimerOn;    // Whether to show onscreen text overlay with stats
     QAction* _renderLookatOn;        // Whether to show lookat vectors from avatar eyes if looking at something
     QAction* _manualFirstPerson;     // Whether to force first-person mode
+    QAction* _manualThirdPerson;     // Whether to force third-person mode
     QAction* _logOn;                 // Whether to show on-screen log
     QActionGroup* _voxelModeActions; // The group of voxel edit mode actions
     QAction* _addVoxelMode;          // Whether add voxel mode is enabled
@@ -263,12 +276,21 @@ private:
     int _headMouseX, _headMouseY;
     float _headCameraPitchYawScale;
     
+    int _audioJitterBufferSamples;     // Number of extra samples to wait before starting audio playback
+    
     HandControl _handControl;
     
     int _mouseX;
     int _mouseY;
     int _mouseDragStartedX;
     int _mouseDragStartedY;
+
+    float _touchAvgX;
+    float _touchAvgY;
+    float _touchDragStartedAvgX;
+    float _touchDragStartedAvgY;
+    bool _isTouchPressed; //  true if multitouch has been pressed (clear when finished)
+    
     VoxelDetail _mouseVoxelDragging;
     glm::vec3 _voxelThrust;
     bool _mousePressed; //  true if mouse has been pressed (clear when finished)

interface/src/Audio.cpp

@@ -26,18 +26,14 @@
 #include "Util.h"
 #include "Log.h"
 
+//  Uncomment the following definition to test audio device latency by copying output to input
+//#define TEST_AUDIO_LOOPBACK
+//#define SHOW_AUDIO_DEBUG
+
 #define VISUALIZE_ECHO_CANCELLATION
 
-static const int NUM_AUDIO_CHANNELS = 2;
-
-static const int PACKET_LENGTH_BYTES = 1024;
-static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2;
-static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t);
-static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2;
-
 static const int PHASE_DELAY_AT_90 = 20;
 static const float AMPLITUDE_RATIO_AT_90 = 0.5;
-
 static const int MIN_FLANGE_EFFECT_THRESHOLD = 600;
 static const int MAX_FLANGE_EFFECT_THRESHOLD = 1500;
 static const float FLANGE_BASE_RATE = 4;
@@ -156,31 +152,53 @@ inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* o
     
     AudioRingBuffer* ringBuffer = &_ringBuffer;
     
-    // if we've been reset, and there isn't any new packets yet
+    // if there is anything in the ring buffer, decide what to do:
-    // just play some silence
     
     if (ringBuffer->getEndOfLastWrite()) {
-        
+        if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (ringBuffer->isStereo() ? 2 : 1))) {
-        if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES) {
+            //
-//            printLog("Held back, buffer has %d of %d samples required.\n",
+            //  If not enough audio has arrived to start playback, keep waiting
-//                     ringBuffer->diffLastWriteNextOutput(), PACKET_LENGTH_SAMPLES + JITTER_BUFFER_SAMPLES);
+            //
-        } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) {
+#ifdef SHOW_AUDIO_DEBUG
+            printLog("%i,%i,%i,%i\n",
+                     _packetsReceivedThisPlayback,
+                     ringBuffer->diffLastWriteNextOutput(),
+                     PACKET_LENGTH_SAMPLES,
+                     _jitterBufferSamples);
+#endif
+        } else if (ringBuffer->isStarted() && (ringBuffer->diffLastWriteNextOutput()
+                                               < PACKET_LENGTH_SAMPLES * (ringBuffer->isStereo() ? 2 : 1))) {
+            //
+            //  If we have started and now have run out of audio to send to the audio device, 
+            //  this means we've starved and should restart.  
+            //  
             ringBuffer->setStarted(false);
             
             _numStarves++;
             _packetsReceivedThisPlayback = 0;
+            _wasStarved = 10;          //   Frames for which to render the indication that the system was starved.
+#ifdef SHOW_AUDIO_DEBUG
+            printLog("Starved, remaining samples = %.0f\n",
+                     ringBuffer->diffLastWriteNextOutput());
+#endif
 
-            // printLog("Starved #%d\n", starve_counter);
-            _wasStarved = 10;      //   Frames to render the indication that the system was starved.
         } else {
+            //
+            //  We are either already playing back, or we have enough audio to start playing back.
+            // 
             if (!ringBuffer->isStarted()) {
                 ringBuffer->setStarted(true);
-                // printLog("starting playback %3.1f msecs delayed \n", (usecTimestampNow() - usecTimestamp(&firstPlaybackTimer))/1000.0);
+#ifdef SHOW_AUDIO_DEBUG
-            } else {
+                printLog("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n",
-                // printLog("pushing buffer\n");
+                         (usecTimestampNow() - usecTimestamp(&_firstPacketReceivedTime))/1000.0,
+                         _jitterBufferSamples,
+                         _packetsReceivedThisPlayback);
+#endif
             }
-            // play whatever we have in the audio buffer
 
+            //
+            // play whatever we have in the audio buffer
+            //
             // if we haven't fired off the flange effect, check if we should
             // TODO: lastMeasuredHeadYaw is now relative to body - check if this still works.
             
@@ -241,9 +259,13 @@ inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* o
                         }
                     }
                 }
-                
+#ifndef TEST_AUDIO_LOOPBACK
                 outputLeft[s] = leftSample;
                 outputRight[s] = rightSample;
+#else 
+                outputLeft[s] = inputLeft[s];
+                outputRight[s] = inputLeft[s];                    
+#endif
             }
             ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES);
             
@@ -300,22 +322,25 @@ static void outputPortAudioError(PaError error) {
     }
 }
 
-Audio::Audio(Oscilloscope* scope) :
+void Audio::reset() {
+    _packetsReceivedThisPlayback = 0;
+    _ringBuffer.reset();
+}
+
+Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) :
     _stream(NULL),
     _ringBuffer(true),
     _scope(scope),
     _averagedLatency(0.0),
     _measuredJitter(0),
-//    _jitterBufferLengthMsecs(12.0),
+    _jitterBufferSamples(initialJitterBufferSamples),
-//    _jitterBufferSamples(_jitterBufferLengthMsecs *
-//                     NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0)),
     _wasStarved(0),
     _numStarves(0),
     _lastInputLoudness(0),
     _lastVelocity(0),
     _lastAcceleration(0),
     _totalPacketsReceived(0),
-    _firstPlaybackTime(),
+    _firstPacketReceivedTime(),
     _packetsReceivedThisPlayback(0),
     _isCancellingEcho(false),
     _echoDelay(0),
@@ -332,14 +357,36 @@ Audio::Audio(Oscilloscope* scope) :
     _flangeWeight(0.0f)
 {
     outputPortAudioError(Pa_Initialize());
-    outputPortAudioError(Pa_OpenDefaultStream(&_stream,
+    
-                                              2,
+    //  NOTE:  Portaudio documentation is unclear as to whether it is safe to specify the
-                                              2,
+    //         number of frames per buffer explicitly versus setting this value to zero.
-                                              (paInt16 | paNonInterleaved),
+    //         Possible source of latency that we need to investigate further.
-                                              SAMPLE_RATE,
+    // 
-                                              BUFFER_LENGTH_SAMPLES_PER_CHANNEL,
+    unsigned long FRAMES_PER_BUFFER = BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
-                                              audioCallback,
+    
-                                              (void*) this));
+    //  Manually initialize the portaudio stream to ask for minimum latency
+    PaStreamParameters inputParameters, outputParameters;
+    
+    inputParameters.device = Pa_GetDefaultInputDevice(); 
+    inputParameters.channelCount = 2;                    //  Stereo input
+    inputParameters.sampleFormat = (paInt16 | paNonInterleaved);
+    inputParameters.suggestedLatency = Pa_GetDeviceInfo( inputParameters.device )->defaultLowInputLatency;
+    inputParameters.hostApiSpecificStreamInfo = NULL;
+
+    outputParameters.device = Pa_GetDefaultOutputDevice();
+    outputParameters.channelCount = 2;                    //  Stereo output
+    outputParameters.sampleFormat = (paInt16 | paNonInterleaved);
+    outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency;
+    outputParameters.hostApiSpecificStreamInfo = NULL;
+    
+    outputPortAudioError(Pa_OpenStream(&_stream, 
+                                       &inputParameters,
+                                       &outputParameters,
+                                       SAMPLE_RATE,
+                                       FRAMES_PER_BUFFER,
+                                       paNoFlag,
+                                       audioCallback,
+                                       (void*) this));
         
     if (! _stream) {
         return;
@@ -382,6 +429,15 @@ Audio::Audio(Oscilloscope* scope) :
     // start the stream now that sources are good to go
     outputPortAudioError(Pa_StartStream(_stream));
     
+    // Uncomment these lines to see the system-reported latency
+    //printLog("Default low input, output latency (secs): %0.4f, %0.4f\n",
+    //         Pa_GetDeviceInfo(Pa_GetDefaultInputDevice())->defaultLowInputLatency,
+    //         Pa_GetDeviceInfo(Pa_GetDefaultOutputDevice())->defaultLowOutputLatency);
+    
+    const PaStreamInfo* streamInfo = Pa_GetStreamInfo(_stream);
+    printLog("Started audio with reported latency msecs In/Out: %.0f, %.0f\n", streamInfo->inputLatency * 1000.f,
+             streamInfo->outputLatency * 1000.f);
+
     gettimeofday(&_lastReceiveTime, NULL);
 }
 
@@ -399,6 +455,7 @@ Audio::~Audio() {
 
 void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) {
     const int NUM_INITIAL_PACKETS_DISCARD = 3;
+    const int STANDARD_DEVIATION_SAMPLE_COUNT = 500;
     
     timeval currentReceiveTime;
     gettimeofday(&currentReceiveTime, NULL);
@@ -411,9 +468,18 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy
         _stdev.addValue(timeDiff);
     }
     
-    if (_stdev.getSamples() > 500) {
+    if (_stdev.getSamples() > STANDARD_DEVIATION_SAMPLE_COUNT) {
         _measuredJitter = _stdev.getStDev();
         _stdev.reset();
+        //  Set jitter buffer to be a multiple of the measured standard deviation
+        const int MAX_JITTER_BUFFER_SAMPLES = RING_BUFFER_LENGTH_SAMPLES / 2;
+        const float NUM_STANDARD_DEVIATIONS = 3.f;
+        if (Application::getInstance()->shouldDynamicallySetJitterBuffer()) {
+            float newJitterBufferSamples = (NUM_STANDARD_DEVIATIONS * _measuredJitter)
+                                            / 1000.f
+                                            * SAMPLE_RATE;
+            setJitterBufferSamples(glm::clamp((int)newJitterBufferSamples, 0, MAX_JITTER_BUFFER_SAMPLES));
+        }
     }
     
     if (!_ringBuffer.isStarted()) {
@@ -421,9 +487,10 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy
     }
     
     if (_packetsReceivedThisPlayback == 1) {
-        gettimeofday(&_firstPlaybackTime, NULL);
+        gettimeofday(&_firstPacketReceivedTime, NULL);
     }
     
+    //printf("Got audio packet %d\n", _packetsReceivedThisPlayback);
     _ringBuffer.parseData((unsigned char*) receivedData, PACKET_LENGTH_BYTES + sizeof(PACKET_HEADER));
     
     _lastReceiveTime = currentReceiveTime;
@@ -447,7 +514,7 @@ void Audio::render(int screenWidth, int screenHeight) {
         glVertex2f(currentX, topY);
         glVertex2f(currentX, bottomY);
         
-        for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES; i++) {
+        for (int i = 0; i < RING_BUFFER_LENGTH_FRAMES / 2; i++) {
             glVertex2f(currentX, halfY);
             glVertex2f(currentX + frameWidth, halfY);
             currentX += frameWidth;
@@ -500,29 +567,60 @@ void Audio::render(int screenWidth, int screenHeight) {
         
         char out[40];
         sprintf(out, "%3.0f\n", _averagedLatency);
-        drawtext(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 10, topY - 10, 0.10, 0, 1, 0, out, 1,1,0);
+        drawtext(startX + _averagedLatency / AUDIO_CALLBACK_MSECS * frameWidth - 10, topY - 9, 0.10, 0, 1, 0, out, 1,1,0);
-        //drawtext(startX + 0, topY-10, 0.08, 0, 1, 0, out, 1,1,0);
         
-        //  Show a Cyan bar with the most recently measured jitter stdev
+        //  Show a red bar with the 'start' point of one frame plus the jitter buffer
         
-        int jitterPels = _measuredJitter / ((1000.0f * PACKET_LENGTH_SAMPLES / SAMPLE_RATE)) * frameWidth;
+        glColor3f(1, 0, 0);
+        int jitterBufferPels = (1.f + (float)getJitterBufferSamples() / (float)PACKET_LENGTH_SAMPLES_PER_CHANNEL) * frameWidth;
+        sprintf(out, "%.0f\n", getJitterBufferSamples() / SAMPLE_RATE * 1000.f);
+        drawtext(startX + jitterBufferPels - 5, topY - 9, 0.10, 0, 1, 0, out, 1, 0, 0);
+        sprintf(out, "j %.1f\n", _measuredJitter);
+        if (Application::getInstance()->shouldDynamicallySetJitterBuffer()) {
+            drawtext(startX + jitterBufferPels - 5, bottomY + 12, 0.10, 0, 1, 0, out, 1, 0, 0);
+        } else {
+            drawtext(startX, bottomY + 12, 0.10, 0, 1, 0, out, 1, 0, 0);
+        }
     
-        glColor3f(0,1,1);
         glBegin(GL_QUADS);
-        glVertex2f(startX + jitterPels - 2, topY - 2);
+        glVertex2f(startX + jitterBufferPels - 2, topY - 2);
-        glVertex2f(startX + jitterPels + 2, topY - 2);
+        glVertex2f(startX + jitterBufferPels + 2, topY - 2);
-        glVertex2f(startX + jitterPels + 2, bottomY + 2);
+        glVertex2f(startX + jitterBufferPels + 2, bottomY + 2);
-        glVertex2f(startX + jitterPels - 2, bottomY + 2);
+        glVertex2f(startX + jitterBufferPels - 2, bottomY + 2);
         glEnd();
 
-        sprintf(out,"%3.1f\n", _measuredJitter);
-        drawtext(startX + jitterPels - 5, topY-10, 0.10, 0, 1, 0, out, 0,1,1);
-        
-        sprintf(out, "%3.1fms\n", JITTER_BUFFER_LENGTH_MSECS);
-        drawtext(startX - 10, bottomY + 15, 0.1, 0, 1, 0, out, 1, 0, 0);
     }
 }
 
+//
+//  Very Simple LowPass filter which works by averaging a bunch of samples with a moving window
+//
+//#define lowpass 1
+void Audio::lowPassFilter(int16_t* inputBuffer) {
+    static int16_t outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL];
+    for (int i = 2; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2; i++) {
+#ifdef lowpass
+        outputBuffer[i] = (int16_t)(0.125f * (float)inputBuffer[i - 2] +
+                            0.25f * (float)inputBuffer[i - 1] +
+                            0.25f * (float)inputBuffer[i] +
+                            0.25f * (float)inputBuffer[i + 1] +
+                            0.125f * (float)inputBuffer[i + 2] );
+#else
+        outputBuffer[i] = (int16_t)(0.125f * -(float)inputBuffer[i - 2] +
+                                    0.25f * -(float)inputBuffer[i - 1] +
+                                    1.75f * (float)inputBuffer[i] +
+                                    0.25f * -(float)inputBuffer[i + 1] +
+                                    0.125f * -(float)inputBuffer[i + 2] );
+
+#endif
+    }
+    outputBuffer[0] = inputBuffer[0];
+    outputBuffer[1] = inputBuffer[1];
+    outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 2];
+    outputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1] = inputBuffer[BUFFER_LENGTH_SAMPLES_PER_CHANNEL - 1];
+    memcpy(inputBuffer, outputBuffer, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t));
+}
+
 //  Take a pointer to the acquired microphone input samples and add procedural sounds
 void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) {
     const float MAX_AUDIBLE_VELOCITY = 6.0;

interface/src/Audio.h

@@ -20,12 +20,20 @@
 #include "Oscilloscope.h"
 #include "Avatar.h"
 
+static const int NUM_AUDIO_CHANNELS = 2;
+
+static const int PACKET_LENGTH_BYTES = 1024;
+static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2;
+static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t);
+static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2;
+
 class Audio {
 public:
     // initializes audio I/O
-    Audio(Oscilloscope* scope);
+    Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples);
     ~Audio();
 
+    void reset(); 
     void render(int screenWidth, int screenHeight);
     
     void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes);
@@ -35,6 +43,13 @@ public:
     void setLastAcceleration(glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; };
     void setLastVelocity(glm::vec3 lastVelocity) { _lastVelocity = lastVelocity; };
     
+    void setJitterBufferSamples(int samples) { _jitterBufferSamples = samples; };
+    int getJitterBufferSamples() { return _jitterBufferSamples; };
+    
+    void lowPassFilter(int16_t* inputBuffer);
+    
+    void startEchoTest();
+    void renderEchoCompare();
     void setIsCancellingEcho(bool enabled);
     bool isCancellingEcho() const;
 
@@ -45,6 +60,7 @@ public:
     // The results of the analysis are written to the log.
     bool eventuallyAnalyzePing();
 
+
 private:    
     PaStream* _stream;
     AudioRingBuffer _ringBuffer;
@@ -54,15 +70,14 @@ private:
     timeval _lastReceiveTime;
     float _averagedLatency;
     float _measuredJitter;
-//    float _jitterBufferLengthMsecs; // currently unused
+    int16_t _jitterBufferSamples;
-//    short _jitterBufferSamples; // currently unsused
     int _wasStarved;
     int _numStarves;
     float _lastInputLoudness;
     glm::vec3 _lastVelocity;
     glm::vec3 _lastAcceleration;
     int _totalPacketsReceived;
-    timeval _firstPlaybackTime;
+    timeval _firstPacketReceivedTime;
     int _packetsReceivedThisPlayback;
     // Echo cancellation
     volatile bool _isCancellingEcho;
@@ -101,6 +116,7 @@ private:
     // Determines round trip time of the audio system. Called from 'eventuallyAnalyzePing'.
     inline void analyzePing();
 
+    // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
     void addProceduralSounds(int16_t* inputBuffer, int numSamples);
 
 

interface/src/Avatar.cpp

@@ -280,22 +280,36 @@ void Avatar::reset() {
 }
 
 //  Update avatar head rotation with sensor data
-void Avatar::updateHeadFromGyros(float deltaTime, SerialInterface* serialInterface) {
+void Avatar::updateHeadFromGyrosAndOrWebcam() {
-    const float AMPLIFY_PITCH = 2.f;
+    const float AMPLIFY_PITCH = 1.f;
-    const float AMPLIFY_YAW = 2.f;
+    const float AMPLIFY_YAW = 1.f;
-    const float AMPLIFY_ROLL = 2.f;
+    const float AMPLIFY_ROLL = 1.f;
 
-    glm::vec3 estimatedRotation = serialInterface->getEstimatedRotation();
+    SerialInterface* gyros = Application::getInstance()->getSerialHeadSensor();
+    Webcam* webcam = Application::getInstance()->getWebcam();
+    glm::vec3 estimatedPosition, estimatedRotation;
+    if (gyros->isActive()) {
+        estimatedPosition = gyros->getEstimatedPosition();
+        estimatedRotation = gyros->getEstimatedRotation();
+        
+    } else if (webcam->isActive()) {
+        estimatedPosition = webcam->getEstimatedPosition();
+        estimatedRotation = webcam->getEstimatedRotation();
+    
+    } else {
+        return;
+    }
     _head.setPitch(estimatedRotation.x * AMPLIFY_PITCH);
     _head.setYaw(estimatedRotation.y * AMPLIFY_YAW);
     _head.setRoll(estimatedRotation.z * AMPLIFY_ROLL);
         
     //  Update torso lean distance based on accelerometer data
-    glm::vec3 estimatedPosition = serialInterface->getEstimatedPosition() * _leanScale;
     const float TORSO_LENGTH = 0.5f;
     const float MAX_LEAN = 45.0f;
-    _head.setLeanSideways(glm::clamp(glm::degrees(atanf(-estimatedPosition.x / TORSO_LENGTH)), -MAX_LEAN, MAX_LEAN));
+    _head.setLeanSideways(glm::clamp(glm::degrees(atanf(-estimatedPosition.x * _leanScale / TORSO_LENGTH)),
-    _head.setLeanForward(glm::clamp(glm::degrees(atanf(estimatedPosition.z / TORSO_LENGTH)), -MAX_LEAN, MAX_LEAN));
+        -MAX_LEAN, MAX_LEAN));
+    _head.setLeanForward(glm::clamp(glm::degrees(atanf(estimatedPosition.z * _leanScale / TORSO_LENGTH)),
+        -MAX_LEAN, MAX_LEAN));
 }
 
 float Avatar::getAbsoluteHeadYaw() const {
@@ -339,6 +353,14 @@ void  Avatar::updateFromMouse(int mouseX, int mouseY, int screenWidth, int scree
     }
 }
 
+void  Avatar::updateFromTouch(float touchAvgDistX, float touchAvgDistY) {
+    const float TOUCH_ROTATE_SPEED = 0.01f;
+    const float TOUCH_PITCH_SPEED = 0.02f;
+    
+    _head.addYaw(-touchAvgDistX * TOUCH_ROTATE_SPEED);
+    _head.addPitch(-touchAvgDistY * TOUCH_PITCH_SPEED);
+}
+
 void Avatar::updateThrust(float deltaTime, Transmitter * transmitter) {
     //
     //  Gather thrust information from keyboard and sensors to apply to avatar motion 
@@ -443,7 +465,16 @@ void Avatar::simulate(float deltaTime, Transmitter* transmitter) {
     }
 
     // update balls
-    if (_balls) { _balls->simulate(deltaTime); }
+    if (_balls) {
+        _balls->moveOrigin(_position);
+        glm::vec3 lookAt = _head.getLookAtPosition();
+        if (glm::length(lookAt) > EPSILON) {
+            _balls->moveOrigin(lookAt);
+        } else {
+            _balls->moveOrigin(_position);
+        }
+        _balls->simulate(deltaTime);
+    }
     
     // update torso rotation based on head lean
     _skeleton.joint[AVATAR_JOINT_TORSO].rotation = glm::quat(glm::radians(glm::vec3(
@@ -978,7 +1009,6 @@ void Avatar::render(bool lookingInMirror, bool renderAvatarBalls) {
     //  Render the balls
     if (_balls) {
         glPushMatrix();
-        glTranslatef(_position.x, _position.y, _position.z);
         _balls->render();
         glPopMatrix();
     }

interface/src/Avatar.h

@@ -86,8 +86,9 @@ public:
     void reset();
     void simulate(float deltaTime, Transmitter* transmitter);
     void updateThrust(float deltaTime, Transmitter * transmitter);
-    void updateHeadFromGyros(float frametime, SerialInterface * serialInterface);
+    void updateHeadFromGyrosAndOrWebcam();
     void updateFromMouse(int mouseX, int mouseY, int screenWidth, int screenHeight);
+    void updateFromTouch(float touchAvgDistX, float touchAvgDistY);
     void addBodyYaw(float y) {_bodyYaw += y;};
     void render(bool lookingInMirror, bool renderAvatarBalls);
 

interface/src/AvatarVoxelSystem.cpp

@@ -22,7 +22,7 @@ const int BONE_ELEMENTS_PER_VOXEL = BONE_ELEMENTS_PER_VERTEX * VERTICES_PER_VOXE
 
 AvatarVoxelSystem::AvatarVoxelSystem(Avatar* avatar) :
     VoxelSystem(AVATAR_TREE_SCALE, MAX_VOXELS_PER_AVATAR),
-    _avatar(avatar), _voxelReply(0) {
+    _mode(0), _avatar(avatar), _voxelReply(0) {
 
     // we may have been created in the network thread, but we live in the main thread
     moveToThread(Application::getInstance()->thread());
@@ -77,6 +77,30 @@ void AvatarVoxelSystem::removeOutOfView() {
     // no-op for now
 }
 
+class Mode {
+public:
+    
+    bool bindVoxelsTogether;
+    int maxBonesPerBind;
+    bool includeBonesOutsideBindRadius;
+};
+
+const Mode MODES[] = {
+    { false, BONE_ELEMENTS_PER_VERTEX, false }, // original
+    { false, 1, true }, // one bone per vertex
+    { true, 1, true }, // one bone per voxel
+    { true, BONE_ELEMENTS_PER_VERTEX, false } }; // four bones per voxel
+
+void AvatarVoxelSystem::cycleMode() {
+    _mode = (_mode + 1) % (sizeof(MODES) / sizeof(MODES[0]));
+    printLog("Voxeltar bind mode %d.\n", _mode);
+    
+    // rebind
+    QUrl url = _voxelURL;
+    setVoxelURL(QUrl());
+    setVoxelURL(url);
+}
+
 void AvatarVoxelSystem::setVoxelURL(const QUrl& url) {
     // don't restart the download if it's the same URL
     if (_voxelURL == url) {
@@ -118,13 +142,27 @@ void AvatarVoxelSystem::updateNodeInArrays(glBufferIndex nodeIndex, const glm::v
     
     GLubyte* writeBoneIndicesAt = _writeBoneIndicesArray + (nodeIndex * BONE_ELEMENTS_PER_VOXEL);
     GLfloat* writeBoneWeightsAt = _writeBoneWeightsArray + (nodeIndex * BONE_ELEMENTS_PER_VOXEL);
-    for (int i = 0; i < VERTICES_PER_VOXEL; i++) {
+    
+    if (MODES[_mode].bindVoxelsTogether) {
         BoneIndices boneIndices;
         glm::vec4 boneWeights;
-        computeBoneIndicesAndWeights(computeVoxelVertex(startVertex, voxelScale, i), boneIndices, boneWeights);
+        computeBoneIndicesAndWeights(startVertex + glm::vec3(voxelScale, voxelScale, voxelScale) * 0.5f,
-        for (int j = 0; j < BONE_ELEMENTS_PER_VERTEX; j++) {
+            boneIndices, boneWeights);
-            *(writeBoneIndicesAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneIndices[j];
+        for (int i = 0; i < VERTICES_PER_VOXEL; i++) {
-            *(writeBoneWeightsAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneWeights[j];
+            for (int j = 0; j < BONE_ELEMENTS_PER_VERTEX; j++) {
+                *(writeBoneIndicesAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneIndices[j];
+                *(writeBoneWeightsAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneWeights[j];
+            }
+        }
+    } else {
+        for (int i = 0; i < VERTICES_PER_VOXEL; i++) {
+            BoneIndices boneIndices;
+            glm::vec4 boneWeights;
+            computeBoneIndicesAndWeights(computeVoxelVertex(startVertex, voxelScale, i), boneIndices, boneWeights);
+            for (int j = 0; j < BONE_ELEMENTS_PER_VERTEX; j++) {
+                *(writeBoneIndicesAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneIndices[j];
+                *(writeBoneWeightsAt + i * BONE_ELEMENTS_PER_VERTEX + j) = boneWeights[j];
+            }
         }
     }
 }
@@ -244,7 +282,7 @@ void AvatarVoxelSystem::computeBoneIndicesAndWeights(const glm::vec3& vertex, Bo
         float distance = glm::length(computeVectorFromPointToSegment(jointVertex,
             skeleton.joint[parent == AVATAR_JOINT_NULL ? i : parent].absoluteBindPosePosition,
             skeleton.joint[i].absoluteBindPosePosition));
-        if (distance > skeleton.joint[i].bindRadius) {
+        if (!MODES[_mode].includeBonesOutsideBindRadius && distance > skeleton.joint[i].bindRadius) {
             continue;
         }
         for (int j = 0; j < BONE_ELEMENTS_PER_VERTEX; j++) {
@@ -261,7 +299,7 @@ void AvatarVoxelSystem::computeBoneIndicesAndWeights(const glm::vec3& vertex, Bo
     
     // compute the weights based on inverse distance
     float totalWeight = 0.0f;
-    for (int i = 0; i < BONE_ELEMENTS_PER_VERTEX; i++) {
+    for (int i = 0; i < MODES[_mode].maxBonesPerBind; i++) {
         indices[i] = nearest[i].index;
         if (nearest[i].distance != FLT_MAX) {
             weights[i] = 1.0f / glm::max(nearest[i].distance, EPSILON);

interface/src/AvatarVoxelSystem.h

@@ -36,6 +36,10 @@ public:
     Q_INVOKABLE void setVoxelURL(const QUrl& url);
     const QUrl& getVoxelURL() const { return _voxelURL; }
 
+public slots:
+    
+    void cycleMode();    
+    
 protected:
     
     virtual void updateNodeInArrays(glBufferIndex nodeIndex, const glm::vec3& startVertex,
@@ -54,6 +58,8 @@ private:
     
     void computeBoneIndicesAndWeights(const glm::vec3& vertex, BoneIndices& indices, glm::vec4& weights) const;
     
+    int _mode;
+    
     Avatar* _avatar;
     
     QUrl _voxelURL;

interface/src/Balls.cpp

@@ -9,34 +9,55 @@
 //  spring, while responding physically to other avatars.   
 //
 
+#include <glm/glm.hpp>
+
+#include <SharedUtil.h>
+
 #include "Balls.h"
+#include "InterfaceConfig.h"
+#include "Util.h"
+#include "world.h"
+
+const float INITIAL_AREA = 0.2f;
+const float BALL_RADIUS = 0.025f;
+const glm::vec3 INITIAL_COLOR(0.62f, 0.74f, 0.91f);
 
 Balls::Balls(int numberOfBalls) {
     _numberOfBalls = numberOfBalls;
     _balls = new Ball[_numberOfBalls];
     for (unsigned int i = 0; i < _numberOfBalls; ++i) {
-        _balls[i].position = glm::vec3(1.0 + randFloat() * 0.5,
+        _balls[i].position = randVector() * INITIAL_AREA;
-                                       0.5 + randFloat() * 0.5,
+        _balls[i].targetPosition = _balls[i].position;
-                                       1.0 + randFloat() * 0.5);
+        _balls[i].velocity = glm::vec3(0, 0, 0);
-        _balls[i].radius = 0.02 + randFloat() * 0.06;
+        _balls[i].radius = BALL_RADIUS;
         for (unsigned int j = 0; j < NUMBER_SPRINGS; ++j) {
-            _balls[i].links[j] = rand() % (numberOfBalls + 1);
+            _balls[i].links[j] = 0;
-            if (_balls[i].links[j]-1 == i) { _balls[i].links[j] = 0; }
+          }
-            _balls[i].springLength[j] = 0.5;
+    }
-         }
+    _color = INITIAL_COLOR;
+    _origin = glm::vec3(0, 0, 0);
+}
+
+void Balls::moveOrigin(const glm::vec3& newOrigin) {
+    glm::vec3 delta = newOrigin - _origin;
+    if (glm::length(delta) > EPSILON) {
+        _origin = newOrigin;
+        for (unsigned int i = 0; i < _numberOfBalls; ++i) {
+            _balls[i].targetPosition += delta;
+        }
     }
 }
 
-const bool RENDER_SPRINGS = true;
+const bool RENDER_SPRINGS = false;
 
 void Balls::render() {
     
     //  Render Balls    NOTE:  This needs to become something other that GlutSpheres!
-    glColor3f(0.62,0.74,0.91);
+    glColor3fv(&_color.x);
     for (unsigned int i = 0; i < _numberOfBalls; ++i) {
         glPushMatrix();
         glTranslatef(_balls[i].position.x, _balls[i].position.y, _balls[i].position.z);
-        glutSolidSphere(_balls[i].radius, 15, 15);
+        glutSolidSphere(_balls[i].radius, 8, 8);
         glPopMatrix();
     }
     
@@ -71,18 +92,22 @@ void Balls::simulate(float deltaTime) {
         
         // Move particles
         _balls[i].position += _balls[i].velocity * deltaTime;
+        _balls[i].targetPosition += _balls[i].velocity * deltaTime;
         
         // Drag: decay velocity
         _balls[i].velocity *= (1.f - CONSTANT_VELOCITY_DAMPING * deltaTime);
         
         // Add noise
-        _balls[i].velocity += glm::vec3((randFloat() - 0.5) * NOISE_SCALE,
+        _balls[i].velocity += randVector() * NOISE_SCALE;
-                                        (randFloat() - 0.5) * NOISE_SCALE,
+        
-                                        (randFloat() - 0.5) * NOISE_SCALE);
+        // Approach target position
+        for (unsigned int i = 0; i < _numberOfBalls; ++i) {
+            _balls[i].position += randFloat() * deltaTime * (_balls[i].targetPosition - _balls[i].position);
+        }
 
         // Spring Force
         
-        
+        /*
         for (unsigned int j = 0; j < NUMBER_SPRINGS; ++j) {
             if(_balls[i].links[j] > 0) {
                 float separation = glm::distance(_balls[i].position,
@@ -96,7 +121,7 @@ void Balls::simulate(float deltaTime) {
                 //_balls[i].velocity *= (1.f - SPRING_DAMPING*deltaTime);
 
             }
-        }
+        } */
          
          
         

interface/src/Balls.h

@@ -9,12 +9,6 @@
 #ifndef hifi_Balls_h
 #define hifi_Balls_h
 
-#include <glm/glm.hpp>
-#include "Util.h"
-#include "world.h"
-#include "InterfaceConfig.h"
-
-
 const int NUMBER_SPRINGS = 4;
 
 class Balls {
@@ -24,14 +18,19 @@ public:
     void simulate(float deltaTime);
     void render();
     
+    void setColor(const glm::vec3& c) { _color = c; };
+    void moveOrigin(const glm::vec3& newOrigin);
+    
 private:
     struct Ball {
-        glm::vec3   position, velocity;
+        glm::vec3   position, targetPosition, velocity;
         int         links[NUMBER_SPRINGS];
         float       springLength[NUMBER_SPRINGS];
         float       radius;
     } *_balls;
     int _numberOfBalls;
+    glm::vec3 _origin;
+    glm::vec3 _color;
 };
 
 #endif

interface/src/Camera.cpp

@@ -22,7 +22,7 @@ const float CAMERA_THIRD_PERSON_MODE_DISTANCE  = 1.5f;
 const float CAMERA_THIRD_PERSON_MODE_TIGHTNESS = 8.0f;
 
 const float CAMERA_MIRROR_MODE_UP_SHIFT        = 0.0f;
-const float CAMERA_MIRROR_MODE_DISTANCE        = 0.2f;
+const float CAMERA_MIRROR_MODE_DISTANCE        = 0.3f;
 const float CAMERA_MIRROR_MODE_TIGHTNESS       = 100.0f;
 
 Camera::Camera() {

interface/src/SerialInterface.cpp

@@ -5,18 +5,23 @@
 //  Read interface data from the gyros/accelerometer Invensense board using the SerialUSB
 //
 
-#include "SerialInterface.h"
-#include "SharedUtil.h"
-#include "Util.h"
-#include <glm/gtx/vector_angle.hpp>
-#include <math.h>
-
 #ifdef __APPLE__
 #include <regex.h>
 #include <sys/time.h>
 #include <string>
 #endif
 
+#include <math.h>
+
+#include <glm/gtx/vector_angle.hpp>
+
+#include <SharedUtil.h>
+
+#include "Application.h"
+#include "SerialInterface.h"
+#include "Util.h"
+#include "Webcam.h"
+
 const short NO_READ_MAXIMUM_MSECS = 3000;
 const int GRAVITY_SAMPLES = 60;                     //  Use the first few samples to baseline values
 const int SENSOR_FUSION_SAMPLES = 20;
@@ -103,7 +108,7 @@ void SerialInterface::initializePort(char* portname) {
     printLog("Connected.\n");
     resetSerial();
     
-    active = true;
+    _active = true;
  #endif
 }
 
@@ -301,7 +306,16 @@ void SerialInterface::readData(float deltaTime) {
         _estimatedVelocity += deltaTime * _estimatedAcceleration;
         _estimatedPosition += deltaTime * _estimatedVelocity;
         _estimatedVelocity *= DECAY_VELOCITY;
-        _estimatedPosition *= DECAY_POSITION;
+        
+        //  Attempt to fuse gyro position with webcam position
+        Webcam* webcam = Application::getInstance()->getWebcam();
+        if (webcam->isActive()) {
+            const float WEBCAM_POSITION_FUSION = 0.5f;
+            _estimatedPosition = glm::mix(_estimatedPosition, webcam->getEstimatedPosition(), WEBCAM_POSITION_FUSION);
+               
+        } else {
+            _estimatedPosition *= DECAY_POSITION;
+        }
             
         //  Accumulate a set of initial baseline readings for setting gravity
         if (totalSamples == 0) {
@@ -327,6 +341,12 @@ void SerialInterface::readData(float deltaTime) {
         
         _estimatedRotation = safeEulerAngles(estimatedRotation); 
         
+        //  Fuse gyro roll with webcam roll
+        if (webcam->isActive()) {
+            _estimatedRotation.z = glm::mix(_estimatedRotation.z, webcam->getEstimatedRotation().z,
+                1.0f / SENSOR_FUSION_SAMPLES);
+        }
+        
         totalSamples++;
     } 
     
@@ -359,7 +379,7 @@ void SerialInterface::resetAverages() {
 void SerialInterface::resetSerial() {
 #ifdef __APPLE__
     resetAverages();
-    active = false;
+    _active = false;
     gettimeofday(&lastGoodRead, NULL);
 #endif
 }

interface/src/SerialInterface.h

@@ -24,7 +24,7 @@ extern const bool USING_INVENSENSE_MPU9150;
 
 class SerialInterface {
 public:
-    SerialInterface() : active(false),
+    SerialInterface() : _active(false),
                         _gravity(0, 0, 0),
                         _averageRotationRates(0, 0, 0),
                         _averageAcceleration(0, 0, 0),
@@ -58,12 +58,13 @@ public:
     
     void renderLevels(int width, int height);
     void resetAverages();
-    bool active;
+    bool isActive() const { return _active; }
     
 private:
     void initializePort(char* portname);
     void resetSerial();
 
+    bool _active;
     int _serialDescriptor;
     int totalSamples;
     timeval lastGoodRead;

interface/src/Webcam.cpp

@@ -8,8 +8,6 @@
 #include <QTimer>
 #include <QtDebug>
 
-#include <opencv2/opencv.hpp>
-
 #include <Log.h>
 #include <SharedUtil.h>
 
@@ -20,7 +18,14 @@
 #include "Application.h"
 #include "Webcam.h"
 
-Webcam::Webcam() : _enabled(false), _frameTextureID(0)  {
+using namespace cv;
+using namespace std;
+
+// register OpenCV matrix type with Qt metatype system
+int matMetaType = qRegisterMetaType<Mat>("cv::Mat");
+int rotatedRectMetaType = qRegisterMetaType<RotatedRect>("cv::RotatedRect");
+
+Webcam::Webcam() : _enabled(false), _active(false), _frameTextureID(0) {
     // the grabber simply runs as fast as possible
     _grabber = new FrameGrabber();
     _grabber->moveToThread(&_grabberThread);
@@ -36,10 +41,21 @@ void Webcam::setEnabled(bool enabled) {
         _frameCount = 0;
         
         // let the grabber know we're ready for the first frame
+        QMetaObject::invokeMethod(_grabber, "reset");
         QMetaObject::invokeMethod(_grabber, "grabFrame");
     
     } else {
         _grabberThread.quit();
+        _active = false;
+    }
+}
+
+void Webcam::reset() {
+    _initialFaceRect = RotatedRect();
+    
+    if (_enabled) {
+        // send a message to the grabber
+        QMetaObject::invokeMethod(_grabber, "reset");
     }
 }
 
@@ -66,6 +82,17 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) {
         glBindTexture(GL_TEXTURE_2D, 0);
         glDisable(GL_TEXTURE_2D);
         
+        glBegin(GL_LINE_LOOP);
+            Point2f facePoints[4];
+            _faceRect.points(facePoints);
+            float xScale = previewWidth / (float)_frameWidth;
+            float yScale = PREVIEW_HEIGHT / (float)_frameHeight;
+            glVertex2f(left + facePoints[0].x * xScale, top + facePoints[0].y * yScale);
+            glVertex2f(left + facePoints[1].x * xScale, top + facePoints[1].y * yScale);
+            glVertex2f(left + facePoints[2].x * xScale, top + facePoints[2].y * yScale);
+            glVertex2f(left + facePoints[3].x * xScale, top + facePoints[3].y * yScale);
+        glEnd();
+        
         char fps[20];
         sprintf(fps, "FPS: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp))));
         drawtext(left, top + PREVIEW_HEIGHT + 20, 0.10, 0, 1, 0, fps);
@@ -80,25 +107,26 @@ Webcam::~Webcam() {
     delete _grabber;
 }
 
-void Webcam::setFrame(void* image) {
+void Webcam::setFrame(const Mat& frame, const RotatedRect& faceRect) {
-    IplImage* img = static_cast<IplImage*>(image);
+    IplImage image = frame;
-    glPixelStorei(GL_UNPACK_ROW_LENGTH, img->widthStep / 3);
+    glPixelStorei(GL_UNPACK_ROW_LENGTH, image.widthStep / 3);
     if (_frameTextureID == 0) {
         glGenTextures(1, &_frameTextureID);
         glBindTexture(GL_TEXTURE_2D, _frameTextureID);
-        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, _frameWidth = img->width, _frameHeight = img->height, 0, GL_BGR,
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, _frameWidth = image.width, _frameHeight = image.height, 0, GL_BGR,
-            GL_UNSIGNED_BYTE, img->imageData);
+            GL_UNSIGNED_BYTE, image.imageData);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
         printLog("Capturing webcam at %dx%d.\n", _frameWidth, _frameHeight);
     
     } else {
         glBindTexture(GL_TEXTURE_2D, _frameTextureID);
-        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _frameWidth, _frameHeight, GL_BGR, GL_UNSIGNED_BYTE, img->imageData);    
+        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _frameWidth, _frameHeight, GL_BGR, GL_UNSIGNED_BYTE, image.imageData);    
     }
     glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
     glBindTexture(GL_TEXTURE_2D, 0);
     
-    // update our frame count for fps computation
+    // store our face rect, update our frame count for fps computation
+    _faceRect = faceRect;
     _frameCount++;
     
     const int MAX_FPS = 60;
@@ -112,16 +140,55 @@ void Webcam::setFrame(void* image) {
     }
     _lastFrameTimestamp = now;
     
+    // roll is just the angle of the face rect (correcting for 180 degree rotations)
+    float roll = faceRect.angle;
+    if (roll < -90.0f) {
+        roll += 180.0f;
+        
+    } else if (roll > 90.0f) {
+        roll -= 180.0f;
+    }
+    const float ROTATION_SMOOTHING = 0.95f;
+    _estimatedRotation.z = glm::mix(roll, _estimatedRotation.z, ROTATION_SMOOTHING);
+    
+    // determine position based on translation and scaling of the face rect
+    if (_initialFaceRect.size.area() == 0) {
+        _initialFaceRect = faceRect;
+        _estimatedPosition = glm::vec3();
+    
+    } else {
+        float proportion = sqrtf(_initialFaceRect.size.area() / (float)faceRect.size.area());
+        const float DISTANCE_TO_CAMERA = 0.333f;
+        const float POSITION_SCALE = 0.5f;
+        float z = DISTANCE_TO_CAMERA * proportion - DISTANCE_TO_CAMERA;
+        glm::vec3 position = glm::vec3(
+            (faceRect.center.x - _initialFaceRect.center.x) * proportion * POSITION_SCALE / _frameWidth,
+            (faceRect.center.y - _initialFaceRect.center.y) * proportion * POSITION_SCALE / _frameWidth,
+            z);
+        const float POSITION_SMOOTHING = 0.95f;
+        _estimatedPosition = glm::mix(position, _estimatedPosition, POSITION_SMOOTHING);
+    }
+    
+    // note that we have data
+    _active = true;
+    
     // let the grabber know we're ready for the next frame
     QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame()));
 }
 
+FrameGrabber::FrameGrabber() : _capture(0), _searchWindow(0, 0, 0, 0) {
+}
+
 FrameGrabber::~FrameGrabber() {
     if (_capture != 0) {
         cvReleaseCapture(&_capture);
     }
 }
 
+void FrameGrabber::reset() {
+    _searchWindow = Rect(0, 0, 0, 0);
+}
+
 void FrameGrabber::grabFrame() {
     if (_capture == 0) {
         if ((_capture = cvCaptureFromCAM(-1)) == 0) {
@@ -134,8 +201,20 @@ void FrameGrabber::grabFrame() {
         cvSetCaptureProperty(_capture, CV_CAP_PROP_FRAME_HEIGHT, IDEAL_FRAME_HEIGHT);
         
 #ifdef __APPLE__
-        configureCamera(0x5ac, 0x8510, false, 0.99, 0.5, 0.5, 0.5, true, 0.5);
+        configureCamera(0x5ac, 0x8510, false, 0.975, 0.5, 1.0, 0.5, true, 0.5);
+#else
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_EXPOSURE, 0.5);
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_CONTRAST, 0.5);
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_SATURATION, 0.5);
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_BRIGHTNESS, 0.5);
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_HUE, 0.5);
+        cvSetCaptureProperty(_capture, CV_CAP_PROP_GAIN, 0.5);
 #endif
+
+        switchToResourcesParentIfRequired();
+        if (!_faceCascade.load("resources/haarcascades/haarcascade_frontalface_alt.xml")) {
+            printLog("Failed to load Haar cascade for face tracking.\n");
+        }
     }
     IplImage* image = cvQueryFrame(_capture);
     if (image == 0) {
@@ -149,5 +228,43 @@ void FrameGrabber::grabFrame() {
         printLog("Invalid webcam image format.\n");
         return;
     }
-    QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", Q_ARG(void*, image));
+    
+    // if we don't have a search window (yet), try using the face cascade
+    Mat frame = image;
+    int channels = 0;
+    float ranges[] = { 0, 180 };
+    const float* range = ranges;
+    if (_searchWindow.area() == 0) {
+        vector<Rect> faces;
+        _faceCascade.detectMultiScale(frame, faces, 1.1, 6);
+        if (!faces.empty()) {
+            _searchWindow = faces.front();
+            updateHSVFrame(frame);
+        
+            Mat faceHsv(_hsvFrame, _searchWindow);
+            Mat faceMask(_mask, _searchWindow);
+            int sizes = 30;
+            calcHist(&faceHsv, 1, &channels, faceMask, _histogram, 1, &sizes, &range);
+            double min, max;
+            minMaxLoc(_histogram, &min, &max);
+            _histogram.convertTo(_histogram, -1, (max == 0.0) ? 0.0 : 255.0 / max);
+        }
+    }
+    RotatedRect faceRect;
+    if (_searchWindow.area() > 0) {
+        updateHSVFrame(frame);
+        
+        calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range);
+        bitwise_and(_backProject, _mask, _backProject);
+        
+        faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1));
+        _searchWindow = faceRect.boundingRect();
+    }   
+    QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame",
+        Q_ARG(cv::Mat, frame), Q_ARG(cv::RotatedRect, faceRect));
+}
+
+void FrameGrabber::updateHSVFrame(const Mat& frame) {
+    cvtColor(frame, _hsvFrame, CV_BGR2HSV);
+    inRange(_hsvFrame, Scalar(0, 55, 65), Scalar(180, 256, 256), _mask);
 }

interface/src/Webcam.h

@@ -9,9 +9,14 @@
 #ifndef __interface__Webcam__
 #define __interface__Webcam__
 
+#include <QMetaType>
 #include <QObject>
 #include <QThread>
 
+#include <glm/glm.hpp>
+
+#include <opencv2/opencv.hpp>
+
 #include "InterfaceConfig.h"
 
 class QImage;
@@ -28,12 +33,17 @@ public:
     Webcam();
     ~Webcam();
 
+    const bool isActive() const { return _active; }
+    const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; }
+    const glm::vec3& getEstimatedRotation() const { return _estimatedRotation; }
+
+    void reset();
     void renderPreview(int screenWidth, int screenHeight);
 
 public slots:
     
     void setEnabled(bool enabled);
-    void setFrame(void* image);
+    void setFrame(const cv::Mat& image, const cv::RotatedRect& faceRect);
     
 private:
     
@@ -41,14 +51,20 @@ private:
     FrameGrabber* _grabber;
     
     bool _enabled;
+    bool _active;
     int _frameWidth;
     int _frameHeight;
     GLuint _frameTextureID;
+    cv::RotatedRect _faceRect;
+    cv::RotatedRect _initialFaceRect;
     
     long long _startTimestamp;
     int _frameCount;
     
     long long _lastFrameTimestamp;
+    
+    glm::vec3 _estimatedPosition;
+    glm::vec3 _estimatedRotation;
 };
 
 class FrameGrabber : public QObject {
@@ -56,16 +72,28 @@ class FrameGrabber : public QObject {
     
 public:
     
-    FrameGrabber() : _capture(0) { }
+    FrameGrabber();
     virtual ~FrameGrabber();
 
 public slots:
     
+    void reset();
     void grabFrame();
     
 private:
     
+    void updateHSVFrame(const cv::Mat& frame);
+    
     CvCapture* _capture;
+    cv::CascadeClassifier _faceCascade;
+    cv::Mat _hsvFrame;
+    cv::Mat _mask;
+    cv::SparseMat _histogram;
+    cv::Mat _backProject;
+    cv::Rect _searchWindow;
 };
 
+Q_DECLARE_METATYPE(cv::Mat)
+Q_DECLARE_METATYPE(cv::RotatedRect)
+
 #endif /* defined(__interface__Webcam__) */

libraries/audio/src/AudioRingBuffer.cpp

@@ -27,6 +27,12 @@ AudioRingBuffer::~AudioRingBuffer() {
     delete[] _buffer;
 }
 
+void AudioRingBuffer::reset() {
+    _endOfLastWrite = _buffer;
+    _nextOutput = _buffer;
+    _isStarted = false;
+}
+
 int AudioRingBuffer::parseData(unsigned char* sourceBuffer, int numBytes) {
     return parseAudioSamples(sourceBuffer + sizeof(PACKET_HEADER_MIXED_AUDIO), numBytes - sizeof(PACKET_HEADER_MIXED_AUDIO));
 }

libraries/audio/src/AudioRingBuffer.h

@@ -22,7 +22,7 @@ const int BUFFER_LENGTH_BYTES_STEREO = 1024;
 const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512;
 const int BUFFER_LENGTH_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_BYTES_PER_CHANNEL / sizeof(int16_t);
 
-const short RING_BUFFER_LENGTH_FRAMES = 10;
+const short RING_BUFFER_LENGTH_FRAMES = 20;
 const short RING_BUFFER_LENGTH_SAMPLES = RING_BUFFER_LENGTH_FRAMES * BUFFER_LENGTH_SAMPLES_PER_CHANNEL;
 
 class AudioRingBuffer : public AgentData {
@@ -30,6 +30,7 @@ public:
     AudioRingBuffer(bool isStereo);
     ~AudioRingBuffer();
 
+    void reset();
 
     int parseData(unsigned char* sourceBuffer, int numBytes);
     int parseAudioSamples(unsigned char* sourceBuffer, int numBytes);
@@ -46,6 +47,9 @@ public:
     void setStarted(bool isStarted) { _isStarted = isStarted; }
     
     int diffLastWriteNextOutput() const;
+    
+    bool isStereo() const { return _isStereo; }
+    
 protected:
     // disallow copying of AudioRingBuffer objects
     AudioRingBuffer(const AudioRingBuffer&);

libraries/shared/src/Logstash.cpp

@@ -11,6 +11,7 @@
 #include <netdb.h>
 
 #include "SharedUtil.h"
+#include "AgentList.h"
 
 #include "Logstash.h"
 
@@ -43,3 +44,17 @@ bool Logstash::shouldSendStats() {
     static bool shouldSendStats = isInEnvironment("production");
     return shouldSendStats;
 }
+
+void Logstash::stashValue(const char* key, float value) {
+    static char logstashPacket[MAX_PACKET_SIZE];
+    
+    // load up the logstash packet with the key and the passed float value
+    // send it to 4 decimal places
+    int numPacketBytes = sprintf(logstashPacket, "%s %.4f", key, value);
+    
+    AgentList *agentList = AgentList::getInstance();
+    
+    if (agentList) {
+        agentList->getAgentSocket()->send(socket(), logstashPacket, numPacketBytes);
+    }
+}

libraries/shared/src/Logstash.h

@@ -18,6 +18,7 @@ class Logstash {
 public:
     static sockaddr* socket();
     static bool shouldSendStats();
+    static void stashValue(const char* key, float value);
 private:
     static sockaddr_in logstashSocket;
 };