Merge branch 'master' of https://github.com/highfidelity/hifi into project-freeloco-fargrab-rotation

.gitignore

@@ -24,6 +24,7 @@ android/**/bin
 android/**/src/main/res/values/libs.xml
 android/**/src/main/assets
 android/**/gradle*
+*.class
 
 # VSCode
 # List taken from Github Global Ignores master@435c4d92

BUILD_QUEST.md

@@ -0,0 +1,65 @@
+Please read the [general build guide](BUILD.md) for information on building other platform. Only Quest specific instructions are found in this file.
+
+# Dependencies
+
+Building is currently supported on OSX, Windows and Linux platforms, but developers intending to do work on the library dependencies are strongly urged to use 64 bit Linux as a build platform
+
+You will need the following tools to build Android targets.
+
+* [Android Studio](https://developer.android.com/studio/index.html)
+
+### Android Studio
+
+Download the Android Studio installer and run it.   Once installed, at the welcome screen, click configure in the lower right corner and select SDK manager
+
+From the SDK Platforms tab, select API levels 24 and 26.  
+
+From the SDK Tools tab select the following
+
+* Android SDK Build-Tools
+* GPU Debugging Tools
+* CMake (even if you have a separate CMake installation)
+* LLDB 
+* Android SDK Platform-Tools
+* Android SDK Tools
+* NDK (even if you have the NDK installed separately)
+
+Make sure the NDK installed version is 18 (or higher)
+
+# Environment 
+
+Setting up the environment for android builds requires some additional steps
+
+#### Set up machine specific Gradle properties
+
+Create a `gradle.properties` file in $HOME/.gradle.   Edit the file to contain the following
+
+    HIFI_ANDROID_PRECOMPILED=<your_home_directory>/Android/hifi_externals
+    HIFI_ANDROID_KEYSTORE=<key_store_directory>/<keystore_name>.jks
+    HIFI_ANDROID_KEYSTORE_PASSWORD=<password>
+    HIFI_ANDROID_KEY_ALIAS=<key_alias>
+    HIFI_ANDROID_KEY_PASSWORD=<key_password>
+
+Note, do not use `$HOME` for the path.  It must be a fully qualified path name.
+
+### Setup the repository
+
+Clone the repository
+
+`git clone https://github.com/highfidelity/hifi.git`
+
+Enter the repository `android` directory
+
+`cd hifi/android`
+
+# Building & Running
+
+* Open Android Studio
+* Choose _Open Existing Android Studio Project_
+* Navigate to the `hifi` repository and choose the `android` folder and select _OK_
+* Open Gradle.settings and comment out any projects not necessary
+* From _File_ menu select _Sync with File System_ to resync Gradle settings
+* From the _Build_ menu select _Make Project_
+* From 
+* Once the build completes, from the _Run_ menu select _Run App_ 
+

CMakeLists.txt

@@ -52,6 +52,7 @@ else()
   set(MOBILE 0)
 endif()
 
+set(HIFI_USE_OPTIMIZED_IK OFF)
 set(BUILD_CLIENT_OPTION ON)
 set(BUILD_SERVER_OPTION ON)
 set(BUILD_TESTS_OPTION OFF)
@@ -100,6 +101,13 @@ if (ANDROID)
     add_definitions(-DCUSTOM_DISPLAY_PLUGINS) 
     set(PLATFORM_PLUGIN_LIBRARIES oculusMobile oculusMobilePlugin)
   endif()
+
+  # Allow client code to use preprocessor macros to distinguish between quest and non-quest builds
+  if (${HIFI_ANDROID_APP} STREQUAL "questInterface")
+    add_definitions(-DANDROID_APP_QUEST_INTERFACE) 
+  elseif(${HIFI_ANDROID_APP} STREQUAL "interface")
+    add_definitions(-DANDROID_APP_INTERFACE) 
+  endif()
 else ()
   set(PLATFORM_QT_COMPONENTS WebEngine Xml)
 endif ()
@@ -108,7 +116,7 @@ if (USE_GLES AND (NOT ANDROID))
   set(DISABLE_QML_OPTION ON)
 endif()
 
-
+option(HIFI_USE_OPTIMIZED_IK "USE OPTIMIZED IK" ${HIFI_USE_OPTIMIZED_IK_OPTION})
 option(BUILD_CLIENT "Build client components" ${BUILD_CLIENT_OPTION})
 option(BUILD_SERVER "Build server components" ${BUILD_SERVER_OPTION})
 option(BUILD_TESTS "Build tests" ${BUILD_TESTS_OPTION})
@@ -139,6 +147,7 @@ foreach(PLATFORM_QT_COMPONENT ${PLATFORM_QT_COMPONENTS})
   list(APPEND PLATFORM_QT_LIBRARIES "Qt5::${PLATFORM_QT_COMPONENT}")
 endforeach()
 
+MESSAGE(STATUS "USE OPTIMIZED IK:      " ${HIFI_USE_OPTIMIZED_IK})
 MESSAGE(STATUS "Build server:          " ${BUILD_SERVER})
 MESSAGE(STATUS "Build client:          " ${BUILD_CLIENT})
 MESSAGE(STATUS "Build tests:           " ${BUILD_TESTS})
@@ -184,6 +193,10 @@ find_package( Threads )
 add_definitions(-DGLM_FORCE_RADIANS)
 add_definitions(-DGLM_ENABLE_EXPERIMENTAL)
 add_definitions(-DGLM_FORCE_CTOR_INIT)
+if (HIFI_USE_OPTIMIZED_IK)
+  MESSAGE(STATUS "SET THE USE IK DEFINITION ")
+  add_definitions(-DHIFI_USE_OPTIMIZED_IK)
+endif()
 set(HIFI_LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libraries")
 
 set(EXTERNAL_PROJECT_PREFIX "project")

android/apps/interface/src/main/java/io/highfidelity/hifiinterface/InterfaceActivity.java

@@ -81,6 +81,7 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
     private boolean nativeEnterBackgroundCallEnqueued = false;
     private SlidingDrawer mWebSlidingDrawer;
     private boolean mStartInDomain;
+    private boolean isLoading;
 //    private GvrApi gvrApi;
     // Opaque native pointer to the Application C++ object.
     // This object is owned by the InterfaceActivity instance and passed to the native methods.
@@ -94,7 +95,7 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
 
     @Override
     public void onCreate(Bundle savedInstanceState) {
-        super.isLoading = true;
+        isLoading = true;
         Intent intent = getIntent();
         if (intent.hasExtra(DOMAIN_URL) && !TextUtils.isEmpty(intent.getStringExtra(DOMAIN_URL))) {
             intent.putExtra("applicationArguments", "--url " + intent.getStringExtra(DOMAIN_URL));
@@ -145,7 +146,7 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
     @Override
     protected void onPause() {
         super.onPause();
-        if (super.isLoading) {
+        if (isLoading) {
             nativeEnterBackgroundCallEnqueued = true;
         } else {
             nativeEnterBackground();
@@ -172,7 +173,6 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
         super.onResume();
         nativeEnterForeground();
         surfacesWorkaround();
-        keepInterfaceRunning = false;
         registerReceiver(headsetStateReceiver, new IntentFilter(Intent.ACTION_HEADSET_PLUG));
         //gvrApi.resumeTracking();
     }
@@ -382,7 +382,7 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
     }
 
     public void onAppLoadedComplete() {
-        super.isLoading = false;
+        isLoading = false;
         if (nativeEnterBackgroundCallEnqueued) {
             nativeEnterBackground();
         }
@@ -413,7 +413,6 @@ public class InterfaceActivity extends QtActivity implements WebViewFragment.OnW
 
     @Override
     public void onExpand() {
-        keepInterfaceRunning = true;
     }
 
     @Override

android/apps/questFramePlayer/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(TARGET_NAME questFramePlayer)
 setup_hifi_library(AndroidExtras)
 link_hifi_libraries(shared ktx shaders gpu gl oculusMobile ${PLATFORM_GL_BACKEND})
-target_include_directories(${TARGET_NAME} PRIVATE ${HIFI_ANDROID_PRECOMPILED}/ovr/VrApi/Include)
+
 target_link_libraries(${TARGET_NAME} android log m)
 target_opengl()
 target_oculus_mobile()

android/apps/questFramePlayer/src/main/AndroidManifest.xml

@@ -19,24 +19,6 @@
         android:name="org.qtproject.qt5.android.bindings.QtApplication"
         tools:ignore="GoogleAppIndexingWarning,MissingApplicationIcon">
         <meta-data android:name="com.samsung.android.vr.application.mode" android:value="vr_only"/>
-        <activity
-            android:name=".QuestQtActivity"
-            android:theme="@android:style/Theme.Black.NoTitleBar.Fullscreen"
-            android:launchMode="singleTask"
-            android:label="@string/app_name"
-            android:screenOrientation="landscape"
-            android:excludeFromRecents="false"
-            android:alwaysRetainTaskState="true"
-            android:configChanges="screenSize|screenLayout|orientation|keyboardHidden|keyboard|navigation|uiMode"
-            >
-            <!-- JNI nonsense -->
-            <meta-data android:name="android.app.lib_name" android:value="questFramePlayer"/>
-            <!-- Qt nonsense -->
-            <meta-data android:name="android.app.qt_libs_resource_id" android:resource="@array/qt_libs"/>
-            <meta-data android:name="android.app.bundled_in_lib_resource_id" android:resource="@array/bundled_in_lib"/>
-            <meta-data android:name="android.app.bundled_in_assets_resource_id" android:resource="@array/bundled_in_assets"/>
-            <meta-data android:name="android.app.load_local_libs" android:value="plugins/platforms/android/libqtforandroid.so:plugins/bearer/libqandroidbearer.so:lib/libQt5QuickParticles.so"/>
-        </activity>
 
         <activity
             android:theme="@android:style/Theme.Black.NoTitleBar.Fullscreen"
@@ -50,6 +32,13 @@
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
             </intent-filter>
+            <!-- JNI nonsense -->
+            <meta-data android:name="android.app.lib_name" android:value="questFramePlayer"/>
+            <!-- Qt nonsense -->
+            <meta-data android:name="android.app.qt_libs_resource_id" android:resource="@array/qt_libs"/>
+            <meta-data android:name="android.app.bundled_in_lib_resource_id" android:resource="@array/bundled_in_lib"/>
+            <meta-data android:name="android.app.bundled_in_assets_resource_id" android:resource="@array/bundled_in_assets"/>
+            <meta-data android:name="android.app.load_local_libs" android:value="plugins/platforms/android/libqtforandroid.so:plugins/bearer/libqandroidbearer.so:lib/libQt5QuickParticles.so"/>
         </activity>
     </application>
 </manifest>

android/apps/questFramePlayer/src/main/cpp/AndroidHelper.cpp

@@ -0,0 +1,30 @@
+//
+//  Created by Bradley Austin Davis on 2019/02/15
+//  Copyright 2013-2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+#include "AndroidHelper.h"
+
+#include <QtCore/QDebug>
+#include <QtGui/QGuiApplication>
+
+AndroidHelper::AndroidHelper() {
+}
+
+AndroidHelper::~AndroidHelper() {
+}
+
+void AndroidHelper::notifyLoadComplete() {
+    emit qtAppLoadComplete();
+}
+
+void AndroidHelper::notifyEnterForeground() {
+    emit enterForeground();
+}
+
+void AndroidHelper::notifyEnterBackground() {
+    emit enterBackground();
+}
+

android/apps/questFramePlayer/src/main/cpp/AndroidHelper.h

@@ -0,0 +1,43 @@
+//
+//  Created by Bradley Austin Davis on 2019/02/15
+//  Copyright 2013-2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_Android_Helper_h
+#define hifi_Android_Helper_h
+
+#include <QtCore/QObject>
+#include <QtCore/QMap>
+#include <QtCore/QUrl>
+#include <QtCore/QEventLoop>
+
+class AndroidHelper : public QObject {
+    Q_OBJECT
+public:
+    AndroidHelper(AndroidHelper const&)  = delete;
+    void operator=(AndroidHelper const&) = delete;
+
+    static AndroidHelper& instance() {
+        static AndroidHelper instance;
+        return instance;
+    }
+
+    void notifyLoadComplete();
+    void notifyEnterForeground();
+    void notifyEnterBackground();
+
+
+signals:
+    void qtAppLoadComplete();
+    void enterForeground();
+    void enterBackground();
+
+private:
+    AndroidHelper();
+    ~AndroidHelper();
+};
+
+#endif

android/apps/questFramePlayer/src/main/cpp/PlayerWindow.cpp

@@ -11,15 +11,11 @@
 #include <QtWidgets/QFileDialog>
 
 PlayerWindow::PlayerWindow() {
-    installEventFilter(this);
-    setFlags(Qt::MSWindowsOwnDC | Qt::Window | Qt::Dialog | Qt::WindowMinMaxButtonsHint | Qt::WindowTitleHint);
+    setFlags(Qt::Window);
     setSurfaceType(QSurface::OpenGLSurface);
     create();
     showFullScreen();
     // Ensure the window is visible and the GL context is valid
     QCoreApplication::processEvents();
-    _renderThread.initialize(this);
-}
-
-PlayerWindow::~PlayerWindow() {
+    _renderThread.initialize();
 }

android/apps/questFramePlayer/src/main/cpp/PlayerWindow.h

@@ -8,22 +8,13 @@
 
 #pragma once
 #include <QtGui/QWindow>
-#include <QtCore/QSettings>
-
-#include <gpu/Forward.h>
 #include "RenderThread.h"
 
-// Create a simple OpenGL window that renders text in various ways
 class PlayerWindow : public QWindow {
 public:
     PlayerWindow();
-    virtual ~PlayerWindow();
-
-protected:
-    //bool eventFilter(QObject* obj, QEvent* event) override;
-    //void keyPressEvent(QKeyEvent* event) override;
+    virtual ~PlayerWindow() {}
 
 private:
-    QSettings _settings;
     RenderThread _renderThread;
 };

android/apps/questFramePlayer/src/main/cpp/RenderThread.cpp

@@ -16,6 +16,7 @@
 #include <QtCore/QFileInfo>
 #include <QtGui/QWindow>
 #include <QtGui/QImageReader>
+#include <QtAndroidExtras/QAndroidJniObject>
 
 #include <gl/QOpenGLContextWrapper.h>
 #include <gpu/FrameIO.h>
@@ -29,9 +30,7 @@
 #include <VrApi.h>
 #include <VrApi_Input.h>
 
-static JNIEnv* _env { nullptr };
-static JavaVM* _vm { nullptr };
-static jobject _activity { nullptr };
+#include "AndroidHelper.h"
 
 struct HandController{
     ovrInputTrackedRemoteCapabilities caps {};
@@ -48,21 +47,43 @@ struct HandController{
 };
 
 std::vector<HandController> devices;
+QAndroidJniObject __interfaceActivity;
 
 extern "C" {
 
-JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *, void *) {
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void *) {
     __android_log_write(ANDROID_LOG_WARN, "QQQ", __FUNCTION__);
     return JNI_VERSION_1_6;
 }
 
+JNIEXPORT void JNICALL
+Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnCreate(JNIEnv *env, jobject obj) {
+    __android_log_print(ANDROID_LOG_INFO, "QQQ", __FUNCTION__);
+    __interfaceActivity = QAndroidJniObject(obj);
+    QObject::connect(&AndroidHelper::instance(), &AndroidHelper::qtAppLoadComplete, []() {
+        __interfaceActivity.callMethod<void>("onAppLoadedComplete", "()V");
+        QObject::disconnect(&AndroidHelper::instance(), &AndroidHelper::qtAppLoadComplete, nullptr, nullptr);
+    });
+}
 
-JNIEXPORT void JNICALL Java_io_highfidelity_frameplayer_QuestQtActivity_nativeOnCreate(JNIEnv* env, jobject obj) {
-    env->GetJavaVM(&_vm);
-    _activity = env->NewGlobalRef(obj);
+JNIEXPORT void
+Java_io_highfidelity_oculus_OculusMobileActivity_questOnAppAfterLoad(JNIEnv *env, jobject obj) {
+    AndroidHelper::instance().moveToThread(qApp->thread());
 }
+
+JNIEXPORT void JNICALL
+Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnPause(JNIEnv *env, jobject obj) {
+    AndroidHelper::instance().notifyEnterBackground();
 }
 
+JNIEXPORT void JNICALL
+Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnResume(JNIEnv *env, jobject obj) {
+    AndroidHelper::instance().notifyEnterForeground();
+}
+
+}
+
+
 static const char* FRAME_FILE = "assets:/frames/20190121_1220.json";
 
 static void textureLoader(const std::string& filename, const gpu::TexturePointer& texture, uint16_t layer) {
@@ -84,11 +105,10 @@ void RenderThread::move(const glm::vec3& v) {
     _correction = glm::inverse(glm::translate(mat4(), v)) * _correction;
 }
 
-void RenderThread::initialize(QWindow* window) {
+void RenderThread::initialize() {
     std::unique_lock<std::mutex> lock(_frameLock);
     setObjectName("RenderThread");
     Parent::initialize();
-    _window = window;
     _thread->setObjectName("RenderThread");
 }
 
@@ -96,14 +116,7 @@ void RenderThread::setup() {
     // Wait until the context has been moved to this thread
     { std::unique_lock<std::mutex> lock(_frameLock); }
 
-
     ovr::VrHandler::initVr();
-    __android_log_write(ANDROID_LOG_WARN, "QQQ", "Launching oculus activity");
-    _vm->AttachCurrentThread(&_env, nullptr);
-    jclass cls = _env->GetObjectClass(_activity);
-    jmethodID mid = _env->GetMethodID(cls, "launchOculusActivity", "()V");
-    _env->CallVoidMethod(_activity, mid);
-    __android_log_write(ANDROID_LOG_WARN, "QQQ", "Launching oculus activity done");
     ovr::VrHandler::setHandler(this);
 
     makeCurrent();
@@ -169,7 +182,6 @@ void RenderThread::handleInput() {
                 const auto &remote = controller.state;
                 if (remote.Joystick.x != 0.0f || remote.Joystick.y != 0.0f) {
                     glm::vec3 translation;
-                    float rotation = 0.0f;
                     if (caps.ControllerCapabilities & ovrControllerCaps_LeftHand) {
                         translation = glm::vec3{0.0f, -remote.Joystick.y, 0.0f};
                     } else {

android/apps/questFramePlayer/src/main/cpp/RenderThread.h

@@ -20,11 +20,9 @@
 class RenderThread : public GenericThread, ovr::VrHandler {
     using Parent = GenericThread;
 public:
-    QWindow* _window{ nullptr };
     std::mutex _mutex;
     gpu::ContextPointer _gpuContext;  // initialized during window creation
     std::shared_ptr<gpu::Backend> _backend;
-    std::atomic<size_t> _presentCount{ 0 };
     std::mutex _frameLock;
     std::queue<gpu::FramePointer> _pendingFrames;
     gpu::FramePointer _activeFrame;
@@ -39,6 +37,6 @@ public:
     void handleInput();
 
     void submitFrame(const gpu::FramePointer& frame);
-    void initialize(QWindow* window);
+    void initialize();
     void renderFrame();
 };

android/apps/questFramePlayer/src/main/cpp/main.cpp

@@ -11,30 +11,33 @@
 #include <QtGui/QGuiApplication>
 #include <QtCore/QTimer>
 #include <QtCore/QFileInfo>
+#include <QtAndroidExtras/QAndroidJniObject>
 
 #include <Trace.h>
 
 #include "PlayerWindow.h"
+#include "AndroidHelper.h"
+
 
 void messageHandler(QtMsgType type, const QMessageLogContext& context, const QString& message) {
     if (!message.isEmpty()) {
-        const char * local=message.toStdString().c_str();
+        const char* local = message.toStdString().c_str();
         switch (type) {
             case QtDebugMsg:
-                __android_log_write(ANDROID_LOG_DEBUG,"Interface",local);
+                __android_log_write(ANDROID_LOG_DEBUG, "Interface", local);
                 break;
             case QtInfoMsg:
-                __android_log_write(ANDROID_LOG_INFO,"Interface",local);
+                __android_log_write(ANDROID_LOG_INFO, "Interface", local);
                 break;
             case QtWarningMsg:
-                __android_log_write(ANDROID_LOG_WARN,"Interface",local);
+                __android_log_write(ANDROID_LOG_WARN, "Interface", local);
                 break;
             case QtCriticalMsg:
-                __android_log_write(ANDROID_LOG_ERROR,"Interface",local);
+                __android_log_write(ANDROID_LOG_ERROR, "Interface", local);
                 break;
             case QtFatalMsg:
             default:
-                __android_log_write(ANDROID_LOG_FATAL,"Interface",local);
+                __android_log_write(ANDROID_LOG_FATAL, "Interface", local);
                 abort();
         }
     }
@@ -46,11 +49,13 @@ int main(int argc, char** argv) {
     auto oldMessageHandler = qInstallMessageHandler(messageHandler);
     DependencyManager::set<tracing::Tracer>();
     PlayerWindow window;
-    __android_log_write(ANDROID_LOG_FATAL,"QQQ","Exec");
+    QTimer::singleShot(10, []{
+        __android_log_write(ANDROID_LOG_WARN, "QQQ", "notifyLoadComplete");
+        AndroidHelper::instance().notifyLoadComplete();
+    });
+    __android_log_write(ANDROID_LOG_WARN, "QQQ", "Exec");
     app.exec();
-    __android_log_write(ANDROID_LOG_FATAL,"QQQ","Exec done");
+    __android_log_write(ANDROID_LOG_WARN, "QQQ", "Exec done");
     qInstallMessageHandler(oldMessageHandler);
     return 0;
 }
-
-

android/apps/questFramePlayer/src/main/java/io/highfidelity/frameplayer/QuestQtActivity.java

@@ -1,53 +0,0 @@
-//
-//  Created by Bradley Austin Davis on 2018/11/20
-//  Copyright 2013-2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-package io.highfidelity.frameplayer;
-
-import android.content.Intent;
-import android.os.Bundle;
-import android.util.Log;
-
-import org.qtproject.qt5.android.bindings.QtActivity;
-
-import io.highfidelity.oculus.OculusMobileActivity;
-
-
-public class QuestQtActivity extends QtActivity {
-    private native void nativeOnCreate();
-    private boolean launchedQuestMode = false;
-
-    @Override
-    public void onCreate(Bundle savedInstanceState) {
-        Log.w("QQQ_Qt", "QuestQtActivity::onCreate");
-        super.onCreate(savedInstanceState);
-        nativeOnCreate();
-    }
-
-    @Override
-    public void onDestroy() {
-        Log.w("QQQ_Qt", "QuestQtActivity::onDestroy");
-        super.onDestroy();
-    }
-
-    public void launchOculusActivity() {
-        Log.w("QQQ_Qt", "QuestQtActivity::launchOculusActivity");
-        runOnUiThread(()->{
-            keepInterfaceRunning = true;
-            launchedQuestMode = true;
-            moveTaskToBack(true);
-            startActivity(new Intent(this, QuestRenderActivity.class));
-        });
-    }
-
-    @Override
-    public void onResume() {
-        super.onResume();
-        if (launchedQuestMode) {
-            moveTaskToBack(true);
-        }
-    }
-}

android/apps/questFramePlayer/src/main/java/io/highfidelity/frameplayer/QuestRenderActivity.java

@@ -1,14 +1,6 @@
 package io.highfidelity.frameplayer;
 
-import android.content.Intent;
-import android.os.Bundle;
-
 import io.highfidelity.oculus.OculusMobileActivity;
 
 public class QuestRenderActivity extends OculusMobileActivity {
-    @Override
-    public void onCreate(Bundle savedState) {
-        super.onCreate(savedState);
-        startActivity(new Intent(this, QuestQtActivity.class));
-    }
 }

android/apps/questInterface/CMakeLists.txt

@@ -0,0 +1,16 @@
+set(TARGET_NAME questInterface)
+setup_hifi_library()
+link_hifi_libraries(
+    shared task networking qml
+    image fbx hfm render-utils physics entities octree 
+    oculusMobile oculusMobilePlugin
+    gl gpu ${PLATFORM_GL_BACKEND}
+)
+target_opengl()
+target_bullet()
+target_oculus_mobile()
+
+add_subdirectory("${CMAKE_SOURCE_DIR}/interface" "libraries/interface")
+include_directories("${CMAKE_SOURCE_DIR}/interface/src")
+add_subdirectory("${CMAKE_SOURCE_DIR}/plugins/hifiCodec" "libraries/hifiCodecPlugin")
+target_link_libraries(questInterface android log m interface)

android/apps/questInterface/build.gradle

@@ -0,0 +1,149 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+apply plugin: 'com.android.application'
+
+task renameHifiACTaskDebug() {
+    doLast {
+        def sourceFile = new File("${appDir}/build/intermediates/cmake/debug/obj/arm64-v8a/","libhifiCodec.so")
+        def destinationFile = new File("${appDir}/src/main/jniLibs/arm64-v8a", "libplugins_libhifiCodec.so")
+        copy { from sourceFile;  into destinationFile.parent;  rename(sourceFile.name, destinationFile.name) }
+    }
+}
+task renameHifiACTaskRelease(type: Copy) {
+    doLast {
+        def sourceFile = new File("${appDir}/build/intermediates/cmake/release/obj/arm64-v8a/","libhifiCodec.so")
+        def destinationFile = new File("${appDir}/src/main/jniLibs/arm64-v8a", "libplugins_libhifiCodec.so")
+        copy { from sourceFile;  into destinationFile.parent;  rename(sourceFile.name, destinationFile.name) }
+    }
+}
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        applicationId "io.highfidelity.questInterface"
+        minSdkVersion 24
+        targetSdkVersion 28
+        versionCode 1
+        versionName appVersionName
+        ndk { abiFilters 'arm64-v8a' }
+        externalNativeBuild {
+            cmake {
+                arguments '-DHIFI_ANDROID=1',
+                    '-DHIFI_ANDROID_APP=questInterface',
+                    '-DANDROID_TOOLCHAIN=clang',
+                    '-DANDROID_STL=c++_shared',
+                    '-DCMAKE_VERBOSE_MAKEFILE=ON',
+                    '-DRELEASE_NUMBER=' + RELEASE_NUMBER,
+                    '-DRELEASE_TYPE=' + RELEASE_TYPE,
+                    '-DSTABLE_BUILD=' + STABLE_BUILD,
+                    '-DDISABLE_QML=OFF',
+                    '-DDISABLE_KTX_CACHE=OFF',
+                    '-DUSE_BREAKPAD=OFF'
+                targets = ['questInterface']
+            }
+        }
+        signingConfigs {
+            release {
+                storeFile project.hasProperty("HIFI_ANDROID_KEYSTORE") ? file(HIFI_ANDROID_KEYSTORE) : null
+                storePassword project.hasProperty("HIFI_ANDROID_KEYSTORE_PASSWORD") ? HIFI_ANDROID_KEYSTORE_PASSWORD : ''
+                keyAlias project.hasProperty("HIFI_ANDROID_KEY_ALIAS") ? HIFI_ANDROID_KEY_ALIAS : ''
+                keyPassword project.hasProperty("HIFI_ANDROID_KEY_PASSWORD") ? HIFI_ANDROID_KEY_PASSWORD : ''
+                v2SigningEnabled false
+            }
+        }
+    }
+
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+
+    buildTypes {
+        debug {
+            buildConfigField "String", "BACKTRACE_URL", "\"" + (System.getenv("CMAKE_BACKTRACE_URL") ? System.getenv("CMAKE_BACKTRACE_URL") : '') + "\""
+            buildConfigField "String", "BACKTRACE_TOKEN", "\"" + (System.getenv("CMAKE_BACKTRACE_TOKEN") ? System.getenv("CMAKE_BACKTRACE_TOKEN") : '') + "\""
+            buildConfigField "String", "OAUTH_CLIENT_ID", "\"" + (System.getenv("OAUTH_CLIENT_ID") ? System.getenv("OAUTH_CLIENT_ID") : '') + "\""
+            buildConfigField "String", "OAUTH_CLIENT_SECRET", "\"" + (System.getenv("OAUTH_CLIENT_SECRET") ? System.getenv("OAUTH_CLIENT_SECRET") : '') + "\""
+            buildConfigField "String", "OAUTH_REDIRECT_URI", "\"" + (System.getenv("OAUTH_REDIRECT_URI") ? System.getenv("OAUTH_REDIRECT_URI") : '') + "\""
+        }
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+            signingConfig signingConfigs.release
+            buildConfigField "String", "BACKTRACE_URL", "\"" + (System.getenv("CMAKE_BACKTRACE_URL") ? System.getenv("CMAKE_BACKTRACE_URL") : '') + "\""
+            buildConfigField "String", "BACKTRACE_TOKEN", "\"" + (System.getenv("CMAKE_BACKTRACE_TOKEN") ? System.getenv("CMAKE_BACKTRACE_TOKEN") : '') + "\""
+            buildConfigField "String", "OAUTH_CLIENT_ID", "\"" + (System.getenv("OAUTH_CLIENT_ID") ? System.getenv("OAUTH_CLIENT_ID") : '') + "\""
+            buildConfigField "String", "OAUTH_CLIENT_SECRET", "\"" + (System.getenv("OAUTH_CLIENT_SECRET") ? System.getenv("OAUTH_CLIENT_SECRET") : '') + "\""
+            buildConfigField "String", "OAUTH_REDIRECT_URI", "\"" + (System.getenv("OAUTH_REDIRECT_URI") ? System.getenv("OAUTH_REDIRECT_URI") : '') + "\""
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path '../../../CMakeLists.txt'
+        }
+    }
+
+    applicationVariants.all { variant ->
+        // Our asset contents depend on items produced in the CMake build
+        // so our merge has to depend on the external native build
+        variant.externalNativeBuildTasks.each { task ->
+            variant.mergeResources.dependsOn(task)
+            if (Os.isFamily(Os.FAMILY_UNIX)) {
+                // FIXME
+                def uploadDumpSymsTask = rootProject.getTasksByName("uploadBreakpadDumpSyms${variant.name.capitalize()}", false).first()
+                def runDumpSymsTask = rootProject.getTasksByName("runBreakpadDumpSyms${variant.name.capitalize()}", false).first()
+                def renameHifiACTask = rootProject.getTasksByName("renameHifiACTask${variant.name.capitalize()}", false).first()
+                runDumpSymsTask.dependsOn(task)
+                variant.assemble.dependsOn(uploadDumpSymsTask)
+                variant.mergeResources.dependsOn(renameHifiACTask)
+            }
+        }
+
+        variant.mergeAssets.doLast {
+            def assetList = new LinkedList<String>()
+            def youngestLastModified = 0
+
+            // Copy the compiled resources generated by the external native build
+            copy {
+                from new File(projectDir, "../../../interface/compiledResources")
+                into outputDir
+                duplicatesStrategy DuplicatesStrategy.INCLUDE
+                eachFile { details ->
+                    youngestLastModified = Math.max(youngestLastModified, details.lastModified)
+                    assetList.add(details.path)
+                }
+            }
+
+            // Copy the scripts directory
+            copy {
+                from new File(projectDir, "../../../scripts")
+                into new File(outputDir, "scripts")
+                duplicatesStrategy DuplicatesStrategy.INCLUDE
+                eachFile { details->
+                    youngestLastModified = Math.max(youngestLastModified, details.lastModified)
+                    assetList.add("scripts/" + details.path)
+                }
+            }
+
+            // Write a list of files to be unpacked to the cache folder
+            new File(outputDir, 'cache_assets.txt').withWriter { out ->
+                out.println(Long.toString(youngestLastModified))
+                assetList.each { file -> out.println(file) }
+            }
+        }
+
+        variant.outputs.all {
+            if (RELEASE_NUMBER != '0') {
+                outputFileName = "app_" + RELEASE_NUMBER + "_" + RELEASE_TYPE + ".apk"
+            }
+        }
+    }
+}
+
+dependencies {
+    implementation fileTree(include: ['*.jar'], dir: '../../libraries/qt/libs')
+    implementation project(':oculus')
+    implementation project(':qt')
+}

android/apps/questInterface/proguard-rules.pro

@@ -0,0 +1,25 @@
+# Add project specific ProGuard rules here.
+# By default, the flags in this file are appended to flags specified
+# in C:\Android\SDK/tools/proguard/proguard-android.txt
+# You can edit the include path and order by changing the proguardFiles
+# directive in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# Add any project specific keep options here:
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/apps/questInterface/src/main/AndroidManifest.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="io.highfidelity.questInterface"
+    android:installLocation="auto">
+
+    <uses-feature android:glEsVersion="0x00030002" android:required="true" />
+    <uses-permission android:name="android.permission.VIBRATE"/>
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+    <uses-feature android:name="android.hardware.sensor.accelerometer" android:required="true"/>
+    <uses-feature android:name="android.hardware.sensor.gyroscope" android:required="true"/>
+    <uses-feature android:name="android.software.vr.mode" android:required="true"/>
+    <uses-feature android:name="android.hardware.vr.high_performance" android:required="true"/>
+
+    <application
+        android:name="org.qtproject.qt5.android.bindings.QtApplication"
+        android:label="@string/app_name"
+        android:allowBackup="true">
+        <meta-data android:name="com.samsung.android.vr.application.mode" android:value="vr_only"/>
+        <activity
+            android:name=".PermissionsChecker"
+            android:launchMode="singleTask"
+            android:screenOrientation="landscape"
+            android:excludeFromRecents="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.INFO" />
+            </intent-filter>
+        </activity>
+        <activity
+            android:configChanges="orientation|uiMode|screenLayout|screenSize|smallestScreenSize|locale|fontScale|keyboard|keyboardHidden|navigation"
+            android:name=".InterfaceActivity"
+            android:label="@string/app_name"
+            android:launchMode="singleTask">
+            <meta-data android:name="android.app.lib_name" android:value="questInterface"/>
+            <meta-data android:name="android.app.qt_libs_resource_id" android:resource="@array/qt_libs"/>
+            <meta-data android:name="android.app.bundled_in_lib_resource_id" android:resource="@array/bundled_in_lib"/>
+            <meta-data android:name="android.app.bundled_in_assets_resource_id" android:resource="@array/bundled_in_assets"/>
+            <meta-data android:name="android.app.load_local_libs" android:value="plugins/platforms/android/libqtforandroid.so:plugins/bearer/libqandroidbearer.so:lib/libQt5QuickParticles.so"/>
+            <meta-data android:name="android.app.background_running" android:value="false"/>
+            <meta-data android:name="android.app.auto_screen_scale_factor" android:value="false"/>
+            <meta-data android:name="android.app.extract_android_style" android:value="full"/>
+        </activity>
+    </application>
+</manifest>

android/apps/questInterface/src/main/cpp/native.cpp

@@ -0,0 +1,106 @@
+#include <functional>
+
+#include <QtCore/QBuffer>
+#include <QtCore/QDebug>
+#include <QtCore/QDir>
+#include <QtCore/QFile>
+#include <QtCore/QThread>
+#include <QtCore/QStringList>
+#include <QtCore/QStandardPaths>
+#include <QtCore/QTextStream>
+#include <QtCore/QObject>
+
+#include <QtAndroidExtras/QAndroidJniObject>
+#include <QtAndroidExtras/QtAndroid>
+#include <android/log.h>
+#include <android/asset_manager.h>
+#include <android/asset_manager_jni.h>
+
+#include <shared/Storage.h>
+
+#include <AddressManager.h>
+#include <AndroidHelper.h>
+#include <udt/PacketHeaders.h>
+
+#include <OVR_Platform.h>
+#include <OVR_Functions_Voip.h>
+
+void initOculusPlatform(JNIEnv* env, jobject obj) {
+    static std::once_flag once;
+    std::call_once(once, [&]{
+        // static const char* appID = "2343652845669354";
+        // if (ovr_PlatformInitializeAndroid(appID, obj, env) != ovrPlatformInitialize_Success) {
+        //     __android_log_write(ANDROID_LOG_WARN, "QQQ", "Failed to init platform SDK");
+        //     return;
+        // }
+        // ovr_Voip_SetSystemVoipSuppressed(true);
+    });
+}
+
+void getClassName(JNIEnv *env, jobject obj){
+    jclass cls = env->GetObjectClass(obj);
+    jmethodID mid = env->GetMethodID(cls,"getClass", "()Ljava/lang/Class;");
+    jobject clsObj = env->CallObjectMethod(obj, mid);
+
+    cls= env->GetObjectClass(clsObj);
+
+    mid= env->GetMethodID(cls, "getName", "()Ljava/lang/String;");
+
+    jstring strObj = (jstring) env->CallObjectMethod(clsObj, mid);
+
+    const char* str = env->GetStringUTFChars(strObj, NULL);
+
+    __android_log_print(ANDROID_LOG_ERROR,__FUNCTION__, "Native Class call: %s",str);
+
+    env->ReleaseStringUTFChars(strObj, str);
+}
+
+
+extern "C" {
+    JNIEXPORT void JNICALL
+    Java_io_highfidelity_oculus_OculusMobileActivity_nativeInitOculusPlatform(JNIEnv *env, jobject obj){
+        initOculusPlatform(env, obj);
+    }
+QAndroidJniObject __interfaceActivity;
+
+    JNIEXPORT void JNICALL
+    Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnCreate(JNIEnv *env, jobject obj) {
+        __android_log_print(ANDROID_LOG_INFO, "QQQ", __FUNCTION__);
+        initOculusPlatform(env, obj);
+        getClassName(env, obj);
+
+        __interfaceActivity = QAndroidJniObject(obj);
+
+        QObject::connect(&AndroidHelper::instance(), &AndroidHelper::qtAppLoadComplete, []() {
+            __interfaceActivity.callMethod<void>("onAppLoadedComplete", "()V");
+
+            QObject::disconnect(&AndroidHelper::instance(), &AndroidHelper::qtAppLoadComplete,
+                                nullptr,
+                                nullptr);
+        });
+    }
+
+
+
+JNIEXPORT void Java_io_highfidelity_oculus_OculusMobileActivity_questOnAppAfterLoad(JNIEnv* env, jobject obj) {
+    AndroidHelper::instance().moveToThread(qApp->thread());
+}
+
+    JNIEXPORT void JNICALL
+    Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnPause(JNIEnv *env, jobject obj) {
+        AndroidHelper::instance().notifyEnterBackground();
+    }
+
+    JNIEXPORT void JNICALL
+    Java_io_highfidelity_oculus_OculusMobileActivity_questNativeOnResume(JNIEnv *env, jobject obj) {
+        AndroidHelper::instance().notifyEnterForeground();
+    }
+
+    JNIEXPORT void JNICALL
+    Java_io_highfidelity_questInterface_receiver_HeadsetStateReceiver_notifyHeadsetOn(JNIEnv *env,
+                                                                                      jobject instance,
+                                                                                      jboolean pluggedIn) {
+        AndroidHelper::instance().notifyHeadsetOn(pluggedIn);
+    }
+
+}

android/apps/questInterface/src/main/java/io/highfidelity/questInterface/InterfaceActivity.java

@@ -0,0 +1,13 @@
+package io.highfidelity.questInterface;
+
+import android.os.Bundle;
+import io.highfidelity.oculus.OculusMobileActivity;
+import io.highfidelity.utils.HifiUtils;
+
+public class InterfaceActivity extends OculusMobileActivity {
+    @Override
+    public void onCreate(Bundle savedInstanceState) {
+        HifiUtils.upackAssets(getAssets(), getCacheDir().getAbsolutePath());
+        super.onCreate(savedInstanceState);
+    }
+}

android/apps/questInterface/src/main/java/io/highfidelity/questInterface/PermissionsChecker.java

@@ -0,0 +1,68 @@
+package io.highfidelity.questInterface;
+
+import android.Manifest;
+import android.app.Activity;
+import android.content.Intent;
+import android.content.pm.PackageManager;
+import android.os.Bundle;
+
+import io.highfidelity.oculus.OculusMobileActivity;
+import io.highfidelity.utils.HifiUtils;
+
+public class PermissionsChecker extends Activity {
+    private static final int REQUEST_PERMISSIONS = 20;
+    private static final String TAG = PermissionsChecker.class.getName();
+    private static final String[] REQUIRED_PERMISSIONS = new String[]{
+        Manifest.permission.READ_EXTERNAL_STORAGE,
+        Manifest.permission.WRITE_EXTERNAL_STORAGE,
+        Manifest.permission.RECORD_AUDIO,
+        Manifest.permission.CAMERA
+    };
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        requestAppPermissions(REQUIRED_PERMISSIONS,REQUEST_PERMISSIONS);
+    }
+
+    public void requestAppPermissions(final String[] requestedPermissions,
+                                      final int requestCode) {
+        int permissionCheck = PackageManager.PERMISSION_GRANTED;
+        boolean shouldShowRequestPermissionRationale = false;
+        for (String permission : requestedPermissions) {
+            permissionCheck = permissionCheck + checkSelfPermission(permission);
+            shouldShowRequestPermissionRationale = shouldShowRequestPermissionRationale || shouldShowRequestPermissionRationale(permission);
+        }
+        if (permissionCheck != PackageManager.PERMISSION_GRANTED) {
+            System.out.println("Permission was not granted. Ask for permissions");
+            if (shouldShowRequestPermissionRationale) {
+                requestPermissions(requestedPermissions, requestCode);
+            } else {
+                requestPermissions(requestedPermissions, requestCode);
+            }
+        } else {
+            System.out.println("Launching the other activity..");
+            launchActivityWithPermissions();
+        }
+    }
+
+    private void launchActivityWithPermissions() {
+        startActivity(new Intent(this, InterfaceActivity.class));
+        finish();
+    }
+
+    @Override
+    public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) {
+        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+        int permissionCheck = PackageManager.PERMISSION_GRANTED;
+        for (int permission : grantResults) {
+            permissionCheck = permissionCheck + permission;
+        }
+        if ((grantResults.length > 0) && permissionCheck == PackageManager.PERMISSION_GRANTED) {
+            launchActivityWithPermissions();
+        } else if (grantResults.length > 0) {
+            System.out.println("User has deliberately denied Permissions. Launching anyways");
+            launchActivityWithPermissions();
+        }
+    }
+}

android/apps/questInterface/src/main/res/drawable/ic_launcher.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--suppress AndroidUnknownAttribute -->
+<vector xmlns:api24="http://schemas.android.com/apk/res/android" xmlns:android="http://schemas.android.com/apk/res/android"
+    android:viewportWidth="192"
+    android:viewportHeight="192"
+    android:width="192dp"
+    android:height="192dp">
+    <path
+        android:pathData="M189.5 96.5A93.5 93.5 0 0 1 96 190 93.5 93.5 0 0 1 2.5 96.5 93.5 93.5 0 0 1 96 3 93.5 93.5 0 0 1 189.5 96.5Z"
+        android:fillColor="#333333" />
+    <path
+        android:pathData="M96.2 173.1c-10.3 0 -20.4 -2.1 -29.8 -6 -9.2 -3.8 -17.3 -9.4 -24.3 -16.4 -7 -7 -12.6 -15.2 -16.4 -24.3 -4.1 -9.6 -6.2 -19.6 -6.2 -30 0 -10.3 2.1 -20.4 6 -29.8 3.8 -9.2 9.4 -17.3 16.4 -24.3 7 -7 15.2 -12.6 24.3 -16.4 9.5 -4 19.5 -6 29.8 -6 10.3 0 20.4 2.1 29.8 6 9.2 3.8 17.3 9.4 24.3 16.4 7 7 12.6 15.2 16.4 24.3 4 9.5 6 19.5 6 29.8 0 10.3 -2.1 20.4 -6 29.8 -3.8 9.2 -9.4 17.3 -16.4 24.3 -7 7 -15.2 12.6 -24.3 16.4 -9.2 4.1 -19.3 6.2 -29.6 6.2zm0 -145.3c-37.8 0 -68.6 30.8 -68.6 68.6 0 37.8 30.8 68.6 68.6 68.6 37.8 0 68.6 -30.8 68.6 -68.6 0 -37.8 -30.8 -68.6 -68.6 -68.6z"
+        android:fillColor="#00b4f0" />
+    <path
+        android:pathData="M119.6 129l0 -53.8c3.4 -1.1 5.8 -4.3 5.8 -8 0 -4.6 -3.8 -8.4 -8.4 -8.4 -4.6 0 -8.4 3.8 -8.4 8.4 0 3.6 2.2 6.6 5.4 7.9l0 25L79 83.8 79 64c3.4 -1.1 5.8 -4.3 5.8 -8 0 -4.6 -3.8 -8.4 -8.4 -8.4 -4.6 0 -8.4 3.8 -8.4 8.4 0 3.6 2.2 6.6 5.4 7.9l0 54.1c-3.1 1.2 -5.4 4.3 -5.4 7.9 0 4.6 3.8 8.4 8.4 8.4 4.6 0 8.4 -3.8 8.4 -8.4 0 -3.7 -2.4 -6.9 -5.8 -8l0 -27.3 35 16.3 0 22.2c-3.1 1.2 -5.4 4.3 -5.4 7.9 0 4.6 3.8 8.4 8.4 8.4 4.6 0 8.4 -3.8 8.4 -8.4 0 -3.8 -2.4 -6.9 -5.8 -8z"
+        android:fillColor="#00b4f0" />
+</vector>

android/apps/questInterface/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name" translatable="false">Interface</string>
+</resources>

android/build.gradle

@@ -42,6 +42,8 @@ ext {
     RELEASE_TYPE = project.hasProperty('RELEASE_TYPE') ? project.getProperty('RELEASE_TYPE') : 'DEV'
     STABLE_BUILD = project.hasProperty('STABLE_BUILD') ? project.getProperty('STABLE_BUILD') : '0'
     EXEC_SUFFIX = Os.isFamily(Os.FAMILY_WINDOWS) ? '.exe' : ''
+    appVersionCode = Integer.valueOf(VERSION_CODE ?: 1)
+    appVersionName = RELEASE_NUMBER ?: "1.0"
 }
 
 def appDir = new File(projectDir, 'apps/interface')

android/gradle.properties

@@ -1 +1,2 @@
 org.gradle.jvmargs=-Xms2g -Xmx4g
+android.debug.obsoleteApi=true

android/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sat Dec 01 08:32:47 PST 2018
+#Wed Dec 19 13:46:46 PST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.1-all.zip

android/libraries/oculus/build.gradle

@@ -15,3 +15,7 @@ android {
         targetCompatibility JavaVersion.VERSION_1_8
     }
 }
+
+dependencies {
+    implementation project(path: ':qt')
+}

android/libraries/oculus/src/main/java/io/highfidelity/oculus/OculusMobileActivity.java

@@ -7,62 +7,65 @@
 //
 package io.highfidelity.oculus;
 
-import android.app.Activity;
-import android.content.Intent;
 import android.os.Bundle;
 import android.util.Log;
 import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
-import android.view.WindowManager;
+
+import org.qtproject.qt5.android.bindings.QtActivity;
+import io.highfidelity.utils.HifiUtils;
 
 /**
  * Contains a native surface and forwards the activity lifecycle and surface lifecycle
  * events to the OculusMobileDisplayPlugin
  */
-public class OculusMobileActivity extends Activity implements SurfaceHolder.Callback {
+public class OculusMobileActivity extends QtActivity implements SurfaceHolder.Callback {
     private static final String TAG = OculusMobileActivity.class.getSimpleName();
     static { System.loadLibrary("oculusMobile"); }
+
     private native void nativeOnCreate();
     private native static void nativeOnResume();
     private native static void nativeOnPause();
-    private native static void nativeOnDestroy();
     private native static void nativeOnSurfaceChanged(Surface s);
 
+    private native void questNativeOnCreate();
+    private native void questNativeOnPause();
+    private native void questNativeOnResume();
+    private native void questOnAppAfterLoad();
+
+
     private SurfaceView mView;
     private SurfaceHolder mSurfaceHolder;
 
-
-    public static void launch(Activity activity) {
-        if (activity != null) {
-            activity.runOnUiThread(()->{
-                activity.startActivity(new Intent(activity, OculusMobileActivity.class));
-            });
-        }
-    }
-
-    @Override
     public void onCreate(Bundle savedInstanceState) {
-        Log.w(TAG, "QQQ onCreate");
         super.onCreate(savedInstanceState);
-        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
+
+        Log.w(TAG, "QQQ onCreate");
         // Create a native surface for VR rendering (Qt GL surfaces are not suitable
         // because of the lack of fine control over the surface callbacks)
+        // Forward the create message to the JNI code
         mView = new SurfaceView(this);
-        setContentView(mView);
         mView.getHolder().addCallback(this);
 
-        // Forward the create message to the JNI code
         nativeOnCreate();
+        questNativeOnCreate();
+    }
+    public void onAppLoadedComplete() {
+        Log.w(TAG, "QQQ Load Completed");
+        runOnUiThread(() -> {
+            setContentView(mView);
+            questOnAppAfterLoad();
+        });
     }
 
     @Override
     protected void onDestroy() {
         Log.w(TAG, "QQQ onDestroy");
-        if (mSurfaceHolder != null) {
-            nativeOnSurfaceChanged(null);
-        }
-        nativeOnDestroy();
+
+        nativeOnSurfaceChanged(null);
+
+        Log.w(TAG, "QQQ onDestroy -- SUPER onDestroy");
         super.onDestroy();
     }
 
@@ -70,19 +73,38 @@ public class OculusMobileActivity extends Activity implements SurfaceHolder.Call
     protected void onResume() {
         Log.w(TAG, "QQQ onResume");
         super.onResume();
+        //Reconnect the global reference back to handler
+        nativeOnCreate();
+
+        questNativeOnResume();
         nativeOnResume();
     }
 
     @Override
     protected void onPause() {
         Log.w(TAG, "QQQ onPause");
-        nativeOnPause();
         super.onPause();
+
+        questNativeOnPause();
+        nativeOnPause();
+    }
+
+    @Override
+    protected void onStop(){
+        super.onStop();
+        Log.w(TAG, "QQQ Onstop called");
+    }
+
+    @Override
+    protected void onRestart(){
+        super.onRestart();
+        Log.w(TAG, "QQQ onRestart called ****");
+        questOnAppAfterLoad();
     }
 
     @Override
     public void surfaceCreated(SurfaceHolder holder) {
-        Log.w(TAG, "QQQ surfaceCreated");
+        Log.w(TAG, "QQQ surfaceCreated ************************************");
         nativeOnSurfaceChanged(holder.getSurface());
         mSurfaceHolder = holder;
     }
@@ -96,8 +118,9 @@ public class OculusMobileActivity extends Activity implements SurfaceHolder.Call
 
     @Override
     public void surfaceDestroyed(SurfaceHolder holder) {
-        Log.w(TAG, "QQQ surfaceDestroyed");
+        Log.w(TAG, "QQQ surfaceDestroyed ***************************************************");
         nativeOnSurfaceChanged(null);
         mSurfaceHolder = null;
+
     }
 }

android/libraries/qt/src/main/java/org/qtproject/qt5/android/bindings/QtActivity.java

@@ -70,9 +70,6 @@ public class QtActivity extends Activity {
     public final String QT_ANDROID_DEFAULT_THEME = QT_ANDROID_THEMES[0]; // sets the default theme.
     private QtActivityLoader m_loader = new QtActivityLoader(this);
 
-    public boolean isLoading;
-    public boolean keepInterfaceRunning;
-
     public QtActivity() {
     }
 
@@ -229,10 +226,13 @@ public class QtActivity extends Activity {
     //---------------------------------------------------------------------------
 
     protected void onCreateHook(Bundle savedInstanceState) {
+
         m_loader.APPLICATION_PARAMETERS = APPLICATION_PARAMETERS;
         m_loader.ENVIRONMENT_VARIABLES = ENVIRONMENT_VARIABLES;
         m_loader.QT_ANDROID_THEMES = QT_ANDROID_THEMES;
         m_loader.QT_ANDROID_DEFAULT_THEME = QT_ANDROID_DEFAULT_THEME;
+
+
         m_loader.onCreate(savedInstanceState);
     }
 
@@ -364,7 +364,10 @@ public class QtActivity extends Activity {
     @Override
     protected void onDestroy() {
         super.onDestroy();
-        QtApplication.invokeDelegate();
+
+        QtNative.terminateQt();
+        QtNative.setActivity(null,null);
+        System.exit(0);
     }
     //---------------------------------------------------------------------------
 
@@ -506,9 +509,9 @@ public class QtActivity extends Activity {
         super.onPause();
         // GC: this trick allow us to show a splash activity until Qt app finishes
         // loading
-        if (!isLoading && !keepInterfaceRunning) {
-            QtApplication.invokeDelegate();
-        }
+        //QtApplication.invokeDelegate();
+
+        //TODO(Amer): looking into why this messes up pause.
     }
     //---------------------------------------------------------------------------
 
@@ -647,11 +650,7 @@ public class QtActivity extends Activity {
     @Override
     protected void onStop() {
         super.onStop();
-        if (!keepInterfaceRunning) {
-            QtApplication.invokeDelegate();
-        }
-        QtNative.terminateQt();
-        QtNative.setActivity(null,null);
+        QtApplication.invokeDelegate();
     }
 
     //---------------------------------------------------------------------------

android/settings.gradle

@@ -12,15 +12,26 @@ project(':qt').projectDir = new File(settingsDir, 'libraries/qt')
 // Applications
 //
 
-include ':interface'
-project(':interface').projectDir = new File(settingsDir, 'apps/interface')
+if (!getSettings().hasProperty("SUPPRESS_INTERFACE")) {
+    include ':interface'
+    project(':interface').projectDir = new File(settingsDir, 'apps/interface')
+}
+
+if (!getSettings().hasProperty("SUPPRESS_QUEST_INTERFACE")) {
+    include ':questInterface'
+    project(':questInterface').projectDir = new File(settingsDir, 'apps/questInterface')
+}
 
 //
 // Test projects
 //
 
-include ':framePlayer'
-project(':framePlayer').projectDir = new File(settingsDir, 'apps/framePlayer')
+if (!getSettings().hasProperty("SUPPRESS_FRAME_PLAYER")) {
+    include ':framePlayer'
+    project(':framePlayer').projectDir = new File(settingsDir, 'apps/framePlayer')
+}
 
-include ':questFramePlayer'
-project(':questFramePlayer').projectDir = new File(settingsDir, 'apps/questFramePlayer')
+if (!getSettings().hasProperty("SUPPRESS_QUEST_FRAME_PLAYER")) {
+    include ':questFramePlayer'
+    project(':questFramePlayer').projectDir = new File(settingsDir, 'apps/questFramePlayer')
+}

cmake/macros/AutoScribeShader.cmake

@@ -270,6 +270,16 @@ macro(AUTOSCRIBE_SHADER_LIBS)
     set(AUTOSCRIBE_SHADERGEN_COMMANDS_FILE ${CMAKE_CURRENT_BINARY_DIR}/shadergen.txt)
     file(WRITE ${AUTOSCRIBE_SHADERGEN_COMMANDS_FILE} "${AUTOSCRIBE_SHADERGEN_COMMANDS}")
 
+    if (HIFI_ANDROID)
+        if (
+            (${HIFI_ANDROID_APP} STREQUAL "questInterface") OR 
+            (${HIFI_ANDROID_APP} STREQUAL "questFramePlayer") OR
+            (${HIFI_ANDROID_APP} STREQUAL "framePlayer") 
+        )
+            set(EXTRA_SHADERGEN_ARGS --extensions EXT_clip_cull_distance)
+        endif()
+    endif()
+
     # A custom python script which will generate all our shader artifacts
     add_custom_command(
         OUTPUT ${SCRIBED_SHADERS} ${SPIRV_SHADERS} ${REFLECTED_SHADERS}
@@ -279,6 +289,7 @@ macro(AUTOSCRIBE_SHADER_LIBS)
             --tools-dir ${VCPKG_TOOLS_DIR}
             --build-dir ${CMAKE_CURRENT_BINARY_DIR}
             --source-dir ${CMAKE_SOURCE_DIR}
+            ${EXTRA_SHADERGEN_ARGS}
         DEPENDS ${AUTOSCRIBE_SHADER_HEADERS} ${CMAKE_SOURCE_DIR}/tools/shadergen.py ${ALL_SCRIBE_SHADERS})
 
     add_custom_target(shadergen DEPENDS ${SCRIBED_SHADERS} ${SPIRV_SHADERS} ${REFLECTED_SHADERS})

hifi_android.py

@@ -53,9 +53,9 @@ ANDROID_PACKAGES = {
         'includeLibs': ['libvrapi.so']
     },
     'oculusPlatform': {
-        'file': 'OVRPlatformSDK_v1.32.0.zip',
-        'versionId': 'jG9DB16zOGxSrmtZy4jcQnwO0TJUuaeL',
-        'checksum': 'ab5b203b3a39a56ab148d68fff769e05',
+        'file': 'OVRPlatformSDK_v1.34.0.zip',
+        'versionId': 'vbRUkkyzUAXfTGSEtuiUr_7.Fm5h5BZk',
+        'checksum': '16e4c5f39520f122bc49cb6d5bb88289',
         'sharedLibFolder': 'Android/libs/arm64-v8a',
         'includeLibs': ['libovrplatformloader.so']
     },

interface/resources/controllers/touchscreenvirtualpad.json

@@ -23,7 +23,9 @@
                 "invert"
             ],
             "to": "Actions.Pitch"
-        }
+        },
+
+        { "from": "TouchscreenVirtualPad.RB", "to": "Standard.RB"}
 
     ]
 }

interface/resources/qml/desktop/Desktop.qml

@@ -425,11 +425,12 @@ FocusScope {
             console.warn("Could not find top level window for " + item);
             return;
         }
-
+/*
         if (typeof Controller === "undefined") {
             console.warn("Controller not yet available... can't center");
             return;
         }
+*/
 
         var newRecommendedRectJS = (typeof Controller === "undefined") ? Qt.rect(0,0,0,0) : Controller.getRecommendedHUDRect();
         var newRecommendedRect = Qt.rect(newRecommendedRectJS.x, newRecommendedRectJS.y,
@@ -455,15 +456,17 @@ FocusScope {
             console.warn("Could not find top level window for " + item);
             return;
         }
-
+/*
         if (typeof Controller === "undefined") {
             console.warn("Controller not yet available... can't reposition targetWindow:" + targetWindow);
             return;
         }
+*/
 
         var oldRecommendedRect = recommendedRect;
         var oldRecommendedDimmensions = { x: oldRecommendedRect.width, y: oldRecommendedRect.height };
-        var newRecommendedRect = Controller.getRecommendedHUDRect();
+        var newRecommendedRect = { width: 1280, height: 720, x: 0, y: 0 };
+		if (typeof Controller !== "undefined") newRecommendedRect = Controller.getRecommendedHUDRect();
         var newRecommendedDimmensions = { x: newRecommendedRect.width, y: newRecommendedRect.height };
         repositionWindow(targetWindow, false, oldRecommendedRect, oldRecommendedDimmensions, newRecommendedRect, newRecommendedDimmensions);
     }
@@ -480,7 +483,8 @@ FocusScope {
             return;
         }
 
-        var recommended = Controller.getRecommendedHUDRect();
+        var recommended = { width: 1280, height: 720, x: 0, y: 0 };
+   	    if (typeof Controller !== "undefined") recommended = Controller.getRecommendedHUDRect();
         var maxX = recommended.x + recommended.width;
         var maxY = recommended.y + recommended.height;
         var newPosition = Qt.vector2d(targetWindow.x, targetWindow.y);

interface/resources/qml/hifi/Desktop.qml

@@ -18,8 +18,8 @@ OriginalDesktop.Desktop {
         hoverEnabled: true
         propagateComposedEvents: true
         scrollGestureEnabled: false // we don't need/want these
-        onEntered: ApplicationCompositor.reticleOverDesktop = true
-        onExited: ApplicationCompositor.reticleOverDesktop = false
+        onEntered: if (typeof ApplicationCompositor !== "undefined") ApplicationCompositor.reticleOverDesktop = true
+        onExited: if (typeof ApplicationCompositor !== "undefined") ApplicationCompositor.reticleOverDesktop = false
         acceptedButtons: Qt.NoButton
     }
 

interface/src/Application.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+
 #include "Application.h"
 
 #include <chrono>
@@ -37,7 +38,6 @@
 
 #include <QtNetwork/QLocalSocket>
 #include <QtNetwork/QLocalServer>
-
 #include <QtQml/QQmlContext>
 #include <QtQml/QQmlEngine>
 #include <QtQuick/QQuickWindow>
@@ -51,6 +51,7 @@
 #include <QProcessEnvironment>
 #include <QTemporaryDir>
 
+
 #include <gl/QOpenGLContextWrapper.h>
 #include <gl/GLWindow.h>
 #include <gl/GLHelpers.h>
@@ -191,6 +192,9 @@
 #include "scripting/WalletScriptingInterface.h"
 #include "scripting/TTSScriptingInterface.h"
 #include "scripting/KeyboardScriptingInterface.h"
+
+
+
 #if defined(Q_OS_MAC) || defined(Q_OS_WIN)
 #include "SpeechRecognizer.h"
 #endif
@@ -239,6 +243,7 @@
 #include "webbrowser/WebBrowserSuggestionsEngine.h"
 #include <DesktopPreviewProvider.h>
 
+
 #include "AboutUtil.h"
 
 #if defined(Q_OS_WIN)
@@ -622,8 +627,6 @@ public:
             switch (type) {
             case NestableType::Entity:
                 return getEntityModelProvider(static_cast<EntityItemID>(uuid));
-            case NestableType::Overlay:
-                return nullptr;
             case NestableType::Avatar:
                 return getAvatarModelProvider(uuid);
             }
@@ -1757,7 +1760,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
     });
     _applicationStateDevice->setInputVariant(STATE_PLATFORM_ANDROID, []() -> float {
 #if defined(Q_OS_ANDROID)
-        return 1;
+        return 1 ;
 #else
         return 0;
 #endif
@@ -1860,6 +1863,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
 
     this->installEventFilter(this);
 
+
+
 #ifdef HAVE_DDE
     auto ddeTracker = DependencyManager::get<DdeFaceTracker>();
     ddeTracker->init();
@@ -2306,31 +2311,31 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         DependencyManager::get<PickManager>()->setPrecisionPicking(rayPickID, value);
     });
 
-    EntityItem::setBillboardRotationOperator([this](const glm::vec3& position, const glm::quat& rotation, BillboardMode billboardMode) {
+    EntityItem::setBillboardRotationOperator([this](const glm::vec3& position, const glm::quat& rotation, BillboardMode billboardMode, const glm::vec3& frustumPos) {
         if (billboardMode == BillboardMode::YAW) {
             //rotate about vertical to face the camera
-            ViewFrustum frustum;
-            copyViewFrustum(frustum);
-            glm::vec3 dPosition = frustum.getPosition() - position;
+            glm::vec3 dPosition = frustumPos - position;
             // If x and z are 0, atan(x, z) is undefined, so default to 0 degrees
             float yawRotation = dPosition.x == 0.0f && dPosition.z == 0.0f ? 0.0f : glm::atan(dPosition.x, dPosition.z);
             return glm::quat(glm::vec3(0.0f, yawRotation, 0.0f));
         } else if (billboardMode == BillboardMode::FULL) {
-            ViewFrustum frustum;
-            copyViewFrustum(frustum);
-            glm::vec3 cameraPos = frustum.getPosition();
             // use the referencial from the avatar, y isn't always up
             glm::vec3 avatarUP = DependencyManager::get<AvatarManager>()->getMyAvatar()->getWorldOrientation() * Vectors::UP;
             // check to see if glm::lookAt will work / using glm::lookAt variable name
-            glm::highp_vec3 s(glm::cross(position - cameraPos, avatarUP));
+            glm::highp_vec3 s(glm::cross(position - frustumPos, avatarUP));
 
             // make sure s is not NaN for any component
             if (glm::length2(s) > 0.0f) {
-                return glm::conjugate(glm::toQuat(glm::lookAt(cameraPos, position, avatarUP)));
+                return glm::conjugate(glm::toQuat(glm::lookAt(frustumPos, position, avatarUP)));
             }
         }
         return rotation;
     });
+    EntityItem::setPrimaryViewFrustumPositionOperator([this]() {
+        ViewFrustum viewFrustum;
+        copyViewFrustum(viewFrustum);
+        return viewFrustum.getPosition();
+    });
 
     render::entities::WebEntityRenderer::setAcquireWebSurfaceOperator([this](const QString& url, bool htmlContent, QSharedPointer<OffscreenQmlSurface>& webSurface, bool& cachedWebSurface) {
         bool isTablet = url == TabletScriptingInterface::QML;
@@ -3097,7 +3102,7 @@ void Application::initializeUi() {
         }
         if (TouchscreenVirtualPadDevice::NAME == inputPlugin->getName()) {
             _touchscreenVirtualPadDevice = std::dynamic_pointer_cast<TouchscreenVirtualPadDevice>(inputPlugin);
-#if defined(Q_OS_ANDROID)
+#if defined(ANDROID_APP_INTERFACE)
             auto& virtualPadManager = VirtualPad::Manager::instance();
             connect(&virtualPadManager, &VirtualPad::Manager::hapticFeedbackRequested,
                     this, [](int duration) {
@@ -3629,10 +3634,14 @@ void Application::handleSandboxStatus(QNetworkReply* reply) {
     }
 
     // Get controller availability
+#ifdef ANDROID_APP_QUEST_INTERFACE
+    bool hasHandControllers = true;
+#else
     bool hasHandControllers = false;
     if (PluginUtils::isViveControllerAvailable() || PluginUtils::isOculusTouchControllerAvailable()) {
         hasHandControllers = true;
     }
+#endif
 
     // Check HMD use (may be technically available without being in use)
     bool hasHMD = PluginUtils::isHMDAvailable();
@@ -8249,6 +8258,7 @@ void Application::loadDomainConnectionDialog() {
 }
 
 void Application::toggleLogDialog() {
+#ifndef ANDROID_APP_QUEST_INTERFACE
     if (getLoginDialogPoppedUp()) {
         return;
     }
@@ -8272,6 +8282,7 @@ void Application::toggleLogDialog() {
     } else {
         _logDialog->show();
     }
+#endif
 }
 
  void Application::recreateLogWindow(int keepOnTop) {
@@ -9137,17 +9148,23 @@ void Application::beforeEnterBackground() {
 void Application::enterBackground() {
     QMetaObject::invokeMethod(DependencyManager::get<AudioClient>().data(),
                               "stop", Qt::BlockingQueuedConnection);
+// Quest only supports one plugin which can't be deactivated currently
+#if !defined(ANDROID_APP_QUEST_INTERFACE)
     if (getActiveDisplayPlugin()->isActive()) {
         getActiveDisplayPlugin()->deactivate();
     }
+#endif
 }
 
 void Application::enterForeground() {
     QMetaObject::invokeMethod(DependencyManager::get<AudioClient>().data(),
                                   "start", Qt::BlockingQueuedConnection);
+// Quest only supports one plugin which can't be deactivated currently
+#if !defined(ANDROID_APP_QUEST_INTERFACE)
     if (!getActiveDisplayPlugin() || getActiveDisplayPlugin()->isActive() || !getActiveDisplayPlugin()->activate()) {
         qWarning() << "Could not re-activate display plugin";
     }
+#endif
     auto nodeList = DependencyManager::get<NodeList>();
     nodeList->setSendDomainServerCheckInEnabled(true);
 }

interface/src/avatar/AvatarManager.cpp

@@ -297,6 +297,9 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
                 avatar->setIsNewAvatar(false);
             }
             avatar->simulate(deltaTime, inView);
+            if (avatar->getSkeletonModel()->isLoaded() && avatar->getWorkloadRegion() == workload::Region::R1) {
+                _myAvatar->addAvatarHandsToFlow(avatar);
+            }
             avatar->updateRenderItem(renderTransaction);
             avatar->updateSpaceProxy(workloadTransaction);
             avatar->setLastRenderUpdateTime(startTime);
@@ -716,7 +719,7 @@ RayToAvatarIntersectionResult AvatarManager::findRayIntersectionVector(const Pic
                 }
             }
 
-            if (rayAvatarResult._intersect && pickAgainstMesh) {
+            if (avatar && rayAvatarResult._intersect && pickAgainstMesh) {
                 glm::vec3 localRayOrigin = avatar->worldToJointPoint(ray.origin, rayAvatarResult._intersectWithJoint);
                 glm::vec3 localRayPoint = avatar->worldToJointPoint(ray.origin + rayAvatarResult._distance * rayDirection, rayAvatarResult._intersectWithJoint);
 

interface/src/avatar/MyAvatar.cpp

@@ -2978,6 +2978,10 @@ void MyAvatar::initAnimGraph() {
         graphUrl = _fstAnimGraphOverrideUrl;
     } else {
         graphUrl = PathUtils::resourcesUrl("avatar/avatar-animation.json");
+
+#if defined(Q_OS_ANDROID) || defined(HIFI_USE_OPTIMIZED_IK)
+        graphUrl = PathUtils::resourcesUrl("avatar/avatar-animation_withSplineIKNode.json");
+#endif
     }
 
     emit animGraphUrlChanged(graphUrl);
@@ -5610,6 +5614,72 @@ void MyAvatar::releaseGrab(const QUuid& grabID) {
     }
 }
 
+void MyAvatar::addAvatarHandsToFlow(const std::shared_ptr<Avatar>& otherAvatar) {
+    auto &flow = _skeletonModel->getRig().getFlow();
+    for (auto &handJointName : HAND_COLLISION_JOINTS) {
+        int jointIndex = otherAvatar->getJointIndex(handJointName);
+        if (jointIndex != -1) {
+            glm::vec3 position = otherAvatar->getJointPosition(jointIndex);
+            flow.setOthersCollision(otherAvatar->getID(), jointIndex, position);
+        }
+    }
+}
+
+void MyAvatar::useFlow(bool isActive, bool isCollidable, const QVariantMap& physicsConfig, const QVariantMap& collisionsConfig) {
+    if (_skeletonModel->isLoaded()) {
+        _skeletonModel->getRig().initFlow(isActive);
+        auto &flow = _skeletonModel->getRig().getFlow();
+        auto &collisionSystem = flow.getCollisionSystem();
+        collisionSystem.setActive(isCollidable);
+        auto physicsGroups = physicsConfig.keys();
+        if (physicsGroups.size() > 0) {
+            for (auto &groupName : physicsGroups) {
+                auto settings = physicsConfig[groupName].toMap();
+                FlowPhysicsSettings physicsSettings;
+                if (settings.contains("active")) {
+                    physicsSettings._active = settings["active"].toBool();
+                }
+                if (settings.contains("damping")) {
+                    physicsSettings._damping = settings["damping"].toFloat();
+                }
+                if (settings.contains("delta")) {
+                    physicsSettings._delta = settings["delta"].toFloat();
+                }
+                if (settings.contains("gravity")) {
+                    physicsSettings._gravity = settings["gravity"].toFloat();
+                }
+                if (settings.contains("inertia")) {
+                    physicsSettings._inertia = settings["inertia"].toFloat();
+                }
+                if (settings.contains("radius")) {
+                    physicsSettings._radius = settings["radius"].toFloat();
+                }
+                if (settings.contains("stiffness")) {
+                    physicsSettings._stiffness = settings["stiffness"].toFloat();
+                }
+                flow.setPhysicsSettingsForGroup(groupName, physicsSettings);
+            }
+        }
+        auto collisionJoints = collisionsConfig.keys();
+        if (collisionJoints.size() > 0) {
+            collisionSystem.resetCollisions();
+            for (auto &jointName : collisionJoints) {
+                int jointIndex = getJointIndex(jointName);
+                FlowCollisionSettings collisionsSettings;
+                auto settings = collisionsConfig[jointName].toMap();
+                collisionsSettings._entityID = getID();
+                if (settings.contains("radius")) {
+                    collisionsSettings._radius = settings["radius"].toFloat();
+                }
+                if (settings.contains("offset")) {
+                    collisionsSettings._offset = vec3FromVariant(settings["offset"]);
+                }
+                collisionSystem.addCollisionSphere(jointIndex, collisionsSettings);
+            }
+        }
+    }
+}
+
 void MyAvatar::sendPacket(const QUuid& entityID, const EntityItemProperties& properties) const {
     auto treeRenderer = DependencyManager::get<EntityTreeRenderer>();
     EntityTreePointer entityTree = treeRenderer ? treeRenderer->getTree() : nullptr;

interface/src/avatar/MyAvatar.h

@@ -1307,6 +1307,20 @@ public:
     void updateAvatarEntity(const QUuid& entityID, const QByteArray& entityData) override;
     void avatarEntityDataToJson(QJsonObject& root) const override;
     int sendAvatarDataPacket(bool sendAll = false) override;
+    
+    void addAvatarHandsToFlow(const std::shared_ptr<Avatar>& otherAvatar);
+
+    /**jsdoc
+    * Init flow simulation on avatar.
+    * @function MyAvatar.useFlow
+    * @param {boolean} - Set to <code>true</code> to activate flow simulation.
+    * @param {boolean} - Set to <code>true</code> to activate collisions.
+    * @param {Object} physicsConfig - object with the customized physic parameters
+    * i.e. {"hair": {"active": true, "stiffness": 0.0, "radius": 0.04, "gravity": -0.035, "damping": 0.8, "inertia": 0.8, "delta": 0.35}}
+    * @param {Object} collisionsConfig - object with the customized collision parameters
+    * i.e. {"Spine2": {"type": "sphere", "radius": 0.14, "offset": {"x": 0.0, "y": 0.2, "z": 0.0}}}
+    */
+    Q_INVOKABLE void useFlow(bool isActive, bool isCollidable, const QVariantMap& physicsConfig = QVariantMap(), const QVariantMap& collisionsConfig = QVariantMap());
 
 public slots:
 

interface/src/avatar/MySkeletonModel.cpp

@@ -10,12 +10,14 @@
 
 #include <avatars-renderer/Avatar.h>
 #include <DebugDraw.h>
+#include <CubicHermiteSpline.h>
 
 #include "Application.h"
 #include "InterfaceLogging.h"
 #include "AnimUtil.h"
 
 
+
 MySkeletonModel::MySkeletonModel(Avatar* owningAvatar, QObject* parent) : SkeletonModel(owningAvatar, parent) {
 }
 
@@ -33,6 +35,22 @@ Rig::CharacterControllerState convertCharacterControllerState(CharacterControlle
     };
 }
 
+#if defined(Q_OS_ANDROID) || defined(HIFI_USE_OPTIMIZED_IK)
+static glm::vec3 computeSpine2WithHeadHipsSpline(MyAvatar* myAvatar, AnimPose hipsIKTargetPose, AnimPose headIKTargetPose) {
+
+    // the the ik targets to compute the spline with
+    CubicHermiteSplineFunctorWithArcLength splineFinal(headIKTargetPose.rot(), headIKTargetPose.trans(), hipsIKTargetPose.rot(), hipsIKTargetPose.trans());
+
+    // measure the total arc length along the spline
+    float totalArcLength = splineFinal.arcLength(1.0f);
+    float tFinal = splineFinal.arcLengthInverse(myAvatar->getSpine2SplineRatio() * totalArcLength);
+    glm::vec3 spine2Translation = splineFinal(tFinal);
+
+    return spine2Translation + myAvatar->getSpine2SplineOffset();
+
+}
+#endif
+
 static AnimPose computeHipsInSensorFrame(MyAvatar* myAvatar, bool isFlying) {
     glm::mat4 worldToSensorMat = glm::inverse(myAvatar->getSensorToWorldMatrix());
 
@@ -233,6 +251,12 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) {
             myAvatar->getControllerPoseInAvatarFrame(controller::Action::LEFT_HAND).isValid() &&
             !(params.primaryControllerFlags[Rig::PrimaryControllerType_Spine2] & (uint8_t)Rig::ControllerFlags::Enabled)) {
 
+#if defined(Q_OS_ANDROID) || defined(HIFI_USE_OPTIMIZED_IK)
+            AnimPose headAvatarSpace(avatarHeadPose.getRotation(), avatarHeadPose.getTranslation());
+            AnimPose headRigSpace = avatarToRigPose * headAvatarSpace;
+            AnimPose hipsRigSpace = sensorToRigPose * sensorHips;
+            glm::vec3 spine2TargetTranslation = computeSpine2WithHeadHipsSpline(myAvatar, hipsRigSpace, headRigSpace);
+#endif
             const float SPINE2_ROTATION_FILTER = 0.5f;
             AnimPose currentSpine2Pose;
             AnimPose currentHeadPose;
@@ -243,6 +267,9 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) {
             if (spine2Exists && headExists && hipsExists) {
 
                 AnimPose rigSpaceYaw(myAvatar->getSpine2RotationRigSpace());
+#if defined(Q_OS_ANDROID) || defined(HIFI_USE_OPTIMIZED_IK)
+                rigSpaceYaw.rot() = safeLerp(Quaternions::IDENTITY, rigSpaceYaw.rot(), 0.5f);
+#endif
                 glm::vec3 u, v, w;
                 glm::vec3 fwd = rigSpaceYaw.rot() * glm::vec3(0.0f, 0.0f, 1.0f);
                 glm::vec3 up = currentHeadPose.trans() - currentHipsPose.trans();
@@ -253,6 +280,9 @@ void MySkeletonModel::updateRig(float deltaTime, glm::mat4 parentTransform) {
                 }
                 generateBasisVectors(up, fwd, u, v, w);
                 AnimPose newSpinePose(glm::mat4(glm::vec4(w, 0.0f), glm::vec4(u, 0.0f), glm::vec4(v, 0.0f), glm::vec4(glm::vec3(0.0f, 0.0f, 0.0f), 1.0f)));
+#if defined(Q_OS_ANDROID) || defined(HIFI_USE_OPTIMIZED_IK)
+                currentSpine2Pose.trans() = spine2TargetTranslation;
+#endif
                 currentSpine2Pose.rot() = safeLerp(currentSpine2Pose.rot(), newSpinePose.rot(), SPINE2_ROTATION_FILTER);
                 params.primaryControllerPoses[Rig::PrimaryControllerType_Spine2] = currentSpine2Pose;
                 params.primaryControllerFlags[Rig::PrimaryControllerType_Spine2] = (uint8_t)Rig::ControllerFlags::Enabled | (uint8_t)Rig::ControllerFlags::Estimated;

interface/src/avatar/OtherAvatar.cpp

@@ -368,7 +368,6 @@ void OtherAvatar::simulate(float deltaTime, bool inView) {
         PROFILE_RANGE(simulation, "grabs");
         applyGrabChanges();
     }
-
     updateFadingStatus();
 }
 

interface/src/avatar/OtherAvatar.h

@@ -48,6 +48,7 @@ public:
     void rebuildCollisionShape() override;
 
     void setWorkloadRegion(uint8_t region);
+    uint8_t getWorkloadRegion() { return _workloadRegion; }
     bool shouldBeInPhysicsSimulation() const;
     bool needsPhysicsUpdate() const;
 

interface/src/raypick/RayPick.h

@@ -85,9 +85,10 @@ public:
     static glm::vec3 intersectRayWithEntityXYPlane(const QUuid& entityID, const glm::vec3& origin, const glm::vec3& direction);
     static glm::vec2 projectOntoEntityXYPlane(const QUuid& entityID, const glm::vec3& worldPos, bool unNormalized = true);
 
+    static glm::vec2 projectOntoXYPlane(const glm::vec3& worldPos, const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions, const glm::vec3& registrationPoint, bool unNormalized);
+
 private:
     static glm::vec3 intersectRayWithXYPlane(const glm::vec3& origin, const glm::vec3& direction, const glm::vec3& point, const glm::quat& rotation, const glm::vec3& registration);
-    static glm::vec2 projectOntoXYPlane(const glm::vec3& worldPos, const glm::vec3& position, const glm::quat& rotation, const glm::vec3& dimensions, const glm::vec3& registrationPoint, bool unNormalized);
 };
 
 #endif // hifi_RayPick_h

interface/src/raypick/StylusPick.cpp

@@ -137,13 +137,14 @@ PickResultPointer StylusPick::getDefaultResult(const QVariantMap& pickVariant) c
 }
 
 PickResultPointer StylusPick::getEntityIntersection(const StylusTip& pick) {
+    auto entityTree = qApp->getEntities()->getTree();
     StylusPickResult nearestTarget(pick.toVariantMap());
     for (const auto& target : getIncludeItems()) {
         if (target.isNull()) {
             continue;
         }
 
-        auto entity = qApp->getEntities()->getTree()->findEntityByEntityItemID(target);
+        auto entity = entityTree->findEntityByEntityItemID(target);
         if (!entity) {
             continue;
         }
@@ -158,8 +159,11 @@ PickResultPointer StylusPick::getEntityIntersection(const StylusTip& pick) {
         glm::vec3 normal = entityRotation * Vectors::UNIT_Z;
         float distance = glm::dot(pick.position - entityPosition, normal);
         if (distance < nearestTarget.distance) {
+            const auto entityDimensions = entity->getScaledDimensions();
+            const auto entityRegistrationPoint = entity->getRegistrationPoint();
             glm::vec3 intersection = pick.position - (normal * distance);
-            glm::vec2 pos2D = RayPick::projectOntoEntityXYPlane(target, intersection, false);
+            glm::vec2 pos2D = RayPick::projectOntoXYPlane(intersection, entityPosition, entityRotation,
+                                                          entityDimensions, entityRegistrationPoint, false);
             if (pos2D == glm::clamp(pos2D, glm::vec2(0), glm::vec2(1))) {
                 IntersectionType type = IntersectionType::ENTITY;
                 if (getFilter().doesPickLocalEntities()) {

interface/src/scripting/Audio.cpp

@@ -27,8 +27,9 @@ QString Audio::HMD { "VR" };
 Setting::Handle<bool> enableNoiseReductionSetting { QStringList { Audio::AUDIO, "NoiseReduction" }, true };
 
 float Audio::loudnessToLevel(float loudness) {
-    float level = 6.02059991f * fastLog2f(loudness);    // level in dBFS
-    level = (level + 48.0f) * (1/39.0f);                // map [-48, -9] dBFS to [0, 1]
+    float level = loudness * (1/32768.0f);  // level in [0, 1]
+    level = 6.02059991f * fastLog2f(level); // convert to dBFS
+    level = (level + 48.0f) * (1/42.0f);    // map [-48, -6] dBFS to [0, 1]
     return glm::clamp(level, 0.0f, 1.0f);
 }
 

interface/src/ui/Keyboard.cpp

@@ -309,12 +309,22 @@ void Keyboard::setRaised(bool raised) {
             _layerIndex = 0;
             _capsEnabled = false;
             _typedCharacters.clear();
+            addIncludeItemsToMallets();
         });
 
         updateTextDisplay();
     }
 }
 
+void Keyboard::addIncludeItemsToMallets() {
+    if (_layerIndex >= 0 && _layerIndex < (int)_keyboardLayers.size()) {
+        QVector<QUuid> includeItems = _keyboardLayers[_layerIndex].keys().toVector();
+        auto pointerManager = DependencyManager::get<PointerManager>();
+        pointerManager->setIncludeItems(_leftHandStylus, includeItems);
+        pointerManager->setIncludeItems(_rightHandStylus, includeItems);
+    }
+}
+
 void Keyboard::updateTextDisplay() {
     auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
     auto entityScriptingInterface = DependencyManager::get<EntityScriptingInterface>();
@@ -463,6 +473,8 @@ void Keyboard::switchToLayer(int layerIndex) {
          properties.setRotation(currentOrientation);
          entityScriptingInterface->editEntity(_anchor.entityID, properties);
 
+         addIncludeItemsToMallets();
+
          startLayerSwitchTimer();
     }
 }
@@ -718,8 +730,6 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
         clearKeyboardKeys();
         auto requestData = request->getData();
 
-        QVector<QUuid> includeItems;
-
         QJsonParseError parseError;
         QJsonDocument jsonDoc = QJsonDocument::fromJson(requestData, &parseError);
 
@@ -840,7 +850,6 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
                 key.setKeyString(keyString);
                 key.saveDimensionsAndLocalPosition();
 
-                includeItems.append(key.getID());
                 _itemsToIgnore.insert(key.getID());
                 keyboardLayerKeys.insert(id, key);
             }
@@ -886,9 +895,7 @@ void Keyboard::loadKeyboardFile(const QString& keyboardFile) {
             _itemsToIgnore.insert(_anchor.entityID);
         });
         _layerIndex = 0;
-        auto pointerManager = DependencyManager::get<PointerManager>();
-        pointerManager->setIncludeItems(_leftHandStylus, includeItems);
-        pointerManager->setIncludeItems(_rightHandStylus, includeItems);
+        addIncludeItemsToMallets();
     });
 
     request->send();

interface/src/ui/Keyboard.h

@@ -157,6 +157,7 @@ private:
     bool shouldProcessEntityAndPointerEvent(const PointerEvent& event) const;
     bool shouldProcessPointerEvent(const PointerEvent& event) const;
     bool shouldProcessEntity() const;
+    void addIncludeItemsToMallets();
 
     void startLayerSwitchTimer();
     bool isLayerSwitchTimerFinished() const;
@@ -178,7 +179,12 @@ private:
     mutable ReadWriteLockable _handLaserLock;
     mutable ReadWriteLockable _preferMalletsOverLasersSettingLock;
     mutable ReadWriteLockable _ignoreItemsLock;
+
+#ifdef Q_OS_ANDROID
+    Setting::Handle<bool> _use3DKeyboard { "use3DKeyboard", false };
+#else
     Setting::Handle<bool> _use3DKeyboard { "use3DKeyboard", true };
+#endif
 
     QString _typedCharacters;
     TextDisplay _textDisplay;

interface/src/ui/overlays/Overlays.cpp

@@ -311,7 +311,11 @@ EntityItemProperties Overlays::convertOverlayToEntityProperties(QVariantMap& ove
         RENAME_PROP(start, position);
     }
     RENAME_PROP(point, position);
-    RENAME_PROP(scale, dimensions);
+    if (type != "Model") {
+        RENAME_PROP(scale, dimensions);
+    } else {
+        RENAME_PROP(scale, modelScale);
+    }
     RENAME_PROP(size, dimensions);
     RENAME_PROP(orientation, rotation);
     RENAME_PROP(localOrientation, localRotation);
@@ -559,6 +563,42 @@ EntityItemProperties Overlays::convertOverlayToEntityProperties(QVariantMap& ove
         SET_OVERLAY_PROP_DEFAULT(textures, PathUtils::resourcesUrl() + "images/whitePixel.png");
     }
 
+    { // Overlays did this conversion for rotation
+        auto iter = overlayProps.find("rotation");
+        if (iter != overlayProps.end() && !overlayProps.contains("localRotation")) {
+            QUuid parentID;
+            {
+                auto iter = overlayProps.find("parentID");
+                if (iter != overlayProps.end()) {
+                    parentID = iter.value().toUuid();
+                } else if (!add) {
+                    EntityPropertyFlags desiredProperties;
+                    desiredProperties += PROP_PARENT_ID;
+                    parentID = DependencyManager::get<EntityScriptingInterface>()->getEntityProperties(id, desiredProperties).getParentID();
+                }
+            }
+
+            int parentJointIndex = -1;
+            {
+                auto iter = overlayProps.find("parentJointIndex");
+                if (iter != overlayProps.end()) {
+                    parentJointIndex = iter.value().toInt();
+                } else if (!add) {
+                    EntityPropertyFlags desiredProperties;
+                    desiredProperties += PROP_PARENT_JOINT_INDEX;
+                    parentJointIndex = DependencyManager::get<EntityScriptingInterface>()->getEntityProperties(id, desiredProperties).getParentJointIndex();
+                }
+            }
+
+            glm::quat rotation = quatFromVariant(iter.value());
+            bool success = false;
+            glm::quat localRotation = SpatiallyNestable::worldToLocal(rotation, parentID, parentJointIndex, false, success);
+            if (success) {
+                overlayProps["rotation"] = quatToVariant(localRotation);
+            }
+        }
+    }
+
     if (type == "Text" || type == "Image" || type == "Grid" || type == "Web") {
         glm::quat originalRotation = ENTITY_ITEM_DEFAULT_ROTATION;
         {
@@ -636,7 +676,11 @@ QVariantMap Overlays::convertEntityToOverlayProperties(const EntityItemPropertie
         RENAME_PROP(position, start);
     }
     RENAME_PROP(position, point);
-    RENAME_PROP(dimensions, scale);
+    if (type != "Model") {
+        RENAME_PROP(dimensions, scale);
+    } else {
+        RENAME_PROP(modelScale, scale);
+    }
     RENAME_PROP(dimensions, size);
     RENAME_PROP(ignorePickIntersection, ignoreRayIntersection);
 
@@ -1718,7 +1762,8 @@ QVector<QUuid> Overlays::findOverlays(const glm::vec3& center, float radius) {
  *
  * @property {Vec3} position - The position of the overlay center. Synonyms: <code>p1</code>, <code>point</code>, and
  *     <code>start</code>.
- * @property {Vec3} dimensions - The dimensions of the overlay. Synonyms: <code>scale</code>, <code>size</code>.
+ * @property {Vec3} dimensions - The dimensions of the overlay. Synonyms: <code>size</code>.
+ * @property {Vec3} scale - The scale factor applied to the model's dimensions.
  * @property {Quat} rotation - The orientation of the overlay. Synonym: <code>orientation</code>.
  * @property {Vec3} localPosition - The local position of the overlay relative to its parent if the overlay has a
  *     <code>parentID</code> set, otherwise the same value as <code>position</code>.

libraries/animation/src/AnimContext.h

@@ -28,6 +28,7 @@ enum class AnimNodeType {
     InverseKinematics,
     DefaultPose,
     TwoBoneIK,
+    SplineIK,
     PoleVectorConstraint,
     NumTypes
 };

libraries/animation/src/AnimInverseKinematics.cpp

@@ -865,10 +865,6 @@ const AnimPoseVec& AnimInverseKinematics::evaluate(const AnimVariantMap& animVar
 
 //virtual
 const AnimPoseVec& AnimInverseKinematics::overlay(const AnimVariantMap& animVars, const AnimContext& context, float dt, AnimVariantMap& triggersOut, const AnimPoseVec& underPoses) {
-#ifdef Q_OS_ANDROID
-    // disable IK on android
-    return underPoses;
-#endif
 
     // allows solutionSource to be overridden by an animVar
     auto solutionSource = animVars.lookup(_solutionSourceVar, (int)_solutionSource);

libraries/animation/src/AnimNodeLoader.cpp

@@ -26,6 +26,7 @@
 #include "AnimInverseKinematics.h"
 #include "AnimDefaultPose.h"
 #include "AnimTwoBoneIK.h"
+#include "AnimSplineIK.h"
 #include "AnimPoleVectorConstraint.h"
 
 using NodeLoaderFunc = AnimNode::Pointer (*)(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
@@ -41,6 +42,7 @@ static AnimNode::Pointer loadManipulatorNode(const QJsonObject& jsonObj, const Q
 static AnimNode::Pointer loadInverseKinematicsNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
 static AnimNode::Pointer loadDefaultPoseNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
 static AnimNode::Pointer loadTwoBoneIKNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
+static AnimNode::Pointer loadSplineIKNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
 static AnimNode::Pointer loadPoleVectorConstraintNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl);
 
 static const float ANIM_GRAPH_LOAD_PRIORITY = 10.0f;
@@ -61,6 +63,7 @@ static const char* animNodeTypeToString(AnimNode::Type type) {
     case AnimNode::Type::InverseKinematics: return "inverseKinematics";
     case AnimNode::Type::DefaultPose: return "defaultPose";
     case AnimNode::Type::TwoBoneIK: return "twoBoneIK";
+    case AnimNode::Type::SplineIK: return "splineIK";
     case AnimNode::Type::PoleVectorConstraint: return "poleVectorConstraint";
     case AnimNode::Type::NumTypes: return nullptr;
     };
@@ -123,6 +126,7 @@ static NodeLoaderFunc animNodeTypeToLoaderFunc(AnimNode::Type type) {
     case AnimNode::Type::InverseKinematics: return loadInverseKinematicsNode;
     case AnimNode::Type::DefaultPose: return loadDefaultPoseNode;
     case AnimNode::Type::TwoBoneIK: return loadTwoBoneIKNode;
+    case AnimNode::Type::SplineIK: return loadSplineIKNode;
     case AnimNode::Type::PoleVectorConstraint: return loadPoleVectorConstraintNode;
     case AnimNode::Type::NumTypes: return nullptr;
     };
@@ -140,6 +144,7 @@ static NodeProcessFunc animNodeTypeToProcessFunc(AnimNode::Type type) {
     case AnimNode::Type::InverseKinematics: return processDoNothing;
     case AnimNode::Type::DefaultPose: return processDoNothing;
     case AnimNode::Type::TwoBoneIK: return processDoNothing;
+    case AnimNode::Type::SplineIK: return processDoNothing;
     case AnimNode::Type::PoleVectorConstraint: return processDoNothing;
     case AnimNode::Type::NumTypes: return nullptr;
     };
@@ -574,6 +579,52 @@ static AnimNode::Pointer loadDefaultPoseNode(const QJsonObject& jsonObj, const Q
     return node;
 }
 
+static AnimNode::Pointer loadSplineIKNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl) {
+    READ_FLOAT(alpha, jsonObj, id, jsonUrl, nullptr);
+    READ_BOOL(enabled, jsonObj, id, jsonUrl, nullptr);
+    READ_FLOAT(interpDuration, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(baseJointName, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(midJointName, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(tipJointName, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(basePositionVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(baseRotationVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(midPositionVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(midRotationVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(tipPositionVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(tipRotationVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(alphaVar, jsonObj, id, jsonUrl, nullptr);
+    READ_STRING(enabledVar, jsonObj, id, jsonUrl, nullptr);
+
+    auto tipFlexCoefficientsValue = jsonObj.value("tipTargetFlexCoefficients");
+    if (!tipFlexCoefficientsValue.isArray()) {
+        qCCritical(animation) << "AnimNodeLoader, bad or missing tip flex array";
+        return nullptr;
+    }
+    auto tipFlexCoefficientsArray = tipFlexCoefficientsValue.toArray();
+    std::vector<float> tipTargetFlexCoefficients;
+    for (const auto& value : tipFlexCoefficientsArray) {
+        tipTargetFlexCoefficients.push_back((float)value.toDouble());
+    }
+
+    auto midFlexCoefficientsValue = jsonObj.value("midTargetFlexCoefficients");
+    if (!midFlexCoefficientsValue.isArray()) {
+        qCCritical(animation) << "AnimNodeLoader, bad or missing mid flex array";
+        return nullptr;
+    }
+    auto midFlexCoefficientsArray = midFlexCoefficientsValue.toArray();
+    std::vector<float> midTargetFlexCoefficients;
+    for (const auto& midValue : midFlexCoefficientsArray) {
+        midTargetFlexCoefficients.push_back((float)midValue.toDouble());
+    }
+
+    auto node = std::make_shared<AnimSplineIK>(id, alpha, enabled, interpDuration,
+        baseJointName, midJointName, tipJointName,
+        basePositionVar, baseRotationVar, midPositionVar, midRotationVar,
+        tipPositionVar, tipRotationVar, alphaVar, enabledVar,
+        tipTargetFlexCoefficients, midTargetFlexCoefficients);
+    return node;
+}
+
 static AnimNode::Pointer loadTwoBoneIKNode(const QJsonObject& jsonObj, const QString& id, const QUrl& jsonUrl) {
     READ_FLOAT(alpha, jsonObj, id, jsonUrl, nullptr);
     READ_BOOL(enabled, jsonObj, id, jsonUrl, nullptr);

libraries/animation/src/AnimPoleVectorConstraint.cpp

@@ -117,7 +117,7 @@ const AnimPoseVec& AnimPoleVectorConstraint::evaluate(const AnimVariantMap& anim
     if (axisLength > MIN_LENGTH && refVectorLength > MIN_LENGTH && sideVectorLength > MIN_LENGTH &&
         refVectorProjLength > MIN_LENGTH && poleVectorProjLength > MIN_LENGTH) {
 
-        float dot = glm::clamp(glm::dot(refVectorProj / refVectorProjLength, poleVectorProj / poleVectorProjLength), 0.0f, 1.0f);
+        float dot = glm::clamp(glm::dot(refVectorProj / refVectorProjLength, poleVectorProj / poleVectorProjLength), -1.0f, 1.0f);
         float sideDot = glm::dot(poleVector, sideVector);
         float theta = copysignf(1.0f, sideDot) * acosf(dot);
 

libraries/animation/src/AnimSplineIK.cpp

@@ -0,0 +1,473 @@
+//
+//  AnimSplineIK.cpp
+//
+//  Created by Angus Antley on 1/7/19.
+//  Copyright (c) 2019 High Fidelity, Inc. All rights reserved.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AnimSplineIK.h"
+#include "AnimationLogging.h"
+#include "CubicHermiteSpline.h"
+#include <DebugDraw.h>
+#include "AnimUtil.h"
+
+static const float FRAMES_PER_SECOND = 30.0f;
+
+AnimSplineIK::AnimSplineIK(const QString& id, float alpha, bool enabled, float interpDuration,
+    const QString& baseJointName,
+    const QString& midJointName,
+    const QString& tipJointName,
+    const QString& basePositionVar,
+    const QString& baseRotationVar,
+    const QString& midPositionVar,
+    const QString& midRotationVar,
+    const QString& tipPositionVar,
+    const QString& tipRotationVar,
+    const QString& alphaVar,
+    const QString& enabledVar,
+    const std::vector<float> tipTargetFlexCoefficients,
+    const std::vector<float> midTargetFlexCoefficients) :
+    AnimNode(AnimNode::Type::SplineIK, id),
+    _alpha(alpha),
+    _enabled(enabled),
+    _interpDuration(interpDuration),
+    _baseJointName(baseJointName),
+    _midJointName(midJointName),
+    _tipJointName(tipJointName),
+    _basePositionVar(basePositionVar),
+    _baseRotationVar(baseRotationVar),
+    _midPositionVar(midPositionVar),
+    _midRotationVar(midRotationVar),
+    _tipPositionVar(tipPositionVar),
+    _tipRotationVar(tipRotationVar),
+    _alphaVar(alphaVar),
+    _enabledVar(enabledVar)
+{
+
+    for (int i = 0; i < (int)tipTargetFlexCoefficients.size(); i++) {
+        if (i < MAX_NUMBER_FLEX_VARIABLES) {
+            _tipTargetFlexCoefficients[i] = tipTargetFlexCoefficients[i];
+        }
+     }
+    _numTipTargetFlexCoefficients = std::min((int)tipTargetFlexCoefficients.size(), MAX_NUMBER_FLEX_VARIABLES);
+
+    for (int i = 0; i < (int)midTargetFlexCoefficients.size(); i++) {
+        if (i < MAX_NUMBER_FLEX_VARIABLES) {
+            _midTargetFlexCoefficients[i] = midTargetFlexCoefficients[i];
+        }
+    }
+    _numMidTargetFlexCoefficients = std::min((int)midTargetFlexCoefficients.size(), MAX_NUMBER_FLEX_VARIABLES);
+
+}
+
+AnimSplineIK::~AnimSplineIK() {
+
+}
+
+const AnimPoseVec& AnimSplineIK::evaluate(const AnimVariantMap& animVars, const AnimContext& context, float dt, AnimVariantMap& triggersOut) {
+    assert(_children.size() == 1);
+    if (_children.size() != 1) {
+        return _poses;
+    }
+
+    const float MIN_ALPHA = 0.0f;
+    const float MAX_ALPHA = 1.0f;
+    float alpha = glm::clamp(animVars.lookup(_alphaVar, _alpha), MIN_ALPHA, MAX_ALPHA);
+
+    // evaluate underPoses
+    AnimPoseVec underPoses = _children[0]->evaluate(animVars, context, dt, triggersOut);
+
+    // if we don't have a skeleton, or jointName lookup failed or the spline alpha is 0 or there are no underposes.
+    if (!_skeleton || _baseJointIndex == -1 || _midJointIndex == -1 || _tipJointIndex == -1 || alpha < EPSILON || underPoses.size() == 0) {
+        // pass underPoses through unmodified.
+        _poses = underPoses;
+        return _poses;
+    }
+
+    // guard against size change
+    if (underPoses.size() != _poses.size()) {
+        _poses = underPoses;
+    }
+
+    // determine if we should interpolate
+    bool enabled = animVars.lookup(_enabledVar, _enabled);
+    if (enabled != _enabled) {
+        AnimChain poseChain;
+        poseChain.buildFromRelativePoses(_skeleton, _poses, _tipJointIndex);
+        if (enabled) {
+            beginInterp(InterpType::SnapshotToSolve, poseChain);
+        } else {
+            beginInterp(InterpType::SnapshotToUnderPoses, poseChain);
+        }
+    }
+    _enabled = enabled;
+
+    // now that we have saved the previous _poses in _snapshotChain, we can update to the current underposes
+    _poses = underPoses;
+
+    // don't build chains or do IK if we are disabled & not interping.
+    if (_interpType == InterpType::None && !enabled) {
+        return _poses;
+    }
+
+    // compute under chain for possible interpolation
+    AnimChain underChain;
+    underChain.buildFromRelativePoses(_skeleton, underPoses, _tipJointIndex);
+
+    AnimPose baseTargetAbsolutePose;
+    // if there is a baseJoint ik target in animvars then set the joint to that
+    // otherwise use the underpose
+    AnimPose baseJointUnderPose = _skeleton->getAbsolutePose(_baseJointIndex, _poses);
+    baseTargetAbsolutePose.rot() = animVars.lookupRigToGeometry(_baseRotationVar, baseJointUnderPose.rot());
+    baseTargetAbsolutePose.trans() = animVars.lookupRigToGeometry(_basePositionVar, baseJointUnderPose.trans());
+
+    int baseParentIndex = _skeleton->getParentIndex(_baseJointIndex);
+    AnimPose baseParentAbsPose(Quaternions::IDENTITY,glm::vec3());
+    if (baseParentIndex >= 0) {
+        baseParentAbsPose = _skeleton->getAbsolutePose(baseParentIndex, _poses);
+    }
+    _poses[_baseJointIndex] = baseParentAbsPose.inverse() * baseTargetAbsolutePose;
+    _poses[_baseJointIndex].scale() = glm::vec3(1.0f);
+
+    // initialize the middle joint target
+    IKTarget midTarget;
+    midTarget.setType((int)IKTarget::Type::Spline);
+    midTarget.setIndex(_midJointIndex);
+    AnimPose absPoseMid = _skeleton->getAbsolutePose(_midJointIndex, _poses);
+    glm::quat midTargetRotation = animVars.lookupRigToGeometry(_midRotationVar, absPoseMid.rot());
+    glm::vec3 midTargetPosition = animVars.lookupRigToGeometry(_midPositionVar, absPoseMid.trans());
+    midTarget.setPose(midTargetRotation, midTargetPosition);
+    midTarget.setWeight(1.0f);
+    midTarget.setFlexCoefficients(_numMidTargetFlexCoefficients, _midTargetFlexCoefficients);
+
+    // solve the lower spine spline
+    AnimChain midJointChain;
+    AnimPoseVec absolutePosesAfterBaseTipSpline;
+    absolutePosesAfterBaseTipSpline.resize(_poses.size());
+    computeAbsolutePoses(absolutePosesAfterBaseTipSpline);
+    midJointChain.buildFromRelativePoses(_skeleton, _poses, midTarget.getIndex());
+    solveTargetWithSpline(context, _baseJointIndex, midTarget, absolutePosesAfterBaseTipSpline, context.getEnableDebugDrawIKChains(), midJointChain);
+    midJointChain.outputRelativePoses(_poses);
+
+    // initialize the tip target
+    IKTarget tipTarget;
+    tipTarget.setType((int)IKTarget::Type::Spline);
+    tipTarget.setIndex(_tipJointIndex);
+    AnimPose absPoseTip = _skeleton->getAbsolutePose(_tipJointIndex, _poses);
+    glm::quat tipRotation = animVars.lookupRigToGeometry(_tipRotationVar, absPoseTip.rot());
+    glm::vec3 tipTranslation = animVars.lookupRigToGeometry(_tipPositionVar, absPoseTip.trans());
+    tipTarget.setPose(tipRotation, tipTranslation);
+    tipTarget.setWeight(1.0f);
+    tipTarget.setFlexCoefficients(_numTipTargetFlexCoefficients, _tipTargetFlexCoefficients);
+
+    // solve the upper spine spline
+    AnimChain upperJointChain;
+    AnimPoseVec finalAbsolutePoses;
+    finalAbsolutePoses.resize(_poses.size());
+    computeAbsolutePoses(finalAbsolutePoses);
+    upperJointChain.buildFromRelativePoses(_skeleton, _poses, tipTarget.getIndex());
+    solveTargetWithSpline(context, _midJointIndex, tipTarget, finalAbsolutePoses, context.getEnableDebugDrawIKChains(), upperJointChain);
+    upperJointChain.buildDirtyAbsolutePoses();
+    upperJointChain.outputRelativePoses(_poses);
+
+    // compute chain
+    AnimChain ikChain;
+    ikChain.buildFromRelativePoses(_skeleton, _poses, _tipJointIndex);
+    // blend with the underChain
+    ikChain.blend(underChain, alpha);
+
+    // apply smooth interpolation when turning ik on and off
+    if (_interpType != InterpType::None) {
+        _interpAlpha += _interpAlphaVel * dt;
+
+        // ease in expo
+        float easeInAlpha = 1.0f - powf(2.0f, -10.0f * _interpAlpha);
+
+        if (_interpAlpha < 1.0f) {
+            AnimChain interpChain;
+            if (_interpType == InterpType::SnapshotToUnderPoses) {
+                interpChain = underChain;
+                interpChain.blend(_snapshotChain, easeInAlpha);
+            } else if (_interpType == InterpType::SnapshotToSolve) {
+                interpChain = ikChain;
+                interpChain.blend(_snapshotChain, easeInAlpha);
+            }
+            // copy interpChain into _poses
+            interpChain.outputRelativePoses(_poses);
+        } else {
+            // interpolation complete
+            _interpType = InterpType::None;
+        }
+    }
+
+    if (_interpType == InterpType::None) {
+        if (enabled) {
+            // copy chain into _poses
+            ikChain.outputRelativePoses(_poses);
+        } else {
+            // copy under chain into _poses
+            underChain.outputRelativePoses(_poses);
+        }
+    }
+
+    // debug render ik targets
+    if (context.getEnableDebugDrawIKTargets()) {
+        const vec4 WHITE(1.0f);
+        const vec4 GREEN(0.0f, 1.0f, 0.0f, 1.0f);
+        glm::mat4 rigToAvatarMat = createMatFromQuatAndPos(Quaternions::Y_180, glm::vec3());
+
+        glm::mat4 geomTargetMat = createMatFromQuatAndPos(tipTarget.getRotation(), tipTarget.getTranslation());
+        glm::mat4 avatarTargetMat = rigToAvatarMat * context.getGeometryToRigMatrix() * geomTargetMat;
+        QString name = QString("ikTargetSplineTip");
+        DebugDraw::getInstance().addMyAvatarMarker(name, glmExtractRotation(avatarTargetMat), extractTranslation(avatarTargetMat), WHITE);
+
+        glm::mat4 geomTargetMat2 = createMatFromQuatAndPos(midTarget.getRotation(), midTarget.getTranslation());
+        glm::mat4 avatarTargetMat2 = rigToAvatarMat * context.getGeometryToRigMatrix() * geomTargetMat2;
+        QString name2 = QString("ikTargetSplineMid");
+        DebugDraw::getInstance().addMyAvatarMarker(name2, glmExtractRotation(avatarTargetMat2), extractTranslation(avatarTargetMat2), WHITE);
+
+        glm::mat4 geomTargetMat3 = createMatFromQuatAndPos(baseTargetAbsolutePose.rot(), baseTargetAbsolutePose.trans());
+        glm::mat4 avatarTargetMat3 = rigToAvatarMat * context.getGeometryToRigMatrix() * geomTargetMat3;
+        QString name3 = QString("ikTargetSplineBase");
+        DebugDraw::getInstance().addMyAvatarMarker(name3, glmExtractRotation(avatarTargetMat3), extractTranslation(avatarTargetMat3), WHITE);
+
+
+    } else if (context.getEnableDebugDrawIKTargets() != _previousEnableDebugIKTargets) {
+
+        // remove markers if they were added last frame.
+        QString name = QString("ikTargetSplineTip");
+        DebugDraw::getInstance().removeMyAvatarMarker(name);
+        QString name2 = QString("ikTargetSplineMid");
+        DebugDraw::getInstance().removeMyAvatarMarker(name2);
+        QString name3 = QString("ikTargetSplineBase");
+        DebugDraw::getInstance().removeMyAvatarMarker(name3);
+    }
+    _previousEnableDebugIKTargets = context.getEnableDebugDrawIKTargets();
+
+    return _poses;
+}
+
+void AnimSplineIK::lookUpIndices() {
+    assert(_skeleton);
+
+    // look up bone indices by name
+    std::vector<int> indices = _skeleton->lookUpJointIndices({ _baseJointName, _tipJointName, _midJointName });
+
+    // cache the results
+    _baseJointIndex = indices[0];
+    _tipJointIndex = indices[1];
+    _midJointIndex = indices[2];
+}
+
+void AnimSplineIK::computeAbsolutePoses(AnimPoseVec& absolutePoses) const {
+    int numJoints = (int)_poses.size();
+    assert(numJoints <= _skeleton->getNumJoints());
+    assert(numJoints == (int)absolutePoses.size());
+    for (int i = 0; i < numJoints; ++i) {
+        int parentIndex = _skeleton->getParentIndex(i);
+        if (parentIndex < 0) {
+            absolutePoses[i] = _poses[i];
+        } else {
+            absolutePoses[i] = absolutePoses[parentIndex] * _poses[i];
+        }
+    }
+}
+
+// for AnimDebugDraw rendering
+const AnimPoseVec& AnimSplineIK::getPosesInternal() const {
+    return _poses;
+}
+
+void AnimSplineIK::setSkeletonInternal(AnimSkeleton::ConstPointer skeleton) {
+    AnimNode::setSkeletonInternal(skeleton);
+    lookUpIndices();
+}
+
+void AnimSplineIK::solveTargetWithSpline(const AnimContext& context, int base, const IKTarget& target, const AnimPoseVec& absolutePoses, bool debug, AnimChain& chainInfoOut) const {
+
+    // build spline from tip to base
+    AnimPose tipPose = AnimPose(glm::vec3(1.0f), target.getRotation(), target.getTranslation());
+    AnimPose basePose = absolutePoses[base];
+
+    CubicHermiteSplineFunctorWithArcLength spline;
+    if (target.getIndex() == _tipJointIndex) {
+        // set gain factors so that more curvature occurs near the tip of the spline.
+        const float HIPS_GAIN = 0.5f;
+        const float HEAD_GAIN = 1.0f;
+        spline = CubicHermiteSplineFunctorWithArcLength(tipPose.rot(), tipPose.trans(), basePose.rot(), basePose.trans(), HIPS_GAIN, HEAD_GAIN);
+    } else {
+        spline = CubicHermiteSplineFunctorWithArcLength(tipPose.rot(),tipPose.trans(), basePose.rot(), basePose.trans());
+    }
+    float totalArcLength = spline.arcLength(1.0f);
+
+    // This prevents the rotation interpolation from rotating the wrong physical way (but correct mathematical way)
+    // when the head is arched backwards very far.
+    glm::quat halfRot = safeLerp(basePose.rot(), tipPose.rot(), 0.5f);
+    if (glm::dot(halfRot * Vectors::UNIT_Z, basePose.rot() * Vectors::UNIT_Z) < 0.0f) {
+        tipPose.rot() = -tipPose.rot();
+    }
+
+    // find or create splineJointInfo for this target
+    const std::vector<SplineJointInfo>* splineJointInfoVec = findOrCreateSplineJointInfo(context, base, target);
+
+    if (splineJointInfoVec && splineJointInfoVec->size() > 0) {
+        const int baseParentIndex = _skeleton->getParentIndex(base);
+        AnimPose parentAbsPose = (baseParentIndex >= 0) ? absolutePoses[baseParentIndex] : AnimPose();
+        // go thru splineJointInfoVec backwards (base to tip)
+        for (int i = (int)splineJointInfoVec->size() - 1; i >= 0; i--) {
+            const SplineJointInfo& splineJointInfo = (*splineJointInfoVec)[i];
+            float t = spline.arcLengthInverse(splineJointInfo.ratio * totalArcLength);
+            glm::vec3 trans = spline(t);
+
+            // for base->tip splines, preform most twist toward the tip by using ease in function. t^2
+            float rotT = t;
+            if (target.getIndex() == _tipJointIndex) {
+                rotT = t * t;
+            }
+            glm::quat twistRot = safeLerp(basePose.rot(), tipPose.rot(), rotT);
+
+            // compute the rotation by using the derivative of the spline as the y-axis, and the twistRot x-axis
+            glm::vec3 y = glm::normalize(spline.d(t));
+            glm::vec3 x = twistRot * Vectors::UNIT_X;
+            glm::vec3 u, v, w;
+            generateBasisVectors(y, x, v, u, w);
+            glm::mat3 m(u, v, glm::cross(u, v));
+            glm::quat rot = glm::normalize(glm::quat_cast(m));
+
+            AnimPose desiredAbsPose = AnimPose(glm::vec3(1.0f), rot, trans) * splineJointInfo.offsetPose;
+
+            // apply flex coefficent
+            AnimPose flexedAbsPose;
+            // get the number of flex coeff for this spline
+            float interpedCoefficient = 1.0f;
+            int numFlexCoeff = target.getNumFlexCoefficients();
+            if (numFlexCoeff == (int)splineJointInfoVec->size()) {
+                // then do nothing special
+                interpedCoefficient = target.getFlexCoefficient(i);
+            } else {
+                // interp based on ratio of the joint.
+                if (splineJointInfo.ratio < 1.0f) {
+                    float flexInterp = splineJointInfo.ratio * (float)(numFlexCoeff - 1);
+                    int startCoeff = (int)glm::floor(flexInterp);
+                    float partial = flexInterp - startCoeff;
+                    interpedCoefficient = target.getFlexCoefficient(startCoeff) * (1.0f - partial) + target.getFlexCoefficient(startCoeff + 1) * partial;
+                } else {
+                    interpedCoefficient = target.getFlexCoefficient(numFlexCoeff - 1);
+                }
+            }
+            ::blend(1, &absolutePoses[splineJointInfo.jointIndex], &desiredAbsPose, interpedCoefficient, &flexedAbsPose);
+
+            AnimPose relPose = parentAbsPose.inverse() * flexedAbsPose;
+
+            if (splineJointInfo.jointIndex != base) {
+                // constrain the amount the spine can stretch or compress
+                float length = glm::length(relPose.trans());
+                const float EPSILON = 0.0001f;
+                if (length > EPSILON) {
+                    float defaultLength = glm::length(_skeleton->getRelativeDefaultPose(splineJointInfo.jointIndex).trans());
+                    const float STRETCH_COMPRESS_PERCENTAGE = 0.15f;
+                    const float MAX_LENGTH = defaultLength * (1.0f + STRETCH_COMPRESS_PERCENTAGE);
+                    const float MIN_LENGTH = defaultLength * (1.0f - STRETCH_COMPRESS_PERCENTAGE);
+                    if (length > MAX_LENGTH) {
+                        relPose.trans() = (relPose.trans() / length) * MAX_LENGTH;
+                    } else if (length < MIN_LENGTH) {
+                        relPose.trans() = (relPose.trans() / length) * MIN_LENGTH;
+                    }
+                } else {
+                    relPose.trans() = glm::vec3(0.0f);
+                }
+            }
+
+            if (!chainInfoOut.setRelativePoseAtJointIndex(splineJointInfo.jointIndex, relPose)) {
+                qCDebug(animation) << "error: joint not found in spline chain";
+            }
+
+            parentAbsPose = flexedAbsPose;
+        }
+    }
+
+    if (debug) {
+        const vec4 CYAN(0.0f, 1.0f, 1.0f, 1.0f);
+        chainInfoOut.debugDraw(context.getRigToWorldMatrix() * context.getGeometryToRigMatrix(), CYAN);
+    }
+}
+
+const std::vector<AnimSplineIK::SplineJointInfo>* AnimSplineIK::findOrCreateSplineJointInfo(const AnimContext& context, int base, const IKTarget& target) const {
+    // find or create splineJointInfo for this target
+    auto iter = _splineJointInfoMap.find(target.getIndex());
+    if (iter != _splineJointInfoMap.end()) {
+        return &(iter->second);
+    } else {
+        computeAndCacheSplineJointInfosForIKTarget(context, base, target);
+        auto iter = _splineJointInfoMap.find(target.getIndex());
+        if (iter != _splineJointInfoMap.end()) {
+            return &(iter->second);
+        }
+    }
+    return nullptr;
+}
+
+// pre-compute information about each joint influenced by this spline IK target.
+void AnimSplineIK::computeAndCacheSplineJointInfosForIKTarget(const AnimContext& context, int base, const IKTarget& target) const {
+    std::vector<SplineJointInfo> splineJointInfoVec;
+
+    // build spline between the default poses.
+    AnimPose tipPose = _skeleton->getAbsoluteDefaultPose(target.getIndex());
+    AnimPose basePose = _skeleton->getAbsoluteDefaultPose(base);
+
+    CubicHermiteSplineFunctorWithArcLength spline;
+    if (target.getIndex() == _tipJointIndex) {
+        // set gain factors so that more curvature occurs near the tip of the spline.
+        const float HIPS_GAIN = 0.5f;
+        const float HEAD_GAIN = 1.0f;
+        spline = CubicHermiteSplineFunctorWithArcLength(tipPose.rot(), tipPose.trans(), basePose.rot(), basePose.trans(), HIPS_GAIN, HEAD_GAIN);
+    } else {
+        spline = CubicHermiteSplineFunctorWithArcLength(tipPose.rot(), tipPose.trans(), basePose.rot(), basePose.trans());
+    }
+    // measure the total arc length along the spline
+    float totalArcLength = spline.arcLength(1.0f);
+
+    glm::vec3 baseToTip = tipPose.trans() - basePose.trans();
+    float baseToTipLength = glm::length(baseToTip);
+    glm::vec3 baseToTipNormal = baseToTip / baseToTipLength;
+
+    int index = target.getIndex();
+    int endIndex = _skeleton->getParentIndex(base);
+
+    while (index != endIndex) {
+        AnimPose defaultPose = _skeleton->getAbsoluteDefaultPose(index);
+        glm::vec3 baseToCurrentJoint = defaultPose.trans() - basePose.trans();
+        float ratio = glm::dot(baseToCurrentJoint, baseToTipNormal) / baseToTipLength;
+
+        // compute offset from spline to the default pose.
+        float t = spline.arcLengthInverse(ratio * totalArcLength);
+
+        // compute the rotation by using the derivative of the spline as the y-axis, and the defaultPose x-axis
+        glm::vec3 y = glm::normalize(spline.d(t));
+        glm::vec3 x = defaultPose.rot() * Vectors::UNIT_X;
+        glm::vec3 u, v, w;
+        generateBasisVectors(y, x, v, u, w);
+        glm::mat3 m(u, v, glm::cross(u, v));
+        glm::quat rot = glm::normalize(glm::quat_cast(m));
+
+        AnimPose pose(glm::vec3(1.0f), rot, spline(t));
+        AnimPose offsetPose = pose.inverse() * defaultPose;
+
+        SplineJointInfo splineJointInfo = { index, ratio, offsetPose };
+        splineJointInfoVec.push_back(splineJointInfo);
+        index = _skeleton->getParentIndex(index);
+    }
+    _splineJointInfoMap[target.getIndex()] = splineJointInfoVec;
+}
+
+void AnimSplineIK::beginInterp(InterpType interpType, const AnimChain& chain) {
+    // capture the current poses in a snapshot.
+    _snapshotChain = chain;
+
+    _interpType = interpType;
+    _interpAlphaVel = FRAMES_PER_SECOND / _interpDuration;
+    _interpAlpha = 0.0f;
+}

libraries/animation/src/AnimSplineIK.h

@@ -0,0 +1,104 @@
+//
+//  AnimSplineIK.h
+//
+//  Created by Angus Antley on 1/7/19.
+//  Copyright (c) 2019 High Fidelity, Inc. All rights reserved.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AnimSplineIK_h
+#define hifi_AnimSplineIK_h
+
+#include "AnimNode.h"
+#include "IKTarget.h"
+#include "AnimChain.h"
+
+static const int MAX_NUMBER_FLEX_VARIABLES = 10;
+
+// Spline IK for the spine
+class AnimSplineIK : public AnimNode {
+public:
+    AnimSplineIK(const QString& id, float alpha, bool enabled, float interpDuration,
+        const QString& baseJointName, const QString& midJointName, const QString& tipJointName,
+        const QString& basePositionVar, const QString& baseRotationVar,
+        const QString& midPositionVar, const QString& midRotationVar,
+        const QString& tipPositionVar, const QString& tipRotationVar,
+        const QString& alphaVar, const QString& enabledVar,
+        const std::vector<float> tipTargetFlexCoefficients,
+        const std::vector<float> midTargetFlexCoefficients);
+
+	virtual ~AnimSplineIK() override;
+    virtual const AnimPoseVec& evaluate(const AnimVariantMap& animVars, const AnimContext& context, float dt, AnimVariantMap& triggersOut) override;
+
+protected:
+
+    enum class InterpType {
+        None = 0,
+        SnapshotToUnderPoses,
+        SnapshotToSolve,
+        NumTypes
+    };
+
+    void computeAbsolutePoses(AnimPoseVec& absolutePoses) const;
+    void loadPoses(const AnimPoseVec& poses);
+
+    // for AnimDebugDraw rendering
+    virtual const AnimPoseVec& getPosesInternal() const override;
+    virtual void setSkeletonInternal(AnimSkeleton::ConstPointer skeleton) override;
+
+    void lookUpIndices();
+    void beginInterp(InterpType interpType, const AnimChain& chain);
+
+    AnimPoseVec _poses;
+
+    float _alpha;
+    bool _enabled;
+    float _interpDuration;
+    QString _baseJointName;
+    QString _midJointName;
+    QString _tipJointName;
+    QString _basePositionVar;
+    QString _baseRotationVar;
+    QString _midPositionVar;
+    QString _midRotationVar;
+    QString _tipPositionVar;
+    QString _tipRotationVar;
+    QString _alphaVar;  // float - (0, 1) 0 means underPoses only, 1 means IK only.
+    QString _enabledVar;
+
+    float _tipTargetFlexCoefficients[MAX_NUMBER_FLEX_VARIABLES];
+    float _midTargetFlexCoefficients[MAX_NUMBER_FLEX_VARIABLES];
+    int _numTipTargetFlexCoefficients { 0 };
+    int _numMidTargetFlexCoefficients { 0 };
+
+    int _baseJointIndex { -1 };
+    int _midJointIndex { -1 };
+    int _tipJointIndex { -1 };
+
+    bool _previousEnableDebugIKTargets { false };
+
+    InterpType _interpType{ InterpType::None };
+    float _interpAlphaVel{ 0.0f };
+    float _interpAlpha{ 0.0f };
+    AnimChain _snapshotChain;
+
+    // used to pre-compute information about each joint influenced by a spline IK target.
+    struct SplineJointInfo {
+        int jointIndex;       // joint in the skeleton that this information pertains to.
+        float ratio;          // percentage (0..1) along the spline for this joint.
+        AnimPose offsetPose;  // local offset from the spline to the joint.
+    };
+
+    void solveTargetWithSpline(const AnimContext& context, int base, const IKTarget& target, const AnimPoseVec& absolutePoses, bool debug, AnimChain& chainInfoOut) const;
+    void computeAndCacheSplineJointInfosForIKTarget(const AnimContext& context, int base, const IKTarget& target) const;
+    const std::vector<SplineJointInfo>* findOrCreateSplineJointInfo(const AnimContext& context, int base, const IKTarget& target) const;
+    mutable std::map<int, std::vector<SplineJointInfo>> _splineJointInfoMap;
+
+    // no copies
+    AnimSplineIK(const AnimSplineIK&) = delete;
+    AnimSplineIK&  operator=(const AnimSplineIK&) = delete;
+
+};
+#endif // hifi_AnimSplineIK_h

libraries/animation/src/AnimStateMachine.cpp

@@ -22,7 +22,6 @@ AnimStateMachine::~AnimStateMachine() {
 }
 
 const AnimPoseVec& AnimStateMachine::evaluate(const AnimVariantMap& animVars, const AnimContext& context, float dt, AnimVariantMap& triggersOut) {
-
     float parentDebugAlpha = context.getDebugAlpha(_id);
 
     QString desiredStateID = animVars.lookup(_currentStateVar, _currentState->getID());

libraries/animation/src/Flow.cpp

@@ -0,0 +1,783 @@
+//
+//  Flow.cpp
+//
+//  Created by Luis Cuenca on 1/21/2019.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "Flow.h"
+#include "Rig.h"
+#include "AnimSkeleton.h"
+
+const std::map<QString, FlowPhysicsSettings> PRESET_FLOW_DATA = { { "hair", FlowPhysicsSettings() },
+{ "skirt", FlowPhysicsSettings(true, 1.0f, DEFAULT_GRAVITY, 0.65f, 0.8f, 0.45f, 0.01f) },
+{ "breast", FlowPhysicsSettings(true, 1.0f, DEFAULT_GRAVITY, 0.65f, 0.8f, 0.45f, 0.01f) } };
+
+const std::map<QString, FlowCollisionSettings> PRESET_COLLISION_DATA = {
+    { "Spine2", FlowCollisionSettings(QUuid(), FlowCollisionType::CollisionSphere, glm::vec3(0.0f, 0.2f, 0.0f), 0.14f) },
+    { "LeftArm", FlowCollisionSettings(QUuid(), FlowCollisionType::CollisionSphere, glm::vec3(0.0f, 0.02f, 0.0f), 0.05f) },
+    { "RightArm", FlowCollisionSettings(QUuid(), FlowCollisionType::CollisionSphere, glm::vec3(0.0f, 0.02f, 0.0f), 0.05f) },
+    { "HeadTop_End", FlowCollisionSettings(QUuid(), FlowCollisionType::CollisionSphere, glm::vec3(0.0f, -0.15f, 0.0f), 0.09f) }
+};
+
+FlowCollisionSphere::FlowCollisionSphere(const int& jointIndex, const FlowCollisionSettings& settings, bool isTouch) {
+    _jointIndex = jointIndex;
+    _radius = _initialRadius = settings._radius;
+    _offset = _initialOffset = settings._offset;
+    _entityID = settings._entityID;
+    _isTouch = isTouch;
+}
+
+FlowCollisionResult FlowCollisionSphere::computeSphereCollision(const glm::vec3& point, float radius) const {
+    FlowCollisionResult result;
+    auto centerToJoint = point - _position;
+    result._distance = glm::length(centerToJoint) - radius;
+    result._offset = _radius - result._distance;
+    result._normal = glm::normalize(centerToJoint);
+    result._radius = _radius;
+    result._position = _position;
+    return result;
+}
+
+FlowCollisionResult FlowCollisionSphere::checkSegmentCollision(const glm::vec3& point1, const glm::vec3& point2, const FlowCollisionResult& collisionResult1, const FlowCollisionResult& collisionResult2) {
+    FlowCollisionResult result;
+    auto segment = point2 - point1;
+    auto segmentLength = glm::length(segment);
+    auto maxDistance = glm::sqrt(powf(collisionResult1._radius, 2.0f) + powf(segmentLength, 2.0f));
+    if (collisionResult1._distance < maxDistance && collisionResult2._distance < maxDistance) {
+        float segmentPercentage = collisionResult1._distance / (collisionResult1._distance + collisionResult2._distance);
+        glm::vec3 collisionPoint = point1 + segment * segmentPercentage;
+        glm::vec3 centerToSegment = collisionPoint - _position;
+        float distance = glm::length(centerToSegment);
+        if (distance < _radius) {
+            result._offset = _radius - distance;
+            result._position = _position;
+            result._radius = _radius;
+            result._normal = glm::normalize(centerToSegment);
+            result._distance = distance;
+        }
+    }
+    return result;
+}
+
+void FlowCollisionSystem::addCollisionSphere(int jointIndex, const FlowCollisionSettings& settings, const glm::vec3& position, bool isSelfCollision, bool isTouch) {
+    auto collision = FlowCollisionSphere(jointIndex, settings, isTouch);
+    collision.setPosition(position);
+    if (isSelfCollision) {
+        _selfCollisions.push_back(collision);
+    } else {
+        _othersCollisions.push_back(collision);
+    }
+
+};
+void FlowCollisionSystem::resetCollisions() {
+    _allCollisions.clear();
+    _othersCollisions.clear();
+    _selfCollisions.clear();
+}
+FlowCollisionResult FlowCollisionSystem::computeCollision(const std::vector<FlowCollisionResult> collisions) {
+    FlowCollisionResult result;
+    if (collisions.size() > 1) {
+        for (size_t i = 0; i < collisions.size(); i++) {
+            result._offset += collisions[i]._offset;
+            result._normal = result._normal + collisions[i]._normal * collisions[i]._distance;
+            result._position = result._position + collisions[i]._position;
+            result._radius += collisions[i]._radius;
+            result._distance += collisions[i]._distance;
+        }
+        result._offset = result._offset / collisions.size();
+        result._radius = 0.5f * glm::length(result._normal);
+        result._normal = glm::normalize(result._normal);
+        result._position = result._position / (float)collisions.size();
+        result._distance = result._distance / collisions.size();
+    } else if (collisions.size() == 1) {
+        result = collisions[0];
+    }
+    result._count = (int)collisions.size();
+    return result;
+};
+
+void FlowCollisionSystem::setScale(float scale) {
+    _scale = scale;
+    for (size_t j = 0; j < _selfCollisions.size(); j++) {
+        _selfCollisions[j]._radius = _selfCollisions[j]._initialRadius * scale;
+        _selfCollisions[j]._offset = _selfCollisions[j]._initialOffset * scale;
+    }
+};
+
+std::vector<FlowCollisionResult> FlowCollisionSystem::checkFlowThreadCollisions(FlowThread* flowThread) {
+    std::vector<std::vector<FlowCollisionResult>> FlowThreadResults;
+    FlowThreadResults.resize(flowThread->_joints.size());
+    for (size_t j = 0; j < _allCollisions.size(); j++) {
+        FlowCollisionSphere &sphere = _allCollisions[j];
+        FlowCollisionResult rootCollision = sphere.computeSphereCollision(flowThread->_positions[0], flowThread->_radius);
+        std::vector<FlowCollisionResult> collisionData = { rootCollision };
+        bool tooFar = rootCollision._distance >(flowThread->_length + rootCollision._radius);
+        FlowCollisionResult nextCollision;
+        if (!tooFar) {
+            if (sphere._isTouch) {
+                for (size_t i = 1; i < flowThread->_joints.size(); i++) {
+                    auto prevCollision = collisionData[i - 1];
+                    nextCollision = _allCollisions[j].computeSphereCollision(flowThread->_positions[i], flowThread->_radius);
+                    collisionData.push_back(nextCollision);
+                    if (prevCollision._offset > 0.0f) {
+                        if (i == 1) {
+                            FlowThreadResults[i - 1].push_back(prevCollision);
+                        }
+                    } else if (nextCollision._offset > 0.0f) {
+                        FlowThreadResults[i].push_back(nextCollision);
+                    } else {
+                        FlowCollisionResult segmentCollision = _allCollisions[j].checkSegmentCollision(flowThread->_positions[i - 1], flowThread->_positions[i], prevCollision, nextCollision);
+                        if (segmentCollision._offset > 0) {
+                            FlowThreadResults[i - 1].push_back(segmentCollision);
+                            FlowThreadResults[i].push_back(segmentCollision);
+                        }
+                    }
+                }
+            } else {
+                if (rootCollision._offset > 0.0f) {
+                    FlowThreadResults[0].push_back(rootCollision);
+                }
+                for (size_t i = 1; i < flowThread->_joints.size(); i++) {
+                    nextCollision = _allCollisions[j].computeSphereCollision(flowThread->_positions[i], flowThread->_radius);
+                    if (nextCollision._offset > 0.0f) {
+                        FlowThreadResults[i].push_back(nextCollision);
+                    }
+                }
+            }
+        }
+    }
+
+    std::vector<FlowCollisionResult> results;
+    for (size_t i = 0; i < flowThread->_joints.size(); i++) {
+        results.push_back(computeCollision(FlowThreadResults[i]));
+    }
+    return results;
+};
+
+FlowCollisionSettings FlowCollisionSystem::getCollisionSettingsByJoint(int jointIndex) {
+    for (auto &collision : _selfCollisions) {
+        if (collision._jointIndex == jointIndex) {
+            return FlowCollisionSettings(collision._entityID, FlowCollisionType::CollisionSphere, collision._initialOffset, collision._initialRadius);
+        }
+    }
+    return FlowCollisionSettings();
+}
+void FlowCollisionSystem::setCollisionSettingsByJoint(int jointIndex, const FlowCollisionSettings& settings) {
+    for (auto &collision : _selfCollisions) {
+        if (collision._jointIndex == jointIndex) {
+            collision._initialRadius = settings._radius;
+            collision._initialOffset = settings._offset;
+            collision._radius = _scale * settings._radius;
+            collision._offset = _scale * settings._offset;
+        }
+    }
+}
+void FlowCollisionSystem::prepareCollisions() {
+    _allCollisions.clear();
+    _allCollisions.resize(_selfCollisions.size() + _othersCollisions.size());
+    std::copy(_selfCollisions.begin(), _selfCollisions.begin() + _selfCollisions.size(), _allCollisions.begin());
+    std::copy(_othersCollisions.begin(), _othersCollisions.begin() + _othersCollisions.size(), _allCollisions.begin() + _selfCollisions.size());
+    _othersCollisions.clear();
+}
+
+FlowNode::FlowNode(const glm::vec3& initialPosition, FlowPhysicsSettings settings) {
+    _initialPosition = _previousPosition = _currentPosition = initialPosition;
+    _initialRadius = settings._radius;
+}
+
+void FlowNode::update(float deltaTime, const glm::vec3& accelerationOffset) {
+    _acceleration = glm::vec3(0.0f, _settings._gravity, 0.0f);
+    _previousVelocity = _currentVelocity;
+    _currentVelocity = _currentPosition - _previousPosition;
+    _previousPosition = _currentPosition;
+    if (!_anchored) {
+        // Add inertia
+        const float FPS = 60.0f;
+        float timeRatio = _scale * (FPS * deltaTime);
+        float invertedTimeRatio = timeRatio > 0.0f ? 1.0f / timeRatio : 1.0f;
+        auto deltaVelocity = _previousVelocity - _currentVelocity;
+        auto centrifugeVector = glm::length(deltaVelocity) != 0.0f ? glm::normalize(deltaVelocity) : glm::vec3();
+        _acceleration = _acceleration + centrifugeVector * _settings._inertia * glm::length(_currentVelocity) * invertedTimeRatio;
+
+        // Add offset
+        _acceleration += accelerationOffset;
+        float accelerationFactor = powf(_settings._delta, 2.0f) * timeRatio;
+        glm::vec3 deltaAcceleration = _acceleration * accelerationFactor;
+        // Calculate new position        
+        _currentPosition = _currentPosition + (_currentVelocity * _settings._damping) + deltaAcceleration;
+    } else {
+        _acceleration = glm::vec3(0.0f);
+        _currentVelocity = glm::vec3(0.0f);
+    }
+};
+
+
+void FlowNode::solve(const glm::vec3& constrainPoint, float maxDistance, const FlowCollisionResult& collision) {
+    solveConstraints(constrainPoint, maxDistance);
+    solveCollisions(collision);
+};
+
+void FlowNode::solveConstraints(const glm::vec3& constrainPoint, float maxDistance) {
+    glm::vec3 constrainVector = _currentPosition - constrainPoint;
+    float difference = maxDistance / glm::length(constrainVector);
+    _currentPosition = difference < 1.0f ? constrainPoint + constrainVector * difference : _currentPosition;
+};
+
+void FlowNode::solveCollisions(const FlowCollisionResult& collision) {
+    _colliding = collision._offset > 0.0f;
+    _collision = collision;
+    if (_colliding) {
+        _currentPosition = _currentPosition + collision._normal * collision._offset;
+        _previousCollision = collision;
+    } else {
+        _previousCollision = FlowCollisionResult();
+    }
+};
+
+FlowJoint::FlowJoint(int jointIndex, int parentIndex, int childIndex, const QString& name, const QString& group, const FlowPhysicsSettings& settings) {
+    _index = jointIndex;
+    _name = name;
+    _group = group;
+    _childIndex = childIndex;
+    _parentIndex = parentIndex;
+    FlowNode(glm::vec3(), settings);
+};
+
+void FlowJoint::setInitialData(const glm::vec3& initialPosition, const glm::vec3& initialTranslation, const glm::quat& initialRotation, const glm::vec3& parentPosition) {
+    _initialPosition = initialPosition;
+    _previousPosition = initialPosition;
+    _currentPosition = initialPosition;
+    _initialTranslation = initialTranslation;
+    _currentRotation = initialRotation;
+    _initialRotation = initialRotation;
+    _translationDirection = glm::normalize(_initialTranslation);
+    _parentPosition = parentPosition;
+    _initialLength = _length = glm::length(_initialPosition - parentPosition);
+}
+
+void FlowJoint::setUpdatedData(const glm::vec3& updatedPosition, const glm::vec3& updatedTranslation, const glm::quat& updatedRotation, const glm::vec3& parentPosition, const glm::quat& parentWorldRotation) {
+    _updatedPosition = updatedPosition;
+    _updatedRotation = updatedRotation;
+    _updatedTranslation = updatedTranslation;
+    _parentPosition = parentPosition;
+    _parentWorldRotation = parentWorldRotation;
+}
+
+void FlowJoint::setRecoveryPosition(const glm::vec3& recoveryPosition) {
+    _recoveryPosition = recoveryPosition;
+    _applyRecovery = true;
+}
+
+void FlowJoint::update(float deltaTime) {
+    glm::vec3 accelerationOffset = glm::vec3(0.0f);
+    if (_settings._stiffness > 0.0f) {
+        glm::vec3 recoveryVector = _recoveryPosition - _currentPosition;
+        float recoveryFactor = powf(_settings._stiffness, 3.0f);
+        accelerationOffset = recoveryVector * recoveryFactor;
+    }
+    FlowNode::update(deltaTime, accelerationOffset);
+    if (_anchored) {
+        if (!_isHelper) {
+            _currentPosition = _updatedPosition;
+        } else {
+            _currentPosition = _parentPosition;
+        }
+    }
+};
+
+void FlowJoint::setScale(float scale, bool initScale) {
+    if (initScale) {
+        _initialLength = _length / scale;
+    }
+    _settings._radius = _initialRadius * scale;
+    _length = _initialLength * scale;
+    _scale = scale;
+}
+
+void FlowJoint::solve(const FlowCollisionResult& collision) {
+    FlowNode::solve(_parentPosition, _length, collision);
+};
+
+void FlowJoint::toHelperJoint(const glm::vec3& initialPosition, float length) {
+    _initialPosition = initialPosition;
+    _isHelper = true;
+    _length = length;
+}
+
+FlowThread::FlowThread(int rootIndex, std::map<int, FlowJoint>* joints) {
+    _jointsPointer = joints;
+    computeFlowThread(rootIndex);
+}
+
+void FlowThread::resetLength() {
+    _length = 0.0f;
+    for (size_t i = 1; i < _joints.size(); i++) {
+        int index = _joints[i];
+        _length += _jointsPointer->at(index)._length;
+    }
+}
+
+void FlowThread::computeFlowThread(int rootIndex) {
+    int parentIndex = rootIndex;
+    if (_jointsPointer->size() == 0) {
+        return;
+    }
+    int childIndex = _jointsPointer->at(parentIndex)._childIndex;
+    std::vector<int> indexes = { parentIndex };
+    for (size_t i = 0; i < _jointsPointer->size(); i++) {
+        if (childIndex > -1) {
+            indexes.push_back(childIndex);
+            childIndex = _jointsPointer->at(childIndex)._childIndex;
+        } else {
+            break;
+        }
+    }
+    _length = 0.0f;
+    for (size_t i = 0; i < indexes.size(); i++) {
+        int index = indexes[i];
+        _joints.push_back(index);
+        if (i > 0) {
+            _length += _jointsPointer->at(index)._length;
+        }
+    }
+};
+
+void FlowThread::computeRecovery() {
+    int parentIndex = _joints[0];
+    auto parentJoint = _jointsPointer->at(parentIndex);
+    _jointsPointer->at(parentIndex)._recoveryPosition = parentJoint._recoveryPosition = parentJoint._currentPosition;
+    glm::quat parentRotation = parentJoint._parentWorldRotation * parentJoint._initialRotation;
+    for (size_t i = 1; i < _joints.size(); i++) {
+        auto joint = _jointsPointer->at(_joints[i]);
+        _jointsPointer->at(_joints[i])._recoveryPosition = joint._recoveryPosition = parentJoint._recoveryPosition + (parentRotation * (joint._initialTranslation * 0.01f));
+        parentJoint = joint;
+    }
+};
+
+void FlowThread::update(float deltaTime) {
+    _positions.clear();
+    auto &firstJoint = _jointsPointer->at(_joints[0]);
+    _radius = firstJoint._settings._radius;
+    computeRecovery();
+    for (size_t i = 0; i < _joints.size(); i++) {
+        auto &joint = _jointsPointer->at(_joints[i]);
+        joint.update(deltaTime);
+        _positions.push_back(joint._currentPosition);
+    }
+};
+
+void FlowThread::solve(FlowCollisionSystem& collisionSystem) {
+    if (collisionSystem.getActive()) {
+        auto bodyCollisions = collisionSystem.checkFlowThreadCollisions(this);
+        for (size_t i = 0; i < _joints.size(); i++) {
+            int index = _joints[i];
+            _jointsPointer->at(index).solve(bodyCollisions[i]);
+        }
+    } else {
+        for (size_t i = 0; i < _joints.size(); i++) {
+            int index = _joints[i];
+            _jointsPointer->at(index).solve(FlowCollisionResult());
+        }
+    }
+};
+
+void FlowThread::computeJointRotations() {
+
+    auto pos0 = _rootFramePositions[0];
+    auto pos1 = _rootFramePositions[1];
+
+    auto joint0 = _jointsPointer->at(_joints[0]);
+    auto joint1 = _jointsPointer->at(_joints[1]);
+
+    auto initial_pos1 = pos0 + (joint0._initialRotation * (joint1._initialTranslation * 0.01f));
+
+    auto vec0 = initial_pos1 - pos0;
+    auto vec1 = pos1 - pos0;
+
+    auto delta = rotationBetween(vec0, vec1);
+
+    joint0._currentRotation = _jointsPointer->at(_joints[0])._currentRotation = delta * joint0._initialRotation;
+    
+    for (size_t i = 1; i < _joints.size() - 1; i++) {
+        auto nextJoint = _jointsPointer->at(_joints[i + 1]);
+        for (size_t j = i; j < _joints.size(); j++) {
+            _rootFramePositions[j] = glm::inverse(joint0._currentRotation) * _rootFramePositions[j] - (joint0._initialTranslation * 0.01f);
+        }
+        pos0 = _rootFramePositions[i];
+        pos1 = _rootFramePositions[i + 1];
+        initial_pos1 = pos0 + joint1._initialRotation * (nextJoint._initialTranslation * 0.01f);
+
+        vec0 = initial_pos1 - pos0;
+        vec1 = pos1 - pos0;
+
+        delta = rotationBetween(vec0, vec1);
+
+        joint1._currentRotation = _jointsPointer->at(joint1._index)._currentRotation = delta * joint1._initialRotation;
+        joint0 = joint1;
+        joint1 = nextJoint;
+    }
+    
+}
+
+void FlowThread::setScale(float scale, bool initScale) {
+    for (size_t i = 0; i < _joints.size(); i++) {
+        auto &joint = _jointsPointer->at(_joints[i]);
+        joint.setScale(scale, initScale);
+    }
+    resetLength();
+}
+
+FlowThread& FlowThread::operator=(const FlowThread& otherFlowThread) {
+    for (int jointIndex: otherFlowThread._joints) {
+        auto& joint = otherFlowThread._jointsPointer->at(jointIndex);
+        auto& myJoint = _jointsPointer->at(jointIndex);
+        myJoint._acceleration = joint._acceleration;
+        myJoint._currentPosition = joint._currentPosition;
+        myJoint._currentRotation = joint._currentRotation;
+        myJoint._currentVelocity = joint._currentVelocity;
+        myJoint._length = joint._length;
+        myJoint._parentPosition = joint._parentPosition;
+        myJoint._parentWorldRotation = joint._parentWorldRotation;
+        myJoint._previousPosition = joint._previousPosition;
+        myJoint._previousVelocity = joint._previousVelocity;
+        myJoint._scale = joint._scale;
+        myJoint._translationDirection = joint._translationDirection;
+        myJoint._updatedPosition = joint._updatedPosition;
+        myJoint._updatedRotation = joint._updatedRotation;
+        myJoint._updatedTranslation = joint._updatedTranslation;
+        myJoint._isHelper = joint._isHelper;
+    }
+    return *this;
+}
+
+void Flow::calculateConstraints(const std::shared_ptr<AnimSkeleton>& skeleton, 
+                                AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses) {
+    cleanUp();
+    if (!skeleton) {
+        return;
+    }
+    auto flowPrefix = FLOW_JOINT_PREFIX.toUpper();
+    auto simPrefix = SIM_JOINT_PREFIX.toUpper();
+    std::vector<int> handsIndices;
+
+    for (int i = 0; i < skeleton->getNumJoints(); i++) {
+        auto name = skeleton->getJointName(i);
+        if (std::find(HAND_COLLISION_JOINTS.begin(), HAND_COLLISION_JOINTS.end(), name) != HAND_COLLISION_JOINTS.end()) {
+            handsIndices.push_back(i);
+        }
+        auto parentIndex = skeleton->getParentIndex(i);
+        if (parentIndex == -1) {
+            continue;
+        }
+        auto jointChildren = skeleton->getChildrenOfJoint(i);
+        // auto childIndex = jointChildren.size() > 0 ? jointChildren[0] : -1;
+        auto group = QStringRef(&name, 0, 3).toString().toUpper();
+        auto split = name.split("_");
+        bool isSimJoint = (group == simPrefix);
+        bool isFlowJoint = split.size() > 2 && split[0].toUpper() == flowPrefix;
+        if (isFlowJoint || isSimJoint) {
+            group = "";
+            if (isSimJoint) {
+                for (int j = 1; j < name.size() - 1; j++) {
+                    bool toFloatSuccess;
+                    QStringRef(&name, (int)(name.size() - j), 1).toString().toFloat(&toFloatSuccess);
+                    if (!toFloatSuccess && (name.size() - j) > (int)simPrefix.size()) {
+                        group = QStringRef(&name, (int)simPrefix.size(), (int)(name.size() - j + 1)).toString();
+                        break;
+                    }
+                }
+                if (group.isEmpty()) {
+                    group = QStringRef(&name, (int)simPrefix.size(), name.size() - 1).toString();
+                }
+                qCDebug(animation) << "Sim joint added to flow: " << name;
+            } else {
+                group = split[1];
+            }
+            if (!group.isEmpty()) {
+                _flowJointKeywords.push_back(group);
+                FlowPhysicsSettings jointSettings;
+                if (PRESET_FLOW_DATA.find(group) != PRESET_FLOW_DATA.end()) {
+                    jointSettings = PRESET_FLOW_DATA.at(group);
+                } else {
+                    jointSettings = DEFAULT_JOINT_SETTINGS;
+                }
+                if (_flowJointData.find(i) ==  _flowJointData.end()) {
+                    auto flowJoint = FlowJoint(i, parentIndex, -1, name, group, jointSettings);
+                    _flowJointData.insert(std::pair<int, FlowJoint>(i, flowJoint));
+                }
+            }
+        } else {
+            if (PRESET_COLLISION_DATA.find(name) != PRESET_COLLISION_DATA.end()) {
+                _collisionSystem.addCollisionSphere(i, PRESET_COLLISION_DATA.at(name));
+            }
+        }
+    }
+
+    for (auto &jointData : _flowJointData) {
+        int jointIndex = jointData.first;
+        glm::vec3 jointPosition, parentPosition, jointTranslation;
+        glm::quat jointRotation;
+        getJointPositionInWorldFrame(absolutePoses, jointIndex, jointPosition, _entityPosition, _entityRotation);
+        getJointTranslation(relativePoses, jointIndex, jointTranslation);
+        getJointRotation(relativePoses, jointIndex, jointRotation);
+        getJointPositionInWorldFrame(absolutePoses, jointData.second.getParentIndex(), parentPosition, _entityPosition, _entityRotation);
+
+        jointData.second.setInitialData(jointPosition, jointTranslation, jointRotation, parentPosition);
+    }
+
+    std::vector<int> roots;
+
+    for (auto &joint :_flowJointData) {
+        if (_flowJointData.find(joint.second.getParentIndex()) == _flowJointData.end()) {
+            joint.second.setAnchored(true);
+            roots.push_back(joint.first);
+        } else {
+            _flowJointData[joint.second.getParentIndex()].setChildIndex(joint.first);
+        }
+    }
+    int extraIndex = -1;
+    for (size_t i = 0; i < roots.size(); i++) {
+        FlowThread thread = FlowThread(roots[i], &_flowJointData);
+        // add threads with at least 2 joints
+        if (thread._joints.size() > 0) {
+            if (thread._joints.size() == 1) {
+                int jointIndex = roots[i];
+                auto &joint = _flowJointData[jointIndex];
+                auto &jointPosition = joint.getUpdatedPosition();
+                auto newSettings = joint.getSettings();
+                extraIndex = extraIndex > -1 ? extraIndex + 1 : skeleton->getNumJoints();
+                joint.setChildIndex(extraIndex);
+                auto newJoint = FlowJoint(extraIndex, jointIndex, -1, joint.getName(), joint.getGroup(), newSettings);
+                newJoint.toHelperJoint(jointPosition, HELPER_JOINT_LENGTH);
+                glm::vec3 translation = glm::vec3(0.0f, HELPER_JOINT_LENGTH, 0.0f);
+                newJoint.setInitialData(jointPosition + translation, 100.0f * translation , Quaternions::IDENTITY, jointPosition);
+                _flowJointData.insert(std::pair<int, FlowJoint>(extraIndex, newJoint));
+                FlowThread newThread = FlowThread(jointIndex, &_flowJointData);
+                if (newThread._joints.size() > 1) {
+                    _jointThreads.push_back(newThread);
+                }
+            } else {
+                _jointThreads.push_back(thread);
+            }
+        }
+    }
+    
+    if (_jointThreads.size() == 0) {
+        onCleanup();
+    }
+    if (handsIndices.size() > 0) {
+        FlowCollisionSettings handSettings;
+        handSettings._radius = HAND_COLLISION_RADIUS;
+        for (size_t i = 0; i < handsIndices.size(); i++) {
+            _collisionSystem.addCollisionSphere(handsIndices[i], handSettings, glm::vec3(), true, true);
+        }
+    }
+    _initialized = _jointThreads.size() > 0;
+}
+
+void Flow::cleanUp() {
+    _flowJointData.clear();
+    _jointThreads.clear();
+    _flowJointKeywords.clear();
+    _collisionSystem.resetCollisions();
+    _initialized = false;
+    _isScaleSet = false;
+    onCleanup();
+ }
+
+void Flow::setTransform(float scale, const glm::vec3& position, const glm::quat& rotation) {
+    _scale = scale;
+    _entityPosition = position;
+    _entityRotation = rotation;
+}
+
+void Flow::setScale(float scale) {
+    _collisionSystem.setScale(_scale);
+    for (size_t i = 0; i < _jointThreads.size(); i++) {
+        _jointThreads[i].setScale(_scale, !_isScaleSet);
+    }
+    if (_lastScale != _scale) {
+        _lastScale = _scale;
+        _isScaleSet = true;
+    }
+    
+}
+
+void Flow::update(float deltaTime, AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses, const std::vector<bool>& overrideFlags) {
+    if (_initialized && _active) {
+        uint64_t startTime = usecTimestampNow();
+        uint64_t updateExpiry = startTime + MAX_UPDATE_FLOW_TIME_BUDGET;
+        if (_scale != _lastScale) {
+            setScale(_scale);
+        }
+        for (size_t i = 0; i < _jointThreads.size(); i++) {
+            size_t index = _invertThreadLoop ? _jointThreads.size() - 1 - i : i;
+            auto &thread = _jointThreads[index];
+            thread.update(deltaTime);
+            thread.solve(_collisionSystem);
+            if (!updateRootFramePositions(absolutePoses, index)) {
+                return;
+            }
+            thread.computeJointRotations();
+            if (usecTimestampNow() > updateExpiry) {
+                break;
+                qWarning(animation) << "Flow Bones ran out of time while updating threads";
+            }
+        }
+        setJoints(relativePoses, overrideFlags);
+        updateJoints(relativePoses, absolutePoses);
+        _invertThreadLoop = !_invertThreadLoop;
+    }    
+}
+
+void Flow::updateAbsolutePoses(const AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses) {
+    for (auto &joint : _flowJointData) {
+        int index = joint.second.getIndex();
+        int parentIndex = joint.second.getParentIndex();
+        if (index >= 0 && index < (int)relativePoses.size() &&
+            parentIndex >= 0 && parentIndex < (int)absolutePoses.size()) {
+            absolutePoses[index] = absolutePoses[parentIndex] * relativePoses[index];
+        }
+    }
+}
+
+bool Flow::worldToJointPoint(const AnimPoseVec& absolutePoses, const glm::vec3& position, const int jointIndex, glm::vec3& jointSpacePosition) const {
+    glm::vec3 jointPos;
+    glm::quat jointRot;
+    if (getJointPositionInWorldFrame(absolutePoses, jointIndex, jointPos, _entityPosition, _entityRotation) && 
+        getJointRotationInWorldFrame(absolutePoses, jointIndex, jointRot, _entityRotation)) {
+        glm::vec3 modelOffset = position - jointPos;
+        jointSpacePosition = glm::inverse(jointRot) * modelOffset;
+        return true;
+    }
+    return false;
+}
+
+bool Flow::updateRootFramePositions(const AnimPoseVec& absolutePoses, size_t threadIndex) {
+    auto &joints = _jointThreads[threadIndex]._joints;
+    int rootIndex = _flowJointData[joints[0]].getParentIndex();
+    _jointThreads[threadIndex]._rootFramePositions.clear();
+    for (size_t j = 0; j < joints.size(); j++) {
+        glm::vec3 jointPos;
+        if (worldToJointPoint(absolutePoses, _flowJointData[joints[j]].getCurrentPosition(), rootIndex, jointPos)) {
+            _jointThreads[threadIndex]._rootFramePositions.push_back(jointPos);
+        } else {
+            return false;
+        }
+    }
+    return true;
+}
+
+void Flow::updateJoints(AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses) {
+    updateAbsolutePoses(relativePoses, absolutePoses);
+    for (auto &jointData : _flowJointData) {
+        int jointIndex = jointData.first;
+        glm::vec3 jointPosition, parentPosition, jointTranslation;
+        glm::quat jointRotation, parentWorldRotation;
+        if (!jointData.second.isHelper()) {
+            getJointPositionInWorldFrame(absolutePoses, jointIndex, jointPosition, _entityPosition, _entityRotation);
+            getJointTranslation(relativePoses, jointIndex, jointTranslation);
+            getJointRotation(relativePoses, jointIndex, jointRotation);
+        } else {
+            jointPosition = jointData.second.getCurrentPosition();
+            jointTranslation = jointData.second.getCurrentTranslation();
+            jointRotation = jointData.second.getCurrentRotation();
+        }
+        getJointPositionInWorldFrame(absolutePoses, jointData.second.getParentIndex(), parentPosition, _entityPosition, _entityRotation);
+        getJointRotationInWorldFrame(absolutePoses, jointData.second.getParentIndex(), parentWorldRotation, _entityRotation);
+        jointData.second.setUpdatedData(jointPosition, jointTranslation, jointRotation, parentPosition, parentWorldRotation);
+    }
+    auto &selfCollisions = _collisionSystem.getSelfCollisions();
+    for (auto &collision : selfCollisions) {
+        glm::quat jointRotation;
+        getJointPositionInWorldFrame(absolutePoses, collision._jointIndex, collision._position, _entityPosition, _entityRotation);
+        getJointRotationInWorldFrame(absolutePoses, collision._jointIndex, jointRotation, _entityRotation);
+        glm::vec3 worldOffset = jointRotation * collision._offset;
+        collision._position = collision._position + worldOffset;
+    }
+    _collisionSystem.prepareCollisions();
+}
+
+void Flow::setJoints(AnimPoseVec& relativePoses, const std::vector<bool>& overrideFlags) {
+    for (auto &thread : _jointThreads) {
+        auto &joints = thread._joints;
+        for (int jointIndex : joints) {
+            auto &joint = _flowJointData[jointIndex];
+            if (jointIndex >= 0 && jointIndex < (int)relativePoses.size() && !overrideFlags[jointIndex]) {
+                relativePoses[jointIndex].rot() = joint.getCurrentRotation();
+            }            
+        }
+    }
+}
+
+void Flow::setOthersCollision(const QUuid& otherId, int jointIndex, const glm::vec3& position) {
+    FlowCollisionSettings settings;
+    settings._entityID = otherId;
+    settings._radius = HAND_COLLISION_RADIUS;
+    _collisionSystem.addCollisionSphere(jointIndex, settings, position, false, true);
+}
+
+void Flow::setPhysicsSettingsForGroup(const QString& group, const FlowPhysicsSettings& settings) {
+    for (auto &joint : _flowJointData) {
+        if (joint.second.getGroup().toUpper() == group.toUpper()) {
+            joint.second.setSettings(settings);
+        }
+    }
+}
+
+bool Flow::getJointPositionInWorldFrame(const AnimPoseVec& absolutePoses, int jointIndex, glm::vec3& position, glm::vec3 translation, glm::quat rotation) const {
+    if (jointIndex >= 0 && jointIndex < (int)absolutePoses.size()) {
+        glm::vec3 poseSetTrans = absolutePoses[jointIndex].trans();
+        position = (rotation * poseSetTrans) + translation;
+        if (!isNaN(position)) {
+            return true;
+        } else {
+            position = glm::vec3(0.0f);
+        }
+    }
+    return false;
+}
+
+bool Flow::getJointRotationInWorldFrame(const AnimPoseVec& absolutePoses, int jointIndex, glm::quat& result, const glm::quat& rotation) const {
+    if (jointIndex >= 0 && jointIndex < (int)absolutePoses.size()) {
+        result = rotation * absolutePoses[jointIndex].rot();
+        return true;
+    } else {
+        return false;
+    }
+}
+
+bool Flow::getJointRotation(const AnimPoseVec& relativePoses, int jointIndex, glm::quat& rotation) const {
+    if (jointIndex >= 0 && jointIndex < (int)relativePoses.size()) {
+        rotation = relativePoses[jointIndex].rot();
+        return true;
+    } else {
+        return false;
+    }
+}
+
+bool Flow::getJointTranslation(const AnimPoseVec& relativePoses, int jointIndex, glm::vec3& translation) const {
+    if (jointIndex >= 0 && jointIndex < (int)relativePoses.size()) {
+        translation = relativePoses[jointIndex].trans();
+        return true;
+    } else {
+        return false;
+    }
+}
+
+Flow& Flow::operator=(const Flow& otherFlow) {
+    _active = otherFlow.getActive();
+    _scale = otherFlow.getScale();
+    _isScaleSet = true;
+    auto &threads = otherFlow.getThreads();
+    if (threads.size() == _jointThreads.size()) {
+        for (size_t i = 0; i < _jointThreads.size(); i++) {
+            _jointThreads[i] = threads[i];
+        }
+    }
+    return *this;
+}

libraries/animation/src/Flow.h

@@ -0,0 +1,328 @@
+//
+//  Flow.h
+//
+//  Created by Luis Cuenca on 1/21/2019.
+//  Copyright 2019 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_Flow_h
+#define hifi_Flow_h
+
+#include <memory>
+#include <qstring.h>
+#include <glm/glm.hpp>
+#include <glm/gtx/quaternion.hpp>
+#include <vector>
+#include <map>
+#include <quuid.h>
+#include "AnimPose.h"
+
+class Rig;
+class AnimSkeleton;
+
+const float HAPTIC_TOUCH_STRENGTH = 0.25f;
+const float HAPTIC_TOUCH_DURATION = 10.0f;
+const float HAPTIC_SLOPE = 0.18f;
+const float HAPTIC_THRESHOLD = 40.0f;
+
+const QString FLOW_JOINT_PREFIX = "flow";
+const QString SIM_JOINT_PREFIX = "sim";
+
+const std::vector<QString> HAND_COLLISION_JOINTS = { "RightHandMiddle1", "RightHandThumb3", "LeftHandMiddle1", "LeftHandThumb3", "RightHandMiddle3", "LeftHandMiddle3" };
+
+const float HAND_COLLISION_RADIUS = 0.03f;
+const float HELPER_JOINT_LENGTH = 0.05f;
+
+const float DEFAULT_STIFFNESS = 0.0f;
+const float DEFAULT_GRAVITY = -0.0096f;
+const float DEFAULT_DAMPING = 0.85f;
+const float DEFAULT_INERTIA = 0.8f;
+const float DEFAULT_DELTA = 0.55f;
+const float DEFAULT_RADIUS = 0.01f;
+
+const uint64_t MAX_UPDATE_FLOW_TIME_BUDGET = 2000;
+
+struct FlowPhysicsSettings {
+    FlowPhysicsSettings() {};
+    FlowPhysicsSettings(bool active, float stiffness, float gravity, float damping, float inertia, float delta, float radius) {
+        _active = active;
+        _stiffness = stiffness;
+        _gravity = gravity;
+        _damping = damping;
+        _inertia = inertia;
+        _delta = delta;
+        _radius = radius;
+    }
+    bool _active{ true };
+    float _stiffness{ DEFAULT_STIFFNESS };
+    float _gravity{ DEFAULT_GRAVITY };
+    float _damping{ DEFAULT_DAMPING };
+    float _inertia{ DEFAULT_INERTIA };
+    float _delta{ DEFAULT_DELTA };
+    float _radius{ DEFAULT_RADIUS };
+};
+
+enum FlowCollisionType {
+    CollisionSphere = 0
+};
+
+struct FlowCollisionSettings {
+    FlowCollisionSettings() {};
+    FlowCollisionSettings(const QUuid& id, const FlowCollisionType& type, const glm::vec3& offset, float radius) {
+        _entityID = id;
+        _type = type;
+        _offset = offset;
+        _radius = radius;
+    };
+    QUuid _entityID;
+    FlowCollisionType _type { FlowCollisionType::CollisionSphere };
+    float _radius { 0.05f };
+    glm::vec3 _offset;
+};
+
+const FlowPhysicsSettings DEFAULT_JOINT_SETTINGS;
+
+struct FlowJointInfo {
+    FlowJointInfo() {};
+    FlowJointInfo(int index, int parentIndex, int childIndex, const QString& name) {
+        _index = index;
+        _parentIndex = parentIndex;
+        _childIndex = childIndex;
+        _name = name;
+    }
+    int _index { -1 };
+    QString _name; 
+    int _parentIndex { -1 };
+    int _childIndex { -1 };
+};
+
+struct FlowCollisionResult {
+    int _count { 0 };
+    float _offset { 0.0f };
+    glm::vec3 _position;
+    float _radius { 0.0f };
+    glm::vec3 _normal;
+    float _distance { 0.0f };
+};
+
+class FlowCollisionSphere {
+public:
+    FlowCollisionSphere() {};
+    FlowCollisionSphere(const int& jointIndex, const FlowCollisionSettings& settings, bool isTouch = false);
+    void setPosition(const glm::vec3& position) { _position = position; }
+    FlowCollisionResult computeSphereCollision(const glm::vec3& point, float radius) const;
+    FlowCollisionResult checkSegmentCollision(const glm::vec3& point1, const glm::vec3& point2, const FlowCollisionResult& collisionResult1, const FlowCollisionResult& collisionResult2);
+
+    QUuid _entityID;
+
+    glm::vec3 _offset;
+    glm::vec3 _initialOffset;
+    glm::vec3 _position;
+
+    bool _isTouch { false };
+    int _jointIndex { -1 };
+    int collisionIndex { -1 };
+    float _radius { 0.0f };
+    float _initialRadius{ 0.0f };
+};
+
+class FlowThread;
+
+class FlowCollisionSystem {
+public:
+    FlowCollisionSystem() {};
+    void addCollisionSphere(int jointIndex, const FlowCollisionSettings& settings, const glm::vec3& position = { 0.0f, 0.0f, 0.0f }, bool isSelfCollision = true, bool isTouch = false);
+    FlowCollisionResult computeCollision(const std::vector<FlowCollisionResult> collisions);
+
+    std::vector<FlowCollisionResult> checkFlowThreadCollisions(FlowThread* flowThread);
+
+    std::vector<FlowCollisionSphere>& getSelfCollisions() { return _selfCollisions; };
+    void setOthersCollisions(const std::vector<FlowCollisionSphere>& othersCollisions) { _othersCollisions = othersCollisions; }
+    void prepareCollisions();
+    void resetCollisions();
+    void resetOthersCollisions() { _othersCollisions.clear(); }
+    void setScale(float scale);
+    FlowCollisionSettings getCollisionSettingsByJoint(int jointIndex);
+    void setCollisionSettingsByJoint(int jointIndex, const FlowCollisionSettings& settings);
+    void setActive(bool active) { _active = active; }
+    bool getActive() const { return _active; }
+protected:
+    std::vector<FlowCollisionSphere> _selfCollisions;
+    std::vector<FlowCollisionSphere> _othersCollisions;
+    std::vector<FlowCollisionSphere> _allCollisions;
+    float _scale { 1.0f };
+    bool _active { false };
+};
+
+class FlowNode {
+public:
+    FlowNode() {};
+    FlowNode(const glm::vec3& initialPosition, FlowPhysicsSettings settings);
+    
+    void update(float deltaTime, const glm::vec3& accelerationOffset);
+    void solve(const glm::vec3& constrainPoint, float maxDistance, const FlowCollisionResult& collision);
+    void solveConstraints(const glm::vec3& constrainPoint, float maxDistance);
+    void solveCollisions(const FlowCollisionResult& collision);    
+
+protected:
+
+    FlowPhysicsSettings _settings;
+    glm::vec3 _initialPosition;
+    glm::vec3 _previousPosition;
+    glm::vec3 _currentPosition;
+
+    glm::vec3 _currentVelocity;
+    glm::vec3 _previousVelocity;
+    glm::vec3 _acceleration;
+
+    FlowCollisionResult _collision;
+    FlowCollisionResult _previousCollision;
+
+    float _initialRadius { 0.0f };
+
+    bool _anchored { false };
+    bool _colliding { false };
+    bool _active { true };
+
+    float _scale{ 1.0f };
+};
+
+class FlowJoint : public FlowNode {
+public:
+    friend class FlowThread;
+
+    FlowJoint(): FlowNode() {};
+    FlowJoint(int jointIndex, int parentIndex, int childIndex, const QString& name, const QString& group, const FlowPhysicsSettings& settings);
+    void toHelperJoint(const glm::vec3& initialPosition, float length);
+    void setInitialData(const glm::vec3& initialPosition, const glm::vec3& initialTranslation, const glm::quat& initialRotation, const glm::vec3& parentPosition);
+    void setUpdatedData(const glm::vec3& updatedPosition, const glm::vec3& updatedTranslation, const glm::quat& updatedRotation, const glm::vec3& parentPosition, const glm::quat& parentWorldRotation);
+    void setRecoveryPosition(const glm::vec3& recoveryPosition);
+    void update(float deltaTime);
+    void solve(const FlowCollisionResult& collision);
+
+    void setScale(float scale, bool initScale);
+    bool isAnchored() const { return _anchored; }
+    void setAnchored(bool anchored) { _anchored = anchored; }
+    bool isHelper() const { return _isHelper; }
+
+    const FlowPhysicsSettings& getSettings() { return _settings; }
+    void setSettings(const FlowPhysicsSettings& settings) { _settings = settings; }
+
+    const glm::vec3& getCurrentPosition() const { return _currentPosition; }
+    int getIndex() const { return _index; }
+    int getParentIndex() const { return _parentIndex; }
+    void setChildIndex(int index) { _childIndex = index; }
+    const glm::vec3& getUpdatedPosition() const { return _updatedPosition; }
+    const QString& getGroup() const { return _group; }
+    const QString& getName() const { return _name; }
+    const glm::quat& getCurrentRotation() const { return _currentRotation; }
+    const glm::vec3& getCurrentTranslation() const { return _initialTranslation; }
+    const glm::vec3& getInitialPosition() const { return _initialPosition; }
+
+protected:
+
+    int _index{ -1 };
+    int _parentIndex{ -1 };
+    int _childIndex{ -1 };
+    QString _name;
+    QString _group;
+
+    bool _isHelper{ false };
+
+    glm::vec3 _initialTranslation;
+    glm::quat _initialRotation;
+
+    glm::vec3 _updatedPosition;
+    glm::vec3 _updatedTranslation;
+    glm::quat _updatedRotation;
+
+    glm::quat _currentRotation;
+    glm::vec3 _recoveryPosition;
+
+    glm::vec3 _parentPosition;
+    glm::quat _parentWorldRotation;
+    glm::vec3 _translationDirection;
+
+    float _length { 0.0f };
+    float _initialLength { 0.0f };
+
+    bool _applyRecovery { false };
+};
+
+class FlowThread {
+public:
+    FlowThread() {};
+    FlowThread& operator=(const FlowThread& otherFlowThread);
+
+    FlowThread(int rootIndex, std::map<int, FlowJoint>* joints);
+
+    void resetLength();
+    void computeFlowThread(int rootIndex);
+    void computeRecovery();
+    void update(float deltaTime);
+    void solve(FlowCollisionSystem& collisionSystem);
+    void computeJointRotations();
+    void setRootFramePositions(const std::vector<glm::vec3>& rootFramePositions) { _rootFramePositions = rootFramePositions; }
+    void setScale(float scale, bool initScale = false);
+
+    std::vector<int> _joints;
+    std::vector<glm::vec3> _positions;
+    float _radius{ 0.0f };
+    float _length{ 0.0f };
+    std::map<int, FlowJoint>* _jointsPointer;
+    std::vector<glm::vec3> _rootFramePositions;
+};
+
+class Flow : public QObject{
+    Q_OBJECT
+public:
+    Flow() { }
+    Flow& operator=(const Flow& otherFlow);
+    bool getActive() const { return _active; }
+    void setActive(bool active) { _active = active; }
+    bool isInitialized() const { return _initialized; }
+    float getScale() const { return _scale; }
+    void calculateConstraints(const std::shared_ptr<AnimSkeleton>& skeleton, AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses);
+    void update(float deltaTime, AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses, const std::vector<bool>& overrideFlags);
+    void setTransform(float scale, const glm::vec3& position, const glm::quat& rotation);
+    const std::map<int, FlowJoint>& getJoints() const { return _flowJointData; }
+    const std::vector<FlowThread>& getThreads() const { return _jointThreads; }
+    void setOthersCollision(const QUuid& otherId, int jointIndex, const glm::vec3& position);
+    FlowCollisionSystem& getCollisionSystem() { return _collisionSystem; }
+    void setPhysicsSettingsForGroup(const QString& group, const FlowPhysicsSettings& settings);
+    void cleanUp();
+
+signals:
+    void onCleanup();
+
+private:    
+    void updateAbsolutePoses(const AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses);
+    bool getJointPositionInWorldFrame(const AnimPoseVec& absolutePoses, int jointIndex, glm::vec3& position, glm::vec3 translation, glm::quat rotation) const;
+    bool getJointRotationInWorldFrame(const AnimPoseVec& absolutePoses, int jointIndex, glm::quat& result, const glm::quat& rotation) const;
+    bool getJointRotation(const AnimPoseVec& relativePoses, int jointIndex, glm::quat& rotation) const;
+    bool getJointTranslation(const AnimPoseVec& relativePoses, int jointIndex, glm::vec3& translation) const;
+    bool worldToJointPoint(const AnimPoseVec& absolutePoses, const glm::vec3& position, const int jointIndex, glm::vec3& jointSpacePosition) const;
+
+    void setJoints(AnimPoseVec& relativePoses, const std::vector<bool>& overrideFlags);
+    void updateJoints(AnimPoseVec& relativePoses, AnimPoseVec& absolutePoses);
+    bool updateRootFramePositions(const AnimPoseVec& absolutePoses, size_t threadIndex);
+    void setScale(float scale);
+    
+    float _scale { 1.0f };
+    float _lastScale{ 1.0f };
+    glm::vec3 _entityPosition;
+    glm::quat _entityRotation;
+    std::map<int, FlowJoint> _flowJointData;
+    std::vector<FlowThread> _jointThreads;
+    std::vector<QString> _flowJointKeywords;
+    FlowCollisionSystem _collisionSystem;
+    bool _initialized { false };
+    bool _active { false };
+    bool _isScaleSet { false };
+    bool _invertThreadLoop { false };
+};
+
+#endif // hifi_Flow_h

libraries/animation/src/IKTarget.h

@@ -35,6 +35,8 @@ public:
     bool getPoleVectorEnabled() const { return _poleVectorEnabled; }
     int getIndex() const { return _index; }
     Type getType() const { return _type; }
+    int getNumFlexCoefficients() const { return (int)_numFlexCoefficients; }
+    float getFlexCoefficient(size_t chainDepth) const;
 
     void setPose(const glm::quat& rotation, const glm::vec3& translation);
     void setPoleVector(const glm::vec3& poleVector) { _poleVector = poleVector; }
@@ -43,7 +45,6 @@ public:
     void setIndex(int index) { _index = index; }
     void setType(int);
     void setFlexCoefficients(size_t numFlexCoefficientsIn, const float* flexCoefficientsIn);
-    float getFlexCoefficient(size_t chainDepth) const;
 
     void setWeight(float weight) { _weight = weight; }
     float getWeight() const { return _weight; }

libraries/animation/src/Rig.cpp

@@ -34,7 +34,6 @@
 #include "IKTarget.h"
 #include "PathUtils.h"
 
-
 static int nextRigId = 1;
 static std::map<int, Rig*> rigRegistry;
 static std::mutex rigRegistryMutex;
@@ -74,6 +73,20 @@ static const QString RIGHT_FOOT_IK_ROTATION_VAR("rightFootIKRotationVar");
 static const QString MAIN_STATE_MACHINE_RIGHT_FOOT_ROTATION("mainStateMachineRightFootRotation");
 static const QString MAIN_STATE_MACHINE_RIGHT_FOOT_POSITION("mainStateMachineRightFootPosition");
 
+static const QString LEFT_HAND_POSITION("leftHandPosition");
+static const QString LEFT_HAND_ROTATION("leftHandRotation");
+static const QString LEFT_HAND_IK_POSITION_VAR("leftHandIKPositionVar");
+static const QString LEFT_HAND_IK_ROTATION_VAR("leftHandIKRotationVar");
+static const QString MAIN_STATE_MACHINE_LEFT_HAND_POSITION("mainStateMachineLeftHandPosition");
+static const QString MAIN_STATE_MACHINE_LEFT_HAND_ROTATION("mainStateMachineLeftHandRotation");
+
+static const QString RIGHT_HAND_POSITION("rightHandPosition");
+static const QString RIGHT_HAND_ROTATION("rightHandRotation");
+static const QString RIGHT_HAND_IK_POSITION_VAR("rightHandIKPositionVar");
+static const QString RIGHT_HAND_IK_ROTATION_VAR("rightHandIKRotationVar");
+static const QString MAIN_STATE_MACHINE_RIGHT_HAND_ROTATION("mainStateMachineRightHandRotation");
+static const QString MAIN_STATE_MACHINE_RIGHT_HAND_POSITION("mainStateMachineRightHandPosition");
+
 
 Rig::Rig() {
     // Ensure thread-safe access to the rigRegistry.
@@ -362,7 +375,6 @@ void Rig::reset(const HFMModel& hfmModel) {
 
     _animSkeleton = std::make_shared<AnimSkeleton>(hfmModel);
 
-
     _internalPoseSet._relativePoses.clear();
     _internalPoseSet._relativePoses = _animSkeleton->getRelativeDefaultPoses();
 
@@ -746,7 +758,8 @@ void Rig::computeMotionAnimationState(float deltaTime, const glm::vec3& worldPos
 
     glm::vec3 forward = worldRotation * IDENTITY_FORWARD;
     glm::vec3 workingVelocity = worldVelocity;
-
+    _internalFlow.setTransform(sensorToWorldScale, worldPosition, worldRotation * Quaternions::Y_180);
+    _networkFlow.setTransform(sensorToWorldScale, worldPosition, worldRotation * Quaternions::Y_180);
     {
         glm::vec3 localVel = glm::inverse(worldRotation) * workingVelocity;
 
@@ -1051,16 +1064,29 @@ void Rig::computeMotionAnimationState(float deltaTime, const glm::vec3& worldPos
 
         t += deltaTime;
 
-        if (_enableInverseKinematics != _lastEnableInverseKinematics) {
-            if (_enableInverseKinematics) {
-                _animVars.set("ikOverlayAlpha", 1.0f);
-            } else {
-                _animVars.set("ikOverlayAlpha", 0.0f);
-            }
+        if (_enableInverseKinematics) {
+            _animVars.set("ikOverlayAlpha", 1.0f);
+            _animVars.set("splineIKEnabled", true);
+            _animVars.set("leftHandIKEnabled", true);
+            _animVars.set("rightHandIKEnabled", true);
+            _animVars.set("leftFootIKEnabled", true);
+            _animVars.set("rightFootIKEnabled", true);
+            _animVars.set("leftFootPoleVectorEnabled", true);
+            _animVars.set("rightFootPoleVectorEnabled", true);
+        } else {
+            _animVars.set("ikOverlayAlpha", 0.0f);
+            _animVars.set("splineIKEnabled", false);
+            _animVars.set("leftHandIKEnabled", false);
+            _animVars.set("rightHandIKEnabled", false);
+            _animVars.set("leftFootIKEnabled", false);
+            _animVars.set("rightFootIKEnabled", false);
+            _animVars.set("leftHandPoleVectorEnabled", false);
+            _animVars.set("rightHandPoleVectorEnabled", false);
+            _animVars.set("leftFootPoleVectorEnabled", false);
+            _animVars.set("rightFootPoleVectorEnabled", false);
         }
         _lastEnableInverseKinematics = _enableInverseKinematics;
     }
-
     _lastForward = forward;
     _lastPosition = worldPosition;
     _lastVelocity = workingVelocity;
@@ -1208,12 +1234,26 @@ void Rig::updateAnimations(float deltaTime, const glm::mat4& rootTransform, cons
         _networkVars = networkTriggersOut;
         _lastContext = context;
     }
+    
     applyOverridePoses();
-    buildAbsoluteRigPoses(_internalPoseSet._relativePoses, _internalPoseSet._absolutePoses);
-    buildAbsoluteRigPoses(_networkPoseSet._relativePoses, _networkPoseSet._absolutePoses);
+
+    buildAbsoluteRigPoses(_internalPoseSet._relativePoses, _internalPoseSet._absolutePoses);    
+    _internalFlow.update(deltaTime, _internalPoseSet._relativePoses, _internalPoseSet._absolutePoses, _internalPoseSet._overrideFlags);
+
+    if (_sendNetworkNode) {
+        if (_internalFlow.getActive() && !_networkFlow.getActive()) {
+            _networkFlow = _internalFlow;
+        }
+        buildAbsoluteRigPoses(_networkPoseSet._relativePoses, _networkPoseSet._absolutePoses);
+        _networkFlow.update(deltaTime, _networkPoseSet._relativePoses, _networkPoseSet._absolutePoses, _internalPoseSet._overrideFlags);
+    } else if (_networkFlow.getActive()) {
+        _networkFlow.setActive(false);
+    }
+
     // copy internal poses to external poses
     {
         QWriteLocker writeLock(&_externalPoseSetLock);
+        
         _externalPoseSet = _internalPoseSet;
     }
 }
@@ -1251,6 +1291,7 @@ void Rig::computeHeadFromHMD(const AnimPose& hmdPose, glm::vec3& headPositionOut
 void Rig::updateHead(bool headEnabled, bool hipsEnabled, const AnimPose& headPose) {
     if (_animSkeleton) {
         if (headEnabled) {
+            _animVars.set("splineIKEnabled", true);
             _animVars.set("headPosition", headPose.trans());
             _animVars.set("headRotation", headPose.rot());
             if (hipsEnabled) {
@@ -1265,6 +1306,7 @@ void Rig::updateHead(bool headEnabled, bool hipsEnabled, const AnimPose& headPos
                 _animVars.set("headWeight", 8.0f);
             }
         } else {
+            _animVars.set("splineIKEnabled", false);
             _animVars.unset("headPosition");
             _animVars.set("headRotation", headPose.rot());
             _animVars.set("headType", (int)IKTarget::Type::RotationOnly);
@@ -1396,8 +1438,22 @@ void Rig::updateHands(bool leftHandEnabled, bool rightHandEnabled, bool hipsEnab
 
     const bool ENABLE_POLE_VECTORS = true;
 
+    if (headEnabled) {
+        // always do IK if head is enabled
+        _animVars.set("leftHandIKEnabled", true);
+        _animVars.set("rightHandIKEnabled", true);
+    } else {
+        // only do IK if we have a valid foot.
+        _animVars.set("leftHandIKEnabled", leftHandEnabled);
+        _animVars.set("rightHandIKEnabled", rightHandEnabled);
+    }
+
     if (leftHandEnabled) {
 
+        // we need this for twoBoneIK version of hands.
+        _animVars.set(LEFT_HAND_IK_POSITION_VAR, LEFT_HAND_POSITION);
+        _animVars.set(LEFT_HAND_IK_ROTATION_VAR, LEFT_HAND_ROTATION);
+
         glm::vec3 handPosition = leftHandPose.trans();
         glm::quat handRotation = leftHandPose.rot();
 
@@ -1430,8 +1486,11 @@ void Rig::updateHands(bool leftHandEnabled, bool rightHandEnabled, bool hipsEnab
             _animVars.set("leftHandPoleVectorEnabled", false);
         }
     } else {
-        _animVars.set("leftHandPoleVectorEnabled", false);
+        // need this for two bone ik
+        _animVars.set(LEFT_HAND_IK_POSITION_VAR, MAIN_STATE_MACHINE_LEFT_HAND_POSITION);
+        _animVars.set(LEFT_HAND_IK_ROTATION_VAR, MAIN_STATE_MACHINE_LEFT_HAND_ROTATION);
 
+        _animVars.set("leftHandPoleVectorEnabled", false);
         _animVars.unset("leftHandPosition");
         _animVars.unset("leftHandRotation");
 
@@ -1445,6 +1504,10 @@ void Rig::updateHands(bool leftHandEnabled, bool rightHandEnabled, bool hipsEnab
 
     if (rightHandEnabled) {
 
+        // need this for two bone IK
+        _animVars.set(RIGHT_HAND_IK_POSITION_VAR, RIGHT_HAND_POSITION);
+        _animVars.set(RIGHT_HAND_IK_ROTATION_VAR, RIGHT_HAND_ROTATION);
+
         glm::vec3 handPosition = rightHandPose.trans();
         glm::quat handRotation = rightHandPose.rot();
 
@@ -1478,8 +1541,12 @@ void Rig::updateHands(bool leftHandEnabled, bool rightHandEnabled, bool hipsEnab
             _animVars.set("rightHandPoleVectorEnabled", false);
         }
     } else {
-        _animVars.set("rightHandPoleVectorEnabled", false);
 
+        // need this for two bone IK
+        _animVars.set(RIGHT_HAND_IK_POSITION_VAR, MAIN_STATE_MACHINE_RIGHT_HAND_POSITION);
+        _animVars.set(RIGHT_HAND_IK_ROTATION_VAR, MAIN_STATE_MACHINE_RIGHT_HAND_ROTATION);
+
+        _animVars.set("rightHandPoleVectorEnabled", false);
         _animVars.unset("rightHandPosition");
         _animVars.unset("rightHandRotation");
 
@@ -1697,6 +1764,7 @@ bool Rig::calculateElbowPoleVector(int handIndex, int elbowIndex, int armIndex,
         correctionVector = forwardAmount * frontVector;
     }
     poleVector = glm::normalize(attenuationVector + fullPoleVector + correctionVector);
+
     return true;
 }
 
@@ -1819,7 +1887,7 @@ void Rig::updateFromControllerParameters(const ControllerParameters& params, flo
     std::shared_ptr<AnimInverseKinematics> ikNode = getAnimInverseKinematicsNode();
     for (int i = 0; i < (int)NumSecondaryControllerTypes; i++) {
         int index = indexOfJoint(secondaryControllerJointNames[i]);
-        if (index >= 0) {
+        if ((index >= 0) && (ikNode)) {
             if (params.secondaryControllerFlags[i] & (uint8_t)ControllerFlags::Enabled) {
                 ikNode->setSecondaryTargetInRigFrame(index, params.secondaryControllerPoses[i]);
             } else {
@@ -1866,7 +1934,6 @@ void Rig::initAnimGraph(const QUrl& url) {
                 auto roleState = roleAnimState.second;
                 overrideRoleAnimation(roleState.role, roleState.url, roleState.fps, roleState.loop, roleState.firstFrame, roleState.lastFrame);
             }
-
             emit onLoadComplete();
         });
         connect(_animLoader.get(), &AnimNodeLoader::error, [url](int error, QString str) {
@@ -2105,3 +2172,16 @@ void Rig::computeAvatarBoundingCapsule(
     glm::vec3 capsuleCenter = transformPoint(_geometryToRigTransform, (0.5f * (totalExtents.maximum + totalExtents.minimum)));
     localOffsetOut = capsuleCenter - hipsPosition;
 }
+
+void Rig::initFlow(bool isActive) {
+    _internalFlow.setActive(isActive);
+    if (isActive) {
+        if (!_internalFlow.isInitialized()) {
+            _internalFlow.calculateConstraints(_animSkeleton, _internalPoseSet._relativePoses, _internalPoseSet._absolutePoses);
+            _networkFlow.calculateConstraints(_animSkeleton, _internalPoseSet._relativePoses, _internalPoseSet._absolutePoses);
+        }
+    } else {
+        _internalFlow.cleanUp();
+        _networkFlow.cleanUp();
+    }
+}

libraries/animation/src/Rig.h

@@ -25,6 +25,7 @@
 #include "AnimNodeLoader.h"
 #include "SimpleMovingAverage.h"
 #include "AnimUtil.h"
+#include "Flow.h"
 
 class Rig;
 class AnimInverseKinematics;
@@ -233,6 +234,9 @@ public:
     const AnimContext::DebugAlphaMap& getDebugAlphaMap() const { return _lastContext.getDebugAlphaMap(); }
     const AnimVariantMap& getAnimVars() const { return _lastAnimVars; }
     const AnimContext::DebugStateMachineMap& getStateMachineMap() const { return _lastContext.getStateMachineMap(); }
+    void initFlow(bool isActive);
+    Flow& getFlow() { return _internalFlow; }
+
 
 signals:
     void onLoadComplete();
@@ -424,6 +428,8 @@ protected:
 
     SnapshotBlendPoseHelper _hipsBlendHelper;
     ControllerParameters _previousControllerParameters;
+    Flow _internalFlow;
+    Flow _networkFlow;
 };
 
 #endif /* defined(__hifi__Rig__) */

libraries/audio-client/src/AudioClient.cpp

@@ -175,7 +175,7 @@ static float computeLoudness(int16_t* samples, int numSamples, int numChannels,
     const int32_t CLIPPING_THRESHOLD = 32392;   // -0.1 dBFS
     const int32_t CLIPPING_DETECTION = 3;       // consecutive samples over threshold
 
-    float scale = numSamples ? 1.0f / (numSamples * 32768.0f) : 0.0f;
+    float scale = numSamples ? 1.0f / numSamples : 0.0f;
 
     int32_t loudness = 0;
     isClipping = false;
@@ -249,6 +249,8 @@ AudioClient::AudioClient() :
     _outputBufferSizeFrames("audioOutputBufferFrames", DEFAULT_BUFFER_FRAMES),
     _sessionOutputBufferSizeFrames(_outputBufferSizeFrames.get()),
     _outputStarveDetectionEnabled("audioOutputStarveDetectionEnabled", DEFAULT_STARVE_DETECTION_ENABLED),
+    _lastRawInputLoudness(0.0f),
+    _lastSmoothedRawInputLoudness(0.0f),
     _lastInputLoudness(0.0f),
     _timeSinceLastClip(-1.0f),
     _muted(false),
@@ -1144,6 +1146,9 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) {
             emit inputReceived(audioBuffer);
         }
 
+        // loudness after mute/gate
+        _lastInputLoudness = (_muted || !audioGateOpen) ? 0.0f : _lastRawInputLoudness;
+
         // detect gate opening and closing
         bool openedInLastBlock = !_audioGateOpen && audioGateOpen;  // the gate just opened
         bool closedInLastBlock = _audioGateOpen && !audioGateOpen;  // the gate just closed
@@ -1222,12 +1227,15 @@ void AudioClient::handleMicAudioInput() {
 
         // detect loudness and clipping on the raw input
         bool isClipping = false;
-        float inputLoudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping);
+        float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount(), isClipping);
+        _lastRawInputLoudness = loudness;
 
-        float tc = (inputLoudness > _lastInputLoudness) ? 0.378f : 0.967f;  // 10ms attack, 300ms release @ 100Hz
-        inputLoudness += tc * (_lastInputLoudness - inputLoudness);
-        _lastInputLoudness = inputLoudness;
+        // envelope detection
+        float tc = (loudness > _lastSmoothedRawInputLoudness) ? 0.378f : 0.967f;  // 10ms attack, 300ms release @ 100Hz
+        loudness += tc * (_lastSmoothedRawInputLoudness - loudness);
+        _lastSmoothedRawInputLoudness = loudness;
 
+        // clipping indicator
         if (isClipping) {
             _timeSinceLastClip = 0.0f;
         } else if (_timeSinceLastClip >= 0.0f) {
@@ -1235,7 +1243,7 @@ void AudioClient::handleMicAudioInput() {
         }
         isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f);   // 2 second hold time
 
-        emit inputLoudnessChanged(_lastInputLoudness, isClipping);
+        emit inputLoudnessChanged(_lastSmoothedRawInputLoudness, isClipping);
 
         if (!_muted) {
             possibleResampling(_inputToNetworkResampler,

libraries/audio-client/src/AudioClient.h

@@ -127,7 +127,7 @@ public:
 
     const QAudioFormat& getOutputFormat() const { return _outputFormat; }
 
-    float getLastInputLoudness() const { return _lastInputLoudness; }   // TODO: relative to noise floor?
+    float getLastInputLoudness() const { return _lastInputLoudness; }
 
     float getTimeSinceLastClip() const { return _timeSinceLastClip; }
     float getAudioAverageInputLoudness() const { return _lastInputLoudness; }
@@ -355,7 +355,9 @@ private:
 
     StDev _stdev;
     QElapsedTimer _timeSinceLastReceived;
-    float _lastInputLoudness;
+    float _lastRawInputLoudness;    // before mute/gate
+    float _lastSmoothedRawInputLoudness;
+    float _lastInputLoudness;       // after mute/gate
     float _timeSinceLastClip;
     int _totalInputAudioSamples;
 

libraries/avatars-renderer/src/avatars-renderer/Avatar.cpp

@@ -37,6 +37,7 @@
 #include "RenderableModelEntityItem.h"
 
 #include <graphics-scripting/Forward.h>
+#include <CubicHermiteSpline.h>
 
 #include "Logging.h"
 
@@ -64,7 +65,11 @@ namespace render {
         return keyBuilder.build();
     }
     template <> const Item::Bound payloadGetBound(const AvatarSharedPointer& avatar) {
-        return static_pointer_cast<Avatar>(avatar)->getRenderBounds();
+        auto avatarPtr = static_pointer_cast<Avatar>(avatar);
+        if (avatarPtr) {
+            return avatarPtr->getRenderBounds();
+        }
+        return Item::Bound();
     }
     template <> void payloadRender(const AvatarSharedPointer& avatar, RenderArgs* args) {
         auto avatarPtr = static_pointer_cast<Avatar>(avatar);
@@ -75,10 +80,15 @@ namespace render {
     }
     template <> uint32_t metaFetchMetaSubItems(const AvatarSharedPointer& avatar, ItemIDs& subItems) {
         auto avatarPtr = static_pointer_cast<Avatar>(avatar);
-        if (avatarPtr->getSkeletonModel()) {
-            auto& metaSubItems = avatarPtr->getSkeletonModel()->fetchRenderItemIDs();
-            subItems.insert(subItems.end(), metaSubItems.begin(), metaSubItems.end());
-            return (uint32_t) metaSubItems.size();
+        if (avatarPtr) {
+            uint32_t total = 0;
+            if (avatarPtr->getSkeletonModel()) {
+                auto& metaSubItems = avatarPtr->getSkeletonModel()->fetchRenderItemIDs();
+                subItems.insert(subItems.end(), metaSubItems.begin(), metaSubItems.end());
+                total += (uint32_t)metaSubItems.size();
+            }
+            total += avatarPtr->appendSubMetaItems(subItems);
+            return total;
         }
         return 0;
     }
@@ -626,12 +636,18 @@ void Avatar::addToScene(AvatarSharedPointer self, const render::ScenePointer& sc
     _skeletonModel->setVisibleInScene(_isMeshVisible, scene);
 
     processMaterials();
+    bool attachmentRenderingNeedsUpdate = false;
     for (auto& attachmentModel : _attachmentModels) {
         attachmentModel->addToScene(scene, transaction);
         attachmentModel->setTagMask(render::hifi::TAG_ALL_VIEWS);
-        attachmentModel->setGroupCulled(false);
+        attachmentModel->setGroupCulled(true);
         attachmentModel->setCanCastShadow(true);
         attachmentModel->setVisibleInScene(_isMeshVisible, scene);
+        attachmentRenderingNeedsUpdate = true;
+    }
+
+    if (attachmentRenderingNeedsUpdate) {
+        updateAttachmentRenderIDs();
     }
 
     _mustFadeIn = true;
@@ -855,15 +871,17 @@ void Avatar::fixupModelsInScene(const render::ScenePointer& scene) {
         canTryFade = true;
         _isAnimatingScale = true;
     }
+    bool attachmentRenderingNeedsUpdate = false;
     for (auto attachmentModel : _attachmentModels) {
         if (attachmentModel->isRenderable() && attachmentModel->needsFixupInScene()) {
             attachmentModel->removeFromScene(scene, transaction);
             attachmentModel->addToScene(scene, transaction);
 
             attachmentModel->setTagMask(render::hifi::TAG_ALL_VIEWS);
-            attachmentModel->setGroupCulled(false);
+            attachmentModel->setGroupCulled(true);
             attachmentModel->setCanCastShadow(true);
             attachmentModel->setVisibleInScene(_isMeshVisible, scene);
+            attachmentRenderingNeedsUpdate = true;
         }
     }
 
@@ -886,9 +904,15 @@ void Avatar::fixupModelsInScene(const render::ScenePointer& scene) {
 
     for (auto attachmentModelToRemove : _attachmentsToRemove) {
         attachmentModelToRemove->removeFromScene(scene, transaction);
+        attachmentRenderingNeedsUpdate = true;
     }
     _attachmentsToDelete.insert(_attachmentsToDelete.end(), _attachmentsToRemove.begin(), _attachmentsToRemove.end());
     _attachmentsToRemove.clear();
+
+    if (attachmentRenderingNeedsUpdate) {
+        updateAttachmentRenderIDs();
+    }
+
     scene->enqueueTransaction(transaction);
 }
 
@@ -931,6 +955,11 @@ void Avatar::simulateAttachments(float deltaTime) {
             }
         }
     }
+
+    if (_ancestorChainRenderableVersion != _lastAncestorChainRenderableVersion) {
+        _lastAncestorChainRenderableVersion = _ancestorChainRenderableVersion;
+        updateDescendantRenderIDs();
+    }
 }
 
 float Avatar::getBoundingRadius() const {
@@ -1506,12 +1535,14 @@ void Avatar::setModelURLFinished(bool success) {
 // rig is ready
 void Avatar::rigReady() {
     buildUnscaledEyeHeightCache();
+    buildSpine2SplineRatioCache();
     computeMultiSphereShapes();
 }
 
 // rig has been reset.
 void Avatar::rigReset() {
     clearUnscaledEyeHeightCache();
+    clearSpine2SplineRatioCache();
 }
 
 void Avatar::computeMultiSphereShapes() {
@@ -1608,7 +1639,6 @@ void Avatar::setAttachmentData(const QVector<AttachmentData>& attachmentData) {
     }
 }
 
-
 int Avatar::parseDataFromBuffer(const QByteArray& buffer) {
     PerformanceTimer perfTimer("unpack");
     if (!_initialized) {
@@ -1967,10 +1997,43 @@ void Avatar::buildUnscaledEyeHeightCache() {
     }
 }
 
+void Avatar::buildSpine2SplineRatioCache() {
+    if (_skeletonModel) {
+        auto& rig = _skeletonModel->getRig();
+        AnimPose hipsRigDefaultPose = rig.getAbsoluteDefaultPose(rig.indexOfJoint("Hips"));
+        AnimPose headRigDefaultPose(rig.getAbsoluteDefaultPose(rig.indexOfJoint("Head")));
+        glm::vec3 basePosition = hipsRigDefaultPose.trans();
+        glm::vec3 tipPosition = headRigDefaultPose.trans();
+        glm::vec3 spine2Position = rig.getAbsoluteDefaultPose(rig.indexOfJoint("Spine2")).trans();
+
+        glm::vec3 baseToTip = tipPosition - basePosition;
+        float baseToTipLength = glm::length(baseToTip);
+        glm::vec3 baseToTipNormal = baseToTip / baseToTipLength;
+        glm::vec3 baseToSpine2 = spine2Position - basePosition;
+
+        _spine2SplineRatio = glm::dot(baseToSpine2, baseToTipNormal) / baseToTipLength;
+
+        CubicHermiteSplineFunctorWithArcLength defaultSpline(headRigDefaultPose.rot(), headRigDefaultPose.trans(), hipsRigDefaultPose.rot(), hipsRigDefaultPose.trans());
+
+        // measure the total arc length along the spline
+        float totalDefaultArcLength = defaultSpline.arcLength(1.0f);
+        float t = defaultSpline.arcLengthInverse(_spine2SplineRatio * totalDefaultArcLength);
+        glm::vec3 defaultSplineSpine2Translation = defaultSpline(t);
+
+        _spine2SplineOffset = spine2Position - defaultSplineSpine2Translation;
+    }
+
+}
+
 void Avatar::clearUnscaledEyeHeightCache() {
     _unscaledEyeHeightCache.set(DEFAULT_AVATAR_EYE_HEIGHT);
 }
 
+void Avatar::clearSpine2SplineRatioCache() {
+    _spine2SplineRatio = DEFAULT_AVATAR_EYE_HEIGHT;
+    _spine2SplineOffset = glm::vec3();
+}
+
 float Avatar::getUnscaledEyeHeightFromSkeleton() const {
 
     // TODO: if performance becomes a concern we can cache this value rather then computing it everytime.
@@ -2084,3 +2147,60 @@ void Avatar::clearAvatarGrabData(const QUuid& id) {
         }
     });
 }
+
+uint32_t Avatar::appendSubMetaItems(render::ItemIDs& subItems) {
+    return _subItemLock.resultWithReadLock<uint32_t>([&] {
+        uint32_t total = 0;
+
+        if (_attachmentRenderIDs.size() > 0) {
+            subItems.insert(subItems.end(), _attachmentRenderIDs.begin(), _attachmentRenderIDs.end());
+            total += (uint32_t)_attachmentRenderIDs.size();
+        }
+
+        if (_descendantRenderIDs.size() > 0) {
+            subItems.insert(subItems.end(), _descendantRenderIDs.begin(), _descendantRenderIDs.end());
+            total += (uint32_t)_descendantRenderIDs.size();
+        }
+
+        return total;
+    });
+}
+
+void Avatar::updateAttachmentRenderIDs() {
+    _subItemLock.withWriteLock([&] {
+        _attachmentRenderIDs.clear();
+        for (auto& attachmentModel : _attachmentModels) {
+            if (attachmentModel && attachmentModel->isRenderable()) {
+                auto& metaSubItems = attachmentModel->fetchRenderItemIDs();
+                _attachmentRenderIDs.insert(_attachmentRenderIDs.end(), metaSubItems.begin(), metaSubItems.end());
+            }
+        }
+    });
+}
+
+void Avatar::updateDescendantRenderIDs() {
+    _subItemLock.withWriteLock([&] {
+        _descendantRenderIDs.clear();
+        auto entityTreeRenderer = DependencyManager::get<EntityTreeRenderer>();
+        EntityTreePointer entityTree = entityTreeRenderer ? entityTreeRenderer->getTree() : nullptr;
+        if (entityTree) {
+            entityTree->withReadLock([&] {
+                forEachDescendant([&](SpatiallyNestablePointer object) {
+                    if (object && object->getNestableType() == NestableType::Entity) {
+                        EntityItemPointer entity = std::static_pointer_cast<EntityItem>(object);
+                        if (entity->isVisible()) {
+                            auto renderer = entityTreeRenderer->renderableForEntityId(object->getID());
+                            if (renderer) {
+                                render::ItemIDs renderableSubItems;
+                                uint32_t numRenderableSubItems = renderer->metaFetchMetaSubItems(renderableSubItems);
+                                if (numRenderableSubItems > 0) {
+                                    _descendantRenderIDs.insert(_descendantRenderIDs.end(), renderableSubItems.begin(), renderableSubItems.end());
+                                }
+                            }
+                        }
+                    }
+                });
+            });
+        }
+    });
+}

libraries/avatars-renderer/src/avatars-renderer/Avatar.h

@@ -235,6 +235,8 @@ public:
     virtual glm::vec3 getAbsoluteJointTranslationInObjectFrame(int index) const override;
     virtual bool setAbsoluteJointRotationInObjectFrame(int index, const glm::quat& rotation) override { return false; }
     virtual bool setAbsoluteJointTranslationInObjectFrame(int index, const glm::vec3& translation) override { return false; }
+    virtual glm::vec3 getSpine2SplineOffset() const { return _spine2SplineOffset; }
+    virtual float getSpine2SplineRatio() const { return _spine2SplineRatio; }
 
     // world-space to avatar-space rigconversion functions
     /**jsdoc
@@ -498,6 +500,8 @@ public:
     const std::vector<MultiSphereShape>& getMultiSphereShapes() const { return _multiSphereShapes; }
     void tearDownGrabs();
 
+    uint32_t appendSubMetaItems(render::ItemIDs& subItems);
+
 signals:
     void targetScaleChanged(float targetScale);
 
@@ -561,7 +565,9 @@ public slots:
 protected:
     float getUnscaledEyeHeightFromSkeleton() const;
     void buildUnscaledEyeHeightCache();
+    void buildSpine2SplineRatioCache();
     void clearUnscaledEyeHeightCache();
+    void clearSpine2SplineRatioCache();
     virtual const QString& getSessionDisplayNameForTransport() const override { return _empty; } // Save a tiny bit of bandwidth. Mixer won't look at what we send.
     QString _empty{};
     virtual void maybeUpdateSessionDisplayNameFromTransport(const QString& sessionDisplayName) override { _sessionDisplayName = sessionDisplayName; } // don't use no-op setter!
@@ -638,8 +644,6 @@ protected:
     RateCounter<> _skeletonModelSimulationRate;
     RateCounter<> _jointDataSimulationRate;
 
-
-protected:
     class AvatarEntityDataHash {
     public:
         AvatarEntityDataHash(uint32_t h) : hash(h) {};
@@ -669,6 +673,8 @@ protected:
     float _displayNameAlpha { 1.0f };
 
     ThreadSafeValueCache<float> _unscaledEyeHeightCache { DEFAULT_AVATAR_EYE_HEIGHT };
+    float _spine2SplineRatio { DEFAULT_SPINE2_SPLINE_PROPORTION };
+    glm::vec3 _spine2SplineOffset;
 
     std::unordered_map<std::string, graphics::MultiMaterial> _materials;
     std::mutex _materialsLock;
@@ -699,6 +705,13 @@ protected:
     MapOfGrabs _avatarGrabs;
     SetOfIDs _grabsToChange; // updated grab IDs -- changes needed to entities or physics
     VectorOfIDs _grabsToDelete; // deleted grab IDs -- changes needed to entities or physics
+
+    ReadWriteLockable _subItemLock;
+    void updateAttachmentRenderIDs();
+    render::ItemIDs _attachmentRenderIDs;
+    void updateDescendantRenderIDs();
+    render::ItemIDs _descendantRenderIDs;
+    uint32_t _lastAncestorChainRenderableVersion { 0 };
 };
 
 #endif // hifi_Avatar_h

libraries/display-plugins/src/display-plugins/Basic2DWindowOpenGLDisplayPlugin.cpp

@@ -74,30 +74,15 @@ void Basic2DWindowOpenGLDisplayPlugin::customizeContext() {
         }
     }
 
-
-    _virtualPadJumpBtnPixelSize = dpi * VirtualPad::Manager::JUMP_BTN_FULL_PIXELS / VirtualPad::Manager::DPI;
-    if (!_virtualPadJumpBtnTexture) {
-        auto iconPath = PathUtils::resourcesPath() + "images/fly.png";
-        auto image = QImage(iconPath);
-        if (image.format() != QImage::Format_ARGB32) {
-            image = image.convertToFormat(QImage::Format_ARGB32);
-        }
-        if ((image.width() > 0) && (image.height() > 0)) {
-            image = image.scaled(_virtualPadJumpBtnPixelSize, _virtualPadJumpBtnPixelSize, Qt::KeepAspectRatio);
-            image = image.mirrored();
-
-            _virtualPadJumpBtnTexture = gpu::Texture::createStrict(
-                    gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA),
-                    image.width(), image.height(),
-                    gpu::Texture::MAX_NUM_MIPS,
-                    gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR));
-            _virtualPadJumpBtnTexture->setSource("virtualPad jump");
-            auto usage = gpu::Texture::Usage::Builder().withColor().withAlpha();
-            _virtualPadJumpBtnTexture->setUsage(usage.build());
-            _virtualPadJumpBtnTexture->setStoredMipFormat(gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA));
-            _virtualPadJumpBtnTexture->assignStoredMip(0, image.byteCount(), image.constBits());
-            _virtualPadJumpBtnTexture->setAutoGenerateMips(true);
-        }
+    if (_virtualPadButtons.size() == 0) {
+        _virtualPadButtons.append(VirtualPadButton(
+                dpi * VirtualPad::Manager::BTN_FULL_PIXELS / VirtualPad::Manager::DPI,
+                PathUtils::resourcesPath() + "images/fly.png",
+                VirtualPad::Manager::Button::JUMP));
+        _virtualPadButtons.append(VirtualPadButton(
+                dpi * VirtualPad::Manager::BTN_FULL_PIXELS / VirtualPad::Manager::DPI,
+                PathUtils::resourcesPath() + "images/handshake.png",
+                VirtualPad::Manager::Button::HANDSHAKE));
     }
 #endif
     Parent::customizeContext();
@@ -133,8 +118,6 @@ void Basic2DWindowOpenGLDisplayPlugin::compositeExtra() {
                                                                                                     _virtualPadPixelSize, _virtualPadPixelSize);
         auto stickTransform = DependencyManager::get<CompositorHelper>()->getPoint2DTransform(virtualPadManager.getLeftVirtualPad()->getCurrentTouch(),
                                                                                               _virtualPadPixelSize, _virtualPadPixelSize);
-        auto jumpTransform = DependencyManager::get<CompositorHelper>()->getPoint2DTransform(virtualPadManager.getJumpButtonPosition(),
-                                                                                             _virtualPadJumpBtnPixelSize, _virtualPadJumpBtnPixelSize);
 
         render([&](gpu::Batch& batch) {
             batch.enableStereo(false);
@@ -151,9 +134,9 @@ void Basic2DWindowOpenGLDisplayPlugin::compositeExtra() {
             batch.setModelTransform(stickTransform);
             batch.draw(gpu::TRIANGLE_STRIP, 4);
 
-            batch.setResourceTexture(0, _virtualPadJumpBtnTexture);
-            batch.setModelTransform(jumpTransform);
-            batch.draw(gpu::TRIANGLE_STRIP, 4);
+            foreach(VirtualPadButton virtualPadButton, _virtualPadButtons) {
+                virtualPadButton.draw(batch, virtualPadManager.getButtonPosition(virtualPadButton._button));
+            }
         });
     }
 #endif
@@ -178,3 +161,47 @@ bool Basic2DWindowOpenGLDisplayPlugin::isThrottled() const {
 QScreen* Basic2DWindowOpenGLDisplayPlugin::getFullscreenTarget() {
     return qApp->primaryScreen();
 }
+
+#if defined(Q_OS_ANDROID)
+
+Basic2DWindowOpenGLDisplayPlugin::VirtualPadButton::VirtualPadButton(qreal pixelSize,
+                                                                     QString iconPath,
+                                                                     VirtualPad::Manager::Button button) :
+    _pixelSize { pixelSize },
+    _button { button }
+{
+    if (!_texture) {
+        auto image = QImage(iconPath);
+        if (image.format() != QImage::Format_ARGB32) {
+            image = image.convertToFormat(QImage::Format_ARGB32);
+        }
+        if ((image.width() > 0) && (image.height() > 0)) {
+            image = image.scaled(_pixelSize, _pixelSize, Qt::KeepAspectRatio);
+            image = image.mirrored();
+
+            _texture = gpu::Texture::createStrict(
+                    gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA),
+                    image.width(), image.height(),
+                    gpu::Texture::MAX_NUM_MIPS,
+                    gpu::Sampler(gpu::Sampler::FILTER_MIN_MAG_MIP_LINEAR));
+            _texture->setSource(iconPath.toStdString());
+            auto usage = gpu::Texture::Usage::Builder().withColor().withAlpha();
+            _texture->setUsage(usage.build());
+            _texture->setStoredMipFormat(gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA));
+            _texture->assignStoredMip(0, image.byteCount(), image.constBits());
+            _texture->setAutoGenerateMips(true);
+        }
+    }
+}
+
+void Basic2DWindowOpenGLDisplayPlugin::VirtualPadButton::draw(gpu::Batch &batch,
+                                                              glm::vec2 buttonPosition) {
+    auto transform = DependencyManager::get<CompositorHelper>()->getPoint2DTransform(
+            buttonPosition,
+            _pixelSize, _pixelSize);
+    batch.setResourceTexture(0, _texture);
+    batch.setModelTransform(transform);
+    batch.draw(gpu::TRIANGLE_STRIP, 4);
+}
+
+#endif

libraries/display-plugins/src/display-plugins/Basic2DWindowOpenGLDisplayPlugin.h

@@ -9,6 +9,10 @@
 
 #include "OpenGLDisplayPlugin.h"
 
+#if defined(Q_OS_ANDROID)
+#include "VirtualPadManager.h"
+#endif
+
 const float TARGET_FRAMERATE_Basic2DWindowOpenGL = 60.0f;
 
 class QScreen;
@@ -51,5 +55,23 @@ private:
 
     gpu::TexturePointer _virtualPadJumpBtnTexture;
     qreal _virtualPadJumpBtnPixelSize;
+
+    gpu::TexturePointer _virtualPadRbBtnTexture;
+    qreal _virtualPadRbBtnPixelSize;
+
+    class VirtualPadButton {
+    public:
+
+        VirtualPadButton() {}
+        VirtualPadButton(qreal pixelSize, QString iconPath, VirtualPad::Manager::Button button);
+
+        void draw(gpu::Batch& batch, glm::vec2 buttonPosition);
+
+        gpu::TexturePointer _texture;
+        qreal _pixelSize;
+        VirtualPad::Manager::Button _button;
+    };
+    QVector<VirtualPadButton> _virtualPadButtons;
+
 #endif
 };

libraries/entities-renderer/src/EntityTreeRenderer.cpp

@@ -219,7 +219,6 @@ void EntityTreeRenderer::clearNonLocalEntities() {
 
     std::unordered_map<EntityItemID, EntityRendererPointer> savedEntities;
     // remove all entities from the scene
-    _space->clear();
     auto scene = _viewState->getMain3DScene();
     if (scene) {
         render::Transaction transaction;
@@ -259,8 +258,6 @@ void EntityTreeRenderer::clear() {
         resetEntitiesScriptEngine();
     }
     // remove all entities from the scene
-
-    _space->clear();
     auto scene = _viewState->getMain3DScene();
     if (scene) {
         render::Transaction transaction;
@@ -1392,4 +1389,4 @@ bool EntityTreeRenderer::removeMaterialFromAvatar(const QUuid& avatarID, graphic
         return _removeMaterialFromAvatarOperator(avatarID, material, parentMaterialName);
     }
     return false;
-}
+}

libraries/entities-renderer/src/RenderableEntityItem.cpp

@@ -320,6 +320,7 @@ bool EntityRenderer::addToScene(const ScenePointer& scene, Transaction& transact
     transaction.resetItem(_renderItemID, renderPayload);
     onAddToScene(_entity);
     updateInScene(scene, transaction);
+    _entity->bumpAncestorChainRenderableVersion();
     return true;
 }
 
@@ -327,6 +328,7 @@ void EntityRenderer::removeFromScene(const ScenePointer& scene, Transaction& tra
     onRemoveFromScene(_entity);
     transaction.removeItem(_renderItemID);
     Item::clearID(_renderItemID);
+    _entity->bumpAncestorChainRenderableVersion();
 }
 
 void EntityRenderer::updateInScene(const ScenePointer& scene, Transaction& transaction) {
@@ -352,14 +354,6 @@ void EntityRenderer::updateInScene(const ScenePointer& scene, Transaction& trans
     });
 }
 
-void EntityRenderer::clearSubRenderItemIDs() {
-    _subRenderItemIDs.clear();
-}
-
-void EntityRenderer::setSubRenderItemIDs(const render::ItemIDs& ids) {
-    _subRenderItemIDs = ids;
-}
-
 //
 // Internal methods
 //

libraries/entities-renderer/src/RenderableEntityItem.h

@@ -52,9 +52,6 @@ public:
     virtual bool addToScene(const ScenePointer& scene, Transaction& transaction) final;
     virtual void removeFromScene(const ScenePointer& scene, Transaction& transaction);
 
-    void clearSubRenderItemIDs();
-    void setSubRenderItemIDs(const render::ItemIDs& ids);
-
     const uint64_t& getUpdateTime() const { return _updateTime; }
 
     virtual void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName);
@@ -64,6 +61,9 @@ public:
 
     static glm::vec4 calculatePulseColor(const glm::vec4& color, const PulsePropertyGroup& pulseProperties, quint64 start);
 
+    virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) override;
+    virtual Item::Bound getBound() override;
+
 protected:
     virtual bool needsRenderUpdateFromEntity() const final { return needsRenderUpdateFromEntity(_entity); }
     virtual void onAddToScene(const EntityItemPointer& entity);
@@ -75,9 +75,7 @@ protected:
     // Implementing the PayloadProxyInterface methods
     virtual ItemKey getKey() override;
     virtual ShapeKey getShapeKey() override;
-    virtual Item::Bound getBound() override;
     virtual void render(RenderArgs* args) override final;
-    virtual uint32_t metaFetchMetaSubItems(ItemIDs& subItems) override;
     virtual render::hifi::Tag getTagMask() const;
     virtual render::hifi::Layer getHifiRenderLayer() const;
 
@@ -133,7 +131,6 @@ protected:
     SharedSoundPointer _collisionSound;
     QUuid _changeHandlerId;
     ItemID _renderItemID{ Item::INVALID_ITEM_ID };
-    ItemIDs _subRenderItemIDs;
     uint64_t _fadeStartTime{ usecTimestampNow() };
     uint64_t _updateTime{ usecTimestampNow() }; // used when sorting/throttling render updates
     bool _isFading { EntityTreeRenderer::getEntitiesShouldFadeFunction()() };

libraries/entities-renderer/src/RenderableImageEntityItem.cpp

@@ -170,7 +170,7 @@ void ImageEntityRenderer::doRender(RenderArgs* args) {
     Q_ASSERT(args->_batch);
     gpu::Batch* batch = args->_batch;
 
-    transform.setRotation(EntityItem::getBillboardRotation(transform.getTranslation(), transform.getRotation(), _billboardMode));
+    transform.setRotation(EntityItem::getBillboardRotation(transform.getTranslation(), transform.getRotation(), _billboardMode, args->getViewFrustum().getPosition()));
     transform.postScale(dimensions);
 
     batch->setModelTransform(transform);

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -1079,7 +1079,7 @@ render::hifi::Tag ModelEntityRenderer::getTagMask() const {
 
 uint32_t ModelEntityRenderer::metaFetchMetaSubItems(ItemIDs& subItems) { 
     if (_model) {
-        auto metaSubItems = _subRenderItemIDs;
+        auto metaSubItems = _model->fetchRenderItemIDs();
         subItems.insert(subItems.end(), metaSubItems.begin(), metaSubItems.end());
         return (uint32_t)metaSubItems.size();
     }
@@ -1321,11 +1321,8 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
     if (!_hasModel) {
         if (model) {
             model->removeFromScene(scene, transaction);
+            entity->bumpAncestorChainRenderableVersion();
             withWriteLock([&] { _model.reset(); });
-            transaction.updateItem<PayloadProxyInterface>(getRenderItemID(), [](PayloadProxyInterface& data) {
-                auto entityRenderer = static_cast<EntityRenderer*>(&data);
-                entityRenderer->clearSubRenderItemIDs();
-            });
             emit DependencyManager::get<scriptable::ModelProviderFactory>()->
                 modelRemovedFromScene(entity->getEntityItemID(), NestableType::Entity, _model);
         }
@@ -1442,12 +1439,7 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
             render::Item::Status::Getters statusGetters;
             makeStatusGetters(entity, statusGetters);
             model->addToScene(scene, transaction, statusGetters);
-
-            auto newRenderItemIDs{ model->fetchRenderItemIDs() };
-            transaction.updateItem<PayloadProxyInterface>(getRenderItemID(), [newRenderItemIDs](PayloadProxyInterface& data) {
-                auto entityRenderer = static_cast<EntityRenderer*>(&data);
-                entityRenderer->setSubRenderItemIDs(newRenderItemIDs);
-            });
+            entity->bumpAncestorChainRenderableVersion();
             processMaterials();
         }
     }

libraries/entities-renderer/src/RenderableTextEntityItem.cpp

@@ -181,7 +181,7 @@ void TextEntityRenderer::doRender(RenderArgs* args) {
     gpu::Batch& batch = *args->_batch;
 
     auto transformToTopLeft = modelTransform;
-    transformToTopLeft.setRotation(EntityItem::getBillboardRotation(transformToTopLeft.getTranslation(), transformToTopLeft.getRotation(), _billboardMode));
+    transformToTopLeft.setRotation(EntityItem::getBillboardRotation(transformToTopLeft.getTranslation(), transformToTopLeft.getRotation(), _billboardMode, args->getViewFrustum().getPosition()));
     transformToTopLeft.postTranslate(dimensions * glm::vec3(-0.5f, 0.5f, 0.0f)); // Go to the top left
     transformToTopLeft.setScale(1.0f); // Use a scale of one so that the text is not deformed
 

libraries/entities-renderer/src/RenderableWebEntityItem.cpp

@@ -323,7 +323,7 @@ void WebEntityRenderer::doRender(RenderArgs* args) {
     });
     batch.setResourceTexture(0, _texture);
 
-    transform.setRotation(EntityItem::getBillboardRotation(transform.getTranslation(), transform.getRotation(), _billboardMode));
+    transform.setRotation(EntityItem::getBillboardRotation(transform.getTranslation(), transform.getRotation(), _billboardMode, args->getViewFrustum().getPosition()));
     batch.setModelTransform(transform);
 
     // Turn off jitter for these entities

libraries/entities/src/EntityItem.cpp

@@ -49,7 +49,8 @@ int EntityItem::_maxActionsDataSize = 800;
 quint64 EntityItem::_rememberDeletedActionTime = 20 * USECS_PER_SECOND;
 QString EntityItem::_marketplacePublicKey;
 
-std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode)> EntityItem::_getBillboardRotationOperator = [](const glm::vec3&, const glm::quat& rotation, BillboardMode) { return rotation; };
+std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode, const glm::vec3&)> EntityItem::_getBillboardRotationOperator = [](const glm::vec3&, const glm::quat& rotation, BillboardMode, const glm::vec3&) { return rotation; };
+std::function<glm::vec3()> EntityItem::_getPrimaryViewFrustumPositionOperator = []() { return glm::vec3(0.0f); };
 
 EntityItem::EntityItem(const EntityItemID& entityItemID) :
     SpatiallyNestable(NestableType::Entity, entityItemID)
@@ -266,7 +267,7 @@ OctreeElement::AppendState EntityItem::appendEntityData(OctreePacketData* packet
         APPEND_ENTITY_PROPERTY(PROP_HREF, getHref());
         APPEND_ENTITY_PROPERTY(PROP_DESCRIPTION, getDescription());
         APPEND_ENTITY_PROPERTY(PROP_POSITION, getLocalPosition());
-        APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, getUnscaledDimensions());
+        APPEND_ENTITY_PROPERTY(PROP_DIMENSIONS, getScaledDimensions());
         APPEND_ENTITY_PROPERTY(PROP_ROTATION, getLocalOrientation());
         APPEND_ENTITY_PROPERTY(PROP_REGISTRATION_POINT, getRegistrationPoint());
         APPEND_ENTITY_PROPERTY(PROP_CREATED, getCreated());
@@ -818,7 +819,7 @@ int EntityItem::readEntityDataFromBuffer(const unsigned char* data, int bytesLef
         };
         READ_ENTITY_PROPERTY(PROP_POSITION, glm::vec3, customUpdatePositionFromNetwork);
     }
-    READ_ENTITY_PROPERTY(PROP_DIMENSIONS, glm::vec3, setUnscaledDimensions);
+    READ_ENTITY_PROPERTY(PROP_DIMENSIONS, glm::vec3, setScaledDimensions);
     {   // See comment above
         auto customUpdateRotationFromNetwork = [this, shouldUpdate, lastEdited](glm::quat value) {
             if (shouldUpdate(_lastUpdatedRotationTimestamp, value != _lastUpdatedRotationValue)) {
@@ -1315,7 +1316,7 @@ EntityItemProperties EntityItem::getProperties(const EntityPropertyFlags& desire
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(href, getHref);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(description, getDescription);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(position, getLocalPosition);
-    COPY_ENTITY_PROPERTY_TO_PROPERTIES(dimensions, getUnscaledDimensions);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(dimensions, getScaledDimensions);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(rotation, getLocalOrientation);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(registrationPoint, getRegistrationPoint);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(created, getCreated);
@@ -1462,7 +1463,7 @@ bool EntityItem::setProperties(const EntityItemProperties& properties) {
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(href, setHref);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(description, setDescription);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(position, setPosition);
-    SET_ENTITY_PROPERTY_FROM_PROPERTIES(dimensions, setUnscaledDimensions);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(dimensions, setScaledDimensions);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(rotation, setRotation);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(registrationPoint, setRegistrationPoint);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(created, setCreated);
@@ -1872,7 +1873,7 @@ glm::vec3 EntityItem::getScaledDimensions() const {
 
 void EntityItem::setScaledDimensions(const glm::vec3& value) {
     glm::vec3 parentScale = getSNScale();
-    setUnscaledDimensions(value * parentScale);
+    setUnscaledDimensions(value / parentScale);
 }
 
 void EntityItem::setUnscaledDimensions(const glm::vec3& value) {
@@ -2935,6 +2936,7 @@ void EntityItem::setVisible(bool value) {
     });
 
     if (changed) {
+        bumpAncestorChainRenderableVersion();
         emit requestRenderUpdate();
     }
 }

libraries/entities/src/EntityItem.h

@@ -557,8 +557,10 @@ public:
     virtual void removeGrab(GrabPointer grab) override;
     virtual void disableGrab(GrabPointer grab) override;
 
-    static void setBillboardRotationOperator(std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode)> getBillboardRotationOperator) { _getBillboardRotationOperator = getBillboardRotationOperator; }
-    static glm::quat getBillboardRotation(const glm::vec3& position, const glm::quat& rotation, BillboardMode billboardMode) { return _getBillboardRotationOperator(position, rotation, billboardMode); }
+    static void setBillboardRotationOperator(std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode, const glm::vec3&)> getBillboardRotationOperator) { _getBillboardRotationOperator = getBillboardRotationOperator; }
+    static glm::quat getBillboardRotation(const glm::vec3& position, const glm::quat& rotation, BillboardMode billboardMode, const glm::vec3& frustumPos) { return _getBillboardRotationOperator(position, rotation, billboardMode, frustumPos); }
+    static void setPrimaryViewFrustumPositionOperator(std::function<glm::vec3()> getPrimaryViewFrustumPositionOperator) { _getPrimaryViewFrustumPositionOperator = getPrimaryViewFrustumPositionOperator; }
+    static glm::vec3 getPrimaryViewFrustumPosition() { return _getPrimaryViewFrustumPositionOperator(); }
 
 signals:
     void requestRenderUpdate();
@@ -748,7 +750,8 @@ protected:
     QHash<QUuid, EntityDynamicPointer> _grabActions;
 
 private:
-    static std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode)> _getBillboardRotationOperator;
+    static std::function<glm::quat(const glm::vec3&, const glm::quat&, BillboardMode, const glm::vec3&)> _getBillboardRotationOperator;
+    static std::function<glm::vec3()> _getPrimaryViewFrustumPositionOperator;
 };
 
 #endif // hifi_EntityItem_h

libraries/entities/src/EntityItemProperties.cpp

@@ -580,6 +580,7 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const {
 
     // Model
     CHECK_PROPERTY_CHANGE(PROP_MODEL_URL, modelURL);
+    CHECK_PROPERTY_CHANGE(PROP_MODEL_SCALE, modelScale);
     CHECK_PROPERTY_CHANGE(PROP_JOINT_ROTATIONS_SET, jointRotationsSet);
     CHECK_PROPERTY_CHANGE(PROP_JOINT_ROTATIONS, jointRotations);
     CHECK_PROPERTY_CHANGE(PROP_JOINT_TRANSLATIONS_SET, jointTranslationsSet);
@@ -1012,6 +1013,7 @@ EntityPropertyFlags EntityItemProperties::getChangedProperties() const {
  * @property {Vec3} dimensions=0.1,0.1,0.1 - The dimensions of the entity. When adding an entity, if no <code>dimensions</code> 
  *     value is specified then the model is automatically sized to its 
  *     <code>{@link Entities.EntityProperties|naturalDimensions}</code>.
+ * @property {Vec3} modelScale - The scale factor applied to the model's dimensions.  Deprecated.
  * @property {Color} color=255,255,255 - <em>Currently not used.</em>
  * @property {string} modelURL="" - The URL of the FBX of OBJ model. Baked FBX models' URLs end in ".baked.fbx".<br />
  * @property {string} textures="" - A JSON string of texture name, URL pairs used when rendering the model in place of the
@@ -1683,6 +1685,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_TEXTURES, textures);
 
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_MODEL_URL, modelURL);
+        COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_MODEL_SCALE, modelScale);
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS_SET, jointRotationsSet);
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_ROTATIONS, jointRotations);
         COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_JOINT_TRANSLATIONS_SET, jointTranslationsSet);
@@ -2078,6 +2081,7 @@ void EntityItemProperties::copyFromScriptValue(const QScriptValue& object, bool
 
     // Model
     COPY_PROPERTY_FROM_QSCRIPTVALUE(modelURL, QString, setModelURL);
+    COPY_PROPERTY_FROM_QSCRIPTVALUE(modelScale, vec3, setModelScale);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(jointRotationsSet, qVectorBool, setJointRotationsSet);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(jointRotations, qVectorQuat, setJointRotations);
     COPY_PROPERTY_FROM_QSCRIPTVALUE(jointTranslationsSet, qVectorBool, setJointTranslationsSet);
@@ -2357,6 +2361,7 @@ void EntityItemProperties::merge(const EntityItemProperties& other) {
 
     // Model
     COPY_PROPERTY_IF_CHANGED(modelURL);
+    COPY_PROPERTY_IF_CHANGED(modelScale);
     COPY_PROPERTY_IF_CHANGED(jointRotationsSet);
     COPY_PROPERTY_IF_CHANGED(jointRotations);
     COPY_PROPERTY_IF_CHANGED(jointTranslationsSet);
@@ -2700,6 +2705,7 @@ bool EntityItemProperties::getPropertyInfo(const QString& propertyName, EntityPr
 
         // Model
         ADD_PROPERTY_TO_MAP(PROP_MODEL_URL, ModelURL, modelURL, QString);
+        ADD_PROPERTY_TO_MAP(PROP_MODEL_SCALE, ModelScale, modelScale, vec3);
         ADD_PROPERTY_TO_MAP(PROP_JOINT_ROTATIONS_SET, JointRotationsSet, jointRotationsSet, QVector<bool>);
         ADD_PROPERTY_TO_MAP(PROP_JOINT_ROTATIONS, JointRotations, jointRotations, QVector<quat>);
         ADD_PROPERTY_TO_MAP(PROP_JOINT_TRANSLATIONS_SET, JointTranslationsSet, jointTranslationsSet, QVector<bool>);
@@ -3989,6 +3995,7 @@ void EntityItemProperties::markAllChanged() {
 
     // Model
     _modelURLChanged = true;
+    _modelScaleChanged = true;
     _jointRotationsSetChanged = true;
     _jointRotationsChanged = true;
     _jointTranslationsSetChanged = true;
@@ -4526,6 +4533,9 @@ QList<QString> EntityItemProperties::listChangedProperties() {
     if (modelURLChanged()) {
         out += "modelURL";
     }
+    if (modelScaleChanged()) {
+        out += "scale";
+    }
     if (jointRotationsSetChanged()) {
         out += "jointRotationsSet";
     }

libraries/entities/src/EntityItemProperties.h

@@ -279,6 +279,7 @@ public:
 
     // Model
     DEFINE_PROPERTY_REF(PROP_MODEL_URL, ModelURL, modelURL, QString, "");
+    DEFINE_PROPERTY_REF(PROP_MODEL_SCALE, ModelScale, modelScale, glm::vec3, glm::vec3(1.0f));
     DEFINE_PROPERTY_REF(PROP_JOINT_ROTATIONS_SET, JointRotationsSet, jointRotationsSet, QVector<bool>, QVector<bool>());
     DEFINE_PROPERTY_REF(PROP_JOINT_ROTATIONS, JointRotations, jointRotations, QVector<glm::quat>, QVector<glm::quat>());
     DEFINE_PROPERTY_REF(PROP_JOINT_TRANSLATIONS_SET, JointTranslationsSet, jointTranslationsSet, QVector<bool>, QVector<bool>());

libraries/entities/src/EntityPropertyFlags.h

@@ -202,22 +202,23 @@ enum EntityPropertyList {
 
     // Model
     PROP_MODEL_URL = PROP_DERIVED_0,
-    PROP_JOINT_ROTATIONS_SET = PROP_DERIVED_1,
-    PROP_JOINT_ROTATIONS = PROP_DERIVED_2,
-    PROP_JOINT_TRANSLATIONS_SET = PROP_DERIVED_3,
-    PROP_JOINT_TRANSLATIONS = PROP_DERIVED_4,
-    PROP_RELAY_PARENT_JOINTS = PROP_DERIVED_5,
-    PROP_GROUP_CULLED = PROP_DERIVED_6,
+    PROP_MODEL_SCALE = PROP_DERIVED_1,
+    PROP_JOINT_ROTATIONS_SET = PROP_DERIVED_2,
+    PROP_JOINT_ROTATIONS = PROP_DERIVED_3,
+    PROP_JOINT_TRANSLATIONS_SET = PROP_DERIVED_4,
+    PROP_JOINT_TRANSLATIONS = PROP_DERIVED_5,
+    PROP_RELAY_PARENT_JOINTS = PROP_DERIVED_6,
+    PROP_GROUP_CULLED = PROP_DERIVED_7,
     // Animation
-    PROP_ANIMATION_URL = PROP_DERIVED_7,
-    PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_8,
-    PROP_ANIMATION_FPS = PROP_DERIVED_9,
-    PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_10,
-    PROP_ANIMATION_PLAYING = PROP_DERIVED_11,
-    PROP_ANIMATION_LOOP = PROP_DERIVED_12,
-    PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_13,
-    PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_14,
-    PROP_ANIMATION_HOLD = PROP_DERIVED_15,
+    PROP_ANIMATION_URL = PROP_DERIVED_8,
+    PROP_ANIMATION_ALLOW_TRANSLATION = PROP_DERIVED_9,
+    PROP_ANIMATION_FPS = PROP_DERIVED_10,
+    PROP_ANIMATION_FRAME_INDEX = PROP_DERIVED_11,
+    PROP_ANIMATION_PLAYING = PROP_DERIVED_12,
+    PROP_ANIMATION_LOOP = PROP_DERIVED_13,
+    PROP_ANIMATION_FIRST_FRAME = PROP_DERIVED_14,
+    PROP_ANIMATION_LAST_FRAME = PROP_DERIVED_15,
+    PROP_ANIMATION_HOLD = PROP_DERIVED_16,
 
     // Light
     PROP_IS_SPOTLIGHT = PROP_DERIVED_0,

libraries/entities/src/EntityScriptingInterface.cpp

@@ -941,7 +941,7 @@ QUuid EntityScriptingInterface::editEntity(const QUuid& id, const EntityItemProp
                 auto nestable = nestableWP.lock();
                 if (nestable) {
                     NestableType nestableType = nestable->getNestableType();
-                    if (nestableType == NestableType::Overlay || nestableType == NestableType::Avatar) {
+                    if (nestableType == NestableType::Avatar) {
                         qCWarning(entities) << "attempted edit on non-entity: " << id << nestable->getName();
                         return QUuid(); // null script value to indicate failure
                     }

libraries/entities/src/ImageEntityItem.cpp

@@ -159,7 +159,7 @@ bool ImageEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const
     glm::vec2 xyDimensions(dimensions.x, dimensions.y);
     glm::quat rotation = getWorldOrientation();
     glm::vec3 position = getWorldPosition() + rotation * (dimensions * (ENTITY_ITEM_DEFAULT_REGISTRATION_POINT - getRegistrationPoint()));
-    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode);
+    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode, EntityItem::getPrimaryViewFrustumPosition());
 
     if (findRayRectangleIntersection(origin, direction, rotation, position, xyDimensions, distance)) {
         glm::vec3 forward = rotation * Vectors::FRONT;

libraries/entities/src/ModelEntityItem.cpp

@@ -63,6 +63,7 @@ EntityItemProperties ModelEntityItem::getProperties(const EntityPropertyFlags& d
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(textures, getTextures);
 
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(modelURL, getModelURL);
+    COPY_ENTITY_PROPERTY_TO_PROPERTIES(modelScale, getModelScale);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointRotationsSet, getJointRotationsSet);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointRotations, getJointRotations);
     COPY_ENTITY_PROPERTY_TO_PROPERTIES(jointTranslationsSet, getJointTranslationsSet);
@@ -85,6 +86,7 @@ bool ModelEntityItem::setProperties(const EntityItemProperties& properties) {
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(textures, setTextures);
 
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(modelURL, setModelURL);
+    SET_ENTITY_PROPERTY_FROM_PROPERTIES(modelScale, setModelScale);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointRotationsSet, setJointRotationsSet);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointRotations, setJointRotations);
     SET_ENTITY_PROPERTY_FROM_PROPERTIES(jointTranslationsSet, setJointTranslationsSet);
@@ -128,6 +130,7 @@ int ModelEntityItem::readEntitySubclassDataFromBuffer(const unsigned char* data,
     READ_ENTITY_PROPERTY(PROP_TEXTURES, QString, setTextures);
 
     READ_ENTITY_PROPERTY(PROP_MODEL_URL, QString, setModelURL);
+    READ_ENTITY_PROPERTY(PROP_MODEL_SCALE, glm::vec3, setModelScale);
     READ_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS_SET, QVector<bool>, setJointRotationsSet);
     READ_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS, QVector<glm::quat>, setJointRotations);
     READ_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS_SET, QVector<bool>, setJointTranslationsSet);
@@ -165,6 +168,7 @@ EntityPropertyFlags ModelEntityItem::getEntityProperties(EncodeBitstreamParams&
     requestedProperties += PROP_TEXTURES;
 
     requestedProperties += PROP_MODEL_URL;
+    requestedProperties += PROP_MODEL_SCALE;
     requestedProperties += PROP_JOINT_ROTATIONS_SET;
     requestedProperties += PROP_JOINT_ROTATIONS;
     requestedProperties += PROP_JOINT_TRANSLATIONS_SET;
@@ -192,6 +196,7 @@ void ModelEntityItem::appendSubclassData(OctreePacketData* packetData, EncodeBit
     APPEND_ENTITY_PROPERTY(PROP_TEXTURES, getTextures());
 
     APPEND_ENTITY_PROPERTY(PROP_MODEL_URL, getModelURL());
+    APPEND_ENTITY_PROPERTY(PROP_MODEL_SCALE, getModelScale());
     APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS_SET, getJointRotationsSet());
     APPEND_ENTITY_PROPERTY(PROP_JOINT_ROTATIONS, getJointRotations());
     APPEND_ENTITY_PROPERTY(PROP_JOINT_TRANSLATIONS_SET, getJointTranslationsSet());
@@ -708,3 +713,15 @@ bool ModelEntityItem::applyNewAnimationProperties(AnimationPropertyGroup newProp
     }
     return somethingChanged;
 }
+
+glm::vec3 ModelEntityItem::getModelScale() const {
+    return _modelScaleLock.resultWithReadLock<glm::vec3>([&] {
+        return getSNScale();
+    });
+}
+
+void ModelEntityItem::setModelScale(const glm::vec3& modelScale) {
+    _modelScaleLock.withWriteLock([&] {
+        setSNScale(modelScale);
+    });
+}

libraries/entities/src/ModelEntityItem.h

@@ -126,6 +126,9 @@ public:
     QVector<glm::vec3> getJointTranslations() const;
     QVector<bool> getJointTranslationsSet() const;
 
+    glm::vec3 getModelScale() const;
+    void setModelScale(const glm::vec3& modelScale);
+
 private:
     void setAnimationSettings(const QString& value); // only called for old bitstream format
     bool applyNewAnimationProperties(AnimationPropertyGroup newProperties);
@@ -141,6 +144,7 @@ protected:
     // they aren't currently updated from data in the model/rig, and they don't have a direct effect
     // on what's rendered.
     ReadWriteLockable _jointDataLock;
+    ReadWriteLockable _modelScaleLock;
 
     bool _jointRotationsExplicitlySet { false }; // were the joints set as a property or just side effect of animations
     bool _jointTranslationsExplicitlySet{ false }; // were the joints set as a property or just side effect of animations

libraries/entities/src/TextEntityItem.cpp

@@ -199,7 +199,7 @@ bool TextEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const
     glm::vec2 xyDimensions(dimensions.x, dimensions.y);
     glm::quat rotation = getWorldOrientation();
     glm::vec3 position = getWorldPosition() + rotation * (dimensions * (ENTITY_ITEM_DEFAULT_REGISTRATION_POINT - getRegistrationPoint()));
-    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode);
+    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode, EntityItem::getPrimaryViewFrustumPosition());
 
     if (findRayRectangleIntersection(origin, direction, rotation, position, xyDimensions, distance)) {
         glm::vec3 forward = rotation * Vectors::FRONT;

libraries/entities/src/WebEntityItem.cpp

@@ -180,7 +180,7 @@ bool WebEntityItem::findDetailedRayIntersection(const glm::vec3& origin, const g
     glm::vec2 xyDimensions(dimensions.x, dimensions.y);
     glm::quat rotation = getWorldOrientation();
     glm::vec3 position = getWorldPosition() + rotation * (dimensions * (ENTITY_ITEM_DEFAULT_REGISTRATION_POINT - getRegistrationPoint()));
-    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode);
+    rotation = EntityItem::getBillboardRotation(position, rotation, _billboardMode, EntityItem::getPrimaryViewFrustumPosition());
 
     if (findRayRectangleIntersection(origin, direction, rotation, position, xyDimensions, distance)) {
         glm::vec3 forward = rotation * Vectors::FRONT;

libraries/gpu/src/gpu/FrameReader.cpp

@@ -388,10 +388,23 @@ ShaderPointer Deserializer::readShader(const json& node) {
         return nullptr;
     }
 
+    static std::map<std::string, uint32_t> shadersIdsByName;
+    if (shadersIdsByName.empty()) {
+        for (const auto id : shader::allShaders()) {
+            const auto& shaderSource = shader::Source::get(id);
+            shadersIdsByName[shaderSource.name] = id;
+        }
+    }
+
     // FIXME support procedural shaders
     Shader::Type type = node[keys::type];
     std::string name = node[keys::name];
-    uint32_t id = node[keys::id];
+    // Using the serialized ID is bad, because it's generated at 
+    // cmake time, and can change across platforms or when 
+    // shaders are added or removed
+    // uint32_t id = node[keys::id];
+    
+    uint32_t id = shadersIdsByName[name];
     ShaderPointer result;
     switch (type) {
         //case Shader::Type::GEOMETRY:

libraries/gpu/src/gpu/Transform.slh

@@ -167,7 +167,7 @@ TransformObject getTransformObject() {
         vec4 eyeClipEdge[2]= vec4[2](vec4(-1,0,0,1), vec4(1,0,0,1));
         vec2 eyeOffsetScale = vec2(-0.5, +0.5);
         uint eyeIndex = uint(_stereoSide);
-#ifndef GPU_GLES
+#if !defined(GPU_GLES) || (defined(HAVE_EXT_clip_cull_distance) && !defined(VULKAN))
         gl_ClipDistance[0] = dot(<$clipPos$>, eyeClipEdge[eyeIndex]);
 #endif
         float newClipPosX = <$clipPos$>.x * 0.5 + eyeOffsetScale[eyeIndex] * <$clipPos$>.w;

libraries/input-plugins/src/input-plugins/TouchscreenVirtualPadDevice.cpp

@@ -66,7 +66,7 @@ void TouchscreenVirtualPadDevice::resize() {
         _fixedRadius = _screenDPI * 0.5f * VirtualPad::Manager::BASE_DIAMETER_PIXELS / VirtualPad::Manager::DPI;
         _fixedRadiusForCalc = _fixedRadius - _screenDPI * VirtualPad::Manager::STICK_RADIUS_PIXELS / VirtualPad::Manager::DPI;
 
-        _jumpButtonRadius = _screenDPI * VirtualPad::Manager::JUMP_BTN_TRIMMED_RADIUS_PIXELS / VirtualPad::Manager::DPI;
+        _buttonRadius = _screenDPI * VirtualPad::Manager::BTN_TRIMMED_RADIUS_PIXELS / VirtualPad::Manager::DPI;
     }
 
     auto& virtualPadManager = VirtualPad::Manager::instance();
@@ -86,11 +86,21 @@ void TouchscreenVirtualPadDevice::setupControlsPositions(VirtualPad::Manager& vi
     virtualPadManager.getLeftVirtualPad()->setFirstTouch(_moveRefTouchPoint);
 
     // Jump button
-    float jumpBtnPixelSize = _screenDPI * VirtualPad::Manager::JUMP_BTN_FULL_PIXELS / VirtualPad::Manager::DPI;
-    float rightMargin = _screenDPI * VirtualPad::Manager::JUMP_BTN_RIGHT_MARGIN_PIXELS / VirtualPad::Manager::DPI;
-    float bottomMargin = _screenDPI * VirtualPad::Manager::JUMP_BTN_BOTTOM_MARGIN_PIXELS/ VirtualPad::Manager::DPI;
-    _jumpButtonPosition = glm::vec2( eventScreen->availableSize().width() - rightMargin - jumpBtnPixelSize, eventScreen->availableSize().height() - bottomMargin - _jumpButtonRadius - _extraBottomMargin);
-    virtualPadManager.setJumpButtonPosition(_jumpButtonPosition);
+    float btnPixelSize = _screenDPI * VirtualPad::Manager::BTN_FULL_PIXELS / VirtualPad::Manager::DPI;
+    float rightMargin = _screenDPI * VirtualPad::Manager::BTN_RIGHT_MARGIN_PIXELS / VirtualPad::Manager::DPI;
+    float bottomMargin = _screenDPI * VirtualPad::Manager::BTN_BOTTOM_MARGIN_PIXELS/ VirtualPad::Manager::DPI;
+    glm::vec2 jumpButtonPosition = glm::vec2( eventScreen->availableSize().width() - rightMargin - btnPixelSize, eventScreen->availableSize().height() - bottomMargin - _buttonRadius - _extraBottomMargin);
+    glm::vec2 rbButtonPosition = glm::vec2( eventScreen->availableSize().width() - rightMargin - btnPixelSize, eventScreen->availableSize().height() - 2 * bottomMargin - 3 * _buttonRadius - _extraBottomMargin);
+
+    // Avoid generating buttons in portrait mode
+    if ( eventScreen->availableSize().width() > eventScreen->availableSize().height() && _buttonsManager.buttonsCount() == 0) {
+        _buttonsManager.addButton(TouchscreenButton(JUMP, JUMP_BUTTON, _buttonRadius, jumpButtonPosition, _inputDevice ));
+        _buttonsManager.addButton(TouchscreenButton(RB, RB_BUTTON, _buttonRadius, rbButtonPosition, _inputDevice ));
+
+        virtualPadManager.setButtonPosition(VirtualPad::Manager::Button::JUMP, jumpButtonPosition);
+        virtualPadManager.setButtonPosition(VirtualPad::Manager::Button::HANDSHAKE, rbButtonPosition);
+    }
+
 }
 
 float clip(float n, float lower, float upper) {
@@ -237,7 +247,7 @@ void TouchscreenVirtualPadDevice::touchEndEvent(const QTouchEvent* event) {
     if (!virtualPadManager.isEnabled() && !virtualPadManager.isHidden()) {
         moveTouchEnd();
         viewTouchEnd();
-        jumpTouchEnd();
+        _buttonsManager.endTouchForAll();
         return;
     }
     // touch end here is a big reset -> resets both pads
@@ -246,7 +256,7 @@ void TouchscreenVirtualPadDevice::touchEndEvent(const QTouchEvent* event) {
     debugPoints(event, " END ----------------");
     moveTouchEnd();
     viewTouchEnd();
-    jumpTouchEnd();
+    _buttonsManager.endTouchForAll();
     _inputDevice->_axisStateMap.clear();
     _inputDevice->_buttonPressedMap.clear();
 }
@@ -282,11 +292,11 @@ void TouchscreenVirtualPadDevice::touchUpdateEvent(const QTouchEvent* event) {
     const QList<QTouchEvent::TouchPoint>& tPoints = event->touchPoints();
     bool moveTouchFound = false;
     bool viewTouchFound = false;
-    bool jumpTouchFound = false;
 
     int idxMoveStartingPointCandidate = -1;
     int idxViewStartingPointCandidate = -1;
-    int idxJumpStartingPointCandidate = -1;
+
+    _buttonsManager.resetEventValues();
 
     glm::vec2 thisPoint;
     int thisPointId;
@@ -311,10 +321,7 @@ void TouchscreenVirtualPadDevice::touchUpdateEvent(const QTouchEvent* event) {
             continue;
         }
 
-        if (!jumpTouchFound && _jumpHasValidTouch && _jumpCurrentTouchId == thisPointId) {
-            // valid if it's an ongoing touch
-            jumpTouchFound = true;
-            jumpTouchUpdate(thisPoint);
+        if (_buttonsManager.processOngoingTouch(thisPoint, thisPointId)) {
             continue;
         }
 
@@ -330,18 +337,16 @@ void TouchscreenVirtualPadDevice::touchUpdateEvent(const QTouchEvent* event) {
             continue;
         }
 
-        if (!jumpTouchFound && idxJumpStartingPointCandidate == -1 && jumpTouchBeginIsValid(thisPoint) &&
-                (!_unusedTouches.count(thisPointId) || _unusedTouches[thisPointId] == JUMP )) {
-            idxJumpStartingPointCandidate = i;
+        if (_buttonsManager.findStartingTouchPointCandidate(thisPoint, thisPointId, i, _unusedTouches)) {
             continue;
         }
 
         if (moveTouchBeginIsValid(thisPoint)) {
             unusedTouchesInEvent[thisPointId] = MOVE;
-        } else if (jumpTouchBeginIsValid(thisPoint)) {
-            unusedTouchesInEvent[thisPointId] = JUMP;
         } else if (viewTouchBeginIsValid(thisPoint))  {
             unusedTouchesInEvent[thisPointId] = VIEW;
+        } else {
+            _buttonsManager.saveUnusedTouches(unusedTouchesInEvent, thisPoint, thisPointId);
         }
 
     }
@@ -370,24 +375,13 @@ void TouchscreenVirtualPadDevice::touchUpdateEvent(const QTouchEvent* event) {
             viewTouchEnd();
         }
     }
-    if (!jumpTouchFound) {
-        if (idxJumpStartingPointCandidate != -1) {
-            _jumpCurrentTouchId = tPoints[idxJumpStartingPointCandidate].id();
-            _unusedTouches.erase(_jumpCurrentTouchId);
-            thisPoint.x = tPoints[idxJumpStartingPointCandidate].pos().x();
-            thisPoint.y = tPoints[idxJumpStartingPointCandidate].pos().y();
-            jumpTouchBegin(thisPoint);
-        } else {
-            if (_jumpHasValidTouch) {
-                jumpTouchEnd();
-            }
-        }
-    }
+
+    _buttonsManager.processBeginOrEnd(thisPoint, tPoints, _unusedTouches);
 
 }
 
 bool TouchscreenVirtualPadDevice::viewTouchBeginIsValid(glm::vec2 touchPoint) {
-    return !moveTouchBeginIsValid(touchPoint) && !jumpTouchBeginIsValid(touchPoint);
+    return !moveTouchBeginIsValid(touchPoint) && _buttonsManager.touchBeginInvalidForAllButtons(touchPoint);
 }
 
 bool TouchscreenVirtualPadDevice::moveTouchBeginIsValid(glm::vec2 touchPoint) {
@@ -400,30 +394,6 @@ bool TouchscreenVirtualPadDevice::moveTouchBeginIsValid(glm::vec2 touchPoint) {
     }
 }
 
-bool TouchscreenVirtualPadDevice::jumpTouchBeginIsValid(glm::vec2 touchPoint) {
-    // position of button and boundaries
-    return glm::distance2(touchPoint, _jumpButtonPosition) < _jumpButtonRadius * _jumpButtonRadius;
-}
-
-void TouchscreenVirtualPadDevice::jumpTouchBegin(glm::vec2 touchPoint) {
-    auto& virtualPadManager = VirtualPad::Manager::instance();
-    if (virtualPadManager.isEnabled() && !virtualPadManager.isHidden()) {
-        _jumpHasValidTouch = true;
-
-        _inputDevice->_buttonPressedMap.insert(TouchButtonChannel::JUMP_BUTTON_PRESS);
-    }
-}
-
-void TouchscreenVirtualPadDevice::jumpTouchUpdate(glm::vec2 touchPoint) {}
-
-void TouchscreenVirtualPadDevice::jumpTouchEnd() {
-    if (_jumpHasValidTouch) {
-        _jumpHasValidTouch = false;
-
-        _inputDevice->_buttonPressedMap.erase(TouchButtonChannel::JUMP_BUTTON_PRESS);
-    }    
-}
-
 void TouchscreenVirtualPadDevice::moveTouchBegin(glm::vec2 touchPoint) {
     auto& virtualPadManager = VirtualPad::Manager::instance();
     if (virtualPadManager.isEnabled() && !virtualPadManager.isHidden()) {
@@ -498,7 +468,8 @@ controller::Input::NamedVector TouchscreenVirtualPadDevice::InputDevice::getAvai
         Input::NamedPair(makeInput(TouchAxisChannel::LY), "LY"),
         Input::NamedPair(makeInput(TouchAxisChannel::RX), "RX"),
         Input::NamedPair(makeInput(TouchAxisChannel::RY), "RY"),
-        Input::NamedPair(makeInput(TouchButtonChannel::JUMP_BUTTON_PRESS), "JUMP_BUTTON_PRESS")
+        Input::NamedPair(makeInput(TouchButtonChannel::JUMP), "JUMP_BUTTON_PRESS"),
+        Input::NamedPair(makeInput(TouchButtonChannel::RB), "RB")
     };
     return availableInputs;
 }
@@ -507,3 +478,146 @@ QString TouchscreenVirtualPadDevice::InputDevice::getDefaultMappingConfig() cons
     static const QString MAPPING_JSON = PathUtils::resourcesPath() + "/controllers/touchscreenvirtualpad.json";
     return MAPPING_JSON;
 }
+
+TouchscreenVirtualPadDevice::TouchscreenButton::TouchscreenButton(
+        TouchscreenVirtualPadDevice::TouchButtonChannel channelIn,
+        TouchscreenVirtualPadDevice::TouchType touchTypeIn, float buttonRadiusIn,
+        glm::vec2 buttonPositionIn, std::shared_ptr<InputDevice> inputDeviceIn) :
+    buttonPosition(buttonPositionIn),
+    buttonRadius(buttonRadiusIn),
+    touchType(touchTypeIn),
+    channel(channelIn),
+    _inputDevice(inputDeviceIn)
+{
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButton::touchBegin(glm::vec2 touchPoint) {
+    auto& virtualPadManager = VirtualPad::Manager::instance();
+    if (virtualPadManager.isEnabled() && !virtualPadManager.isHidden()) {
+        hasValidTouch = true;
+
+        _inputDevice->_buttonPressedMap.insert(channel);
+    }
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButton::touchUpdate(glm::vec2 touchPoint) {
+
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButton::touchEnd() {
+    if (hasValidTouch) {
+        hasValidTouch = false;
+
+        _inputDevice->_buttonPressedMap.erase(channel);
+    }
+}
+
+bool TouchscreenVirtualPadDevice::TouchscreenButton::touchBeginIsValid(glm::vec2 touchPoint) {
+    return glm::distance2(touchPoint, buttonPosition) < buttonRadius * buttonRadius;
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButton::resetEventValues() {
+    _candidatePointIdx = -1;
+    _found = false;
+}
+
+TouchscreenVirtualPadDevice::TouchscreenButtonsManager::TouchscreenButtonsManager() {}
+
+void TouchscreenVirtualPadDevice::TouchscreenButtonsManager::addButton(
+        TouchscreenVirtualPadDevice::TouchscreenButton button) {
+    buttons.push_back(button);
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButtonsManager::resetEventValues() {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        button.resetEventValues();
+    }
+}
+
+bool
+TouchscreenVirtualPadDevice::TouchscreenButtonsManager::processOngoingTouch(glm::vec2 thisPoint,
+                                                                            int thisPointId) {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+
+        if (!button._found && button.hasValidTouch && button.currentTouchId == thisPointId) {
+            // valid if it's an ongoing touch
+            button._found = true;
+            button.touchUpdate(thisPoint);
+            return true;
+        }
+    }
+    return false;
+
+}
+
+bool TouchscreenVirtualPadDevice::TouchscreenButtonsManager::findStartingTouchPointCandidate(
+        glm::vec2 thisPoint, int thisPointId, int thisPointIdx, std::map<int, TouchType> &globalUnusedTouches) {
+
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        if (!button._found && button._candidatePointIdx == -1 && button.touchBeginIsValid(thisPoint)) {
+            if (!globalUnusedTouches.count(thisPointId) ) {
+                button._candidatePointIdx = thisPointIdx;
+                return true;
+            } else if (globalUnusedTouches[thisPointId] == button.touchType) {
+                button._candidatePointIdx = thisPointIdx;
+                return true;
+            }
+        }
+    }
+    return false;
+
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButtonsManager::saveUnusedTouches(
+        std::map<int, TouchscreenVirtualPadDevice::TouchType> &unusedTouchesInEvent, glm::vec2 thisPoint,
+        int thisPointId) {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        if (button.touchBeginIsValid(thisPoint)) {
+            unusedTouchesInEvent[thisPointId] = button.touchType;
+            return;
+        }
+    }
+
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButtonsManager::processBeginOrEnd(
+        glm::vec2 thisPoint, const QList<QTouchEvent::TouchPoint> &tPoints, std::map<int, TouchType> globalUnusedTouches) {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        if (!button._found) {
+            if (button._candidatePointIdx != -1) {
+                button.currentTouchId = tPoints[button._candidatePointIdx].id();
+                globalUnusedTouches.erase(button.currentTouchId);
+                thisPoint.x = tPoints[button._candidatePointIdx].pos().x();
+                thisPoint.y = tPoints[button._candidatePointIdx].pos().y();
+                button.touchBegin(thisPoint);
+            } else {
+                if (button.hasValidTouch) {
+                    button.touchEnd();
+                }
+            }
+        }
+    }
+
+}
+
+void TouchscreenVirtualPadDevice::TouchscreenButtonsManager::endTouchForAll() {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        button.touchEnd();
+    }
+}
+
+bool TouchscreenVirtualPadDevice::TouchscreenButtonsManager::touchBeginInvalidForAllButtons(glm::vec2 touchPoint) {
+    for(int i = 0; i < buttons.size(); i++) {
+        TouchscreenButton &button = buttons[i];
+        if (button.touchBeginIsValid(touchPoint)) {
+            return false;
+        }
+    }
+    return true;
+}

libraries/input-plugins/src/input-plugins/TouchscreenVirtualPadDevice.h

@@ -15,6 +15,7 @@
 #include <controllers/InputDevice.h>
 #include "InputPlugin.h"
 #include <QtGui/qtouchdevice.h>
+#include <QtGui/QList>
 #include "VirtualPadManager.h"
 
 class QTouchEvent;
@@ -51,7 +52,8 @@ public:
     };
 
     enum TouchButtonChannel {
-        JUMP_BUTTON_PRESS
+        JUMP,
+        RB
     };
 
 protected:
@@ -82,7 +84,60 @@ protected:
     enum TouchType {
         MOVE = 1,
         VIEW,
-        JUMP
+        JUMP_BUTTON,
+        RB_BUTTON
+    };
+
+    class TouchscreenButton {
+    public:
+
+        TouchscreenButton() {};
+
+        TouchscreenButton(TouchButtonChannel channelIn, TouchType touchTypeIn, float buttonRadiusIn, glm::vec2 buttonPositionIn,
+                          std::shared_ptr<InputDevice> inputDeviceIn);
+
+        void touchBegin(glm::vec2 touchPoint);
+        void touchUpdate(glm::vec2 touchPoint);
+        void touchEnd();
+        bool touchBeginIsValid(glm::vec2 touchPoint);
+
+        bool hasValidTouch { false };
+        int currentTouchId;
+
+        // per event tmp values
+        int _candidatePointIdx { -1 };
+        bool _found { false };
+        void resetEventValues();
+
+        glm::vec2 buttonPosition;
+        float buttonRadius;
+        TouchType touchType;
+        TouchButtonChannel channel;
+
+        std::shared_ptr<InputDevice> _inputDevice;
+
+    };
+
+    class TouchscreenButtonsManager {
+    public:
+
+        TouchscreenButtonsManager();
+
+        QVector<TouchscreenButton> buttons;
+
+        void addButton(TouchscreenButton button);
+        int buttonsCount() {
+            return buttons.size();
+        }
+
+        void resetEventValues();
+        bool processOngoingTouch(glm::vec2 thisPoint, int thisPointId);
+        bool findStartingTouchPointCandidate(glm::vec2 thisPoint, int thisPointId, int thisPointIdx, std::map<int, TouchType> &globalUnusedTouches);
+        void saveUnusedTouches(std::map<int, TouchType> &unusedTouchesInEvent, glm::vec2 thisPoint, int thisPointId);
+        void processBeginOrEnd(glm::vec2 thisPoint, const QList<QTouchEvent::TouchPoint>& tPoints, std::map<int, TouchType> globalUnusedTouches);
+
+        void endTouchForAll();
+        bool touchBeginInvalidForAllButtons(glm::vec2 touchPoint);
     };
 
     float _lastPinchScale;
@@ -101,9 +156,6 @@ protected:
     glm::vec2 _viewCurrentTouchPoint;
     int _viewCurrentTouchId;
 
-    bool _jumpHasValidTouch;
-    int _jumpCurrentTouchId;
-
     std::map<int, TouchType> _unusedTouches;
 
     int _touchPointCount;
@@ -116,8 +168,9 @@ protected:
     float _fixedRadiusForCalc;
     int _extraBottomMargin {0};
 
-    glm::vec2 _jumpButtonPosition;
-    float _jumpButtonRadius;
+    float _buttonRadius;
+
+    TouchscreenButtonsManager _buttonsManager;
 
     void moveTouchBegin(glm::vec2 touchPoint);
     void moveTouchUpdate(glm::vec2 touchPoint);
@@ -129,11 +182,6 @@ protected:
     void viewTouchEnd();
     bool viewTouchBeginIsValid(glm::vec2 touchPoint);
 
-    void jumpTouchBegin(glm::vec2 touchPoint);
-    void jumpTouchUpdate(glm::vec2 touchPoint);
-    void jumpTouchEnd();
-    bool jumpTouchBeginIsValid(glm::vec2 touchPoint);
-
     void setupControlsPositions(VirtualPad::Manager& virtualPadManager, bool force = false);
 
     void processInputDeviceForMove(VirtualPad::Manager& virtualPadManager);

libraries/networking/src/udt/PacketHeaders.h

@@ -262,6 +262,7 @@ enum class EntityVersion : PacketVersion {
     RingGizmoEntities,
     ShowKeyboardFocusHighlight,
     WebBillboardMode,
+    ModelScale,
 
     // Add new versions above here
     NUM_PACKET_TYPE,

libraries/oculusMobile/src/ovr/VrHandler.cpp

@@ -315,10 +315,6 @@ JNIEXPORT void JNICALL Java_io_highfidelity_oculus_OculusMobileActivity_nativeOn
     SURFACE.onCreate(env, obj);
 }
 
-JNIEXPORT void JNICALL Java_io_highfidelity_oculus_OculusMobileActivity_nativeOnDestroy(JNIEnv*, jclass) {
-    __android_log_write(ANDROID_LOG_WARN, "QQQ_JNI", __FUNCTION__);
-}
-
 JNIEXPORT void JNICALL Java_io_highfidelity_oculus_OculusMobileActivity_nativeOnResume(JNIEnv*, jclass) {
     __android_log_write(ANDROID_LOG_WARN, "QQQ_JNI", __FUNCTION__);
     SURFACE.setResumed(true);

libraries/render-utils/src/CauterizedModel.cpp

@@ -178,6 +178,7 @@ void CauterizedModel::updateClusterMatrices() {
             }
         }
     }
+    computeMeshPartLocalBounds();
 
     // post the blender if we're not currently waiting for one to finish
     auto modelBlender = DependencyManager::get<ModelBlender>();

libraries/render-utils/src/Model.cpp

@@ -1169,6 +1169,7 @@ void Model::setURL(const QUrl& url) {
         resource->setLoadPriority(this, _loadingPriority);
         _renderWatcher.setResource(resource);
     }
+    _rig.initFlow(false);
     onInvalidate();
 }
 
@@ -1385,6 +1386,7 @@ void Model::updateClusterMatrices() {
             }
         }
     }
+    computeMeshPartLocalBounds();
 
     // post the blender if we're not currently waiting for one to finish
     auto modelBlender = DependencyManager::get<ModelBlender>();

libraries/render-utils/src/parabola_forward.slv

@@ -0,0 +1,7 @@
+layout(location=0) in vec4 _color;
+
+layout(location=0) out vec4 _fragColor0;
+
+void main(void) {
+    _fragColor0 = _color;
+}

libraries/shaders/headers/310es/header.glsl

@@ -1,6 +1,3 @@
-#version 310 es
-#define GPU_GLES
-#define GPU_GLES_310
 #define BITFIELD highp int
 #define LAYOUT(X) layout(X)
 #define LAYOUT_STD140(X) layout(std140, X)
@@ -9,6 +6,9 @@
     #define gl_VertexID  gl_VertexIndex
 #endif
 #extension GL_EXT_texture_buffer : enable
+#if defined(HAVE_EXT_clip_cull_distance) && !defined(VULKAN)
+#extension GL_EXT_clip_cull_distance : enable
+#endif
 precision highp float;
 precision highp samplerBuffer;
 precision highp sampler2DShadow;

libraries/shaders/headers/310es/version.glsl

@@ -0,0 +1,3 @@
+#version 310 es
+#define GPU_GLES
+#define GPU_GLES_310

libraries/shaders/headers/410/header.glsl

@@ -1,5 +1,3 @@
-#version 410 core
-#define GPU_GL410
 #define BITFIELD int
 #if defined(VULKAN)
   #extension GL_ARB_shading_language_420pack : require

libraries/shaders/headers/410/version.glsl

@@ -0,0 +1,2 @@
+#version 410 core
+#define GPU_GL410