From c530c530ef05862af210e3085465936c0522b4ea Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 20 May 2015 17:26:31 -0700 Subject: [PATCH 1/5] Make eyelids follow iris as head tilts - for DDE and no face tracking --- interface/src/avatar/Head.cpp | 58 ++++++++++++++++++++++++++--------- interface/src/avatar/Head.h | 1 + 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index e0cdef109e..8691243c07 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -93,22 +93,24 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (_isFaceTrackerConnected) { _blendshapeCoefficients = faceTracker->getBlendshapeCoefficients(); - if (typeid(*faceTracker) == typeid(DdeFaceTracker) - && Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { + if (typeid(*faceTracker) == typeid(DdeFaceTracker)) { - calculateMouthShapes(); + if (Menu::getInstance()->isOptionChecked(MenuOption::UseAudioForMouth)) { + calculateMouthShapes(); - const int JAW_OPEN_BLENDSHAPE = 21; - const int MMMM_BLENDSHAPE = 34; - const int FUNNEL_BLENDSHAPE = 40; - const int SMILE_LEFT_BLENDSHAPE = 28; - const int SMILE_RIGHT_BLENDSHAPE = 29; - _blendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; - _blendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; - _blendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; - _blendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; - _blendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; + const int JAW_OPEN_BLENDSHAPE = 21; + const int MMMM_BLENDSHAPE = 34; + const int FUNNEL_BLENDSHAPE = 40; + const int SMILE_LEFT_BLENDSHAPE = 28; + const int SMILE_RIGHT_BLENDSHAPE = 29; + _blendshapeCoefficients[JAW_OPEN_BLENDSHAPE] += _audioJawOpen; + _blendshapeCoefficients[SMILE_LEFT_BLENDSHAPE] += _mouth4; + _blendshapeCoefficients[SMILE_RIGHT_BLENDSHAPE] += _mouth4; + _blendshapeCoefficients[MMMM_BLENDSHAPE] += _mouth2; + _blendshapeCoefficients[FUNNEL_BLENDSHAPE] += _mouth3; + } + applyEyelidOffset(getFinalOrientationInWorldFrame()); } } } @@ -203,6 +205,9 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { _mouth3, _mouth4, _blendshapeCoefficients); + + applyEyelidOffset(getOrientation()); + } else { _saccade = glm::vec3(); } @@ -218,7 +223,6 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } } _eyePosition = calculateAverageEyePosition(); - } void Head::calculateMouthShapes() { @@ -249,6 +253,32 @@ void Head::calculateMouthShapes() { _mouth4 = glm::mix(_audioJawOpen, _mouth4, SMILE_PERIOD + randFloat() * SMILE_RANDOM_PERIOD); } +void Head::applyEyelidOffset(glm::quat headOrientation) { + // Adjusts the eyelid blendshape coefficients so that the eyelid follows the iris as the head pitches. + + glm::quat eyeRotation = rotationBetween(headOrientation * IDENTITY_FRONT, getCorrectedLookAtPosition() - _eyePosition); + eyeRotation = eyeRotation * glm::angleAxis(safeEulerAngles(headOrientation).y, IDENTITY_UP); // Rotation w.r.t. head + float eyePitch = safeEulerAngles(eyeRotation).x; + + const float EYE_PITCH_TO_COEFFICIENT = 1.6f; // Empirically determined + const float MAX_EYELID_OFFSET = 0.8f; // So that don't fully close eyes when looking way down + float eyelidOffset = glm::clamp(-eyePitch * EYE_PITCH_TO_COEFFICIENT, -1.0f, MAX_EYELID_OFFSET); + + for (int i = 0; i < 2; i++) { + const int LEFT_EYE = 8; + float eyeCoefficient = _blendshapeCoefficients[i] - _blendshapeCoefficients[LEFT_EYE + i]; // Raw value + eyeCoefficient = glm::clamp(eyelidOffset + eyeCoefficient * (1.0f - eyelidOffset), -1.0f, 1.0f); + if (eyeCoefficient > 0.0f) { + _blendshapeCoefficients[i] = eyeCoefficient; + _blendshapeCoefficients[LEFT_EYE + i] = 0.0f; + + } else { + _blendshapeCoefficients[i] = 0.0f; + _blendshapeCoefficients[LEFT_EYE + i] = -eyeCoefficient; + } + } +} + void Head::relaxLean(float deltaTime) { // restore rotation, lean to neutral positions const float LEAN_RELAXATION_PERIOD = 0.25f; // seconds diff --git a/interface/src/avatar/Head.h b/interface/src/avatar/Head.h index e523d958a6..6eba16824e 100644 --- a/interface/src/avatar/Head.h +++ b/interface/src/avatar/Head.h @@ -155,6 +155,7 @@ private: // private methods void renderLookatVectors(glm::vec3 leftEyePosition, glm::vec3 rightEyePosition, glm::vec3 lookatPosition); void calculateMouthShapes(); + void applyEyelidOffset(glm::quat headOrientation); friend class FaceModel; }; From 94131908fb856b2f60fc0b217c2a0d1404824e2e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 1 Jun 2015 14:53:11 -0700 Subject: [PATCH 2/5] update polyvox for xcode C++11 build --- cmake/externals/polyvox/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/externals/polyvox/CMakeLists.txt b/cmake/externals/polyvox/CMakeLists.txt index 76302f9b4a..28aec6dab7 100644 --- a/cmake/externals/polyvox/CMakeLists.txt +++ b/cmake/externals/polyvox/CMakeLists.txt @@ -3,8 +3,8 @@ set(EXTERNAL_NAME polyvox) include(ExternalProject) ExternalProject_Add( ${EXTERNAL_NAME} - URL http://hifi-public.s3.amazonaws.com/dependencies/polyvox-master-2015-5-27.zip - URL_MD5 e3dd09a24df4db29ba370e3bea753388 + URL http://hifi-public.s3.amazonaws.com/dependencies/polyvox.zip + URL_MD5 904b840328278c9b36fa7a14be730c34 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= BINARY_DIR ${EXTERNAL_PROJECT_PREFIX}/build LOG_DOWNLOAD 1 @@ -45,7 +45,7 @@ else () endif () -if (WIN32) +if (WIN32) set(${EXTERNAL_NAME_UPPER}_CORE_LIBRARY ${INSTALL_DIR}/PolyVoxCore/lib/PolyVoxCore.lib CACHE FILEPATH "polyvox core library") # set(${EXTERNAL_NAME_UPPER}_UTIL_LIBRARY ${INSTALL_DIR}/PolyVoxUtil/lib/PolyVoxUtil.lib CACHE FILEPATH "polyvox util library") elseif (APPLE) From 622a1e30688982863eb73ef1f5edc229c2f5db6f Mon Sep 17 00:00:00 2001 From: Howard Stearns Date: Mon, 1 Jun 2015 21:19:42 -0700 Subject: [PATCH 3/5] Pitch-shift collision sounds, and make entities-renderer depend on soxr. --- libraries/entities-renderer/CMakeLists.txt | 6 +++++ .../src/EntityTreeRenderer.cpp | 26 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/libraries/entities-renderer/CMakeLists.txt b/libraries/entities-renderer/CMakeLists.txt index a94bfb5f97..394d156252 100644 --- a/libraries/entities-renderer/CMakeLists.txt +++ b/libraries/entities-renderer/CMakeLists.txt @@ -17,4 +17,10 @@ find_package(PolyVox REQUIRED) target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${POLYVOX_INCLUDE_DIRS}) target_link_libraries(${TARGET_NAME} ${POLYVOX_LIBRARIES}) +# for changing the pitch of collision sounds +add_dependency_external_projects(soxr) +find_package(Soxr REQUIRED) +target_link_libraries(${TARGET_NAME} ${SOXR_LIBRARIES}) +target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${SOXR_INCLUDE_DIRS}) + link_hifi_libraries(shared gpu script-engine render-utils) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index 8f034409ae..e3680d4614 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "EntityTreeRenderer.h" @@ -1158,7 +1159,30 @@ void EntityTreeRenderer::playEntityCollisionSound(const QUuid& myNodeID, EntityT options.stereo = sound->isStereo(); options.position = position; options.volume = volume; - AudioInjector* injector = new AudioInjector(sound.data(), options); + + // Shift the pitch down by ln(1 + (size / COLLISION_SIZE_FOR_STANDARD_PITCH)) / ln(2) + const float COLLISION_SIZE_FOR_STANDARD_PITCH = 0.2f; + QByteArray samples = sound->getByteArray(); + soxr_io_spec_t spec = soxr_io_spec(SOXR_INT16_I, SOXR_INT16_I); + soxr_quality_spec_t qualitySpec = soxr_quality_spec(SOXR_MQ, 0); + const int channelCount = sound->isStereo() ? 2 : 1; + const float factor = log(1.0f + (entity->getMaximumAACube().getLargestDimension() / COLLISION_SIZE_FOR_STANDARD_PITCH)) / log(2); + const int HIFI_SAMPLE_RATE = 24000; //AudioConstants::SAMPLE_RATE + const int resampledRate = HIFI_SAMPLE_RATE * factor; + const int nInputSamples = samples.size() / sizeof(int16_t); + const int nOutputSamples = nInputSamples * factor; + QByteArray resampled(nOutputSamples * sizeof(int16_t), '\0'); + const int16_t* receivedSamples = reinterpret_cast(samples.data()); + soxr_error_t soxError = soxr_oneshot(HIFI_SAMPLE_RATE, resampledRate, channelCount, + receivedSamples, nInputSamples, NULL, + reinterpret_cast(resampled.data()), nOutputSamples, NULL, + &spec, &qualitySpec, 0); + if (soxError) { + qCDebug(entitiesrenderer) << "Unable to resample" << collisionSoundURL << "from" << nInputSamples << "@" << HIFI_SAMPLE_RATE << "to" << nOutputSamples << "@" << resampledRate; + resampled = samples; + } + + AudioInjector* injector = new AudioInjector(resampled, options); injector->setLocalAudioInterface(_localAudioInterface); injector->triggerDeleteAfterFinish(); QThread* injectorThread = new QThread(); From 0837c016c52237ca6dcffdac98b8ca4d514bf4af Mon Sep 17 00:00:00 2001 From: Howard Stearns Date: Tue, 2 Jun 2015 09:18:45 -0700 Subject: [PATCH 4/5] Update constant to use AudioConstants.h --- libraries/entities-renderer/src/EntityTreeRenderer.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index e3680d4614..23e504bfb8 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "EntityTreeRenderer.h" @@ -1167,18 +1168,18 @@ void EntityTreeRenderer::playEntityCollisionSound(const QUuid& myNodeID, EntityT soxr_quality_spec_t qualitySpec = soxr_quality_spec(SOXR_MQ, 0); const int channelCount = sound->isStereo() ? 2 : 1; const float factor = log(1.0f + (entity->getMaximumAACube().getLargestDimension() / COLLISION_SIZE_FOR_STANDARD_PITCH)) / log(2); - const int HIFI_SAMPLE_RATE = 24000; //AudioConstants::SAMPLE_RATE - const int resampledRate = HIFI_SAMPLE_RATE * factor; + const int standardRate = AudioConstants::SAMPLE_RATE; + const int resampledRate = standardRate * factor; const int nInputSamples = samples.size() / sizeof(int16_t); const int nOutputSamples = nInputSamples * factor; QByteArray resampled(nOutputSamples * sizeof(int16_t), '\0'); const int16_t* receivedSamples = reinterpret_cast(samples.data()); - soxr_error_t soxError = soxr_oneshot(HIFI_SAMPLE_RATE, resampledRate, channelCount, + soxr_error_t soxError = soxr_oneshot(standardRate, resampledRate, channelCount, receivedSamples, nInputSamples, NULL, reinterpret_cast(resampled.data()), nOutputSamples, NULL, &spec, &qualitySpec, 0); if (soxError) { - qCDebug(entitiesrenderer) << "Unable to resample" << collisionSoundURL << "from" << nInputSamples << "@" << HIFI_SAMPLE_RATE << "to" << nOutputSamples << "@" << resampledRate; + qCDebug(entitiesrenderer) << "Unable to resample" << collisionSoundURL << "from" << nInputSamples << "@" << standardRate << "to" << nOutputSamples << "@" << resampledRate; resampled = samples; } From c90114e1ef5a2e702e7e2040797539b58c70a102 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 2 Jun 2015 10:49:45 -0700 Subject: [PATCH 5/5] Update BUILD_OSX.md --- BUILD_OSX.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/BUILD_OSX.md b/BUILD_OSX.md index 2598507c90..60fbb7cf92 100644 --- a/BUILD_OSX.md +++ b/BUILD_OSX.md @@ -12,6 +12,12 @@ We have a [homebrew formulas repository](https://github.com/highfidelity/homebre *Our [qt5 homebrew formula](https://raw.github.com/highfidelity/homebrew-formulas/master/qt5.rb) is for a patched version of Qt 5.4.x stable that removes wireless network scanning that can reduce real-time audio performance. We recommended you use this formula to install Qt.* +###Qt + +Assuming you've installed Qt 5 using the homebrew instructions above, you'll need to set QT_CMAKE_PREFIX_PATH so CMake can find your installation of Qt. For Qt 5.4.1 installed via homebrew, set QT_CMAKE_PREFIX_PATH as follows. + + export QT_CMAKE_PREFIX_PATH=/usr/local/Cellar/qt5/5.4.1/lib/cmake + ###Xcode If Xcode is your editor of choice, you can ask CMake to generate Xcode project files instead of Unix Makefiles. @@ -19,4 +25,4 @@ If Xcode is your editor of choice, you can ask CMake to generate Xcode project f After running cmake, you will have the make files or Xcode project file necessary to build all of the components. Open the hifi.xcodeproj file, choose ALL_BUILD from the Product > Scheme menu (or target drop down), and click Run. -If the build completes successfully, you will have built targets for all components located in the `build/${target_name}/Debug` directories. \ No newline at end of file +If the build completes successfully, you will have built targets for all components located in the `build/${target_name}/Debug` directories.