From 8336b318e1cf6af3da551cfb249caf32e428d710 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 15 Jul 2015 10:17:18 -0700 Subject: [PATCH 01/36] Scale saccade per target distance when calculating eye rotation So that saccades still show when looking into distance. --- interface/src/avatar/FaceModel.cpp | 4 ++-- interface/src/avatar/Head.cpp | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/interface/src/avatar/FaceModel.cpp b/interface/src/avatar/FaceModel.cpp index 170965bb4d..bb8e58cb6f 100644 --- a/interface/src/avatar/FaceModel.cpp +++ b/interface/src/avatar/FaceModel.cpp @@ -73,8 +73,8 @@ void FaceModel::maybeUpdateEyeRotation(Model* model, const JointState& parentSta glm::translate(state.getDefaultTranslationInConstrainedFrame()) * joint.preTransform * glm::mat4_cast(joint.preRotation * joint.rotation)); glm::vec3 front = glm::vec3(inverse * glm::vec4(_owningHead->getFinalOrientationInWorldFrame() * IDENTITY_FRONT, 0.0f)); - glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(_owningHead->getCorrectedLookAtPosition() + - _owningHead->getSaccade() - model->getTranslation(), 1.0f)); + glm::vec3 lookAtDelta = _owningHead->getCorrectedLookAtPosition() - model->getTranslation(); + glm::vec3 lookAt = glm::vec3(inverse * glm::vec4(lookAtDelta + glm::length(lookAtDelta) * _owningHead->getSaccade(), 1.0f)); glm::quat between = rotationBetween(front, lookAt); const float MAX_ANGLE = 30.0f * RADIANS_PER_DEGREE; state.setRotationInConstrainedFrame(glm::angleAxis(glm::clamp(glm::angle(between), -MAX_ANGLE, MAX_ANGLE), glm::axis(between)) * diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 98126c3a22..d41d5c5dd1 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -343,7 +343,8 @@ glm::quat Head::getCameraOrientation() const { glm::quat Head::getEyeRotation(const glm::vec3& eyePosition) const { glm::quat orientation = getOrientation(); - return rotationBetween(orientation * IDENTITY_FRONT, _lookAtPosition + _saccade - eyePosition) * orientation; + glm::vec3 lookAtDelta = _lookAtPosition - eyePosition; + return rotationBetween(orientation * IDENTITY_FRONT, lookAtDelta + glm::length(lookAtDelta) * _saccade) * orientation; } glm::vec3 Head::getScalePivot() const { From ba733f730b5b35e198261e27687cee52960c30e0 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 15 Jul 2015 10:19:51 -0700 Subject: [PATCH 02/36] Move towards saccade target at speed taking into account frame rate --- interface/src/avatar/Head.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index d41d5c5dd1..149154da23 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -130,13 +130,14 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { const float AVERAGE_SACCADE_INTERVAL = 6.0f; const float MICROSACCADE_MAGNITUDE = 0.002f; const float SACCADE_MAGNITUDE = 0.04f; + const float NOMINAL_FRAME_RATE = 60.0f; if (randFloat() < deltaTime / AVERAGE_MICROSACCADE_INTERVAL) { _saccadeTarget = MICROSACCADE_MAGNITUDE * randVector(); } else if (randFloat() < deltaTime / AVERAGE_SACCADE_INTERVAL) { _saccadeTarget = SACCADE_MAGNITUDE * randVector(); } - _saccade += (_saccadeTarget - _saccade) * 0.5f; + _saccade += (_saccadeTarget - _saccade) * pow(0.5f, NOMINAL_FRAME_RATE * deltaTime); // Detect transition from talking to not; force blink after that and a delay bool forceBlink = false; From 6b523ada2b86d38f901386126e88a24ed94132f1 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 16 Jul 2015 10:48:21 -0700 Subject: [PATCH 03/36] Calculate eye pitch and yaw from DDE coefficients --- interface/src/devices/DdeFaceTracker.cpp | 15 +++++++++++++++ interface/src/devices/DdeFaceTracker.h | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 6facf44b91..f91178b041 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -157,6 +157,10 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _reset(false), _leftBlinkIndex(0), // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes _rightBlinkIndex(1), + _leftEyeDownIndex(4), + _rightEyeDownIndex(5), + _leftEyeInIndex(6), + _rightEyeInIndex(7), _leftEyeOpenIndex(8), _rightEyeOpenIndex(9), _browDownLeftIndex(14), @@ -173,6 +177,8 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _filteredHeadTranslation(glm::vec3(0.0f)), _lastBrowUp(0.0f), _filteredBrowUp(0.0f), + _eyeGazePitch(0.0f), + _eyeGazeYaw(0.0f), _lastEyeBlinks(), _filteredEyeBlinks(), _lastEyeCoefficients(), @@ -436,6 +442,15 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { _coefficients[_mouthSmileLeftIndex] = _coefficients[_mouthSmileLeftIndex] - SMILE_THRESHOLD; _coefficients[_mouthSmileRightIndex] = _coefficients[_mouthSmileRightIndex] - SMILE_THRESHOLD; + // Eye pitch and yaw + // EyeDown coefficients work better over both +ve and -ve values than EyeUp values. + // EyeIn coefficients work better over both +ve and -ve values than EyeOut values. + // Confusingly, the sign of the EyeIn_R value is the same of EyeIn_L. + const float EYE_GAZE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. + _eyeGazePitch = EYE_GAZE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); + const float EYE_GAZE_YAW_SCALE = 2500.0f; // Scale and average to be similar to Faceshift values. + _eyeGazeYaw = EYE_GAZE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); + // Velocity filter EyeBlink values const float DDE_EYEBLINK_SCALE = 3.0f; float eyeBlinks[] = { DDE_EYEBLINK_SCALE * _coefficients[_leftBlinkIndex], diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index b3318d0cb0..025937d25c 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -93,6 +93,11 @@ private: int _leftEyeOpenIndex; int _rightEyeOpenIndex; + int _leftEyeDownIndex; + int _rightEyeDownIndex; + int _leftEyeInIndex; + int _rightEyeInIndex; + int _browDownLeftIndex; int _browDownRightIndex; int _browUpCenterIndex; @@ -115,6 +120,9 @@ private: float _lastBrowUp; float _filteredBrowUp; + float _eyeGazePitch; // Degrees + float _eyeGazeYaw; + enum EyeState { EYE_UNCONTROLLED, EYE_OPEN, From 228e8d4fc0d56076059f6e974c82aa1e630b4d16 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 16 Jul 2015 12:37:47 -0700 Subject: [PATCH 04/36] Velocity filter eye pitch and yaw --- interface/src/devices/DdeFaceTracker.cpp | 33 +++++++++++++++++++----- interface/src/devices/DdeFaceTracker.h | 11 ++++++-- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index f91178b041..307e3ccd39 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -177,8 +177,14 @@ DdeFaceTracker::DdeFaceTracker(const QHostAddress& host, quint16 serverPort, qui _filteredHeadTranslation(glm::vec3(0.0f)), _lastBrowUp(0.0f), _filteredBrowUp(0.0f), - _eyeGazePitch(0.0f), - _eyeGazeYaw(0.0f), + _eyePitch(0.0f), + _eyeYaw(0.0f), + _lastEyePitch(0.0f), + _lastEyeYaw(0.0f), + _filteredEyePitch(0.0f), + _filteredEyeYaw(0.0f), + _longTermAverageEyePitch(0.0f), + _longTermAverageEyeYaw(0.0f), _lastEyeBlinks(), _filteredEyeBlinks(), _lastEyeCoefficients(), @@ -445,11 +451,24 @@ void DdeFaceTracker::decodePacket(const QByteArray& buffer) { // Eye pitch and yaw // EyeDown coefficients work better over both +ve and -ve values than EyeUp values. // EyeIn coefficients work better over both +ve and -ve values than EyeOut values. - // Confusingly, the sign of the EyeIn_R value is the same of EyeIn_L. - const float EYE_GAZE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. - _eyeGazePitch = EYE_GAZE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); - const float EYE_GAZE_YAW_SCALE = 2500.0f; // Scale and average to be similar to Faceshift values. - _eyeGazeYaw = EYE_GAZE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); + // Pitch and yaw values are relative to the screen. + const float EYE_PITCH_SCALE = -1500.0f; // Sign, scale, and average to be similar to Faceshift values. + _eyePitch = EYE_PITCH_SCALE * (_coefficients[_leftEyeDownIndex] + _coefficients[_rightEyeDownIndex]); + const float EYE_YAW_SCALE = 2000.0f; // Scale and average to be similar to Faceshift values. + _eyeYaw = EYE_YAW_SCALE * (_coefficients[_leftEyeInIndex] + _coefficients[_rightEyeInIndex]); + if (isFiltering) { + const float EYE_VELOCITY_FILTER_STRENGTH = 0.005f; + float pitchVelocity = fabsf(_eyePitch - _lastEyePitch) / _averageMessageTime; + float pitchVelocityFilter = glm::clamp(pitchVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredEyePitch = pitchVelocityFilter * _eyePitch + (1.0f - pitchVelocityFilter) * _filteredEyePitch; + _lastEyePitch = _eyePitch; + _eyePitch = _filteredEyePitch; + float yawVelocity = fabsf(_eyeYaw - _lastEyeYaw) / _averageMessageTime; + float yawVelocityFilter = glm::clamp(yawVelocity * EYE_VELOCITY_FILTER_STRENGTH, 0.0f, 1.0f); + _filteredEyeYaw = yawVelocityFilter * _eyeYaw + (1.0f - yawVelocityFilter) * _filteredEyeYaw; + _lastEyeYaw = _eyeYaw; + _eyeYaw = _filteredEyeYaw; + } // Velocity filter EyeBlink values const float DDE_EYEBLINK_SCALE = 3.0f; diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index 025937d25c..bf51f85778 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -120,8 +120,15 @@ private: float _lastBrowUp; float _filteredBrowUp; - float _eyeGazePitch; // Degrees - float _eyeGazeYaw; + float _eyePitch; // Degrees, relative to screen + float _eyeYaw; + float _lastEyePitch; + float _lastEyeYaw; + float _filteredEyePitch; + float _filteredEyeYaw; + float _longTermAverageEyePitch = 0.0f; + float _longTermAverageEyeYaw = 0.0f; + bool _longTermAverageInitialized = false; enum EyeState { EYE_UNCONTROLLED, From 9980233a0df5e57176b607cc60bffaa83119fab9 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 16 Jul 2015 13:03:33 -0700 Subject: [PATCH 05/36] Use estimated eye pitch and yaw from DDE to deflect eyes --- interface/src/Application.cpp | 29 ++++++++++++------------ interface/src/devices/DdeFaceTracker.cpp | 11 +++++++++ interface/src/devices/DdeFaceTracker.h | 1 + 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 027ad159d0..931cdc47ab 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2359,23 +2359,22 @@ void Application::updateMyAvatarLookAtPosition() { lookAtSpot = _myAvatar->getHead()->getEyePosition() + (_myAvatar->getHead()->getFinalOrientationInWorldFrame() * glm::vec3(0.0f, 0.0f, -TREE_SCALE)); } - } - // Deflect the eyes a bit to match the detected gaze from Faceshift if active. - // DDE doesn't track eyes. - if (tracker && typeid(*tracker) == typeid(Faceshift) && !tracker->isMuted()) { - float eyePitch = tracker->getEstimatedEyePitch(); - float eyeYaw = tracker->getEstimatedEyeYaw(); - const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; - glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); - float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; - float deflection = DependencyManager::get()->getEyeDeflection(); - if (isLookingAtSomeone) { - deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; - } - lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3( - eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) * + // Deflect the eyes a bit to match the detected gaze from the face tracker if active. + if (tracker && !tracker->isMuted()) { + float eyePitch = tracker->getEstimatedEyePitch(); + float eyeYaw = tracker->getEstimatedEyeYaw(); + const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; + glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); + float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; + float deflection = DependencyManager::get()->getEyeDeflection(); + if (isLookingAtSomeone) { + deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; + } + lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3( + eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) * glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin); + } } _myAvatar->getHead()->setLookAtPosition(lookAtSpot); diff --git a/interface/src/devices/DdeFaceTracker.cpp b/interface/src/devices/DdeFaceTracker.cpp index 307e3ccd39..d09cba3b61 100644 --- a/interface/src/devices/DdeFaceTracker.cpp +++ b/interface/src/devices/DdeFaceTracker.cpp @@ -294,6 +294,17 @@ void DdeFaceTracker::reset() { } } +void DdeFaceTracker::update(float deltaTime) { + if (!isActive()) { + return; + } + FaceTracker::update(deltaTime); + + glm::vec3 headEulers = glm::degrees(glm::eulerAngles(_headRotation)); + _estimatedEyePitch = _eyePitch - headEulers.x; + _estimatedEyeYaw = _eyeYaw - headEulers.y; +} + bool DdeFaceTracker::isActive() const { return (_ddeProcess != NULL); } diff --git a/interface/src/devices/DdeFaceTracker.h b/interface/src/devices/DdeFaceTracker.h index bf51f85778..5536fa14bd 100644 --- a/interface/src/devices/DdeFaceTracker.h +++ b/interface/src/devices/DdeFaceTracker.h @@ -31,6 +31,7 @@ class DdeFaceTracker : public FaceTracker, public Dependency { public: virtual void init(); virtual void reset(); + virtual void update(float deltaTime); virtual bool isActive() const; virtual bool isTracking() const; From 6c6e3eb758eeadc543eee3ce9768e163e8df0338 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 16 Jul 2015 14:05:34 -0700 Subject: [PATCH 06/36] Rename and refactor eye deflection slider --- interface/src/Application.cpp | 2 +- interface/src/devices/FaceTracker.cpp | 7 +++++++ interface/src/devices/FaceTracker.h | 7 +++++++ interface/src/devices/Faceshift.cpp | 6 ------ interface/src/devices/Faceshift.h | 4 ---- interface/src/ui/PreferencesDialog.cpp | 12 ++++++------ interface/ui/preferencesDialog.ui | 6 +++--- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 931cdc47ab..27858533b5 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2367,7 +2367,7 @@ void Application::updateMyAvatarLookAtPosition() { const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; - float deflection = DependencyManager::get()->getEyeDeflection(); + float deflection = tracker->getEyeDeflection(); if (isLookingAtSomeone) { deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; } diff --git a/interface/src/devices/FaceTracker.cpp b/interface/src/devices/FaceTracker.cpp index 25a76ff2b1..76a4534952 100644 --- a/interface/src/devices/FaceTracker.cpp +++ b/interface/src/devices/FaceTracker.cpp @@ -20,6 +20,9 @@ const int FPS_TIMER_DELAY = 2000; // ms const int FPS_TIMER_DURATION = 2000; // ms +const float DEFAULT_EYE_DEFLECTION = 0.25f; +Setting::Handle FaceTracker::_eyeDeflection("faceshiftEyeDeflection", DEFAULT_EYE_DEFLECTION); + void FaceTracker::init() { _isMuted = Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking); _isInitialized = true; // FaceTracker can be used now @@ -106,3 +109,7 @@ void FaceTracker::toggleMute() { _isMuted = !_isMuted; emit muteToggled(); } + +void FaceTracker::setEyeDeflection(float eyeDeflection) { + _eyeDeflection.set(eyeDeflection); +} diff --git a/interface/src/devices/FaceTracker.h b/interface/src/devices/FaceTracker.h index 2a0c4438a4..193262d121 100644 --- a/interface/src/devices/FaceTracker.h +++ b/interface/src/devices/FaceTracker.h @@ -18,6 +18,8 @@ #include #include +#include + /// Base class for face trackers (Faceshift, DDE). class FaceTracker : public QObject { Q_OBJECT @@ -47,6 +49,9 @@ public: void setIsMuted(bool isMuted) { _isMuted = isMuted; } void toggleMute(); + static float getEyeDeflection() { return _eyeDeflection.get(); } + static void setEyeDeflection(float eyeDeflection); + signals: void muteToggled(); @@ -77,6 +82,8 @@ private slots: private: bool _isCalculatingFPS = false; int _frameCount = 0; + + static Setting::Handle _eyeDeflection; }; #endif // hifi_FaceTracker_h diff --git a/interface/src/devices/Faceshift.cpp b/interface/src/devices/Faceshift.cpp index b517f9d67b..9e7357af67 100644 --- a/interface/src/devices/Faceshift.cpp +++ b/interface/src/devices/Faceshift.cpp @@ -28,10 +28,8 @@ using namespace std; const QString DEFAULT_FACESHIFT_HOSTNAME = "localhost"; const quint16 FACESHIFT_PORT = 33433; -const float DEFAULT_FACESHIFT_EYE_DEFLECTION = 0.25f; Faceshift::Faceshift() : - _eyeDeflection("faceshiftEyeDeflection", DEFAULT_FACESHIFT_EYE_DEFLECTION), _hostname("faceshiftHostname", DEFAULT_FACESHIFT_HOSTNAME) { #ifdef HAVE_FACESHIFT @@ -306,10 +304,6 @@ void Faceshift::receive(const QByteArray& buffer) { FaceTracker::countFrame(); } -void Faceshift::setEyeDeflection(float faceshiftEyeDeflection) { - _eyeDeflection.set(faceshiftEyeDeflection); -} - void Faceshift::setHostname(const QString& hostname) { _hostname.set(hostname); } diff --git a/interface/src/devices/Faceshift.h b/interface/src/devices/Faceshift.h index 9be1766170..4cb7557410 100644 --- a/interface/src/devices/Faceshift.h +++ b/interface/src/devices/Faceshift.h @@ -68,9 +68,6 @@ public: float getMouthSmileLeft() const { return getBlendshapeCoefficient(_mouthSmileLeftIndex); } float getMouthSmileRight() const { return getBlendshapeCoefficient(_mouthSmileRightIndex); } - float getEyeDeflection() { return _eyeDeflection.get(); } - void setEyeDeflection(float faceshiftEyeDeflection); - QString getHostname() { return _hostname.get(); } void setHostname(const QString& hostname); @@ -134,7 +131,6 @@ private: float _longTermAverageEyeYaw = 0.0f; bool _longTermAverageInitialized = false; - Setting::Handle _eyeDeflection; Setting::Handle _hostname; // see http://support.faceshift.com/support/articles/35129-export-of-blendshapes diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index 35a049db3c..121b7d611c 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -142,10 +142,10 @@ void PreferencesDialog::loadPreferences() { ui.ddeEyeClosingThresholdSlider->setValue(dde->getEyeClosingThreshold() * ui.ddeEyeClosingThresholdSlider->maximum()); - auto faceshift = DependencyManager::get(); - ui.faceshiftEyeDeflectionSider->setValue(faceshift->getEyeDeflection() * - ui.faceshiftEyeDeflectionSider->maximum()); + ui.faceTrackerEyeDeflectionSider->setValue(FaceTracker::getEyeDeflection() * + ui.faceTrackerEyeDeflectionSider->maximum()); + auto faceshift = DependencyManager::get(); ui.faceshiftHostnameEdit->setText(faceshift->getHostname()); auto audio = DependencyManager::get(); @@ -233,10 +233,10 @@ void PreferencesDialog::savePreferences() { dde->setEyeClosingThreshold(ui.ddeEyeClosingThresholdSlider->value() / (float)ui.ddeEyeClosingThresholdSlider->maximum()); - auto faceshift = DependencyManager::get(); - faceshift->setEyeDeflection(ui.faceshiftEyeDeflectionSider->value() / - (float)ui.faceshiftEyeDeflectionSider->maximum()); + FaceTracker::setEyeDeflection(ui.faceTrackerEyeDeflectionSider->value() / + (float)ui.faceTrackerEyeDeflectionSider->maximum()); + auto faceshift = DependencyManager::get(); faceshift->setHostname(ui.faceshiftHostnameEdit->text()); qApp->setMaxOctreePacketsPerSecond(ui.maxOctreePPSSpin->value()); diff --git a/interface/ui/preferencesDialog.ui b/interface/ui/preferencesDialog.ui index 78f9f5bf09..df6d28c07b 100644 --- a/interface/ui/preferencesDialog.ui +++ b/interface/ui/preferencesDialog.ui @@ -1468,13 +1468,13 @@ - Faceshift eye deflection + Face tracker eye deflection 0 - faceshiftEyeDeflectionSider + faceTrackerEyeDeflectionSider @@ -1497,7 +1497,7 @@ - + 0 From f5b02c173109c862df9a231c068619964f473404 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 16 Jul 2015 14:06:42 -0700 Subject: [PATCH 07/36] Remove dead code Camera mode cannot be mirror --- interface/src/Application.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 27858533b5..7b2e172117 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2366,13 +2366,12 @@ void Application::updateMyAvatarLookAtPosition() { float eyeYaw = tracker->getEstimatedEyeYaw(); const float GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT = 0.1f; glm::vec3 origin = _myAvatar->getHead()->getEyePosition(); - float pitchSign = (_myCamera.getMode() == CAMERA_MODE_MIRROR) ? -1.0f : 1.0f; float deflection = tracker->getEyeDeflection(); if (isLookingAtSomeone) { deflection *= GAZE_DEFLECTION_REDUCTION_DURING_EYE_CONTACT; } lookAtSpot = origin + _myCamera.getRotation() * glm::quat(glm::radians(glm::vec3( - eyePitch * pitchSign * deflection, eyeYaw * deflection, 0.0f))) * + eyePitch * deflection, eyeYaw * deflection, 0.0f))) * glm::inverse(_myCamera.getRotation()) * (lookAtSpot - origin); } } From 5d7485850049d43f5a426c13e0c3699680ed46d0 Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Tue, 21 Jul 2015 17:01:54 +0200 Subject: [PATCH 08/36] edit.js - entityList - delete key deletes entities, hold shift to select a group of entities, moved appending single entity to selection to ctrl key --- examples/html/entityList.html | 37 ++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/examples/html/entityList.html b/examples/html/entityList.html index 9506c78994..62bbbd08a6 100644 --- a/examples/html/entityList.html +++ b/examples/html/entityList.html @@ -35,8 +35,32 @@ function onRowClicked(e) { var id = this.dataset.entityId; var selection = [this.dataset.entityId]; - if (e.shiftKey) { + if (e.ctrlKey) { selection = selection.concat(selectedEntities); + } else if (e.shiftKey && selectedEntities.length > 0) { + var previousItemFound = -1; + var clickedItemFound = -1; + for (var i in entityList.visibleItems) { + if (clickedItemFound === -1 && this.dataset.entityId == entityList.visibleItems[i].values().id) { + clickedItemFound = i; + } else if(previousItemFound === -1 && selectedEntities[0] == entityList.visibleItems[i].values().id) { + previousItemFound = i; + } + } + if (previousItemFound !== -1 && clickedItemFound !== -1) { + var betweenItems = []; + var toItem = Math.max(previousItemFound, clickedItemFound); + // skip first and last item in this loop, we add them to selection after the loop + for (var i = (Math.min(previousItemFound, clickedItemFound) + 1); i < toItem; i++) { + entityList.visibleItems[i].elm.className = 'selected'; + betweenItems.push(entityList.visibleItems[i].values().id); + } + if (previousItemFound > clickedItemFound) { + // always make sure that we add the items in the right order + betweenItems.reverse(); + } + selection = selection.concat(betweenItems, selectedEntities); + } } selectedEntities = selection; @@ -151,6 +175,17 @@ refreshEntities(); } + document.addEventListener("keydown", function (e) { + if (e.target.nodeName === "INPUT") { + return; + } + var keyCode = e.keyCode; + if (keyCode === 46) { + EventBridge.emitWebEvent(JSON.stringify({ type: 'delete' })); + refreshEntities(); + } + }, false); + if (window.EventBridge !== undefined) { EventBridge.scriptEventReceived.connect(function(data) { data = JSON.parse(data); From 3350d638dd5cd2b8fc5983bafab2aca92746a8cc Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 09:29:27 -0700 Subject: [PATCH 09/36] always use the _viewFrustum for caluclating the pick ray, not the _displayViewFrustum which can be the mirror --- interface/src/Application.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8f825a2768..3130e172af 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2963,11 +2963,7 @@ PickRay Application::computePickRay(float x, float y) const { if (isHMDMode()) { getApplicationCompositor().computeHmdPickRay(glm::vec2(x, y), result.origin, result.direction); } else { - if (QThread::currentThread() == activeRenderingThread) { - getDisplayViewFrustum()->computePickRay(x, y, result.origin, result.direction); - } else { - getViewFrustum()->computePickRay(x, y, result.origin, result.direction); - } + getViewFrustum()->computePickRay(x, y, result.origin, result.direction); } return result; } From 91d317f964b3287a51d408d199f0aa6ebdef2822 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 12:42:55 -0700 Subject: [PATCH 10/36] testing windows render bug --- libraries/render-utils/src/simple.slv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/simple.slv b/libraries/render-utils/src/simple.slv index 380734e70c..43b139c565 100644 --- a/libraries/render-utils/src/simple.slv +++ b/libraries/render-utils/src/simple.slv @@ -23,7 +23,7 @@ void main(void) { gl_TexCoord[0] = gl_MultiTexCoord0; // pass along the diffuse color - gl_FrontColor = gl_Color; + gl_FrontColor = vec4(0,1,0,1); //gl_Color; // standard transform TransformCamera cam = getTransformCamera(); From a0666d5e99a4877c1bdd478bd975d771e226e04c Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 13:17:12 -0700 Subject: [PATCH 11/36] testing windows render bug --- libraries/render-utils/src/DeferredLightingEffect.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index 7da314cdf2..9a45ea488e 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -142,6 +142,10 @@ void DeferredLightingEffect::bindSimpleProgram(gpu::Batch& batch, bool textured, bool emmisive, bool depthBias) { SimpleProgramKey config{textured, culled, emmisive, depthBias}; batch.setPipeline(getPipeline(config)); + + gpu::ShaderPointer program = (config.isEmissive()) ? _emissiveShader : _simpleShader; + int glowIntensity = program->getUniforms().findLocation("glowIntensity"); + batch._glUniform1f(glowIntensity, 1.0f); if (!config.isTextured()) { // If it is not textured, bind white texture and keep using textured pipeline From 5774af08560ae0619ecab25c5dc9f8449cef794a Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 13:38:08 -0700 Subject: [PATCH 12/36] more hacking --- libraries/render-utils/src/simple.slf | 6 +++--- libraries/render-utils/src/simple_textured.slf | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/render-utils/src/simple.slf b/libraries/render-utils/src/simple.slf index bbbb44cc51..8d49de4e46 100644 --- a/libraries/render-utils/src/simple.slf +++ b/libraries/render-utils/src/simple.slf @@ -21,7 +21,7 @@ void main(void) { packDeferredFragment( normalize(interpolatedNormal.xyz), glowIntensity, - gl_Color.rgb, - gl_FrontMaterial.specular.rgb, - gl_FrontMaterial.shininess); + vec3(0,0,1), //gl_Color.rgb, + vec3(0,0,1), //gl_FrontMaterial.specular.rgb, + 1); //gl_FrontMaterial.shininess); } diff --git a/libraries/render-utils/src/simple_textured.slf b/libraries/render-utils/src/simple_textured.slf index 7444d512e4..5a3909c7ce 100644 --- a/libraries/render-utils/src/simple_textured.slf +++ b/libraries/render-utils/src/simple_textured.slf @@ -25,8 +25,8 @@ void main(void) { packDeferredFragment( normalize(interpolatedNormal.xyz), - glowIntensity * texel.a, - gl_Color.rgb * texel.rgb, - gl_FrontMaterial.specular.rgb, - gl_FrontMaterial.shininess); + glowIntensity, // glowIntensity * texel.a, + vec3(0,0,1), // gl_Color.rgb * texel.rgb, + vec3(0,0,1), // gl_FrontMaterial.specular.rgb, + 1); // gl_FrontMaterial.shininess); } \ No newline at end of file From 9496b6c024d1d6c304c42a2a02ce0abf0bcb99b7 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 13:56:59 -0700 Subject: [PATCH 13/36] more hacking --- libraries/render-utils/src/simple_textured.slf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/render-utils/src/simple_textured.slf b/libraries/render-utils/src/simple_textured.slf index 5a3909c7ce..814854e169 100644 --- a/libraries/render-utils/src/simple_textured.slf +++ b/libraries/render-utils/src/simple_textured.slf @@ -26,7 +26,7 @@ void main(void) { packDeferredFragment( normalize(interpolatedNormal.xyz), glowIntensity, // glowIntensity * texel.a, - vec3(0,0,1), // gl_Color.rgb * texel.rgb, - vec3(0,0,1), // gl_FrontMaterial.specular.rgb, - 1); // gl_FrontMaterial.shininess); + gl_Color.rgb, // gl_Color.rgb * texel.rgb, + gl_FrontMaterial.specular.rgb, + gl_FrontMaterial.shininess); } \ No newline at end of file From 96e83e3ed496fa582fcf2483fd094ca0d9c0d855 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 14:11:28 -0700 Subject: [PATCH 14/36] remove gl_FrontMaterial --- libraries/render-utils/src/simple.slf | 6 +++--- libraries/render-utils/src/simple_textured.slf | 8 ++++---- libraries/render-utils/src/simple_textured_emisive.slf | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libraries/render-utils/src/simple.slf b/libraries/render-utils/src/simple.slf index 8d49de4e46..1d77dde0d3 100644 --- a/libraries/render-utils/src/simple.slf +++ b/libraries/render-utils/src/simple.slf @@ -21,7 +21,7 @@ void main(void) { packDeferredFragment( normalize(interpolatedNormal.xyz), glowIntensity, - vec3(0,0,1), //gl_Color.rgb, - vec3(0,0,1), //gl_FrontMaterial.specular.rgb, - 1); //gl_FrontMaterial.shininess); + gl_Color.rgb, + vec3(0.1), + 10); } diff --git a/libraries/render-utils/src/simple_textured.slf b/libraries/render-utils/src/simple_textured.slf index 814854e169..88b196543c 100644 --- a/libraries/render-utils/src/simple_textured.slf +++ b/libraries/render-utils/src/simple_textured.slf @@ -25,8 +25,8 @@ void main(void) { packDeferredFragment( normalize(interpolatedNormal.xyz), - glowIntensity, // glowIntensity * texel.a, - gl_Color.rgb, // gl_Color.rgb * texel.rgb, - gl_FrontMaterial.specular.rgb, - gl_FrontMaterial.shininess); + glowIntensity * texel.a, + gl_Color.rgb * texel.rgb, + vec3(0.1), + 10); } \ No newline at end of file diff --git a/libraries/render-utils/src/simple_textured_emisive.slf b/libraries/render-utils/src/simple_textured_emisive.slf index 643dcde190..c5749114f3 100644 --- a/libraries/render-utils/src/simple_textured_emisive.slf +++ b/libraries/render-utils/src/simple_textured_emisive.slf @@ -27,7 +27,7 @@ void main(void) { normalize(interpolatedNormal.xyz), glowIntensity * texel.a, gl_Color.rgb, - gl_FrontMaterial.specular.rgb, - gl_FrontMaterial.shininess, + vec3(0.1), + 10, texel.rgb); } \ No newline at end of file From 1177ffd43954333ec4fb105934a33ca28c094c23 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 14:12:52 -0700 Subject: [PATCH 15/36] more hacking --- libraries/render-utils/src/simple.slv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/render-utils/src/simple.slv b/libraries/render-utils/src/simple.slv index 43b139c565..380734e70c 100644 --- a/libraries/render-utils/src/simple.slv +++ b/libraries/render-utils/src/simple.slv @@ -23,7 +23,7 @@ void main(void) { gl_TexCoord[0] = gl_MultiTexCoord0; // pass along the diffuse color - gl_FrontColor = vec4(0,1,0,1); //gl_Color; + gl_FrontColor = gl_Color; // standard transform TransformCamera cam = getTransformCamera(); From b26f4b0c8d034f298bc317db6fbb157cc66992d1 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 23 Jul 2015 14:49:15 -0700 Subject: [PATCH 16/36] fix for underdamped avatar rotation momentum --- interface/src/avatar/MyAvatar.cpp | 40 ++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index ac0cb3feda..980a4d3e14 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -54,7 +54,7 @@ using namespace std; const glm::vec3 DEFAULT_UP_DIRECTION(0.0f, 1.0f, 0.0f); -const float YAW_SPEED = 500.0f; // degrees/sec +const float YAW_SPEED = 150.0f; // degrees/sec const float PITCH_SPEED = 100.0f; // degrees/sec const float DEFAULT_REAL_WORLD_FIELD_OF_VIEW_DEGREES = 30.0f; @@ -1290,22 +1290,34 @@ bool MyAvatar::shouldRenderHead(const RenderArgs* renderArgs) const { void MyAvatar::updateOrientation(float deltaTime) { // Smoothly rotate body with arrow keys - _bodyYawDelta -= _driveKeys[ROT_RIGHT] * YAW_SPEED * deltaTime; - _bodyYawDelta += _driveKeys[ROT_LEFT] * YAW_SPEED * deltaTime; - getHead()->setBasePitch(getHead()->getBasePitch() + (_driveKeys[ROT_UP] - _driveKeys[ROT_DOWN]) * PITCH_SPEED * deltaTime); + float driveLeft = _driveKeys[ROT_LEFT] - _driveKeys[ROT_RIGHT]; + float targetSpeed = (_driveKeys[ROT_LEFT] - _driveKeys[ROT_RIGHT]) * YAW_SPEED; + if (targetSpeed != 0.0f) { + const float ROTATION_RAMP_TIMESCALE = 0.1f; + float blend = deltaTime / ROTATION_RAMP_TIMESCALE; + if (blend > 1.0f) { + blend = 1.0f; + } + _bodyYawDelta = (1.0f - blend) * _bodyYawDelta + blend * targetSpeed; + } else if (_bodyYawDelta != 0.0f) { + // attenuate body rotation speed + const float ROTATION_DECAY_TIMESCALE = 0.05f; + float attenuation = 1.0f - deltaTime / ROTATION_DECAY_TIMESCALE; + if (attenuation < 0.0f) { + attenuation = 0.0f; + } + _bodyYawDelta *= attenuation; + float MINIMUM_ROTATION_RATE = 2.0f; + if (fabsf(_bodyYawDelta) < MINIMUM_ROTATION_RATE) { + _bodyYawDelta = 0.0f; + } + } + + getHead()->setBasePitch(getHead()->getBasePitch() + (_driveKeys[ROT_UP] - _driveKeys[ROT_DOWN]) * PITCH_SPEED * deltaTime); // update body orientation by movement inputs setOrientation(getOrientation() * - glm::quat(glm::radians(glm::vec3(0.0f, _bodyYawDelta, 0.0f) * deltaTime))); - - // decay body rotation momentum - const float BODY_SPIN_FRICTION = 7.5f; - float bodySpinMomentum = 1.0f - BODY_SPIN_FRICTION * deltaTime; - if (bodySpinMomentum < 0.0f) { bodySpinMomentum = 0.0f; } - _bodyYawDelta *= bodySpinMomentum; - - float MINIMUM_ROTATION_RATE = 2.0f; - if (fabs(_bodyYawDelta) < MINIMUM_ROTATION_RATE) { _bodyYawDelta = 0.0f; } + glm::quat(glm::radians(glm::vec3(0.0f, _bodyYawDelta * deltaTime, 0.0f)))); if (qApp->isHMDMode()) { // these angles will be in radians From a561874ce3c918b87f0fda8cc69054e4f9ccba74 Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 14:56:14 -0700 Subject: [PATCH 17/36] Render the rear view mirror in a separate target and then display it during overlay pass --- interface/src/Application.cpp | 93 +++++++++++++++---- interface/src/Application.h | 2 + interface/src/avatar/MyAvatar.cpp | 1 + interface/src/ui/ApplicationOverlay.cpp | 30 ++++++ libraries/gpu/src/gpu/GLBackendOutput.cpp | 6 ++ .../src/DeferredLightingEffect.cpp | 10 +- .../render-utils/src/FramebufferCache.cpp | 9 +- libraries/render-utils/src/FramebufferCache.h | 7 ++ libraries/render-utils/src/GeometryCache.cpp | 14 ++- libraries/render-utils/src/GeometryCache.h | 3 +- 10 files changed, 147 insertions(+), 28 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7de69ef602..eebdf61c5b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -875,6 +875,33 @@ void Application::paintGL() { PerformanceWarning warn(showWarnings, "Application::paintGL()"); resizeGL(); + // Before anything else, let's sync up the gpuContext with the true glcontext used in case anything happened + renderArgs._context->syncCache(); + + if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { + auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); + { + gpu::Batch batch; + batch.setFramebuffer(nullptr); + batch.setFramebuffer(primaryFbo); + renderArgs._context->render(batch); + } + renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; + renderRearViewMirror(&renderArgs, _mirrorViewRect); + renderArgs._renderMode = RenderArgs::NORMAL_RENDER_MODE; + + { + auto mirrorViewport = glm::ivec4(0, 0, _mirrorViewRect.width(), _mirrorViewRect.height()); + float ratio = 1.0f / ((float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale()); + + auto mirrorViewportDest = glm::ivec4(0, 0, ratio * mirrorViewport.z, ratio * mirrorViewport.w); + + auto selfieFbo = DependencyManager::get()->getSelfieFramebuffer(); + gpu::Batch batch; + batch.blit(primaryFbo, mirrorViewport, selfieFbo, mirrorViewportDest); + renderArgs._context->render(batch); + } + } { PerformanceTimer perfTimer("renderOverlay"); @@ -936,7 +963,7 @@ void Application::paintGL() { } // Sync up the View Furstum with the camera - loadViewFrustum(_myCamera, _viewFrustum); + // loadViewFrustum(_myCamera, _viewFrustum); renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; @@ -974,19 +1001,17 @@ void Application::paintGL() { displaySide(&renderArgs, _myCamera); - if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { - renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; - renderRearViewMirror(&renderArgs, _mirrorViewRect); - renderArgs._renderMode = RenderArgs::NORMAL_RENDER_MODE; - } - { auto geometryCache = DependencyManager::get(); auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); gpu::Batch batch; batch.blit(primaryFbo, glm::ivec4(0, 0, _renderResolution.x, _renderResolution.y), nullptr, glm::ivec4(0, 0, _glWidget->getDeviceSize().width(), _glWidget->getDeviceSize().height())); + + batch.setFramebuffer(nullptr); + renderArgs._context->render(batch); + } _compositor.displayOverlayTexture(&renderArgs); @@ -2958,7 +2983,7 @@ PickRay Application::computePickRay(float x, float y) const { } QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { - auto primaryFramebuffer = DependencyManager::get()->getPrimaryFramebuffer(); +/* auto primaryFramebuffer = DependencyManager::get()->getPrimaryFramebuffer(); glBindFramebuffer(GL_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(primaryFramebuffer)); // clear the alpha channel so the background is transparent @@ -2966,17 +2991,41 @@ QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { glClearColor(0.0, 0.0, 0.0, 0.0); glClear(GL_COLOR_BUFFER_BIT); glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE); + */ - const int BILLBOARD_SIZE = 64; - // TODO: Pass a RenderArgs to renderAvatarBillboard - renderRearViewMirror(renderArgs, QRect(0, _glWidget->getDeviceHeight() - BILLBOARD_SIZE, + // const int BILLBOARD_SIZE = 64; + const int BILLBOARD_SIZE = 256; + _glWidget->makeCurrent(); + + auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); + + { + gpu::Batch batch; + batch.setFramebuffer(nullptr); + batch.setFramebuffer(primaryFbo); + renderArgs->_context->render(batch); + } + renderArgs->_renderMode = RenderArgs::MIRROR_RENDER_MODE; + renderRearViewMirror(renderArgs, QRect(0, /*_glWidget->getDeviceHeight() - BILLBOARD_SIZE*/ 0, BILLBOARD_SIZE, BILLBOARD_SIZE), - true); - QImage image(BILLBOARD_SIZE, BILLBOARD_SIZE, QImage::Format_ARGB32); - glReadPixels(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE, GL_BGRA, GL_UNSIGNED_BYTE, image.bits()); - glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + false); + renderArgs->_renderMode = RenderArgs::NORMAL_RENDER_MODE; + auto selfieFbo = DependencyManager::get()->getSelfieFramebuffer(); + { + auto mirrorViewport = glm::ivec4(0, 0,BILLBOARD_SIZE, BILLBOARD_SIZE); + gpu::Batch batch; + // batch.blit(primaryFbo, mirrorViewport, selfieFbo, mirrorViewport); + batch.setFramebuffer(nullptr); + renderArgs->_context->render(batch); + } + + + QImage image(BILLBOARD_SIZE, BILLBOARD_SIZE, QImage::Format_ARGB32); + glBindFramebuffer(GL_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(primaryFbo)); + glReadPixels(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE, GL_BGRA, GL_UNSIGNED_BYTE, image.bits()); glBindFramebuffer(GL_FRAMEBUFFER, 0); + return image; } @@ -3390,13 +3439,18 @@ void Application::renderRearViewMirror(RenderArgs* renderArgs, const QRect& regi gpu::Vec4i viewport; if (billboard) { QSize size = DependencyManager::get()->getFrameBufferSize(); - viewport = gpu::Vec4i(region.x(), size.height() - region.y() - region.height(), region.width(), region.height()); + // viewport = gpu::Vec4i(region.x(), size.height() - region.y() - region.height(), region.width(), region.height()); + viewport = gpu::Vec4i(0, 0, region.width(), region.height()); } else { // if not rendering the billboard, the region is in device independent coordinates; must convert to device QSize size = DependencyManager::get()->getFrameBufferSize(); float ratio = (float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale(); - int x = region.x() * ratio, y = region.y() * ratio, width = region.width() * ratio, height = region.height() * ratio; - viewport = gpu::Vec4i(x, size.height() - y - height, width, height); + int x = region.x() * ratio; + int y = region.y() * ratio; + int width = region.width() * ratio; + int height = region.height() * ratio; + // viewport = gpu::Vec4i(x, size.height() - y - height, width, height); + viewport = gpu::Vec4i(0, 0, width, height); } renderArgs->_viewport = viewport; @@ -3414,8 +3468,7 @@ void Application::renderRearViewMirror(RenderArgs* renderArgs, const QRect& regi renderArgs->_context->render(batch); } - bool updateViewFrustum = false; - loadViewFrustum(_mirrorCamera, _viewFrustum); + // loadViewFrustum(_mirrorCamera, _viewFrustum); // render rear mirror view displaySide(renderArgs, _mirrorCamera, true, billboard); diff --git a/interface/src/Application.h b/interface/src/Application.h index 3a07fb656d..b335b8a333 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -328,6 +328,8 @@ public: gpu::ContextPointer getGPUContext() const { return _gpuContext; } + const QRect& getMirrorViewRect() const { return _mirrorViewRect; } + signals: /// Fired when we're simulating; allows external parties to hook in. diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 4147d24b15..f92c3c6700 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -1539,6 +1539,7 @@ void MyAvatar::maybeUpdateBillboard() { QBuffer buffer(&_billboard); buffer.open(QIODevice::WriteOnly); image.save(&buffer, "PNG"); + image.save("billboard.png", "PNG"); _billboardValid = true; sendBillboardPacket(); diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index a7f03db24f..d1154bda03 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -96,6 +97,7 @@ void ApplicationOverlay::renderOverlay(RenderArgs* renderArgs) { renderQmlUi(renderArgs); // renders a unit quad with the QML UI texture, and the text overlays from scripts renderOverlays(renderArgs); // renders Scripts Overlay and AudioScope renderStatsAndLogs(renderArgs); // currently renders nothing + renderRearView(renderArgs); // renders the mirror view selfie renderArgs->_context->syncCache(); renderArgs->_context->render(batch); @@ -167,6 +169,34 @@ void ApplicationOverlay::renderRearViewToFbo(RenderArgs* renderArgs) { } void ApplicationOverlay::renderRearView(RenderArgs* renderArgs) { + if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { + gpu::Batch& batch = *renderArgs->_batch; + + auto geometryCache = DependencyManager::get(); + + auto framebuffer = DependencyManager::get(); + auto selfieTexture = framebuffer->getSelfieFramebuffer()->getRenderBuffer(0); + + int width = renderArgs->_viewport.z; + int height = renderArgs->_viewport.w; + mat4 legacyProjection = glm::ortho(0, width, height, 0, ORTHO_NEAR_CLIP, ORTHO_FAR_CLIP); + batch.setProjectionTransform(legacyProjection); + batch.setModelTransform(Transform()); + batch.setViewTransform(Transform()); + + auto viewport = qApp->getMirrorViewRect(); + glm::vec2 bottomLeft(viewport.left(), viewport.bottom()); + glm::vec2 topRight(viewport.right(), viewport.top()); + glm::vec2 texCoordMinCorner(0.0f, 0.0f); + glm::vec2 texCoordMaxCorner(viewport.width() / float(selfieTexture->getWidth()), viewport.height() / float(selfieTexture->getHeight())); + + geometryCache->useSimpleDrawPipeline(batch, false); + batch.setResourceTexture(0, renderArgs->_whiteTexture); + geometryCache->renderQuad(batch, bottomLeft, topRight, glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); + geometryCache->useSimpleDrawPipeline(batch, true); + batch.setResourceTexture(0, selfieTexture); + geometryCache->renderQuad(batch, bottomLeft, topRight, texCoordMinCorner, texCoordMaxCorner, glm::vec4(1.0f, 1.0f, 1.0f, 1.0f)); + } } void ApplicationOverlay::renderStatsAndLogs(RenderArgs* renderArgs) { diff --git a/libraries/gpu/src/gpu/GLBackendOutput.cpp b/libraries/gpu/src/gpu/GLBackendOutput.cpp index 8876db95ac..621f04b71b 100755 --- a/libraries/gpu/src/gpu/GLBackendOutput.cpp +++ b/libraries/gpu/src/gpu/GLBackendOutput.cpp @@ -184,4 +184,10 @@ void GLBackend::do_blit(Batch& batch, uint32 paramOffset) { glBlitFramebuffer(srcvp.x, srcvp.y, srcvp.z, srcvp.w, dstvp.x, dstvp.y, dstvp.z, dstvp.w, GL_COLOR_BUFFER_BIT, GL_LINEAR); + + (void) CHECK_GL_ERROR(); + + if (_output._framebuffer) { + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, getFramebufferID(_output._framebuffer)); + } } diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index e6f2b520c7..5680165969 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -217,11 +217,12 @@ void DeferredLightingEffect::addSpotLight(const glm::vec3& position, float radiu void DeferredLightingEffect::prepare(RenderArgs* args) { gpu::Batch batch; - + + batch.setStateScissorRect(args->_viewport); // clear the normal and specular buffers - batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR1, glm::vec4(0.0f, 0.0f, 0.0f, 0.0f)); + batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR1, glm::vec4(0.0f, 0.0f, 0.0f, 0.0f), true); const float MAX_SPECULAR_EXPONENT = 128.0f; - batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR2, glm::vec4(0.0f, 0.0f, 0.0f, 1.0f / MAX_SPECULAR_EXPONENT)); + batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR2, glm::vec4(0.0f, 0.0f, 0.0f, 1.0f / MAX_SPECULAR_EXPONENT), true); args->_context->syncCache(); args->_context->render(batch); @@ -242,8 +243,9 @@ void DeferredLightingEffect::render(RenderArgs* args) { batch.setFramebuffer(_copyFBO); batch.setViewportTransform(args->_viewport); + batch.setStateScissorRect(args->_viewport); - batch.clearColorFramebuffer(_copyFBO->getBufferMask(), glm::vec4(0.0f, 0.0f, 0.0f, 0.0f)); + batch.clearColorFramebuffer(_copyFBO->getBufferMask(), glm::vec4(0.0f, 0.0f, 0.0f, 0.0f), true); batch.setResourceTexture(0, framebufferCache->getPrimaryColorTexture()); diff --git a/libraries/render-utils/src/FramebufferCache.cpp b/libraries/render-utils/src/FramebufferCache.cpp index c7cb6451f7..0bcb54fe2c 100644 --- a/libraries/render-utils/src/FramebufferCache.cpp +++ b/libraries/render-utils/src/FramebufferCache.cpp @@ -39,6 +39,7 @@ void FramebufferCache::setFrameBufferSize(QSize frameBufferSize) { _primaryColorTexture.reset(); _primaryNormalTexture.reset(); _primarySpecularTexture.reset(); + _selfieFramebuffer.reset(); _cachedFramebuffers.clear(); } } @@ -112,7 +113,6 @@ gpu::FramebufferPointer FramebufferCache::getFramebuffer() { return result; } - void FramebufferCache::releaseFramebuffer(const gpu::FramebufferPointer& framebuffer) { if (QSize(framebuffer->getSize().x, framebuffer->getSize().y) == _frameBufferSize) { _cachedFramebuffers.push_back(framebuffer); @@ -126,3 +126,10 @@ gpu::FramebufferPointer FramebufferCache::getShadowFramebuffer() { } return _shadowFramebuffer; } + +gpu::FramebufferPointer FramebufferCache::getSelfieFramebuffer() { + if (!_selfieFramebuffer) { + _selfieFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create(gpu::Element::COLOR_RGBA_32, _frameBufferSize.width(), _frameBufferSize.height())); + } + return _selfieFramebuffer; +} diff --git a/libraries/render-utils/src/FramebufferCache.h b/libraries/render-utils/src/FramebufferCache.h index ca01a470d9..d65c467be1 100644 --- a/libraries/render-utils/src/FramebufferCache.h +++ b/libraries/render-utils/src/FramebufferCache.h @@ -39,8 +39,12 @@ public: /// Returns the framebuffer object used to render shadow maps; gpu::FramebufferPointer getShadowFramebuffer(); + /// Returns the framebuffer object used to render selfie maps; + gpu::FramebufferPointer getSelfieFramebuffer(); + /// Returns a free framebuffer with a single color attachment for temp or intra-frame operations gpu::FramebufferPointer getFramebuffer(); + // TODO add sync functionality to the release, so we don't reuse a framebuffer being read from /// Releases a free framebuffer back for reuse void releaseFramebuffer(const gpu::FramebufferPointer& framebuffer); @@ -58,6 +62,9 @@ private: gpu::TexturePointer _primarySpecularTexture; gpu::FramebufferPointer _shadowFramebuffer; + + gpu::FramebufferPointer _selfieFramebuffer; + QSize _frameBufferSize{ 100, 100 }; }; diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 6c03d57de3..d9b40a70f9 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1749,7 +1749,7 @@ QSharedPointer GeometryCache::createResource(const QUrl& url, const QS return geometry.staticCast(); } -void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch) { +void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { if (!_standardDrawPipeline) { auto vs = gpu::ShaderPointer(gpu::Shader::createVertex(std::string(standardTransformPNTC_vert))); auto ps = gpu::ShaderPointer(gpu::Shader::createPixel(std::string(standardDrawTexture_frag))); @@ -1762,8 +1762,18 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch) { state->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); _standardDrawPipeline.reset(gpu::Pipeline::create(program, state)); + + auto stateNoBlend = std::make_shared(); + stateNoBlend->setColorWriteMask(true, true, true, false); + stateNoBlend->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); + + _standardDrawPipelineNoBlend.reset(gpu::Pipeline::create(program, stateNoBlend)); + } + if (noBlend) { + batch.setPipeline(_standardDrawPipelineNoBlend); + } else { + batch.setPipeline(_standardDrawPipeline); } - batch.setPipeline(_standardDrawPipeline); } const float NetworkGeometry::NO_HYSTERESIS = -1.0f; diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 147b4f8093..dbf9c4a95d 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -248,7 +248,7 @@ public: QSharedPointer getGeometry(const QUrl& url, const QUrl& fallback = QUrl(), bool delayLoad = false); /// Set a batch to the simple pipeline, returning the previous pipeline - void useSimpleDrawPipeline(gpu::Batch& batch); + void useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend = false); protected: @@ -263,6 +263,7 @@ private: typedef QPair VerticesIndices; gpu::PipelinePointer _standardDrawPipeline; + gpu::PipelinePointer _standardDrawPipelineNoBlend; QHash _cubeVerticies; QHash _cubeColors; gpu::BufferPointer _wireCubeIndexBuffer; From e6262de864f1607c3be7ba52153ee6f21240e462 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 14:56:52 -0700 Subject: [PATCH 18/36] CR feedback --- libraries/render-utils/src/DeferredBufferWrite.slh | 3 +++ libraries/render-utils/src/simple.slf | 3 +-- libraries/render-utils/src/simple_textured.slf | 3 +-- libraries/render-utils/src/simple_textured_emisive.slf | 3 +-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/render-utils/src/DeferredBufferWrite.slh b/libraries/render-utils/src/DeferredBufferWrite.slh index a7f4055bba..ab070db9e9 100755 --- a/libraries/render-utils/src/DeferredBufferWrite.slh +++ b/libraries/render-utils/src/DeferredBufferWrite.slh @@ -21,6 +21,9 @@ float evalOpaqueFinalAlpha(float alpha, float mapAlpha) { return mix(alpha * glowIntensity, 1.0 - alpha * glowIntensity, step(mapAlpha, alphaThreshold)); } +const DEFAULT_SPECULAR = vec3(0.1); +const DEFAULT_SHININESS = 10; + void packDeferredFragment(vec3 normal, float alpha, vec3 diffuse, vec3 specular, float shininess) { if (alpha != glowIntensity) { discard; diff --git a/libraries/render-utils/src/simple.slf b/libraries/render-utils/src/simple.slf index 1d77dde0d3..84f1752c37 100644 --- a/libraries/render-utils/src/simple.slf +++ b/libraries/render-utils/src/simple.slf @@ -22,6 +22,5 @@ void main(void) { normalize(interpolatedNormal.xyz), glowIntensity, gl_Color.rgb, - vec3(0.1), - 10); + DEFAULT_SPECULAR, DEFAULT_SHININESS); } diff --git a/libraries/render-utils/src/simple_textured.slf b/libraries/render-utils/src/simple_textured.slf index 88b196543c..68d235050d 100644 --- a/libraries/render-utils/src/simple_textured.slf +++ b/libraries/render-utils/src/simple_textured.slf @@ -27,6 +27,5 @@ void main(void) { normalize(interpolatedNormal.xyz), glowIntensity * texel.a, gl_Color.rgb * texel.rgb, - vec3(0.1), - 10); + DEFAULT_SPECULAR, DEFAULT_SHININESS); } \ No newline at end of file diff --git a/libraries/render-utils/src/simple_textured_emisive.slf b/libraries/render-utils/src/simple_textured_emisive.slf index c5749114f3..d2c7d403c3 100644 --- a/libraries/render-utils/src/simple_textured_emisive.slf +++ b/libraries/render-utils/src/simple_textured_emisive.slf @@ -27,7 +27,6 @@ void main(void) { normalize(interpolatedNormal.xyz), glowIntensity * texel.a, gl_Color.rgb, - vec3(0.1), - 10, + DEFAULT_SPECULAR, DEFAULT_SHININESS, texel.rgb); } \ No newline at end of file From 5f33ac545907426624c2c3ed17f430d714b552dc Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:30:54 -0700 Subject: [PATCH 19/36] Update script print() to be variadic --- libraries/script-engine/src/ScriptEngine.cpp | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 7bb62a01ab..99497c7f55 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -46,13 +46,21 @@ static QScriptValue debugPrint(QScriptContext* context, QScriptEngine* engine){ - qCDebug(scriptengine) << "script:print()<<" << context->argument(0).toString(); - QString message = context->argument(0).toString() - .replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("'", "\\'"); - engine->evaluate("Script.print('" + message + "')"); + QString message = ""; + for (int i = 0; i < context->argumentCount(); i++) { + if (i > 0) { + message += " "; + } + message += context->argument(i).toString(); + } + qCDebug(scriptengine) << "script:print()<<" << message; + + QString scriptMessage = message.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("'", "\\'"); + engine->evaluate("Script.print('" + scriptMessage + "')"); + return QScriptValue(); } From 396bb613f8a1a3a9c382c0179bce5bce972f3339 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 23 Jul 2015 15:32:40 -0700 Subject: [PATCH 20/36] Avoid creating a new variable in js print --- libraries/script-engine/src/ScriptEngine.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 99497c7f55..d5e727657c 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -55,11 +55,11 @@ static QScriptValue debugPrint(QScriptContext* context, QScriptEngine* engine){ } qCDebug(scriptengine) << "script:print()<<" << message; - QString scriptMessage = message.replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("'", "\\'"); - engine->evaluate("Script.print('" + scriptMessage + "')"); + message = message.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("'", "\\'"); + engine->evaluate("Script.print('" + message + "')"); return QScriptValue(); } From 8007fba909961905a36077c6867925625ae284bb Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 23 Jul 2015 15:37:56 -0700 Subject: [PATCH 21/36] add type to shader include --- libraries/render-utils/src/DeferredBufferWrite.slh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/render-utils/src/DeferredBufferWrite.slh b/libraries/render-utils/src/DeferredBufferWrite.slh index ab070db9e9..a3edfbdbbb 100755 --- a/libraries/render-utils/src/DeferredBufferWrite.slh +++ b/libraries/render-utils/src/DeferredBufferWrite.slh @@ -21,8 +21,8 @@ float evalOpaqueFinalAlpha(float alpha, float mapAlpha) { return mix(alpha * glowIntensity, 1.0 - alpha * glowIntensity, step(mapAlpha, alphaThreshold)); } -const DEFAULT_SPECULAR = vec3(0.1); -const DEFAULT_SHININESS = 10; +const vec3 DEFAULT_SPECULAR = vec3(0.1); +const float DEFAULT_SHININESS = 10; void packDeferredFragment(vec3 normal, float alpha, vec3 diffuse, vec3 specular, float shininess) { if (alpha != glowIntensity) { From be9d99264feec6b26df35820a86773af18e7133f Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 16:15:10 -0700 Subject: [PATCH 22/36] Introducing the download Framebuffer call on the context to capture a snapshot --- interface/src/Application.cpp | 26 ++++------------------- libraries/gpu/src/gpu/Batch.h | 1 - libraries/gpu/src/gpu/Context.cpp | 6 +++++- libraries/gpu/src/gpu/Context.h | 6 ++++++ libraries/gpu/src/gpu/Framebuffer.h | 2 -- libraries/gpu/src/gpu/GLBackend.h | 4 ++++ libraries/gpu/src/gpu/GLBackendOutput.cpp | 12 +++++++++++ 7 files changed, 31 insertions(+), 26 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index e7479b94d8..ced6d2b301 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2998,18 +2998,8 @@ PickRay Application::computePickRay(float x, float y) const { } QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { -/* auto primaryFramebuffer = DependencyManager::get()->getPrimaryFramebuffer(); - glBindFramebuffer(GL_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(primaryFramebuffer)); - // clear the alpha channel so the background is transparent - glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_TRUE); - glClearColor(0.0, 0.0, 0.0, 0.0); - glClear(GL_COLOR_BUFFER_BIT); - glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE); - */ - - // const int BILLBOARD_SIZE = 64; - const int BILLBOARD_SIZE = 256; + const int BILLBOARD_SIZE = 64; _glWidget->makeCurrent(); auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); @@ -3020,26 +3010,18 @@ QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { batch.setFramebuffer(primaryFbo); renderArgs->_context->render(batch); } - renderArgs->_renderMode = RenderArgs::MIRROR_RENDER_MODE; - renderRearViewMirror(renderArgs, QRect(0, /*_glWidget->getDeviceHeight() - BILLBOARD_SIZE*/ 0, - BILLBOARD_SIZE, BILLBOARD_SIZE), - false); - renderArgs->_renderMode = RenderArgs::NORMAL_RENDER_MODE; + renderArgs->_renderMode = RenderArgs::NORMAL_RENDER_MODE; + renderRearViewMirror(renderArgs, QRect(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE), true); - auto selfieFbo = DependencyManager::get()->getSelfieFramebuffer(); { auto mirrorViewport = glm::ivec4(0, 0,BILLBOARD_SIZE, BILLBOARD_SIZE); gpu::Batch batch; - // batch.blit(primaryFbo, mirrorViewport, selfieFbo, mirrorViewport); batch.setFramebuffer(nullptr); renderArgs->_context->render(batch); } - QImage image(BILLBOARD_SIZE, BILLBOARD_SIZE, QImage::Format_ARGB32); - glBindFramebuffer(GL_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(primaryFbo)); - glReadPixels(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE, GL_BGRA, GL_UNSIGNED_BYTE, image.bits()); - glBindFramebuffer(GL_FRAMEBUFFER, 0); + renderArgs->_context->downloadFramebuffer(primaryFbo, glm::ivec4(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE), image); return image; } diff --git a/libraries/gpu/src/gpu/Batch.h b/libraries/gpu/src/gpu/Batch.h index 2f1d2e8ece..acc1f6fdac 100644 --- a/libraries/gpu/src/gpu/Batch.h +++ b/libraries/gpu/src/gpu/Batch.h @@ -108,7 +108,6 @@ public: void blit(const FramebufferPointer& src, const Vec4i& srcViewport, const FramebufferPointer& dst, const Vec4i& dstViewport); - // Query Section void beginQuery(const QueryPointer& query); void endQuery(const QueryPointer& query); diff --git a/libraries/gpu/src/gpu/Context.cpp b/libraries/gpu/src/gpu/Context.cpp index 51335f78df..604f39f46f 100644 --- a/libraries/gpu/src/gpu/Context.cpp +++ b/libraries/gpu/src/gpu/Context.cpp @@ -40,4 +40,8 @@ void Context::render(Batch& batch) { void Context::syncCache() { PROFILE_RANGE(__FUNCTION__); _backend->syncCache(); -} \ No newline at end of file +} + +void Context::downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage) { + _backend->downloadFramebuffer(srcFramebuffer, region, destImage); +} diff --git a/libraries/gpu/src/gpu/Context.h b/libraries/gpu/src/gpu/Context.h index 484c772c71..0c4ead6eae 100644 --- a/libraries/gpu/src/gpu/Context.h +++ b/libraries/gpu/src/gpu/Context.h @@ -28,6 +28,8 @@ public: virtual~ Backend() {}; virtual void render(Batch& batch) = 0; virtual void syncCache() = 0; + virtual void downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage) = 0; + class TransformObject { public: @@ -121,6 +123,10 @@ public: void syncCache(); + // Downloading the Framebuffer is a synchronous action that is not efficient. + // It s here for convenience to easily capture a snapshot + void downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage); + protected: Context(const Context& context); diff --git a/libraries/gpu/src/gpu/Framebuffer.h b/libraries/gpu/src/gpu/Framebuffer.h index 5a1504d25c..c1b5358d3c 100755 --- a/libraries/gpu/src/gpu/Framebuffer.h +++ b/libraries/gpu/src/gpu/Framebuffer.h @@ -14,8 +14,6 @@ #include "Texture.h" #include -class QImage; - namespace gpu { typedef Element Format; diff --git a/libraries/gpu/src/gpu/GLBackend.h b/libraries/gpu/src/gpu/GLBackend.h index 0e6f181028..c924395334 100644 --- a/libraries/gpu/src/gpu/GLBackend.h +++ b/libraries/gpu/src/gpu/GLBackend.h @@ -38,6 +38,10 @@ public: // Let's try to avoid to do that as much as possible! virtual void syncCache(); + // This is the ugly "download the pixels to sysmem for taking a snapshot" + // Just avoid using it, it's ugly and will break performances + virtual void downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage); + static bool checkGLError(const char* name = nullptr); // Only checks in debug builds diff --git a/libraries/gpu/src/gpu/GLBackendOutput.cpp b/libraries/gpu/src/gpu/GLBackendOutput.cpp index 621f04b71b..ec1b9de341 100755 --- a/libraries/gpu/src/gpu/GLBackendOutput.cpp +++ b/libraries/gpu/src/gpu/GLBackendOutput.cpp @@ -11,6 +11,8 @@ #include "GPULogging.h" #include "GLBackendShared.h" +#include "qimage.h" + using namespace gpu; GLBackend::GLFramebuffer::GLFramebuffer() {} @@ -191,3 +193,13 @@ void GLBackend::do_blit(Batch& batch, uint32 paramOffset) { glBindFramebuffer(GL_DRAW_FRAMEBUFFER, getFramebufferID(_output._framebuffer)); } } + + +void GLBackend::downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage) { + + glBindFramebuffer(GL_READ_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(srcFramebuffer)); + glReadPixels(region.x, region.y, region.z, region.w, GL_BGRA, GL_UNSIGNED_BYTE, destImage.bits()); + glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); + + (void) CHECK_GL_ERROR(); +} \ No newline at end of file From d56b5f39fb3a3d9bb95bbbdd86bcb7a2e0aa221b Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 16:31:11 -0700 Subject: [PATCH 23/36] strenghening the checks in GLBackend::downloadFramebuffer before actually calling the true glReadPixels --- interface/src/Application.cpp | 2 -- libraries/gpu/src/gpu/GLBackendOutput.cpp | 22 ++++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ced6d2b301..f073e5cb26 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3436,7 +3436,6 @@ void Application::renderRearViewMirror(RenderArgs* renderArgs, const QRect& regi gpu::Vec4i viewport; if (billboard) { QSize size = DependencyManager::get()->getFrameBufferSize(); - // viewport = gpu::Vec4i(region.x(), size.height() - region.y() - region.height(), region.width(), region.height()); viewport = gpu::Vec4i(0, 0, region.width(), region.height()); } else { // if not rendering the billboard, the region is in device independent coordinates; must convert to device @@ -3446,7 +3445,6 @@ void Application::renderRearViewMirror(RenderArgs* renderArgs, const QRect& regi int y = region.y() * ratio; int width = region.width() * ratio; int height = region.height() * ratio; - // viewport = gpu::Vec4i(x, size.height() - y - height, width, height); viewport = gpu::Vec4i(0, 0, width, height); } renderArgs->_viewport = viewport; diff --git a/libraries/gpu/src/gpu/GLBackendOutput.cpp b/libraries/gpu/src/gpu/GLBackendOutput.cpp index ec1b9de341..44dee90fa7 100755 --- a/libraries/gpu/src/gpu/GLBackendOutput.cpp +++ b/libraries/gpu/src/gpu/GLBackendOutput.cpp @@ -196,9 +196,27 @@ void GLBackend::do_blit(Batch& batch, uint32 paramOffset) { void GLBackend::downloadFramebuffer(const FramebufferPointer& srcFramebuffer, const Vec4i& region, QImage& destImage) { - + auto readFBO = gpu::GLBackend::getFramebufferID(srcFramebuffer); + if (srcFramebuffer && readFBO) { + if ((srcFramebuffer->getWidth() < (region.x + region.z)) || (srcFramebuffer->getHeight() < (region.y + region.w))) { + qCDebug(gpulogging) << "GLBackend::downloadFramebuffer : srcFramebuffer is too small to provide the region queried"; + return; + } + } + + if ((destImage.width() < region.z) || (destImage.height() < region.w)) { + qCDebug(gpulogging) << "GLBackend::downloadFramebuffer : destImage is too small to receive the region of the framebuffer"; + return; + } + + GLenum format = GL_BGRA; + if (destImage.format() != QImage::Format_ARGB32) { + qCDebug(gpulogging) << "GLBackend::downloadFramebuffer : destImage format must be FORMAT_ARGB32 to receive the region of the framebuffer"; + return; + } + glBindFramebuffer(GL_READ_FRAMEBUFFER, gpu::GLBackend::getFramebufferID(srcFramebuffer)); - glReadPixels(region.x, region.y, region.z, region.w, GL_BGRA, GL_UNSIGNED_BYTE, destImage.bits()); + glReadPixels(region.x, region.y, region.z, region.w, format, GL_UNSIGNED_BYTE, destImage.bits()); glBindFramebuffer(GL_READ_FRAMEBUFFER, 0); (void) CHECK_GL_ERROR(); From a424ff98ca06157d94ffa04c9be9221bf1f54d08 Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 16:55:18 -0700 Subject: [PATCH 24/36] Fixing issues with the review process --- interface/src/Application.cpp | 7 ++----- interface/src/avatar/MyAvatar.cpp | 2 ++ libraries/gpu/src/gpu/Context.h | 2 ++ libraries/gpu/src/gpu/GLBackendOutput.cpp | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f073e5cb26..efe208a1b8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -962,10 +962,6 @@ void Application::paintGL() { _myCamera.update(1.0f / _fps); } - // Sync up the View Furstum with the camera - // loadViewFrustum(_myCamera, _viewFrustum); - - renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; if (OculusManager::isConnected()) { @@ -3010,7 +3006,8 @@ QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { batch.setFramebuffer(primaryFbo); renderArgs->_context->render(batch); } - renderArgs->_renderMode = RenderArgs::NORMAL_RENDER_MODE; + renderArgs->_renderMode = RenderArgs::DEFAULT_RENDER_MODE; + renderRearViewMirror(renderArgs, QRect(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE), true); { diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 9752536f60..c6c6919325 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -1551,7 +1551,9 @@ void MyAvatar::maybeUpdateBillboard() { QBuffer buffer(&_billboard); buffer.open(QIODevice::WriteOnly); image.save(&buffer, "PNG"); +#ifdef DEBUG image.save("billboard.png", "PNG"); +#endif _billboardValid = true; sendBillboardPacket(); diff --git a/libraries/gpu/src/gpu/Context.h b/libraries/gpu/src/gpu/Context.h index 0c4ead6eae..ab7a1d1c11 100644 --- a/libraries/gpu/src/gpu/Context.h +++ b/libraries/gpu/src/gpu/Context.h @@ -20,6 +20,8 @@ #include "Pipeline.h" #include "Framebuffer.h" +class QImage; + namespace gpu { class Backend { diff --git a/libraries/gpu/src/gpu/GLBackendOutput.cpp b/libraries/gpu/src/gpu/GLBackendOutput.cpp index 44dee90fa7..7245272131 100755 --- a/libraries/gpu/src/gpu/GLBackendOutput.cpp +++ b/libraries/gpu/src/gpu/GLBackendOutput.cpp @@ -8,10 +8,11 @@ // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include + #include "GPULogging.h" #include "GLBackendShared.h" -#include "qimage.h" using namespace gpu; From ded95a3dde482f21d48577d370559561b3f91fbf Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 16:58:21 -0700 Subject: [PATCH 25/36] removing dead code --- libraries/gpu/src/gpu/Framebuffer.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/gpu/src/gpu/Framebuffer.h b/libraries/gpu/src/gpu/Framebuffer.h index c1b5358d3c..6f2b762bb0 100755 --- a/libraries/gpu/src/gpu/Framebuffer.h +++ b/libraries/gpu/src/gpu/Framebuffer.h @@ -132,8 +132,6 @@ public: static const uint32 MAX_NUM_RENDER_BUFFERS = 8; static uint32 getMaxNumRenderBuffers() { return MAX_NUM_RENDER_BUFFERS; } - void getImage(QImage* result) const; - protected: SwapchainPointer _swapchain; From f919d2985b89600443132aec16571ee3f30e81bf Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 23 Jul 2015 17:17:15 -0700 Subject: [PATCH 26/36] don't attempt to send the avatar billboard if too large --- libraries/avatars/src/AvatarData.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 3a3b895c66..23549cc69a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1114,11 +1114,13 @@ void AvatarData::sendIdentityPacket() { void AvatarData::sendBillboardPacket() { if (!_billboard.isEmpty()) { auto nodeList = DependencyManager::get(); - - auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, _billboard.size()); - billboardPacket->write(_billboard); - - nodeList->broadcastToNodes(std::move(billboardPacket), NodeSet() << NodeType::AvatarMixer); + + if (_billboard.size() <= NLPacket::maxPayloadSize(PacketType::AvatarBillboard)) { + auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, _billboard.size()); + billboardPacket->write(_billboard); + + nodeList->broadcastToNodes(std::move(billboardPacket), NodeSet() << NodeType::AvatarMixer); + } } } From 974108734adca48a99cafd6427e2488596d664d5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 23 Jul 2015 17:17:59 -0700 Subject: [PATCH 27/36] add a comment for size guard --- libraries/avatars/src/AvatarData.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 23549cc69a..5b970a95a3 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1115,6 +1115,8 @@ void AvatarData::sendBillboardPacket() { if (!_billboard.isEmpty()) { auto nodeList = DependencyManager::get(); + // This makes sure the billboard won't be too large to send. + // Once more protocol changes are done and we can send blocks of data we can support sending > MTU sized billboards. if (_billboard.size() <= NLPacket::maxPayloadSize(PacketType::AvatarBillboard)) { auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, _billboard.size()); billboardPacket->write(_billboard); From 2fc158246dd8504bc524182f708286d2a2529993 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 23 Jul 2015 17:30:50 -0700 Subject: [PATCH 28/36] decouple FPS from rotation rate during mouse drag --- interface/src/Application.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8f825a2768..34791b29f1 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2495,10 +2495,18 @@ void Application::update(float deltaTime) { _myAvatar->setDriveKeys(DOWN, _userInputMapper.getActionState(UserInputMapper::VERTICAL_DOWN)); _myAvatar->setDriveKeys(LEFT, _userInputMapper.getActionState(UserInputMapper::LATERAL_LEFT)); _myAvatar->setDriveKeys(RIGHT, _userInputMapper.getActionState(UserInputMapper::LATERAL_RIGHT)); - _myAvatar->setDriveKeys(ROT_UP, _userInputMapper.getActionState(UserInputMapper::PITCH_UP)); - _myAvatar->setDriveKeys(ROT_DOWN, _userInputMapper.getActionState(UserInputMapper::PITCH_DOWN)); - _myAvatar->setDriveKeys(ROT_LEFT, _userInputMapper.getActionState(UserInputMapper::YAW_LEFT)); - _myAvatar->setDriveKeys(ROT_RIGHT, _userInputMapper.getActionState(UserInputMapper::YAW_RIGHT)); + if (deltaTime > FLT_EPSILON) { + // For rotations what we really want are meausures of "angles per second" (in order to prevent + // fps-dependent spin rates) so we need to scale the units of the controller contribution. + // (TODO?: maybe we should similarly scale ALL action state info, or change the expected behavior + // controllers to provide a delta_per_second value rather than a raw delta.) + const float EXPECTED_FRAME_RATE = 60.0f; + float timeFactor = EXPECTED_FRAME_RATE * deltaTime; + _myAvatar->setDriveKeys(ROT_UP, _userInputMapper.getActionState(UserInputMapper::PITCH_UP) / timeFactor); + _myAvatar->setDriveKeys(ROT_DOWN, _userInputMapper.getActionState(UserInputMapper::PITCH_DOWN) / timeFactor); + _myAvatar->setDriveKeys(ROT_LEFT, _userInputMapper.getActionState(UserInputMapper::YAW_LEFT) / timeFactor); + _myAvatar->setDriveKeys(ROT_RIGHT, _userInputMapper.getActionState(UserInputMapper::YAW_RIGHT) / timeFactor); + } } _myAvatar->setDriveKeys(BOOM_IN, _userInputMapper.getActionState(UserInputMapper::BOOM_IN)); _myAvatar->setDriveKeys(BOOM_OUT, _userInputMapper.getActionState(UserInputMapper::BOOM_OUT)); From cdcab299b65c9ca400881e1e25c13be104279b5f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 23 Jul 2015 17:58:39 -0700 Subject: [PATCH 29/36] bring back delayed queue of Application events --- interface/src/Application.cpp | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8e8b4544b7..b237d003e0 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1798,13 +1798,6 @@ void Application::idle() { } double timeSinceLastUpdate = (double)_lastTimeUpdated.nsecsElapsed() / 1000000.0; if (timeSinceLastUpdate > targetFramePeriod) { - - { - static const int IDLE_EVENT_PROCESS_MAX_TIME_MS = 2; - PerformanceTimer perfTimer("processEvents"); - processEvents(QEventLoop::AllEvents, IDLE_EVENT_PROCESS_MAX_TIME_MS); - } - _lastTimeUpdated.start(); { PerformanceTimer perfTimer("update"); @@ -1836,12 +1829,8 @@ void Application::idle() { // Once rendering is off on another thread we should be able to have Application::idle run at start(0) in // perpetuity and not expect events to get backed up. - static const int IDLE_TIMER_DELAY_MS = 0; - int desiredInterval = _glWidget->isThrottleRendering() ? THROTTLED_IDLE_TIMER_DELAY : IDLE_TIMER_DELAY_MS; - - if (idleTimer->interval() != desiredInterval) { - idleTimer->start(desiredInterval); - } + static const int IDLE_TIMER_DELAY_MS = 2; + idleTimer->start(_glWidget->isThrottleRendering() ? THROTTLED_IDLE_TIMER_DELAY : IDLE_TIMER_DELAY_MS); } // check for any requested background downloads. From 8421aafe33efc21215c2cc67bbe82f192365a43f Mon Sep 17 00:00:00 2001 From: samcake Date: Thu, 23 Jul 2015 18:17:10 -0700 Subject: [PATCH 30/36] try to fix the scaling issues on mac --- interface/src/Application.cpp | 7 +++---- interface/src/ui/ApplicationOverlay.cpp | 13 +++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index efe208a1b8..2e1a708fa2 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -891,10 +891,9 @@ void Application::paintGL() { renderArgs._renderMode = RenderArgs::NORMAL_RENDER_MODE; { - auto mirrorViewport = glm::ivec4(0, 0, _mirrorViewRect.width(), _mirrorViewRect.height()); - float ratio = 1.0f / ((float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale()); - - auto mirrorViewportDest = glm::ivec4(0, 0, ratio * mirrorViewport.z, ratio * mirrorViewport.w); + float ratio = ((float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale()); + auto mirrorViewport = glm::ivec4(0, 0, _mirrorViewRect.width() * ratio, _mirrorViewRect.height() * ratio); + auto mirrorViewportDest = mirrorViewport; auto selfieFbo = DependencyManager::get()->getSelfieFramebuffer(); gpu::Batch batch; diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index d1154bda03..643663e658 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -183,12 +183,17 @@ void ApplicationOverlay::renderRearView(RenderArgs* renderArgs) { batch.setProjectionTransform(legacyProjection); batch.setModelTransform(Transform()); batch.setViewTransform(Transform()); - + + float screenRatio = ((float)qApp->getDevicePixelRatio()); + float renderRatio = ((float)screenRatio * qApp->getRenderResolutionScale()); + auto viewport = qApp->getMirrorViewRect(); - glm::vec2 bottomLeft(viewport.left(), viewport.bottom()); - glm::vec2 topRight(viewport.right(), viewport.top()); + glm::vec2 bottomLeft(viewport.left(), viewport.top() + viewport.height()); + glm::vec2 topRight(viewport.left() + viewport.width(), viewport.top()); + bottomLeft *= screenRatio; + topRight *= screenRatio; glm::vec2 texCoordMinCorner(0.0f, 0.0f); - glm::vec2 texCoordMaxCorner(viewport.width() / float(selfieTexture->getWidth()), viewport.height() / float(selfieTexture->getHeight())); + glm::vec2 texCoordMaxCorner(viewport.width() * renderRatio / float(selfieTexture->getWidth()), viewport.height() * renderRatio / float(selfieTexture->getHeight())); geometryCache->useSimpleDrawPipeline(batch, false); batch.setResourceTexture(0, renderArgs->_whiteTexture); From c1e1caa7092c30d8bc9398990621e13d6296c064 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 23 Jul 2015 21:59:19 -0700 Subject: [PATCH 31/36] Add ability to update title of WebWindow after it's created --- interface/src/scripting/WebWindowClass.cpp | 4 ++++ interface/src/scripting/WebWindowClass.h | 1 + 2 files changed, 5 insertions(+) diff --git a/interface/src/scripting/WebWindowClass.cpp b/interface/src/scripting/WebWindowClass.cpp index 67e5e86280..3bd7e390ec 100644 --- a/interface/src/scripting/WebWindowClass.cpp +++ b/interface/src/scripting/WebWindowClass.cpp @@ -142,3 +142,7 @@ QScriptValue WebWindowClass::constructor(QScriptContext* context, QScriptEngine* return engine->newQObject(retVal); } + +void WebWindowClass::setTitle(const QString& title) { + _windowWidget->setWindowTitle(title); +} diff --git a/interface/src/scripting/WebWindowClass.h b/interface/src/scripting/WebWindowClass.h index f5a292874c..c44898fa97 100644 --- a/interface/src/scripting/WebWindowClass.h +++ b/interface/src/scripting/WebWindowClass.h @@ -48,6 +48,7 @@ public slots: void raise(); ScriptEventBridge* getEventBridge() const { return _eventBridge; } void addEventBridgeToWindowObject(); + void setTitle(const QString& title); signals: void closed(); From 66cd17ec6663e0800e20753e8a552ee0a7e1d94f Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Thu, 23 Jul 2015 23:09:40 -0700 Subject: [PATCH 32/36] REmove the crappy solution for drawing transparent texture into overlay by a proper shader --- interface/src/ui/ApplicationOverlay.cpp | 8 +++---- libraries/gpu/src/gpu/DrawTextureOpaque.slf | 22 ++++++++++++++++++++ libraries/gpu/src/gpu/StandardShaderLib.cpp | 11 ++++++++++ libraries/gpu/src/gpu/StandardShaderLib.h | 2 ++ libraries/model/src/model/Skybox.cpp | 2 +- libraries/render-utils/src/GeometryCache.cpp | 14 +++++++++---- 6 files changed, 50 insertions(+), 9 deletions(-) create mode 100755 libraries/gpu/src/gpu/DrawTextureOpaque.slf diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 643663e658..8ec2183681 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -92,12 +92,12 @@ void ApplicationOverlay::renderOverlay(RenderArgs* renderArgs) { batch.clearFramebuffer(gpu::Framebuffer::BUFFER_COLOR0 | gpu::Framebuffer::BUFFER_DEPTH, color, depth, stencil); // Now render the overlay components together into a single texture + renderRearView(renderArgs); // renders the mirror view selfie renderDomainConnectionStatusBorder(renderArgs); // renders the connected domain line renderAudioScope(renderArgs); // audio scope in the very back renderQmlUi(renderArgs); // renders a unit quad with the QML UI texture, and the text overlays from scripts renderOverlays(renderArgs); // renders Scripts Overlay and AudioScope renderStatsAndLogs(renderArgs); // currently renders nothing - renderRearView(renderArgs); // renders the mirror view selfie renderArgs->_context->syncCache(); renderArgs->_context->render(batch); @@ -195,12 +195,12 @@ void ApplicationOverlay::renderRearView(RenderArgs* renderArgs) { glm::vec2 texCoordMinCorner(0.0f, 0.0f); glm::vec2 texCoordMaxCorner(viewport.width() * renderRatio / float(selfieTexture->getWidth()), viewport.height() * renderRatio / float(selfieTexture->getHeight())); - geometryCache->useSimpleDrawPipeline(batch, false); - batch.setResourceTexture(0, renderArgs->_whiteTexture); - geometryCache->renderQuad(batch, bottomLeft, topRight, glm::vec4(0.0f, 0.0f, 0.0f, 1.0f)); geometryCache->useSimpleDrawPipeline(batch, true); batch.setResourceTexture(0, selfieTexture); geometryCache->renderQuad(batch, bottomLeft, topRight, texCoordMinCorner, texCoordMaxCorner, glm::vec4(1.0f, 1.0f, 1.0f, 1.0f)); + + batch.setResourceTexture(0, renderArgs->_whiteTexture); + geometryCache->useSimpleDrawPipeline(batch, false); } } diff --git a/libraries/gpu/src/gpu/DrawTextureOpaque.slf b/libraries/gpu/src/gpu/DrawTextureOpaque.slf new file mode 100755 index 0000000000..1ce3a34c02 --- /dev/null +++ b/libraries/gpu/src/gpu/DrawTextureOpaque.slf @@ -0,0 +1,22 @@ +<@include gpu/Config.slh@> +<$VERSION_HEADER$> +// Generated on <$_SCRIBE_DATE$> +// +// Draw texture 0 fetched at texcoord.xy +// Alpha is 1 +// +// Created by Sam Gateau on 6/22/2015 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +uniform sampler2D colorMap; + +varying vec2 varTexcoord; + +void main(void) { + gl_FragColor = vec4(texture2D(colorMap, varTexcoord).xyz, 1.0); +} diff --git a/libraries/gpu/src/gpu/StandardShaderLib.cpp b/libraries/gpu/src/gpu/StandardShaderLib.cpp index 1c1cca458a..3f27a7fc35 100755 --- a/libraries/gpu/src/gpu/StandardShaderLib.cpp +++ b/libraries/gpu/src/gpu/StandardShaderLib.cpp @@ -16,6 +16,7 @@ #include "DrawTexcoordRectTransformUnitQuad_vert.h" #include "DrawViewportQuadTransformTexcoord_vert.h" #include "DrawTexture_frag.h" +#include "DrawTextureOpaque_frag.h" #include "DrawColoredTexture_frag.h" using namespace gpu; @@ -24,6 +25,7 @@ ShaderPointer StandardShaderLib::_drawTransformUnitQuadVS; ShaderPointer StandardShaderLib::_drawTexcoordRectTransformUnitQuadVS; ShaderPointer StandardShaderLib::_drawViewportQuadTransformTexcoordVS; ShaderPointer StandardShaderLib::_drawTexturePS; +ShaderPointer StandardShaderLib::_drawTextureOpaquePS; ShaderPointer StandardShaderLib::_drawColoredTexturePS; StandardShaderLib::ProgramMap StandardShaderLib::_programs; @@ -82,6 +84,15 @@ ShaderPointer StandardShaderLib::getDrawTexturePS() { return _drawTexturePS; } +ShaderPointer StandardShaderLib::getDrawTextureOpaquePS() { + if (!_drawTextureOpaquePS) { + _drawTextureOpaquePS = gpu::ShaderPointer(gpu::Shader::createPixel(std::string(DrawTextureOpaque_frag))); + } + return _drawTextureOpaquePS; +} + + + ShaderPointer StandardShaderLib::getDrawColoredTexturePS() { if (!_drawColoredTexturePS) { _drawColoredTexturePS = gpu::ShaderPointer(gpu::Shader::createPixel(std::string(DrawColoredTexture_frag))); diff --git a/libraries/gpu/src/gpu/StandardShaderLib.h b/libraries/gpu/src/gpu/StandardShaderLib.h index d5a3685feb..2d9c168473 100755 --- a/libraries/gpu/src/gpu/StandardShaderLib.h +++ b/libraries/gpu/src/gpu/StandardShaderLib.h @@ -35,6 +35,7 @@ public: static ShaderPointer getDrawViewportQuadTransformTexcoordVS(); static ShaderPointer getDrawTexturePS(); + static ShaderPointer getDrawTextureOpaquePS(); static ShaderPointer getDrawColoredTexturePS(); // The shader program combining the shaders available above, so they are unique @@ -47,6 +48,7 @@ protected: static ShaderPointer _drawTexcoordRectTransformUnitQuadVS; static ShaderPointer _drawViewportQuadTransformTexcoordVS; static ShaderPointer _drawTexturePS; + static ShaderPointer _drawTextureOpaquePS; static ShaderPointer _drawColoredTexturePS; typedef std::map, ShaderPointer> ProgramMap; diff --git a/libraries/model/src/model/Skybox.cpp b/libraries/model/src/model/Skybox.cpp index 1464afa45d..314492881f 100755 --- a/libraries/model/src/model/Skybox.cpp +++ b/libraries/model/src/model/Skybox.cpp @@ -110,7 +110,7 @@ void Skybox::render(gpu::Batch& batch, const ViewFrustum& viewFrustum, const Sky } else { // skybox has no cubemap, just clear the color buffer auto color = skybox.getColor(); - batch.clearFramebuffer(gpu::Framebuffer::BUFFER_COLOR0, glm::vec4(color, 0.0f), 0.0f, 0); + batch.clearFramebuffer(gpu::Framebuffer::BUFFER_COLOR0, glm::vec4(color, 0.0f), 0.0f, 0, true); } } diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 45106cf1b6..595e901919 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -29,6 +29,8 @@ #include "standardTransformPNTC_vert.h" #include "standardDrawTexture_frag.h" +#include "gpu/StandardShaderLib.h" + //#define WANT_DEBUG const int GeometryCache::UNKNOWN_ID = -1; @@ -1660,16 +1662,20 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { auto state = std::make_shared(); + // enable decal blend state->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); _standardDrawPipeline.reset(gpu::Pipeline::create(program, state)); - auto stateNoBlend = std::make_shared(); - stateNoBlend->setColorWriteMask(true, true, true, false); - stateNoBlend->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); - _standardDrawPipelineNoBlend.reset(gpu::Pipeline::create(program, stateNoBlend)); + auto stateNoBlend = std::make_shared(); + // stateNoBlend->setColorWriteMask(true, true, true, false); + // stateNoBlend->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); + + auto programNoBlend = gpu::ShaderPointer(gpu::Shader::createProgram(vs, gpu::StandardShaderLib::getDrawTextureOpaquePS())); + gpu::Shader::makeProgram((*programNoBlend)); + _standardDrawPipelineNoBlend.reset(gpu::Pipeline::create(programNoBlend, stateNoBlend)); } if (noBlend) { batch.setPipeline(_standardDrawPipelineNoBlend); From cb375e8ecee5ce439e2714eb3635bfa44050c1c4 Mon Sep 17 00:00:00 2001 From: Sam Gateau Date: Fri, 24 Jul 2015 02:15:50 -0700 Subject: [PATCH 33/36] FIund a nasty bug with the scissor which was the reason for many issues, tried to integrate the framebuffer setup and clear as part of the enginepipeline so calling display side is simpler --- interface/src/Application.cpp | 75 ++++--------------- libraries/gpu/src/gpu/Batch.cpp | 5 +- libraries/gpu/src/gpu/GLBackendState.cpp | 7 +- .../src/DeferredLightingEffect.cpp | 14 ++-- .../render-utils/src/FramebufferCache.cpp | 27 +++++-- libraries/render-utils/src/FramebufferCache.h | 4 +- .../render-utils/src/RenderDeferredTask.cpp | 48 ++++++------ .../render-utils/src/RenderDeferredTask.h | 7 ++ 8 files changed, 79 insertions(+), 108 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 2e1a708fa2..8c5769c26c 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -879,16 +879,11 @@ void Application::paintGL() { renderArgs._context->syncCache(); if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { - auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); - { - gpu::Batch batch; - batch.setFramebuffer(nullptr); - batch.setFramebuffer(primaryFbo); - renderArgs._context->render(batch); - } + auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); + renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; renderRearViewMirror(&renderArgs, _mirrorViewRect); - renderArgs._renderMode = RenderArgs::NORMAL_RENDER_MODE; + renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; { float ratio = ((float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale()); @@ -918,6 +913,9 @@ void Application::paintGL() { Application::getInstance()->cameraMenuChanged(); } + // The render mode is default or mirror if the camera is in mirror mode, assigned further below + renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; + if (_myCamera.getMode() == CAMERA_MODE_FIRST_PERSON) { // Always use the default eye position, not the actual head eye position. // Using the latter will cause the camera to wobble with idle animations, @@ -954,6 +952,7 @@ void Application::paintGL() { glm::vec3(0, _raiseMirror * _myAvatar->getScale(), 0) + (_myAvatar->getOrientation() * glm::quat(glm::vec3(0.0f, _rotateMirror, 0.0f))) * glm::vec3(0.0f, 0.0f, -1.0f) * MIRROR_FULLSCREEN_DISTANCE * _scaleMirror); + renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; } // Update camera position @@ -961,8 +960,6 @@ void Application::paintGL() { _myCamera.update(1.0f / _fps); } - renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; - if (OculusManager::isConnected()) { //When in mirror mode, use camera rotation. Otherwise, use body rotation if (_myCamera.getMode() == CAMERA_MODE_MIRROR) { @@ -974,31 +971,16 @@ void Application::paintGL() { TV3DManager::display(&renderArgs, _myCamera); } else { PROFILE_RANGE(__FUNCTION__ "/mainRender"); + // Viewport is assigned to the size of the framebuffer + QSize size = DependencyManager::get()->getFrameBufferSize(); + renderArgs._viewport = glm::ivec4(0, 0, size.width(), size.height()); - { - gpu::Batch batch; - auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); - batch.setFramebuffer(primaryFbo); - // clear the normal and specular buffers - batch.clearFramebuffer( - gpu::Framebuffer::BUFFER_COLOR0 | - gpu::Framebuffer::BUFFER_COLOR1 | - gpu::Framebuffer::BUFFER_COLOR2 | - gpu::Framebuffer::BUFFER_DEPTH, - vec4(vec3(0), 1), 1.0, 0.0); - - // Viewport is assigned to the size of the framebuffer - QSize size = DependencyManager::get()->getFrameBufferSize(); - renderArgs._viewport = glm::ivec4(0, 0, size.width(), size.height()); - batch.setViewportTransform(renderArgs._viewport); - renderArgs._context->render(batch); - } displaySide(&renderArgs, _myCamera); { auto geometryCache = DependencyManager::get(); - auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); + auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); gpu::Batch batch; batch.blit(primaryFbo, glm::ivec4(0, 0, _renderResolution.x, _renderResolution.y), nullptr, glm::ivec4(0, 0, _glWidget->getDeviceSize().width(), _glWidget->getDeviceSize().height())); @@ -1006,7 +988,6 @@ void Application::paintGL() { batch.setFramebuffer(nullptr); renderArgs._context->render(batch); - } _compositor.displayOverlayTexture(&renderArgs); @@ -2995,27 +2976,14 @@ PickRay Application::computePickRay(float x, float y) const { QImage Application::renderAvatarBillboard(RenderArgs* renderArgs) { const int BILLBOARD_SIZE = 64; + + // Need to make sure the gl context is current here _glWidget->makeCurrent(); - auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); - - { - gpu::Batch batch; - batch.setFramebuffer(nullptr); - batch.setFramebuffer(primaryFbo); - renderArgs->_context->render(batch); - } renderArgs->_renderMode = RenderArgs::DEFAULT_RENDER_MODE; - renderRearViewMirror(renderArgs, QRect(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE), true); - { - auto mirrorViewport = glm::ivec4(0, 0,BILLBOARD_SIZE, BILLBOARD_SIZE); - gpu::Batch batch; - batch.setFramebuffer(nullptr); - renderArgs->_context->render(batch); - } - + auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); QImage image(BILLBOARD_SIZE, BILLBOARD_SIZE, QImage::Format_ARGB32); renderArgs->_context->downloadFramebuffer(primaryFbo, glm::ivec4(0, 0, BILLBOARD_SIZE, BILLBOARD_SIZE), image); @@ -3445,25 +3413,10 @@ void Application::renderRearViewMirror(RenderArgs* renderArgs, const QRect& regi } renderArgs->_viewport = viewport; - { - gpu::Batch batch; - batch.setViewportTransform(viewport); - batch.setStateScissorRect(viewport); - batch.clearFramebuffer( - gpu::Framebuffer::BUFFER_COLOR0 | - gpu::Framebuffer::BUFFER_COLOR1 | - gpu::Framebuffer::BUFFER_COLOR2 | - gpu::Framebuffer::BUFFER_DEPTH, - vec4(vec3(0), 1), 1.0, 0.0, true); - // Viewport is assigned to the size of the framebuffer - renderArgs->_context->render(batch); - } - // render rear mirror view displaySide(renderArgs, _mirrorCamera, true, billboard); renderArgs->_viewport = originalViewport; - } void Application::resetSensors() { diff --git a/libraries/gpu/src/gpu/Batch.cpp b/libraries/gpu/src/gpu/Batch.cpp index 01c3c4ade7..567ce66cd8 100644 --- a/libraries/gpu/src/gpu/Batch.cpp +++ b/libraries/gpu/src/gpu/Batch.cpp @@ -220,10 +220,7 @@ void Batch::setStateBlendFactor(const Vec4& factor) { void Batch::setStateScissorRect(const Vec4i& rect) { ADD_COMMAND(setStateScissorRect); - _params.push_back(rect.x); - _params.push_back(rect.y); - _params.push_back(rect.z); - _params.push_back(rect.w); + _params.push_back(cacheData(sizeof(Vec4i), &rect)); } void Batch::setUniformBuffer(uint32 slot, const BufferPointer& buffer, Offset offset, Offset size) { diff --git a/libraries/gpu/src/gpu/GLBackendState.cpp b/libraries/gpu/src/gpu/GLBackendState.cpp index f4f27a5ee9..18fc9ddd3c 100644 --- a/libraries/gpu/src/gpu/GLBackendState.cpp +++ b/libraries/gpu/src/gpu/GLBackendState.cpp @@ -757,11 +757,8 @@ void GLBackend::do_setStateBlendFactor(Batch& batch, uint32 paramOffset) { } void GLBackend::do_setStateScissorRect(Batch& batch, uint32 paramOffset) { - - Vec4 rect(batch._params[paramOffset + 0]._float, - batch._params[paramOffset + 1]._float, - batch._params[paramOffset + 2]._float, - batch._params[paramOffset + 3]._float); + Vec4i rect; + memcpy(&rect, batch.editData(batch._params[paramOffset]._uint), sizeof(Vec4i)); glScissor(rect.x, rect.y, rect.z, rect.w); (void) CHECK_GL_ERROR(); diff --git a/libraries/render-utils/src/DeferredLightingEffect.cpp b/libraries/render-utils/src/DeferredLightingEffect.cpp index b527390589..c14bbfcb1d 100644 --- a/libraries/render-utils/src/DeferredLightingEffect.cpp +++ b/libraries/render-utils/src/DeferredLightingEffect.cpp @@ -222,12 +222,15 @@ void DeferredLightingEffect::prepare(RenderArgs* args) { gpu::Batch batch; batch.setStateScissorRect(args->_viewport); + + auto primaryFbo = DependencyManager::get()->getPrimaryFramebuffer(); + + batch.setFramebuffer(primaryFbo); // clear the normal and specular buffers batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR1, glm::vec4(0.0f, 0.0f, 0.0f, 0.0f), true); const float MAX_SPECULAR_EXPONENT = 128.0f; batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR2, glm::vec4(0.0f, 0.0f, 0.0f, 1.0f / MAX_SPECULAR_EXPONENT), true); - args->_context->syncCache(); args->_context->render(batch); } @@ -535,7 +538,6 @@ void DeferredLightingEffect::render(RenderArgs* args) { batch.setResourceTexture(2, nullptr); batch.setResourceTexture(3, nullptr); - args->_context->syncCache(); args->_context->render(batch); // End of the Lighting pass @@ -548,7 +550,8 @@ void DeferredLightingEffect::copyBack(RenderArgs* args) { QSize framebufferSize = framebufferCache->getFrameBufferSize(); // TODO why doesn't this blit work? It only seems to affect a small area below the rear view mirror. - auto destFbo = framebufferCache->getPrimaryFramebuffer(); + // auto destFbo = framebufferCache->getPrimaryFramebuffer(); + auto destFbo = framebufferCache->getPrimaryFramebufferDepthColor(); // gpu::Vec4i vp = args->_viewport; // batch.blit(_copyFBO, vp, framebufferCache->getPrimaryFramebuffer(), vp); batch.setFramebuffer(destFbo); @@ -567,11 +570,6 @@ void DeferredLightingEffect::copyBack(RenderArgs* args) { batch.setModelTransform(model); } - GLenum buffers[3]; - int bufferCount = 0; - buffers[bufferCount++] = GL_COLOR_ATTACHMENT0; - batch._glDrawBuffers(bufferCount, buffers); - batch.setResourceTexture(0, _copyFBO->getRenderBuffer(0)); batch.draw(gpu::TRIANGLE_STRIP, 4); diff --git a/libraries/render-utils/src/FramebufferCache.cpp b/libraries/render-utils/src/FramebufferCache.cpp index 0bcb54fe2c..b4865ef58c 100644 --- a/libraries/render-utils/src/FramebufferCache.cpp +++ b/libraries/render-utils/src/FramebufferCache.cpp @@ -34,7 +34,8 @@ void FramebufferCache::setFrameBufferSize(QSize frameBufferSize) { //If the size changed, we need to delete our FBOs if (_frameBufferSize != frameBufferSize) { _frameBufferSize = frameBufferSize; - _primaryFramebuffer.reset(); + _primaryFramebufferFull.reset(); + _primaryFramebufferDepthColor.reset(); _primaryDepthTexture.reset(); _primaryColorTexture.reset(); _primaryNormalTexture.reset(); @@ -45,7 +46,8 @@ void FramebufferCache::setFrameBufferSize(QSize frameBufferSize) { } void FramebufferCache::createPrimaryFramebuffer() { - _primaryFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create()); + _primaryFramebufferFull = gpu::FramebufferPointer(gpu::Framebuffer::create()); + _primaryFramebufferDepthColor = gpu::FramebufferPointer(gpu::Framebuffer::create()); auto colorFormat = gpu::Element(gpu::VEC4, gpu::NUINT8, gpu::RGBA); auto width = _frameBufferSize.width(); @@ -56,24 +58,33 @@ void FramebufferCache::createPrimaryFramebuffer() { _primaryNormalTexture = gpu::TexturePointer(gpu::Texture::create2D(colorFormat, width, height, defaultSampler)); _primarySpecularTexture = gpu::TexturePointer(gpu::Texture::create2D(colorFormat, width, height, defaultSampler)); - _primaryFramebuffer->setRenderBuffer(0, _primaryColorTexture); - _primaryFramebuffer->setRenderBuffer(1, _primaryNormalTexture); - _primaryFramebuffer->setRenderBuffer(2, _primarySpecularTexture); + _primaryFramebufferFull->setRenderBuffer(0, _primaryColorTexture); + _primaryFramebufferFull->setRenderBuffer(1, _primaryNormalTexture); + _primaryFramebufferFull->setRenderBuffer(2, _primarySpecularTexture); + _primaryFramebufferDepthColor->setRenderBuffer(0, _primaryColorTexture); auto depthFormat = gpu::Element(gpu::SCALAR, gpu::FLOAT, gpu::DEPTH); _primaryDepthTexture = gpu::TexturePointer(gpu::Texture::create2D(depthFormat, width, height, defaultSampler)); - _primaryFramebuffer->setDepthStencilBuffer(_primaryDepthTexture, depthFormat); + _primaryFramebufferFull->setDepthStencilBuffer(_primaryDepthTexture, depthFormat); + + _primaryFramebufferDepthColor->setDepthStencilBuffer(_primaryDepthTexture, depthFormat); } gpu::FramebufferPointer FramebufferCache::getPrimaryFramebuffer() { - if (!_primaryFramebuffer) { + if (!_primaryFramebufferFull) { createPrimaryFramebuffer(); } - return _primaryFramebuffer; + return _primaryFramebufferFull; } +gpu::FramebufferPointer FramebufferCache::getPrimaryFramebufferDepthColor() { + if (!_primaryFramebufferDepthColor) { + createPrimaryFramebuffer(); + } + return _primaryFramebufferDepthColor; +} gpu::TexturePointer FramebufferCache::getPrimaryDepthTexture() { diff --git a/libraries/render-utils/src/FramebufferCache.h b/libraries/render-utils/src/FramebufferCache.h index d65c467be1..c2274a77e8 100644 --- a/libraries/render-utils/src/FramebufferCache.h +++ b/libraries/render-utils/src/FramebufferCache.h @@ -30,6 +30,7 @@ public: /// Returns a pointer to the primary framebuffer object. This render target includes a depth component, and is /// used for scene rendering. gpu::FramebufferPointer getPrimaryFramebuffer(); + gpu::FramebufferPointer getPrimaryFramebufferDepthColor(); gpu::TexturePointer getPrimaryDepthTexture(); gpu::TexturePointer getPrimaryColorTexture(); @@ -55,7 +56,8 @@ private: void createPrimaryFramebuffer(); - gpu::FramebufferPointer _primaryFramebuffer; + gpu::FramebufferPointer _primaryFramebufferFull; + gpu::FramebufferPointer _primaryFramebufferDepthColor; gpu::TexturePointer _primaryDepthTexture; gpu::TexturePointer _primaryColorTexture; gpu::TexturePointer _primaryNormalTexture; diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index 6a627bb6ab..0c8d19250b 100755 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -17,6 +17,7 @@ #include #include +#include "FramebufferCache.h" #include "DeferredLightingEffect.h" #include "TextureCache.h" @@ -27,6 +28,26 @@ using namespace render; +void SetupDeferred::run(const SceneContextPointer& sceneContext, const RenderContextPointer& renderContext) { + RenderArgs* args = renderContext->args; + + auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); + + gpu::Batch batch; + batch.setFramebuffer(nullptr); + batch.setFramebuffer(primaryFbo); + + batch.setViewportTransform(args->_viewport); + batch.setStateScissorRect(args->_viewport); + + batch.clearFramebuffer( + gpu::Framebuffer::BUFFER_COLOR0 | + gpu::Framebuffer::BUFFER_DEPTH, + vec4(vec3(0), 1), 1.0, 0.0, true); + + args->_context->render(batch); +} + void PrepareDeferred::run(const SceneContextPointer& sceneContext, const RenderContextPointer& renderContext) { DependencyManager::get()->prepare(renderContext->args); } @@ -41,6 +62,7 @@ void ResolveDeferred::run(const SceneContextPointer& sceneContext, const RenderC } RenderDeferredTask::RenderDeferredTask() : Task() { + _jobs.push_back(Job(new SetupDeferred::JobModel("SetupFramebuffer"))); _jobs.push_back(Job(new DrawBackground::JobModel("DrawBackground"))); _jobs.push_back(Job(new PrepareDeferred::JobModel("PrepareDeferred"))); @@ -56,7 +78,6 @@ RenderDeferredTask::RenderDeferredTask() : Task() { auto& renderedOpaques = _jobs.back().getOutput(); _jobs.push_back(Job(new DrawOpaqueDeferred::JobModel("DrawOpaqueDeferred", _jobs.back().getOutput()))); _jobs.push_back(Job(new DrawLight::JobModel("DrawLight"))); - _jobs.push_back(Job(new ResetGLState::JobModel())); _jobs.push_back(Job(new RenderDeferred::JobModel("RenderDeferred"))); _jobs.push_back(Job(new ResolveDeferred::JobModel("ResolveDeferred"))); _jobs.push_back(Job(new FetchItems::JobModel("FetchTransparent", @@ -133,21 +154,12 @@ void DrawOpaqueDeferred::run(const SceneContextPointer& sceneContext, const Rend batch.setViewTransform(viewMat); { - GLenum buffers[3]; - int bufferCount = 0; - buffers[bufferCount++] = GL_COLOR_ATTACHMENT0; - buffers[bufferCount++] = GL_COLOR_ATTACHMENT1; - buffers[bufferCount++] = GL_COLOR_ATTACHMENT2; - batch._glDrawBuffers(bufferCount, buffers); const float OPAQUE_ALPHA_THRESHOLD = 0.5f; args->_alphaThreshold = OPAQUE_ALPHA_THRESHOLD; } renderItems(sceneContext, renderContext, inItems, renderContext->_maxDrawnOpaqueItems); - // Before rendering the batch make sure we re in sync with gl state - args->_context->syncCache(); - renderContext->args->_context->syncCache(); args->_context->render((*args->_batch)); args->_batch = nullptr; } @@ -171,21 +183,15 @@ void DrawTransparentDeferred::run(const SceneContextPointer& sceneContext, const } batch.setProjectionTransform(projMat); batch.setViewTransform(viewMat); - const float TRANSPARENT_ALPHA_THRESHOLD = 0.0f; { - GLenum buffers[3]; - int bufferCount = 0; - buffers[bufferCount++] = GL_COLOR_ATTACHMENT0; - batch._glDrawBuffers(bufferCount, buffers); + const float TRANSPARENT_ALPHA_THRESHOLD = 0.0f; args->_alphaThreshold = TRANSPARENT_ALPHA_THRESHOLD; } renderItems(sceneContext, renderContext, inItems, renderContext->_maxDrawnTransparentItems); - - // Before rendering the batch make sure we re in sync with gl state - args->_context->syncCache(); + args->_context->render((*args->_batch)); args->_batch = nullptr; } @@ -239,17 +245,17 @@ void DrawOverlay3D::run(const SceneContextPointer& sceneContext, const RenderCon } batch.setProjectionTransform(projMat); batch.setViewTransform(viewMat); + batch.setViewportTransform(args->_viewport); + batch.setStateScissorRect(args->_viewport); batch.setPipeline(getOpaquePipeline()); batch.setResourceTexture(0, args->_whiteTexture); if (!inItems.empty()) { - batch.clearFramebuffer(gpu::Framebuffer::BUFFER_DEPTH, glm::vec4(), 1.f, 0); + batch.clearFramebuffer(gpu::Framebuffer::BUFFER_DEPTH, glm::vec4(), 1.f, 0, true); renderItems(sceneContext, renderContext, inItems, renderContext->_maxDrawnOverlay3DItems); } - // Before rendering the batch make sure we re in sync with gl state - args->_context->syncCache(); args->_context->render((*args->_batch)); args->_batch = nullptr; args->_whiteTexture.reset(); diff --git a/libraries/render-utils/src/RenderDeferredTask.h b/libraries/render-utils/src/RenderDeferredTask.h index 4040606c62..1fec1c936f 100755 --- a/libraries/render-utils/src/RenderDeferredTask.h +++ b/libraries/render-utils/src/RenderDeferredTask.h @@ -16,6 +16,13 @@ #include "gpu/Pipeline.h" +class SetupDeferred { +public: + void run(const render::SceneContextPointer& sceneContext, const render::RenderContextPointer& renderContext); + + typedef render::Job::Model JobModel; +}; + class PrepareDeferred { public: void run(const render::SceneContextPointer& sceneContext, const render::RenderContextPointer& renderContext); From 52ac5b3ef63331d193f6abe4c8bab39a1410b3d4 Mon Sep 17 00:00:00 2001 From: samcake Date: Fri, 24 Jul 2015 03:38:21 -0700 Subject: [PATCH 34/36] On mac cannot get ther erendering to properly display the overlay if the rear view mirror is active on resize --- interface/src/Application.cpp | 19 ++++++++------- interface/src/Application.h | 1 + interface/src/ui/ApplicationOverlay.cpp | 2 +- libraries/gpu/src/gpu/GLBackend.cpp | 1 + libraries/gpu/src/gpu/GLBackend.h | 7 ++++-- libraries/gpu/src/gpu/GLBackendOutput.cpp | 23 ++++++++++++++++++- .../render-utils/src/FramebufferCache.cpp | 6 ++++- libraries/render-utils/src/GeometryCache.cpp | 6 ++--- 8 files changed, 48 insertions(+), 17 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 8c5769c26c..cb1291337d 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -877,22 +877,25 @@ void Application::paintGL() { // Before anything else, let's sync up the gpuContext with the true glcontext used in case anything happened renderArgs._context->syncCache(); - - if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { + + if ((_numFramesSinceLastResize > 1) && Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); - + renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; renderRearViewMirror(&renderArgs, _mirrorViewRect); renderArgs._renderMode = RenderArgs::DEFAULT_RENDER_MODE; - + { float ratio = ((float)QApplication::desktop()->windowHandle()->devicePixelRatio() * getRenderResolutionScale()); auto mirrorViewport = glm::ivec4(0, 0, _mirrorViewRect.width() * ratio, _mirrorViewRect.height() * ratio); auto mirrorViewportDest = mirrorViewport; - + auto selfieFbo = DependencyManager::get()->getSelfieFramebuffer(); gpu::Batch batch; + batch.setFramebuffer(selfieFbo); + batch.clearColorFramebuffer(gpu::Framebuffer::BUFFER_COLOR0, glm::vec4(0.0f, 0.0f, 0.0f, 0.0f)); batch.blit(primaryFbo, mirrorViewport, selfieFbo, mirrorViewportDest); + batch.setFramebuffer(nullptr); renderArgs._context->render(batch); } } @@ -992,8 +995,7 @@ void Application::paintGL() { _compositor.displayOverlayTexture(&renderArgs); } - - + if (!OculusManager::isConnected() || OculusManager::allowSwap()) { PROFILE_RANGE(__FUNCTION__ "/bufferSwap"); _glWidget->swapBuffers(); @@ -1003,6 +1005,7 @@ void Application::paintGL() { OculusManager::endFrameTiming(); } _frameCount++; + _numFramesSinceLastResize++; Stats::getInstance()->setRenderDetails(renderArgs._details); } @@ -1056,6 +1059,7 @@ void Application::resizeGL() { } if (_renderResolution != toGlm(renderSize)) { + _numFramesSinceLastResize = 0; _renderResolution = toGlm(renderSize); DependencyManager::get()->setFrameBufferSize(renderSize); @@ -1069,7 +1073,6 @@ void Application::resizeGL() { auto canvasSize = _glWidget->size(); offscreenUi->resize(canvasSize); _glWidget->makeCurrent(); - } bool Application::importSVOFromURL(const QString& urlString) { diff --git a/interface/src/Application.h b/interface/src/Application.h index b335b8a333..d1886862d2 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -642,6 +642,7 @@ private: Overlays _overlays; ApplicationOverlay _applicationOverlay; ApplicationCompositor _compositor; + int _numFramesSinceLastResize = 0; }; #endif // hifi_Application_h diff --git a/interface/src/ui/ApplicationOverlay.cpp b/interface/src/ui/ApplicationOverlay.cpp index 8ec2183681..bc10b555e4 100644 --- a/interface/src/ui/ApplicationOverlay.cpp +++ b/interface/src/ui/ApplicationOverlay.cpp @@ -92,9 +92,9 @@ void ApplicationOverlay::renderOverlay(RenderArgs* renderArgs) { batch.clearFramebuffer(gpu::Framebuffer::BUFFER_COLOR0 | gpu::Framebuffer::BUFFER_DEPTH, color, depth, stencil); // Now render the overlay components together into a single texture - renderRearView(renderArgs); // renders the mirror view selfie renderDomainConnectionStatusBorder(renderArgs); // renders the connected domain line renderAudioScope(renderArgs); // audio scope in the very back + renderRearView(renderArgs); // renders the mirror view selfie renderQmlUi(renderArgs); // renders a unit quad with the QML UI texture, and the text overlays from scripts renderOverlays(renderArgs); // renders Scripts Overlay and AudioScope renderStatsAndLogs(renderArgs); // currently renders nothing diff --git a/libraries/gpu/src/gpu/GLBackend.cpp b/libraries/gpu/src/gpu/GLBackend.cpp index 7fd0f9be76..d704a4d17f 100644 --- a/libraries/gpu/src/gpu/GLBackend.cpp +++ b/libraries/gpu/src/gpu/GLBackend.cpp @@ -192,6 +192,7 @@ void GLBackend::syncCache() { syncTransformStateCache(); syncPipelineStateCache(); syncInputStateCache(); + syncOutputStateCache(); glEnable(GL_LINE_SMOOTH); } diff --git a/libraries/gpu/src/gpu/GLBackend.h b/libraries/gpu/src/gpu/GLBackend.h index c924395334..894e2c4548 100644 --- a/libraries/gpu/src/gpu/GLBackend.h +++ b/libraries/gpu/src/gpu/GLBackend.h @@ -387,11 +387,14 @@ protected: void do_setFramebuffer(Batch& batch, uint32 paramOffset); void do_blit(Batch& batch, uint32 paramOffset); - + // Synchronize the state cache of this Backend with the actual real state of the GL Context + void syncOutputStateCache(); + struct OutputStageState { FramebufferPointer _framebuffer = nullptr; - + GLuint _drawFBO = 0; + OutputStageState() {} } _output; diff --git a/libraries/gpu/src/gpu/GLBackendOutput.cpp b/libraries/gpu/src/gpu/GLBackendOutput.cpp index 7245272131..1b22649ad6 100755 --- a/libraries/gpu/src/gpu/GLBackendOutput.cpp +++ b/libraries/gpu/src/gpu/GLBackendOutput.cpp @@ -37,6 +37,9 @@ GLBackend::GLFramebuffer* GLBackend::syncGPUObject(const Framebuffer& framebuffe // need to have a gpu object? if (!object) { + GLint currentFBO; + glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, ¤tFBO); + GLuint fbo; glGenFramebuffers(1, &fbo); (void) CHECK_GL_ERROR(); @@ -87,6 +90,8 @@ GLBackend::GLFramebuffer* GLBackend::syncGPUObject(const Framebuffer& framebuffe glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, renderBuffer); (void) CHECK_GL_ERROR(); } + + // glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT); #endif @@ -142,6 +147,9 @@ GLBackend::GLFramebuffer* GLBackend::syncGPUObject(const Framebuffer& framebuffe object->_fbo = fbo; object->_colorBuffers = colorBuffers; Backend::setGPUObject(framebuffer, object); + + // restore the current framebuffer + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, currentFBO); } return object; @@ -161,11 +169,24 @@ GLuint GLBackend::getFramebufferID(const FramebufferPointer& framebuffer) { } } +void GLBackend::syncOutputStateCache() { + GLint currentFBO; + glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, ¤tFBO); + + _output._drawFBO = currentFBO; + _output._framebuffer.reset(); +} + + void GLBackend::do_setFramebuffer(Batch& batch, uint32 paramOffset) { auto framebuffer = batch._framebuffers.get(batch._params[paramOffset]._uint); if (_output._framebuffer != framebuffer) { - glBindFramebuffer(GL_DRAW_FRAMEBUFFER, getFramebufferID(framebuffer)); + auto newFBO = getFramebufferID(framebuffer); + if (_output._drawFBO != newFBO) { + _output._drawFBO = newFBO; + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, newFBO); + } _output._framebuffer = framebuffer; } } diff --git a/libraries/render-utils/src/FramebufferCache.cpp b/libraries/render-utils/src/FramebufferCache.cpp index b4865ef58c..601d99108d 100644 --- a/libraries/render-utils/src/FramebufferCache.cpp +++ b/libraries/render-utils/src/FramebufferCache.cpp @@ -70,6 +70,10 @@ void FramebufferCache::createPrimaryFramebuffer() { _primaryFramebufferFull->setDepthStencilBuffer(_primaryDepthTexture, depthFormat); _primaryFramebufferDepthColor->setDepthStencilBuffer(_primaryDepthTexture, depthFormat); + + _selfieFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create()); + auto tex = gpu::TexturePointer(gpu::Texture::create2D(colorFormat, width * 0.5, height * 0.5, defaultSampler)); + _selfieFramebuffer->setRenderBuffer(0, tex); } gpu::FramebufferPointer FramebufferCache::getPrimaryFramebuffer() { @@ -140,7 +144,7 @@ gpu::FramebufferPointer FramebufferCache::getShadowFramebuffer() { gpu::FramebufferPointer FramebufferCache::getSelfieFramebuffer() { if (!_selfieFramebuffer) { - _selfieFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create(gpu::Element::COLOR_RGBA_32, _frameBufferSize.width(), _frameBufferSize.height())); + createPrimaryFramebuffer(); } return _selfieFramebuffer; } diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 595e901919..8550f8d8b6 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1670,10 +1670,8 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { auto stateNoBlend = std::make_shared(); - // stateNoBlend->setColorWriteMask(true, true, true, false); - // stateNoBlend->setBlendFunction(true, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA); - - auto programNoBlend = gpu::ShaderPointer(gpu::Shader::createProgram(vs, gpu::StandardShaderLib::getDrawTextureOpaquePS())); + auto noBlendPS = gpu::StandardShaderLib::getDrawTextureOpaquePS(); + auto programNoBlend = gpu::ShaderPointer(gpu::Shader::createProgram(vs, noBlendPS)); gpu::Shader::makeProgram((*programNoBlend)); _standardDrawPipelineNoBlend.reset(gpu::Pipeline::create(programNoBlend, stateNoBlend)); } From 6ba135de0937641f55a4743a15b7f98f972604f8 Mon Sep 17 00:00:00 2001 From: Seth Alves Date: Fri, 24 Jul 2015 07:16:40 -0700 Subject: [PATCH 35/36] when creating a walking surface collision hull from a flat mesh, ignore triangles with a normal that's not mostly up or down --- tools/vhacd-util/src/VHACDUtil.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/vhacd-util/src/VHACDUtil.cpp b/tools/vhacd-util/src/VHACDUtil.cpp index f1ff0e9e4f..4860785091 100644 --- a/tools/vhacd-util/src/VHACDUtil.cpp +++ b/tools/vhacd-util/src/VHACDUtil.cpp @@ -118,6 +118,13 @@ void vhacd::VHACDUtil::fattenMeshes(const FBXMesh& mesh, FBXMesh& result, glm::vec3 p2 = result.vertices[index2]; glm::vec3 av = (p0 + p1 + p2) / 3.0f; // center of the triangular face + glm::vec3 normal = glm::normalize(glm::cross(p1 - p0, p2 - p0)); + float threshold = 1.0f / sqrtf(3.0f); + if (normal.y > -threshold && normal.y < threshold) { + // this triangle is more a wall than a floor, skip it. + continue; + } + float dropAmount = 0; dropAmount = glm::max(glm::length(p1 - p0), dropAmount); dropAmount = glm::max(glm::length(p2 - p1), dropAmount); From e6f2504514313538689b721d7fa2b3e86c0df688 Mon Sep 17 00:00:00 2001 From: samcake Date: Fri, 24 Jul 2015 09:26:32 -0700 Subject: [PATCH 36/36] FIx the problem of clear when the color mask is not WRITE_ALL --- interface/src/Application.cpp | 2 +- libraries/gpu/src/gpu/GLBackend.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index cb1291337d..f6433cfb45 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -878,7 +878,7 @@ void Application::paintGL() { // Before anything else, let's sync up the gpuContext with the true glcontext used in case anything happened renderArgs._context->syncCache(); - if ((_numFramesSinceLastResize > 1) && Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { + if (Menu::getInstance()->isOptionChecked(MenuOption::Mirror)) { auto primaryFbo = DependencyManager::get()->getPrimaryFramebufferDepthColor(); renderArgs._renderMode = RenderArgs::MIRROR_RENDER_MODE; diff --git a/libraries/gpu/src/gpu/GLBackend.cpp b/libraries/gpu/src/gpu/GLBackend.cpp index d704a4d17f..ae50b96bc5 100644 --- a/libraries/gpu/src/gpu/GLBackend.cpp +++ b/libraries/gpu/src/gpu/GLBackend.cpp @@ -282,6 +282,9 @@ void GLBackend::do_clearFramebuffer(Batch& batch, uint32 paramOffset) { glClearColor(color.x, color.y, color.z, color.w); glmask |= GL_COLOR_BUFFER_BIT; } + + // Force the color mask cache to WRITE_ALL if not the case + do_setStateColorWriteMask(State::ColorMask::WRITE_ALL); } // Apply scissor if needed and if not already on