Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-04-19 01:39:19 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #10268 from druiz17/input-recorder

Browse source 
Input recorder/ input playback system  for all input actions
This commit is contained in:
anshuman64 2017-04-26 16:20:20 -07:00 committed by GitHub
commit d146431e9b
12 changed files with 695 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

170
interface/resources/qml/hifi/tablet/InputRecorder.qml Normal file
View file

@ -0,0 +1,170 @@
//
// Created by Dante Ruiz 2017/04/17
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
import QtQuick 2.5
import Hifi 1.0
import QtQuick.Controls 1.4
import QtQuick.Dialogs 1.2 as OriginalDialogs
import "../../styles-uit"
import "../../controls-uit" as HifiControls
import "../../windows"
import "../../dialogs"
Rectangle {
id: inputRecorder
property var eventBridge;
HifiConstants { id: hifi }
signal sendToScript(var message);
color: hifi.colors.baseGray;
property string path: ""
property string dir: ""
property var dialog: null;
property bool recording: false;
Component { id: fileDialog; TabletFileDialog { } }
Row {
id: topButtons
width: parent.width
height: 40
spacing: 40
anchors {
left: parent.left
right: parent.right
top: parent.top
topMargin: 10
}
HifiControls.Button {
id: start
text: "Start Recoring"
color: hifi.buttons.black
enabled: true
onClicked: {
if (inputRecorder.recording) {
sendToScript({method: "Stop"});
inputRecorder.recording = false;
start.text = "Start Recording";
selectedFile.text = "Current recording is not saved";
} else {
sendToScript({method: "Start"});
inputRecorder.recording = true;
start.text = "Stop Recording";
}
}
}
HifiControls.Button {
id: save
text: "Save Recording"
color: hifi.buttons.black
enabled: true
onClicked: {
sendToScript({method: "Save"});
selectedFile.text = "";
}
}
HifiControls.Button {
id: playBack
anchors.right: browse.left
anchors.top: selectedFile.bottom
anchors.topMargin: 10
text: "Play Recording"
color: hifi.buttons.black
enabled: true
onClicked: {
sendToScript({method: "playback"});
HMD.closeTablet();
}
}
}
HifiControls.VerticalSpacer {}
HifiControls.TextField {
id: selectedFile
anchors.left: parent.left
anchors.right: parent.right
anchors.top: topButtons.top
anchors.topMargin: 40
colorScheme: hifi.colorSchemes.dark
readOnly: true
}
HifiControls.Button {
id: browse
anchors.right: parent.right
anchors.top: selectedFile.bottom
anchors.topMargin: 10
text: "Load..."
color: hifi.buttons.black
enabled: true
onClicked: {
dialog = fileDialog.createObject(inputRecorder);
dialog.caption = "InputRecorder";
console.log(dialog.dir);
dialog.dir = "file:///" + inputRecorder.dir;
dialog.selectedFile.connect(getFileSelected);
}
}
Column {
id: notes
anchors.centerIn: parent;
spacing: 20
Text {
text: "All files are saved under the folder 'hifi-input-recording' in AppData directory";
color: "white"
font.pointSize: 10
}
Text {
text: "To cancel a recording playback press Alt-B"
color: "white"
font.pointSize: 10
}
}
function getFileSelected(file) {
selectedFile.text = file;
inputRecorder.path = file;
sendToScript({method: "Load", params: {file: path }});
}
function fromScript(message) {
switch (message.method) {
case "update":
updateButtonStatus(message.params);
break;
case "path":
console.log(message.params);
inputRecorder.dir = message.params;
break;
}
}
function updateButtonStatus(status) {
inputRecorder.recording = status;
if (inputRecorder.recording) {
start.text = "Stop Recording";
} else {
start.text = "Start Recording";
}
}
}

4
interface/src/Application.cpp
View file

@ -78,6 +78,7 @@
#include <InfoView.h>
#include <input-plugins/InputPlugin.h>
#include <controllers/UserInputMapper.h>
#include <controllers/InputRecorder.h>
#include <controllers/ScriptingInterface.h>
#include <controllers/StateController.h>
#include <UserActivityLoggerScriptingInterface.h>
@ -2753,6 +2754,9 @@ void Application::keyPressEvent(QKeyEvent* event) {
if (isMeta) {
auto offscreenUi = DependencyManager::get<OffscreenUi>();
offscreenUi->load("Browser.qml");
} else if (isOption) {
controller::InputRecorder* inputRecorder = controller::InputRecorder::getInstance();
inputRecorder->stopPlayback();
}
break;

2
libraries/controllers/CMakeLists.txt
View file

@ -10,4 +10,4 @@ GroupSources("src/controllers")
add_dependency_external_projects(glm)
find_package(GLM REQUIRED)
target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS})
target_include_directories(${TARGET_NAME} PUBLIC ${GLM_INCLUDE_DIRS} "${CMAKE_BINARY_DIR}/includes")

290
libraries/controllers/src/controllers/InputRecorder.cpp Normal file
View file

@ -0,0 +1,290 @@
//
// Created by Dante Ruiz 2017/04/16
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "InputRecorder.h"
#include <QJsonArray>
#include <QJsonObject>
#include <QJsonDocument>
#include <QFile>
#include <QDir>
#include <QDirIterator>
#include <QStandardPaths>
#include <QDateTime>
#include <QByteArray>
#include <QStandardPaths>
#include <PathUtils.h>
#include <BuildInfo.h>
#include <GLMHelpers.h>
QString SAVE_DIRECTORY = QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation) + "/" + BuildInfo::MODIFIED_ORGANIZATION + "/" + BuildInfo::INTERFACE_NAME + "/hifi-input-recordings/";
QString FILE_PREFIX_NAME = "input-recording-";
QString COMPRESS_EXTENSION = ".tar.gz";
namespace controller {
QJsonObject poseToJsonObject(const Pose pose) {
QJsonObject newPose;
QJsonArray translation;
translation.append(pose.translation.x);
translation.append(pose.translation.y);
translation.append(pose.translation.z);
QJsonArray rotation;
rotation.append(pose.rotation.x);
rotation.append(pose.rotation.y);
rotation.append(pose.rotation.z);
rotation.append(pose.rotation.w);
QJsonArray velocity;
velocity.append(pose.velocity.x);
velocity.append(pose.velocity.y);
velocity.append(pose.velocity.z);
QJsonArray angularVelocity;
angularVelocity.append(pose.angularVelocity.x);
angularVelocity.append(pose.angularVelocity.y);
angularVelocity.append(pose.angularVelocity.z);
newPose["translation"] = translation;
newPose["rotation"] = rotation;
newPose["velocity"] = velocity;
newPose["angularVelocity"] = angularVelocity;
newPose["valid"] = pose.valid;
return newPose;
}
Pose jsonObjectToPose(const QJsonObject object) {
Pose pose;
QJsonArray translation = object["translation"].toArray();
QJsonArray rotation = object["rotation"].toArray();
QJsonArray velocity = object["velocity"].toArray();
QJsonArray angularVelocity = object["angularVelocity"].toArray();
pose.valid = object["valid"].toBool();
pose.translation.x = translation[0].toDouble();
pose.translation.y = translation[1].toDouble();
pose.translation.z = translation[2].toDouble();
pose.rotation.x = rotation[0].toDouble();
pose.rotation.y = rotation[1].toDouble();
pose.rotation.z = rotation[2].toDouble();
pose.rotation.w = rotation[3].toDouble();
pose.velocity.x = velocity[0].toDouble();
pose.velocity.y = velocity[1].toDouble();
pose.velocity.z = velocity[2].toDouble();
pose.angularVelocity.x = angularVelocity[0].toDouble();
pose.angularVelocity.y = angularVelocity[1].toDouble();
pose.angularVelocity.z = angularVelocity[2].toDouble();
return pose;
}
void exportToFile(QJsonObject& object) {
if (!QDir(SAVE_DIRECTORY).exists()) {
QDir().mkdir(SAVE_DIRECTORY);
}
QString timeStamp = QDateTime::currentDateTime().toString(Qt::ISODate);
timeStamp.replace(":", "-");
QString fileName = SAVE_DIRECTORY + FILE_PREFIX_NAME + timeStamp + COMPRESS_EXTENSION;
qDebug() << fileName;
QFile saveFile (fileName);
if (!saveFile.open(QIODevice::WriteOnly)) {
qWarning() << "could not open file: " << fileName;
return;
}
QJsonDocument saveData(object);
QByteArray compressedData = qCompress(saveData.toJson(QJsonDocument::Compact));
saveFile.write(compressedData);
}
QJsonObject openFile(const QString& file, bool& status) {
QJsonObject object;
QFile openFile(file);
if (!openFile.open(QIODevice::ReadOnly)) {
qWarning() << "could not open file: " << file;
status = false;
return object;
}
QByteArray compressedData = qUncompress(openFile.readAll());
QJsonDocument jsonDoc = QJsonDocument::fromJson(compressedData);
object = jsonDoc.object();
status = true;
return object;
}
InputRecorder::InputRecorder() {}
InputRecorder::~InputRecorder() {}
InputRecorder* InputRecorder::getInstance() {
static InputRecorder inputRecorder;
return &inputRecorder;
}
QString InputRecorder::getSaveDirectory() {
return SAVE_DIRECTORY;
}
void InputRecorder::startRecording() {
_recording = true;
_playback = false;
_framesRecorded = 0;
_poseStateList.clear();
_actionStateList.clear();
}
void InputRecorder::saveRecording() {
QJsonObject data;
data["frameCount"] = _framesRecorded;
QJsonArray actionArrayList;
QJsonArray poseArrayList;
for(const ActionStates actionState: _actionStateList) {
QJsonArray actionArray;
for (const float value: actionState) {
actionArray.append(value);
}
actionArrayList.append(actionArray);
}
for (const PoseStates poseState: _poseStateList) {
QJsonArray poseArray;
for (const Pose pose: poseState) {
poseArray.append(poseToJsonObject(pose));
}
poseArrayList.append(poseArray);
}
data["actionList"] = actionArrayList;
data["poseList"] = poseArrayList;
exportToFile(data);
}
void InputRecorder::loadRecording(const QString& path) {
_recording = false;
_playback = false;
_loading = true;
_playCount = 0;
resetFrame();
_poseStateList.clear();
_actionStateList.clear();
QString filePath = path;
filePath.remove(0,8);
QFileInfo info(filePath);
QString extension = info.suffix();
if (extension != "gz") {
qWarning() << "can not load file with exentsion of " << extension;
return;
}
bool success = false;
QJsonObject data = openFile(info.absoluteFilePath(), success);
if (success) {
_framesRecorded = data["frameCount"].toInt();
QJsonArray actionArrayList = data["actionList"].toArray();
QJsonArray poseArrayList = data["poseList"].toArray();
for (int actionIndex = 0; actionIndex < actionArrayList.size(); actionIndex++) {
QJsonArray actionState = actionArrayList[actionIndex].toArray();
for (int index = 0; index < actionState.size(); index++) {
_currentFrameActions[index] = actionState[index].toInt();
}
_actionStateList.push_back(_currentFrameActions);
_currentFrameActions = ActionStates(toInt(Action::NUM_ACTIONS));
}
for (int poseIndex = 0; poseIndex < poseArrayList.size(); poseIndex++) {
QJsonArray poseState = poseArrayList[poseIndex].toArray();
for (int index = 0; index < poseState.size(); index++) {
_currentFramePoses[index] = jsonObjectToPose(poseState[index].toObject());
}
_poseStateList.push_back(_currentFramePoses);
_currentFramePoses = PoseStates(toInt(Action::NUM_ACTIONS));
}
}
_loading = false;
}
void InputRecorder::stopRecording() {
_recording = false;
}
void InputRecorder::startPlayback() {
_playback = true;
_recording = false;
_playCount = 0;
}
void InputRecorder::stopPlayback() {
_playback = false;
_playCount = 0;
}
void InputRecorder::setActionState(controller::Action action, float value) {
if (_recording) {
_currentFrameActions[toInt(action)] += value;
}
}
void InputRecorder::setActionState(controller::Action action, const controller::Pose pose) {
if (_recording) {
_currentFramePoses[toInt(action)] = pose;
}
}
void InputRecorder::resetFrame() {
if (_recording) {
for(auto& channel : _currentFramePoses) {
channel = Pose();
}
for(auto& channel : _currentFrameActions) {
channel = 0.0f;
}
}
}
float InputRecorder::getActionState(controller::Action action) {
if (_actionStateList.size() > 0 ) {
return _actionStateList[_playCount][toInt(action)];
}
return 0.0f;
}
controller::Pose InputRecorder::getPoseState(controller::Action action) {
if (_poseStateList.size() > 0) {
return _poseStateList[_playCount][toInt(action)];
}
return Pose();
}
void InputRecorder::frameTick() {
if (_recording) {
_framesRecorded++;
_poseStateList.push_back(_currentFramePoses);
_actionStateList.push_back(_currentFrameActions);
}
if (_playback) {
_playCount++;
if (_playCount == _framesRecorded) {
_playCount = 0;
}
}
}
}

62
libraries/controllers/src/controllers/InputRecorder.h Normal file
View file

@ -0,0 +1,62 @@
//
// Created by Dante Ruiz on 2017/04/16
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_InputRecorder_h
#define hifi_InputRecorder_h
#include <mutex>
#include <atomic>
#include <vector>
#include <QString>
#include "Pose.h"
#include "Actions.h"
namespace controller {
class InputRecorder {
public:
using PoseStates = std::vector<Pose>;
using ActionStates = std::vector<float>;
InputRecorder();
~InputRecorder();
static InputRecorder* getInstance();
void saveRecording();
void loadRecording(const QString& path);
void startRecording();
void startPlayback();
void stopPlayback();
void stopRecording();
void toggleRecording() { _recording = !_recording; }
void togglePlayback() { _playback = !_playback; }
void resetFrame();
bool isRecording() { return _recording; }
bool isPlayingback() { return (_playback && !_loading); }
void setActionState(controller::Action action, float value);
void setActionState(controller::Action action, const controller::Pose pose);
float getActionState(controller::Action action);
controller::Pose getPoseState(controller::Action action);
QString getSaveDirectory();
void frameTick();
private:
bool _recording { false };
bool _playback { false };
bool _loading { false };
std::vector<PoseStates> _poseStateList = std::vector<PoseStates>();
std::vector<ActionStates> _actionStateList = std::vector<ActionStates>();
PoseStates _currentFramePoses = PoseStates(toInt(Action::NUM_ACTIONS));
ActionStates _currentFrameActions = ActionStates(toInt(Action::NUM_ACTIONS));
int _framesRecorded { 0 };
int _playCount { 0 };
};
}
#endif

36
libraries/controllers/src/controllers/ScriptingInterface.cpp
View file

@ -23,6 +23,7 @@
#include "impl/MappingBuilderProxy.h"
#include "Logging.h"
#include "InputDevice.h"
#include "InputRecorder.h"
static QRegularExpression SANITIZE_NAME_EXPRESSION{ "[\\(\\)\\.\\s]" };
@ -154,6 +155,41 @@ namespace controller {
return DependencyManager::get<UserInputMapper>()->triggerHapticPulse(strength, SHORT_HAPTIC_DURATION_MS, hand);
}
void ScriptingInterface::startInputRecording() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->startRecording();
}
void ScriptingInterface::stopInputRecording() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->stopRecording();
}
void ScriptingInterface::startInputPlayback() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->startPlayback();
}
void ScriptingInterface::stopInputPlayback() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->stopPlayback();
}
void ScriptingInterface::saveInputRecording() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->saveRecording();
}
void ScriptingInterface::loadInputRecording(const QString& file) {
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->loadRecording(file);
}
QString ScriptingInterface::getInputRecorderSaveDirectory() {
InputRecorder* inputRecorder = InputRecorder::getInstance();
return inputRecorder->getSaveDirectory();
}
bool ScriptingInterface::triggerHapticPulseOnDevice(unsigned int device, float strength, float duration, controller::Hand hand) const {
return DependencyManager::get<UserInputMapper>()->triggerHapticPulseOnDevice(device, strength, duration, hand);
}

7
libraries/controllers/src/controllers/ScriptingInterface.h
View file

@ -99,6 +99,13 @@ namespace controller {
Q_INVOKABLE const QVariantMap& getHardware() { return _hardware; }
Q_INVOKABLE const QVariantMap& getActions() { return _actions; }
Q_INVOKABLE const QVariantMap& getStandard() { return _standard; }
Q_INVOKABLE void startInputRecording();
Q_INVOKABLE void stopInputRecording();
Q_INVOKABLE void startInputPlayback();
Q_INVOKABLE void stopInputPlayback();
Q_INVOKABLE void saveInputRecording();
Q_INVOKABLE void loadInputRecording(const QString& file);
Q_INVOKABLE QString getInputRecorderSaveDirectory();
bool isMouseCaptured() const { return _mouseCaptured; }
bool isTouchCaptured() const { return _touchCaptured; }

6
libraries/controllers/src/controllers/UserInputMapper.cpp
View file

@ -22,7 +22,7 @@
#include "StandardController.h"
#include "StateController.h"
#include "InputRecorder.h"
#include "Logging.h"
#include "impl/conditionals/AndConditional.h"
@ -243,10 +243,11 @@ void fixBisectedAxis(float& full, float& negative, float& positive) {
void UserInputMapper::update(float deltaTime) {
Locker locker(_lock);
InputRecorder* inputRecorder = InputRecorder::getInstance();
static uint64_t updateCount = 0;
++updateCount;
inputRecorder->resetFrame();
// Reset the axis state for next loop
for (auto& channel : _actionStates) {
channel = 0.0f;
@ -298,6 +299,7 @@ void UserInputMapper::update(float deltaTime) {
emit inputEvent(input.id, value);
}
}
inputRecorder->frameTick();
}
Input::NamedVector UserInputMapper::getAvailableInputs(uint16 deviceID) const {

13
libraries/controllers/src/controllers/impl/endpoints/ActionEndpoint.cpp
View file

@ -11,19 +11,32 @@
#include <DependencyManager.h>
#include "../../UserInputMapper.h"
#include "../../InputRecorder.h"
using namespace controller;
void ActionEndpoint::apply(float newValue, const Pointer& source) {
InputRecorder* inputRecorder = InputRecorder::getInstance();
if(inputRecorder->isPlayingback()) {
newValue = inputRecorder->getActionState(Action(_input.getChannel()));
}
_currentValue += newValue;
if (_input != Input::INVALID_INPUT) {
auto userInputMapper = DependencyManager::get<UserInputMapper>();
userInputMapper->deltaActionState(Action(_input.getChannel()), newValue);
}
inputRecorder->setActionState(Action(_input.getChannel()), newValue);
}
void ActionEndpoint::apply(const Pose& value, const Pointer& source) {
_currentPose = value;
InputRecorder* inputRecorder = InputRecorder::getInstance();
inputRecorder->setActionState(Action(_input.getChannel()), _currentPose);
if (inputRecorder->isPlayingback()) {
_currentPose = inputRecorder->getPoseState(Action(_input.getChannel()));
}
if (!_currentPose.isValid()) {
return;
}

4
libraries/ui/src/FileDialogHelper.cpp
View file

@ -26,6 +26,10 @@ QStringList FileDialogHelper::standardPath(StandardLocation location) {
return QStandardPaths::standardLocations(static_cast<QStandardPaths::StandardLocation>(location));
}
QString FileDialogHelper::writableLocation(StandardLocation location) {
return QStandardPaths::writableLocation(static_cast<QStandardPaths::StandardLocation>(location));
}
QString FileDialogHelper::urlToPath(const QUrl& url) {
return url.toLocalFile();
}

1
libraries/ui/src/FileDialogHelper.h
View file

@ -48,6 +48,7 @@ public:
Q_INVOKABLE QUrl home();
Q_INVOKABLE QStringList standardPath(StandardLocation location);
Q_INVOKABLE QString writableLocation(StandardLocation location);
Q_INVOKABLE QStringList drives();
Q_INVOKABLE QString urlToPath(const QUrl& url);
Q_INVOKABLE bool urlIsDir(const QUrl& url);

103
scripts/developer/inputRecording.js Normal file
View file

@ -0,0 +1,103 @@
//
// Created by Dante Ruiz 2017/04/17
// Copyright 2017 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
(function() {
var recording = false;
var onRecordingScreen = false;
var passedSaveDirectory = false;
var tablet = Tablet.getTablet("com.highfidelity.interface.tablet.system");
var button = tablet.addButton({
text: "IRecord"
});
function onClick() {
if (onRecordingScreen) {
tablet.gotoHomeScreen();
onRecordingScreen = false;
} else {
tablet.loadQMLSource("InputRecorder.qml");
onRecordingScreen = true;
}
}
function onScreenChanged(type, url) {
onRecordingScreen = false;
passedSaveDirectory = false;
}
button.clicked.connect(onClick);
tablet.fromQml.connect(fromQml);
tablet.screenChanged.connect(onScreenChanged);
function fromQml(message) {
switch (message.method) {
case "Start":
startRecording();
break;
case "Stop":
stopRecording();
break;
case "Save":
saveRecording();
break;
case "Load":
loadRecording(message.params.file);
break;
case "playback":
startPlayback();
break;
}
}
function startRecording() {
Controller.startInputRecording();
recording = true;
}
function stopRecording() {
Controller.stopInputRecording();
recording = false;
}
function saveRecording() {
Controller.saveInputRecording();
}
function loadRecording(file) {
Controller.loadInputRecording(file);
}
function startPlayback() {
Controller.startInputPlayback();
}
function sendToQml(message) {
tablet.sendToQml(message);
}
function update() {
if (!passedSaveDirectory) {
var directory = Controller.getInputRecorderSaveDirectory();
sendToQml({method: "path", params: directory});
passedSaveDirectory = true;
}
sendToQml({method: "update", params: recording});
}
Script.setInterval(update, 60);
Script.scriptEnding.connect(function () {
button.clicked.disconnect(onClick);
if (tablet) {
tablet.removeButton(button);
}
Controller.stopInputRecording();
});
}());