From 9b87e31739f18da26bc22a88308b1c96e8e8d671 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 18:47:55 -0700 Subject: [PATCH 01/21] Fixed typo --- libraries/avatars/src/Referential.cpp | 2 +- libraries/avatars/src/Referential.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/Referential.cpp b/libraries/avatars/src/Referential.cpp index ebfb809471..d5203a0a9d 100644 --- a/libraries/avatars/src/Referential.cpp +++ b/libraries/avatars/src/Referential.cpp @@ -83,7 +83,7 @@ int Referential::pack(unsigned char* destinationBuffer) const { int Referential::unpack(const unsigned char* sourceBuffer) { const unsigned char* startPosition = sourceBuffer; _type = (Type)*sourceBuffer++; - if (_type < 0 || _type >= NUM_TYPE) { + if (_type < 0 || _type >= NUM_TYPES) { _type = UNKNOWN; } memcpy(&_version, sourceBuffer, sizeof(_version)); diff --git a/libraries/avatars/src/Referential.h b/libraries/avatars/src/Referential.h index f24d66c160..d824cad42b 100644 --- a/libraries/avatars/src/Referential.h +++ b/libraries/avatars/src/Referential.h @@ -26,7 +26,7 @@ public: JOINT, AVATAR, - NUM_TYPE + NUM_TYPES }; Referential(const unsigned char*& sourceBuffer, AvatarData* avatar); From 145e11a478890608c04bf9a5c8d38d655f78a6b7 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:01:40 -0700 Subject: [PATCH 02/21] Move one-line method to .h Moved method to .h f line less than 120 characters Added _lookAtPosition to RecordingFrame --- libraries/avatars/src/Recorder.cpp | 34 ++++++------------------------ libraries/avatars/src/Recorder.h | 17 +++++++++------ 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 70568a3487..ce4bd3056d 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -20,38 +20,16 @@ #include "AvatarData.h" #include "Recorder.h" + + + +} +} + void RecordingFrame::setBlendshapeCoefficients(QVector blendshapeCoefficients) { _blendshapeCoefficients = blendshapeCoefficients; } -void RecordingFrame::setJointRotations(QVector jointRotations) { - _jointRotations = jointRotations; -} - -void RecordingFrame::setTranslation(glm::vec3 translation) { - _translation = translation; -} - -void RecordingFrame::setRotation(glm::quat rotation) { - _rotation = rotation; -} - -void RecordingFrame::setScale(float scale) { - _scale = scale; -} - -void RecordingFrame::setHeadRotation(glm::quat headRotation) { - _headRotation = headRotation; -} - -void RecordingFrame::setLeanSideways(float leanSideways) { - _leanSideways = leanSideways; -} - -void RecordingFrame::setLeanForward(float leanForward) { - _leanForward = leanForward; -} - Recording::Recording() : _audio(NULL) { } diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index 447c3eabe8..a2eb34399b 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -48,16 +48,18 @@ public: glm::quat getHeadRotation() const { return _headRotation; } float getLeanSideways() const { return _leanSideways; } float getLeanForward() const { return _leanForward; } + glm::vec3 getLookAtPosition() const { return _lookAtPosition; } protected: void setBlendshapeCoefficients(QVector blendshapeCoefficients); - void setJointRotations(QVector jointRotations); - void setTranslation(glm::vec3 translation); - void setRotation(glm::quat rotation); - void setScale(float scale); - void setHeadRotation(glm::quat headRotation); - void setLeanSideways(float leanSideways); - void setLeanForward(float leanForward); + void setJointRotations(QVector jointRotations) { _jointRotations = jointRotations; } + void setTranslation(glm::vec3 translation) { _translation = translation; } + void setRotation(glm::quat rotation) { _rotation = rotation; } + void setScale(float scale) { _scale = scale; } + void setHeadRotation(glm::quat headRotation) { _headRotation = headRotation; } + void setLeanSideways(float leanSideways) { _leanSideways = leanSideways; } + void setLeanForward(float leanForward) { _leanForward = leanForward; } + void setLookAtPosition(glm::vec3 lookAtPosition) { _lookAtPosition = lookAtPosition; } private: QVector _blendshapeCoefficients; @@ -68,6 +70,7 @@ private: glm::quat _headRotation; float _leanSideways; float _leanForward; + glm::vec3 _lookAtPosition; friend class Recorder; friend void writeRecordingToFile(RecordingPointer recording, QString file); From 8f65f036ddadb7a282c1c193cfa1e3ca09e367fd Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:06:09 -0700 Subject: [PATCH 03/21] Remove deprecated getters --- libraries/avatars/src/Recorder.cpp | 31 ------------------------------ libraries/avatars/src/Recorder.h | 6 ------ 2 files changed, 37 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index ce4bd3056d..340146fa92 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -186,37 +186,6 @@ qint64 Player::elapsed() const { } } -glm::quat Player::getHeadRotation() { - if (!computeCurrentFrame()) { - qWarning() << "Incorrect use of Player::getHeadRotation()"; - return glm::quat(); - } - - if (_currentFrame == 0) { - return _recording->getFrame(_currentFrame).getHeadRotation(); - } - return _recording->getFrame(0).getHeadRotation() * - _recording->getFrame(_currentFrame).getHeadRotation(); -} - -float Player::getLeanSideways() { - if (!computeCurrentFrame()) { - qWarning() << "Incorrect use of Player::getLeanSideways()"; - return 0.0f; - } - - return _recording->getFrame(_currentFrame).getLeanSideways(); -} - -float Player::getLeanForward() { - if (!computeCurrentFrame()) { - qWarning() << "Incorrect use of Player::getLeanForward()"; - return 0.0f; - } - - return _recording->getFrame(_currentFrame).getLeanForward(); -} - void Player::startPlaying() { if (_recording && _recording->getFrameNumber() > 0) { qDebug() << "Recorder::startPlaying()"; diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index a2eb34399b..e1cfd77cba 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -142,12 +142,6 @@ public: RecordingPointer getRecording() const { return _recording; } - // Those should only be called if isPlaying() returns true - glm::quat getHeadRotation(); - float getLeanSideways(); - float getLeanForward(); - - public slots: void startPlaying(); void stopPlaying(); From 29f85ecca42c5193ac2bd63438ef966703201904 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:09:40 -0700 Subject: [PATCH 04/21] Introduced RecordingContext class --- libraries/avatars/src/Recorder.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index e1cfd77cba..51b59b11df 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -26,6 +26,7 @@ #include #include +class AttachmentData; class AvatarData; class Recorder; class Recording; @@ -77,6 +78,21 @@ private: friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); }; +class RecordingContext { +public: + QString domain; + glm::vec3 position; + glm::quat orientation; + float scale; + QString headModel; + QString skeletonModel; + QString displayName; + QVector attachments; + + // This avoids recomputation every frame while recording. + glm::quat orientationInv; +}; + /// Stores a recording class Recording { public: @@ -86,6 +102,7 @@ public: bool isEmpty() const { return _timestamps.isEmpty(); } int getLength() const; // in ms + RecordingContext& getContext() { return _context; } int getFrameNumber() const { return _frames.size(); } qint32 getFrameTimestamp(int i) const; const RecordingFrame& getFrame(int i) const; @@ -97,6 +114,7 @@ protected: void clear(); private: + RecordingContext _context; QVector _timestamps; QVector _frames; From b28e7e45620438743d3a64fa7236478428d630c8 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:14:47 -0700 Subject: [PATCH 05/21] Added RecordingContext support to Player/Recorder Changed meaning of certain values stored in recordings --- libraries/avatars/src/Recorder.cpp | 172 ++++++++++++++++++++--------- libraries/avatars/src/Recorder.h | 4 +- 2 files changed, 118 insertions(+), 58 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 340146fa92..9c908d7d6c 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -18,12 +18,18 @@ #include #include "AvatarData.h" +#include #include "Recorder.h" +// TODO: remove operators +void operator<<(QDebug& stream, glm::vec3 vector) { + stream << vector.x << vector.y << vector.z; } +void operator<<(QDebug& stream, glm::quat quat) { + stream << quat.x << quat.y << quat.z << quat.w; } void RecordingFrame::setBlendshapeCoefficients(QVector blendshapeCoefficients) { @@ -97,24 +103,42 @@ qint64 Recorder::elapsed() const { void Recorder::startRecording() { qDebug() << "Recorder::startRecording()"; _recording->clear(); + + RecordingContext& context = _recording->getContext(); + context.domain = NodeList::getInstance()->getDomainHandler().getHostname(); + context.position = _avatar->getPosition(); + context.orientation = _avatar->getOrientation(); + context.scale = _avatar->getTargetScale(); + context.headModel = _avatar->getFaceModelURL().toString(); + context.skeletonModel = _avatar->getSkeletonModelURL().toString(); + context.displayName = _avatar->getDisplayName(); + context.attachments = _avatar->getAttachmentData(); + + context.orientationInv = glm::inverse(context.orientation); + + bool wantDebug = false; + if (wantDebug) { + qDebug() << "Recorder::startRecording(): Recording Context"; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head URL:" << context.headModel; + qDebug() << "Skeleton URL:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << context.attachments.size(); + + for (int i = 0; i < context.attachments.size(); ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + } + _timer.start(); - - RecordingFrame frame; - frame.setBlendshapeCoefficients(_avatar->getHeadData()->getBlendshapeCoefficients()); - frame.setJointRotations(_avatar->getJointRotations()); - frame.setTranslation(_avatar->getPosition()); - frame.setRotation(_avatar->getOrientation()); - frame.setScale(_avatar->getTargetScale()); - - const HeadData* head = _avatar->getHeadData(); - glm::quat rotation = glm::quat(glm::radians(glm::vec3(head->getFinalPitch(), - head->getFinalYaw(), - head->getFinalRoll()))); - frame.setHeadRotation(rotation); - frame.setLeanForward(_avatar->getHeadData()->getLeanForward()); - frame.setLeanSideways(_avatar->getHeadData()->getLeanSideways()); - - _recording->addFrame(0, frame); + record(); } void Recorder::stopRecording() { @@ -134,22 +158,41 @@ void Recorder::saveToFile(QString file) { void Recorder::record() { if (isRecording()) { - const RecordingFrame& referenceFrame = _recording->getFrame(0); + const RecordingContext& context = _recording->getContext(); RecordingFrame frame; frame.setBlendshapeCoefficients(_avatar->getHeadData()->getBlendshapeCoefficients()); frame.setJointRotations(_avatar->getJointRotations()); - frame.setTranslation(_avatar->getPosition() - referenceFrame.getTranslation()); - frame.setRotation(glm::inverse(referenceFrame.getRotation()) * _avatar->getOrientation()); - frame.setScale(_avatar->getTargetScale() / referenceFrame.getScale()); + frame.setTranslation(context.orientationInv * (_avatar->getPosition() - context.position)); + frame.setRotation(context.orientationInv * _avatar->getOrientation()); + frame.setScale(_avatar->getTargetScale() / context.scale); const HeadData* head = _avatar->getHeadData(); - glm::quat rotation = glm::quat(glm::radians(glm::vec3(head->getFinalPitch(), - head->getFinalYaw(), - head->getFinalRoll()))); - frame.setHeadRotation(rotation); - frame.setLeanForward(_avatar->getHeadData()->getLeanForward()); - frame.setLeanSideways(_avatar->getHeadData()->getLeanSideways()); + if (head) { + glm::vec3 rotationDegrees = glm::vec3(head->getFinalPitch(), + head->getFinalYaw(), + head->getFinalRoll()); + frame.setHeadRotation(glm::quat(glm::radians(rotationDegrees))); + frame.setLeanForward(head->getLeanForward()); + frame.setLeanSideways(head->getLeanSideways()); + glm::vec3 relativeLookAt = context.orientationInv * + (head->getLookAtPosition() - context.position); + frame.setLookAtPosition(relativeLookAt); + } + + bool wantDebug = false; + if (wantDebug) { + qDebug() << "Recording frame #" << _recording->getFrameNumber(); + qDebug() << "Blendshapes:" << frame.getBlendshapeCoefficients().size(); + qDebug() << "JointRotations:" << frame.getJointRotations().size(); + qDebug() << "Translation:" << frame.getTranslation(); + qDebug() << "Rotation:" << frame.getRotation(); + qDebug() << "Scale:" << frame.getScale(); + qDebug() << "Head rotation:" << frame.getHeadRotation(); + qDebug() << "Lean Forward:" << frame.getLeanForward(); + qDebug() << "Lean Sideways:" << frame.getLeanSideways(); + qDebug() << "LookAtPosition:" << frame.getLookAtPosition(); + } _recording->addFrame(_timer.elapsed(), frame); } @@ -165,7 +208,6 @@ Player::Player(AvatarData* avatar) : _recording(new Recording()), _avatar(avatar), _audioThread(NULL), - _startingScale(1.0f), _playFromCurrentPosition(true), _loop(false) { @@ -188,9 +230,6 @@ qint64 Player::elapsed() const { void Player::startPlaying() { if (_recording && _recording->getFrameNumber() > 0) { - qDebug() << "Recorder::startPlaying()"; - _currentFrame = 0; - // Setup audio thread _audioThread = new QThread(); _options.setPosition(_avatar->getPosition()); @@ -203,16 +242,40 @@ void Player::startPlaying() { // Fake faceshift connection _avatar->setForceFaceshiftConnected(true); - if (_playFromCurrentPosition) { - _startingPosition = _avatar->getPosition(); - _startingRotation = _avatar->getOrientation(); - _startingScale = _avatar->getTargetScale(); - } else { - _startingPosition = _recording->getFrame(0).getTranslation(); - _startingRotation = _recording->getFrame(0).getRotation(); - _startingScale = _recording->getFrame(0).getScale(); + _currentContext.domain = NodeList::getInstance()->getDomainHandler().getHostname(); + _currentContext.position = _avatar->getPosition(); + _currentContext.orientation = _avatar->getOrientation(); + _currentContext.scale = _avatar->getTargetScale(); + _currentContext.headModel = _avatar->getFaceModelURL().toString(); + _currentContext.skeletonModel = _avatar->getSkeletonModelURL().toString(); + _currentContext.displayName = _avatar->getDisplayName(); + _currentContext.attachments = _avatar->getAttachmentData(); + + _currentContext.orientationInv = glm::inverse(_currentContext.orientation); + + bool wantDebug = false; + if (wantDebug) { + qDebug() << "Player::startPlaying(): Recording Context"; + qDebug() << "Domain:" << _currentContext.domain; + qDebug() << "Position:" << _currentContext.position; + qDebug() << "Orientation:" << _currentContext.orientation; + qDebug() << "Scale:" << _currentContext.scale; + qDebug() << "Head URL:" << _currentContext.headModel; + qDebug() << "Skeleton URL:" << _currentContext.skeletonModel; + qDebug() << "Display Name:" << _currentContext.displayName; + qDebug() << "Num Attachments:" << _currentContext.attachments.size(); + + for (int i = 0; i < _currentContext.attachments.size(); ++i) { + qDebug() << "Model URL:" << _currentContext.attachments[i].modelURL; + qDebug() << "Joint Name:" << _currentContext.attachments[i].jointName; + qDebug() << "Translation:" << _currentContext.attachments[i].translation; + qDebug() << "Rotation:" << _currentContext.attachments[i].rotation; + qDebug() << "Scale:" << _currentContext.attachments[i].scale; + } } + qDebug() << "Recorder::startPlaying()"; + _currentFrame = 0; _timer.start(); } } @@ -268,30 +331,29 @@ void Player::play() { return; } - if (_currentFrame == 0) { - // Don't play frame 0 - // only meant to store absolute values - return; + RecordingContext& context = _recording->getContext(); + if (_playFromCurrentPosition) { + context = _currentContext; } + const RecordingFrame& currentFrame = _recording->getFrame(_currentFrame); - _avatar->setPosition(_startingPosition + - glm::inverse(_recording->getFrame(0).getRotation()) * _startingRotation * - _recording->getFrame(_currentFrame).getTranslation()); - _avatar->setOrientation(_startingRotation * - _recording->getFrame(_currentFrame).getRotation()); - _avatar->setTargetScale(_startingScale * - _recording->getFrame(_currentFrame).getScale()); - _avatar->setJointRotations(_recording->getFrame(_currentFrame).getJointRotations()); + _avatar->setPosition(context.position + context.orientation * currentFrame.getTranslation()); + _avatar->setOrientation(context.orientation * currentFrame.getRotation()); + _avatar->setTargetScale(context.scale * currentFrame.getScale()); + _avatar->setJointRotations(currentFrame.getJointRotations()); HeadData* head = const_cast(_avatar->getHeadData()); if (head) { - head->setBlendshapeCoefficients(_recording->getFrame(_currentFrame).getBlendshapeCoefficients()); - head->setLeanSideways(_recording->getFrame(_currentFrame).getLeanSideways()); - head->setLeanForward(_recording->getFrame(_currentFrame).getLeanForward()); - glm::vec3 eulers = glm::degrees(safeEulerAngles(_recording->getFrame(_currentFrame).getHeadRotation())); + head->setBlendshapeCoefficients(currentFrame.getBlendshapeCoefficients()); + head->setLeanSideways(currentFrame.getLeanSideways()); + head->setLeanForward(currentFrame.getLeanForward()); + glm::vec3 eulers = glm::degrees(safeEulerAngles(currentFrame.getHeadRotation())); head->setFinalPitch(eulers.x); head->setFinalYaw(eulers.y); head->setFinalRoll(eulers.z); + head->setLookAtPosition(currentFrame.getLookAtPosition()); + } else { + qDebug() << "WARNING: Player couldn't find head data."; } _options.setPosition(_avatar->getPosition()); diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index 51b59b11df..84015703a6 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -183,10 +183,8 @@ private: AvatarData* _avatar; QThread* _audioThread; - glm::vec3 _startingPosition; - glm::quat _startingRotation; - float _startingScale; + RecordingContext _currentContext; bool _playFromCurrentPosition; bool _loop; }; From ffba243a4fbfab983e10862419cb6aa0e73741d2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:17:16 -0700 Subject: [PATCH 06/21] Added methods to read/write glm types --- libraries/avatars/src/Recorder.cpp | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 9c908d7d6c..fd0d30da80 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -22,6 +22,7 @@ #include "Recorder.h" +#include // TODO: remove operators @@ -386,6 +387,57 @@ bool Player::computeCurrentFrame() { return true; } +void writeVec3(QDataStream& stream, glm::vec3 value) { + unsigned char buffer[256]; + int writtenToBuffer = packFloatVec3ToSignedTwoByteFixed(buffer, value, 0); + stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); +} + +bool readVec3(QDataStream& stream, glm::vec3& value) { + int vec3ByteSize = 3 * 2; // 3 floats * 2 bytes + unsigned char buffer[256]; + stream.readRawData(reinterpret_cast(buffer), vec3ByteSize); + int readFromBuffer = unpackFloatVec3FromSignedTwoByteFixed(buffer, value, 0); + if (readFromBuffer != vec3ByteSize) { + return false; + } + return true; +} + +void writeQuat(QDataStream& stream, glm::quat value) { + unsigned char buffer[256]; + int writtenToBuffer = packOrientationQuatToBytes(buffer, value); + stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); +} + +bool readQuat(QDataStream& stream, glm::quat& value) { + int quatByteSize = 4 * 2; // 4 floats * 2 bytes + unsigned char buffer[256]; + stream.readRawData(reinterpret_cast(buffer), quatByteSize); + int readFromBuffer = unpackOrientationQuatFromBytes(buffer, value); + if (readFromBuffer != quatByteSize) { + return false; + } + return true; +} + +void writeFloat(QDataStream& stream, float value) { + unsigned char buffer[256]; + int writtenToBuffer = packFloatScalarToSignedTwoByteFixed(buffer, value, 0); + stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); +} + +bool readFloat(QDataStream& stream, float& value) { + int floatByteSize = 2; // 1 floats * 2 bytes + int16_t buffer[256]; + stream.readRawData(reinterpret_cast(buffer), floatByteSize); + int readFromBuffer = unpackFloatScalarFromSignedTwoByteFixed(buffer, &value, 0); + if (readFromBuffer != floatByteSize) { + return false; + } + return true; +} + void writeRecordingToFile(RecordingPointer recording, QString filename) { if (!recording || recording->getFrameNumber() < 1) { qDebug() << "Can't save empty recording"; From d80a42b80000b58c55103e004b95581f89def8c3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 15 Sep 2014 19:19:10 -0700 Subject: [PATCH 07/21] New recording file format Added header containing: - Magic number - Version number - Data Offset - Data Length - CRC-32 Planned space for metadata Added context block (domain, models, transform, display name, attachments) Improved glm types packing (SHould take about half the space it used to)) --- libraries/avatars/src/Recorder.cpp | 392 ++++++++++++++++++++--------- 1 file changed, 274 insertions(+), 118 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index fd0d30da80..74ea541bd8 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -24,6 +24,14 @@ #include +// HFR file format magic number +// (decimal) 17 72 70 82 13 10 26 10 +// (hexadecimal) 11 48 46 52 0d 0a 1a 0a +// (ASCII C notation) \021 H F R \r \n \032 \n +static const int MAGIC_NUMBER_SIZE = 8; +static const char MAGIC_NUMBER[MAGIC_NUMBER_SIZE] = {17, 72, 70, 82, 13, 10, 26, 10}; +// Version (Major, Minor) +static const QPair VERSION(0, 1); // TODO: remove operators void operator<<(QDebug& stream, glm::vec3 vector) { @@ -444,122 +452,173 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { return; } - qDebug() << "Writing recording to " << filename << "."; QElapsedTimer timer; QFile file(filename); - if (!file.open(QIODevice::WriteOnly)){ + if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)){ + qDebug() << "Couldn't open " << filename; return; } timer.start(); - + qDebug() << "Writing recording to " << filename << "."; QDataStream fileStream(&file); + // HEADER + file.write(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); // Magic number + fileStream << VERSION; // File format version + const qint64 dataOffsetPos = file.pos(); + fileStream << (quint16)17; // Save two empty bytes for the data offset + const qint64 dataLengthPos = file.pos(); + fileStream << (quint32)42; // Save four empty bytes for the data offset + const quint64 crc32Pos = file.pos(); + fileStream << (quint32)121; // Save four empty bytes for the CRC-32 + + + // METADATA + // TODO + + + + // CONTEXT + RecordingContext& context = recording->getContext(); + // Domain + fileStream << context.domain; + // Position + writeVec3(fileStream, context.position); + // Orientation + writeQuat(fileStream, context.orientation); + // Scale + writeFloat(fileStream, context.scale); + // Head model + fileStream << context.headModel; + // Skeleton model + fileStream << context.skeletonModel; + // Display name + fileStream << context.displayName; + // Attachements + fileStream << (quint8)context.attachments.size(); + foreach (AttachmentData data, context.attachments) { + // Model + fileStream << data.modelURL.toString(); + // Joint name + fileStream << data.jointName; + // Position + writeVec3(fileStream, data.translation); + // Orientation + writeQuat(fileStream, data.rotation); + // Scale + writeFloat(fileStream, data.scale); + } + + // RECORDING fileStream << recording->_timestamps; - RecordingFrame& baseFrame = recording->_frames[0]; - int totalLength = 0; + QBitArray mask; - // Blendshape coefficients - fileStream << baseFrame._blendshapeCoefficients; - totalLength += baseFrame._blendshapeCoefficients.size(); - - // Joint Rotations - int jointRotationSize = baseFrame._jointRotations.size(); - fileStream << jointRotationSize; - for (int i = 0; i < jointRotationSize; ++i) { - fileStream << baseFrame._jointRotations[i].x << baseFrame._jointRotations[i].y << baseFrame._jointRotations[i].z << baseFrame._jointRotations[i].w; - } - totalLength += jointRotationSize; - - // Translation - fileStream << baseFrame._translation.x << baseFrame._translation.y << baseFrame._translation.z; - totalLength += 1; - - // Rotation - fileStream << baseFrame._rotation.x << baseFrame._rotation.y << baseFrame._rotation.z << baseFrame._rotation.w; - totalLength += 1; - - // Scale - fileStream << baseFrame._scale; - totalLength += 1; - - // Head Rotation - fileStream << baseFrame._headRotation.x << baseFrame._headRotation.y << baseFrame._headRotation.z << baseFrame._headRotation.w; - totalLength += 1; - - // Lean Sideways - fileStream << baseFrame._leanSideways; - totalLength += 1; - - // Lean Forward - fileStream << baseFrame._leanForward; - totalLength += 1; - - for (int i = 1; i < recording->_timestamps.size(); ++i) { - QBitArray mask(totalLength); + for (int i = 0; i < recording->_timestamps.size(); ++i) { + mask.fill(false); int maskIndex = 0; QByteArray buffer; QDataStream stream(&buffer, QIODevice::WriteOnly); - RecordingFrame& previousFrame = recording->_frames[i - 1]; + RecordingFrame& previousFrame = recording->_frames[(i != 0) ? i - 1 : i]; RecordingFrame& frame = recording->_frames[i]; - // Blendshape coefficients - for (int i = 0; i < frame._blendshapeCoefficients.size(); ++i) { - if (frame._blendshapeCoefficients[i] != previousFrame._blendshapeCoefficients[i]) { - stream << frame._blendshapeCoefficients[i]; + // Blendshape Coefficients + quint32 numBlenshapes = frame.getBlendshapeCoefficients().size(); + stream << numBlenshapes; + if (i == 0) { + mask.resize(mask.size() + numBlenshapes); + } + for (int j = 0; j < numBlenshapes; ++j) { + if (i == 0 || + frame._blendshapeCoefficients[j] != previousFrame._blendshapeCoefficients[j]) { + writeFloat(stream, frame.getBlendshapeCoefficients()[j]); mask.setBit(maskIndex); } - maskIndex++; + ++maskIndex; } // Joint Rotations - for (int i = 0; i < frame._jointRotations.size(); ++i) { - if (frame._jointRotations[i] != previousFrame._jointRotations[i]) { - stream << frame._jointRotations[i].x << frame._jointRotations[i].y << frame._jointRotations[i].z << frame._jointRotations[i].w; + quint32 numJoints = frame.getJointRotations().size(); + stream << numJoints; + if (i == 0) { + mask.resize(mask.size() + numJoints); + } + for (int j = 0; j < numJoints; ++j) { + if (i == 0 || + frame._jointRotations[j] != previousFrame._jointRotations[j]) { + writeQuat(stream, frame._jointRotations[j]); mask.setBit(maskIndex); } maskIndex++; } // Translation - if (frame._translation != previousFrame._translation) { - stream << frame._translation.x << frame._translation.y << frame._translation.z; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._translation != previousFrame._translation) { + writeVec3(stream, frame._translation); mask.setBit(maskIndex); } maskIndex++; // Rotation - if (frame._rotation != previousFrame._rotation) { - stream << frame._rotation.x << frame._rotation.y << frame._rotation.z << frame._rotation.w; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._rotation != previousFrame._rotation) { + writeQuat(stream, frame._rotation); mask.setBit(maskIndex); } maskIndex++; // Scale - if (frame._scale != previousFrame._scale) { - stream << frame._scale; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._scale != previousFrame._scale) { + writeFloat(stream, frame._scale); mask.setBit(maskIndex); } maskIndex++; // Head Rotation - if (frame._headRotation != previousFrame._headRotation) { - stream << frame._headRotation.x << frame._headRotation.y << frame._headRotation.z << frame._headRotation.w; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._headRotation != previousFrame._headRotation) { + writeQuat(stream, frame._headRotation); mask.setBit(maskIndex); } maskIndex++; // Lean Sideways - if (frame._leanSideways != previousFrame._leanSideways) { - stream << frame._leanSideways; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._leanSideways != previousFrame._leanSideways) { + writeFloat(stream, frame._leanSideways); mask.setBit(maskIndex); } maskIndex++; // Lean Forward - if (frame._leanForward != previousFrame._leanForward) { - stream << frame._leanForward; + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._leanForward != previousFrame._leanForward) { + writeFloat(stream, frame._leanForward); + mask.setBit(maskIndex); + } + maskIndex++; + + // LookAt Position + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._lookAtPosition != previousFrame._lookAtPosition) { + writeVec3(stream, frame._lookAtPosition); mask.setBit(maskIndex); } maskIndex++; @@ -570,22 +629,30 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { fileStream << recording->_audio->getByteArray(); + // TODO: Complete empty bytes + file.seek(dataOffsetPos); + file.seek(dataLengthPos); + file.seek(crc32Pos); + qDebug() << "Wrote " << file.size() << " bytes in " << timer.elapsed() << " ms."; } RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename) { - QElapsedTimer timer; - timer.start(); - QByteArray byteArray; QUrl url(filename); + QElapsedTimer timer; + timer.start(); // timer used for debug informations (download/parsing time) + + // Aquire the data and place it in byteArray + // Return if data unavailable if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { + // Download file if necessary qDebug() << "Downloading recording at" << url; NetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); QNetworkReply* reply = networkAccessManager.get(QNetworkRequest(url)); QEventLoop loop; QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); - loop.exec(); + loop.exec(); // wait for file if (reply->error() != QNetworkReply::NoError) { qDebug() << "Error while downloading recording: " << reply->error(); reply->deleteLater(); @@ -593,112 +660,201 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen } byteArray = reply->readAll(); reply->deleteLater(); + // print debug + restart timer + qDebug() << "Downloaded " << byteArray.size() << " bytes in " << timer.restart() << " ms."; } else { + // If local file, just read it. qDebug() << "Reading recording from " << filename << "."; QFile file(filename); if (!file.open(QIODevice::ReadOnly)){ + qDebug() << "Could not open local file: " << url; return recording; } byteArray = file.readAll(); file.close(); } + // Reset the recording passed in the arguments if (!recording) { recording.reset(new Recording()); } QDataStream fileStream(byteArray); - fileStream >> recording->_timestamps; - RecordingFrame baseFrame; + // HEADER + QByteArray magicNumber(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); + if (!byteArray.startsWith(magicNumber)) { + qDebug() << "ERROR: This is not a .HFR file. (Magic Number incorrect)"; + return recording; + } + fileStream.skipRawData(MAGIC_NUMBER_SIZE); - // Blendshape coefficients - fileStream >> baseFrame._blendshapeCoefficients; - - // Joint Rotations - int jointRotationSize; - fileStream >> jointRotationSize; - baseFrame._jointRotations.resize(jointRotationSize); - for (int i = 0; i < jointRotationSize; ++i) { - fileStream >> baseFrame._jointRotations[i].x >> baseFrame._jointRotations[i].y >> baseFrame._jointRotations[i].z >> baseFrame._jointRotations[i].w; + QPair version; + fileStream >> version; // File format version + if (version != VERSION) { + qDebug() << "ERROR: This file format version is not supported."; + return recording; } - fileStream >> baseFrame._translation.x >> baseFrame._translation.y >> baseFrame._translation.z; - fileStream >> baseFrame._rotation.x >> baseFrame._rotation.y >> baseFrame._rotation.z >> baseFrame._rotation.w; - fileStream >> baseFrame._scale; - fileStream >> baseFrame._headRotation.x >> baseFrame._headRotation.y >> baseFrame._headRotation.z >> baseFrame._headRotation.w; - fileStream >> baseFrame._leanSideways; - fileStream >> baseFrame._leanForward; + quint16 dataLength = 0; + fileStream >> dataLength; + quint32 dataOffset = 0; + fileStream >> dataOffset; + quint32 crc32 = 0; + fileStream >> crc32; - recording->_frames << baseFrame; + // METADATA + // TODO - for (int i = 1; i < recording->_timestamps.size(); ++i) { + + + // CONTEXT + RecordingContext& context = recording->getContext(); + + // Domain + fileStream >> context.domain; + // Position + if (!readVec3(fileStream, context.position)) { + qDebug() << "Couldn't read file correctly. (Invalid vec3)"; + recording.clear(); + return recording; + } + // Orientation + if (!readQuat(fileStream, context.orientation)) { + qDebug() << "Couldn't read file correctly. (Invalid quat)"; + recording.clear(); + return recording; + } + + // Scale + if (!readFloat(fileStream, context.scale)) { + qDebug() << "Couldn't read file correctly. (Invalid float)"; + recording.clear(); + return recording; + } + // Head model + fileStream >> context.headModel; + // Skeleton model + fileStream >> context.skeletonModel; + // Display Name + fileStream >> context.displayName; + + // Attachements + quint8 numAttachments = 0; + fileStream >> numAttachments; + for (int i = 0; i < numAttachments; ++i) { + AttachmentData data; + // Model + fileStream >> data.modelURL; + // Joint name + fileStream >> data.jointName; + // Translation + if (!readVec3(fileStream, data.translation)) { + qDebug() << "Couldn't read attachment correctly. (Invalid vec3)"; + continue; + } + // Rotation + if (!readQuat(fileStream, data.rotation)) { + qDebug() << "Couldn't read attachment correctly. (Invalid quat)"; + continue; + } + + // Scale + if (!readFloat(fileStream, data.scale)) { + qDebug() << "Couldn't read attachment correctly. (Invalid float)"; + continue; + } + context.attachments << data; + } + + bool wantDebug = false; + if (wantDebug) { + qDebug() << "File Format version:" << VERSION; + qDebug() << "Data length:" << dataLength; + qDebug() << "Data offset:" << dataOffset; + qDebug() << "CRC-32:" << crc32; + + qDebug() << "Context block:"; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head Model:" << context.headModel; + qDebug() << "Skeleton Model:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << numAttachments; + + for (int i = 0; i < numAttachments; ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + } + + // RECORDING + fileStream >> recording->_timestamps; + + for (int i = 0; i < recording->_timestamps.size(); ++i) { QBitArray mask; QByteArray buffer; QDataStream stream(&buffer, QIODevice::ReadOnly); RecordingFrame frame; - RecordingFrame& previousFrame = recording->_frames.last(); + RecordingFrame& previousFrame = (i == 0) ? frame : recording->_frames.last(); fileStream >> mask; fileStream >> buffer; int maskIndex = 0; // Blendshape Coefficients - frame._blendshapeCoefficients.resize(baseFrame._blendshapeCoefficients.size()); - for (int i = 0; i < baseFrame._blendshapeCoefficients.size(); ++i) { - if (mask[maskIndex++]) { - stream >> frame._blendshapeCoefficients[i]; - } else { - frame._blendshapeCoefficients[i] = previousFrame._blendshapeCoefficients[i]; + quint32 numBlendshapes = 0; + stream >> numBlendshapes; + frame._blendshapeCoefficients.resize(numBlendshapes); + for (int j = 0; j < numBlendshapes; ++j) { + if (!mask[maskIndex++] || !readFloat(stream, frame._blendshapeCoefficients[j])) { + frame._blendshapeCoefficients[j] = previousFrame._blendshapeCoefficients[j]; } } // Joint Rotations - frame._jointRotations.resize(baseFrame._jointRotations.size()); - for (int i = 0; i < baseFrame._jointRotations.size(); ++i) { - if (mask[maskIndex++]) { - stream >> frame._jointRotations[i].x >> frame._jointRotations[i].y >> frame._jointRotations[i].z >> frame._jointRotations[i].w; - } else { - frame._jointRotations[i] = previousFrame._jointRotations[i]; + quint32 numJoints = 0; + stream >> numJoints; + frame._jointRotations.resize(numJoints); + for (int j = 0; j < numJoints; ++j) { + if (!mask[maskIndex++] || !readQuat(stream, frame._jointRotations[j])) { + frame._jointRotations[j] = previousFrame._jointRotations[j]; } } - if (mask[maskIndex++]) { - stream >> frame._translation.x >> frame._translation.y >> frame._translation.z; - } else { + if (!mask[maskIndex++] || !readVec3(stream, frame._translation)) { frame._translation = previousFrame._translation; } - if (mask[maskIndex++]) { - stream >> frame._rotation.x >> frame._rotation.y >> frame._rotation.z >> frame._rotation.w; - } else { + if (!mask[maskIndex++] || !readQuat(stream, frame._rotation)) { frame._rotation = previousFrame._rotation; } - if (mask[maskIndex++]) { - stream >> frame._scale; - } else { + if (!mask[maskIndex++] || !readFloat(stream, frame._scale)) { frame._scale = previousFrame._scale; } - if (mask[maskIndex++]) { - stream >> frame._headRotation.x >> frame._headRotation.y >> frame._headRotation.z >> frame._headRotation.w; - } else { + if (!mask[maskIndex++] || !readQuat(stream, frame._headRotation)) { frame._headRotation = previousFrame._headRotation; } - if (mask[maskIndex++]) { - stream >> frame._leanSideways; - } else { + if (!mask[maskIndex++] || !readFloat(stream, frame._leanSideways)) { frame._leanSideways = previousFrame._leanSideways; } - if (mask[maskIndex++]) { - stream >> frame._leanForward; - } else { + if (!mask[maskIndex++] || !readFloat(stream, frame._leanForward)) { frame._leanForward = previousFrame._leanForward; } + if (!mask[maskIndex++] || !readVec3(stream, frame._lookAtPosition)) { + frame._lookAtPosition = previousFrame._lookAtPosition; + } + recording->_frames << frame; } From 59637390a23e9ee1ffc11b783df5823c61f643a7 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 12:26:55 -0700 Subject: [PATCH 08/21] Add global timestamp to recordings --- libraries/avatars/src/Recorder.cpp | 7 +++++-- libraries/avatars/src/Recorder.h | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 74ea541bd8..267d362f89 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -33,7 +33,6 @@ static const char MAGIC_NUMBER[MAGIC_NUMBER_SIZE] = {17, 72, 70, 82, 13, 10, 26, // Version (Major, Minor) static const QPair VERSION(0, 1); -// TODO: remove operators void operator<<(QDebug& stream, glm::vec3 vector) { stream << vector.x << vector.y << vector.z; } @@ -114,6 +113,7 @@ void Recorder::startRecording() { _recording->clear(); RecordingContext& context = _recording->getContext(); + context.globalTimestamp = usecTimestampNow(); context.domain = NodeList::getInstance()->getDomainHandler().getHostname(); context.position = _avatar->getPosition(); context.orientation = _avatar->getOrientation(); @@ -481,6 +481,8 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { // CONTEXT RecordingContext& context = recording->getContext(); + // Global Timestamp + fileStream << context.globalTimestamp; // Domain fileStream << context.domain; // Position @@ -710,7 +712,8 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen // CONTEXT RecordingContext& context = recording->getContext(); - + // Global Timestamp + fileStream >> context.globalTimestamp; // Domain fileStream >> context.domain; // Position diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index 84015703a6..f3125bff05 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -80,6 +80,7 @@ private: class RecordingContext { public: + quint64 globalTimestamp; QString domain; glm::vec3 position; glm::quat orientation; From 8573130f079c80aa89c482a82f5f74fd713e4ea1 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 12:34:22 -0700 Subject: [PATCH 09/21] Write correct Offset ad Length of the data --- libraries/avatars/src/Recorder.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 267d362f89..60bcaf8dee 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -467,11 +467,11 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { file.write(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); // Magic number fileStream << VERSION; // File format version const qint64 dataOffsetPos = file.pos(); - fileStream << (quint16)17; // Save two empty bytes for the data offset + fileStream << (quint16)0; // Save two empty bytes for the data offset const qint64 dataLengthPos = file.pos(); - fileStream << (quint32)42; // Save four empty bytes for the data offset + fileStream << (quint32)0; // Save four empty bytes for the data offset const quint64 crc32Pos = file.pos(); - fileStream << (quint32)121; // Save four empty bytes for the CRC-32 + fileStream << (quint32)0; // Save four empty bytes for the CRC-32 // METADATA @@ -479,6 +479,12 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { + // Write data offset + quint16 dataOffset = file.pos(); + file.seek(dataOffsetPos); + fileStream << dataOffset; + file.seek(dataOffset); + // CONTEXT RecordingContext& context = recording->getContext(); // Global Timestamp @@ -631,10 +637,11 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { fileStream << recording->_audio->getByteArray(); - // TODO: Complete empty bytes - file.seek(dataOffsetPos); + // Write data length + quint16 dataLength = file.pos() - dataOffset; file.seek(dataLengthPos); - file.seek(crc32Pos); + fileStream << dataLength; + file.seek(dataOffset + dataLength); qDebug() << "Wrote " << file.size() << " bytes in " << timer.elapsed() << " ms."; } From 72481999dfa0cd8889db45e60fb1612931acaaf3 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 12:46:42 -0700 Subject: [PATCH 10/21] Compute actual checksum --- libraries/avatars/src/Recorder.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 60bcaf8dee..51b2938290 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -470,8 +470,8 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { fileStream << (quint16)0; // Save two empty bytes for the data offset const qint64 dataLengthPos = file.pos(); fileStream << (quint32)0; // Save four empty bytes for the data offset - const quint64 crc32Pos = file.pos(); - fileStream << (quint32)0; // Save four empty bytes for the CRC-32 + const quint64 crc16Pos = file.pos(); + fileStream << (quint32)0; // Save four empty bytes for the CRC-16 // METADATA @@ -637,13 +637,19 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { fileStream << recording->_audio->getByteArray(); - // Write data length + qint64 writtingTime = timer.restart(); + // Write data length and CRC-16 quint16 dataLength = file.pos() - dataOffset; + quint16 crc16= qChecksum(file.readAll().constData() + dataOffset, + dataLength); file.seek(dataLengthPos); fileStream << dataLength; + file.seek(crc16Pos); + fileStream << crc16; file.seek(dataOffset + dataLength); - qDebug() << "Wrote " << file.size() << " bytes in " << timer.elapsed() << " ms."; + qint64 checksumTime = timer.elapsed(); + qDebug() << "Wrote" << file.size() << "bytes in" << writtingTime + checksumTime << "ms. (" << checksumTime << "ms for checksum)"; } RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename) { @@ -709,8 +715,8 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen fileStream >> dataLength; quint32 dataOffset = 0; fileStream >> dataOffset; - quint32 crc32 = 0; - fileStream >> crc32; + quint32 crc16 = 0; + fileStream >> crc16; // METADATA // TODO @@ -782,7 +788,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen qDebug() << "File Format version:" << VERSION; qDebug() << "Data length:" << dataLength; qDebug() << "Data offset:" << dataOffset; - qDebug() << "CRC-32:" << crc32; + qDebug() << "CRC-16:" << crc16; qDebug() << "Context block:"; qDebug() << "Domain:" << context.domain; @@ -872,7 +878,6 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen fileStream >> audioArray; recording->addAudioPacket(audioArray); - qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; return recording; } From 1c5c9cb31bc2d6f6a368b80181cb03df701441f4 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 13:57:46 -0700 Subject: [PATCH 11/21] Fix bug adding first samples twice --- libraries/avatars/src/Recorder.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 51b2938290..8e0bbdeb95 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -78,6 +78,7 @@ void Recording::addFrame(int timestamp, RecordingFrame &frame) { void Recording::addAudioPacket(QByteArray byteArray) { if (!_audio) { _audio = new Sound(byteArray); + return; } _audio->append(byteArray); } From 96995e3aed5431bc9174d0d6598f11154827c523 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 13:59:30 -0700 Subject: [PATCH 12/21] Enforce checksums equality --- libraries/avatars/src/Recorder.cpp | 118 ++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 8e0bbdeb95..a394d850b4 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -455,7 +455,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { QElapsedTimer timer; QFile file(filename); - if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)){ + if (!file.open(QIODevice::ReadWrite | QIODevice::Truncate)){ qDebug() << "Couldn't open " << filename; return; } @@ -472,7 +472,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { const qint64 dataLengthPos = file.pos(); fileStream << (quint32)0; // Save four empty bytes for the data offset const quint64 crc16Pos = file.pos(); - fileStream << (quint32)0; // Save four empty bytes for the CRC-16 + fileStream << (quint16)0; // Save two empty bytes for the CRC-16 // METADATA @@ -640,15 +640,47 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { qint64 writtingTime = timer.restart(); // Write data length and CRC-16 - quint16 dataLength = file.pos() - dataOffset; - quint16 crc16= qChecksum(file.readAll().constData() + dataOffset, - dataLength); + quint32 dataLength = file.pos() - dataOffset; + file.seek(dataOffset); // Go to beginning of data for checksum + quint16 crc16 = qChecksum(file.readAll().constData(), dataLength); + file.seek(dataLengthPos); fileStream << dataLength; file.seek(crc16Pos); fileStream << crc16; file.seek(dataOffset + dataLength); + bool wantDebug = true; + if (wantDebug) { + qDebug() << "Header:"; + qDebug() << "File Format version:" << VERSION; + qDebug() << "Data length:" << dataLength; + qDebug() << "Data offset:" << dataOffset; + qDebug() << "CRC-16:" << crc16; + + qDebug() << "Context block:"; + qDebug() << "Global timestamp:" << context.globalTimestamp; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head Model:" << context.headModel; + qDebug() << "Skeleton Model:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << context.attachments.size(); + for (int i = 0; i < context.attachments.size(); ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + + qDebug() << "Recording:"; + qDebug() << "Total frames:" << recording->getFrameNumber(); + qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + } + qint64 checksumTime = timer.elapsed(); qDebug() << "Wrote" << file.size() << "bytes in" << writtingTime + checksumTime << "ms. (" << checksumTime << "ms for checksum)"; } @@ -712,13 +744,23 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen return recording; } - quint16 dataLength = 0; - fileStream >> dataLength; - quint32 dataOffset = 0; + quint16 dataOffset = 0; fileStream >> dataOffset; - quint32 crc16 = 0; + quint32 dataLength = 0; + fileStream >> dataLength; + quint16 crc16 = 0; fileStream >> crc16; + + // Check checksum + + quint16 computedCRC16 = qChecksum(byteArray.constData() + dataOffset, dataLength); + if (computedCRC16 != crc16) { + qDebug() << "Checksum does not match. Bailling!"; + recording.clear(); + return recording; + } + // METADATA // TODO @@ -784,32 +826,6 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen context.attachments << data; } - bool wantDebug = false; - if (wantDebug) { - qDebug() << "File Format version:" << VERSION; - qDebug() << "Data length:" << dataLength; - qDebug() << "Data offset:" << dataOffset; - qDebug() << "CRC-16:" << crc16; - - qDebug() << "Context block:"; - qDebug() << "Domain:" << context.domain; - qDebug() << "Position:" << context.position; - qDebug() << "Orientation:" << context.orientation; - qDebug() << "Scale:" << context.scale; - qDebug() << "Head Model:" << context.headModel; - qDebug() << "Skeleton Model:" << context.skeletonModel; - qDebug() << "Display Name:" << context.displayName; - qDebug() << "Num Attachments:" << numAttachments; - - for (int i = 0; i < numAttachments; ++i) { - qDebug() << "Model URL:" << context.attachments[i].modelURL; - qDebug() << "Joint Name:" << context.attachments[i].jointName; - qDebug() << "Translation:" << context.attachments[i].translation; - qDebug() << "Rotation:" << context.attachments[i].rotation; - qDebug() << "Scale:" << context.attachments[i].scale; - } - } - // RECORDING fileStream >> recording->_timestamps; @@ -879,6 +895,38 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen fileStream >> audioArray; recording->addAudioPacket(audioArray); + bool wantDebug = true; + if (wantDebug) { + qDebug() << "Header:"; + qDebug() << "File Format version:" << VERSION; + qDebug() << "Data length:" << dataLength; + qDebug() << "Data offset:" << dataOffset; + qDebug() << "CRC-16:" << crc16; + + qDebug() << "Context block:"; + qDebug() << "Global timestamp:" << context.globalTimestamp; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head Model:" << context.headModel; + qDebug() << "Skeleton Model:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << numAttachments; + for (int i = 0; i < numAttachments; ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + + qDebug() << "Recording:"; + qDebug() << "Total frames:" << recording->getFrameNumber(); + qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + + } + qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; return recording; } From f649cd1a6e660c124d8d81772601b4483b74b69e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 14:49:47 -0700 Subject: [PATCH 13/21] Changed how looping occurs in recordings --- libraries/avatars/src/Recorder.cpp | 67 +++++++++++++++++------------- libraries/avatars/src/Recorder.h | 3 ++ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index a394d850b4..a453c045d0 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -129,6 +129,7 @@ void Recorder::startRecording() { bool wantDebug = false; if (wantDebug) { qDebug() << "Recorder::startRecording(): Recording Context"; + qDebug() << "Global timestamp:" << context.globalTimestamp; qDebug() << "Domain:" << context.domain; qDebug() << "Position:" << context.position; qDebug() << "Orientation:" << context.orientation; @@ -240,18 +241,7 @@ qint64 Player::elapsed() const { void Player::startPlaying() { if (_recording && _recording->getFrameNumber() > 0) { - // Setup audio thread - _audioThread = new QThread(); - _options.setPosition(_avatar->getPosition()); - _options.setOrientation(_avatar->getOrientation()); - _injector.reset(new AudioInjector(_recording->getAudio(), _options), &QObject::deleteLater); - _injector->moveToThread(_audioThread); - _audioThread->start(); - QMetaObject::invokeMethod(_injector.data(), "injectAudio", Qt::QueuedConnection); - - // Fake faceshift connection - _avatar->setForceFaceshiftConnected(true); - + _currentContext.globalTimestamp = usecTimestampNow(); _currentContext.domain = NodeList::getInstance()->getDomainHandler().getHostname(); _currentContext.position = _avatar->getPosition(); _currentContext.orientation = _avatar->getOrientation(); @@ -284,8 +274,12 @@ void Player::startPlaying() { } } + // Fake faceshift connection + _avatar->setForceFaceshiftConnected(true); + qDebug() << "Recorder::startPlaying()"; _currentFrame = 0; + setupAudioThread(); _timer.start(); } } @@ -294,12 +288,27 @@ void Player::stopPlaying() { if (!isPlaying()) { return; } - _timer.invalidate(); - + cleanupAudioThread(); _avatar->clearJointsData(); - // Cleanup audio thread + // Turn off fake faceshift connection + _avatar->setForceFaceshiftConnected(false); + + qDebug() << "Recorder::stopPlaying()"; +} + +void Player::setupAudioThread() { + _audioThread = new QThread(); + _options.setPosition(_avatar->getPosition()); + _options.setOrientation(_avatar->getOrientation()); + _injector.reset(new AudioInjector(_recording->getAudio(), _options), &QObject::deleteLater); + _injector->moveToThread(_audioThread); + _audioThread->start(); + QMetaObject::invokeMethod(_injector.data(), "injectAudio", Qt::QueuedConnection); +} + +void Player::cleanupAudioThread() { _injector->stop(); QObject::connect(_injector.data(), &AudioInjector::finished, _injector.data(), &AudioInjector::deleteLater); @@ -309,11 +318,14 @@ void Player::stopPlaying() { _audioThread, &QThread::deleteLater); _injector.clear(); _audioThread = NULL; +} + +void Player::loopRecording() { + cleanupAudioThread(); + setupAudioThread(); + _currentFrame = 0; + _timer.restart(); - // Turn off fake faceshift connection - _avatar->setForceFaceshiftConnected(false); - - qDebug() << "Recorder::stopPlaying()"; } void Player::loadFromFile(QString file) { @@ -332,24 +344,23 @@ void Player::loadRecording(RecordingPointer recording) { void Player::play() { computeCurrentFrame(); if (_currentFrame < 0 || (_currentFrame >= _recording->getFrameNumber() - 1)) { - // If it's the end of the recording, stop playing - stopPlaying(); - if (_loop) { - startPlaying(); + loopRecording(); + } else { + stopPlaying(); } return; } - RecordingContext& context = _recording->getContext(); + const RecordingContext* context = &_recording->getContext(); if (_playFromCurrentPosition) { - context = _currentContext; + context = &_currentContext; } const RecordingFrame& currentFrame = _recording->getFrame(_currentFrame); - _avatar->setPosition(context.position + context.orientation * currentFrame.getTranslation()); - _avatar->setOrientation(context.orientation * currentFrame.getRotation()); - _avatar->setTargetScale(context.scale * currentFrame.getScale()); + _avatar->setPosition(context->position + context->orientation * currentFrame.getTranslation()); + _avatar->setOrientation(context->orientation * currentFrame.getRotation()); + _avatar->setTargetScale(context->scale * currentFrame.getScale()); _avatar->setJointRotations(currentFrame.getJointRotations()); HeadData* head = const_cast(_avatar->getHeadData()); diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index f3125bff05..65488e0696 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -172,6 +172,9 @@ public slots: void setLoop(bool loop); private: + void setupAudioThread(); + void cleanupAudioThread(); + void loopRecording(); bool computeCurrentFrame(); QElapsedTimer _timer; From 81fc0d3696ca49630d312000be591862dcbe0ea9 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 14:50:20 -0700 Subject: [PATCH 14/21] Added options to use recording context --- libraries/avatars/src/Recorder.cpp | 33 +++++++++++++++++++++++++++++- libraries/avatars/src/Recorder.h | 5 +++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index a453c045d0..6b6d9dc64e 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -220,7 +220,11 @@ Player::Player(AvatarData* avatar) : _avatar(avatar), _audioThread(NULL), _playFromCurrentPosition(true), - _loop(false) + _loop(false), + _useAttachments(true), + _useDisplayName(true), + _useHeadURL(true), + _useSkeletonURL(true) { _timer.invalidate(); _options.setLoop(false); @@ -253,6 +257,20 @@ void Player::startPlaying() { _currentContext.orientationInv = glm::inverse(_currentContext.orientation); + RecordingContext& context = _recording->getContext(); + if (_useAttachments) { + _avatar->setAttachmentData(context.attachments); + } + if (_useDisplayName) { + _avatar->setDisplayName(context.displayName); + } + if (_useHeadURL) { + _avatar->setFaceModelURL(context.headModel); + } + if (_useSkeletonURL) { + _avatar->setSkeletonModelURL(context.skeletonModel); + } + bool wantDebug = false; if (wantDebug) { qDebug() << "Player::startPlaying(): Recording Context"; @@ -295,6 +313,19 @@ void Player::stopPlaying() { // Turn off fake faceshift connection _avatar->setForceFaceshiftConnected(false); + if (_useAttachments) { + _avatar->setAttachmentData(_currentContext.attachments); + } + if (_useDisplayName) { + _avatar->setDisplayName(_currentContext.displayName); + } + if (_useHeadURL) { + _avatar->setFaceModelURL(_currentContext.headModel); + } + if (_useSkeletonURL) { + _avatar->setSkeletonModelURL(_currentContext.skeletonModel); + } + qDebug() << "Recorder::stopPlaying()"; } diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index 65488e0696..f40904cd9b 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -191,6 +191,11 @@ private: RecordingContext _currentContext; bool _playFromCurrentPosition; bool _loop; + bool _useAttachments; + bool _useDisplayName; + bool _useHeadURL; + bool _useSkeletonURL; + }; void writeRecordingToFile(RecordingPointer recording, QString file); From c699726ae8e6edea072393b0fdc1775bcd495c1a Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 16 Sep 2014 16:32:14 -0700 Subject: [PATCH 15/21] Fixed URLs not read properly --- libraries/avatars/src/Recorder.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 6b6d9dc64e..77cc23d806 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -694,6 +694,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { bool wantDebug = true; if (wantDebug) { + qDebug() << "[DEBUG] WRITE recording"; qDebug() << "Header:"; qDebug() << "File Format version:" << VERSION; qDebug() << "Data length:" << dataLength; @@ -846,7 +847,9 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen for (int i = 0; i < numAttachments; ++i) { AttachmentData data; // Model - fileStream >> data.modelURL; + QString modelURL; + fileStream >> modelURL; + data.modelURL = modelURL; // Joint name fileStream >> data.jointName; // Translation @@ -939,6 +942,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen bool wantDebug = true; if (wantDebug) { + qDebug() << "[DEBUG] READ recording"; qDebug() << "Header:"; qDebug() << "File Format version:" << VERSION; qDebug() << "Data length:" << dataLength; From 6e55c66d09572461584603c98193ec171cab0a3b Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 17 Sep 2014 16:24:54 -0700 Subject: [PATCH 16/21] Fix radix bug --- libraries/avatars/src/Recorder.cpp | 79 ++++++++++++++++-------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 77cc23d806..76343ff156 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -33,6 +33,10 @@ static const char MAGIC_NUMBER[MAGIC_NUMBER_SIZE] = {17, 72, 70, 82, 13, 10, 26, // Version (Major, Minor) static const QPair VERSION(0, 1); +int SCALE_RADIX = 10; +int BLENDSHAPE_RADIX = 15; +int LEAN_RADIX = 7; + void operator<<(QDebug& stream, glm::vec3 vector) { stream << vector.x << vector.y << vector.z; } @@ -439,19 +443,15 @@ bool Player::computeCurrentFrame() { } void writeVec3(QDataStream& stream, glm::vec3 value) { - unsigned char buffer[256]; - int writtenToBuffer = packFloatVec3ToSignedTwoByteFixed(buffer, value, 0); - stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); + unsigned char buffer[sizeof(value)]; + memcpy(buffer, &value, sizeof(value)); + stream.writeRawData(reinterpret_cast(buffer), sizeof(value)); } bool readVec3(QDataStream& stream, glm::vec3& value) { - int vec3ByteSize = 3 * 2; // 3 floats * 2 bytes - unsigned char buffer[256]; - stream.readRawData(reinterpret_cast(buffer), vec3ByteSize); - int readFromBuffer = unpackFloatVec3FromSignedTwoByteFixed(buffer, value, 0); - if (readFromBuffer != vec3ByteSize) { - return false; - } + unsigned char buffer[sizeof(value)]; + stream.readRawData(reinterpret_cast(buffer), sizeof(value)); + memcpy(&value, buffer, sizeof(value)); return true; } @@ -472,17 +472,17 @@ bool readQuat(QDataStream& stream, glm::quat& value) { return true; } -void writeFloat(QDataStream& stream, float value) { +void writeFloat(QDataStream& stream, float value, int radix) { unsigned char buffer[256]; - int writtenToBuffer = packFloatScalarToSignedTwoByteFixed(buffer, value, 0); + int writtenToBuffer = packFloatScalarToSignedTwoByteFixed(buffer, value, radix); stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); } -bool readFloat(QDataStream& stream, float& value) { +bool readFloat(QDataStream& stream, float& value, int radix) { int floatByteSize = 2; // 1 floats * 2 bytes int16_t buffer[256]; stream.readRawData(reinterpret_cast(buffer), floatByteSize); - int readFromBuffer = unpackFloatScalarFromSignedTwoByteFixed(buffer, &value, 0); + int readFromBuffer = unpackFloatScalarFromSignedTwoByteFixed(buffer, &value, radix); if (readFromBuffer != floatByteSize) { return false; } @@ -539,7 +539,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { // Orientation writeQuat(fileStream, context.orientation); // Scale - writeFloat(fileStream, context.scale); + writeFloat(fileStream, context.scale, SCALE_RADIX); // Head model fileStream << context.headModel; // Skeleton model @@ -558,13 +558,15 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { // Orientation writeQuat(fileStream, data.rotation); // Scale - writeFloat(fileStream, data.scale); + writeFloat(fileStream, data.scale, SCALE_RADIX); } // RECORDING fileStream << recording->_timestamps; QBitArray mask; + quint32 numBlendshapes = 0; + quint32 numJoints = 0; for (int i = 0; i < recording->_timestamps.size(); ++i) { mask.fill(false); @@ -575,24 +577,24 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { RecordingFrame& frame = recording->_frames[i]; // Blendshape Coefficients - quint32 numBlenshapes = frame.getBlendshapeCoefficients().size(); - stream << numBlenshapes; if (i == 0) { - mask.resize(mask.size() + numBlenshapes); + numBlendshapes = frame.getBlendshapeCoefficients().size(); + stream << numBlendshapes; + mask.resize(mask.size() + numBlendshapes); } - for (int j = 0; j < numBlenshapes; ++j) { + for (int j = 0; j < numBlendshapes; ++j) { if (i == 0 || frame._blendshapeCoefficients[j] != previousFrame._blendshapeCoefficients[j]) { - writeFloat(stream, frame.getBlendshapeCoefficients()[j]); + writeFloat(stream, frame.getBlendshapeCoefficients()[j], BLENDSHAPE_RADIX); mask.setBit(maskIndex); } ++maskIndex; } // Joint Rotations - quint32 numJoints = frame.getJointRotations().size(); - stream << numJoints; if (i == 0) { + numJoints = frame.getJointRotations().size(); + stream << numJoints; mask.resize(mask.size() + numJoints); } for (int j = 0; j < numJoints; ++j) { @@ -629,7 +631,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { mask.resize(mask.size() + 1); } if (i == 0 || frame._scale != previousFrame._scale) { - writeFloat(stream, frame._scale); + writeFloat(stream, frame._scale, SCALE_RADIX); mask.setBit(maskIndex); } maskIndex++; @@ -649,7 +651,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { mask.resize(mask.size() + 1); } if (i == 0 || frame._leanSideways != previousFrame._leanSideways) { - writeFloat(stream, frame._leanSideways); + writeFloat(stream, frame._leanSideways, LEAN_RADIX); mask.setBit(maskIndex); } maskIndex++; @@ -659,7 +661,7 @@ void writeRecordingToFile(RecordingPointer recording, QString filename) { mask.resize(mask.size() + 1); } if (i == 0 || frame._leanForward != previousFrame._leanForward) { - writeFloat(stream, frame._leanForward); + writeFloat(stream, frame._leanForward, LEAN_RADIX); mask.setBit(maskIndex); } maskIndex++; @@ -829,7 +831,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen } // Scale - if (!readFloat(fileStream, context.scale)) { + if (!readFloat(fileStream, context.scale, SCALE_RADIX)) { qDebug() << "Couldn't read file correctly. (Invalid float)"; recording.clear(); return recording; @@ -864,13 +866,15 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen } // Scale - if (!readFloat(fileStream, data.scale)) { + if (!readFloat(fileStream, data.scale, SCALE_RADIX)) { qDebug() << "Couldn't read attachment correctly. (Invalid float)"; continue; } context.attachments << data; } + quint32 numBlendshapes = 0; + quint32 numJoints = 0; // RECORDING fileStream >> recording->_timestamps; @@ -886,18 +890,19 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen int maskIndex = 0; // Blendshape Coefficients - quint32 numBlendshapes = 0; - stream >> numBlendshapes; + if (i == 0) { + stream >> numBlendshapes; + } frame._blendshapeCoefficients.resize(numBlendshapes); for (int j = 0; j < numBlendshapes; ++j) { - if (!mask[maskIndex++] || !readFloat(stream, frame._blendshapeCoefficients[j])) { + if (!mask[maskIndex++] || !readFloat(stream, frame._blendshapeCoefficients[j], BLENDSHAPE_RADIX)) { frame._blendshapeCoefficients[j] = previousFrame._blendshapeCoefficients[j]; } } - // Joint Rotations - quint32 numJoints = 0; - stream >> numJoints; + if (i == 0) { + stream >> numJoints; + } frame._jointRotations.resize(numJoints); for (int j = 0; j < numJoints; ++j) { if (!mask[maskIndex++] || !readQuat(stream, frame._jointRotations[j])) { @@ -913,7 +918,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen frame._rotation = previousFrame._rotation; } - if (!mask[maskIndex++] || !readFloat(stream, frame._scale)) { + if (!mask[maskIndex++] || !readFloat(stream, frame._scale, SCALE_RADIX)) { frame._scale = previousFrame._scale; } @@ -921,11 +926,11 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen frame._headRotation = previousFrame._headRotation; } - if (!mask[maskIndex++] || !readFloat(stream, frame._leanSideways)) { + if (!mask[maskIndex++] || !readFloat(stream, frame._leanSideways, LEAN_RADIX)) { frame._leanSideways = previousFrame._leanSideways; } - if (!mask[maskIndex++] || !readFloat(stream, frame._leanForward)) { + if (!mask[maskIndex++] || !readFloat(stream, frame._leanForward, LEAN_RADIX)) { frame._leanForward = previousFrame._leanForward; } From 34c16b8a74c1afb17fb4e8c9d668af68638b8f8e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 17 Sep 2014 16:25:48 -0700 Subject: [PATCH 17/21] Convert .rec to new format if necessary --- libraries/avatars/src/Recorder.cpp | 178 +++++++++++++++++++++++++++++ libraries/avatars/src/Recorder.h | 7 +- 2 files changed, 183 insertions(+), 2 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 76343ff156..b67a21ebc0 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -14,6 +14,8 @@ #include #include +#include +#include #include #include @@ -767,6 +769,18 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen file.close(); } + if (filename.endsWith(".rec") || filename.endsWith(".REC")) { + qDebug() << "Old .rec format"; + QMessageBox::warning(NULL, + QString("Old recording format"), + QString("Converting your file to the new format."), + QMessageBox::Ok); + readRecordingFromRecFile(recording, filename, byteArray); + return recording; + } else if (!filename.endsWith(".hfr") && !filename.endsWith(".HFR")) { + qDebug() << "File extension not recognized"; + } + // Reset the recording passed in the arguments if (!recording) { recording.reset(new Recording()); @@ -983,3 +997,167 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen } +RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray) { + QElapsedTimer timer; + timer.start(); + + if (!recording) { + recording.reset(new Recording()); + } + + QDataStream fileStream(byteArray); + + fileStream >> recording->_timestamps; + RecordingFrame baseFrame; + + // Blendshape coefficients + fileStream >> baseFrame._blendshapeCoefficients; + + // Joint Rotations + int jointRotationSize; + fileStream >> jointRotationSize; + baseFrame._jointRotations.resize(jointRotationSize); + for (int i = 0; i < jointRotationSize; ++i) { + fileStream >> baseFrame._jointRotations[i].x >> baseFrame._jointRotations[i].y >> baseFrame._jointRotations[i].z >> baseFrame._jointRotations[i].w; + } + + fileStream >> baseFrame._translation.x >> baseFrame._translation.y >> baseFrame._translation.z; + fileStream >> baseFrame._rotation.x >> baseFrame._rotation.y >> baseFrame._rotation.z >> baseFrame._rotation.w; + fileStream >> baseFrame._scale; + fileStream >> baseFrame._headRotation.x >> baseFrame._headRotation.y >> baseFrame._headRotation.z >> baseFrame._headRotation.w; + fileStream >> baseFrame._leanSideways; + fileStream >> baseFrame._leanForward; + + + // Fake context + RecordingContext& context = recording->getContext(); + context.globalTimestamp = usecTimestampNow(); + context.domain = NodeList::getInstance()->getDomainHandler().getHostname(); + context.position = glm::vec3(144.5f, 3.3f, 181.3f); + context.orientation = glm::angleAxis(glm::radians(-92.5f), glm::vec3(0, 1, 0));; + context.scale = baseFrame._scale; + context.headModel = "http://public.highfidelity.io/models/heads/Emily_v4.fst"; + context.skeletonModel = "http://public.highfidelity.io/models/skeletons/EmilyCutMesh_A.fst"; + context.displayName = "Leslie"; + context.attachments.clear(); + AttachmentData data; + data.modelURL = "http://public.highfidelity.io/models/attachments/fbx.fst"; + data.jointName = "RightHand" ; + data.translation = glm::vec3(0.04f, 0.07f, 0.0f); + data.rotation = glm::angleAxis(glm::radians(102.0f), glm::vec3(0, 1, 0)); + data.scale = 0.20f; + context.attachments << data; + + context.orientationInv = glm::inverse(context.orientation); + + baseFrame._translation = glm::vec3(); + baseFrame._rotation = glm::quat(); + baseFrame._scale = 1.0f; + + recording->_frames << baseFrame; + + for (int i = 1; i < recording->_timestamps.size(); ++i) { + QBitArray mask; + QByteArray buffer; + QDataStream stream(&buffer, QIODevice::ReadOnly); + RecordingFrame frame; + RecordingFrame& previousFrame = recording->_frames.last(); + + fileStream >> mask; + fileStream >> buffer; + int maskIndex = 0; + + // Blendshape Coefficients + frame._blendshapeCoefficients.resize(baseFrame._blendshapeCoefficients.size()); + for (int i = 0; i < baseFrame._blendshapeCoefficients.size(); ++i) { + if (mask[maskIndex++]) { + stream >> frame._blendshapeCoefficients[i]; + } else { + frame._blendshapeCoefficients[i] = previousFrame._blendshapeCoefficients[i]; + } + } + + // Joint Rotations + frame._jointRotations.resize(baseFrame._jointRotations.size()); + for (int i = 0; i < baseFrame._jointRotations.size(); ++i) { + if (mask[maskIndex++]) { + stream >> frame._jointRotations[i].x >> frame._jointRotations[i].y >> frame._jointRotations[i].z >> frame._jointRotations[i].w; + } else { + frame._jointRotations[i] = previousFrame._jointRotations[i]; + } + } + + if (mask[maskIndex++]) { + stream >> frame._translation.x >> frame._translation.y >> frame._translation.z; + frame._translation = context.orientationInv * frame._translation; + } else { + frame._translation = previousFrame._translation; + } + + if (mask[maskIndex++]) { + stream >> frame._rotation.x >> frame._rotation.y >> frame._rotation.z >> frame._rotation.w; + } else { + frame._rotation = previousFrame._rotation; + } + + if (mask[maskIndex++]) { + stream >> frame._scale; + } else { + frame._scale = previousFrame._scale; + } + + if (mask[maskIndex++]) { + stream >> frame._headRotation.x >> frame._headRotation.y >> frame._headRotation.z >> frame._headRotation.w; + } else { + frame._headRotation = previousFrame._headRotation; + } + + if (mask[maskIndex++]) { + stream >> frame._leanSideways; + } else { + frame._leanSideways = previousFrame._leanSideways; + } + + if (mask[maskIndex++]) { + stream >> frame._leanForward; + } else { + frame._leanForward = previousFrame._leanForward; + } + + recording->_frames << frame; + } + + QByteArray audioArray; + fileStream >> audioArray; + + // Cut down audio if necessary + int SAMPLE_RATE = 48000; // 48 kHz + int SAMPLE_SIZE = 2; // 16 bits + int MSEC_PER_SEC = 1000; + int audioLength = recording->getLength() * SAMPLE_SIZE * (SAMPLE_RATE / MSEC_PER_SEC); + audioArray.chop(audioArray.size() - audioLength); + + recording->addAudioPacket(audioArray); + + qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; + + // Set new filename + if (filename.startsWith("http") || filename.startsWith("https") || filename.startsWith("ftp")) { + filename = QUrl(filename).fileName(); + } + if (filename.endsWith(".rec") || filename.endsWith(".REC")) { + filename.chop(qstrlen(".rec")); + } + filename.append(".hfr"); + filename = QFileInfo(filename).absoluteFilePath(); + + // Set recording to new format + writeRecordingToFile(recording, filename); + QMessageBox::warning(NULL, + QString("New recording location"), + QString("The new recording was saved at:\n" + filename), + QMessageBox::Ok); + qDebug() << "Recording has been successfully converted at" << filename; + return recording; +} + diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index f40904cd9b..f1783da436 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -76,6 +76,7 @@ private: friend class Recorder; friend void writeRecordingToFile(RecordingPointer recording, QString file); friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); }; class RecordingContext { @@ -125,6 +126,7 @@ private: friend class Player; friend void writeRecordingToFile(RecordingPointer recording, QString file); friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); }; /// Records a recording @@ -198,7 +200,8 @@ private: }; -void writeRecordingToFile(RecordingPointer recording, QString file); -RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); +void writeRecordingToFile(RecordingPointer recording, QString filename); +RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename); +RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); #endif // hifi_Recorder_h \ No newline at end of file From d27134a7ac39150476b4dd6f475a525fe43495eb Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 17 Sep 2014 17:06:47 -0700 Subject: [PATCH 18/21] Added getters for the different Player options --- libraries/avatars/src/Recorder.cpp | 6 +----- libraries/avatars/src/Recorder.h | 8 ++++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index b67a21ebc0..953da2feca 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -409,7 +409,7 @@ void Player::play() { head->setFinalPitch(eulers.x); head->setFinalYaw(eulers.y); head->setFinalRoll(eulers.z); - head->setLookAtPosition(currentFrame.getLookAtPosition()); + //head->setLookAtPosition(currentFrame.getLookAtPosition()); } else { qDebug() << "WARNING: Player couldn't find head data."; } @@ -423,10 +423,6 @@ void Player::setPlayFromCurrentLocation(bool playFromCurrentLocation) { _playFromCurrentPosition = playFromCurrentLocation; } -void Player::setLoop(bool loop) { - _loop = loop; -} - bool Player::computeCurrentFrame() { if (!isPlaying()) { _currentFrame = -1; diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index f1783da436..961beca0a9 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -170,8 +170,12 @@ public slots: void loadRecording(RecordingPointer recording); void play(); - void setPlayFromCurrentLocation(bool playFromCurrentLocation); - void setLoop(bool loop); + void setPlayFromCurrentLocation(bool playFromCurrentPosition); + void setLoop(bool loop) { _loop = loop; } + void useAttachements(bool useAttachments) { _useAttachments = useAttachments; } + void useDisplayName(bool useDisplayName) { _useDisplayName = useDisplayName; } + void useHeadModel(bool useHeadURL) { _useHeadURL = useHeadURL; } + void useSkeletonModel(bool useSkeletonURL) { _useSkeletonURL = useSkeletonURL; } private: void setupAudioThread(); From 06dd9d037695a640206f0a7c77061e3cdd0d4500 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 17 Sep 2014 23:52:21 -0700 Subject: [PATCH 19/21] Moved Player and Recording to their own files --- libraries/avatars/src/AvatarData.h | 5 +- libraries/avatars/src/Player.cpp | 236 ++++++ libraries/avatars/src/Player.h | 78 ++ libraries/avatars/src/Recorder.cpp | 1023 +-------------------------- libraries/avatars/src/Recorder.h | 164 +---- libraries/avatars/src/Recording.cpp | 807 +++++++++++++++++++++ libraries/avatars/src/Recording.h | 127 ++++ 7 files changed, 1257 insertions(+), 1183 deletions(-) create mode 100644 libraries/avatars/src/Player.cpp create mode 100644 libraries/avatars/src/Player.h create mode 100644 libraries/avatars/src/Recording.cpp create mode 100644 libraries/avatars/src/Recording.h diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 5b0c6b97dd..65090b093f 100755 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -49,10 +49,11 @@ typedef unsigned long long quint64; #include +#include "HandData.h" +#include "HeadData.h" +#include "Player.h" #include "Recorder.h" #include "Referential.h" -#include "HeadData.h" -#include "HandData.h" // avatar motion behaviors const quint32 AVATAR_MOTION_MOTOR_KEYBOARD_ENABLED = 1U << 0; diff --git a/libraries/avatars/src/Player.cpp b/libraries/avatars/src/Player.cpp new file mode 100644 index 0000000000..554df1a758 --- /dev/null +++ b/libraries/avatars/src/Player.cpp @@ -0,0 +1,236 @@ +// +// Player.cpp +// +// +// Created by Clement on 9/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include + +#include "AvatarData.h" +#include "Player.h" + +Player::Player(AvatarData* avatar) : +_recording(new Recording()), +_avatar(avatar), +_audioThread(NULL), +_playFromCurrentPosition(true), +_loop(false), +_useAttachments(true), +_useDisplayName(true), +_useHeadURL(true), +_useSkeletonURL(true) +{ + _timer.invalidate(); + _options.setLoop(false); + _options.setVolume(1.0f); +} + +bool Player::isPlaying() const { + return _timer.isValid(); +} + +qint64 Player::elapsed() const { + if (isPlaying()) { + return _timer.elapsed(); + } else { + return 0; + } +} + +void Player::startPlaying() { + if (_recording && _recording->getFrameNumber() > 0) { + _currentContext.globalTimestamp = usecTimestampNow(); + _currentContext.domain = NodeList::getInstance()->getDomainHandler().getHostname(); + _currentContext.position = _avatar->getPosition(); + _currentContext.orientation = _avatar->getOrientation(); + _currentContext.scale = _avatar->getTargetScale(); + _currentContext.headModel = _avatar->getFaceModelURL().toString(); + _currentContext.skeletonModel = _avatar->getSkeletonModelURL().toString(); + _currentContext.displayName = _avatar->getDisplayName(); + _currentContext.attachments = _avatar->getAttachmentData(); + + _currentContext.orientationInv = glm::inverse(_currentContext.orientation); + + RecordingContext& context = _recording->getContext(); + if (_useAttachments) { + _avatar->setAttachmentData(context.attachments); + } + if (_useDisplayName) { + _avatar->setDisplayName(context.displayName); + } + if (_useHeadURL) { + _avatar->setFaceModelURL(context.headModel); + } + if (_useSkeletonURL) { + _avatar->setSkeletonModelURL(context.skeletonModel); + } + + bool wantDebug = false; + if (wantDebug) { + qDebug() << "Player::startPlaying(): Recording Context"; + qDebug() << "Domain:" << _currentContext.domain; + qDebug() << "Position:" << _currentContext.position; + qDebug() << "Orientation:" << _currentContext.orientation; + qDebug() << "Scale:" << _currentContext.scale; + qDebug() << "Head URL:" << _currentContext.headModel; + qDebug() << "Skeleton URL:" << _currentContext.skeletonModel; + qDebug() << "Display Name:" << _currentContext.displayName; + qDebug() << "Num Attachments:" << _currentContext.attachments.size(); + + for (int i = 0; i < _currentContext.attachments.size(); ++i) { + qDebug() << "Model URL:" << _currentContext.attachments[i].modelURL; + qDebug() << "Joint Name:" << _currentContext.attachments[i].jointName; + qDebug() << "Translation:" << _currentContext.attachments[i].translation; + qDebug() << "Rotation:" << _currentContext.attachments[i].rotation; + qDebug() << "Scale:" << _currentContext.attachments[i].scale; + } + } + + // Fake faceshift connection + _avatar->setForceFaceshiftConnected(true); + + qDebug() << "Recorder::startPlaying()"; + _currentFrame = 0; + setupAudioThread(); + _timer.start(); + } +} + +void Player::stopPlaying() { + if (!isPlaying()) { + return; + } + _timer.invalidate(); + cleanupAudioThread(); + _avatar->clearJointsData(); + + // Turn off fake faceshift connection + _avatar->setForceFaceshiftConnected(false); + + if (_useAttachments) { + _avatar->setAttachmentData(_currentContext.attachments); + } + if (_useDisplayName) { + _avatar->setDisplayName(_currentContext.displayName); + } + if (_useHeadURL) { + _avatar->setFaceModelURL(_currentContext.headModel); + } + if (_useSkeletonURL) { + _avatar->setSkeletonModelURL(_currentContext.skeletonModel); + } + + qDebug() << "Recorder::stopPlaying()"; +} + +void Player::setupAudioThread() { + _audioThread = new QThread(); + _options.setPosition(_avatar->getPosition()); + _options.setOrientation(_avatar->getOrientation()); + _injector.reset(new AudioInjector(_recording->getAudio(), _options), &QObject::deleteLater); + _injector->moveToThread(_audioThread); + _audioThread->start(); + QMetaObject::invokeMethod(_injector.data(), "injectAudio", Qt::QueuedConnection); +} + +void Player::cleanupAudioThread() { + _injector->stop(); + QObject::connect(_injector.data(), &AudioInjector::finished, + _injector.data(), &AudioInjector::deleteLater); + QObject::connect(_injector.data(), &AudioInjector::destroyed, + _audioThread, &QThread::quit); + QObject::connect(_audioThread, &QThread::finished, + _audioThread, &QThread::deleteLater); + _injector.clear(); + _audioThread = NULL; +} + +void Player::loopRecording() { + cleanupAudioThread(); + setupAudioThread(); + _currentFrame = 0; + _timer.restart(); + +} + +void Player::loadFromFile(QString file) { + if (_recording) { + _recording->clear(); + } else { + _recording = RecordingPointer(new Recording()); + } + readRecordingFromFile(_recording, file); +} + +void Player::loadRecording(RecordingPointer recording) { + _recording = recording; +} + +void Player::play() { + computeCurrentFrame(); + if (_currentFrame < 0 || (_currentFrame >= _recording->getFrameNumber() - 1)) { + if (_loop) { + loopRecording(); + } else { + stopPlaying(); + } + return; + } + + const RecordingContext* context = &_recording->getContext(); + if (_playFromCurrentPosition) { + context = &_currentContext; + } + const RecordingFrame& currentFrame = _recording->getFrame(_currentFrame); + + _avatar->setPosition(context->position + context->orientation * currentFrame.getTranslation()); + _avatar->setOrientation(context->orientation * currentFrame.getRotation()); + _avatar->setTargetScale(context->scale * currentFrame.getScale()); + _avatar->setJointRotations(currentFrame.getJointRotations()); + + HeadData* head = const_cast(_avatar->getHeadData()); + if (head) { + head->setBlendshapeCoefficients(currentFrame.getBlendshapeCoefficients()); + head->setLeanSideways(currentFrame.getLeanSideways()); + head->setLeanForward(currentFrame.getLeanForward()); + glm::vec3 eulers = glm::degrees(safeEulerAngles(currentFrame.getHeadRotation())); + head->setFinalPitch(eulers.x); + head->setFinalYaw(eulers.y); + head->setFinalRoll(eulers.z); + //head->setLookAtPosition(currentFrame.getLookAtPosition()); + } else { + qDebug() << "WARNING: Player couldn't find head data."; + } + + _options.setPosition(_avatar->getPosition()); + _options.setOrientation(_avatar->getOrientation()); + _injector->setOptions(_options); +} + +void Player::setPlayFromCurrentLocation(bool playFromCurrentLocation) { + _playFromCurrentPosition = playFromCurrentLocation; +} + +bool Player::computeCurrentFrame() { + if (!isPlaying()) { + _currentFrame = -1; + return false; + } + if (_currentFrame < 0) { + _currentFrame = 0; + } + + while (_currentFrame < _recording->getFrameNumber() - 1 && + _recording->getFrameTimestamp(_currentFrame) < _timer.elapsed()) { + ++_currentFrame; + } + + return true; +} diff --git a/libraries/avatars/src/Player.h b/libraries/avatars/src/Player.h new file mode 100644 index 0000000000..ba2ab9951a --- /dev/null +++ b/libraries/avatars/src/Player.h @@ -0,0 +1,78 @@ +// +// Player.h +// +// +// Created by Clement on 9/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_Player_h +#define hifi_Player_h + +#include + +#include + +#include "Recording.h" + +class AvatarData; +class Player; + +typedef QSharedPointer PlayerPointer; +typedef QWeakPointer WeakPlayerPointer; + +/// Plays back a recording +class Player { +public: + Player(AvatarData* avatar); + + bool isPlaying() const; + qint64 elapsed() const; + + RecordingPointer getRecording() const { return _recording; } + + public slots: + void startPlaying(); + void stopPlaying(); + void loadFromFile(QString file); + void loadRecording(RecordingPointer recording); + void play(); + + void setPlayFromCurrentLocation(bool playFromCurrentPosition); + void setLoop(bool loop) { _loop = loop; } + void useAttachements(bool useAttachments) { _useAttachments = useAttachments; } + void useDisplayName(bool useDisplayName) { _useDisplayName = useDisplayName; } + void useHeadModel(bool useHeadURL) { _useHeadURL = useHeadURL; } + void useSkeletonModel(bool useSkeletonURL) { _useSkeletonURL = useSkeletonURL; } + +private: + void setupAudioThread(); + void cleanupAudioThread(); + void loopRecording(); + bool computeCurrentFrame(); + + QElapsedTimer _timer; + RecordingPointer _recording; + int _currentFrame; + + QSharedPointer _injector; + AudioInjectorOptions _options; + + AvatarData* _avatar; + QThread* _audioThread; + + + RecordingContext _currentContext; + bool _playFromCurrentPosition; + bool _loop; + bool _useAttachments; + bool _useDisplayName; + bool _useHeadURL; + bool _useSkeletonURL; + +}; + +#endif // hifi_Player_h \ No newline at end of file diff --git a/libraries/avatars/src/Recorder.cpp b/libraries/avatars/src/Recorder.cpp index 953da2feca..4c71dc1a18 100644 --- a/libraries/avatars/src/Recorder.cpp +++ b/libraries/avatars/src/Recorder.cpp @@ -9,93 +9,14 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include "AvatarData.h" -#include #include "Recorder.h" - -#include - -// HFR file format magic number -// (decimal) 17 72 70 82 13 10 26 10 -// (hexadecimal) 11 48 46 52 0d 0a 1a 0a -// (ASCII C notation) \021 H F R \r \n \032 \n -static const int MAGIC_NUMBER_SIZE = 8; -static const char MAGIC_NUMBER[MAGIC_NUMBER_SIZE] = {17, 72, 70, 82, 13, 10, 26, 10}; -// Version (Major, Minor) -static const QPair VERSION(0, 1); - -int SCALE_RADIX = 10; -int BLENDSHAPE_RADIX = 15; -int LEAN_RADIX = 7; - -void operator<<(QDebug& stream, glm::vec3 vector) { - stream << vector.x << vector.y << vector.z; -} -void operator<<(QDebug& stream, glm::quat quat) { - stream << quat.x << quat.y << quat.z << quat.w; -} - -void RecordingFrame::setBlendshapeCoefficients(QVector blendshapeCoefficients) { - _blendshapeCoefficients = blendshapeCoefficients; -} - -Recording::Recording() : _audio(NULL) { -} - -Recording::~Recording() { - delete _audio; -} - -int Recording::getLength() const { - if (_timestamps.isEmpty()) { - return 0; - } - return _timestamps.last(); -} - -qint32 Recording::getFrameTimestamp(int i) const { - if (i >= _timestamps.size()) { - return getLength(); - } - return _timestamps[i]; -} - -const RecordingFrame& Recording::getFrame(int i) const { - assert(i < _timestamps.size()); - return _frames[i]; -} - -void Recording::addFrame(int timestamp, RecordingFrame &frame) { - _timestamps << timestamp; - _frames << frame; -} - -void Recording::addAudioPacket(QByteArray byteArray) { - if (!_audio) { - _audio = new Sound(byteArray); - return; - } - _audio->append(byteArray); -} - -void Recording::clear() { - _timestamps.clear(); - _frames.clear(); - delete _audio; - _audio = NULL; -} - Recorder::Recorder(AvatarData* avatar) : _recording(new Recording()), _avatar(avatar) @@ -219,941 +140,3 @@ void Recorder::record(char* samples, int size) { QByteArray byteArray(samples, size); _recording->addAudioPacket(byteArray); } - - -Player::Player(AvatarData* avatar) : - _recording(new Recording()), - _avatar(avatar), - _audioThread(NULL), - _playFromCurrentPosition(true), - _loop(false), - _useAttachments(true), - _useDisplayName(true), - _useHeadURL(true), - _useSkeletonURL(true) -{ - _timer.invalidate(); - _options.setLoop(false); - _options.setVolume(1.0f); -} - -bool Player::isPlaying() const { - return _timer.isValid(); -} - -qint64 Player::elapsed() const { - if (isPlaying()) { - return _timer.elapsed(); - } else { - return 0; - } -} - -void Player::startPlaying() { - if (_recording && _recording->getFrameNumber() > 0) { - _currentContext.globalTimestamp = usecTimestampNow(); - _currentContext.domain = NodeList::getInstance()->getDomainHandler().getHostname(); - _currentContext.position = _avatar->getPosition(); - _currentContext.orientation = _avatar->getOrientation(); - _currentContext.scale = _avatar->getTargetScale(); - _currentContext.headModel = _avatar->getFaceModelURL().toString(); - _currentContext.skeletonModel = _avatar->getSkeletonModelURL().toString(); - _currentContext.displayName = _avatar->getDisplayName(); - _currentContext.attachments = _avatar->getAttachmentData(); - - _currentContext.orientationInv = glm::inverse(_currentContext.orientation); - - RecordingContext& context = _recording->getContext(); - if (_useAttachments) { - _avatar->setAttachmentData(context.attachments); - } - if (_useDisplayName) { - _avatar->setDisplayName(context.displayName); - } - if (_useHeadURL) { - _avatar->setFaceModelURL(context.headModel); - } - if (_useSkeletonURL) { - _avatar->setSkeletonModelURL(context.skeletonModel); - } - - bool wantDebug = false; - if (wantDebug) { - qDebug() << "Player::startPlaying(): Recording Context"; - qDebug() << "Domain:" << _currentContext.domain; - qDebug() << "Position:" << _currentContext.position; - qDebug() << "Orientation:" << _currentContext.orientation; - qDebug() << "Scale:" << _currentContext.scale; - qDebug() << "Head URL:" << _currentContext.headModel; - qDebug() << "Skeleton URL:" << _currentContext.skeletonModel; - qDebug() << "Display Name:" << _currentContext.displayName; - qDebug() << "Num Attachments:" << _currentContext.attachments.size(); - - for (int i = 0; i < _currentContext.attachments.size(); ++i) { - qDebug() << "Model URL:" << _currentContext.attachments[i].modelURL; - qDebug() << "Joint Name:" << _currentContext.attachments[i].jointName; - qDebug() << "Translation:" << _currentContext.attachments[i].translation; - qDebug() << "Rotation:" << _currentContext.attachments[i].rotation; - qDebug() << "Scale:" << _currentContext.attachments[i].scale; - } - } - - // Fake faceshift connection - _avatar->setForceFaceshiftConnected(true); - - qDebug() << "Recorder::startPlaying()"; - _currentFrame = 0; - setupAudioThread(); - _timer.start(); - } -} - -void Player::stopPlaying() { - if (!isPlaying()) { - return; - } - _timer.invalidate(); - cleanupAudioThread(); - _avatar->clearJointsData(); - - // Turn off fake faceshift connection - _avatar->setForceFaceshiftConnected(false); - - if (_useAttachments) { - _avatar->setAttachmentData(_currentContext.attachments); - } - if (_useDisplayName) { - _avatar->setDisplayName(_currentContext.displayName); - } - if (_useHeadURL) { - _avatar->setFaceModelURL(_currentContext.headModel); - } - if (_useSkeletonURL) { - _avatar->setSkeletonModelURL(_currentContext.skeletonModel); - } - - qDebug() << "Recorder::stopPlaying()"; -} - -void Player::setupAudioThread() { - _audioThread = new QThread(); - _options.setPosition(_avatar->getPosition()); - _options.setOrientation(_avatar->getOrientation()); - _injector.reset(new AudioInjector(_recording->getAudio(), _options), &QObject::deleteLater); - _injector->moveToThread(_audioThread); - _audioThread->start(); - QMetaObject::invokeMethod(_injector.data(), "injectAudio", Qt::QueuedConnection); -} - -void Player::cleanupAudioThread() { - _injector->stop(); - QObject::connect(_injector.data(), &AudioInjector::finished, - _injector.data(), &AudioInjector::deleteLater); - QObject::connect(_injector.data(), &AudioInjector::destroyed, - _audioThread, &QThread::quit); - QObject::connect(_audioThread, &QThread::finished, - _audioThread, &QThread::deleteLater); - _injector.clear(); - _audioThread = NULL; -} - -void Player::loopRecording() { - cleanupAudioThread(); - setupAudioThread(); - _currentFrame = 0; - _timer.restart(); - -} - -void Player::loadFromFile(QString file) { - if (_recording) { - _recording->clear(); - } else { - _recording = RecordingPointer(new Recording()); - } - readRecordingFromFile(_recording, file); -} - -void Player::loadRecording(RecordingPointer recording) { - _recording = recording; -} - -void Player::play() { - computeCurrentFrame(); - if (_currentFrame < 0 || (_currentFrame >= _recording->getFrameNumber() - 1)) { - if (_loop) { - loopRecording(); - } else { - stopPlaying(); - } - return; - } - - const RecordingContext* context = &_recording->getContext(); - if (_playFromCurrentPosition) { - context = &_currentContext; - } - const RecordingFrame& currentFrame = _recording->getFrame(_currentFrame); - - _avatar->setPosition(context->position + context->orientation * currentFrame.getTranslation()); - _avatar->setOrientation(context->orientation * currentFrame.getRotation()); - _avatar->setTargetScale(context->scale * currentFrame.getScale()); - _avatar->setJointRotations(currentFrame.getJointRotations()); - - HeadData* head = const_cast(_avatar->getHeadData()); - if (head) { - head->setBlendshapeCoefficients(currentFrame.getBlendshapeCoefficients()); - head->setLeanSideways(currentFrame.getLeanSideways()); - head->setLeanForward(currentFrame.getLeanForward()); - glm::vec3 eulers = glm::degrees(safeEulerAngles(currentFrame.getHeadRotation())); - head->setFinalPitch(eulers.x); - head->setFinalYaw(eulers.y); - head->setFinalRoll(eulers.z); - //head->setLookAtPosition(currentFrame.getLookAtPosition()); - } else { - qDebug() << "WARNING: Player couldn't find head data."; - } - - _options.setPosition(_avatar->getPosition()); - _options.setOrientation(_avatar->getOrientation()); - _injector->setOptions(_options); -} - -void Player::setPlayFromCurrentLocation(bool playFromCurrentLocation) { - _playFromCurrentPosition = playFromCurrentLocation; -} - -bool Player::computeCurrentFrame() { - if (!isPlaying()) { - _currentFrame = -1; - return false; - } - if (_currentFrame < 0) { - _currentFrame = 0; - } - - while (_currentFrame < _recording->getFrameNumber() - 1 && - _recording->getFrameTimestamp(_currentFrame) < _timer.elapsed()) { - ++_currentFrame; - } - - return true; -} - -void writeVec3(QDataStream& stream, glm::vec3 value) { - unsigned char buffer[sizeof(value)]; - memcpy(buffer, &value, sizeof(value)); - stream.writeRawData(reinterpret_cast(buffer), sizeof(value)); -} - -bool readVec3(QDataStream& stream, glm::vec3& value) { - unsigned char buffer[sizeof(value)]; - stream.readRawData(reinterpret_cast(buffer), sizeof(value)); - memcpy(&value, buffer, sizeof(value)); - return true; -} - -void writeQuat(QDataStream& stream, glm::quat value) { - unsigned char buffer[256]; - int writtenToBuffer = packOrientationQuatToBytes(buffer, value); - stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); -} - -bool readQuat(QDataStream& stream, glm::quat& value) { - int quatByteSize = 4 * 2; // 4 floats * 2 bytes - unsigned char buffer[256]; - stream.readRawData(reinterpret_cast(buffer), quatByteSize); - int readFromBuffer = unpackOrientationQuatFromBytes(buffer, value); - if (readFromBuffer != quatByteSize) { - return false; - } - return true; -} - -void writeFloat(QDataStream& stream, float value, int radix) { - unsigned char buffer[256]; - int writtenToBuffer = packFloatScalarToSignedTwoByteFixed(buffer, value, radix); - stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); -} - -bool readFloat(QDataStream& stream, float& value, int radix) { - int floatByteSize = 2; // 1 floats * 2 bytes - int16_t buffer[256]; - stream.readRawData(reinterpret_cast(buffer), floatByteSize); - int readFromBuffer = unpackFloatScalarFromSignedTwoByteFixed(buffer, &value, radix); - if (readFromBuffer != floatByteSize) { - return false; - } - return true; -} - -void writeRecordingToFile(RecordingPointer recording, QString filename) { - if (!recording || recording->getFrameNumber() < 1) { - qDebug() << "Can't save empty recording"; - return; - } - - QElapsedTimer timer; - QFile file(filename); - if (!file.open(QIODevice::ReadWrite | QIODevice::Truncate)){ - qDebug() << "Couldn't open " << filename; - return; - } - timer.start(); - qDebug() << "Writing recording to " << filename << "."; - - QDataStream fileStream(&file); - - // HEADER - file.write(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); // Magic number - fileStream << VERSION; // File format version - const qint64 dataOffsetPos = file.pos(); - fileStream << (quint16)0; // Save two empty bytes for the data offset - const qint64 dataLengthPos = file.pos(); - fileStream << (quint32)0; // Save four empty bytes for the data offset - const quint64 crc16Pos = file.pos(); - fileStream << (quint16)0; // Save two empty bytes for the CRC-16 - - - // METADATA - // TODO - - - - // Write data offset - quint16 dataOffset = file.pos(); - file.seek(dataOffsetPos); - fileStream << dataOffset; - file.seek(dataOffset); - - // CONTEXT - RecordingContext& context = recording->getContext(); - // Global Timestamp - fileStream << context.globalTimestamp; - // Domain - fileStream << context.domain; - // Position - writeVec3(fileStream, context.position); - // Orientation - writeQuat(fileStream, context.orientation); - // Scale - writeFloat(fileStream, context.scale, SCALE_RADIX); - // Head model - fileStream << context.headModel; - // Skeleton model - fileStream << context.skeletonModel; - // Display name - fileStream << context.displayName; - // Attachements - fileStream << (quint8)context.attachments.size(); - foreach (AttachmentData data, context.attachments) { - // Model - fileStream << data.modelURL.toString(); - // Joint name - fileStream << data.jointName; - // Position - writeVec3(fileStream, data.translation); - // Orientation - writeQuat(fileStream, data.rotation); - // Scale - writeFloat(fileStream, data.scale, SCALE_RADIX); - } - - // RECORDING - fileStream << recording->_timestamps; - - QBitArray mask; - quint32 numBlendshapes = 0; - quint32 numJoints = 0; - - for (int i = 0; i < recording->_timestamps.size(); ++i) { - mask.fill(false); - int maskIndex = 0; - QByteArray buffer; - QDataStream stream(&buffer, QIODevice::WriteOnly); - RecordingFrame& previousFrame = recording->_frames[(i != 0) ? i - 1 : i]; - RecordingFrame& frame = recording->_frames[i]; - - // Blendshape Coefficients - if (i == 0) { - numBlendshapes = frame.getBlendshapeCoefficients().size(); - stream << numBlendshapes; - mask.resize(mask.size() + numBlendshapes); - } - for (int j = 0; j < numBlendshapes; ++j) { - if (i == 0 || - frame._blendshapeCoefficients[j] != previousFrame._blendshapeCoefficients[j]) { - writeFloat(stream, frame.getBlendshapeCoefficients()[j], BLENDSHAPE_RADIX); - mask.setBit(maskIndex); - } - ++maskIndex; - } - - // Joint Rotations - if (i == 0) { - numJoints = frame.getJointRotations().size(); - stream << numJoints; - mask.resize(mask.size() + numJoints); - } - for (int j = 0; j < numJoints; ++j) { - if (i == 0 || - frame._jointRotations[j] != previousFrame._jointRotations[j]) { - writeQuat(stream, frame._jointRotations[j]); - mask.setBit(maskIndex); - } - maskIndex++; - } - - // Translation - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._translation != previousFrame._translation) { - writeVec3(stream, frame._translation); - mask.setBit(maskIndex); - } - maskIndex++; - - // Rotation - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._rotation != previousFrame._rotation) { - writeQuat(stream, frame._rotation); - mask.setBit(maskIndex); - } - maskIndex++; - - // Scale - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._scale != previousFrame._scale) { - writeFloat(stream, frame._scale, SCALE_RADIX); - mask.setBit(maskIndex); - } - maskIndex++; - - // Head Rotation - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._headRotation != previousFrame._headRotation) { - writeQuat(stream, frame._headRotation); - mask.setBit(maskIndex); - } - maskIndex++; - - // Lean Sideways - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._leanSideways != previousFrame._leanSideways) { - writeFloat(stream, frame._leanSideways, LEAN_RADIX); - mask.setBit(maskIndex); - } - maskIndex++; - - // Lean Forward - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._leanForward != previousFrame._leanForward) { - writeFloat(stream, frame._leanForward, LEAN_RADIX); - mask.setBit(maskIndex); - } - maskIndex++; - - // LookAt Position - if (i == 0) { - mask.resize(mask.size() + 1); - } - if (i == 0 || frame._lookAtPosition != previousFrame._lookAtPosition) { - writeVec3(stream, frame._lookAtPosition); - mask.setBit(maskIndex); - } - maskIndex++; - - fileStream << mask; - fileStream << buffer; - } - - fileStream << recording->_audio->getByteArray(); - - qint64 writtingTime = timer.restart(); - // Write data length and CRC-16 - quint32 dataLength = file.pos() - dataOffset; - file.seek(dataOffset); // Go to beginning of data for checksum - quint16 crc16 = qChecksum(file.readAll().constData(), dataLength); - - file.seek(dataLengthPos); - fileStream << dataLength; - file.seek(crc16Pos); - fileStream << crc16; - file.seek(dataOffset + dataLength); - - bool wantDebug = true; - if (wantDebug) { - qDebug() << "[DEBUG] WRITE recording"; - qDebug() << "Header:"; - qDebug() << "File Format version:" << VERSION; - qDebug() << "Data length:" << dataLength; - qDebug() << "Data offset:" << dataOffset; - qDebug() << "CRC-16:" << crc16; - - qDebug() << "Context block:"; - qDebug() << "Global timestamp:" << context.globalTimestamp; - qDebug() << "Domain:" << context.domain; - qDebug() << "Position:" << context.position; - qDebug() << "Orientation:" << context.orientation; - qDebug() << "Scale:" << context.scale; - qDebug() << "Head Model:" << context.headModel; - qDebug() << "Skeleton Model:" << context.skeletonModel; - qDebug() << "Display Name:" << context.displayName; - qDebug() << "Num Attachments:" << context.attachments.size(); - for (int i = 0; i < context.attachments.size(); ++i) { - qDebug() << "Model URL:" << context.attachments[i].modelURL; - qDebug() << "Joint Name:" << context.attachments[i].jointName; - qDebug() << "Translation:" << context.attachments[i].translation; - qDebug() << "Rotation:" << context.attachments[i].rotation; - qDebug() << "Scale:" << context.attachments[i].scale; - } - - qDebug() << "Recording:"; - qDebug() << "Total frames:" << recording->getFrameNumber(); - qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); - } - - qint64 checksumTime = timer.elapsed(); - qDebug() << "Wrote" << file.size() << "bytes in" << writtingTime + checksumTime << "ms. (" << checksumTime << "ms for checksum)"; -} - -RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename) { - QByteArray byteArray; - QUrl url(filename); - QElapsedTimer timer; - timer.start(); // timer used for debug informations (download/parsing time) - - // Aquire the data and place it in byteArray - // Return if data unavailable - if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { - // Download file if necessary - qDebug() << "Downloading recording at" << url; - NetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkReply* reply = networkAccessManager.get(QNetworkRequest(url)); - QEventLoop loop; - QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); - loop.exec(); // wait for file - if (reply->error() != QNetworkReply::NoError) { - qDebug() << "Error while downloading recording: " << reply->error(); - reply->deleteLater(); - return recording; - } - byteArray = reply->readAll(); - reply->deleteLater(); - // print debug + restart timer - qDebug() << "Downloaded " << byteArray.size() << " bytes in " << timer.restart() << " ms."; - } else { - // If local file, just read it. - qDebug() << "Reading recording from " << filename << "."; - QFile file(filename); - if (!file.open(QIODevice::ReadOnly)){ - qDebug() << "Could not open local file: " << url; - return recording; - } - byteArray = file.readAll(); - file.close(); - } - - if (filename.endsWith(".rec") || filename.endsWith(".REC")) { - qDebug() << "Old .rec format"; - QMessageBox::warning(NULL, - QString("Old recording format"), - QString("Converting your file to the new format."), - QMessageBox::Ok); - readRecordingFromRecFile(recording, filename, byteArray); - return recording; - } else if (!filename.endsWith(".hfr") && !filename.endsWith(".HFR")) { - qDebug() << "File extension not recognized"; - } - - // Reset the recording passed in the arguments - if (!recording) { - recording.reset(new Recording()); - } - - QDataStream fileStream(byteArray); - - // HEADER - QByteArray magicNumber(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); - if (!byteArray.startsWith(magicNumber)) { - qDebug() << "ERROR: This is not a .HFR file. (Magic Number incorrect)"; - return recording; - } - fileStream.skipRawData(MAGIC_NUMBER_SIZE); - - QPair version; - fileStream >> version; // File format version - if (version != VERSION) { - qDebug() << "ERROR: This file format version is not supported."; - return recording; - } - - quint16 dataOffset = 0; - fileStream >> dataOffset; - quint32 dataLength = 0; - fileStream >> dataLength; - quint16 crc16 = 0; - fileStream >> crc16; - - - // Check checksum - - quint16 computedCRC16 = qChecksum(byteArray.constData() + dataOffset, dataLength); - if (computedCRC16 != crc16) { - qDebug() << "Checksum does not match. Bailling!"; - recording.clear(); - return recording; - } - - // METADATA - // TODO - - - - // CONTEXT - RecordingContext& context = recording->getContext(); - // Global Timestamp - fileStream >> context.globalTimestamp; - // Domain - fileStream >> context.domain; - // Position - if (!readVec3(fileStream, context.position)) { - qDebug() << "Couldn't read file correctly. (Invalid vec3)"; - recording.clear(); - return recording; - } - // Orientation - if (!readQuat(fileStream, context.orientation)) { - qDebug() << "Couldn't read file correctly. (Invalid quat)"; - recording.clear(); - return recording; - } - - // Scale - if (!readFloat(fileStream, context.scale, SCALE_RADIX)) { - qDebug() << "Couldn't read file correctly. (Invalid float)"; - recording.clear(); - return recording; - } - // Head model - fileStream >> context.headModel; - // Skeleton model - fileStream >> context.skeletonModel; - // Display Name - fileStream >> context.displayName; - - // Attachements - quint8 numAttachments = 0; - fileStream >> numAttachments; - for (int i = 0; i < numAttachments; ++i) { - AttachmentData data; - // Model - QString modelURL; - fileStream >> modelURL; - data.modelURL = modelURL; - // Joint name - fileStream >> data.jointName; - // Translation - if (!readVec3(fileStream, data.translation)) { - qDebug() << "Couldn't read attachment correctly. (Invalid vec3)"; - continue; - } - // Rotation - if (!readQuat(fileStream, data.rotation)) { - qDebug() << "Couldn't read attachment correctly. (Invalid quat)"; - continue; - } - - // Scale - if (!readFloat(fileStream, data.scale, SCALE_RADIX)) { - qDebug() << "Couldn't read attachment correctly. (Invalid float)"; - continue; - } - context.attachments << data; - } - - quint32 numBlendshapes = 0; - quint32 numJoints = 0; - // RECORDING - fileStream >> recording->_timestamps; - - for (int i = 0; i < recording->_timestamps.size(); ++i) { - QBitArray mask; - QByteArray buffer; - QDataStream stream(&buffer, QIODevice::ReadOnly); - RecordingFrame frame; - RecordingFrame& previousFrame = (i == 0) ? frame : recording->_frames.last(); - - fileStream >> mask; - fileStream >> buffer; - int maskIndex = 0; - - // Blendshape Coefficients - if (i == 0) { - stream >> numBlendshapes; - } - frame._blendshapeCoefficients.resize(numBlendshapes); - for (int j = 0; j < numBlendshapes; ++j) { - if (!mask[maskIndex++] || !readFloat(stream, frame._blendshapeCoefficients[j], BLENDSHAPE_RADIX)) { - frame._blendshapeCoefficients[j] = previousFrame._blendshapeCoefficients[j]; - } - } - // Joint Rotations - if (i == 0) { - stream >> numJoints; - } - frame._jointRotations.resize(numJoints); - for (int j = 0; j < numJoints; ++j) { - if (!mask[maskIndex++] || !readQuat(stream, frame._jointRotations[j])) { - frame._jointRotations[j] = previousFrame._jointRotations[j]; - } - } - - if (!mask[maskIndex++] || !readVec3(stream, frame._translation)) { - frame._translation = previousFrame._translation; - } - - if (!mask[maskIndex++] || !readQuat(stream, frame._rotation)) { - frame._rotation = previousFrame._rotation; - } - - if (!mask[maskIndex++] || !readFloat(stream, frame._scale, SCALE_RADIX)) { - frame._scale = previousFrame._scale; - } - - if (!mask[maskIndex++] || !readQuat(stream, frame._headRotation)) { - frame._headRotation = previousFrame._headRotation; - } - - if (!mask[maskIndex++] || !readFloat(stream, frame._leanSideways, LEAN_RADIX)) { - frame._leanSideways = previousFrame._leanSideways; - } - - if (!mask[maskIndex++] || !readFloat(stream, frame._leanForward, LEAN_RADIX)) { - frame._leanForward = previousFrame._leanForward; - } - - if (!mask[maskIndex++] || !readVec3(stream, frame._lookAtPosition)) { - frame._lookAtPosition = previousFrame._lookAtPosition; - } - - recording->_frames << frame; - } - - QByteArray audioArray; - fileStream >> audioArray; - recording->addAudioPacket(audioArray); - - bool wantDebug = true; - if (wantDebug) { - qDebug() << "[DEBUG] READ recording"; - qDebug() << "Header:"; - qDebug() << "File Format version:" << VERSION; - qDebug() << "Data length:" << dataLength; - qDebug() << "Data offset:" << dataOffset; - qDebug() << "CRC-16:" << crc16; - - qDebug() << "Context block:"; - qDebug() << "Global timestamp:" << context.globalTimestamp; - qDebug() << "Domain:" << context.domain; - qDebug() << "Position:" << context.position; - qDebug() << "Orientation:" << context.orientation; - qDebug() << "Scale:" << context.scale; - qDebug() << "Head Model:" << context.headModel; - qDebug() << "Skeleton Model:" << context.skeletonModel; - qDebug() << "Display Name:" << context.displayName; - qDebug() << "Num Attachments:" << numAttachments; - for (int i = 0; i < numAttachments; ++i) { - qDebug() << "Model URL:" << context.attachments[i].modelURL; - qDebug() << "Joint Name:" << context.attachments[i].jointName; - qDebug() << "Translation:" << context.attachments[i].translation; - qDebug() << "Rotation:" << context.attachments[i].rotation; - qDebug() << "Scale:" << context.attachments[i].scale; - } - - qDebug() << "Recording:"; - qDebug() << "Total frames:" << recording->getFrameNumber(); - qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); - - } - - qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; - return recording; -} - - -RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray) { - QElapsedTimer timer; - timer.start(); - - if (!recording) { - recording.reset(new Recording()); - } - - QDataStream fileStream(byteArray); - - fileStream >> recording->_timestamps; - RecordingFrame baseFrame; - - // Blendshape coefficients - fileStream >> baseFrame._blendshapeCoefficients; - - // Joint Rotations - int jointRotationSize; - fileStream >> jointRotationSize; - baseFrame._jointRotations.resize(jointRotationSize); - for (int i = 0; i < jointRotationSize; ++i) { - fileStream >> baseFrame._jointRotations[i].x >> baseFrame._jointRotations[i].y >> baseFrame._jointRotations[i].z >> baseFrame._jointRotations[i].w; - } - - fileStream >> baseFrame._translation.x >> baseFrame._translation.y >> baseFrame._translation.z; - fileStream >> baseFrame._rotation.x >> baseFrame._rotation.y >> baseFrame._rotation.z >> baseFrame._rotation.w; - fileStream >> baseFrame._scale; - fileStream >> baseFrame._headRotation.x >> baseFrame._headRotation.y >> baseFrame._headRotation.z >> baseFrame._headRotation.w; - fileStream >> baseFrame._leanSideways; - fileStream >> baseFrame._leanForward; - - - // Fake context - RecordingContext& context = recording->getContext(); - context.globalTimestamp = usecTimestampNow(); - context.domain = NodeList::getInstance()->getDomainHandler().getHostname(); - context.position = glm::vec3(144.5f, 3.3f, 181.3f); - context.orientation = glm::angleAxis(glm::radians(-92.5f), glm::vec3(0, 1, 0));; - context.scale = baseFrame._scale; - context.headModel = "http://public.highfidelity.io/models/heads/Emily_v4.fst"; - context.skeletonModel = "http://public.highfidelity.io/models/skeletons/EmilyCutMesh_A.fst"; - context.displayName = "Leslie"; - context.attachments.clear(); - AttachmentData data; - data.modelURL = "http://public.highfidelity.io/models/attachments/fbx.fst"; - data.jointName = "RightHand" ; - data.translation = glm::vec3(0.04f, 0.07f, 0.0f); - data.rotation = glm::angleAxis(glm::radians(102.0f), glm::vec3(0, 1, 0)); - data.scale = 0.20f; - context.attachments << data; - - context.orientationInv = glm::inverse(context.orientation); - - baseFrame._translation = glm::vec3(); - baseFrame._rotation = glm::quat(); - baseFrame._scale = 1.0f; - - recording->_frames << baseFrame; - - for (int i = 1; i < recording->_timestamps.size(); ++i) { - QBitArray mask; - QByteArray buffer; - QDataStream stream(&buffer, QIODevice::ReadOnly); - RecordingFrame frame; - RecordingFrame& previousFrame = recording->_frames.last(); - - fileStream >> mask; - fileStream >> buffer; - int maskIndex = 0; - - // Blendshape Coefficients - frame._blendshapeCoefficients.resize(baseFrame._blendshapeCoefficients.size()); - for (int i = 0; i < baseFrame._blendshapeCoefficients.size(); ++i) { - if (mask[maskIndex++]) { - stream >> frame._blendshapeCoefficients[i]; - } else { - frame._blendshapeCoefficients[i] = previousFrame._blendshapeCoefficients[i]; - } - } - - // Joint Rotations - frame._jointRotations.resize(baseFrame._jointRotations.size()); - for (int i = 0; i < baseFrame._jointRotations.size(); ++i) { - if (mask[maskIndex++]) { - stream >> frame._jointRotations[i].x >> frame._jointRotations[i].y >> frame._jointRotations[i].z >> frame._jointRotations[i].w; - } else { - frame._jointRotations[i] = previousFrame._jointRotations[i]; - } - } - - if (mask[maskIndex++]) { - stream >> frame._translation.x >> frame._translation.y >> frame._translation.z; - frame._translation = context.orientationInv * frame._translation; - } else { - frame._translation = previousFrame._translation; - } - - if (mask[maskIndex++]) { - stream >> frame._rotation.x >> frame._rotation.y >> frame._rotation.z >> frame._rotation.w; - } else { - frame._rotation = previousFrame._rotation; - } - - if (mask[maskIndex++]) { - stream >> frame._scale; - } else { - frame._scale = previousFrame._scale; - } - - if (mask[maskIndex++]) { - stream >> frame._headRotation.x >> frame._headRotation.y >> frame._headRotation.z >> frame._headRotation.w; - } else { - frame._headRotation = previousFrame._headRotation; - } - - if (mask[maskIndex++]) { - stream >> frame._leanSideways; - } else { - frame._leanSideways = previousFrame._leanSideways; - } - - if (mask[maskIndex++]) { - stream >> frame._leanForward; - } else { - frame._leanForward = previousFrame._leanForward; - } - - recording->_frames << frame; - } - - QByteArray audioArray; - fileStream >> audioArray; - - // Cut down audio if necessary - int SAMPLE_RATE = 48000; // 48 kHz - int SAMPLE_SIZE = 2; // 16 bits - int MSEC_PER_SEC = 1000; - int audioLength = recording->getLength() * SAMPLE_SIZE * (SAMPLE_RATE / MSEC_PER_SEC); - audioArray.chop(audioArray.size() - audioLength); - - recording->addAudioPacket(audioArray); - - qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; - - // Set new filename - if (filename.startsWith("http") || filename.startsWith("https") || filename.startsWith("ftp")) { - filename = QUrl(filename).fileName(); - } - if (filename.endsWith(".rec") || filename.endsWith(".REC")) { - filename.chop(qstrlen(".rec")); - } - filename.append(".hfr"); - filename = QFileInfo(filename).absoluteFilePath(); - - // Set recording to new format - writeRecordingToFile(recording, filename); - QMessageBox::warning(NULL, - QString("New recording location"), - QString("The new recording was saved at:\n" + filename), - QMessageBox::Ok); - qDebug() << "Recording has been successfully converted at" << filename; - return recording; -} - diff --git a/libraries/avatars/src/Recorder.h b/libraries/avatars/src/Recorder.h index 961beca0a9..454e8016fa 100644 --- a/libraries/avatars/src/Recorder.h +++ b/libraries/avatars/src/Recorder.h @@ -12,122 +12,18 @@ #ifndef hifi_Recorder_h #define hifi_Recorder_h -#include -#include -#include -#include -#include -#include +#include "Recording.h" -#include -#include - -#include -#include -#include +template +class QSharedPointer; class AttachmentData; class AvatarData; class Recorder; class Recording; -class Player; -typedef QSharedPointer RecordingPointer; typedef QSharedPointer RecorderPointer; typedef QWeakPointer WeakRecorderPointer; -typedef QSharedPointer PlayerPointer; -typedef QWeakPointer WeakPlayerPointer; - -/// Stores the different values associated to one recording frame -class RecordingFrame { -public: - QVector getBlendshapeCoefficients() const { return _blendshapeCoefficients; } - QVector getJointRotations() const { return _jointRotations; } - glm::vec3 getTranslation() const { return _translation; } - glm::quat getRotation() const { return _rotation; } - float getScale() const { return _scale; } - glm::quat getHeadRotation() const { return _headRotation; } - float getLeanSideways() const { return _leanSideways; } - float getLeanForward() const { return _leanForward; } - glm::vec3 getLookAtPosition() const { return _lookAtPosition; } - -protected: - void setBlendshapeCoefficients(QVector blendshapeCoefficients); - void setJointRotations(QVector jointRotations) { _jointRotations = jointRotations; } - void setTranslation(glm::vec3 translation) { _translation = translation; } - void setRotation(glm::quat rotation) { _rotation = rotation; } - void setScale(float scale) { _scale = scale; } - void setHeadRotation(glm::quat headRotation) { _headRotation = headRotation; } - void setLeanSideways(float leanSideways) { _leanSideways = leanSideways; } - void setLeanForward(float leanForward) { _leanForward = leanForward; } - void setLookAtPosition(glm::vec3 lookAtPosition) { _lookAtPosition = lookAtPosition; } - -private: - QVector _blendshapeCoefficients; - QVector _jointRotations; - glm::vec3 _translation; - glm::quat _rotation; - float _scale; - glm::quat _headRotation; - float _leanSideways; - float _leanForward; - glm::vec3 _lookAtPosition; - - friend class Recorder; - friend void writeRecordingToFile(RecordingPointer recording, QString file); - friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); - friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); -}; - -class RecordingContext { -public: - quint64 globalTimestamp; - QString domain; - glm::vec3 position; - glm::quat orientation; - float scale; - QString headModel; - QString skeletonModel; - QString displayName; - QVector attachments; - - // This avoids recomputation every frame while recording. - glm::quat orientationInv; -}; - -/// Stores a recording -class Recording { -public: - Recording(); - ~Recording(); - - bool isEmpty() const { return _timestamps.isEmpty(); } - int getLength() const; // in ms - - RecordingContext& getContext() { return _context; } - int getFrameNumber() const { return _frames.size(); } - qint32 getFrameTimestamp(int i) const; - const RecordingFrame& getFrame(int i) const; - Sound* getAudio() const { return _audio; } - -protected: - void addFrame(int timestamp, RecordingFrame& frame); - void addAudioPacket(QByteArray byteArray); - void clear(); - -private: - RecordingContext _context; - QVector _timestamps; - QVector _frames; - - Sound* _audio; - - friend class Recorder; - friend class Player; - friend void writeRecordingToFile(RecordingPointer recording, QString file); - friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); - friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); -}; /// Records a recording class Recorder { @@ -153,59 +49,5 @@ private: AvatarData* _avatar; }; -/// Plays back a recording -class Player { -public: - Player(AvatarData* avatar); - - bool isPlaying() const; - qint64 elapsed() const; - - RecordingPointer getRecording() const { return _recording; } - -public slots: - void startPlaying(); - void stopPlaying(); - void loadFromFile(QString file); - void loadRecording(RecordingPointer recording); - void play(); - - void setPlayFromCurrentLocation(bool playFromCurrentPosition); - void setLoop(bool loop) { _loop = loop; } - void useAttachements(bool useAttachments) { _useAttachments = useAttachments; } - void useDisplayName(bool useDisplayName) { _useDisplayName = useDisplayName; } - void useHeadModel(bool useHeadURL) { _useHeadURL = useHeadURL; } - void useSkeletonModel(bool useSkeletonURL) { _useSkeletonURL = useSkeletonURL; } - -private: - void setupAudioThread(); - void cleanupAudioThread(); - void loopRecording(); - bool computeCurrentFrame(); - - QElapsedTimer _timer; - RecordingPointer _recording; - int _currentFrame; - - QSharedPointer _injector; - AudioInjectorOptions _options; - - AvatarData* _avatar; - QThread* _audioThread; - - - RecordingContext _currentContext; - bool _playFromCurrentPosition; - bool _loop; - bool _useAttachments; - bool _useDisplayName; - bool _useHeadURL; - bool _useSkeletonURL; - -}; - -void writeRecordingToFile(RecordingPointer recording, QString filename); -RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename); -RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); #endif // hifi_Recorder_h \ No newline at end of file diff --git a/libraries/avatars/src/Recording.cpp b/libraries/avatars/src/Recording.cpp new file mode 100644 index 0000000000..ce67a7676c --- /dev/null +++ b/libraries/avatars/src/Recording.cpp @@ -0,0 +1,807 @@ +// +// Recording.cpp +// +// +// Created by Clement on 9/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "AvatarData.h" +#include "Recording.h" + +// HFR file format magic number (Inspired by PNG) +// (decimal) 17 72 70 82 13 10 26 10 +// (hexadecimal) 11 48 46 52 0d 0a 1a 0a +// (ASCII C notation) \021 H F R \r \n \032 \n +static const int MAGIC_NUMBER_SIZE = 8; +static const char MAGIC_NUMBER[MAGIC_NUMBER_SIZE] = {17, 72, 70, 82, 13, 10, 26, 10}; +// Version (Major, Minor) +static const QPair VERSION(0, 1); + +int SCALE_RADIX = 10; +int BLENDSHAPE_RADIX = 15; +int LEAN_RADIX = 7; + +void RecordingFrame::setBlendshapeCoefficients(QVector blendshapeCoefficients) { + _blendshapeCoefficients = blendshapeCoefficients; +} + +Recording::Recording() : _audio(NULL) { +} + +Recording::~Recording() { + delete _audio; +} + +int Recording::getLength() const { + if (_timestamps.isEmpty()) { + return 0; + } + return _timestamps.last(); +} + +qint32 Recording::getFrameTimestamp(int i) const { + if (i >= _timestamps.size()) { + return getLength(); + } + return _timestamps[i]; +} + +const RecordingFrame& Recording::getFrame(int i) const { + assert(i < _timestamps.size()); + return _frames[i]; +} + +void Recording::addFrame(int timestamp, RecordingFrame &frame) { + _timestamps << timestamp; + _frames << frame; +} + +void Recording::addAudioPacket(QByteArray byteArray) { + if (!_audio) { + _audio = new Sound(byteArray); + return; + } + _audio->append(byteArray); +} + +void Recording::clear() { + _timestamps.clear(); + _frames.clear(); + delete _audio; + _audio = NULL; +} + +void writeVec3(QDataStream& stream, glm::vec3 value) { + unsigned char buffer[sizeof(value)]; + memcpy(buffer, &value, sizeof(value)); + stream.writeRawData(reinterpret_cast(buffer), sizeof(value)); +} + +bool readVec3(QDataStream& stream, glm::vec3& value) { + unsigned char buffer[sizeof(value)]; + stream.readRawData(reinterpret_cast(buffer), sizeof(value)); + memcpy(&value, buffer, sizeof(value)); + return true; +} + +void writeQuat(QDataStream& stream, glm::quat value) { + unsigned char buffer[256]; + int writtenToBuffer = packOrientationQuatToBytes(buffer, value); + stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); +} + +bool readQuat(QDataStream& stream, glm::quat& value) { + int quatByteSize = 4 * 2; // 4 floats * 2 bytes + unsigned char buffer[256]; + stream.readRawData(reinterpret_cast(buffer), quatByteSize); + int readFromBuffer = unpackOrientationQuatFromBytes(buffer, value); + if (readFromBuffer != quatByteSize) { + return false; + } + return true; +} + +void writeFloat(QDataStream& stream, float value, int radix) { + unsigned char buffer[256]; + int writtenToBuffer = packFloatScalarToSignedTwoByteFixed(buffer, value, radix); + stream.writeRawData(reinterpret_cast(buffer), writtenToBuffer); +} + +bool readFloat(QDataStream& stream, float& value, int radix) { + int floatByteSize = 2; // 1 floats * 2 bytes + int16_t buffer[256]; + stream.readRawData(reinterpret_cast(buffer), floatByteSize); + int readFromBuffer = unpackFloatScalarFromSignedTwoByteFixed(buffer, &value, radix); + if (readFromBuffer != floatByteSize) { + return false; + } + return true; +} + +void writeRecordingToFile(RecordingPointer recording, QString filename) { + if (!recording || recording->getFrameNumber() < 1) { + qDebug() << "Can't save empty recording"; + return; + } + + QElapsedTimer timer; + QFile file(filename); + if (!file.open(QIODevice::ReadWrite | QIODevice::Truncate)){ + qDebug() << "Couldn't open " << filename; + return; + } + timer.start(); + qDebug() << "Writing recording to " << filename << "."; + + QDataStream fileStream(&file); + + // HEADER + file.write(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); // Magic number + fileStream << VERSION; // File format version + const qint64 dataOffsetPos = file.pos(); + fileStream << (quint16)0; // Save two empty bytes for the data offset + const qint64 dataLengthPos = file.pos(); + fileStream << (quint32)0; // Save four empty bytes for the data offset + const quint64 crc16Pos = file.pos(); + fileStream << (quint16)0; // Save two empty bytes for the CRC-16 + + + // METADATA + // TODO + + + + // Write data offset + quint16 dataOffset = file.pos(); + file.seek(dataOffsetPos); + fileStream << dataOffset; + file.seek(dataOffset); + + // CONTEXT + RecordingContext& context = recording->getContext(); + // Global Timestamp + fileStream << context.globalTimestamp; + // Domain + fileStream << context.domain; + // Position + writeVec3(fileStream, context.position); + // Orientation + writeQuat(fileStream, context.orientation); + // Scale + writeFloat(fileStream, context.scale, SCALE_RADIX); + // Head model + fileStream << context.headModel; + // Skeleton model + fileStream << context.skeletonModel; + // Display name + fileStream << context.displayName; + // Attachements + fileStream << (quint8)context.attachments.size(); + foreach (AttachmentData data, context.attachments) { + // Model + fileStream << data.modelURL.toString(); + // Joint name + fileStream << data.jointName; + // Position + writeVec3(fileStream, data.translation); + // Orientation + writeQuat(fileStream, data.rotation); + // Scale + writeFloat(fileStream, data.scale, SCALE_RADIX); + } + + // RECORDING + fileStream << recording->_timestamps; + + QBitArray mask; + quint32 numBlendshapes = 0; + quint32 numJoints = 0; + + for (int i = 0; i < recording->_timestamps.size(); ++i) { + mask.fill(false); + int maskIndex = 0; + QByteArray buffer; + QDataStream stream(&buffer, QIODevice::WriteOnly); + RecordingFrame& previousFrame = recording->_frames[(i != 0) ? i - 1 : i]; + RecordingFrame& frame = recording->_frames[i]; + + // Blendshape Coefficients + if (i == 0) { + numBlendshapes = frame.getBlendshapeCoefficients().size(); + stream << numBlendshapes; + mask.resize(mask.size() + numBlendshapes); + } + for (int j = 0; j < numBlendshapes; ++j) { + if (i == 0 || + frame._blendshapeCoefficients[j] != previousFrame._blendshapeCoefficients[j]) { + writeFloat(stream, frame.getBlendshapeCoefficients()[j], BLENDSHAPE_RADIX); + mask.setBit(maskIndex); + } + ++maskIndex; + } + + // Joint Rotations + if (i == 0) { + numJoints = frame.getJointRotations().size(); + stream << numJoints; + mask.resize(mask.size() + numJoints); + } + for (int j = 0; j < numJoints; ++j) { + if (i == 0 || + frame._jointRotations[j] != previousFrame._jointRotations[j]) { + writeQuat(stream, frame._jointRotations[j]); + mask.setBit(maskIndex); + } + maskIndex++; + } + + // Translation + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._translation != previousFrame._translation) { + writeVec3(stream, frame._translation); + mask.setBit(maskIndex); + } + maskIndex++; + + // Rotation + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._rotation != previousFrame._rotation) { + writeQuat(stream, frame._rotation); + mask.setBit(maskIndex); + } + maskIndex++; + + // Scale + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._scale != previousFrame._scale) { + writeFloat(stream, frame._scale, SCALE_RADIX); + mask.setBit(maskIndex); + } + maskIndex++; + + // Head Rotation + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._headRotation != previousFrame._headRotation) { + writeQuat(stream, frame._headRotation); + mask.setBit(maskIndex); + } + maskIndex++; + + // Lean Sideways + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._leanSideways != previousFrame._leanSideways) { + writeFloat(stream, frame._leanSideways, LEAN_RADIX); + mask.setBit(maskIndex); + } + maskIndex++; + + // Lean Forward + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._leanForward != previousFrame._leanForward) { + writeFloat(stream, frame._leanForward, LEAN_RADIX); + mask.setBit(maskIndex); + } + maskIndex++; + + // LookAt Position + if (i == 0) { + mask.resize(mask.size() + 1); + } + if (i == 0 || frame._lookAtPosition != previousFrame._lookAtPosition) { + writeVec3(stream, frame._lookAtPosition); + mask.setBit(maskIndex); + } + maskIndex++; + + fileStream << mask; + fileStream << buffer; + } + + fileStream << recording->_audio->getByteArray(); + + qint64 writtingTime = timer.restart(); + // Write data length and CRC-16 + quint32 dataLength = file.pos() - dataOffset; + file.seek(dataOffset); // Go to beginning of data for checksum + quint16 crc16 = qChecksum(file.readAll().constData(), dataLength); + + file.seek(dataLengthPos); + fileStream << dataLength; + file.seek(crc16Pos); + fileStream << crc16; + file.seek(dataOffset + dataLength); + + bool wantDebug = true; + if (wantDebug) { + qDebug() << "[DEBUG] WRITE recording"; + qDebug() << "Header:"; + qDebug() << "File Format version:" << VERSION; + qDebug() << "Data length:" << dataLength; + qDebug() << "Data offset:" << dataOffset; + qDebug() << "CRC-16:" << crc16; + + qDebug() << "Context block:"; + qDebug() << "Global timestamp:" << context.globalTimestamp; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head Model:" << context.headModel; + qDebug() << "Skeleton Model:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << context.attachments.size(); + for (int i = 0; i < context.attachments.size(); ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + + qDebug() << "Recording:"; + qDebug() << "Total frames:" << recording->getFrameNumber(); + qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + } + + qint64 checksumTime = timer.elapsed(); + qDebug() << "Wrote" << file.size() << "bytes in" << writtingTime + checksumTime << "ms. (" << checksumTime << "ms for checksum)"; +} + +RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename) { + QByteArray byteArray; + QUrl url(filename); + QElapsedTimer timer; + timer.start(); // timer used for debug informations (download/parsing time) + + // Aquire the data and place it in byteArray + // Return if data unavailable + if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") { + // Download file if necessary + qDebug() << "Downloading recording at" << url; + NetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkReply* reply = networkAccessManager.get(QNetworkRequest(url)); + QEventLoop loop; + QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + loop.exec(); // wait for file + if (reply->error() != QNetworkReply::NoError) { + qDebug() << "Error while downloading recording: " << reply->error(); + reply->deleteLater(); + return recording; + } + byteArray = reply->readAll(); + reply->deleteLater(); + // print debug + restart timer + qDebug() << "Downloaded " << byteArray.size() << " bytes in " << timer.restart() << " ms."; + } else { + // If local file, just read it. + qDebug() << "Reading recording from " << filename << "."; + QFile file(filename); + if (!file.open(QIODevice::ReadOnly)){ + qDebug() << "Could not open local file: " << url; + return recording; + } + byteArray = file.readAll(); + file.close(); + } + + if (filename.endsWith(".rec") || filename.endsWith(".REC")) { + qDebug() << "Old .rec format"; + QMessageBox::warning(NULL, + QString("Old recording format"), + QString("Converting your file to the new format."), + QMessageBox::Ok); + readRecordingFromRecFile(recording, filename, byteArray); + return recording; + } else if (!filename.endsWith(".hfr") && !filename.endsWith(".HFR")) { + qDebug() << "File extension not recognized"; + } + + // Reset the recording passed in the arguments + if (!recording) { + recording.reset(new Recording()); + } + + QDataStream fileStream(byteArray); + + // HEADER + QByteArray magicNumber(MAGIC_NUMBER, MAGIC_NUMBER_SIZE); + if (!byteArray.startsWith(magicNumber)) { + qDebug() << "ERROR: This is not a .HFR file. (Magic Number incorrect)"; + return recording; + } + fileStream.skipRawData(MAGIC_NUMBER_SIZE); + + QPair version; + fileStream >> version; // File format version + if (version != VERSION) { + qDebug() << "ERROR: This file format version is not supported."; + return recording; + } + + quint16 dataOffset = 0; + fileStream >> dataOffset; + quint32 dataLength = 0; + fileStream >> dataLength; + quint16 crc16 = 0; + fileStream >> crc16; + + + // Check checksum + + quint16 computedCRC16 = qChecksum(byteArray.constData() + dataOffset, dataLength); + if (computedCRC16 != crc16) { + qDebug() << "Checksum does not match. Bailling!"; + recording.clear(); + return recording; + } + + // METADATA + // TODO + + + + // CONTEXT + RecordingContext& context = recording->getContext(); + // Global Timestamp + fileStream >> context.globalTimestamp; + // Domain + fileStream >> context.domain; + // Position + if (!readVec3(fileStream, context.position)) { + qDebug() << "Couldn't read file correctly. (Invalid vec3)"; + recording.clear(); + return recording; + } + // Orientation + if (!readQuat(fileStream, context.orientation)) { + qDebug() << "Couldn't read file correctly. (Invalid quat)"; + recording.clear(); + return recording; + } + + // Scale + if (!readFloat(fileStream, context.scale, SCALE_RADIX)) { + qDebug() << "Couldn't read file correctly. (Invalid float)"; + recording.clear(); + return recording; + } + // Head model + fileStream >> context.headModel; + // Skeleton model + fileStream >> context.skeletonModel; + // Display Name + fileStream >> context.displayName; + + // Attachements + quint8 numAttachments = 0; + fileStream >> numAttachments; + for (int i = 0; i < numAttachments; ++i) { + AttachmentData data; + // Model + QString modelURL; + fileStream >> modelURL; + data.modelURL = modelURL; + // Joint name + fileStream >> data.jointName; + // Translation + if (!readVec3(fileStream, data.translation)) { + qDebug() << "Couldn't read attachment correctly. (Invalid vec3)"; + continue; + } + // Rotation + if (!readQuat(fileStream, data.rotation)) { + qDebug() << "Couldn't read attachment correctly. (Invalid quat)"; + continue; + } + + // Scale + if (!readFloat(fileStream, data.scale, SCALE_RADIX)) { + qDebug() << "Couldn't read attachment correctly. (Invalid float)"; + continue; + } + context.attachments << data; + } + + quint32 numBlendshapes = 0; + quint32 numJoints = 0; + // RECORDING + fileStream >> recording->_timestamps; + + for (int i = 0; i < recording->_timestamps.size(); ++i) { + QBitArray mask; + QByteArray buffer; + QDataStream stream(&buffer, QIODevice::ReadOnly); + RecordingFrame frame; + RecordingFrame& previousFrame = (i == 0) ? frame : recording->_frames.last(); + + fileStream >> mask; + fileStream >> buffer; + int maskIndex = 0; + + // Blendshape Coefficients + if (i == 0) { + stream >> numBlendshapes; + } + frame._blendshapeCoefficients.resize(numBlendshapes); + for (int j = 0; j < numBlendshapes; ++j) { + if (!mask[maskIndex++] || !readFloat(stream, frame._blendshapeCoefficients[j], BLENDSHAPE_RADIX)) { + frame._blendshapeCoefficients[j] = previousFrame._blendshapeCoefficients[j]; + } + } + // Joint Rotations + if (i == 0) { + stream >> numJoints; + } + frame._jointRotations.resize(numJoints); + for (int j = 0; j < numJoints; ++j) { + if (!mask[maskIndex++] || !readQuat(stream, frame._jointRotations[j])) { + frame._jointRotations[j] = previousFrame._jointRotations[j]; + } + } + + if (!mask[maskIndex++] || !readVec3(stream, frame._translation)) { + frame._translation = previousFrame._translation; + } + + if (!mask[maskIndex++] || !readQuat(stream, frame._rotation)) { + frame._rotation = previousFrame._rotation; + } + + if (!mask[maskIndex++] || !readFloat(stream, frame._scale, SCALE_RADIX)) { + frame._scale = previousFrame._scale; + } + + if (!mask[maskIndex++] || !readQuat(stream, frame._headRotation)) { + frame._headRotation = previousFrame._headRotation; + } + + if (!mask[maskIndex++] || !readFloat(stream, frame._leanSideways, LEAN_RADIX)) { + frame._leanSideways = previousFrame._leanSideways; + } + + if (!mask[maskIndex++] || !readFloat(stream, frame._leanForward, LEAN_RADIX)) { + frame._leanForward = previousFrame._leanForward; + } + + if (!mask[maskIndex++] || !readVec3(stream, frame._lookAtPosition)) { + frame._lookAtPosition = previousFrame._lookAtPosition; + } + + recording->_frames << frame; + } + + QByteArray audioArray; + fileStream >> audioArray; + recording->addAudioPacket(audioArray); + + bool wantDebug = true; + if (wantDebug) { + qDebug() << "[DEBUG] READ recording"; + qDebug() << "Header:"; + qDebug() << "File Format version:" << VERSION; + qDebug() << "Data length:" << dataLength; + qDebug() << "Data offset:" << dataOffset; + qDebug() << "CRC-16:" << crc16; + + qDebug() << "Context block:"; + qDebug() << "Global timestamp:" << context.globalTimestamp; + qDebug() << "Domain:" << context.domain; + qDebug() << "Position:" << context.position; + qDebug() << "Orientation:" << context.orientation; + qDebug() << "Scale:" << context.scale; + qDebug() << "Head Model:" << context.headModel; + qDebug() << "Skeleton Model:" << context.skeletonModel; + qDebug() << "Display Name:" << context.displayName; + qDebug() << "Num Attachments:" << numAttachments; + for (int i = 0; i < numAttachments; ++i) { + qDebug() << "Model URL:" << context.attachments[i].modelURL; + qDebug() << "Joint Name:" << context.attachments[i].jointName; + qDebug() << "Translation:" << context.attachments[i].translation; + qDebug() << "Rotation:" << context.attachments[i].rotation; + qDebug() << "Scale:" << context.attachments[i].scale; + } + + qDebug() << "Recording:"; + qDebug() << "Total frames:" << recording->getFrameNumber(); + qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + + } + + qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; + return recording; +} + + +RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray) { + QElapsedTimer timer; + timer.start(); + + if (!recording) { + recording.reset(new Recording()); + } + + QDataStream fileStream(byteArray); + + fileStream >> recording->_timestamps; + RecordingFrame baseFrame; + + // Blendshape coefficients + fileStream >> baseFrame._blendshapeCoefficients; + + // Joint Rotations + int jointRotationSize; + fileStream >> jointRotationSize; + baseFrame._jointRotations.resize(jointRotationSize); + for (int i = 0; i < jointRotationSize; ++i) { + fileStream >> baseFrame._jointRotations[i].x >> baseFrame._jointRotations[i].y >> baseFrame._jointRotations[i].z >> baseFrame._jointRotations[i].w; + } + + fileStream >> baseFrame._translation.x >> baseFrame._translation.y >> baseFrame._translation.z; + fileStream >> baseFrame._rotation.x >> baseFrame._rotation.y >> baseFrame._rotation.z >> baseFrame._rotation.w; + fileStream >> baseFrame._scale; + fileStream >> baseFrame._headRotation.x >> baseFrame._headRotation.y >> baseFrame._headRotation.z >> baseFrame._headRotation.w; + fileStream >> baseFrame._leanSideways; + fileStream >> baseFrame._leanForward; + + + // Fake context + RecordingContext& context = recording->getContext(); + context.globalTimestamp = usecTimestampNow(); + context.domain = NodeList::getInstance()->getDomainHandler().getHostname(); + context.position = glm::vec3(144.5f, 3.3f, 181.3f); + context.orientation = glm::angleAxis(glm::radians(-92.5f), glm::vec3(0, 1, 0));; + context.scale = baseFrame._scale; + context.headModel = "http://public.highfidelity.io/models/heads/Emily_v4.fst"; + context.skeletonModel = "http://public.highfidelity.io/models/skeletons/EmilyCutMesh_A.fst"; + context.displayName = "Leslie"; + context.attachments.clear(); + AttachmentData data; + data.modelURL = "http://public.highfidelity.io/models/attachments/fbx.fst"; + data.jointName = "RightHand" ; + data.translation = glm::vec3(0.04f, 0.07f, 0.0f); + data.rotation = glm::angleAxis(glm::radians(102.0f), glm::vec3(0, 1, 0)); + data.scale = 0.20f; + context.attachments << data; + + context.orientationInv = glm::inverse(context.orientation); + + baseFrame._translation = glm::vec3(); + baseFrame._rotation = glm::quat(); + baseFrame._scale = 1.0f; + + recording->_frames << baseFrame; + + for (int i = 1; i < recording->_timestamps.size(); ++i) { + QBitArray mask; + QByteArray buffer; + QDataStream stream(&buffer, QIODevice::ReadOnly); + RecordingFrame frame; + RecordingFrame& previousFrame = recording->_frames.last(); + + fileStream >> mask; + fileStream >> buffer; + int maskIndex = 0; + + // Blendshape Coefficients + frame._blendshapeCoefficients.resize(baseFrame._blendshapeCoefficients.size()); + for (int i = 0; i < baseFrame._blendshapeCoefficients.size(); ++i) { + if (mask[maskIndex++]) { + stream >> frame._blendshapeCoefficients[i]; + } else { + frame._blendshapeCoefficients[i] = previousFrame._blendshapeCoefficients[i]; + } + } + + // Joint Rotations + frame._jointRotations.resize(baseFrame._jointRotations.size()); + for (int i = 0; i < baseFrame._jointRotations.size(); ++i) { + if (mask[maskIndex++]) { + stream >> frame._jointRotations[i].x >> frame._jointRotations[i].y >> frame._jointRotations[i].z >> frame._jointRotations[i].w; + } else { + frame._jointRotations[i] = previousFrame._jointRotations[i]; + } + } + + if (mask[maskIndex++]) { + stream >> frame._translation.x >> frame._translation.y >> frame._translation.z; + frame._translation = context.orientationInv * frame._translation; + } else { + frame._translation = previousFrame._translation; + } + + if (mask[maskIndex++]) { + stream >> frame._rotation.x >> frame._rotation.y >> frame._rotation.z >> frame._rotation.w; + } else { + frame._rotation = previousFrame._rotation; + } + + if (mask[maskIndex++]) { + stream >> frame._scale; + } else { + frame._scale = previousFrame._scale; + } + + if (mask[maskIndex++]) { + stream >> frame._headRotation.x >> frame._headRotation.y >> frame._headRotation.z >> frame._headRotation.w; + } else { + frame._headRotation = previousFrame._headRotation; + } + + if (mask[maskIndex++]) { + stream >> frame._leanSideways; + } else { + frame._leanSideways = previousFrame._leanSideways; + } + + if (mask[maskIndex++]) { + stream >> frame._leanForward; + } else { + frame._leanForward = previousFrame._leanForward; + } + + recording->_frames << frame; + } + + QByteArray audioArray; + fileStream >> audioArray; + + // Cut down audio if necessary + int SAMPLE_RATE = 48000; // 48 kHz + int SAMPLE_SIZE = 2; // 16 bits + int MSEC_PER_SEC = 1000; + int audioLength = recording->getLength() * SAMPLE_SIZE * (SAMPLE_RATE / MSEC_PER_SEC); + audioArray.chop(audioArray.size() - audioLength); + + recording->addAudioPacket(audioArray); + + qDebug() << "Read " << byteArray.size() << " bytes in " << timer.elapsed() << " ms."; + + // Set new filename + if (filename.startsWith("http") || filename.startsWith("https") || filename.startsWith("ftp")) { + filename = QUrl(filename).fileName(); + } + if (filename.endsWith(".rec") || filename.endsWith(".REC")) { + filename.chop(qstrlen(".rec")); + } + filename.append(".hfr"); + filename = QFileInfo(filename).absoluteFilePath(); + + // Set recording to new format + writeRecordingToFile(recording, filename); + QMessageBox::warning(NULL, + QString("New recording location"), + QString("The new recording was saved at:\n" + filename), + QMessageBox::Ok); + qDebug() << "Recording has been successfully converted at" << filename; + return recording; +} diff --git a/libraries/avatars/src/Recording.h b/libraries/avatars/src/Recording.h new file mode 100644 index 0000000000..ddfebaeb5e --- /dev/null +++ b/libraries/avatars/src/Recording.h @@ -0,0 +1,127 @@ +// +// Recording.h +// +// +// Created by Clement on 9/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_Recording_h +#define hifi_Recording_h + +#include +#include + +#include +#include + +template +class QSharedPointer; + +class AttachmentData; +class Recording; +class RecordingFrame; +class Sound; + +typedef QSharedPointer RecordingPointer; + +/// Stores recordings static data +class RecordingContext { +public: + quint64 globalTimestamp; + QString domain; + glm::vec3 position; + glm::quat orientation; + float scale; + QString headModel; + QString skeletonModel; + QString displayName; + QVector attachments; + + // This avoids recomputation every frame while recording. + glm::quat orientationInv; +}; + +/// Stores a recording +class Recording { +public: + Recording(); + ~Recording(); + + bool isEmpty() const { return _timestamps.isEmpty(); } + int getLength() const; // in ms + + RecordingContext& getContext() { return _context; } + int getFrameNumber() const { return _frames.size(); } + qint32 getFrameTimestamp(int i) const; + const RecordingFrame& getFrame(int i) const; + Sound* getAudio() const { return _audio; } + +protected: + void addFrame(int timestamp, RecordingFrame& frame); + void addAudioPacket(QByteArray byteArray); + void clear(); + +private: + RecordingContext _context; + QVector _timestamps; + QVector _frames; + + Sound* _audio; + + friend class Recorder; + friend class Player; + friend void writeRecordingToFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); +}; + +/// Stores the different values associated to one recording frame +class RecordingFrame { +public: + QVector getBlendshapeCoefficients() const { return _blendshapeCoefficients; } + QVector getJointRotations() const { return _jointRotations; } + glm::vec3 getTranslation() const { return _translation; } + glm::quat getRotation() const { return _rotation; } + float getScale() const { return _scale; } + glm::quat getHeadRotation() const { return _headRotation; } + float getLeanSideways() const { return _leanSideways; } + float getLeanForward() const { return _leanForward; } + glm::vec3 getLookAtPosition() const { return _lookAtPosition; } + +protected: + void setBlendshapeCoefficients(QVector blendshapeCoefficients); + void setJointRotations(QVector jointRotations) { _jointRotations = jointRotations; } + void setTranslation(glm::vec3 translation) { _translation = translation; } + void setRotation(glm::quat rotation) { _rotation = rotation; } + void setScale(float scale) { _scale = scale; } + void setHeadRotation(glm::quat headRotation) { _headRotation = headRotation; } + void setLeanSideways(float leanSideways) { _leanSideways = leanSideways; } + void setLeanForward(float leanForward) { _leanForward = leanForward; } + void setLookAtPosition(glm::vec3 lookAtPosition) { _lookAtPosition = lookAtPosition; } + +private: + QVector _blendshapeCoefficients; + QVector _jointRotations; + glm::vec3 _translation; + glm::quat _rotation; + float _scale; + glm::quat _headRotation; + float _leanSideways; + float _leanForward; + glm::vec3 _lookAtPosition; + + friend class Recorder; + friend void writeRecordingToFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromFile(RecordingPointer recording, QString file); + friend RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); +}; + +void writeRecordingToFile(RecordingPointer recording, QString filename); +RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filename); +RecordingPointer readRecordingFromRecFile(RecordingPointer recording, QString filename, QByteArray byteArray); + +#endif // hifi_Recording_h \ No newline at end of file From e0739e4aa869199aab351472363a5b14f7655c30 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 18 Sep 2014 00:03:02 -0700 Subject: [PATCH 20/21] PreCR fixes --- libraries/avatars/src/Player.cpp | 2 +- libraries/avatars/src/Recording.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/avatars/src/Player.cpp b/libraries/avatars/src/Player.cpp index 554df1a758..f18c7f04aa 100644 --- a/libraries/avatars/src/Player.cpp +++ b/libraries/avatars/src/Player.cpp @@ -204,7 +204,7 @@ void Player::play() { head->setFinalPitch(eulers.x); head->setFinalYaw(eulers.y); head->setFinalRoll(eulers.z); - //head->setLookAtPosition(currentFrame.getLookAtPosition()); + head->setLookAtPosition(currentFrame.getLookAtPosition()); } else { qDebug() << "WARNING: Player couldn't find head data."; } diff --git a/libraries/avatars/src/Recording.cpp b/libraries/avatars/src/Recording.cpp index ce67a7676c..63f37a6885 100644 --- a/libraries/avatars/src/Recording.cpp +++ b/libraries/avatars/src/Recording.cpp @@ -457,7 +457,6 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, QString filen // Check checksum - quint16 computedCRC16 = qChecksum(byteArray.constData() + dataOffset, dataLength); if (computedCRC16 != crc16) { qDebug() << "Checksum does not match. Bailling!"; From 881ca61a09cf6bbe7c17bda89a1433be2ac9a705 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Thu, 18 Sep 2014 00:30:07 -0700 Subject: [PATCH 21/21] Modified recordings namefilter --- examples/Recorder.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/Recorder.js b/examples/Recorder.js index b3a4c7a4c3..657e2d5206 100644 --- a/examples/Recorder.js +++ b/examples/Recorder.js @@ -210,14 +210,14 @@ function mousePressEvent(event) { } } else if (saveIcon === toolBar.clicked(clickedOverlay)) { if (!MyAvatar.isRecording() && !MyAvatar.isPlaying() && MyAvatar.playerLength() != 0) { - recordingFile = Window.save("Save recording to file", ".", "*.rec"); + recordingFile = Window.save("Save recording to file", ".", "Recordings (*.hfr)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { MyAvatar.saveRecording(recordingFile); } } } else if (loadIcon === toolBar.clicked(clickedOverlay)) { if (!MyAvatar.isRecording() && !MyAvatar.isPlaying()) { - recordingFile = Window.browse("Load recorcding from file", ".", "*.rec"); + recordingFile = Window.browse("Load recorcding from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { MyAvatar.loadRecording(recordingFile); }