Merge pull request #3474 from Atlante45/fix_attachment_bug_with_recording

Play/Pause + interactive slider bar

examples/Recorder.js

@@ -40,6 +40,7 @@ var timerOffset;
 setupToolBar();
 
 var timer = null;
+var slider = null;
 setupTimer();
 
 var watchStop = false;
@@ -115,6 +116,30 @@ function setupTimer() {
 		alpha: 1.0,
 		visible: true
 	});
+
+    slider = { x: 0, y: 0,
+                w: 200, h: 20,
+                pos: 0.0, // 0.0 <= pos <= 1.0
+    };
+    slider.background = Overlays.addOverlay("text", {
+                                            text: "",
+                                            backgroundColor: { red: 128, green: 128, blue: 128 },
+                                            x: slider.x, y: slider.y,
+                                            width: slider.w,
+                                            height: slider.h,
+                                            alpha: 1.0,
+                                            visible: true
+                                            });
+    slider.foreground = Overlays.addOverlay("text", {
+                                            text: "",
+                                            backgroundColor: { red: 200, green: 200, blue: 200 },
+                                            x: slider.x, y: slider.y,
+                                            width: slider.pos * slider.w,
+                                            height: slider.h,
+                                            alpha: 1.0,
+                                            visible: true
+                                            });
+
 }
 
 function updateTimer() {
@@ -131,6 +156,16 @@ function updateTimer() {
 		text: text
 	})
     toolBar.changeSpacing(text.length * 8 + ((MyAvatar.isRecording()) ? 15 : 0), spacing);
+    
+    if (MyAvatar.isRecording()) {
+        slider.pos = 1.0;
+    } else if (MyAvatar.playerLength() > 0) {
+        slider.pos = MyAvatar.playerElapsed() / MyAvatar.playerLength();
+    }
+    
+    Overlays.editOverlay(slider.foreground, {
+                         width: slider.pos * slider.w
+                         });
 }
 
 function formatTime(time) {
@@ -163,7 +198,19 @@ function moveUI() {
 	Overlays.editOverlay(timer, {
 		x: relative.x + timerOffset - ToolBar.SPACING,
 		y: windowDimensions.y - relative.y - ToolBar.SPACING
-	});
+                         });
+    
+    slider.x = relative.x - ToolBar.SPACING;
+    slider.y = windowDimensions.y - relative.y - slider.h - ToolBar.SPACING;
+    
+    Overlays.editOverlay(slider.background, {
+                         x: slider.x,
+                         y: slider.y,
+                         });
+    Overlays.editOverlay(slider.foreground, {
+                         x: slider.x,
+                         y: slider.y,
+                         });
 }
 
 function mousePressEvent(event) {
@@ -188,7 +235,7 @@ function mousePressEvent(event) {
         }
     } else if (playIcon === toolBar.clicked(clickedOverlay) && !MyAvatar.isRecording()) {
         if (MyAvatar.isPlaying()) {
-            MyAvatar.stopPlaying();
+            MyAvatar.pausePlayer();
             toolBar.setAlpha(ALPHA_ON, recordIcon);
             toolBar.setAlpha(ALPHA_ON, saveIcon);
             toolBar.setAlpha(ALPHA_ON, loadIcon);
@@ -203,7 +250,7 @@ function mousePressEvent(event) {
         }
     } else if (playLoopIcon === toolBar.clicked(clickedOverlay) && !MyAvatar.isRecording()) {
         if (MyAvatar.isPlaying()) {
-            MyAvatar.stopPlaying();
+            MyAvatar.pausePlayer();
             toolBar.setAlpha(ALPHA_ON, recordIcon);
             toolBar.setAlpha(ALPHA_ON, saveIcon);
             toolBar.setAlpha(ALPHA_ON, loadIcon);
@@ -234,10 +281,30 @@ function mousePressEvent(event) {
                 toolBar.setAlpha(ALPHA_ON, saveIcon);
             }
         }
-    } else {
+    } else if (MyAvatar.playerLength() > 0 &&
-        
+               slider.x < event.x && event.x < slider.x + slider.w &&
+               slider.y < event.y && event.y < slider.y + slider.h) {
+        isSliding = true;
+        slider.pos = (event.x - slider.x) / slider.w;
+        MyAvatar.setPlayerTime(slider.pos * MyAvatar.playerLength());
     }
 }
+var isSliding = false;
+
+function mouseMoveEvent(event) {
+    if (isSliding) {
+        slider.pos = (event.x - slider.x) / slider.w;
+        if (slider.pos < 0.0 || slider.pos > 1.0) {
+            MyAvatar.stopPlaying();
+            slider.pos = 0.0;
+        }
+        MyAvatar.setPlayerTime(slider.pos * MyAvatar.playerLength());
+    }
+}
+
+function mouseReleaseEvent(event) {
+    isSliding = false;
+}
 
 function update() {
 	var newDimensions = Controller.getViewportDimensions();
@@ -264,11 +331,15 @@ function scriptEnding() {
 	if (MyAvatar.isPlaying()) {
 		MyAvatar.stopPlaying();
 	}
-	toolBar.cleanup();
+    toolBar.cleanup();
-	Overlays.deleteOverlay(timer);
+    Overlays.deleteOverlay(timer);
+    Overlays.deleteOverlay(slider.background);
+    Overlays.deleteOverlay(slider.foreground);
 }
 
 Controller.mousePressEvent.connect(mousePressEvent);
+Controller.mouseMoveEvent.connect(mouseMoveEvent);
+Controller.mouseReleaseEvent.connect(mouseReleaseEvent);
 Script.update.connect(update);
 Script.scriptEnding.connect(scriptEnding);
 

examples/editVoxels.js

@@ -1127,8 +1127,6 @@ function keyPressEvent(event) {
         } else if (event.text == "z") {
             undoSound.playRandom();
         }
-        
-        
     }
     
     trackKeyPressEvent(event); // used by preview support

libraries/audio/src/AudioInjector.cpp

@@ -25,7 +25,8 @@ AudioInjector::AudioInjector(QObject* parent) :
     QObject(parent),
     _sound(NULL),
     _options(),
-    _shouldStop(false)
+    _shouldStop(false),
+    _currentSendPosition(0)
 {
 }
 
@@ -95,17 +96,15 @@ void AudioInjector::injectAudio() {
         timer.start();
         int nextFrame = 0;
         
-        int currentSendPosition = 0;
-        
         int numPreAudioDataBytes = injectAudioPacket.size();
         bool shouldLoop = _options.getLoop();
         
         // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
         quint16 outgoingInjectedAudioSequenceNumber = 0;
-        while (currentSendPosition < soundByteArray.size() && !_shouldStop) {
+        while (_currentSendPosition < soundByteArray.size() && !_shouldStop) {
             
             int bytesToCopy = std::min(NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
-                                       soundByteArray.size() - currentSendPosition);
+                                       soundByteArray.size() - _currentSendPosition);
             memcpy(injectAudioPacket.data() + positionOptionOffset,
                    &_options.getPosition(),
                    sizeof(_options.getPosition()));
@@ -120,7 +119,7 @@ void AudioInjector::injectAudio() {
             memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes, &outgoingInjectedAudioSequenceNumber, sizeof(quint16));
             
             // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
-            memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + currentSendPosition, bytesToCopy);
+            memcpy(injectAudioPacket.data() + numPreAudioDataBytes, soundByteArray.data() + _currentSendPosition, bytesToCopy);
             
             // grab our audio mixer from the NodeList, if it exists
             NodeList* nodeList = NodeList::getInstance();
@@ -130,22 +129,22 @@ void AudioInjector::injectAudio() {
             nodeList->writeDatagram(injectAudioPacket, audioMixer);
             outgoingInjectedAudioSequenceNumber++;
             
-            currentSendPosition += bytesToCopy;
+            _currentSendPosition += bytesToCopy;
             
             // send two packets before the first sleep so the mixer can start playback right away
             
-            if (currentSendPosition != bytesToCopy && currentSendPosition < soundByteArray.size()) {
+            if (_currentSendPosition != bytesToCopy && _currentSendPosition < soundByteArray.size()) {
                 // not the first packet and not done
                 // sleep for the appropriate time
                 int usecToSleep = (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - timer.nsecsElapsed() / 1000;
                 
                 if (usecToSleep > 0) {
                     usleep(usecToSleep);
-                }
+                } 
             }
 
-            if (shouldLoop && currentSendPosition == soundByteArray.size()) {
+            if (shouldLoop && _currentSendPosition >= soundByteArray.size()) {
-                currentSendPosition = 0;
+                _currentSendPosition = 0;
             }
         }
     }

libraries/audio/src/AudioInjector.h

@@ -26,16 +26,20 @@ class AudioInjector : public QObject {
 public:
     AudioInjector(QObject* parent);
     AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions);
+    
+    int getCurrentSendPosition() const { return _currentSendPosition; }
 public slots:
     void injectAudio();
     void stop() { _shouldStop = true; }
     void setOptions(AudioInjectorOptions& options);
+    void setCurrentSendPosition(int currentSendPosition) { _currentSendPosition = currentSendPosition; }
 signals:
     void finished();
 private:
     Sound* _sound;
     AudioInjectorOptions _options;
     bool _shouldStop;
+    int _currentSendPosition;
 };
 
 Q_DECLARE_METATYPE(AudioInjector*)

libraries/avatars/src/AvatarData.cpp

@@ -587,18 +587,13 @@ bool AvatarData::hasReferential() {
 }
 
 bool AvatarData::isPlaying() {
-    if (!_player) {
-        return false;
-    }
-    if (QThread::currentThread() != thread()) {
-        bool result;
-        QMetaObject::invokeMethod(this, "isPlaying", Qt::BlockingQueuedConnection,
-                                  Q_RETURN_ARG(bool, result));
-        return result;
-    }
     return _player && _player->isPlaying();
 }
 
+bool AvatarData::isPaused() {
+    return _player && _player->isPaused();
+}
+
 qint64 AvatarData::playerElapsed() {
     if (!_player) {
         return 0;
@@ -625,6 +620,14 @@ qint64 AvatarData::playerLength() {
     return _player->getRecording()->getLength();
 }
 
+int AvatarData::playerCurrentFrame() {
+    return (_player) ? _player->getCurrentFrame() : 0;
+}
+
+int AvatarData::playerFrameNumber() {
+    return (_player && _player->getRecording()) ? _player->getRecording()->getFrameNumber() : 0;
+}
+
 void AvatarData::loadRecording(QString filename) {
     if (QThread::currentThread() != thread()) {
         QMetaObject::invokeMethod(this, "loadRecording", Qt::BlockingQueuedConnection,
@@ -649,6 +652,18 @@ void AvatarData::startPlaying() {
     _player->startPlaying();
 }
 
+void AvatarData::setPlayerFrame(int frame) {
+    if (_player) {
+        _player->setCurrentFrame(frame);
+    }
+}
+
+void AvatarData::setPlayerTime(qint64 time) {
+    if (_player) {
+        _player->setCurrentTime(time);
+    }
+}
+
 void AvatarData::setPlayFromCurrentLocation(bool playFromCurrentLocation) {
     if (_player) {
         _player->setPlayFromCurrentLocation(playFromCurrentLocation);
@@ -696,6 +711,19 @@ void AvatarData::play() {
     }
 }
 
+void AvatarData::pausePlayer() {
+    if (!_player) {
+        return;
+    }
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "pausePlayer", Qt::BlockingQueuedConnection);
+        return;
+    }
+    if (_player) {
+        _player->pausePlayer();
+    }
+}
+
 void AvatarData::stopPlaying() {
     if (!_player) {
         return;

libraries/avatars/src/AvatarData.h

@@ -296,10 +296,16 @@ public slots:
     bool hasReferential();
     
     bool isPlaying();
+    bool isPaused();
     qint64 playerElapsed();
     qint64 playerLength();
+    int playerCurrentFrame();
+    int playerFrameNumber();
+    
     void loadRecording(QString filename);
     void startPlaying();
+    void setPlayerFrame(int frame);
+    void setPlayerTime(qint64 time);
     void setPlayFromCurrentLocation(bool playFromCurrentLocation);
     void setPlayerLoop(bool loop);
     void setPlayerUseDisplayName(bool useDisplayName);
@@ -307,6 +313,7 @@ public slots:
     void setPlayerUseHeadModel(bool useHeadModel);
     void setPlayerUseSkeletonModel(bool useSkeletonModel);
     void play();
+    void pausePlayer();
     void stopPlaying();
     
 protected:

libraries/avatars/src/Player.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <AudioRingBuffer.h>
 #include <GLMHelpers.h>
 #include <NodeList.h>
 #include <StreamUtils.h>
@@ -18,6 +19,8 @@
 
 Player::Player(AvatarData* avatar) :
     _recording(new Recording()),
+    _pausedFrame(-1),
+    _timerOffset(0),
     _avatar(avatar),
     _audioThread(NULL),
     _playFromCurrentPosition(true),
@@ -36,16 +39,26 @@ bool Player::isPlaying() const {
     return _timer.isValid();
 }
 
+bool Player::isPaused() const {
+    return (_pausedFrame != -1);
+}
+
 qint64 Player::elapsed() const {
     if (isPlaying()) {
-        return _timer.elapsed();
+        return _timerOffset + _timer.elapsed();
+    } else if (isPaused()) {
+        return _timerOffset;
     } else {
         return 0;
     }
 }
 
 void Player::startPlaying() {
-    if (_recording && _recording->getFrameNumber() > 0) {
+    if (!_recording || _recording->getFrameNumber() <= 1) {
+        return;
+    }
+    
+    if (!isPaused()) {
         _currentContext.globalTimestamp = usecTimestampNow();
         _currentContext.domain = NodeList::getInstance()->getDomainHandler().getHostname();
         _currentContext.position = _avatar->getPosition();
@@ -97,9 +110,17 @@ void Player::startPlaying() {
         _avatar->setForceFaceshiftConnected(true);
         
         qDebug() << "Recorder::startPlaying()";
+        setupAudioThread();
         _currentFrame = 0;
+        _timerOffset = 0;
+        _timer.start();
+    } else {
+        qDebug() << "Recorder::startPlaying(): Unpause";
         setupAudioThread();
         _timer.start();
+        
+        setCurrentFrame(_pausedFrame);
+        _pausedFrame = -1;
     }
 }
 
@@ -107,6 +128,7 @@ void Player::stopPlaying() {
     if (!isPlaying()) {
         return;
     }
+    _pausedFrame = -1;
     _timer.invalidate();
     cleanupAudioThread();
     _avatar->clearJointsData();
@@ -130,6 +152,15 @@ void Player::stopPlaying() {
     qDebug() << "Recorder::stopPlaying()";
 }
 
+void Player::pausePlayer() {
+    _timerOffset = elapsed();
+    _timer.invalidate();
+    cleanupAudioThread();
+    
+    _pausedFrame = _currentFrame;
+    qDebug() << "Recorder::pausePlayer()";
+}
+
 void Player::setupAudioThread() {
     _audioThread = new QThread();
     _options.setPosition(_avatar->getPosition());
@@ -156,6 +187,7 @@ void Player::loopRecording() {
     cleanupAudioThread();
     setupAudioThread();
     _currentFrame = 0;
+    _timerOffset = 0;
     _timer.restart();
 }
 
@@ -166,10 +198,13 @@ void Player::loadFromFile(const QString& file) {
         _recording = RecordingPointer(new Recording());
     }
     readRecordingFromFile(_recording, file);
+    
+    _pausedFrame = -1;
 }
 
 void Player::loadRecording(RecordingPointer recording) {
     _recording = recording;
+    _pausedFrame = -1;
 }
 
 void Player::play() {
@@ -213,6 +248,77 @@ void Player::play() {
     _injector->setOptions(_options);
 }
 
+void Player::setCurrentFrame(int currentFrame) {
+    if (_recording && (currentFrame < 0 || currentFrame >= _recording->getFrameNumber())) {
+        stopPlaying();
+        return;
+    }
+    
+    _currentFrame = currentFrame;
+    _timerOffset = _recording->getFrameTimestamp(_currentFrame);
+    
+    if (isPlaying()) {
+        _timer.start();
+        setAudionInjectorPosition();
+    } else {
+        _pausedFrame = currentFrame;
+    }
+}
+
+void Player::setCurrentTime(qint64 currentTime) {
+    if (currentTime < 0 || currentTime >= _recording->getLength()) {
+        stopPlaying();
+        return;
+    }
+    
+    // Find correct frame
+    int lowestBound = 0;
+    int highestBound = _recording->getFrameNumber() - 1;
+    while (lowestBound + 1 != highestBound) {
+        assert(lowestBound < highestBound);
+        
+        int bestGuess = lowestBound +
+                    (highestBound - lowestBound) *
+                    (float)(currentTime - _recording->getFrameTimestamp(lowestBound)) /
+                    (float)(_recording->getFrameTimestamp(highestBound) - _recording->getFrameTimestamp(lowestBound));
+        
+        if (_recording->getFrameTimestamp(bestGuess) <= currentTime) {
+            if (currentTime < _recording->getFrameTimestamp(bestGuess + 1)) {
+                lowestBound = bestGuess;
+                highestBound = bestGuess + 1;
+            } else {
+                lowestBound = bestGuess + 1;
+            }
+        } else {
+            if (_recording->getFrameTimestamp(bestGuess - 1) <= currentTime) {
+                lowestBound = bestGuess - 1;
+                highestBound = bestGuess;
+            } else {
+                highestBound = bestGuess - 1;
+            }
+        }
+    }
+    
+    _currentFrame = lowestBound;
+    _timerOffset = _recording->getFrameTimestamp(lowestBound);
+    
+    if (isPlaying()) {
+        _timer.start();
+        setAudionInjectorPosition();
+    } else {
+        _pausedFrame = lowestBound;
+    }
+}
+
+void Player::setAudionInjectorPosition() {
+    int MSEC_PER_SEC = 1000;
+    int SAMPLE_SIZE = 2; // 16 bits
+    int CHANNEL_COUNT = 1;
+    int FRAME_SIZE = SAMPLE_SIZE * CHANNEL_COUNT;
+    int currentAudioFrame = elapsed() * FRAME_SIZE * (SAMPLE_RATE / MSEC_PER_SEC);
+    _injector->setCurrentSendPosition(currentAudioFrame);
+}
+
 void Player::setPlayFromCurrentLocation(bool playFromCurrentLocation) {
     _playFromCurrentPosition = playFromCurrentLocation;
 }
@@ -227,7 +333,7 @@ bool Player::computeCurrentFrame() {
     }
     
     while (_currentFrame < _recording->getFrameNumber() - 1 &&
-           _recording->getFrameTimestamp(_currentFrame) < _timer.elapsed()) {
+           _recording->getFrameTimestamp(_currentFrame) < elapsed()) {
         ++_currentFrame;
     }
     

libraries/avatars/src/Player.h

@@ -30,17 +30,23 @@ public:
     Player(AvatarData* avatar);
     
     bool isPlaying() const;
+    bool isPaused() const;
     qint64 elapsed() const;
     
     RecordingPointer getRecording() const { return _recording; }
+    int getCurrentFrame() const { return _currentFrame; }
     
 public slots:
     void startPlaying();
     void stopPlaying();
+    void pausePlayer();
     void loadFromFile(const QString& file);
     void loadRecording(RecordingPointer recording);
     void play();
     
+    void setCurrentFrame(int currentFrame);
+    void setCurrentTime(qint64 currentTime);
+    
     void setPlayFromCurrentLocation(bool playFromCurrentPosition);
     void setLoop(bool loop) { _loop = loop; }
     void useAttachements(bool useAttachments) { _useAttachments = useAttachments; }
@@ -52,11 +58,14 @@ private:
     void setupAudioThread();
     void cleanupAudioThread();
     void loopRecording();
+    void setAudionInjectorPosition();
     bool computeCurrentFrame();
     
     QElapsedTimer _timer;
     RecordingPointer _recording;
     int _currentFrame;
+    int _pausedFrame;
+    qint64 _timerOffset;
     
     QSharedPointer<AudioInjector> _injector;
     AudioInjectorOptions _options;
@@ -64,7 +73,6 @@ private:
     AvatarData* _avatar;
     QThread* _audioThread;
     
-    
     RecordingContext _currentContext;
     bool _playFromCurrentPosition;
     bool _loop;
@@ -72,7 +80,6 @@ private:
     bool _useDisplayName;
     bool _useHeadURL;
     bool _useSkeletonURL;
-    
 };
 
 #endif // hifi_Player_h

libraries/avatars/src/Recording.cpp

@@ -9,6 +9,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <AudioRingBuffer.h>
 #include <GLMHelpers.h>
 #include <NetworkAccessManager.h>
 #include <NodeList.h>
@@ -60,6 +61,9 @@ qint32 Recording::getFrameTimestamp(int i) const {
     if (i >= _timestamps.size()) {
         return getLength();
     }
+    if (i < 0) {
+        return 0;
+    }
     return _timestamps[i];
 }
 
@@ -781,7 +785,6 @@ RecordingPointer readRecordingFromRecFile(RecordingPointer recording, const QStr
     fileStream >> audioArray;
     
     // Cut down audio if necessary
-    int SAMPLE_RATE = 48000; // 48 kHz
     int SAMPLE_SIZE = 2; // 16 bits
     int MSEC_PER_SEC = 1000;
     int audioLength = recording->getLength() * SAMPLE_SIZE * (SAMPLE_RATE / MSEC_PER_SEC);