Merge pull request #10721 from davidkelly/dk/spectatorCameraPreview

Spectator camera preview

interface/resources/qml/hifi/SpectatorCamera.qml

@@ -11,6 +11,7 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+import Hifi 1.0 as Hifi
 import QtQuick 2.5
 import QtQuick.Controls 1.4
 import "../styles-uit"
@@ -212,21 +213,26 @@ Rectangle {
             boxSize: 24;
             onClicked: {
                 sendToScript({method: (checked ? 'spectatorCameraOn' : 'spectatorCameraOff')});
+                spectatorCameraPreview.ready = checked;
             }
         }
 
         // Spectator Camera Preview
-        Image {
+        Hifi.ResourceImageItem {
             id: spectatorCameraPreview;
+            url: "resource://spectatorCameraFrame";
+            ready: cameraToggleCheckBox.checked;
+            mirrorVertically: true;
             height: 250;
             anchors.left: parent.left;
             anchors.top: cameraToggleCheckBox.bottom;
             anchors.topMargin: 20;
             anchors.right: parent.right;
-            fillMode: Image.PreserveAspectFit;
-            horizontalAlignment: Image.AlignHCenter;
-            verticalAlignment: Image.AlignVCenter;
-            source: "http://1.bp.blogspot.com/-1GABEq__054/T03B00j_OII/AAAAAAAAAa8/jo55LcvEPHI/s1600/Winning.jpg";
+            onVisibleChanged: {
+                ready = cameraToggleCheckBox.checked;
+                update();
+            }
+
         }
 
         // "Monitor Shows" Switch Label Glyph
@@ -236,7 +242,7 @@ Rectangle {
             size: 32;
             color: hifi.colors.blueHighlight;
             anchors.top: spectatorCameraPreview.bottom;
-            anchors.topMargin: 12;
+            anchors.topMargin: 20;
             anchors.left: parent.left;
         }
         // "Monitor Shows" Switch Label

interface/src/Application.cpp

@@ -169,6 +169,7 @@
 #if defined(Q_OS_MAC) || defined(Q_OS_WIN)
 #include "SpeechRecognizer.h"
 #endif
+#include "ui/ResourceImageItem.h"
 #include "ui/AddressBarDialog.h"
 #include "ui/AvatarInputs.h"
 #include "ui/DialogsManager.h"
@@ -2004,6 +2005,7 @@ void Application::initializeUi() {
     LoginDialog::registerType();
     Tooltip::registerType();
     UpdateDialog::registerType();
+    qmlRegisterType<ResourceImageItem>("Hifi", 1, 0, "ResourceImageItem");
     qmlRegisterType<Preference>("Hifi", 1, 0, "Preference");
 
     auto offscreenUi = DependencyManager::get<OffscreenUi>();

interface/src/ui/ResourceImageItem.cpp

@@ -0,0 +1,111 @@
+//
+// ResourceImageItem.cpp
+//
+// Created by David Kelly and Howard Stearns on 2017/06/08
+// Copyright 2017 High Fidelity, Inc.
+
+// Distributed under the Apache License, Version 2.0
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+//#include "Application.h"
+#include "ResourceImageItem.h"
+
+#include <QOpenGLFramebufferObjectFormat>
+#include <QOpenGLFunctions>
+#include <QOpenGLExtraFunctions>
+#include <QOpenGLContext>
+
+
+void ResourceImageItem::setUrl(const QString& url) {
+    if (url != m_url) {
+        m_url = url;
+        update();
+    }
+}
+
+void ResourceImageItem::setReady(bool ready) {
+    if (ready != m_ready) {
+        m_ready = ready;
+        update();
+    }
+}
+
+void ResourceImageItemRenderer::onUpdateTimer() {
+    if (_ready && _networkTexture && _networkTexture->isLoaded()) {
+        if(_fboMutex.tryLock()) {
+            invalidateFramebufferObject();
+            qApp->getActiveDisplayPlugin()->copyTextureToQuickFramebuffer(_networkTexture, _copyFbo, &_fenceSync);
+            _fboMutex.unlock();
+        } else {
+            qDebug() << "couldn't get a lock, using last frame";
+        }
+    }
+    update();
+}
+
+ResourceImageItemRenderer::ResourceImageItemRenderer() : QQuickFramebufferObject::Renderer() {
+    connect(&_updateTimer, SIGNAL(timeout()), this, SLOT(onUpdateTimer()));
+}
+
+void ResourceImageItemRenderer::synchronize(QQuickFramebufferObject* item) {
+    ResourceImageItem* resourceImageItem = static_cast<ResourceImageItem*>(item);
+
+    resourceImageItem->setFlag(QQuickItem::ItemHasContents);
+    bool urlChanged = false;
+    if (_url != resourceImageItem->getUrl()) {
+        _url = resourceImageItem->getUrl();
+        urlChanged = true;
+    }
+    bool readyChanged = false;
+    if (_ready != resourceImageItem->getReady()) {
+        _ready = resourceImageItem->getReady();
+        readyChanged = true;
+    }
+
+    _window = resourceImageItem->window();
+    if (_ready && !_url.isNull() && !_url.isEmpty() && (urlChanged || readyChanged || !_networkTexture)) {
+        _networkTexture = DependencyManager::get<TextureCache>()->getTexture(_url);
+    }
+    static const int UPDATE_TIMER_DELAY_IN_MS = 100; // 100 ms = 10 hz for now
+    if (_ready && !_updateTimer.isActive()) {
+        _updateTimer.start(UPDATE_TIMER_DELAY_IN_MS);
+    } else if (!_ready && _updateTimer.isActive()) {
+        _updateTimer.stop();
+    }
+}
+
+QOpenGLFramebufferObject* ResourceImageItemRenderer::createFramebufferObject(const QSize& size) {
+    QOpenGLFramebufferObjectFormat format;
+    format.setAttachment(QOpenGLFramebufferObject::CombinedDepthStencil);
+    _copyFbo = new QOpenGLFramebufferObject(size, format);
+    _copyFbo->bind();
+    return new QOpenGLFramebufferObject(size, format);
+}
+
+void ResourceImageItemRenderer::render() {
+    auto f = QOpenGLContext::currentContext()->extraFunctions();
+    bool doUpdate = false;
+    // black background
+    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    if (_fenceSync) {
+        f->glWaitSync(_fenceSync, 0, GL_TIMEOUT_IGNORED);
+        f->glDeleteSync(_fenceSync);
+        _fenceSync = 0;
+        doUpdate = true;
+    }
+    if (_ready) {
+        _fboMutex.lock();
+        _copyFbo->bind();
+        QOpenGLFramebufferObject::blitFramebuffer(framebufferObject(), _copyFbo, GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT, GL_NEAREST);
+        _copyFbo->release();
+        _fboMutex.unlock();
+        if (doUpdate) {
+            update();
+        }
+    }
+    glFlush();
+    _window->resetOpenGLState();
+}

interface/src/ui/ResourceImageItem.h

@@ -0,0 +1,61 @@
+//
+// ResourceImageItem.h
+//
+// Created by David Kelly and Howard Stearns on 2017/06/08
+// Copyright 2017 High Fidelity, Inc.
+
+// Distributed under the Apache License, Version 2.0
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#pragma once
+#ifndef hifi_ResourceImageItem_h
+#define hifi_ResourceImageItem_h
+
+#include "Application.h"
+
+#include <QQuickFramebufferObject>
+#include <QQuickWindow>
+#include <QTimer>
+
+#include <TextureCache.h>
+
+class ResourceImageItemRenderer : public QObject, public QQuickFramebufferObject::Renderer {
+    Q_OBJECT
+public:
+    ResourceImageItemRenderer();
+    QOpenGLFramebufferObject* createFramebufferObject(const QSize& size) override;
+    void synchronize(QQuickFramebufferObject* item) override;
+    void render() override;
+private:
+    bool _ready;
+    QString _url;
+
+    NetworkTexturePointer _networkTexture;
+    QQuickWindow* _window;
+    QMutex _fboMutex;
+    QOpenGLFramebufferObject* _copyFbo;
+    GLsync _fenceSync { 0 };
+    QTimer _updateTimer;
+public slots:
+    void onUpdateTimer();
+};
+
+class ResourceImageItem : public QQuickFramebufferObject {
+    Q_OBJECT
+    Q_PROPERTY(QString url READ getUrl WRITE setUrl)
+    Q_PROPERTY(bool ready READ getReady WRITE setReady)
+public:
+    QString getUrl() const { return m_url; }
+    void setUrl(const QString& url);
+    bool getReady() const { return m_ready; }
+    void setReady(bool ready);
+    QQuickFramebufferObject::Renderer* createRenderer() const override { return new ResourceImageItemRenderer; }
+
+private:
+    QString m_url;
+    bool m_ready { false };
+
+};
+
+#endif // hifi_ResourceImageItem_h

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -16,6 +16,7 @@
 
 #include <QtOpenGL/QGLWidget>
 #include <QtGui/QImage>
+#include <QOpenGLFramebufferObject>
 #if defined(Q_OS_MAC)
 #include <OpenGL/CGLCurrent.h>
 #endif
@@ -41,7 +42,7 @@
 #include <ui-plugins/PluginContainer.h>
 #include <ui/Menu.h>
 #include <CursorManager.h>
-
+#include <TextureCache.h>
 #include "CompositorHelper.h"
 #include "Logging.h"
 
@@ -821,3 +822,51 @@ void OpenGLDisplayPlugin::updateCompositeFramebuffer() {
         _compositeFramebuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("OpenGLDisplayPlugin::composite", gpu::Element::COLOR_RGBA_32, renderSize.x, renderSize.y));
     }
 }
+
+void OpenGLDisplayPlugin::copyTextureToQuickFramebuffer(NetworkTexturePointer networkTexture, QOpenGLFramebufferObject* target, GLsync* fenceSync) {
+    auto glBackend = const_cast<OpenGLDisplayPlugin&>(*this).getGLBackend();
+    withMainThreadContext([&] {
+        GLuint sourceTexture = glBackend->getTextureID(networkTexture->getGPUTexture());
+        GLuint targetTexture = target->texture();
+        GLuint fbo[2] {0, 0};
+
+        // need mipmaps for blitting texture
+        glGenerateTextureMipmap(sourceTexture);
+
+        // create 2 fbos (one for initial texture, second for scaled one)
+        glCreateFramebuffers(2, fbo);
+
+        // setup source fbo
+        glBindFramebuffer(GL_FRAMEBUFFER, fbo[0]);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, sourceTexture, 0);
+        GLint texWidth, texHeight;
+        glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &texWidth);
+        glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &texHeight);
+
+        // setup destination fbo
+        glBindFramebuffer(GL_FRAMEBUFFER, fbo[1]);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, targetTexture, 0);
+
+
+        // maintain aspect ratio, filling the width first if possible.  If that makes the height too
+        // much, fill height instead. TODO: only do this when texture changes
+        GLint newX = 0;
+        GLint newY = 0;
+        float aspectRatio = (float)texHeight / (float)texWidth;
+        GLint newWidth = target->width();
+        GLint newHeight = std::round(aspectRatio * (float) target->width());
+        if (newHeight > target->height()) {
+            newHeight = target->height();
+            newWidth = std::round((float)target->height() / aspectRatio);
+            newX = (target->width() - newWidth) / 2;
+        } else {
+            newY = (target->height() - newHeight) / 2;
+        }
+        glBlitNamedFramebuffer(fbo[0], fbo[1], 0, 0, texWidth, texHeight, newX, newY, newWidth, newHeight, GL_DEPTH_BUFFER_BIT|GL_COLOR_BUFFER_BIT, GL_NEAREST);
+
+        // don't delete the textures!
+        glDeleteFramebuffers(2, fbo);
+        *fenceSync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
+    });
+}
+

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -79,6 +79,8 @@ public:
     // Three threads, one for rendering, one for texture transfers, one reserved for the GL driver
     int getRequiredThreadCount() const override { return 3; }
 
+    void copyTextureToQuickFramebuffer(NetworkTexturePointer source, QOpenGLFramebufferObject* target, GLsync* fenceSync) override;
+
 protected:
     friend class PresentThread;
 

libraries/model-networking/src/model-networking/TextureCache.cpp

@@ -980,10 +980,12 @@ NetworkTexturePointer TextureCache::getResourceTexture(QUrl resourceTextureUrl)
         if (!_spectatorCameraNetworkTexture) {
             _spectatorCameraNetworkTexture.reset(new NetworkTexture(resourceTextureUrl));
         }
-        texture = _spectatorCameraFramebuffer->getRenderBuffer(0);
-        if (texture) {
-            _spectatorCameraNetworkTexture->setImage(texture, texture->getWidth(), texture->getHeight());
-            return _spectatorCameraNetworkTexture;
+        if (_spectatorCameraFramebuffer) {
+            texture = _spectatorCameraFramebuffer->getRenderBuffer(0);
+            if (texture) {
+                _spectatorCameraNetworkTexture->setImage(texture, texture->getWidth(), texture->getHeight());
+                return _spectatorCameraNetworkTexture;
+            }
         }
     }
 

libraries/plugins/src/plugins/DisplayPlugin.h

@@ -22,9 +22,10 @@
 #include <RegisteredMetaTypes.h>
 #include <shared/Bilateral.h>
 #include <gpu/Forward.h>
-
 #include "Plugin.h"
 
+class QOpenGLFramebufferObject;
+
 class QImage;
 
 enum Eye {
@@ -60,6 +61,10 @@ namespace gpu {
     using TexturePointer = std::shared_ptr<Texture>;
 }
 
+class NetworkTexture;
+using NetworkTexturePointer = QSharedPointer<NetworkTexture>;
+typedef struct __GLsync *GLsync;
+
 // Stereo display functionality
 // TODO move out of this file don't derive DisplayPlugin from this.  Instead use dynamic casting when
 // displayPlugin->isStereo returns true
@@ -208,6 +213,8 @@ public:
     // Hardware specific stats
     virtual QJsonObject getHardwareStats() const { return QJsonObject(); }
 
+    virtual void copyTextureToQuickFramebuffer(NetworkTexturePointer source, QOpenGLFramebufferObject* target, GLsync* fenceSync) = 0;
+
     uint32_t presentCount() const { return _presentedFrameIndex; }
     // Time since last call to incrementPresentCount (only valid if DEBUG_PAINT_DELAY is defined)
     int64_t getPaintDelayUsecs() const;