Merge pull request #10057 from huffman/limitless-stable

Beta Release 36 - Adding support for Limitless VR Interactive Character platform

assignment-client/src/Agent.cpp

@@ -403,6 +403,7 @@ void Agent::executeScript() {
     _scriptEngine->registerGlobalObject("Agent", this);
 
     _scriptEngine->registerGlobalObject("SoundCache", DependencyManager::get<SoundCache>().data());
+    _scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get<AnimationCache>().data());
 
     QScriptValue webSocketServerConstructorValue = _scriptEngine->newFunction(WebSocketServerClass::constructor);
     _scriptEngine->globalObject().setProperty("WebSocketServer", webSocketServerConstructorValue);

interface/src/Application.cpp

@@ -177,6 +177,7 @@
 #include "FrameTimingsScriptingInterface.h"
 #include <GPUIdent.h>
 #include <gl/GLHelpers.h>
+#include <src/scripting/LimitlessVoiceRecognitionScriptingInterface.h>
 
 // On Windows PC, NVidia Optimus laptop, we want to enable NVIDIA GPU
 // FIXME seems to be broken.
@@ -522,6 +523,7 @@ bool setupEssentials(int& argc, char** argv) {
     DependencyManager::set<OffscreenQmlSurfaceCache>();
     DependencyManager::set<EntityScriptClient>();
     DependencyManager::set<EntityScriptServerLogClient>();
+    DependencyManager::set<LimitlessVoiceRecognitionScriptingInterface>();
     return previousSessionCrashed;
 }
 
@@ -4557,6 +4559,8 @@ void Application::update(float deltaTime) {
     }
 
     AnimDebugDraw::getInstance().update();
+
+    DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>()->update();
 }
 
 void Application::sendAvatarViewFrustum() {
@@ -5548,6 +5552,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
     scriptEngine->registerGlobalObject("UserActivityLogger", DependencyManager::get<UserActivityLoggerScriptingInterface>().data());
     scriptEngine->registerGlobalObject("Users", DependencyManager::get<UsersScriptingInterface>().data());
 
+    scriptEngine->registerGlobalObject("LimitlessSpeechRecognition", DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>().data());
+
     if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) {
         scriptEngine->registerGlobalObject("Steam", new SteamScriptingInterface(scriptEngine, steamClient.get()));
     }

interface/src/scripting/LimitlessConnection.cpp

@@ -0,0 +1,91 @@
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include "LimitlessConnection.h"
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+LimitlessConnection::LimitlessConnection() :
+        _streamingAudioForTranscription(false)
+{
+}
+
+void LimitlessConnection::startListening(QString authCode) {
+    _transcribeServerSocket.reset(new QTcpSocket(this));
+    connect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+            &LimitlessConnection::transcriptionReceived);
+    connect(_transcribeServerSocket.get(), &QTcpSocket::disconnected, this, [this](){stopListening();});
+
+    static const auto host = "gserv_devel.studiolimitless.com";
+    _transcribeServerSocket->connectToHost(host, 1407);
+    _transcribeServerSocket->waitForConnected();
+    QString requestHeader = QString::asprintf("Authorization: %s\r\nfs: %i\r\n",
+                                              authCode.toLocal8Bit().data(), AudioConstants::SAMPLE_RATE);
+    qCDebug(interfaceapp) << "Sending Limitless Audio Stream Request: " << requestHeader;
+    _transcribeServerSocket->write(requestHeader.toLocal8Bit());
+    _transcribeServerSocket->waitForBytesWritten();
+}
+
+void LimitlessConnection::stopListening() {
+    emit onFinishedSpeaking(_currentTranscription);
+    _streamingAudioForTranscription = false;
+    _currentTranscription = "";
+    if (!isConnected())
+        return;
+    _transcribeServerSocket->close();
+    disconnect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+            &LimitlessConnection::transcriptionReceived);
+    _transcribeServerSocket.release()->deleteLater();
+    disconnect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+            &LimitlessConnection::audioInputReceived);
+    qCDebug(interfaceapp) << "Connection to Limitless Voice Server closed.";
+}
+
+void LimitlessConnection::audioInputReceived(const QByteArray& inputSamples) {
+    if (isConnected()) {
+        _transcribeServerSocket->write(inputSamples.data(), inputSamples.size());
+        _transcribeServerSocket->waitForBytesWritten();
+    }
+}
+
+void LimitlessConnection::transcriptionReceived() {
+    while (_transcribeServerSocket && _transcribeServerSocket->bytesAvailable() > 0) {
+        const QByteArray data = _transcribeServerSocket->readAll();
+        _serverDataBuffer.append(data);
+        int begin = _serverDataBuffer.indexOf('<');
+        int end = _serverDataBuffer.indexOf('>');
+        while (begin > -1 && end > -1) {
+            const int len = end - begin;
+            const QByteArray serverMessage = _serverDataBuffer.mid(begin+1, len-1);
+            if (serverMessage.contains("1407")) {
+                qCDebug(interfaceapp) << "Limitless Speech Server denied the request.";
+                // Don't spam the server with further false requests please.
+                DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>()->setListeningToVoice(true);
+                stopListening();
+                return;
+            } else if (serverMessage.contains("1408")) {
+                qCDebug(interfaceapp) << "Limitless Audio request authenticated!";
+                _serverDataBuffer.clear();
+                connect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+                        &LimitlessConnection::audioInputReceived);
+                return;
+            }
+            QJsonObject json = QJsonDocument::fromJson(serverMessage.data()).object();
+            _serverDataBuffer.remove(begin, len+1);
+            _currentTranscription = json["alternatives"].toArray()[0].toObject()["transcript"].toString();
+            emit onReceivedTranscription(_currentTranscription);
+            if (json["isFinal"] == true) {
+                qCDebug(interfaceapp) << "Final transcription: " << _currentTranscription;
+                stopListening();
+                return;
+            }
+            begin = _serverDataBuffer.indexOf('<');
+            end = _serverDataBuffer.indexOf('>');
+        }
+    }
+}
+
+bool LimitlessConnection::isConnected() const {
+    return _transcribeServerSocket.get() && _transcribeServerSocket->isWritable()
+    && _transcribeServerSocket->state() != QAbstractSocket::SocketState::UnconnectedState;
+}

interface/src/scripting/LimitlessConnection.h

@@ -0,0 +1,44 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/24/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_LimitlessConnection_h
+#define hifi_LimitlessConnection_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+
+class LimitlessConnection : public QObject {
+    Q_OBJECT
+public:
+    LimitlessConnection();
+
+    Q_INVOKABLE void startListening(QString authCode);
+    Q_INVOKABLE void stopListening();
+
+    std::atomic<bool> _streamingAudioForTranscription;
+
+signals:
+    void onReceivedTranscription(QString speech);
+    void onFinishedSpeaking(QString speech);
+
+private:
+    void transcriptionReceived();
+    void audioInputReceived(const QByteArray& inputSamples);
+
+    bool isConnected() const;
+
+    std::unique_ptr<QTcpSocket> _transcribeServerSocket;
+    QByteArray _serverDataBuffer;
+    QString _currentTranscription;
+};
+
+#endif //hifi_LimitlessConnection_h

interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp

@@ -0,0 +1,64 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/20/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include <QtConcurrent/QtConcurrentRun>
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+const float LimitlessVoiceRecognitionScriptingInterface::_audioLevelThreshold = 0.33f;
+const int LimitlessVoiceRecognitionScriptingInterface::_voiceTimeoutDuration = 2000;
+
+LimitlessVoiceRecognitionScriptingInterface::LimitlessVoiceRecognitionScriptingInterface() :
+        _shouldStartListeningForVoice(false)
+{
+    _voiceTimer.setSingleShot(true);
+    connect(&_voiceTimer, &QTimer::timeout, this, &LimitlessVoiceRecognitionScriptingInterface::voiceTimeout);
+    connect(&_connection, &LimitlessConnection::onReceivedTranscription, this, [this](QString transcription){emit onReceivedTranscription(transcription);});
+    connect(&_connection, &LimitlessConnection::onFinishedSpeaking, this, [this](QString transcription){emit onFinishedSpeaking(transcription);});
+    _connection.moveToThread(&_connectionThread);
+    _connectionThread.setObjectName("Limitless Connection");
+    _connectionThread.start();
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::update() {
+    const float audioLevel = AvatarInputs::getInstance()->loudnessToAudioLevel(DependencyManager::get<AudioClient>()->getAudioAverageInputLoudness());
+
+    if (_shouldStartListeningForVoice) {
+        if (_connection._streamingAudioForTranscription) {
+            if (audioLevel > _audioLevelThreshold) {
+                if (_voiceTimer.isActive()) {
+                    _voiceTimer.stop();
+                }
+            } else if (!_voiceTimer.isActive()){
+                _voiceTimer.start(_voiceTimeoutDuration);
+            }
+        } else if (audioLevel > _audioLevelThreshold) {
+            // to make sure invoke doesn't get called twice before the method actually gets called
+            _connection._streamingAudioForTranscription = true;
+            QMetaObject::invokeMethod(&_connection, "startListening", Q_ARG(QString, authCode));
+        }
+    }
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setListeningToVoice(bool listening) {
+    _shouldStartListeningForVoice = listening;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setAuthKey(QString key) {
+    authCode = key;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::voiceTimeout() {
+    if (_connection._streamingAudioForTranscription) {
+        QMetaObject::invokeMethod(&_connection, "stopListening");
+    }
+}

interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h

@@ -0,0 +1,50 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/20/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SpeechRecognitionScriptingInterface_h
+#define hifi_SpeechRecognitionScriptingInterface_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+#include "LimitlessConnection.h"
+
+class LimitlessVoiceRecognitionScriptingInterface : public QObject, public Dependency {
+    Q_OBJECT
+public:
+    LimitlessVoiceRecognitionScriptingInterface();
+
+    void update();
+
+    QString authCode;
+
+public slots:
+    void setListeningToVoice(bool listening);
+    void setAuthKey(QString key);
+
+signals:
+    void onReceivedTranscription(QString speech);
+    void onFinishedSpeaking(QString speech);
+
+private:
+
+    bool _shouldStartListeningForVoice;
+    static const float _audioLevelThreshold;
+    static const int _voiceTimeoutDuration;
+
+    QTimer _voiceTimer;
+    QThread _connectionThread;
+    LimitlessConnection _connection;
+
+    void voiceTimeout();
+};
+
+#endif //hifi_SpeechRecognitionScriptingInterface_h

interface/src/ui/AvatarInputs.cpp

@@ -58,25 +58,13 @@ AvatarInputs::AvatarInputs(QQuickItem* parent) :  QQuickItem(parent) {
         } \
     }
 
-void AvatarInputs::update() {
-    if (!Menu::getInstance()) {
-        return;
-    }
-    AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode()
-        && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror));
-    AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking));
-    AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking));
-    AI_UPDATE(isHMD, qApp->isHMDMode());
-    AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools));
-
-    auto audioIO = DependencyManager::get<AudioClient>();
+float AvatarInputs::loudnessToAudioLevel(float loudness) {
     const float AUDIO_METER_AVERAGING = 0.5;
     const float LOG2 = log(2.0f);
     const float METER_LOUDNESS_SCALE = 2.8f / 5.0f;
     const float LOG2_LOUDNESS_FLOOR = 11.0f;
     float audioLevel = 0.0f;
-    auto audio = DependencyManager::get<AudioClient>();
-    float loudness = audio->getLastInputLoudness() + 1.0f;
+    loudness += 1.0f;
 
     _trailingAudioLoudness = AUDIO_METER_AVERAGING * _trailingAudioLoudness + (1.0f - AUDIO_METER_AVERAGING) * loudness;
 
@@ -90,6 +78,24 @@ void AvatarInputs::update() {
     if (audioLevel > 1.0f) {
         audioLevel = 1.0;
     }
+    return audioLevel;
+}
+
+void AvatarInputs::update() {
+    if (!Menu::getInstance()) {
+        return;
+    }
+
+    AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode()
+         && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror));
+    AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking));
+    AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking));
+    AI_UPDATE(isHMD, qApp->isHMDMode());
+    AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools));
+
+    auto audioIO = DependencyManager::get<AudioClient>();
+
+    const float audioLevel = loudnessToAudioLevel(DependencyManager::get<AudioClient>()->getLastInputLoudness());
     AI_UPDATE_FLOAT(audioLevel, audioLevel, 0.01);
     AI_UPDATE(audioClipping, ((audioIO->getTimeSinceLastClip() > 0.0f) && (audioIO->getTimeSinceLastClip() < 1.0f)));
     AI_UPDATE(audioMuted, audioIO->isMuted());

interface/src/ui/AvatarInputs.h

@@ -35,6 +35,7 @@ class AvatarInputs : public QQuickItem {
 
 public:
     static AvatarInputs* getInstance();
+    float loudnessToAudioLevel(float loudness);
     AvatarInputs(QQuickItem* parent = nullptr);
     void update();