From 4f7cbd5b7caa70bf382673967f9906762e164ab2 Mon Sep 17 00:00:00 2001
From: Delamare2112 <trevor@studiolimitless.com>
Date: Wed, 29 Mar 2017 12:47:58 -0700
Subject: [PATCH 1/4] Add limitless connection and scripting interface
---
.../src/scripting/LimitlessConnection.cpp | 91 +++++++++++++++++++
interface/src/scripting/LimitlessConnection.h | 44 +++++++++
...lessVoiceRecognitionScriptingInterface.cpp | 64 +++++++++++++
...itlessVoiceRecognitionScriptingInterface.h | 50 ++++++++++
4 files changed, 249 insertions(+)
create mode 100644 interface/src/scripting/LimitlessConnection.cpp
create mode 100644 interface/src/scripting/LimitlessConnection.h
create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp
create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h
diff --git a/interface/src/scripting/LimitlessConnection.cpp b/interface/src/scripting/LimitlessConnection.cpp
new file mode 100644
index 0000000000..b9f4eacd4b
--- /dev/null
+++ b/interface/src/scripting/LimitlessConnection.cpp
@@ -0,0 +1,91 @@
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include "LimitlessConnection.h"
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+LimitlessConnection::LimitlessConnection() :
+ _streamingAudioForTranscription(false)
+{
+}
+
+void LimitlessConnection::startListening(QString authCode) {
+ _transcribeServerSocket.reset(new QTcpSocket(this));
+ connect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+ &LimitlessConnection::transcriptionReceived);
+ connect(_transcribeServerSocket.get(), &QTcpSocket::disconnected, this, [this](){stopListening();});
+
+ static const auto host = "gserv_devel.studiolimitless.com";
+ _transcribeServerSocket->connectToHost(host, 1407);
+ _transcribeServerSocket->waitForConnected();
+ QString requestHeader = QString::asprintf("Authorization: %s\r\nfs: %i\r\n",
+ authCode.toLocal8Bit().data(), AudioConstants::SAMPLE_RATE);
+ qCDebug(interfaceapp) << "Sending Limitless Audio Stream Request: " << requestHeader;
+ _transcribeServerSocket->write(requestHeader.toLocal8Bit());
+ _transcribeServerSocket->waitForBytesWritten();
+}
+
+void LimitlessConnection::stopListening() {
+ emit onFinishedSpeaking(_currentTranscription);
+ _streamingAudioForTranscription = false;
+ _currentTranscription = "";
+ if (!isConnected())
+ return;
+ _transcribeServerSocket->close();
+ disconnect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+ &LimitlessConnection::transcriptionReceived);
+ _transcribeServerSocket.release()->deleteLater();
+ disconnect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+ &LimitlessConnection::audioInputReceived);
+ qCDebug(interfaceapp) << "Connection to Limitless Voice Server closed.";
+}
+
+void LimitlessConnection::audioInputReceived(const QByteArray& inputSamples) {
+ if (isConnected()) {
+ _transcribeServerSocket->write(inputSamples.data(), inputSamples.size());
+ _transcribeServerSocket->waitForBytesWritten();
+ }
+}
+
+void LimitlessConnection::transcriptionReceived() {
+ while (_transcribeServerSocket && _transcribeServerSocket->bytesAvailable() > 0) {
+ const QByteArray data = _transcribeServerSocket->readAll();
+ _serverDataBuffer.append(data);
+ int begin = _serverDataBuffer.indexOf('<');
+ int end = _serverDataBuffer.indexOf('>');
+ while (begin > -1 && end > -1) {
+ const int len = end - begin;
+ const QByteArray serverMessage = _serverDataBuffer.mid(begin+1, len-1);
+ if (serverMessage.contains("1407")) {
+ qCDebug(interfaceapp) << "Limitless Speech Server denied the request.";
+ // Don't spam the server with further false requests please.
+ DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>()->setListeningToVoice(true);
+ stopListening();
+ return;
+ } else if (serverMessage.contains("1408")) {
+ qCDebug(interfaceapp) << "Limitless Audio request authenticated!";
+ _serverDataBuffer.clear();
+ connect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+ &LimitlessConnection::audioInputReceived);
+ return;
+ }
+ QJsonObject json = QJsonDocument::fromJson(serverMessage.data()).object();
+ _serverDataBuffer.remove(begin, len+1);
+ _currentTranscription = json["alternatives"].toArray()[0].toObject()["transcript"].toString();
+ emit onReceivedTranscription(_currentTranscription);
+ if (json["isFinal"] == true) {
+ qCDebug(interfaceapp) << "Final transcription: " << _currentTranscription;
+ stopListening();
+ return;
+ }
+ begin = _serverDataBuffer.indexOf('<');
+ end = _serverDataBuffer.indexOf('>');
+ }
+ }
+}
+
+bool LimitlessConnection::isConnected() const {
+ return _transcribeServerSocket.get() && _transcribeServerSocket->isWritable()
+ && _transcribeServerSocket->state() != QAbstractSocket::SocketState::UnconnectedState;
+}
diff --git a/interface/src/scripting/LimitlessConnection.h b/interface/src/scripting/LimitlessConnection.h
new file mode 100644
index 0000000000..ee049aff8e
--- /dev/null
+++ b/interface/src/scripting/LimitlessConnection.h
@@ -0,0 +1,44 @@
+//
+// SpeechRecognitionScriptingInterface.h
+// interface/src/scripting
+//
+// Created by Trevor Berninger on 3/24/17.
+// Copyright 2017 Limitless ltd.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_LimitlessConnection_h
+#define hifi_LimitlessConnection_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+
+class LimitlessConnection : public QObject {
+ Q_OBJECT
+public:
+ LimitlessConnection();
+
+ Q_INVOKABLE void startListening(QString authCode);
+ Q_INVOKABLE void stopListening();
+
+ std::atomic<bool> _streamingAudioForTranscription;
+
+signals:
+ void onReceivedTranscription(QString speech);
+ void onFinishedSpeaking(QString speech);
+
+private:
+ void transcriptionReceived();
+ void audioInputReceived(const QByteArray& inputSamples);
+
+ bool isConnected() const;
+
+ std::unique_ptr<QTcpSocket> _transcribeServerSocket;
+ QByteArray _serverDataBuffer;
+ QString _currentTranscription;
+};
+
+#endif //hifi_LimitlessConnection_h
diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp
new file mode 100644
index 0000000000..1352630f84
--- /dev/null
+++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp
@@ -0,0 +1,64 @@
+//
+// SpeechRecognitionScriptingInterface.h
+// interface/src/scripting
+//
+// Created by Trevor Berninger on 3/20/17.
+// Copyright 2017 Limitless ltd.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include <QtConcurrent/QtConcurrentRun>
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+const float LimitlessVoiceRecognitionScriptingInterface::_audioLevelThreshold = 0.33f;
+const int LimitlessVoiceRecognitionScriptingInterface::_voiceTimeoutDuration = 2000;
+
+LimitlessVoiceRecognitionScriptingInterface::LimitlessVoiceRecognitionScriptingInterface() :
+ _shouldStartListeningForVoice(false)
+{
+ _voiceTimer.setSingleShot(true);
+ connect(&_voiceTimer, &QTimer::timeout, this, &LimitlessVoiceRecognitionScriptingInterface::voiceTimeout);
+ connect(&_connection, &LimitlessConnection::onReceivedTranscription, this, [this](QString transcription){emit onReceivedTranscription(transcription);});
+ connect(&_connection, &LimitlessConnection::onFinishedSpeaking, this, [this](QString transcription){emit onFinishedSpeaking(transcription);});
+ _connection.moveToThread(&_connectionThread);
+ _connectionThread.setObjectName("Limitless Connection");
+ _connectionThread.start();
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::update() {
+ const float audioLevel = AvatarInputs::getInstance()->loudnessToAudioLevel(DependencyManager::get<AudioClient>()->getAudioAverageInputLoudness());
+
+ if (_shouldStartListeningForVoice) {
+ if (_connection._streamingAudioForTranscription) {
+ if (audioLevel > _audioLevelThreshold) {
+ if (_voiceTimer.isActive()) {
+ _voiceTimer.stop();
+ }
+ } else if (!_voiceTimer.isActive()){
+ _voiceTimer.start(_voiceTimeoutDuration);
+ }
+ } else if (audioLevel > _audioLevelThreshold) {
+ // to make sure invoke doesn't get called twice before the method actually gets called
+ _connection._streamingAudioForTranscription = true;
+ QMetaObject::invokeMethod(&_connection, "startListening", Q_ARG(QString, authCode));
+ }
+ }
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setListeningToVoice(bool listening) {
+ _shouldStartListeningForVoice = listening;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setAuthKey(QString key) {
+ authCode = key;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::voiceTimeout() {
+ if (_connection._streamingAudioForTranscription) {
+ QMetaObject::invokeMethod(&_connection, "stopListening");
+ }
+}
diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h
new file mode 100644
index 0000000000..d1b1139695
--- /dev/null
+++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h
@@ -0,0 +1,50 @@
+//
+// SpeechRecognitionScriptingInterface.h
+// interface/src/scripting
+//
+// Created by Trevor Berninger on 3/20/17.
+// Copyright 2017 Limitless ltd.
+//
+// Distributed under the Apache License, Version 2.0.
+// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SpeechRecognitionScriptingInterface_h
+#define hifi_SpeechRecognitionScriptingInterface_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+#include "LimitlessConnection.h"
+
+class LimitlessVoiceRecognitionScriptingInterface : public QObject, public Dependency {
+ Q_OBJECT
+public:
+ LimitlessVoiceRecognitionScriptingInterface();
+
+ void update();
+
+ QString authCode;
+
+public slots:
+ void setListeningToVoice(bool listening);
+ void setAuthKey(QString key);
+
+signals:
+ void onReceivedTranscription(QString speech);
+ void onFinishedSpeaking(QString speech);
+
+private:
+
+ bool _shouldStartListeningForVoice;
+ static const float _audioLevelThreshold;
+ static const int _voiceTimeoutDuration;
+
+ QTimer _voiceTimer;
+ QThread _connectionThread;
+ LimitlessConnection _connection;
+
+ void voiceTimeout();
+};
+
+#endif //hifi_SpeechRecognitionScriptingInterface_h
From db11c40a08daa38ca3f2b6fb2f86b3f7d511f2ba Mon Sep 17 00:00:00 2001
From: Delamare2112 <trevor@studiolimitless.com>
Date: Wed, 29 Mar 2017 12:54:25 -0700
Subject: [PATCH 2/4] Add AnimationCache to Agent
---
assignment-client/src/Agent.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp
index 355e47be46..baed421df7 100644
--- a/assignment-client/src/Agent.cpp
+++ b/assignment-client/src/Agent.cpp
@@ -403,6 +403,7 @@ void Agent::executeScript() {
_scriptEngine->registerGlobalObject("Agent", this);
_scriptEngine->registerGlobalObject("SoundCache", DependencyManager::get<SoundCache>().data());
+ _scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get<AnimationCache>().data());
QScriptValue webSocketServerConstructorValue = _scriptEngine->newFunction(WebSocketServerClass::constructor);
_scriptEngine->globalObject().setProperty("WebSocketServer", webSocketServerConstructorValue);
From 2f6add3398a1b25747a1678f552699c0e5d73087 Mon Sep 17 00:00:00 2001
From: Delamare2112 <trevor@studiolimitless.com>
Date: Wed, 29 Mar 2017 12:56:08 -0700
Subject: [PATCH 3/4] Expose LimitlessSpeechRecognition to scripts
---
interface/src/Application.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index d48fe19a99..9f4e4a6bc6 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -177,6 +177,7 @@
#include "FrameTimingsScriptingInterface.h"
#include <GPUIdent.h>
#include <gl/GLHelpers.h>
+#include <src/scripting/LimitlessVoiceRecognitionScriptingInterface.h>
// On Windows PC, NVidia Optimus laptop, we want to enable NVIDIA GPU
// FIXME seems to be broken.
@@ -522,6 +523,7 @@ bool setupEssentials(int& argc, char** argv) {
DependencyManager::set<OffscreenQmlSurfaceCache>();
DependencyManager::set<EntityScriptClient>();
DependencyManager::set<EntityScriptServerLogClient>();
+ DependencyManager::set<LimitlessVoiceRecognitionScriptingInterface>();
return previousSessionCrashed;
}
@@ -4557,6 +4559,8 @@ void Application::update(float deltaTime) {
}
AnimDebugDraw::getInstance().update();
+
+ DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>()->update();
}
void Application::sendAvatarViewFrustum() {
@@ -5548,6 +5552,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri
scriptEngine->registerGlobalObject("UserActivityLogger", DependencyManager::get<UserActivityLoggerScriptingInterface>().data());
scriptEngine->registerGlobalObject("Users", DependencyManager::get<UsersScriptingInterface>().data());
+ scriptEngine->registerGlobalObject("LimitlessSpeechRecognition", DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>().data());
+
if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) {
scriptEngine->registerGlobalObject("Steam", new SteamScriptingInterface(scriptEngine, steamClient.get()));
}
From 3ebe1a0015364520104b918d89730e2229fa02dc Mon Sep 17 00:00:00 2001
From: Delamare2112 <trevor@studiolimitless.com>
Date: Wed, 29 Mar 2017 13:05:31 -0700
Subject: [PATCH 4/4] Add AvatarInputs::loudnessToAudioLevel
---
interface/src/ui/AvatarInputs.cpp | 34 ++++++++++++++++++-------------
interface/src/ui/AvatarInputs.h | 1 +
2 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/interface/src/ui/AvatarInputs.cpp b/interface/src/ui/AvatarInputs.cpp
index b09289c78a..49911c08bb 100644
--- a/interface/src/ui/AvatarInputs.cpp
+++ b/interface/src/ui/AvatarInputs.cpp
@@ -58,25 +58,13 @@ AvatarInputs::AvatarInputs(QQuickItem* parent) : QQuickItem(parent) {
} \
}
-void AvatarInputs::update() {
- if (!Menu::getInstance()) {
- return;
- }
- AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode()
- && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror));
- AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking));
- AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking));
- AI_UPDATE(isHMD, qApp->isHMDMode());
- AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools));
-
- auto audioIO = DependencyManager::get<AudioClient>();
+float AvatarInputs::loudnessToAudioLevel(float loudness) {
const float AUDIO_METER_AVERAGING = 0.5;
const float LOG2 = log(2.0f);
const float METER_LOUDNESS_SCALE = 2.8f / 5.0f;
const float LOG2_LOUDNESS_FLOOR = 11.0f;
float audioLevel = 0.0f;
- auto audio = DependencyManager::get<AudioClient>();
- float loudness = audio->getLastInputLoudness() + 1.0f;
+ loudness += 1.0f;
_trailingAudioLoudness = AUDIO_METER_AVERAGING * _trailingAudioLoudness + (1.0f - AUDIO_METER_AVERAGING) * loudness;
@@ -90,6 +78,24 @@ void AvatarInputs::update() {
if (audioLevel > 1.0f) {
audioLevel = 1.0;
}
+ return audioLevel;
+}
+
+void AvatarInputs::update() {
+ if (!Menu::getInstance()) {
+ return;
+ }
+
+ AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode()
+ && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror));
+ AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking));
+ AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking));
+ AI_UPDATE(isHMD, qApp->isHMDMode());
+ AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools));
+
+ auto audioIO = DependencyManager::get<AudioClient>();
+
+ const float audioLevel = loudnessToAudioLevel(DependencyManager::get<AudioClient>()->getLastInputLoudness());
AI_UPDATE_FLOAT(audioLevel, audioLevel, 0.01);
AI_UPDATE(audioClipping, ((audioIO->getTimeSinceLastClip() > 0.0f) && (audioIO->getTimeSinceLastClip() < 1.0f)));
AI_UPDATE(audioMuted, audioIO->isMuted());
diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h
index 85570ecd3c..2d203a10b9 100644
--- a/interface/src/ui/AvatarInputs.h
+++ b/interface/src/ui/AvatarInputs.h
@@ -35,6 +35,7 @@ class AvatarInputs : public QQuickItem {
public:
static AvatarInputs* getInstance();
+ float loudnessToAudioLevel(float loudness);
AvatarInputs(QQuickItem* parent = nullptr);
void update();