From 4f7cbd5b7caa70bf382673967f9906762e164ab2 Mon Sep 17 00:00:00 2001 From: Delamare2112 Date: Wed, 29 Mar 2017 12:47:58 -0700 Subject: [PATCH 1/4] Add limitless connection and scripting interface --- .../src/scripting/LimitlessConnection.cpp | 91 +++++++++++++++++++ interface/src/scripting/LimitlessConnection.h | 44 +++++++++ ...lessVoiceRecognitionScriptingInterface.cpp | 64 +++++++++++++ ...itlessVoiceRecognitionScriptingInterface.h | 50 ++++++++++ 4 files changed, 249 insertions(+) create mode 100644 interface/src/scripting/LimitlessConnection.cpp create mode 100644 interface/src/scripting/LimitlessConnection.h create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h diff --git a/interface/src/scripting/LimitlessConnection.cpp b/interface/src/scripting/LimitlessConnection.cpp new file mode 100644 index 0000000000..b9f4eacd4b --- /dev/null +++ b/interface/src/scripting/LimitlessConnection.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include "LimitlessConnection.h" +#include "LimitlessVoiceRecognitionScriptingInterface.h" + +LimitlessConnection::LimitlessConnection() : + _streamingAudioForTranscription(false) +{ +} + +void LimitlessConnection::startListening(QString authCode) { + _transcribeServerSocket.reset(new QTcpSocket(this)); + connect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this, + &LimitlessConnection::transcriptionReceived); + connect(_transcribeServerSocket.get(), &QTcpSocket::disconnected, this, [this](){stopListening();}); + + static const auto host = "gserv_devel.studiolimitless.com"; + _transcribeServerSocket->connectToHost(host, 1407); + _transcribeServerSocket->waitForConnected(); + QString requestHeader = QString::asprintf("Authorization: %s\r\nfs: %i\r\n", + authCode.toLocal8Bit().data(), AudioConstants::SAMPLE_RATE); + qCDebug(interfaceapp) << "Sending Limitless Audio Stream Request: " << requestHeader; + _transcribeServerSocket->write(requestHeader.toLocal8Bit()); + _transcribeServerSocket->waitForBytesWritten(); +} + +void LimitlessConnection::stopListening() { + emit onFinishedSpeaking(_currentTranscription); + _streamingAudioForTranscription = false; + _currentTranscription = ""; + if (!isConnected()) + return; + _transcribeServerSocket->close(); + disconnect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this, + &LimitlessConnection::transcriptionReceived); + _transcribeServerSocket.release()->deleteLater(); + disconnect(DependencyManager::get().data(), &AudioClient::inputReceived, this, + &LimitlessConnection::audioInputReceived); + qCDebug(interfaceapp) << "Connection to Limitless Voice Server closed."; +} + +void LimitlessConnection::audioInputReceived(const QByteArray& inputSamples) { + if (isConnected()) { + _transcribeServerSocket->write(inputSamples.data(), inputSamples.size()); + _transcribeServerSocket->waitForBytesWritten(); + } +} + +void LimitlessConnection::transcriptionReceived() { + while (_transcribeServerSocket && _transcribeServerSocket->bytesAvailable() > 0) { + const QByteArray data = _transcribeServerSocket->readAll(); + _serverDataBuffer.append(data); + int begin = _serverDataBuffer.indexOf('<'); + int end = _serverDataBuffer.indexOf('>'); + while (begin > -1 && end > -1) { + const int len = end - begin; + const QByteArray serverMessage = _serverDataBuffer.mid(begin+1, len-1); + if (serverMessage.contains("1407")) { + qCDebug(interfaceapp) << "Limitless Speech Server denied the request."; + // Don't spam the server with further false requests please. + DependencyManager::get()->setListeningToVoice(true); + stopListening(); + return; + } else if (serverMessage.contains("1408")) { + qCDebug(interfaceapp) << "Limitless Audio request authenticated!"; + _serverDataBuffer.clear(); + connect(DependencyManager::get().data(), &AudioClient::inputReceived, this, + &LimitlessConnection::audioInputReceived); + return; + } + QJsonObject json = QJsonDocument::fromJson(serverMessage.data()).object(); + _serverDataBuffer.remove(begin, len+1); + _currentTranscription = json["alternatives"].toArray()[0].toObject()["transcript"].toString(); + emit onReceivedTranscription(_currentTranscription); + if (json["isFinal"] == true) { + qCDebug(interfaceapp) << "Final transcription: " << _currentTranscription; + stopListening(); + return; + } + begin = _serverDataBuffer.indexOf('<'); + end = _serverDataBuffer.indexOf('>'); + } + } +} + +bool LimitlessConnection::isConnected() const { + return _transcribeServerSocket.get() && _transcribeServerSocket->isWritable() + && _transcribeServerSocket->state() != QAbstractSocket::SocketState::UnconnectedState; +} diff --git a/interface/src/scripting/LimitlessConnection.h b/interface/src/scripting/LimitlessConnection.h new file mode 100644 index 0000000000..ee049aff8e --- /dev/null +++ b/interface/src/scripting/LimitlessConnection.h @@ -0,0 +1,44 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/24/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_LimitlessConnection_h +#define hifi_LimitlessConnection_h + +#include +#include +#include + +class LimitlessConnection : public QObject { + Q_OBJECT +public: + LimitlessConnection(); + + Q_INVOKABLE void startListening(QString authCode); + Q_INVOKABLE void stopListening(); + + std::atomic _streamingAudioForTranscription; + +signals: + void onReceivedTranscription(QString speech); + void onFinishedSpeaking(QString speech); + +private: + void transcriptionReceived(); + void audioInputReceived(const QByteArray& inputSamples); + + bool isConnected() const; + + std::unique_ptr _transcribeServerSocket; + QByteArray _serverDataBuffer; + QString _currentTranscription; +}; + +#endif //hifi_LimitlessConnection_h diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp new file mode 100644 index 0000000000..1352630f84 --- /dev/null +++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp @@ -0,0 +1,64 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/20/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include +#include "LimitlessVoiceRecognitionScriptingInterface.h" + +const float LimitlessVoiceRecognitionScriptingInterface::_audioLevelThreshold = 0.33f; +const int LimitlessVoiceRecognitionScriptingInterface::_voiceTimeoutDuration = 2000; + +LimitlessVoiceRecognitionScriptingInterface::LimitlessVoiceRecognitionScriptingInterface() : + _shouldStartListeningForVoice(false) +{ + _voiceTimer.setSingleShot(true); + connect(&_voiceTimer, &QTimer::timeout, this, &LimitlessVoiceRecognitionScriptingInterface::voiceTimeout); + connect(&_connection, &LimitlessConnection::onReceivedTranscription, this, [this](QString transcription){emit onReceivedTranscription(transcription);}); + connect(&_connection, &LimitlessConnection::onFinishedSpeaking, this, [this](QString transcription){emit onFinishedSpeaking(transcription);}); + _connection.moveToThread(&_connectionThread); + _connectionThread.setObjectName("Limitless Connection"); + _connectionThread.start(); +} + +void LimitlessVoiceRecognitionScriptingInterface::update() { + const float audioLevel = AvatarInputs::getInstance()->loudnessToAudioLevel(DependencyManager::get()->getAudioAverageInputLoudness()); + + if (_shouldStartListeningForVoice) { + if (_connection._streamingAudioForTranscription) { + if (audioLevel > _audioLevelThreshold) { + if (_voiceTimer.isActive()) { + _voiceTimer.stop(); + } + } else if (!_voiceTimer.isActive()){ + _voiceTimer.start(_voiceTimeoutDuration); + } + } else if (audioLevel > _audioLevelThreshold) { + // to make sure invoke doesn't get called twice before the method actually gets called + _connection._streamingAudioForTranscription = true; + QMetaObject::invokeMethod(&_connection, "startListening", Q_ARG(QString, authCode)); + } + } +} + +void LimitlessVoiceRecognitionScriptingInterface::setListeningToVoice(bool listening) { + _shouldStartListeningForVoice = listening; +} + +void LimitlessVoiceRecognitionScriptingInterface::setAuthKey(QString key) { + authCode = key; +} + +void LimitlessVoiceRecognitionScriptingInterface::voiceTimeout() { + if (_connection._streamingAudioForTranscription) { + QMetaObject::invokeMethod(&_connection, "stopListening"); + } +} diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h new file mode 100644 index 0000000000..d1b1139695 --- /dev/null +++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h @@ -0,0 +1,50 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/20/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SpeechRecognitionScriptingInterface_h +#define hifi_SpeechRecognitionScriptingInterface_h + +#include +#include +#include +#include "LimitlessConnection.h" + +class LimitlessVoiceRecognitionScriptingInterface : public QObject, public Dependency { + Q_OBJECT +public: + LimitlessVoiceRecognitionScriptingInterface(); + + void update(); + + QString authCode; + +public slots: + void setListeningToVoice(bool listening); + void setAuthKey(QString key); + +signals: + void onReceivedTranscription(QString speech); + void onFinishedSpeaking(QString speech); + +private: + + bool _shouldStartListeningForVoice; + static const float _audioLevelThreshold; + static const int _voiceTimeoutDuration; + + QTimer _voiceTimer; + QThread _connectionThread; + LimitlessConnection _connection; + + void voiceTimeout(); +}; + +#endif //hifi_SpeechRecognitionScriptingInterface_h From db11c40a08daa38ca3f2b6fb2f86b3f7d511f2ba Mon Sep 17 00:00:00 2001 From: Delamare2112 Date: Wed, 29 Mar 2017 12:54:25 -0700 Subject: [PATCH 2/4] Add AnimationCache to Agent --- assignment-client/src/Agent.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 355e47be46..baed421df7 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -403,6 +403,7 @@ void Agent::executeScript() { _scriptEngine->registerGlobalObject("Agent", this); _scriptEngine->registerGlobalObject("SoundCache", DependencyManager::get().data()); + _scriptEngine->registerGlobalObject("AnimationCache", DependencyManager::get().data()); QScriptValue webSocketServerConstructorValue = _scriptEngine->newFunction(WebSocketServerClass::constructor); _scriptEngine->globalObject().setProperty("WebSocketServer", webSocketServerConstructorValue); From 2f6add3398a1b25747a1678f552699c0e5d73087 Mon Sep 17 00:00:00 2001 From: Delamare2112 Date: Wed, 29 Mar 2017 12:56:08 -0700 Subject: [PATCH 3/4] Expose LimitlessSpeechRecognition to scripts --- interface/src/Application.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index d48fe19a99..9f4e4a6bc6 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -177,6 +177,7 @@ #include "FrameTimingsScriptingInterface.h" #include #include +#include // On Windows PC, NVidia Optimus laptop, we want to enable NVIDIA GPU // FIXME seems to be broken. @@ -522,6 +523,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); return previousSessionCrashed; } @@ -4557,6 +4559,8 @@ void Application::update(float deltaTime) { } AnimDebugDraw::getInstance().update(); + + DependencyManager::get()->update(); } void Application::sendAvatarViewFrustum() { @@ -5548,6 +5552,8 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerGlobalObject("UserActivityLogger", DependencyManager::get().data()); scriptEngine->registerGlobalObject("Users", DependencyManager::get().data()); + scriptEngine->registerGlobalObject("LimitlessSpeechRecognition", DependencyManager::get().data()); + if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) { scriptEngine->registerGlobalObject("Steam", new SteamScriptingInterface(scriptEngine, steamClient.get())); } From 3ebe1a0015364520104b918d89730e2229fa02dc Mon Sep 17 00:00:00 2001 From: Delamare2112 Date: Wed, 29 Mar 2017 13:05:31 -0700 Subject: [PATCH 4/4] Add AvatarInputs::loudnessToAudioLevel --- interface/src/ui/AvatarInputs.cpp | 34 ++++++++++++++++++------------- interface/src/ui/AvatarInputs.h | 1 + 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/interface/src/ui/AvatarInputs.cpp b/interface/src/ui/AvatarInputs.cpp index b09289c78a..49911c08bb 100644 --- a/interface/src/ui/AvatarInputs.cpp +++ b/interface/src/ui/AvatarInputs.cpp @@ -58,25 +58,13 @@ AvatarInputs::AvatarInputs(QQuickItem* parent) : QQuickItem(parent) { } \ } -void AvatarInputs::update() { - if (!Menu::getInstance()) { - return; - } - AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode() - && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror)); - AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking)); - AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking)); - AI_UPDATE(isHMD, qApp->isHMDMode()); - AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools)); - - auto audioIO = DependencyManager::get(); +float AvatarInputs::loudnessToAudioLevel(float loudness) { const float AUDIO_METER_AVERAGING = 0.5; const float LOG2 = log(2.0f); const float METER_LOUDNESS_SCALE = 2.8f / 5.0f; const float LOG2_LOUDNESS_FLOOR = 11.0f; float audioLevel = 0.0f; - auto audio = DependencyManager::get(); - float loudness = audio->getLastInputLoudness() + 1.0f; + loudness += 1.0f; _trailingAudioLoudness = AUDIO_METER_AVERAGING * _trailingAudioLoudness + (1.0f - AUDIO_METER_AVERAGING) * loudness; @@ -90,6 +78,24 @@ void AvatarInputs::update() { if (audioLevel > 1.0f) { audioLevel = 1.0; } + return audioLevel; +} + +void AvatarInputs::update() { + if (!Menu::getInstance()) { + return; + } + + AI_UPDATE(mirrorVisible, Menu::getInstance()->isOptionChecked(MenuOption::MiniMirror) && !qApp->isHMDMode() + && !Menu::getInstance()->isOptionChecked(MenuOption::FullscreenMirror)); + AI_UPDATE(cameraEnabled, !Menu::getInstance()->isOptionChecked(MenuOption::NoFaceTracking)); + AI_UPDATE(cameraMuted, Menu::getInstance()->isOptionChecked(MenuOption::MuteFaceTracking)); + AI_UPDATE(isHMD, qApp->isHMDMode()); + AI_UPDATE(showAudioTools, Menu::getInstance()->isOptionChecked(MenuOption::AudioTools)); + + auto audioIO = DependencyManager::get(); + + const float audioLevel = loudnessToAudioLevel(DependencyManager::get()->getLastInputLoudness()); AI_UPDATE_FLOAT(audioLevel, audioLevel, 0.01); AI_UPDATE(audioClipping, ((audioIO->getTimeSinceLastClip() > 0.0f) && (audioIO->getTimeSinceLastClip() < 1.0f))); AI_UPDATE(audioMuted, audioIO->isMuted()); diff --git a/interface/src/ui/AvatarInputs.h b/interface/src/ui/AvatarInputs.h index 85570ecd3c..2d203a10b9 100644 --- a/interface/src/ui/AvatarInputs.h +++ b/interface/src/ui/AvatarInputs.h @@ -35,6 +35,7 @@ class AvatarInputs : public QQuickItem { public: static AvatarInputs* getInstance(); + float loudnessToAudioLevel(float loudness); AvatarInputs(QQuickItem* parent = nullptr); void update();