From 4f7cbd5b7caa70bf382673967f9906762e164ab2 Mon Sep 17 00:00:00 2001 From: Delamare2112 Date: Wed, 29 Mar 2017 12:47:58 -0700 Subject: [PATCH] Add limitless connection and scripting interface --- .../src/scripting/LimitlessConnection.cpp | 91 +++++++++++++++++++ interface/src/scripting/LimitlessConnection.h | 44 +++++++++ ...lessVoiceRecognitionScriptingInterface.cpp | 64 +++++++++++++ ...itlessVoiceRecognitionScriptingInterface.h | 50 ++++++++++ 4 files changed, 249 insertions(+) create mode 100644 interface/src/scripting/LimitlessConnection.cpp create mode 100644 interface/src/scripting/LimitlessConnection.h create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp create mode 100644 interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h diff --git a/interface/src/scripting/LimitlessConnection.cpp b/interface/src/scripting/LimitlessConnection.cpp new file mode 100644 index 0000000000..b9f4eacd4b --- /dev/null +++ b/interface/src/scripting/LimitlessConnection.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include "LimitlessConnection.h" +#include "LimitlessVoiceRecognitionScriptingInterface.h" + +LimitlessConnection::LimitlessConnection() : + _streamingAudioForTranscription(false) +{ +} + +void LimitlessConnection::startListening(QString authCode) { + _transcribeServerSocket.reset(new QTcpSocket(this)); + connect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this, + &LimitlessConnection::transcriptionReceived); + connect(_transcribeServerSocket.get(), &QTcpSocket::disconnected, this, [this](){stopListening();}); + + static const auto host = "gserv_devel.studiolimitless.com"; + _transcribeServerSocket->connectToHost(host, 1407); + _transcribeServerSocket->waitForConnected(); + QString requestHeader = QString::asprintf("Authorization: %s\r\nfs: %i\r\n", + authCode.toLocal8Bit().data(), AudioConstants::SAMPLE_RATE); + qCDebug(interfaceapp) << "Sending Limitless Audio Stream Request: " << requestHeader; + _transcribeServerSocket->write(requestHeader.toLocal8Bit()); + _transcribeServerSocket->waitForBytesWritten(); +} + +void LimitlessConnection::stopListening() { + emit onFinishedSpeaking(_currentTranscription); + _streamingAudioForTranscription = false; + _currentTranscription = ""; + if (!isConnected()) + return; + _transcribeServerSocket->close(); + disconnect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this, + &LimitlessConnection::transcriptionReceived); + _transcribeServerSocket.release()->deleteLater(); + disconnect(DependencyManager::get().data(), &AudioClient::inputReceived, this, + &LimitlessConnection::audioInputReceived); + qCDebug(interfaceapp) << "Connection to Limitless Voice Server closed."; +} + +void LimitlessConnection::audioInputReceived(const QByteArray& inputSamples) { + if (isConnected()) { + _transcribeServerSocket->write(inputSamples.data(), inputSamples.size()); + _transcribeServerSocket->waitForBytesWritten(); + } +} + +void LimitlessConnection::transcriptionReceived() { + while (_transcribeServerSocket && _transcribeServerSocket->bytesAvailable() > 0) { + const QByteArray data = _transcribeServerSocket->readAll(); + _serverDataBuffer.append(data); + int begin = _serverDataBuffer.indexOf('<'); + int end = _serverDataBuffer.indexOf('>'); + while (begin > -1 && end > -1) { + const int len = end - begin; + const QByteArray serverMessage = _serverDataBuffer.mid(begin+1, len-1); + if (serverMessage.contains("1407")) { + qCDebug(interfaceapp) << "Limitless Speech Server denied the request."; + // Don't spam the server with further false requests please. + DependencyManager::get()->setListeningToVoice(true); + stopListening(); + return; + } else if (serverMessage.contains("1408")) { + qCDebug(interfaceapp) << "Limitless Audio request authenticated!"; + _serverDataBuffer.clear(); + connect(DependencyManager::get().data(), &AudioClient::inputReceived, this, + &LimitlessConnection::audioInputReceived); + return; + } + QJsonObject json = QJsonDocument::fromJson(serverMessage.data()).object(); + _serverDataBuffer.remove(begin, len+1); + _currentTranscription = json["alternatives"].toArray()[0].toObject()["transcript"].toString(); + emit onReceivedTranscription(_currentTranscription); + if (json["isFinal"] == true) { + qCDebug(interfaceapp) << "Final transcription: " << _currentTranscription; + stopListening(); + return; + } + begin = _serverDataBuffer.indexOf('<'); + end = _serverDataBuffer.indexOf('>'); + } + } +} + +bool LimitlessConnection::isConnected() const { + return _transcribeServerSocket.get() && _transcribeServerSocket->isWritable() + && _transcribeServerSocket->state() != QAbstractSocket::SocketState::UnconnectedState; +} diff --git a/interface/src/scripting/LimitlessConnection.h b/interface/src/scripting/LimitlessConnection.h new file mode 100644 index 0000000000..ee049aff8e --- /dev/null +++ b/interface/src/scripting/LimitlessConnection.h @@ -0,0 +1,44 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/24/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_LimitlessConnection_h +#define hifi_LimitlessConnection_h + +#include +#include +#include + +class LimitlessConnection : public QObject { + Q_OBJECT +public: + LimitlessConnection(); + + Q_INVOKABLE void startListening(QString authCode); + Q_INVOKABLE void stopListening(); + + std::atomic _streamingAudioForTranscription; + +signals: + void onReceivedTranscription(QString speech); + void onFinishedSpeaking(QString speech); + +private: + void transcriptionReceived(); + void audioInputReceived(const QByteArray& inputSamples); + + bool isConnected() const; + + std::unique_ptr _transcribeServerSocket; + QByteArray _serverDataBuffer; + QString _currentTranscription; +}; + +#endif //hifi_LimitlessConnection_h diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp new file mode 100644 index 0000000000..1352630f84 --- /dev/null +++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp @@ -0,0 +1,64 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/20/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include +#include +#include "LimitlessVoiceRecognitionScriptingInterface.h" + +const float LimitlessVoiceRecognitionScriptingInterface::_audioLevelThreshold = 0.33f; +const int LimitlessVoiceRecognitionScriptingInterface::_voiceTimeoutDuration = 2000; + +LimitlessVoiceRecognitionScriptingInterface::LimitlessVoiceRecognitionScriptingInterface() : + _shouldStartListeningForVoice(false) +{ + _voiceTimer.setSingleShot(true); + connect(&_voiceTimer, &QTimer::timeout, this, &LimitlessVoiceRecognitionScriptingInterface::voiceTimeout); + connect(&_connection, &LimitlessConnection::onReceivedTranscription, this, [this](QString transcription){emit onReceivedTranscription(transcription);}); + connect(&_connection, &LimitlessConnection::onFinishedSpeaking, this, [this](QString transcription){emit onFinishedSpeaking(transcription);}); + _connection.moveToThread(&_connectionThread); + _connectionThread.setObjectName("Limitless Connection"); + _connectionThread.start(); +} + +void LimitlessVoiceRecognitionScriptingInterface::update() { + const float audioLevel = AvatarInputs::getInstance()->loudnessToAudioLevel(DependencyManager::get()->getAudioAverageInputLoudness()); + + if (_shouldStartListeningForVoice) { + if (_connection._streamingAudioForTranscription) { + if (audioLevel > _audioLevelThreshold) { + if (_voiceTimer.isActive()) { + _voiceTimer.stop(); + } + } else if (!_voiceTimer.isActive()){ + _voiceTimer.start(_voiceTimeoutDuration); + } + } else if (audioLevel > _audioLevelThreshold) { + // to make sure invoke doesn't get called twice before the method actually gets called + _connection._streamingAudioForTranscription = true; + QMetaObject::invokeMethod(&_connection, "startListening", Q_ARG(QString, authCode)); + } + } +} + +void LimitlessVoiceRecognitionScriptingInterface::setListeningToVoice(bool listening) { + _shouldStartListeningForVoice = listening; +} + +void LimitlessVoiceRecognitionScriptingInterface::setAuthKey(QString key) { + authCode = key; +} + +void LimitlessVoiceRecognitionScriptingInterface::voiceTimeout() { + if (_connection._streamingAudioForTranscription) { + QMetaObject::invokeMethod(&_connection, "stopListening"); + } +} diff --git a/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h new file mode 100644 index 0000000000..d1b1139695 --- /dev/null +++ b/interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h @@ -0,0 +1,50 @@ +// +// SpeechRecognitionScriptingInterface.h +// interface/src/scripting +// +// Created by Trevor Berninger on 3/20/17. +// Copyright 2017 Limitless ltd. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SpeechRecognitionScriptingInterface_h +#define hifi_SpeechRecognitionScriptingInterface_h + +#include +#include +#include +#include "LimitlessConnection.h" + +class LimitlessVoiceRecognitionScriptingInterface : public QObject, public Dependency { + Q_OBJECT +public: + LimitlessVoiceRecognitionScriptingInterface(); + + void update(); + + QString authCode; + +public slots: + void setListeningToVoice(bool listening); + void setAuthKey(QString key); + +signals: + void onReceivedTranscription(QString speech); + void onFinishedSpeaking(QString speech); + +private: + + bool _shouldStartListeningForVoice; + static const float _audioLevelThreshold; + static const int _voiceTimeoutDuration; + + QTimer _voiceTimer; + QThread _connectionThread; + LimitlessConnection _connection; + + void voiceTimeout(); +}; + +#endif //hifi_SpeechRecognitionScriptingInterface_h