Add limitless connection and scripting interface

interface/src/scripting/LimitlessConnection.cpp

@@ -0,0 +1,91 @@
+#include <QJsonDocument>
+#include <QJsonArray>
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include "LimitlessConnection.h"
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+LimitlessConnection::LimitlessConnection() :
+        _streamingAudioForTranscription(false)
+{
+}
+
+void LimitlessConnection::startListening(QString authCode) {
+    _transcribeServerSocket.reset(new QTcpSocket(this));
+    connect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+            &LimitlessConnection::transcriptionReceived);
+    connect(_transcribeServerSocket.get(), &QTcpSocket::disconnected, this, [this](){stopListening();});
+
+    static const auto host = "gserv_devel.studiolimitless.com";
+    _transcribeServerSocket->connectToHost(host, 1407);
+    _transcribeServerSocket->waitForConnected();
+    QString requestHeader = QString::asprintf("Authorization: %s\r\nfs: %i\r\n",
+                                              authCode.toLocal8Bit().data(), AudioConstants::SAMPLE_RATE);
+    qCDebug(interfaceapp) << "Sending Limitless Audio Stream Request: " << requestHeader;
+    _transcribeServerSocket->write(requestHeader.toLocal8Bit());
+    _transcribeServerSocket->waitForBytesWritten();
+}
+
+void LimitlessConnection::stopListening() {
+    emit onFinishedSpeaking(_currentTranscription);
+    _streamingAudioForTranscription = false;
+    _currentTranscription = "";
+    if (!isConnected())
+        return;
+    _transcribeServerSocket->close();
+    disconnect(_transcribeServerSocket.get(), &QTcpSocket::readyRead, this,
+            &LimitlessConnection::transcriptionReceived);
+    _transcribeServerSocket.release()->deleteLater();
+    disconnect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+            &LimitlessConnection::audioInputReceived);
+    qCDebug(interfaceapp) << "Connection to Limitless Voice Server closed.";
+}
+
+void LimitlessConnection::audioInputReceived(const QByteArray& inputSamples) {
+    if (isConnected()) {
+        _transcribeServerSocket->write(inputSamples.data(), inputSamples.size());
+        _transcribeServerSocket->waitForBytesWritten();
+    }
+}
+
+void LimitlessConnection::transcriptionReceived() {
+    while (_transcribeServerSocket && _transcribeServerSocket->bytesAvailable() > 0) {
+        const QByteArray data = _transcribeServerSocket->readAll();
+        _serverDataBuffer.append(data);
+        int begin = _serverDataBuffer.indexOf('<');
+        int end = _serverDataBuffer.indexOf('>');
+        while (begin > -1 && end > -1) {
+            const int len = end - begin;
+            const QByteArray serverMessage = _serverDataBuffer.mid(begin+1, len-1);
+            if (serverMessage.contains("1407")) {
+                qCDebug(interfaceapp) << "Limitless Speech Server denied the request.";
+                // Don't spam the server with further false requests please.
+                DependencyManager::get<LimitlessVoiceRecognitionScriptingInterface>()->setListeningToVoice(true);
+                stopListening();
+                return;
+            } else if (serverMessage.contains("1408")) {
+                qCDebug(interfaceapp) << "Limitless Audio request authenticated!";
+                _serverDataBuffer.clear();
+                connect(DependencyManager::get<AudioClient>().data(), &AudioClient::inputReceived, this,
+                        &LimitlessConnection::audioInputReceived);
+                return;
+            }
+            QJsonObject json = QJsonDocument::fromJson(serverMessage.data()).object();
+            _serverDataBuffer.remove(begin, len+1);
+            _currentTranscription = json["alternatives"].toArray()[0].toObject()["transcript"].toString();
+            emit onReceivedTranscription(_currentTranscription);
+            if (json["isFinal"] == true) {
+                qCDebug(interfaceapp) << "Final transcription: " << _currentTranscription;
+                stopListening();
+                return;
+            }
+            begin = _serverDataBuffer.indexOf('<');
+            end = _serverDataBuffer.indexOf('>');
+        }
+    }
+}
+
+bool LimitlessConnection::isConnected() const {
+    return _transcribeServerSocket.get() && _transcribeServerSocket->isWritable()
+    && _transcribeServerSocket->state() != QAbstractSocket::SocketState::UnconnectedState;
+}

interface/src/scripting/LimitlessConnection.h

@@ -0,0 +1,44 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/24/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_LimitlessConnection_h
+#define hifi_LimitlessConnection_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+
+class LimitlessConnection : public QObject {
+    Q_OBJECT
+public:
+    LimitlessConnection();
+
+    Q_INVOKABLE void startListening(QString authCode);
+    Q_INVOKABLE void stopListening();
+
+    std::atomic<bool> _streamingAudioForTranscription;
+
+signals:
+    void onReceivedTranscription(QString speech);
+    void onFinishedSpeaking(QString speech);
+
+private:
+    void transcriptionReceived();
+    void audioInputReceived(const QByteArray& inputSamples);
+
+    bool isConnected() const;
+
+    std::unique_ptr<QTcpSocket> _transcribeServerSocket;
+    QByteArray _serverDataBuffer;
+    QString _currentTranscription;
+};
+
+#endif //hifi_LimitlessConnection_h

interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.cpp

@@ -0,0 +1,64 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/20/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <src/InterfaceLogging.h>
+#include <src/ui/AvatarInputs.h>
+#include <QtConcurrent/QtConcurrentRun>
+#include "LimitlessVoiceRecognitionScriptingInterface.h"
+
+const float LimitlessVoiceRecognitionScriptingInterface::_audioLevelThreshold = 0.33f;
+const int LimitlessVoiceRecognitionScriptingInterface::_voiceTimeoutDuration = 2000;
+
+LimitlessVoiceRecognitionScriptingInterface::LimitlessVoiceRecognitionScriptingInterface() :
+        _shouldStartListeningForVoice(false)
+{
+    _voiceTimer.setSingleShot(true);
+    connect(&_voiceTimer, &QTimer::timeout, this, &LimitlessVoiceRecognitionScriptingInterface::voiceTimeout);
+    connect(&_connection, &LimitlessConnection::onReceivedTranscription, this, [this](QString transcription){emit onReceivedTranscription(transcription);});
+    connect(&_connection, &LimitlessConnection::onFinishedSpeaking, this, [this](QString transcription){emit onFinishedSpeaking(transcription);});
+    _connection.moveToThread(&_connectionThread);
+    _connectionThread.setObjectName("Limitless Connection");
+    _connectionThread.start();
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::update() {
+    const float audioLevel = AvatarInputs::getInstance()->loudnessToAudioLevel(DependencyManager::get<AudioClient>()->getAudioAverageInputLoudness());
+
+    if (_shouldStartListeningForVoice) {
+        if (_connection._streamingAudioForTranscription) {
+            if (audioLevel > _audioLevelThreshold) {
+                if (_voiceTimer.isActive()) {
+                    _voiceTimer.stop();
+                }
+            } else if (!_voiceTimer.isActive()){
+                _voiceTimer.start(_voiceTimeoutDuration);
+            }
+        } else if (audioLevel > _audioLevelThreshold) {
+            // to make sure invoke doesn't get called twice before the method actually gets called
+            _connection._streamingAudioForTranscription = true;
+            QMetaObject::invokeMethod(&_connection, "startListening", Q_ARG(QString, authCode));
+        }
+    }
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setListeningToVoice(bool listening) {
+    _shouldStartListeningForVoice = listening;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::setAuthKey(QString key) {
+    authCode = key;
+}
+
+void LimitlessVoiceRecognitionScriptingInterface::voiceTimeout() {
+    if (_connection._streamingAudioForTranscription) {
+        QMetaObject::invokeMethod(&_connection, "stopListening");
+    }
+}

interface/src/scripting/LimitlessVoiceRecognitionScriptingInterface.h

@@ -0,0 +1,50 @@
+//
+//  SpeechRecognitionScriptingInterface.h
+//  interface/src/scripting
+//
+//  Created by Trevor Berninger on 3/20/17.
+//  Copyright 2017 Limitless ltd.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_SpeechRecognitionScriptingInterface_h
+#define hifi_SpeechRecognitionScriptingInterface_h
+
+#include <AudioClient.h>
+#include <QObject>
+#include <QFuture>
+#include "LimitlessConnection.h"
+
+class LimitlessVoiceRecognitionScriptingInterface : public QObject, public Dependency {
+    Q_OBJECT
+public:
+    LimitlessVoiceRecognitionScriptingInterface();
+
+    void update();
+
+    QString authCode;
+
+public slots:
+    void setListeningToVoice(bool listening);
+    void setAuthKey(QString key);
+
+signals:
+    void onReceivedTranscription(QString speech);
+    void onFinishedSpeaking(QString speech);
+
+private:
+
+    bool _shouldStartListeningForVoice;
+    static const float _audioLevelThreshold;
+    static const int _voiceTimeoutDuration;
+
+    QTimer _voiceTimer;
+    QThread _connectionThread;
+    LimitlessConnection _connection;
+
+    void voiceTimeout();
+};
+
+#endif //hifi_SpeechRecognitionScriptingInterface_h