From 7de784ce27d6c9d41e3847e74caf778d90b71c87 Mon Sep 17 00:00:00 2001 From: Zach Fox Date: Wed, 10 Oct 2018 12:32:55 -0700 Subject: [PATCH] First steps (definitely not working) --- assignment-client/src/Agent.cpp | 2 +- interface/src/Application.cpp | 14 ++- interface/src/assets/ATPAssetMigrator.cpp | 4 +- .../scripting/SpeechScriptingInterface.cpp | 96 +++++++++++++++++++ .../src/scripting/SpeechScriptingInterface.h | 76 +++++++++++++++ .../src/RenderableWebEntityItem.cpp | 2 +- libraries/entities/src/EntityEditFilters.cpp | 2 +- .../src/model-networking/TextureCache.cpp | 4 +- libraries/networking/src/AddressManager.cpp | 10 +- libraries/networking/src/DomainHandler.cpp | 2 +- .../networking/src/NetworkingConstants.h | 10 +- libraries/networking/src/ResourceCache.cpp | 2 +- libraries/networking/src/ResourceManager.cpp | 18 ++-- 13 files changed, 209 insertions(+), 33 deletions(-) create mode 100644 interface/src/scripting/SpeechScriptingInterface.cpp create mode 100644 interface/src/scripting/SpeechScriptingInterface.h diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 639e9f924b..ee21fff8c0 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -216,7 +216,7 @@ void Agent::requestScript() { } // make sure this is not a script request for the file scheme - if (scriptURL.scheme() == URL_SCHEME_FILE) { + if (scriptURL.scheme() == HIFI_URL_SCHEME_FILE) { qWarning() << "Cannot load script for Agent from local filesystem."; scriptRequestFinished(); return; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index aa2b382c58..74532ef53a 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -182,6 +182,7 @@ #include "scripting/RatesScriptingInterface.h" #include "scripting/SelectionScriptingInterface.h" #include "scripting/WalletScriptingInterface.h" +#include "scripting/SpeechScriptingInterface.h" #if defined(Q_OS_MAC) || defined(Q_OS_WIN) #include "SpeechRecognizer.h" #endif @@ -528,11 +529,11 @@ bool isDomainURL(QUrl url) { if (url.scheme() == URL_SCHEME_HIFI) { return true; } - if (url.scheme() != URL_SCHEME_FILE) { + if (url.scheme() != HIFI_URL_SCHEME_FILE) { // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can // be loaded over http(s) - // && url.scheme() != URL_SCHEME_HTTP && - // url.scheme() != URL_SCHEME_HTTPS + // && url.scheme() != HIFI_URL_SCHEME_HTTP && + // url.scheme() != HIFI_URL_SCHEME_HTTPS return false; } if (url.path().endsWith(".json", Qt::CaseInsensitive) || @@ -943,6 +944,7 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); @@ -1024,8 +1026,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo // If the URL scheme is http(s) or ftp, then use as is, else - treat it as a local file // This is done so as not break previous command line scripts - if (testScriptPath.left(URL_SCHEME_HTTP.length()) == URL_SCHEME_HTTP || - testScriptPath.left(URL_SCHEME_FTP.length()) == URL_SCHEME_FTP) { + if (testScriptPath.left(HIFI_URL_SCHEME_HTTP.length()) == HIFI_URL_SCHEME_HTTP || + testScriptPath.left(HIFI_URL_SCHEME_FTP.length()) == HIFI_URL_SCHEME_FTP) { setProperty(hifi::properties::TEST, QUrl::fromUserInput(testScriptPath)); } else if (QFileInfo(testScriptPath).exists()) { @@ -3127,6 +3129,7 @@ void Application::onDesktopRootContextCreated(QQmlContext* surfaceContext) { surfaceContext->setContextProperty("ContextOverlay", DependencyManager::get().data()); surfaceContext->setContextProperty("Wallet", DependencyManager::get().data()); surfaceContext->setContextProperty("HiFiAbout", AboutUtil::getInstance()); + surfaceContext->setContextProperty("Speech", DependencyManager::get().data()); if (auto steamClient = PluginManager::getInstance()->getSteamClientPlugin()) { surfaceContext->setContextProperty("Steam", new SteamScriptingInterface(engine, steamClient.get())); @@ -6797,6 +6800,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEnginePointe scriptEngine->registerGlobalObject("Wallet", DependencyManager::get().data()); scriptEngine->registerGlobalObject("AddressManager", DependencyManager::get().data()); scriptEngine->registerGlobalObject("HifiAbout", AboutUtil::getInstance()); + scriptEngine->registerGlobalObject("Speech", DependencyManager::get().data()); qScriptRegisterMetaType(scriptEngine.data(), OverlayIDtoScriptValue, OverlayIDfromScriptValue); diff --git a/interface/src/assets/ATPAssetMigrator.cpp b/interface/src/assets/ATPAssetMigrator.cpp index 45ac80b054..423a4b8509 100644 --- a/interface/src/assets/ATPAssetMigrator.cpp +++ b/interface/src/assets/ATPAssetMigrator.cpp @@ -121,8 +121,8 @@ void ATPAssetMigrator::loadEntityServerFile() { QUrl migrationURL = QUrl(migrationURLString); if (!_ignoredUrls.contains(migrationURL) - && (migrationURL.scheme() == URL_SCHEME_HTTP || migrationURL.scheme() == URL_SCHEME_HTTPS - || migrationURL.scheme() == URL_SCHEME_FILE || migrationURL.scheme() == URL_SCHEME_FTP)) { + && (migrationURL.scheme() == HIFI_URL_SCHEME_HTTP || migrationURL.scheme() == HIFI_URL_SCHEME_HTTPS + || migrationURL.scheme() == HIFI_URL_SCHEME_FILE || migrationURL.scheme() == HIFI_URL_SCHEME_FTP)) { if (_pendingReplacements.contains(migrationURL)) { // we already have a request out for this asset, just store the QJsonValueRef diff --git a/interface/src/scripting/SpeechScriptingInterface.cpp b/interface/src/scripting/SpeechScriptingInterface.cpp new file mode 100644 index 0000000000..a38e1aa824 --- /dev/null +++ b/interface/src/scripting/SpeechScriptingInterface.cpp @@ -0,0 +1,96 @@ +// +// SpeechScriptingInterface.cpp +// interface/src/scripting +// +// Created by Zach Fox on 2018-10-10. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SpeechScriptingInterface.h" +#include "avatar/AvatarManager.h" +#include + +SpeechScriptingInterface::SpeechScriptingInterface() { + // + // Create text to speech engine + // + HRESULT hr = m_tts.CoCreateInstance(CLSID_SpVoice); + if (FAILED(hr)) { + ATLTRACE(TEXT("Text-to-speech creation failed.\n")); + AtlThrow(hr); + } + + // + // Get token corresponding to default voice + // + hr = SpGetDefaultTokenFromCategoryId(SPCAT_VOICES, &m_voiceToken, FALSE); + if (FAILED(hr)) { + ATLTRACE(TEXT("Can't get default voice token.\n")); + AtlThrow(hr); + } + + // + // Set default voice + // + hr = m_tts->SetVoice(m_voiceToken); + if (FAILED(hr)) { + ATLTRACE(TEXT("Can't set default voice.\n")); + AtlThrow(hr); + } + + WAVEFORMATEX fmt; + fmt.wFormatTag = WAVE_FORMAT_PCM; + fmt.nSamplesPerSec = 48000; + fmt.wBitsPerSample = 16; + fmt.nChannels = 1; + fmt.nBlockAlign = fmt.nChannels * fmt.wBitsPerSample / 8; + fmt.nAvgBytesPerSec = fmt.nSamplesPerSec * fmt.nBlockAlign; + fmt.cbSize = 0; + + BYTE* pcontent = new BYTE[1024 * 1000]; + + cpIStream = SHCreateMemStream(NULL, 0); + hr = outputStream->SetBaseStream(cpIStream, SPDFID_WaveFormatEx, &fmt); + + hr = m_tts->SetOutput(outputStream, true); + if (FAILED(hr)) { + ATLTRACE(TEXT("Can't set output stream.\n")); + AtlThrow(hr); + } +} + +SpeechScriptingInterface::~SpeechScriptingInterface() { + +} + +void SpeechScriptingInterface::speakText(const QString& textToSpeak) { + ULONG streamNumber; + HRESULT hr = m_tts->Speak(reinterpret_cast(textToSpeak.utf16()), + SPF_IS_NOT_XML | SPF_ASYNC | SPF_PURGEBEFORESPEAK, + &streamNumber); + if (FAILED(hr)) { + ATLTRACE(TEXT("Speak failed.\n")); + AtlThrow(hr); + } + + m_tts->WaitUntilDone(-1); + + outputStream->GetBaseStream(&cpIStream); + ULARGE_INTEGER StreamSize; + StreamSize.LowPart = 0; + hr = IStream_Size(cpIStream, &StreamSize); + + DWORD dwSize = StreamSize.QuadPart; + char* buf1 = new char[dwSize + 1]; + hr = IStream_Read(cpIStream, buf1, dwSize); + + QByteArray byteArray = QByteArray::QByteArray(buf1, (int)dwSize); + AudioInjectorOptions options; + + options.position = DependencyManager::get()->getMyAvatarPosition(); + + AudioInjector::playSound(byteArray, options); +} diff --git a/interface/src/scripting/SpeechScriptingInterface.h b/interface/src/scripting/SpeechScriptingInterface.h new file mode 100644 index 0000000000..311bd80605 --- /dev/null +++ b/interface/src/scripting/SpeechScriptingInterface.h @@ -0,0 +1,76 @@ +// SpeechScriptingInterface.h +// interface/src/scripting +// +// Created by Zach Fox on 2018-10-10. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SpeechScriptingInterface_h +#define hifi_SpeechScriptingInterface_h + +#include +#include + +#include // SAPI +#include // SAPI Helper + +class SpeechScriptingInterface : public QObject, public Dependency { + Q_OBJECT + +public: + SpeechScriptingInterface(); + ~SpeechScriptingInterface(); + + Q_INVOKABLE void speakText(const QString& textToSpeak); + +private: + + class CComAutoInit { + public: + // Initializes COM using CoInitialize. + // On failure, signals error using AtlThrow. + CComAutoInit() { + HRESULT hr = ::CoInitialize(NULL); + if (FAILED(hr)) { + ATLTRACE(TEXT("CoInitialize() failed in CComAutoInit constructor (hr=0x%08X).\n"), hr); + AtlThrow(hr); + } + } + + // Initializes COM using CoInitializeEx. + // On failure, signals error using AtlThrow. + explicit CComAutoInit(__in DWORD dwCoInit) { + HRESULT hr = ::CoInitializeEx(NULL, dwCoInit); + if (FAILED(hr)) { + ATLTRACE(TEXT("CoInitializeEx() failed in CComAutoInit constructor (hr=0x%08X).\n"), hr); + AtlThrow(hr); + } + } + + // Uninitializes COM using CoUninitialize. + ~CComAutoInit() { ::CoUninitialize(); } + + // + // Ban copy + // + private: + CComAutoInit(const CComAutoInit&); + }; + + // COM initialization and cleanup (must precede other COM related data members) + CComAutoInit m_comInit; + + // Text to speech engine + CComPtr m_tts; + + // Default voice token + CComPtr m_voiceToken; + + CComPtr outputStream; + CComPtr cpIStream; +}; + +#endif // hifi_SpeechScriptingInterface_h diff --git a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp index bc9ac84c91..ac5e43e558 100644 --- a/libraries/entities-renderer/src/RenderableWebEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderableWebEntityItem.cpp @@ -54,7 +54,7 @@ WebEntityRenderer::ContentType WebEntityRenderer::getContentType(const QString& const QUrl url(urlString); auto scheme = url.scheme(); - if (scheme == URL_SCHEME_ABOUT || scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || + if (scheme == HIFI_URL_SCHEME_ABOUT || scheme == HIFI_URL_SCHEME_HTTP || scheme == HIFI_URL_SCHEME_HTTPS || urlString.toLower().endsWith(".htm") || urlString.toLower().endsWith(".html")) { return ContentType::HtmlContent; } diff --git a/libraries/entities/src/EntityEditFilters.cpp b/libraries/entities/src/EntityEditFilters.cpp index 94df7eb465..9a3f056e04 100644 --- a/libraries/entities/src/EntityEditFilters.cpp +++ b/libraries/entities/src/EntityEditFilters.cpp @@ -183,7 +183,7 @@ void EntityEditFilters::addFilter(EntityItemID entityID, QString filterURL) { } // The following should be abstracted out for use in Agent.cpp (and maybe later AvatarMixer.cpp) - if (scriptURL.scheme().isEmpty() || (scriptURL.scheme() == URL_SCHEME_FILE)) { + if (scriptURL.scheme().isEmpty() || (scriptURL.scheme() == HIFI_URL_SCHEME_FILE)) { qWarning() << "Cannot load script from local filesystem, because assignment may be on a different computer."; scriptRequestFinished(entityID); return; diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index e8aec5e60e..11a5b2f167 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -329,7 +329,7 @@ _maxNumPixels(100) static bool isLocalUrl(const QUrl& url) { auto scheme = url.scheme(); - return (scheme == URL_SCHEME_FILE || scheme == URL_SCHEME_QRC || scheme == RESOURCE_SCHEME); + return (scheme == HIFI_URL_SCHEME_FILE || scheme == URL_SCHEME_QRC || scheme == RESOURCE_SCHEME); } NetworkTexture::NetworkTexture(const QUrl& url, image::TextureUsage::Type type, const QByteArray& content, int maxNumPixels) : @@ -502,7 +502,7 @@ void NetworkTexture::handleLocalRequestCompleted() { void NetworkTexture::makeLocalRequest() { const QString scheme = _activeUrl.scheme(); QString path; - if (scheme == URL_SCHEME_FILE) { + if (scheme == HIFI_URL_SCHEME_FILE) { path = PathUtils::expandToLocalDataAbsolutePath(_activeUrl).toLocalFile(); } else { path = ":" + _activeUrl.path(); diff --git a/libraries/networking/src/AddressManager.cpp b/libraries/networking/src/AddressManager.cpp index f8ab8ceaec..e6957728e8 100644 --- a/libraries/networking/src/AddressManager.cpp +++ b/libraries/networking/src/AddressManager.cpp @@ -155,12 +155,12 @@ void AddressManager::goForward() { void AddressManager::storeCurrentAddress() { auto url = currentAddress(); - if (url.scheme() == URL_SCHEME_FILE || + if (url.scheme() == HIFI_URL_SCHEME_FILE || (url.scheme() == URL_SCHEME_HIFI && !url.host().isEmpty())) { // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can // be loaded over http(s) - // url.scheme() == URL_SCHEME_HTTP || - // url.scheme() == URL_SCHEME_HTTPS || + // url.scheme() == HIFI_URL_SCHEME_HTTP || + // url.scheme() == HIFI_URL_SCHEME_HTTPS || bool isInErrorState = DependencyManager::get()->getDomainHandler().isInErrorState(); if (isConnected()) { if (isInErrorState) { @@ -331,11 +331,11 @@ bool AddressManager::handleUrl(const QUrl& lookupUrl, LookupTrigger trigger) { emit lookupResultsFinished(); return true; - } else if (lookupUrl.scheme() == URL_SCHEME_FILE) { + } else if (lookupUrl.scheme() == HIFI_URL_SCHEME_FILE) { // TODO -- once Octree::readFromURL no-longer takes over the main event-loop, serverless-domain urls can // be loaded over http(s) // lookupUrl.scheme() == URL_SCHEME_HTTP || - // lookupUrl.scheme() == URL_SCHEME_HTTPS || + // lookupUrl.scheme() == HIFI_URL_SCHEME_HTTPS || // TODO once a file can return a connection refusal if there were to be some kind of load error, we'd // need to store the previous domain tried in _lastVisitedURL. For now , do not store it. diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index 615546b410..3dda182989 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -194,7 +194,7 @@ void DomainHandler::setURLAndID(QUrl domainURL, QUuid domainID) { _sockAddr.clear(); // if this is a file URL we need to see if it has a ~ for us to expand - if (domainURL.scheme() == URL_SCHEME_FILE) { + if (domainURL.scheme() == HIFI_URL_SCHEME_FILE) { domainURL = PathUtils::expandToLocalDataAbsolutePath(domainURL); } } diff --git a/libraries/networking/src/NetworkingConstants.h b/libraries/networking/src/NetworkingConstants.h index 839e269fd4..302e0efa02 100644 --- a/libraries/networking/src/NetworkingConstants.h +++ b/libraries/networking/src/NetworkingConstants.h @@ -30,14 +30,14 @@ namespace NetworkingConstants { QUrl METAVERSE_SERVER_URL(); } -const QString URL_SCHEME_ABOUT = "about"; +const QString HIFI_URL_SCHEME_ABOUT = "about"; const QString URL_SCHEME_HIFI = "hifi"; const QString URL_SCHEME_HIFIAPP = "hifiapp"; const QString URL_SCHEME_QRC = "qrc"; -const QString URL_SCHEME_FILE = "file"; -const QString URL_SCHEME_HTTP = "http"; -const QString URL_SCHEME_HTTPS = "https"; -const QString URL_SCHEME_FTP = "ftp"; +const QString HIFI_URL_SCHEME_FILE = "file"; +const QString HIFI_URL_SCHEME_HTTP = "http"; +const QString HIFI_URL_SCHEME_HTTPS = "https"; +const QString HIFI_URL_SCHEME_FTP = "ftp"; const QString URL_SCHEME_ATP = "atp"; #endif // hifi_NetworkingConstants_h diff --git a/libraries/networking/src/ResourceCache.cpp b/libraries/networking/src/ResourceCache.cpp index aed9f3b0e5..1328606be4 100644 --- a/libraries/networking/src/ResourceCache.cpp +++ b/libraries/networking/src/ResourceCache.cpp @@ -114,7 +114,7 @@ QSharedPointer ResourceCacheSharedItems::getHighestPendingRequest() { // Check load priority float priority = resource->getLoadPriority(); - bool isFile = resource->getURL().scheme() == URL_SCHEME_FILE; + bool isFile = resource->getURL().scheme() == HIFI_URL_SCHEME_FILE; if (priority >= highestPriority && (isFile || !currentHighestIsFile)) { highestPriority = priority; highestIndex = i; diff --git a/libraries/networking/src/ResourceManager.cpp b/libraries/networking/src/ResourceManager.cpp index 553f0d0a61..40d6570f48 100644 --- a/libraries/networking/src/ResourceManager.cpp +++ b/libraries/networking/src/ResourceManager.cpp @@ -82,10 +82,10 @@ const QSet& getKnownUrls() { static std::once_flag once; std::call_once(once, [] { knownUrls.insert(URL_SCHEME_QRC); - knownUrls.insert(URL_SCHEME_FILE); - knownUrls.insert(URL_SCHEME_HTTP); - knownUrls.insert(URL_SCHEME_HTTPS); - knownUrls.insert(URL_SCHEME_FTP); + knownUrls.insert(HIFI_URL_SCHEME_FILE); + knownUrls.insert(HIFI_URL_SCHEME_HTTP); + knownUrls.insert(HIFI_URL_SCHEME_HTTPS); + knownUrls.insert(HIFI_URL_SCHEME_FTP); knownUrls.insert(URL_SCHEME_ATP); }); return knownUrls; @@ -97,7 +97,7 @@ QUrl ResourceManager::normalizeURL(const QUrl& originalUrl) { if (!getKnownUrls().contains(scheme)) { // check the degenerative file case: on windows we can often have urls of the form c:/filename // this checks for and works around that case. - QUrl urlWithFileScheme{ URL_SCHEME_FILE + ":///" + url.toString() }; + QUrl urlWithFileScheme{ HIFI_URL_SCHEME_FILE + ":///" + url.toString() }; if (!urlWithFileScheme.toLocalFile().isEmpty()) { return urlWithFileScheme; } @@ -118,9 +118,9 @@ ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const Q ResourceRequest* request = nullptr; - if (scheme == URL_SCHEME_FILE || scheme == URL_SCHEME_QRC) { + if (scheme == HIFI_URL_SCHEME_FILE || scheme == URL_SCHEME_QRC) { request = new FileResourceRequest(normalizedURL); - } else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) { + } else if (scheme == HIFI_URL_SCHEME_HTTP || scheme == HIFI_URL_SCHEME_HTTPS || scheme == HIFI_URL_SCHEME_FTP) { request = new HTTPResourceRequest(normalizedURL); } else if (scheme == URL_SCHEME_ATP) { if (!_atpSupportEnabled) { @@ -143,10 +143,10 @@ ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const Q bool ResourceManager::resourceExists(const QUrl& url) { auto scheme = url.scheme(); - if (scheme == URL_SCHEME_FILE) { + if (scheme == HIFI_URL_SCHEME_FILE) { QFileInfo file{ url.toString() }; return file.exists(); - } else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) { + } else if (scheme == HIFI_URL_SCHEME_HTTP || scheme == HIFI_URL_SCHEME_HTTPS || scheme == HIFI_URL_SCHEME_FTP) { auto& networkAccessManager = NetworkAccessManager::getInstance(); QNetworkRequest request{ url };