From 32b5ac50204239e6c693107f6fcaf7f464b3a936 Mon Sep 17 00:00:00 2001 From: seefo Date: Thu, 25 May 2017 16:10:53 -0700 Subject: [PATCH 1/3] Moved audio asset processing to a separate thread --- libraries/audio/src/Sound.cpp | 37 ++++++++++++++++++++++++----------- libraries/audio/src/Sound.h | 29 +++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index d6607424db..b48ea1fed7 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -13,6 +13,8 @@ #include +#include +#include #include #include #include @@ -53,9 +55,25 @@ Sound::Sound(const QUrl& url, bool isStereo, bool isAmbisonic) : } void Sound::downloadFinished(const QByteArray& data) { + SoundProcessor* soundProcessor = new SoundProcessor(_url, data, this); + QThreadPool::globalInstance()->start(soundProcessor); +} + +void Sound::setReady(bool isReady) { + if (isReady) { + finishedLoading(true); + _isReady = true; + emit ready(); + qCDebug(audio) << "Setting ready state for audio asset" << _url; + } +} + +void SoundProcessor::run() { // replace our byte array with the downloaded data - QByteArray rawAudioByteArray = QByteArray(data); - QString fileName = getURL().fileName().toLower(); + QByteArray rawAudioByteArray = QByteArray(_data); + QString fileName = _url.fileName().toLower(); + + //Sound* sound = dynamic_cast(_sound); static const QString WAV_EXTENSION = ".wav"; static const QString RAW_EXTENSION = ".raw"; @@ -63,28 +81,25 @@ void Sound::downloadFinished(const QByteArray& data) { QByteArray outputAudioByteArray; - int sampleRate = interpretAsWav(rawAudioByteArray, outputAudioByteArray); + int sampleRate = _sound->interpretAsWav(rawAudioByteArray, outputAudioByteArray); if (sampleRate != 0) { - downSample(outputAudioByteArray, sampleRate); + _sound->downSample(outputAudioByteArray, sampleRate); } } else if (fileName.endsWith(RAW_EXTENSION)) { // check if this was a stereo raw file // since it's raw the only way for us to know that is if the file was called .stereo.raw if (fileName.toLower().endsWith("stereo.raw")) { - _isStereo = true; - qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << getURL() << "as stereo audio file."; + _sound->setStereo(true); + qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << _url << "as stereo audio file."; } // Process as 48khz RAW file - downSample(rawAudioByteArray, 48000); + _sound->downSample(rawAudioByteArray, 48000); } else { qCDebug(audio) << "Unknown sound file type"; } - finishedLoading(true); - - _isReady = true; - emit ready(); + _sound->setReady(true); } void Sound::downSample(const QByteArray& rawAudioByteArray, int sampleRate) { diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 37d5b40e95..6ec099b071 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -12,6 +12,7 @@ #ifndef hifi_Sound_h #define hifi_Sound_h +#include #include #include #include @@ -28,10 +29,15 @@ public: bool isAmbisonic() const { return _isAmbisonic; } bool isReady() const { return _isReady; } float getDuration() const { return _duration; } - const QByteArray& getByteArray() const { return _byteArray; } + void setStereo(bool stereo) { _isStereo = stereo; } + void setReady(bool ready); + + void downSample(const QByteArray& rawAudioByteArray, int sampleRate); + int interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); + signals: void ready(); @@ -42,12 +48,27 @@ private: bool _isReady; float _duration; // In seconds - void downSample(const QByteArray& rawAudioByteArray, int sampleRate); - int interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); - virtual void downloadFinished(const QByteArray& data) override; }; +class SoundProcessor : public QObject, public QRunnable { + Q_OBJECT + +public: + SoundProcessor(const QUrl& url, const QByteArray& data, Sound* sound) + : _url(url), _data(data), _sound(sound) + { + } + + virtual void run() override; + +private: + QUrl _url; + QByteArray _data; + Sound* _sound; + +}; + typedef QSharedPointer SharedSoundPointer; class SoundScriptingInterface : public QObject { From 90d5b2a6d45671b64dd3fdcfd887880573aff3b0 Mon Sep 17 00:00:00 2001 From: seefo Date: Fri, 26 May 2017 10:26:32 -0700 Subject: [PATCH 2/3] Removed SoundProcessor's dependence on Sound objects --- libraries/audio/src/Sound.cpp | 62 +++++++++++++++++++++-------------- libraries/audio/src/Sound.h | 26 +++++++++------ 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index b48ea1fed7..5ac9f8f53d 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -51,67 +51,81 @@ Sound::Sound(const QUrl& url, bool isStereo, bool isAmbisonic) : _isAmbisonic(isAmbisonic), _isReady(false) { - } void Sound::downloadFinished(const QByteArray& data) { - SoundProcessor* soundProcessor = new SoundProcessor(_url, data, this); + // this is a QRunnable, will delete itself after it has finished running + SoundProcessor* soundProcessor = new SoundProcessor(_url, data, _isStereo, _isAmbisonic); + connect(soundProcessor, SIGNAL(onSuccess(QByteArray, bool, bool, float)), SLOT(soundProcessSuccess(QByteArray, bool, bool, float))); + connect(soundProcessor, SIGNAL(onError(int, QString)), SLOT(soundProcessError(int, QString))); QThreadPool::globalInstance()->start(soundProcessor); } -void Sound::setReady(bool isReady) { - if (isReady) { - finishedLoading(true); - _isReady = true; - emit ready(); - qCDebug(audio) << "Setting ready state for audio asset" << _url; - } +void Sound::soundProcessSuccess(QByteArray data, bool stereo, bool ambisonic, float duration) { + + qCDebug(audio) << "Setting ready state for sound file" << _url.toDisplayString(); + + _byteArray = data; + _isStereo = stereo; + _isAmbisonic = ambisonic; + _duration = duration; + _isReady = true; + finishedLoading(true); + + emit ready(); +} + +void Sound::soundProcessError(int error, QString str) { + qCCritical(audio) << "Failed to process sound file" << _url.toDisplayString() << "code =" << error << str; + emit failed(QNetworkReply::UnknownContentError); + finishedLoading(false); } void SoundProcessor::run() { + + qCDebug(audio) << "Processing sound file" << _url.toDisplayString(); + // replace our byte array with the downloaded data QByteArray rawAudioByteArray = QByteArray(_data); QString fileName = _url.fileName().toLower(); - //Sound* sound = dynamic_cast(_sound); - static const QString WAV_EXTENSION = ".wav"; static const QString RAW_EXTENSION = ".raw"; if (fileName.endsWith(WAV_EXTENSION)) { QByteArray outputAudioByteArray; - int sampleRate = _sound->interpretAsWav(rawAudioByteArray, outputAudioByteArray); + int sampleRate = interpretAsWav(rawAudioByteArray, outputAudioByteArray); if (sampleRate != 0) { - _sound->downSample(outputAudioByteArray, sampleRate); + downSample(outputAudioByteArray, sampleRate); } } else if (fileName.endsWith(RAW_EXTENSION)) { // check if this was a stereo raw file // since it's raw the only way for us to know that is if the file was called .stereo.raw if (fileName.toLower().endsWith("stereo.raw")) { - _sound->setStereo(true); + _isStereo = true; qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << _url << "as stereo audio file."; } // Process as 48khz RAW file - _sound->downSample(rawAudioByteArray, 48000); + downSample(rawAudioByteArray, 48000); } else { qCDebug(audio) << "Unknown sound file type"; + emit onError(300, "Failed to load sound file, reason: unknown sound file type"); + return; } - _sound->setReady(true); + emit onSuccess(_data, _isStereo, _isAmbisonic, _duration); } -void Sound::downSample(const QByteArray& rawAudioByteArray, int sampleRate) { +void SoundProcessor::downSample(const QByteArray& rawAudioByteArray, int sampleRate) { // we want to convert it to the format that the audio-mixer wants // which is signed, 16-bit, 24Khz if (sampleRate == AudioConstants::SAMPLE_RATE) { - // no resampling needed - _byteArray = rawAudioByteArray; - + _data = rawAudioByteArray; } else { int numChannels = _isAmbisonic ? AudioConstants::AMBISONIC : (_isStereo ? AudioConstants::STEREO : AudioConstants::MONO); @@ -121,15 +135,15 @@ void Sound::downSample(const QByteArray& rawAudioByteArray, int sampleRate) { int numSourceFrames = rawAudioByteArray.size() / (numChannels * sizeof(AudioConstants::AudioSample)); int maxDestinationFrames = resampler.getMaxOutput(numSourceFrames); int maxDestinationBytes = maxDestinationFrames * numChannels * sizeof(AudioConstants::AudioSample); - _byteArray.resize(maxDestinationBytes); + _data.resize(maxDestinationBytes); int numDestinationFrames = resampler.render((int16_t*)rawAudioByteArray.data(), - (int16_t*)_byteArray.data(), + (int16_t*)_data.data(), numSourceFrames); // truncate to actual output int numDestinationBytes = numDestinationFrames * numChannels * sizeof(AudioConstants::AudioSample); - _byteArray.resize(numDestinationBytes); + _data.resize(numDestinationBytes); } } @@ -178,7 +192,7 @@ struct WAVEFormat { }; // returns wavfile sample rate, used for resampling -int Sound::interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray) { +int SoundProcessor::interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray) { // Create a data stream to analyze the data QDataStream waveStream(const_cast(&inputAudioByteArray), QIODevice::ReadOnly); diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 6ec099b071..43a4eb7685 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -32,14 +32,12 @@ public: const QByteArray& getByteArray() const { return _byteArray; } - void setStereo(bool stereo) { _isStereo = stereo; } - void setReady(bool ready); - - void downSample(const QByteArray& rawAudioByteArray, int sampleRate); - int interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); - signals: void ready(); + +protected slots: + void soundProcessSuccess(QByteArray data, bool stereo, bool ambisonic, float duration); + void soundProcessError(int error, QString str); private: QByteArray _byteArray; @@ -55,18 +53,26 @@ class SoundProcessor : public QObject, public QRunnable { Q_OBJECT public: - SoundProcessor(const QUrl& url, const QByteArray& data, Sound* sound) - : _url(url), _data(data), _sound(sound) + SoundProcessor(const QUrl& url, const QByteArray& data, const bool stereo, const bool ambisonic) + : _url(url), _data(data), _isStereo(stereo), _isAmbisonic(ambisonic) { } virtual void run() override; + void downSample(const QByteArray& rawAudioByteArray, int sampleRate); + int interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); + +signals: + void onSuccess(QByteArray data, bool stereo, bool ambisonic, float duration); + void onError(int error, QString str); + private: QUrl _url; QByteArray _data; - Sound* _sound; - + bool _isStereo; + bool _isAmbisonic; + float _duration; }; typedef QSharedPointer SharedSoundPointer; From cc57d28cccf1080b723317ee4bcbd81b46308ef4 Mon Sep 17 00:00:00 2001 From: seefo Date: Fri, 26 May 2017 11:39:47 -0700 Subject: [PATCH 3/3] Made requested changes to PR10554 --- libraries/audio/src/Sound.cpp | 4 ++-- libraries/audio/src/Sound.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 5ac9f8f53d..476a8d4d88 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -56,8 +56,8 @@ Sound::Sound(const QUrl& url, bool isStereo, bool isAmbisonic) : void Sound::downloadFinished(const QByteArray& data) { // this is a QRunnable, will delete itself after it has finished running SoundProcessor* soundProcessor = new SoundProcessor(_url, data, _isStereo, _isAmbisonic); - connect(soundProcessor, SIGNAL(onSuccess(QByteArray, bool, bool, float)), SLOT(soundProcessSuccess(QByteArray, bool, bool, float))); - connect(soundProcessor, SIGNAL(onError(int, QString)), SLOT(soundProcessError(int, QString))); + connect(soundProcessor, &SoundProcessor::onSuccess, this, &Sound::soundProcessSuccess); + connect(soundProcessor, &SoundProcessor::onError, this, &Sound::soundProcessError); QThreadPool::globalInstance()->start(soundProcessor); } diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index 43a4eb7685..69dbf5a913 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -53,7 +53,7 @@ class SoundProcessor : public QObject, public QRunnable { Q_OBJECT public: - SoundProcessor(const QUrl& url, const QByteArray& data, const bool stereo, const bool ambisonic) + SoundProcessor(const QUrl& url, const QByteArray& data, bool stereo, bool ambisonic) : _url(url), _data(data), _isStereo(stereo), _isAmbisonic(ambisonic) { }