From fd6b9c3550676af1a00dfdccf68ce1ab02961842 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 13 Nov 2014 10:16:42 -0800 Subject: [PATCH] changes to sound class to allow caching of sounds --- libraries/audio/src/AudioInjector.cpp | 41 +++++++------ libraries/audio/src/AudioInjector.h | 3 +- libraries/audio/src/Sound.cpp | 85 ++------------------------- libraries/audio/src/Sound.h | 24 +++----- libraries/audio/src/SoundCache.cpp | 33 +++++++++++ libraries/audio/src/SoundCache.h | 37 ++++++++++++ libraries/avatars/src/Recording.cpp | 24 ++------ libraries/avatars/src/Recording.h | 9 +-- 8 files changed, 114 insertions(+), 142 deletions(-) create mode 100644 libraries/audio/src/SoundCache.cpp create mode 100644 libraries/audio/src/SoundCache.h diff --git a/libraries/audio/src/AudioInjector.cpp b/libraries/audio/src/AudioInjector.cpp index 1743504883..1c9c93c00e 100644 --- a/libraries/audio/src/AudioInjector.cpp +++ b/libraries/audio/src/AudioInjector.cpp @@ -31,7 +31,6 @@ void injectorFromScriptValue(const QScriptValue& object, AudioInjector*& out) { AudioInjector::AudioInjector(QObject* parent) : QObject(parent), - _sound(NULL), _options(), _shouldStop(false), _loudness(0.0f), @@ -42,7 +41,7 @@ AudioInjector::AudioInjector(QObject* parent) : } AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions) : - _sound(sound), + _audioData(sound->getByteArray()), _options(injectorOptions), _shouldStop(false), _loudness(0.0f), @@ -52,6 +51,18 @@ AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorO { } +AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) : + _audioData(audioData), + _options(injectorOptions), + _shouldStop(false), + _loudness(0.0f), + _isFinished(false), + _currentSendPosition(0), + _localBuffer(NULL) +{ + +} + AudioInjector::~AudioInjector() { if (_localBuffer) { _localBuffer->stop(); @@ -76,11 +87,9 @@ void AudioInjector::injectAudio() { void AudioInjector::injectLocally() { bool success = false; - if (_localAudioInterface) { - const QByteArray& soundByteArray = _sound->getByteArray(); - - if (soundByteArray.size() > 0) { - _localBuffer = new AudioInjectorLocalBuffer(_sound->getByteArray(), this); + if (_localAudioInterface) { + if (_audioData.size() > 0) { + _localBuffer = new AudioInjectorLocalBuffer(_audioData, this); _localBuffer->open(QIODevice::ReadOnly); _localBuffer->setShouldLoop(_options.loop); @@ -114,15 +123,13 @@ void AudioInjector::injectLocally() { const uchar MAX_INJECTOR_VOLUME = 0xFF; void AudioInjector::injectToMixer() { - QByteArray soundByteArray = _sound->getByteArray(); - if (_currentSendPosition < 0 || - _currentSendPosition >= soundByteArray.size()) { + _currentSendPosition >= _audioData.size()) { _currentSendPosition = 0; } // make sure we actually have samples downloaded to inject - if (soundByteArray.size()) { + if (_audioData.size()) { // setup the packet for injected audio QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio); @@ -172,15 +179,15 @@ void AudioInjector::injectToMixer() { // loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks quint16 outgoingInjectedAudioSequenceNumber = 0; - while (_currentSendPosition < soundByteArray.size() && !_shouldStop) { + while (_currentSendPosition < _audioData.size() && !_shouldStop) { int bytesToCopy = std::min(((_options.stereo) ? 2 : 1) * NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL, - soundByteArray.size() - _currentSendPosition); + _audioData.size() - _currentSendPosition); // Measure the loudness of this frame _loudness = 0.0f; for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) { - _loudness += abs(*reinterpret_cast(soundByteArray.data() + _currentSendPosition + i)) / + _loudness += abs(*reinterpret_cast(_audioData.data() + _currentSendPosition + i)) / (MAX_SAMPLE_VALUE / 2.0f); } _loudness /= (float)(bytesToCopy / sizeof(int16_t)); @@ -203,7 +210,7 @@ void AudioInjector::injectToMixer() { // copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet memcpy(injectAudioPacket.data() + numPreAudioDataBytes, - soundByteArray.data() + _currentSendPosition, bytesToCopy); + _audioData.data() + _currentSendPosition, bytesToCopy); // grab our audio mixer from the NodeList, if it exists NodeList* nodeList = NodeList::getInstance(); @@ -217,7 +224,7 @@ void AudioInjector::injectToMixer() { // send two packets before the first sleep so the mixer can start playback right away - if (_currentSendPosition != bytesToCopy && _currentSendPosition < soundByteArray.size()) { + if (_currentSendPosition != bytesToCopy && _currentSendPosition < _audioData.size()) { // not the first packet and not done // sleep for the appropriate time int usecToSleep = (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - timer.nsecsElapsed() / 1000; @@ -227,7 +234,7 @@ void AudioInjector::injectToMixer() { } } - if (shouldLoop && _currentSendPosition >= soundByteArray.size()) { + if (shouldLoop && _currentSendPosition >= _audioData.size()) { _currentSendPosition = 0; } } diff --git a/libraries/audio/src/AudioInjector.h b/libraries/audio/src/AudioInjector.h index 13188c5977..257b538c11 100644 --- a/libraries/audio/src/AudioInjector.h +++ b/libraries/audio/src/AudioInjector.h @@ -29,6 +29,7 @@ class AudioInjector : public QObject { public: AudioInjector(QObject* parent); AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions); + AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions); ~AudioInjector(); bool isFinished() const { return _isFinished; } @@ -51,7 +52,7 @@ private: void injectToMixer(); void injectLocally(); - Sound* _sound; + QByteArray _audioData; AudioInjectorOptions _options; bool _shouldStop; float _loudness; diff --git a/libraries/audio/src/Sound.cpp b/libraries/audio/src/Sound.cpp index 4c520f27ce..8f858e6d3c 100644 --- a/libraries/audio/src/Sound.cpp +++ b/libraries/audio/src/Sound.cpp @@ -38,87 +38,17 @@ void soundFromScriptValue(const QScriptValue& object, Sound*& out) { out = qobject_cast(object.toQObject()); } -// procedural audio version of Sound -Sound::Sound(float volume, float frequency, float duration, float decay, QObject* parent) : - QObject(parent), - _isStereo(false) +Sound::Sound(const QUrl& url, bool isStereo) : + Resource(url), + _isStereo(isStereo) { - static char monoAudioData[MAX_PACKET_SIZE]; - static int16_t* monoAudioSamples = (int16_t*)(monoAudioData); - - float t; - const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; - const float MAX_VOLUME = 32000.f; - const float MAX_DURATION = 2.f; - const float MIN_AUDIBLE_VOLUME = 0.001f; - const float NOISE_MAGNITUDE = 0.02f; - const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); - const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); - int numSamples = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // we add sounds in chunks of this many samples - int chunkStartingSample = 0; - float waveFrequency = (frequency / SAMPLE_RATE) * TWO_PI; - while (volume > 0.f) { - for (int i = 0; i < numSamples; i++) { - t = (float)chunkStartingSample + (float)i; - float sample = sinf(t * waveFrequency); - sample += ((randFloat() - 0.5f) * NOISE_MAGNITUDE); - sample *= volume * MAX_VOLUME; - - monoAudioSamples[i] = glm::clamp((int)sample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); - volume *= (1.f - decay); - } - // add the monoAudioSamples to our actual output Byte Array - _byteArray.append(monoAudioData, numSamples * sizeof(int16_t)); - chunkStartingSample += numSamples; - duration = glm::clamp(duration - (AUDIO_CALLBACK_MSECS / 1000.f), 0.f, MAX_DURATION); - //qDebug() << "decaying... _duration=" << _duration; - if (duration == 0.f || (volume < MIN_AUDIBLE_VOLUME)) { - volume = 0.f; - } - } } -Sound::Sound(const QUrl& sampleURL, bool isStereo, QObject* parent) : - QObject(parent), - _isStereo(isStereo), - _hasDownloaded(false) -{ - // assume we have a QApplication or QCoreApplication instance and use the - // QNetworkAccess manager to grab the raw audio file at the given URL - - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - - qDebug() << "Requesting audio file" << sampleURL.toDisplayString(); - - QNetworkReply* soundDownload = networkAccessManager.get(QNetworkRequest(sampleURL)); - connect(soundDownload, &QNetworkReply::finished, this, &Sound::replyFinished); - connect(soundDownload, SIGNAL(error(QNetworkReply::NetworkError)), - this, SLOT(replyError(QNetworkReply::NetworkError))); -} - -Sound::Sound(const QByteArray byteArray, QObject* parent) : - QObject(parent), - _byteArray(byteArray), - _isStereo(false), - _hasDownloaded(true) -{ -} - -void Sound::append(const QByteArray byteArray) { - _byteArray.append(byteArray); -} - -void Sound::replyFinished() { - - QNetworkReply* reply = reinterpret_cast(sender()); - +void Sound::downloadFinished(QNetworkReply* reply) { // replace our byte array with the downloaded data QByteArray rawAudioByteArray = reply->readAll(); - // foreach(QByteArray b, reply->rawHeaderList()) - // qDebug() << b.constData() << ": " << reply->rawHeader(b).constData(); - if (reply->hasRawHeader("Content-Type")) { QByteArray headerContentType = reply->rawHeader("Content-Type"); @@ -140,13 +70,6 @@ void Sound::replyFinished() { } else { qDebug() << "Network reply without 'Content-Type'."; } - - _hasDownloaded = true; -} - -void Sound::replyError(QNetworkReply::NetworkError code) { - QNetworkReply* reply = reinterpret_cast(sender()); - qDebug() << "Error downloading sound file at" << reply->url().toString() << "-" << reply->errorString(); } void Sound::downSample(const QByteArray& rawAudioByteArray) { diff --git a/libraries/audio/src/Sound.h b/libraries/audio/src/Sound.h index f7b51891f0..2880781ac6 100644 --- a/libraries/audio/src/Sound.h +++ b/libraries/audio/src/Sound.h @@ -16,38 +16,28 @@ #include #include -class Sound : public QObject { +#include + +class Sound : public Resource { Q_OBJECT - Q_PROPERTY(bool downloaded READ hasDownloaded) + Q_PROPERTY(bool downloaded READ isLoaded) public: - Sound(const QUrl& sampleURL, bool isStereo = false, QObject* parent = NULL); - Sound(float volume, float frequency, float duration, float decay, QObject* parent = NULL); - Sound(const QByteArray byteArray, QObject* parent = NULL); - void append(const QByteArray byteArray); + Sound(const QUrl& url, bool isStereo = false); bool isStereo() const { return _isStereo; } - bool hasDownloaded() const { return _hasDownloaded; } const QByteArray& getByteArray() { return _byteArray; } private: QByteArray _byteArray; bool _isStereo; - bool _hasDownloaded; void trimFrames(); void downSample(const QByteArray& rawAudioByteArray); void interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray); - -private slots: - void replyFinished(); - void replyError(QNetworkReply::NetworkError code); + + virtual void downloadFinished(QNetworkReply* reply); }; -Q_DECLARE_METATYPE(Sound*) - -QScriptValue soundToScriptValue(QScriptEngine* engine, Sound* const& in); -void soundFromScriptValue(const QScriptValue& object, Sound*& out); - #endif // hifi_Sound_h diff --git a/libraries/audio/src/SoundCache.cpp b/libraries/audio/src/SoundCache.cpp new file mode 100644 index 0000000000..09b9cfe330 --- /dev/null +++ b/libraries/audio/src/SoundCache.cpp @@ -0,0 +1,33 @@ +// +// SoundCache.cpp +// libraries/audio/src +// +// Created by Stephen Birarda on 2014-11-13. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include "SoundCache.h" + +SoundCache::SoundCache(QObject* parent) : + ResourceCache(parent) { +} + +SharedSoundPointer SoundCache::getSound(const QUrl& url) { + if (QThread::currentThread() != thread()) { + SharedSoundPointer result; + QMetaObject::invokeMethod(this, "getSound", Qt::BlockingQueuedConnection, + Q_RETURN_ARG(SharedSoundPointer, result), Q_ARG(const QUrl&, url)); + return result; + } + return getResource(url).staticCast(); +} + +QSharedPointer SoundCache::createResource(const QUrl& url, const QSharedPointer& fallback, + bool delayLoad, const void* extra) { + return QSharedPointer(new Sound(url), &Resource::allReferencesCleared); +} \ No newline at end of file diff --git a/libraries/audio/src/SoundCache.h b/libraries/audio/src/SoundCache.h new file mode 100644 index 0000000000..4146171794 --- /dev/null +++ b/libraries/audio/src/SoundCache.h @@ -0,0 +1,37 @@ +// +// SoundCache.h +// libraries/audio/src +// +// Created by Stephen Birarda on 2014-11-13. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SoundCache_h +#define hifi_SoundCache_h + +#include + +#include "Sound.h" + +typedef QSharedPointer SharedSoundPointer; + +/// Scriptable interface for FBX animation loading. +class SoundCache : public ResourceCache { + Q_OBJECT +public: + SoundCache(QObject* parent = NULL); + + Q_INVOKABLE SharedSoundPointer getSound(const QUrl& url); + +protected: + + virtual QSharedPointer createResource(const QUrl& url, + const QSharedPointer& fallback, bool delayLoad, const void* extra); +}; + +Q_DECLARE_METATYPE(SharedSoundPointer) + +#endif // hifi_SoundCache_h \ No newline at end of file diff --git a/libraries/avatars/src/Recording.cpp b/libraries/avatars/src/Recording.cpp index 0d089a2bd2..b39421d03a 100644 --- a/libraries/avatars/src/Recording.cpp +++ b/libraries/avatars/src/Recording.cpp @@ -43,13 +43,6 @@ void RecordingFrame::setBlendshapeCoefficients(QVector blendshapeCoeffici _blendshapeCoefficients = blendshapeCoefficients; } -Recording::Recording() : _audio(NULL) { -} - -Recording::~Recording() { - delete _audio; -} - int Recording::getLength() const { if (_timestamps.isEmpty()) { return 0; @@ -77,19 +70,10 @@ void Recording::addFrame(int timestamp, RecordingFrame &frame) { _frames << frame; } -void Recording::addAudioPacket(const QByteArray& byteArray) { - if (!_audio) { - _audio = new Sound(byteArray); - return; - } - _audio->append(byteArray); -} - void Recording::clear() { _timestamps.clear(); _frames.clear(); - delete _audio; - _audio = NULL; + _audioData.clear(); } void writeVec3(QDataStream& stream, const glm::vec3& value) { @@ -324,7 +308,7 @@ void writeRecordingToFile(RecordingPointer recording, const QString& filename) { fileStream << buffer; } - fileStream << recording->_audio->getByteArray(); + fileStream << recording->getAudioData(); qint64 writingTime = timer.restart(); // Write data length and CRC-16 @@ -367,7 +351,7 @@ void writeRecordingToFile(RecordingPointer recording, const QString& filename) { qDebug() << "Recording:"; qDebug() << "Total frames:" << recording->getFrameNumber(); - qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + qDebug() << "Audio array:" << recording->getAudioData().size(); } qint64 checksumTime = timer.elapsed(); @@ -642,7 +626,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, const QString qDebug() << "Recording:"; qDebug() << "Total frames:" << recording->getFrameNumber(); - qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size(); + qDebug() << "Audio array:" << recording->getAudioData().size(); } diff --git a/libraries/avatars/src/Recording.h b/libraries/avatars/src/Recording.h index aa14dc5d76..d1da77560c 100644 --- a/libraries/avatars/src/Recording.h +++ b/libraries/avatars/src/Recording.h @@ -48,9 +48,6 @@ public: /// Stores a recording class Recording { public: - Recording(); - ~Recording(); - bool isEmpty() const { return _timestamps.isEmpty(); } int getLength() const; // in ms @@ -58,11 +55,11 @@ public: int getFrameNumber() const { return _frames.size(); } qint32 getFrameTimestamp(int i) const; const RecordingFrame& getFrame(int i) const; - Sound* getAudio() const { return _audio; } + const QByteArray& getAudioData() const { return _audioData; } protected: void addFrame(int timestamp, RecordingFrame& frame); - void addAudioPacket(const QByteArray& byteArray); + void addAudioPacket(const QByteArray& byteArray) { _audioData.append(byteArray); } void clear(); private: @@ -70,7 +67,7 @@ private: QVector _timestamps; QVector _frames; - Sound* _audio; + QByteArray _audioData; friend class Recorder; friend class Player;