Mirrors/overte
3
0
Fork 0
mirror of https://github.com/overte-org/overte.git synced 2025-04-19 15:03:53 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

changes to sound class to allow caching of sounds

Browse source
This commit is contained in:
Stephen Birarda 2014-11-13 10:16:42 -08:00
parent 324b2fb18f
commit fd6b9c3550
8 changed files with 114 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
libraries/audio/src/AudioInjector.cpp
View file

@ -31,7 +31,6 @@ void injectorFromScriptValue(const QScriptValue& object, AudioInjector*& out) {
AudioInjector::AudioInjector(QObject* parent) :
QObject(parent),
_sound(NULL),
_options(),
_shouldStop(false),
_loudness(0.0f),
@ -42,7 +41,7 @@ AudioInjector::AudioInjector(QObject* parent) :
}
AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions) :
_sound(sound),
_audioData(sound->getByteArray()),
_options(injectorOptions),
_shouldStop(false),
_loudness(0.0f),
@ -52,6 +51,18 @@ AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorO
{
}
AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions) :
_audioData(audioData),
_options(injectorOptions),
_shouldStop(false),
_loudness(0.0f),
_isFinished(false),
_currentSendPosition(0),
_localBuffer(NULL)
{
}
AudioInjector::~AudioInjector() {
if (_localBuffer) {
_localBuffer->stop();
@ -76,11 +87,9 @@ void AudioInjector::injectAudio() {
void AudioInjector::injectLocally() {
bool success = false;
if (_localAudioInterface) {
const QByteArray& soundByteArray = _sound->getByteArray();
if (soundByteArray.size() > 0) {
_localBuffer = new AudioInjectorLocalBuffer(_sound->getByteArray(), this);
if (_localAudioInterface) {
if (_audioData.size() > 0) {
_localBuffer = new AudioInjectorLocalBuffer(_audioData, this);
_localBuffer->open(QIODevice::ReadOnly);
_localBuffer->setShouldLoop(_options.loop);
@ -114,15 +123,13 @@ void AudioInjector::injectLocally() {
const uchar MAX_INJECTOR_VOLUME = 0xFF;
void AudioInjector::injectToMixer() {
QByteArray soundByteArray = _sound->getByteArray();
if (_currentSendPosition < 0 ||
_currentSendPosition >= soundByteArray.size()) {
_currentSendPosition >= _audioData.size()) {
_currentSendPosition = 0;
}
// make sure we actually have samples downloaded to inject
if (soundByteArray.size()) {
if (_audioData.size()) {
// setup the packet for injected audio
QByteArray injectAudioPacket = byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
@ -172,15 +179,15 @@ void AudioInjector::injectToMixer() {
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
quint16 outgoingInjectedAudioSequenceNumber = 0;
while (_currentSendPosition < soundByteArray.size() && !_shouldStop) {
while (_currentSendPosition < _audioData.size() && !_shouldStop) {
int bytesToCopy = std::min(((_options.stereo) ? 2 : 1) * NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
soundByteArray.size() - _currentSendPosition);
_audioData.size() - _currentSendPosition);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(soundByteArray.data() + _currentSendPosition + i)) /
_loudness += abs(*reinterpret_cast<int16_t*>(_audioData.data() + _currentSendPosition + i)) /
(MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(bytesToCopy / sizeof(int16_t));
@ -203,7 +210,7 @@ void AudioInjector::injectToMixer() {
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes,
soundByteArray.data() + _currentSendPosition, bytesToCopy);
_audioData.data() + _currentSendPosition, bytesToCopy);
// grab our audio mixer from the NodeList, if it exists
NodeList* nodeList = NodeList::getInstance();
@ -217,7 +224,7 @@ void AudioInjector::injectToMixer() {
// send two packets before the first sleep so the mixer can start playback right away
if (_currentSendPosition != bytesToCopy && _currentSendPosition < soundByteArray.size()) {
if (_currentSendPosition != bytesToCopy && _currentSendPosition < _audioData.size()) {
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - timer.nsecsElapsed() / 1000;
@ -227,7 +234,7 @@ void AudioInjector::injectToMixer() {
}
}
if (shouldLoop && _currentSendPosition >= soundByteArray.size()) {
if (shouldLoop && _currentSendPosition >= _audioData.size()) {
_currentSendPosition = 0;
}
}

3
libraries/audio/src/AudioInjector.h
View file

@ -29,6 +29,7 @@ class AudioInjector : public QObject {
public:
AudioInjector(QObject* parent);
AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions);
AudioInjector(const QByteArray& audioData, const AudioInjectorOptions& injectorOptions);
~AudioInjector();
bool isFinished() const { return _isFinished; }
@ -51,7 +52,7 @@ private:
void injectToMixer();
void injectLocally();
Sound* _sound;
QByteArray _audioData;
AudioInjectorOptions _options;
bool _shouldStop;
float _loudness;

85
libraries/audio/src/Sound.cpp
View file

@ -38,87 +38,17 @@ void soundFromScriptValue(const QScriptValue& object, Sound*& out) {
out = qobject_cast<Sound*>(object.toQObject());
}
// procedural audio version of Sound
Sound::Sound(float volume, float frequency, float duration, float decay, QObject* parent) :
QObject(parent),
_isStereo(false)
Sound::Sound(const QUrl& url, bool isStereo) :
Resource(url),
_isStereo(isStereo)
{
static char monoAudioData[MAX_PACKET_SIZE];
static int16_t* monoAudioSamples = (int16_t*)(monoAudioData);
float t;
const float AUDIO_CALLBACK_MSECS = (float) NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0;
const float MAX_VOLUME = 32000.f;
const float MAX_DURATION = 2.f;
const float MIN_AUDIBLE_VOLUME = 0.001f;
const float NOISE_MAGNITUDE = 0.02f;
const int MAX_SAMPLE_VALUE = std::numeric_limits<int16_t>::max();
const int MIN_SAMPLE_VALUE = std::numeric_limits<int16_t>::min();
int numSamples = NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // we add sounds in chunks of this many samples
int chunkStartingSample = 0;
float waveFrequency = (frequency / SAMPLE_RATE) * TWO_PI;
while (volume > 0.f) {
for (int i = 0; i < numSamples; i++) {
t = (float)chunkStartingSample + (float)i;
float sample = sinf(t * waveFrequency);
sample += ((randFloat() - 0.5f) * NOISE_MAGNITUDE);
sample *= volume * MAX_VOLUME;
monoAudioSamples[i] = glm::clamp((int)sample, MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
volume *= (1.f - decay);
}
// add the monoAudioSamples to our actual output Byte Array
_byteArray.append(monoAudioData, numSamples * sizeof(int16_t));
chunkStartingSample += numSamples;
duration = glm::clamp(duration - (AUDIO_CALLBACK_MSECS / 1000.f), 0.f, MAX_DURATION);
//qDebug() << "decaying... _duration=" << _duration;
if (duration == 0.f || (volume < MIN_AUDIBLE_VOLUME)) {
volume = 0.f;
}
}
}
Sound::Sound(const QUrl& sampleURL, bool isStereo, QObject* parent) :
QObject(parent),
_isStereo(isStereo),
_hasDownloaded(false)
{
// assume we have a QApplication or QCoreApplication instance and use the
// QNetworkAccess manager to grab the raw audio file at the given URL
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
qDebug() << "Requesting audio file" << sampleURL.toDisplayString();
QNetworkReply* soundDownload = networkAccessManager.get(QNetworkRequest(sampleURL));
connect(soundDownload, &QNetworkReply::finished, this, &Sound::replyFinished);
connect(soundDownload, SIGNAL(error(QNetworkReply::NetworkError)),
this, SLOT(replyError(QNetworkReply::NetworkError)));
}
Sound::Sound(const QByteArray byteArray, QObject* parent) :
QObject(parent),
_byteArray(byteArray),
_isStereo(false),
_hasDownloaded(true)
{
}
void Sound::append(const QByteArray byteArray) {
_byteArray.append(byteArray);
}
void Sound::replyFinished() {
QNetworkReply* reply = reinterpret_cast<QNetworkReply*>(sender());
void Sound::downloadFinished(QNetworkReply* reply) {
// replace our byte array with the downloaded data
QByteArray rawAudioByteArray = reply->readAll();
// foreach(QByteArray b, reply->rawHeaderList())
// qDebug() << b.constData() << ": " << reply->rawHeader(b).constData();
if (reply->hasRawHeader("Content-Type")) {
QByteArray headerContentType = reply->rawHeader("Content-Type");
@ -140,13 +70,6 @@ void Sound::replyFinished() {
} else {
qDebug() << "Network reply without 'Content-Type'.";
}
_hasDownloaded = true;
}
void Sound::replyError(QNetworkReply::NetworkError code) {
QNetworkReply* reply = reinterpret_cast<QNetworkReply*>(sender());
qDebug() << "Error downloading sound file at" << reply->url().toString() << "-" << reply->errorString();
}
void Sound::downSample(const QByteArray& rawAudioByteArray) {

24
libraries/audio/src/Sound.h
View file

@ -16,38 +16,28 @@
#include <QtNetwork/QNetworkReply>
#include <QtScript/qscriptengine.h>
class Sound : public QObject {
#include <ResourceCache.h>
class Sound : public Resource {
Q_OBJECT
Q_PROPERTY(bool downloaded READ hasDownloaded)
Q_PROPERTY(bool downloaded READ isLoaded)
public:
Sound(const QUrl& sampleURL, bool isStereo = false, QObject* parent = NULL);
Sound(float volume, float frequency, float duration, float decay, QObject* parent = NULL);
Sound(const QByteArray byteArray, QObject* parent = NULL);
void append(const QByteArray byteArray);
Sound(const QUrl& url, bool isStereo = false);
bool isStereo() const { return _isStereo; }
bool hasDownloaded() const { return _hasDownloaded; }
const QByteArray& getByteArray() { return _byteArray; }
private:
QByteArray _byteArray;
bool _isStereo;
bool _hasDownloaded;
void trimFrames();
void downSample(const QByteArray& rawAudioByteArray);
void interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray);
private slots:
void replyFinished();
void replyError(QNetworkReply::NetworkError code);
virtual void downloadFinished(QNetworkReply* reply);
};
Q_DECLARE_METATYPE(Sound*)
QScriptValue soundToScriptValue(QScriptEngine* engine, Sound* const& in);
void soundFromScriptValue(const QScriptValue& object, Sound*& out);
#endif // hifi_Sound_h

33
libraries/audio/src/SoundCache.cpp Normal file
View file

@ -0,0 +1,33 @@
//
// SoundCache.cpp
// libraries/audio/src
//
// Created by Stephen Birarda on 2014-11-13.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <qthread.h>
#include "SoundCache.h"
SoundCache::SoundCache(QObject* parent) :
ResourceCache(parent) {
}
SharedSoundPointer SoundCache::getSound(const QUrl& url) {
if (QThread::currentThread() != thread()) {
SharedSoundPointer result;
QMetaObject::invokeMethod(this, "getSound", Qt::BlockingQueuedConnection,
Q_RETURN_ARG(SharedSoundPointer, result), Q_ARG(const QUrl&, url));
return result;
}
return getResource(url).staticCast<Sound>();
}
QSharedPointer<Resource> SoundCache::createResource(const QUrl& url, const QSharedPointer<Resource>& fallback,
bool delayLoad, const void* extra) {
return QSharedPointer<Resource>(new Sound(url), &Resource::allReferencesCleared);
}

37
libraries/audio/src/SoundCache.h Normal file
View file

@ -0,0 +1,37 @@
//
// SoundCache.h
// libraries/audio/src
//
// Created by Stephen Birarda on 2014-11-13.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_SoundCache_h
#define hifi_SoundCache_h
#include <ResourceCache.h>
#include "Sound.h"
typedef QSharedPointer<Sound> SharedSoundPointer;
/// Scriptable interface for FBX animation loading.
class SoundCache : public ResourceCache {
Q_OBJECT
public:
SoundCache(QObject* parent = NULL);
Q_INVOKABLE SharedSoundPointer getSound(const QUrl& url);
protected:
virtual QSharedPointer<Resource> createResource(const QUrl& url,
const QSharedPointer<Resource>& fallback, bool delayLoad, const void* extra);
};
Q_DECLARE_METATYPE(SharedSoundPointer)
#endif // hifi_SoundCache_h

24
libraries/avatars/src/Recording.cpp
View file

@ -43,13 +43,6 @@ void RecordingFrame::setBlendshapeCoefficients(QVector<float> blendshapeCoeffici
_blendshapeCoefficients = blendshapeCoefficients;
}
Recording::Recording() : _audio(NULL) {
}
Recording::~Recording() {
delete _audio;
}
int Recording::getLength() const {
if (_timestamps.isEmpty()) {
return 0;
@ -77,19 +70,10 @@ void Recording::addFrame(int timestamp, RecordingFrame &frame) {
_frames << frame;
}
void Recording::addAudioPacket(const QByteArray& byteArray) {
if (!_audio) {
_audio = new Sound(byteArray);
return;
}
_audio->append(byteArray);
}
void Recording::clear() {
_timestamps.clear();
_frames.clear();
delete _audio;
_audio = NULL;
_audioData.clear();
}
void writeVec3(QDataStream& stream, const glm::vec3& value) {
@ -324,7 +308,7 @@ void writeRecordingToFile(RecordingPointer recording, const QString& filename) {
fileStream << buffer;
}
fileStream << recording->_audio->getByteArray();
fileStream << recording->getAudioData();
qint64 writingTime = timer.restart();
// Write data length and CRC-16
@ -367,7 +351,7 @@ void writeRecordingToFile(RecordingPointer recording, const QString& filename) {
qDebug() << "Recording:";
qDebug() << "Total frames:" << recording->getFrameNumber();
qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size();
qDebug() << "Audio array:" << recording->getAudioData().size();
}
qint64 checksumTime = timer.elapsed();
@ -642,7 +626,7 @@ RecordingPointer readRecordingFromFile(RecordingPointer recording, const QString
qDebug() << "Recording:";
qDebug() << "Total frames:" << recording->getFrameNumber();
qDebug() << "Audio array:" << recording->getAudio()->getByteArray().size();
qDebug() << "Audio array:" << recording->getAudioData().size();
}

9
libraries/avatars/src/Recording.h
View file

@ -48,9 +48,6 @@ public:
/// Stores a recording
class Recording {
public:
Recording();
~Recording();
bool isEmpty() const { return _timestamps.isEmpty(); }
int getLength() const; // in ms
@ -58,11 +55,11 @@ public:
int getFrameNumber() const { return _frames.size(); }
qint32 getFrameTimestamp(int i) const;
const RecordingFrame& getFrame(int i) const;
Sound* getAudio() const { return _audio; }
const QByteArray& getAudioData() const { return _audioData; }
protected:
void addFrame(int timestamp, RecordingFrame& frame);
void addAudioPacket(const QByteArray& byteArray);
void addAudioPacket(const QByteArray& byteArray) { _audioData.append(byteArray); }
void clear();
private:
@ -70,7 +67,7 @@ private:
QVector<qint32> _timestamps;
QVector<RecordingFrame> _frames;
Sound* _audio;
QByteArray _audioData;
friend class Recorder;
friend class Player;