Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-07 19:23:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add a sound once first AM packet received

Browse source
This commit is contained in:
Stephen Birarda 2015-06-08 11:30:12 -07:00
parent 94cd0762bd
commit 666830c04d
11 changed files with 358 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
examples/dialTone.js Normal file
View file

@ -0,0 +1,23 @@
//
// dialTone.js
// examples
//
// Created by Stephen Birarda on 06/08/15.
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
// setup the local sound we're going to use
var connectSound = SoundCache.getSound("file://" + Paths.resources + "sounds/short1.wav");
// setup the options needed for that sound
var connectSoundOptions = {
localOnly: true
}
// play the sound locally once we get the first audio packet from a mixer
Audio.receivedFirstPacket.connect(function(){
Audio.playSound(connectSound, connectSoundOptions);
});

BIN
interface/resources/sounds/short1.wav Normal file
View file

Binary file not shown.

401
interface/src/Application.cpp
View file

File diff suppressed because it is too large Load diff

11
libraries/audio-client/src/AudioClient.cpp
View file

@ -153,6 +153,7 @@ void AudioClient::reset() {
}
void AudioClient::audioMixerKilled() {
_hasReceivedFirstPacket = false;
_outgoingAvatarAudioSequenceNumber = 0;
_stats.reset();
}
@ -481,6 +482,7 @@ void AudioClient::start() {
qCDebug(audioclient) << "Unable to set up audio input because of a problem with input format.";
qCDebug(audioclient) << "The closest format available is" << inputDeviceInfo.nearestFormat(_desiredInputFormat);
}
if (!outputFormatSupported) {
qCDebug(audioclient) << "Unable to set up audio output because of a problem with output format.";
qCDebug(audioclient) << "The closest format available is" << outputDeviceInfo.nearestFormat(_desiredOutputFormat);
@ -489,6 +491,7 @@ void AudioClient::start() {
if (_audioInput) {
_inputFrameBuffer.initialize( _inputFormat.channelCount(), _audioInput->bufferSize() * 8 );
}
_inputGain.initialize();
_sourceGain.initialize();
_noiseSource.initialize();
@ -926,6 +929,14 @@ void AudioClient::addReceivedAudioToStream(const QByteArray& audioByteArray) {
DependencyManager::get<NodeList>()->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::ReceiveFirstAudioPacket);
if (_audioOutput) {
if (!_hasReceivedFirstPacket) {
_hasReceivedFirstPacket = true;
// have the audio scripting interface emit a signal to say we just connected to mixer
emit receivedFirstPacket();
}
// Audio output must exist and be correctly set up if we're going to process received audio
_receivedAudioStream.parseData(audioByteArray);
}

63
libraries/audio-client/src/AudioClient.h
View file

@ -37,6 +37,7 @@
#include <MixedProcessedAudioStream.h>
#include <RingBufferHistory.h>
#include <SettingHandle.h>
#include <Sound.h>
#include <StDev.h>
#include "AudioIOStats.h"
@ -57,7 +58,7 @@ static const int NUM_AUDIO_CHANNELS = 2;
static const int DEFAULT_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 3;
static const int MIN_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 1;
static const int MAX_AUDIO_OUTPUT_BUFFER_SIZE_FRAMES = 20;
#if defined(Q_OS_ANDROID) || defined(Q_OS_WIN)
#if defined(Q_OS_ANDROID) || defined(Q_OS_WIN)
static const int DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_ENABLED = false;
#else
static const int DEFAULT_AUDIO_OUTPUT_STARVE_DETECTION_ENABLED = true;
@ -88,14 +89,14 @@ public:
void stop() { close(); }
qint64 readData(char * data, qint64 maxSize);
qint64 writeData(const char * data, qint64 maxSize) { return 0; }
int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; }
private:
MixedProcessedAudioStream& _receivedAudioStream;
AudioClient* _audio;
int _unfulfilledReads;
};
const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }
MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }
@ -105,30 +106,30 @@ public:
float getAudioAverageInputLoudness() const { return _lastInputLoudness; }
int getDesiredJitterBufferFrames() const { return _receivedAudioStream.getDesiredJitterBufferFrames(); }
bool isMuted() { return _muted; }
const AudioIOStats& getStats() const { return _stats; }
float getInputRingBufferMsecsAvailable() const;
float getAudioOutputMsecsUnplayed() const;
int getOutputBufferSize() { return _outputBufferSizeFrames.get(); }
bool getOutputStarveDetectionEnabled() { return _outputStarveDetectionEnabled.get(); }
void setOutputStarveDetectionEnabled(bool enabled) { _outputStarveDetectionEnabled.set(enabled); }
int getOutputStarveDetectionPeriod() { return _outputStarveDetectionPeriodMsec.get(); }
void setOutputStarveDetectionPeriod(int msecs) { _outputStarveDetectionPeriodMsec.set(msecs); }
int getOutputStarveDetectionThreshold() { return _outputStarveDetectionThreshold.get(); }
void setOutputStarveDetectionThreshold(int threshold) { _outputStarveDetectionThreshold.set(threshold); }
void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }
void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }
static const float CALLBACK_ACCELERATOR_RATIO;
public slots:
void start();
void stop();
@ -140,7 +141,7 @@ public slots:
void reset();
void audioMixerKilled();
void toggleMute();
virtual void enableAudioSourceInject(bool enable);
virtual void selectAudioSourcePinkNoise();
virtual void selectAudioSourceSine440();
@ -148,10 +149,10 @@ public slots:
virtual void setIsStereoInput(bool stereo);
void toggleAudioNoiseReduction() { _isNoiseGateEnabled = !_isNoiseGateEnabled; }
void toggleLocalEcho() { _shouldEchoLocally = !_shouldEchoLocally; }
void toggleServerEcho() { _shouldEchoToServer = !_shouldEchoToServer; }
void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);
void sendMuteEnvironmentPacket();
@ -172,10 +173,10 @@ public slots:
void setReverbOptions(const AudioEffectOptions* options);
void outputNotify();
void loadSettings();
void saveSettings();
signals:
bool muteToggled();
void inputReceived(const QByteArray& inputSamples);
@ -184,14 +185,16 @@ signals:
void deviceChanged();
void receivedFirstPacket();
protected:
AudioClient();
~AudioClient();
virtual void customDeleter() {
deleteLater();
}
private:
void outputFormatChanged();
@ -216,35 +219,35 @@ private:
QString _inputAudioDeviceName;
QString _outputAudioDeviceName;
quint64 _outputStarveDetectionStartTimeMsec;
int _outputStarveDetectionCount;
Setting::Handle<int> _outputBufferSizeFrames;
Setting::Handle<bool> _outputStarveDetectionEnabled;
Setting::Handle<int> _outputStarveDetectionPeriodMsec;
// Maximum number of starves per _outputStarveDetectionPeriod before increasing buffer size
Setting::Handle<int> _outputStarveDetectionThreshold;
StDev _stdev;
QElapsedTimer _timeSinceLastReceived;
float _averagedLatency;
float _lastInputLoudness;
float _timeSinceLastClip;
int _totalInputAudioSamples;
bool _muted;
bool _shouldEchoLocally;
bool _shouldEchoToServer;
bool _isNoiseGateEnabled;
bool _audioSourceInjectEnabled;
bool _reverb;
AudioEffectOptions _scriptReverbOptions;
AudioEffectOptions _zoneReverbOptions;
AudioEffectOptions* _reverbOptions;
ty_gverb* _gverb;
// possible soxr streams needed for resample
soxr* _inputToNetworkResampler;
soxr* _networkToOutputResampler;
@ -268,17 +271,17 @@ private:
// Input framebuffer
AudioBufferFloat32 _inputFrameBuffer;
// Input gain
AudioGain _inputGain;
// Post tone/pink noise generator gain
AudioGain _sourceGain;
// Pink noise source
bool _noiseSourceEnabled;
AudioSourcePinkNoise _noiseSource;
// Tone source
bool _toneSourceEnabled;
AudioSourceTone _toneSource;
@ -286,17 +289,19 @@ private:
quint16 _outgoingAvatarAudioSequenceNumber;
AudioOutputIODevice _audioOutputIODevice;
AudioIOStats _stats;
AudioNoiseGate _inputGate;
AudioPositionGetter _positionGetter;
AudioOrientationGetter _orientationGetter;
QVector<QString> _inputDevices;
QVector<QString> _outputDevices;
void checkDevices();
bool _hasReceivedFirstPacket = false;
};

100
libraries/audio/src/AudioInjector.cpp
View file

@ -26,7 +26,7 @@
AudioInjector::AudioInjector(QObject* parent) :
QObject(parent)
{
}
AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorOptions) :
@ -39,24 +39,24 @@ AudioInjector::AudioInjector(const QByteArray& audioData, const AudioInjectorOpt
_audioData(audioData),
_options(injectorOptions)
{
}
void AudioInjector::setIsFinished(bool isFinished) {
_isFinished = isFinished;
if (_isFinished) {
emit finished();
if (_localBuffer) {
_localBuffer->stop();
_localBuffer->deleteLater();
_localBuffer = NULL;
}
_isStarted = false;
_shouldStop = false;
if (_shouldDeleteAfterFinish) {
// we've been asked to delete after finishing, trigger a queued deleteLater here
qCDebug(audio) << "AudioInjector triggering delete from setIsFinished";
@ -69,16 +69,16 @@ void AudioInjector::injectAudio() {
if (!_isStarted) {
// check if we need to offset the sound by some number of seconds
if (_options.secondOffset > 0.0f) {
// convert the offset into a number of bytes
int byteOffset = (int) floorf(AudioConstants::SAMPLE_RATE * _options.secondOffset * (_options.stereo ? 2.0f : 1.0f));
byteOffset *= sizeof(int16_t);
_currentSendPosition = byteOffset;
} else {
_currentSendPosition = 0;
}
if (_options.localOnly) {
injectLocally();
} else {
@ -86,7 +86,7 @@ void AudioInjector::injectAudio() {
}
} else {
qCDebug(audio) << "AudioInjector::injectAudio called but already started.";
}
}
}
void AudioInjector::restart() {
@ -100,37 +100,37 @@ void AudioInjector::injectLocally() {
bool success = false;
if (_localAudioInterface) {
if (_audioData.size() > 0) {
_localBuffer = new AudioInjectorLocalBuffer(_audioData, this);
_localBuffer->open(QIODevice::ReadOnly);
_localBuffer->setShouldLoop(_options.loop);
_localBuffer->setVolume(_options.volume);
// give our current send position to the local buffer
_localBuffer->setCurrentOffset(_currentSendPosition);
success = _localAudioInterface->outputLocalInjector(_options.stereo, this);
// if we're not looping and the buffer tells us it is empty then emit finished
connect(_localBuffer, &AudioInjectorLocalBuffer::bufferEmpty, this, &AudioInjector::stop);
if (!success) {
qCDebug(audio) << "AudioInjector::injectLocally could not output locally via _localAudioInterface";
}
} else {
qCDebug(audio) << "AudioInjector::injectLocally called without any data in Sound QByteArray";
}
} else {
qCDebug(audio) << "AudioInjector::injectLocally cannot inject locally with no local audio interface present.";
}
if (!success) {
// we never started so we are finished, call our stop method
stop();
}
}
const uchar MAX_INJECTOR_VOLUME = 0xFF;
@ -140,65 +140,65 @@ void AudioInjector::injectToMixer() {
_currentSendPosition >= _audioData.size()) {
_currentSendPosition = 0;
}
auto nodeList = DependencyManager::get<NodeList>();
// make sure we actually have samples downloaded to inject
if (_audioData.size()) {
// setup the packet for injected audio
QByteArray injectAudioPacket = nodeList->byteArrayWithPopulatedHeader(PacketTypeInjectAudio);
QDataStream packetStream(&injectAudioPacket, QIODevice::Append);
// pack some placeholder sequence number for now
int numPreSequenceNumberBytes = injectAudioPacket.size();
packetStream << (quint16)0;
// pack stream identifier (a generated UUID)
packetStream << QUuid::createUuid();
// pack the stereo/mono type of the stream
packetStream << _options.stereo;
// pack the flag for loopback
uchar loopbackFlag = (uchar) true;
packetStream << loopbackFlag;
// pack the position for injected audio
int positionOptionOffset = injectAudioPacket.size();
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.position),
sizeof(_options.position));
// pack our orientation for injected audio
int orientationOptionOffset = injectAudioPacket.size();
packetStream.writeRawData(reinterpret_cast<const char*>(&_options.orientation),
sizeof(_options.orientation));
// pack zero for radius
float radius = 0;
packetStream << radius;
// pack 255 for attenuation byte
int volumeOptionOffset = injectAudioPacket.size();
quint8 volume = MAX_INJECTOR_VOLUME * _options.volume;
packetStream << volume;
packetStream << _options.ignorePenumbra;
QElapsedTimer timer;
timer.start();
int nextFrame = 0;
int numPreAudioDataBytes = injectAudioPacket.size();
bool shouldLoop = _options.loop;
// loop to send off our audio in NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL byte chunks
quint16 outgoingInjectedAudioSequenceNumber = 0;
while (_currentSendPosition < _audioData.size() && !_shouldStop) {
int bytesToCopy = std::min(((_options.stereo) ? 2 : 1) * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL,
_audioData.size() - _currentSendPosition);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
@ -215,45 +215,45 @@ void AudioInjector::injectToMixer() {
sizeof(_options.orientation));
volume = MAX_INJECTOR_VOLUME * _options.volume;
memcpy(injectAudioPacket.data() + volumeOptionOffset, &volume, sizeof(volume));
// resize the QByteArray to the right size
injectAudioPacket.resize(numPreAudioDataBytes + bytesToCopy);
// pack the sequence number
memcpy(injectAudioPacket.data() + numPreSequenceNumberBytes,
&outgoingInjectedAudioSequenceNumber, sizeof(quint16));
// copy the next NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL bytes to the packet
memcpy(injectAudioPacket.data() + numPreAudioDataBytes,
_audioData.data() + _currentSendPosition, bytesToCopy);
// grab our audio mixer from the NodeList, if it exists
SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
// send off this audio packet
nodeList->writeDatagram(injectAudioPacket, audioMixer);
outgoingInjectedAudioSequenceNumber++;
_currentSendPosition += bytesToCopy;
// send two packets before the first sleep so the mixer can start playback right away
if (_currentSendPosition != bytesToCopy && _currentSendPosition < _audioData.size()) {
// process events in case we have been told to stop and be deleted
QCoreApplication::processEvents();
if (_shouldStop) {
break;
}
// not the first packet and not done
// sleep for the appropriate time
int usecToSleep = (++nextFrame * AudioConstants::NETWORK_FRAME_USECS) - timer.nsecsElapsed() / 1000;
if (usecToSleep > 0) {
usleep(usecToSleep);
}
}
}
if (shouldLoop && _currentSendPosition >= _audioData.size()) {
@ -261,13 +261,13 @@ void AudioInjector::injectToMixer() {
}
}
}
setIsFinished(true);
}
void AudioInjector::stop() {
_shouldStop = true;
if (_options.localOnly) {
// we're only a local injector, so we can say we are finished right away too
setIsFinished(true);

34
libraries/audio/src/Sound.cpp
View file

@ -53,21 +53,25 @@ Sound::Sound(const QUrl& url, bool isStereo) :
_isStereo(isStereo),
_isReady(false)
{
}
void Sound::downloadFinished(QNetworkReply* reply) {
// replace our byte array with the downloaded data
QByteArray rawAudioByteArray = reply->readAll();
QString fileName = reply->url().fileName();
if (reply->hasRawHeader("Content-Type")) {
const QString WAV_EXTENSION = ".wav";
if (reply->hasRawHeader("Content-Type") || fileName.endsWith(WAV_EXTENSION)) {
QByteArray headerContentType = reply->rawHeader("Content-Type");
// WAV audio file encountered
if (headerContentType == "audio/x-wav"
|| headerContentType == "audio/wav"
|| headerContentType == "audio/wave") {
|| headerContentType == "audio/wave"
|| fileName.endsWith(WAV_EXTENSION)) {
QByteArray outputAudioByteArray;
@ -80,7 +84,7 @@ void Sound::downloadFinished(QNetworkReply* reply) {
_isStereo = true;
qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << reply->url() << "as stereo audio file.";
}
// Process as RAW file
downSample(rawAudioByteArray);
}
@ -88,7 +92,7 @@ void Sound::downloadFinished(QNetworkReply* reply) {
} else {
qCDebug(audio) << "Network reply without 'Content-Type'.";
}
_isReady = true;
reply->deleteLater();
}
@ -99,16 +103,16 @@ void Sound::downSample(const QByteArray& rawAudioByteArray) {
// we want to convert it to the format that the audio-mixer wants
// which is signed, 16-bit, 24Khz
int numSourceSamples = rawAudioByteArray.size() / sizeof(AudioConstants::AudioSample);
int numDestinationBytes = rawAudioByteArray.size() / sizeof(AudioConstants::AudioSample);
if (_isStereo && numSourceSamples % 2 != 0) {
numDestinationBytes += sizeof(AudioConstants::AudioSample);
}
_byteArray.resize(numDestinationBytes);
int16_t* sourceSamples = (int16_t*) rawAudioByteArray.data();
int16_t* destinationSamples = (int16_t*) _byteArray.data();
@ -134,22 +138,22 @@ void Sound::downSample(const QByteArray& rawAudioByteArray) {
}
void Sound::trimFrames() {
const uint32_t inputFrameCount = _byteArray.size() / sizeof(int16_t);
const uint32_t trimCount = 1024; // number of leading and trailing frames to trim
if (inputFrameCount <= (2 * trimCount)) {
return;
}
int16_t* inputFrameData = (int16_t*)_byteArray.data();
AudioEditBufferFloat32 editBuffer(1, inputFrameCount);
editBuffer.copyFrames(1, inputFrameCount, inputFrameData, false /*copy in*/);
editBuffer.linearFade(0, trimCount, true);
editBuffer.linearFade(inputFrameCount - trimCount, inputFrameCount, false);
editBuffer.copyFrames(1, inputFrameCount, inputFrameData, true /*copy out*/);
}
@ -238,7 +242,7 @@ void Sound::interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& ou
} else if (qFromLittleEndian<quint16>(fileHeader.wave.numChannels) > 2) {
qCDebug(audio) << "Currently not support audio files with more than 2 channels.";
}
if (qFromLittleEndian<quint16>(fileHeader.wave.bitsPerSample) != 16) {
qCDebug(audio) << "Currently not supporting non 16bit audio files.";
return;

6
libraries/script-engine/src/AudioScriptingInterface.cpp
View file

@ -28,19 +28,19 @@ AudioScriptingInterface& AudioScriptingInterface::getInstance() {
AudioScriptingInterface::AudioScriptingInterface() :
_localAudioInterface(NULL)
{
}
ScriptAudioInjector* AudioScriptingInterface::playSound(Sound* sound, const AudioInjectorOptions& injectorOptions) {
if (QThread::currentThread() != thread()) {
ScriptAudioInjector* injector = NULL;
QMetaObject::invokeMethod(this, "playSound", Qt::BlockingQueuedConnection,
Q_RETURN_ARG(ScriptAudioInjector*, injector),
Q_ARG(Sound*, sound), Q_ARG(const AudioInjectorOptions&, injectorOptions));
return injector;
}
if (sound) {
// stereo option isn't set from script, this comes from sound metadata or filename
AudioInjectorOptions optionsCopy = injectorOptions;

9
libraries/script-engine/src/AudioScriptingInterface.h
View file

@ -22,22 +22,23 @@ class AudioScriptingInterface : public QObject {
Q_OBJECT
public:
static AudioScriptingInterface& getInstance();
void setLocalAudioInterface(AbstractAudioInterface* audioInterface) { _localAudioInterface = audioInterface; }
protected:
// this method is protected to stop C++ callers from calling, but invokable from script
Q_INVOKABLE ScriptAudioInjector* playSound(Sound* sound, const AudioInjectorOptions& injectorOptions = AudioInjectorOptions());
Q_INVOKABLE void injectGeneratedNoise(bool inject);
Q_INVOKABLE void selectPinkNoise();
Q_INVOKABLE void selectSine440();
Q_INVOKABLE void setStereoInput(bool stereo);
signals:
void mutedByMixer();
void environmentMuted();
void receivedFirstPacket();
private:
AudioScriptingInterface();

2
libraries/shared/src/PathUtils.cpp
View file

@ -18,7 +18,7 @@
#include "PathUtils.h"
QString& PathUtils::resourcesPath() {
const QString& PathUtils::resourcesPath() {
#ifdef Q_OS_MAC
static QString staticResourcePath = QCoreApplication::applicationDirPath() + "/../Resources/";
#else

13
libraries/shared/src/PathUtils.h
View file

@ -12,12 +12,17 @@
#ifndef hifi_PathUtils_h
#define hifi_PathUtils_h
#include <QtCore/QObject>
#include <QString>
#include "DependencyManager.h"
namespace PathUtils {
QString& resourcesPath();
}
class PathUtils : public QObject, public Dependency {
Q_OBJECT
SINGLETON_DEPENDENCY
Q_PROPERTY(QString resources READ resourcesPath)
public:
static const QString& resourcesPath();
};
QString fileNameWithoutExtension(const QString& fileName, const QVector<QString> possibleExtensions);
QString findMostRecentFileExtension(const QString& originalFileName, QVector<QString> possibleExtensions);