add support for head-shadow/penumbra filter for positional audio streams

This commit is contained in:
Craig Hansen-Sturm 2014-08-24 22:17:07 -07:00
parent 4b7208ac53
commit 4cd1f4afef
12 changed files with 405 additions and 188 deletions

View file

@ -75,6 +75,8 @@ int AudioMixer::_maxFramesOverDesired = 0;
bool AudioMixer::_printStreamStats = false;
bool AudioMixer::_enableFilter = false;
AudioMixer::AudioMixer(const QByteArray& packet) :
ThreadedAssignment(packet),
_trailingSleepRatio(1.0f),
@ -107,7 +109,7 @@ void AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
float weakChannelAmplitudeRatio = 1.0f;
bool shouldAttenuate = (streamToAdd != listeningNodeStream);
if (shouldAttenuate) {
// if the two stream pointers do not match then these are different streams
@ -267,6 +269,43 @@ void AudioMixer::addStreamToMixForListeningNodeWithStream(PositionalAudioStream*
MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
}
}
if ( _enableFilter && shouldAttenuate ) {
glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream->getPosition();
if ( relativePosition.z < 0 ) { // if the source is behind us
AudioFilterPEQ1s& penumbraFilter = streamToAdd->getFilter();
// calculate penumbra angle
float headPenumbraAngle = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f),
glm::normalize(relativePosition));
// normalize penumbra angle
float normalizedHeadPenumbraAngle = headPenumbraAngle / PI_OVER_TWO;
if ( normalizedHeadPenumbraAngle < EPSILON ) {
normalizedHeadPenumbraAngle = EPSILON;
}
float penumbraFilterGain;
float penumbraFilterFrequency;
float penumbraFilterSlope;
// calculate the updated gain
penumbraFilterGain = normalizedHeadPenumbraAngle; // Note this will be tuned - consider this only a crude-first pass at correlating gain with penumbra angle.
penumbraFilterFrequency = 2000.0f;
penumbraFilterSlope = 1.0f; // gentle slope
// printf("gain=%f,angle=%f\n",penumbraFilterGain,headPenumbraAngle);
// set the gain on both filter channels
penumbraFilter.setParameters(0,0,SAMPLE_RATE,penumbraFilterFrequency,penumbraFilterGain,penumbraFilterSlope);
penumbraFilter.setParameters(0,1,SAMPLE_RATE,penumbraFilterFrequency,penumbraFilterGain,penumbraFilterSlope);
penumbraFilter.render( _clientSamples, _clientSamples, NETWORK_BUFFER_LENGTH_SAMPLES_STEREO / 2);
}
}
}
void AudioMixer::prepareMixForListeningNode(Node* node) {
@ -462,6 +501,12 @@ void AudioMixer::run() {
bool ok;
const QString FILTER_KEY = "E-enable-filter";
_enableFilter = audioGroupObject[FILTER_KEY].toBool();
if (_enableFilter) {
qDebug() << "Filter enabled";
}
const QString DESIRED_JITTER_BUFFER_FRAMES_KEY = "B-desired-jitter-buffer-frames";
_staticDesiredJitterBufferFrames = audioGroupObject[DESIRED_JITTER_BUFFER_FRAMES_KEY].toString().toInt(&ok);
if (!ok) {

View file

@ -41,7 +41,7 @@ public slots:
static bool getUseDynamicJitterBuffers() { return _useDynamicJitterBuffers; }
static int getStaticDesiredJitterBufferFrames() { return _staticDesiredJitterBufferFrames; }
static int getMaxFramesOverDesired() { return _maxFramesOverDesired; }
private:
/// adds one stream to the mix for a listening node
void addStreamToMixForListeningNodeWithStream(PositionalAudioStream* streamToAdd,
@ -68,7 +68,8 @@ private:
static int _maxFramesOverDesired;
static bool _printStreamStats;
static bool _enableFilter;
quint64 _lastSendAudioStreamStatsTime;
};

View file

@ -32,7 +32,13 @@
"help": "Boxes for source and listener (corner x, corner y, corner z, size x, size y, size z, corner x, corner y, corner z, size x, size y, size z)",
"placeholder": "no zone",
"default": ""
},
"E-enable-filter": {
"type": "checkbox",
"label": "Enable Positional Filter",
"help": "If enabled, positional audio stream uses lowpass filter",
"default": false
}
}
}
}
}

View file

@ -12,14 +12,14 @@
var bird = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Animals/bushtit_1.raw");
function maybePlaySound(deltaTime) {
if (Math.random() < 0.01) {
// if (Math.random() < 0.01) {
// Set the location and other info for the sound to play
var options = new AudioInjectionOptions();
var position = MyAvatar.position;
options.position = position;
options.volume = 0.5;
Audio.playSound(bird, options);
}
// }
}
// Connect a call back that happens every frame

View file

@ -0,0 +1,42 @@
//
// playSoundPath.js
// examples
//
// Created by Craig Hansen-Sturm on 05/27/14.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
var soundClip = new Sound("https://s3-us-west-1.amazonaws.com/highfidelity-public/sounds/Voxels/voxel create 3.raw");
var currentTime = 1.570079; // pi/2
var deltaTime = 0.05;
var distance = 1;
var debug = 0;
function playSound() {
var options = new AudioInjectionOptions();
currentTime += deltaTime;
var s = distance * Math.sin(currentTime);
var c = distance * Math.cos(currentTime);
var soundOffset = { x:s, y:0, z:c };
if( debug ) {
print("t=" + currentTime + "offset=" + soundOffset.x + "," + soundOffset.y + "," + soundOffset.z);
}
var avatarPosition = MyAvatar.position;
var soundPosition = Vec3.sum(avatarPosition,soundOffset);
options.position = soundPosition
options.volume = 1.0;
Audio.playSound(soundClip, options);
}
Script.setInterval(playSound, 250);

View file

@ -20,6 +20,7 @@
#include "RingBufferHistory.h"
#include "MovingMinMaxAvg.h"
#include "AudioFilter.h"
#include "AudioFilterBank.h"
#include <QAudio>
#include <QAudioInput>
@ -278,8 +279,8 @@ private:
int _samplesPerScope;
// Multi-band parametric EQ
bool _peqEnabled;
AudioFilterPEQ3 _peq;
bool _peqEnabled;
AudioFilterPEQ3m _peq;
QMutex _guard;
QByteArray* _scopeInput;

View file

@ -1,26 +0,0 @@
//
// AudioFilter.cpp
// hifi
//
// Created by Craig Hansen-Sturm on 8/10/14.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <math.h>
#include <vector>
#include <SharedUtil.h>
#include "AudioRingBuffer.h"
#include "AudioFilter.h"
template<>
AudioFilterPEQ3::FilterParameter AudioFilterPEQ3::_profiles[ AudioFilterPEQ3::_profileCount ][ AudioFilterPEQ3::_filterCount ] = {
// Freq Gain Q Freq Gain Q Freq Gain Q
{ { 300.0f, 1.0f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 1.0f, 1.0f } }, // flat response (default)
{ { 300.0f, 1.0f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 0.1f, 1.0f } }, // treble cut
{ { 300.0f, 0.1f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 1.0f, 1.0f } }, // bass cut
{ { 300.0f, 1.5f, 0.71f }, { 1000.0f, 0.5f, 1.0f }, { 4000.0f, 1.50f, 0.71f } } // smiley curve
};

View file

@ -90,209 +90,134 @@ public:
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// Implements a single-band parametric EQ using a biquad "peaking EQ" configuration
//
// gain > 1.0 boosts the center frequency
// gain < 1.0 cuts the center frequency
//
class AudioParametricEQ {
////////////////////////////////////////////////////////////////////////////////////////////
// Implements common base class interface for all Audio Filter Objects
//
template< class T >
class AudioFilterBase {
protected:
//
// private data
// data
//
AudioBiquad _kernel;
float _sampleRate;
float _frequency;
float _gain;
float _slope;
//
// helpers
//
void updateKernel() {
/*
a0 = 1 + alpha*A
a1 = -2*cos(w0)
a2 = 1 - alpha*A
b1 = -2*cos(w0)
b2 = 1 - alpha/A
*/
const float a = _gain;
const float omega = TWO_PI * _frequency / _sampleRate;
const float alpha = 0.5f * sinf(omega) / _slope;
const float gamma = 1.0f / ( 1.0f + (alpha/a) );
const float a0 = 1.0f + (alpha*a);
const float a1 = -2.0f * cosf(omega);
const float a2 = 1.0f - (alpha*a);
const float b1 = a1;
const float b2 = 1.0f - (alpha/a);
_kernel.setParameters( a0*gamma,a1*gamma,a2*gamma,b1*gamma,b2*gamma );
static_cast<T*>(this)->updateKernel();
}
public:
//
// ctor/dtor
//
AudioParametricEQ() {
AudioFilterBase() {
setParameters(0.,0.,0.,0.);
updateKernel();
}
~AudioParametricEQ() {
~AudioFilterBase() {
}
//
// public interface
//
void setParameters( const float sampleRate, const float frequency, const float gain, const float slope ) {
_sampleRate = std::max(sampleRate,1.0f);
_frequency = std::max(frequency,2.0f);
_gain = std::max(gain,0.0f);
_slope = std::max(slope,0.00001f);
updateKernel();
}
void getParameters( float& sampleRate, float& frequency, float& gain, float& slope ) {
sampleRate = _sampleRate; frequency = _frequency; gain = _gain; slope = _slope;
}
void render(const float* in, float* out, const int frames ) {
_kernel.render(in,out,frames);
}
void reset() {
_kernel.reset();
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// Helper/convenience class that implements a bank of EQ objects
// Implements a low-shelf filter using a biquad
//
template< typename T, const int N>
class AudioFilterBank {
//
// types
//
struct FilterParameter {
float _p1;
float _p2;
float _p3;
};
//
// private static data
//
static const int _filterCount = N;
static const int _profileCount = 4;
static FilterParameter _profiles[_profileCount][_filterCount];
//
// private data
//
T _filters[ _filterCount ];
float* _buffer;
float _sampleRate;
uint16_t _frameCount;
class AudioFilterLSF :
public AudioFilterBase< AudioFilterLSF >
{
public:
//
// ctor/dtor
// helpers
//
AudioFilterBank()
: _buffer(NULL)
, _sampleRate(0.)
, _frameCount(0) {
void updateKernel() {
// TBD
}
~AudioFilterBank() {
finalize();
}
//
// public interface
//
void initialize( const float sampleRate, const int frameCount ) {
finalize();
_buffer = (float*)malloc( frameCount * sizeof(float) );
if(!_buffer) {
return;
}
_sampleRate = sampleRate;
_frameCount = frameCount;
reset();
loadProfile(0); // load default profile "flat response" into the bank (see AudioFilter.cpp)
}
void finalize() {
if (_buffer ) {
free (_buffer);
_buffer = NULL;
}
}
void loadProfile( int profileIndex ) {
if (profileIndex >= 0 && profileIndex < _profileCount) {
for (int i = 0; i < _filterCount; ++i) {
FilterParameter p = _profiles[profileIndex][i];
_filters[i].setParameters(_sampleRate,p._p1,p._p2,p._p3);
}
}
}
void render( const float* in, float* out, const int frameCount ) {
for (int i = 0; i < _filterCount; ++i) {
_filters[i].render( in, out, frameCount );
}
}
void render( const int16_t* in, int16_t* out, const int frameCount ) {
if (!_buffer || ( frameCount > _frameCount ))
return;
const int scale = (2 << ((8*sizeof(int16_t))-1));
// convert int16_t to float32 (normalized to -1. ... 1.)
for (int i = 0; i < frameCount; ++i) {
_buffer[i] = ((float)(*in++)) / scale;
}
// for this filter, we share input/output buffers at each stage, but our design does not mandate this
render( _buffer, _buffer, frameCount );
// convert float32 to int16_t
for (int i = 0; i < frameCount; ++i) {
*out++ = (int16_t)(_buffer[i] * scale);
}
}
void reset() {
for (int i = 0; i < _filterCount; ++i ) {
_filters[i].reset();
}
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// Specializations of AudioFilterBank
// Implements a hi-shelf filter using a biquad
//
typedef AudioFilterBank< AudioParametricEQ, 1> AudioFilterPEQ1; // bank with one band of PEQ
typedef AudioFilterBank< AudioParametricEQ, 2> AudioFilterPEQ2; // bank with two bands of PEQ
typedef AudioFilterBank< AudioParametricEQ, 3> AudioFilterPEQ3; // bank with three bands of PEQ
// etc....
class AudioFilterHSF :
public AudioFilterBase< AudioFilterHSF >
{
public:
//
// helpers
//
void updateKernel() {
// TBD
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// Implements a single-band parametric EQ using a biquad "peaking EQ" configuration
//
class AudioFilterPEQ :
public AudioFilterBase< AudioFilterPEQ >
{
public:
//
// helpers
//
void updateKernel() {
const float a = _gain;
const float omega = TWO_PI * _frequency / _sampleRate;
const float alpha = 0.5f * sinf(omega) / _slope;
/*
a0 = 1 + alpha*A
a1 = -2*cos(w0)
a2 = 1 - alpha*A
b1 = -2*cos(w0)
b2 = 1 - alpha/A
*/
const float a0 = 1.0f + (alpha*a);
const float a1 = -2.0f * cosf(omega);
const float a2 = 1.0f - (alpha*a);
const float b1 = a1;
const float b2 = 1.0f - (alpha/a);
const float scale = 1.0f / ( 1.0f + (alpha/a) );
_kernel.setParameters( a0*scale,a1*scale,a2*scale,b1*scale,b2*scale );
}
};
#endif // hifi_AudioFilter_h

View file

@ -0,0 +1,45 @@
//
// AudioFilterBank.cpp
// hifi
//
// Created by Craig Hansen-Sturm on 8/10/14.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include <math.h>
#include <vector.h>
#include <SharedUtil.h>
#include "AudioRingBuffer.h"
#include "AudioFilter.h"
#include "AudioFilterBank.h"
template<>
AudioFilterLSF1s::FilterParameter AudioFilterLSF1s::_profiles[ AudioFilterLSF1s::_profileCount ][ AudioFilterLSF1s::_filterCount ] = {
// Freq Gain Slope
{ { 1000.0f, 1.0f, 1.0f } } // flat response (default)
};
template<>
AudioFilterHSF1s::FilterParameter AudioFilterHSF1s::_profiles[ AudioFilterHSF1s::_profileCount ][ AudioFilterHSF1s::_filterCount ] = {
// Freq Gain Slope
{ { 1000.0f, 1.0f, 1.0f } } // flat response (default)
};
template<>
AudioFilterPEQ1s::FilterParameter AudioFilterPEQ1s::_profiles[ AudioFilterPEQ1s::_profileCount ][ AudioFilterPEQ1s::_filterCount ] = {
// Freq Gain Q
{ { 1000.0f, 1.0f, 1.0f } } // flat response (default)
};
template<>
AudioFilterPEQ3m::FilterParameter AudioFilterPEQ3m::_profiles[ AudioFilterPEQ3m::_profileCount ][ AudioFilterPEQ3m::_filterCount ] = {
// Freq Gain Q Freq Gain Q Freq Gain Q
{ { 300.0f, 1.0f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 1.0f, 1.0f } }, // flat response (default)
{ { 300.0f, 1.0f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 0.1f, 1.0f } }, // treble cut
{ { 300.0f, 0.1f, 1.0f }, { 1000.0f, 1.0f, 1.0f }, { 4000.0f, 1.0f, 1.0f } }, // bass cut
{ { 300.0f, 1.5f, 0.71f }, { 1000.0f, 0.5f, 1.0f }, { 4000.0f, 1.50f, 0.71f } } // smiley curve
};

View file

@ -0,0 +1,169 @@
//
// AudioFilterBank.h
// hifi
//
// Created by Craig Hansen-Sturm on 8/23/14.
// Copyright 2014 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AudioFilterBank_h
#define hifi_AudioFilterBank_h
////////////////////////////////////////////////////////////////////////////////////////////
// Helper/convenience class that implements a bank of Filter objects
//
template< typename T, const int N, const int C >
class AudioFilterBank {
//
// types
//
struct FilterParameter {
float _p1;
float _p2;
float _p3;
};
//
// private static data
//
static const int _filterCount = N;
static const int _channelCount = C;
static const int _profileCount = 4;
static FilterParameter _profiles[_profileCount][_filterCount];
//
// private data
//
T _filters[ _filterCount ][ _channelCount ];
float* _buffer[ _channelCount ];
float _sampleRate;
uint16_t _frameCount;
public:
//
// ctor/dtor
//
AudioFilterBank()
: _sampleRate(0.)
, _frameCount(0) {
for (int i = 0; i < _channelCount; ++i ) {
_buffer[ i ] = NULL;
}
}
~AudioFilterBank() {
finalize();
}
//
// public interface
//
void initialize( const float sampleRate, const int frameCount ) {
finalize();
for (int i = 0; i < _channelCount; ++i ) {
_buffer[i] = (float*)malloc( frameCount * sizeof( float ) );
}
_sampleRate = sampleRate;
_frameCount = frameCount;
reset();
loadProfile(0); // load default profile "flat response" into the bank (see AudioFilterBank.cpp)
}
void finalize() {
for (int i = 0; i < _channelCount; ++i ) {
if (_buffer[i]) {
free (_buffer[i]);
_buffer[i] = NULL;
}
}
}
void loadProfile( int profileIndex ) {
if (profileIndex >= 0 && profileIndex < _profileCount) {
for (int i = 0; i < _filterCount; ++i) {
FilterParameter p = _profiles[profileIndex][i];
for (int j = 0; j < _channelCount; ++j) {
_filters[i][j].setParameters(_sampleRate,p._p1,p._p2,p._p3);
}
}
}
}
void setParameters( int filterStage, int filterChannel, const float sampleRate, const float frequency, const float gain, const float slope ) {
if ( filterStage >= 0 && filterStage < _filterCount && filterChannel >= 0 && filterChannel < _channelCount ) {
_filters[filterStage][filterChannel].setParameters(sampleRate,frequency,gain,slope);
}
}
void getParameters( int filterStage, int filterChannel, float& sampleRate, float& frequency, float& gain, float& slope ) {
if ( filterStage >= 0 && filterStage < _filterCount && filterChannel >= 0 && filterChannel < _channelCount ) {
_filters[filterStage][filterChannel].getParameters(sampleRate,frequency,gain,slope);
}
}
void render( const int16_t* in, int16_t* out, const int frameCount ) {
if (!_buffer || ( frameCount > _frameCount ))
return;
const int scale = (2 << ((8*sizeof(int16_t))-1));
// de-interleave and convert int16_t to float32 (normalized to -1. ... 1.)
for (int i = 0; i < frameCount; ++i) {
for (int j = 0; j < _channelCount; ++j) {
_buffer[j][i] = ((float)(*in++)) / scale;
}
}
// now step through each filter
for (int i = 0; i < _channelCount; ++i) {
for (int j = 0; j < _filterCount; ++j ) {
_filters[j][i].render( &_buffer[i][0], &_buffer[i][0], frameCount );
}
}
// convert float32 to int16_t and interleave
for (int i = 0; i < frameCount; ++i) {
for (int j = 0; j < _channelCount; ++j) {
*out++ = (int16_t)(_buffer[j][i] * scale);
}
}
}
void reset() {
for (int i = 0; i < _filterCount; ++i ) {
for (int j = 0; j < _channelCount; ++j ) {
_filters[i][j].reset();
}
}
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// Specializations of AudioFilterBank
//
typedef AudioFilterBank< AudioFilterLSF, 1, 1> AudioFilterLSF1m; // mono bank with one band of LSF
typedef AudioFilterBank< AudioFilterLSF, 1, 2> AudioFilterLSF1s; // stereo bank with one band of LSF
typedef AudioFilterBank< AudioFilterHSF, 1, 1> AudioFilterHSF1m; // mono bank with one band of HSF
typedef AudioFilterBank< AudioFilterHSF, 1, 2> AudioFilterHSF1s; // stereo bank with one band of HSF
typedef AudioFilterBank< AudioFilterPEQ, 1, 1> AudioFilterPEQ1m; // mono bank with one band of PEQ
typedef AudioFilterBank< AudioFilterPEQ, 2, 1> AudioFilterPEQ2m; // mono bank with two bands of PEQ
typedef AudioFilterBank< AudioFilterPEQ, 3, 1> AudioFilterPEQ3m; // mono bank with three bands of PEQ
typedef AudioFilterBank< AudioFilterPEQ, 1, 2> AudioFilterPEQ1s; // stereo bank with one band of PEQ
typedef AudioFilterBank< AudioFilterPEQ, 2, 2> AudioFilterPEQ2s; // stereo bank with two bands of PEQ
typedef AudioFilterBank< AudioFilterPEQ, 3, 2> AudioFilterPEQ3s; // stereo bank with three bands of PEQ
// etc....
#endif // hifi_AudioFilter_h

View file

@ -33,6 +33,9 @@ PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, b
_lastPopOutputTrailingLoudness(0.0f),
_listenerUnattenuatedZone(NULL)
{
// constant defined in AudioMixer.h. However, we don't want to include this here, since we will soon find a better common home for these audio-related constants
const int SAMPLE_PHASE_DELAY_AT_90 = 20;
_filter.initialize( SAMPLE_RATE, ( NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2) ) / 2);
}
void PositionalAudioStream::updateLastPopOutputTrailingLoudness() {

View file

@ -16,6 +16,8 @@
#include <AABox.h>
#include "InboundAudioStream.h"
#include "AudioFilter.h"
#include "AudioFilterBank.h"
const int AUDIOMIXER_INBOUND_RING_BUFFER_FRAME_CAPACITY = 100;
@ -44,6 +46,8 @@ public:
void setListenerUnattenuatedZone(AABox* listenerUnattenuatedZone) { _listenerUnattenuatedZone = listenerUnattenuatedZone; }
AudioFilterPEQ1s& getFilter() { return _filter; }
protected:
// disallow copying of PositionalAudioStream objects
PositionalAudioStream(const PositionalAudioStream&);
@ -61,6 +65,8 @@ protected:
float _lastPopOutputTrailingLoudness;
AABox* _listenerUnattenuatedZone;
AudioFilterPEQ1s _filter;
};
#endif // hifi_PositionalAudioStream_h