Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-23 20:54:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3747 from PhilipRosedale/master

Browse source 
First pass audio mixer muting, improved tour guide, hair, injector loudness in JS
This commit is contained in:
AndrewMeadows 2014-11-06 10:36:46 -08:00
commit 843aaf4e95
15 changed files with 338 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
assignment-client/src/audio/AudioMixer.cpp
View file

@ -61,7 +61,7 @@
const float LOUDNESS_TO_DISTANCE_RATIO = 0.00001f;
const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.18;
const float DEFAULT_NOISE_MUTING_THRESHOLD = 0.003f;
const QString AUDIO_MIXER_LOGGING_TARGET_NAME = "audio-mixer";
const QString AUDIO_ENV_GROUP_KEY = "audio_env";
const QString AUDIO_BUFFER_GROUP_KEY = "audio_buffer";
@ -78,12 +78,17 @@ bool AudioMixer::_printStreamStats = false;
bool AudioMixer::_enableFilter = true;
bool AudioMixer::shouldMute(float quietestFrame, float loudestFrame) {
return (quietestFrame > _noiseMutingThreshold);
}
AudioMixer::AudioMixer(const QByteArray& packet) :
ThreadedAssignment(packet),
_trailingSleepRatio(1.0f),
_minAudibilityThreshold(LOUDNESS_TO_DISTANCE_RATIO / 2.0f),
_performanceThrottlingRatio(0.0f),
_attenuationPerDoublingInDistance(DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE),
_noiseMutingThreshold(DEFAULT_NOISE_MUTING_THRESHOLD),
_numStatFrames(0),
_sumListeners(0),
_sumMixes(0),
@ -136,6 +141,11 @@ int AudioMixer::addStreamToMixForListeningNodeWithStream(AudioMixerClientData* l
return 0;
}
// if the stream should be muted, bail
if (shouldMute(streamToAdd->getQuietestTrailingFrameLoudness(), streamToAdd->getLoudestTrailingFrameLoudness())) {
return 0;
}
float bearingRelativeAngleToSource = 0.0f;
float attenuationCoefficient = 1.0f;
int numSamplesDelay = 0;
@ -1000,7 +1010,17 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) {
qDebug() << "Attenuation per doubling in distance changed to" << _attenuationPerDoublingInDistance;
}
}
const QString NOISE_MUTING_THRESHOLD = "noise_muting_threshold";
if (audioEnvGroupObject[NOISE_MUTING_THRESHOLD].isString()) {
bool ok = false;
float noiseMutingThreshold = audioEnvGroupObject[NOISE_MUTING_THRESHOLD].toString().toFloat(&ok);
if (ok) {
_noiseMutingThreshold = noiseMutingThreshold;
qDebug() << "Noise muting threshold changed to" << _noiseMutingThreshold;
}
}
const QString FILTER_KEY = "enable_filter";
if (audioEnvGroupObject[FILTER_KEY].isBool()) {
_enableFilter = audioEnvGroupObject[FILTER_KEY].toBool();

3
assignment-client/src/audio/AudioMixer.h
View file

@ -59,6 +59,8 @@ private:
int16_t _mixSamples[NETWORK_BUFFER_LENGTH_SAMPLES_STEREO + (SAMPLE_PHASE_DELAY_AT_90 * 2)];
void perSecondActions();
bool shouldMute(float quietestFrame, float loudestFrame);
QString getReadPendingDatagramsCallsPerSecondsStatsString() const;
QString getReadPendingDatagramsPacketsPerCallStatsString() const;
@ -71,6 +73,7 @@ private:
float _minAudibilityThreshold;
float _performanceThrottlingRatio;
float _attenuationPerDoublingInDistance;
float _noiseMutingThreshold;
int _numStatFrames;
int _sumListeners;
int _sumMixes;

8
domain-server/resources/describe-settings.json
View file

@ -89,6 +89,14 @@
"default": "0.18",
"advanced": false
},
{
"name": "noise_muting_threshold",
"label": "Noise Muting Threshold",
"help": "Loudness value for noise background between 0 and 1.0 (0: mute everyone, 1.0: never mute)",
"placeholder": "0.003",
"default": "0.003",
"advanced": false
},
{
"name": "enable_filter",
"type": "checkbox",

164
examples/birdSongs.js Normal file
View file

@ -0,0 +1,164 @@
//
// birdSongs.js
// examples
//
// Copyright 2014 High Fidelity, Inc.
// Plays a sample audio file at the avatar's current location
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
// First, load a sample sound from a URL
var birds = [];
var playing = [];
var lowerCorner = { x: 0, y: 8, z: 0 };
var upperCorner = { x: 10, y: 10, z: 10 };
var RATE = 0.035;
var numPlaying = 0;
var BIRD_SIZE = 0.1;
var BIRD_VELOCITY = 2.0;
var LIGHT_RADIUS = 10.0;
var BIRD_MASTER_VOLUME = 0.5;
var useLights = true;
function randomVector(scale) {
return { x: Math.random() * scale - scale / 2.0, y: Math.random() * scale - scale / 2.0, z: Math.random() * scale - scale / 2.0 };
}
function maybePlaySound(deltaTime) {
if (Math.random() < RATE) {
// Set the location and other info for the sound to play
var whichBird = Math.floor(Math.random() * birds.length);
//print("playing sound # " + whichBird);
var options = new AudioInjectionOptions();
var position = { x: lowerCorner.x + Math.random() * (upperCorner.x - lowerCorner.x),
y: lowerCorner.y + Math.random() * (upperCorner.y - lowerCorner.y),
z: lowerCorner.z + Math.random() * (upperCorner.z - lowerCorner.z) };
options.position = position;
options.volume = BIRD_MASTER_VOLUME;
//
var entityId = Entities.addEntity({
type: "Sphere",
position: position,
dimensions: { x: BIRD_SIZE, y: BIRD_SIZE, z: BIRD_SIZE },
color: birds[whichBird].color,
lifetime: 10
});
if (useLights) {
var lightId = Entities.addEntity({
type: "Light",
position: position,
dimensions: { x: LIGHT_RADIUS, y: LIGHT_RADIUS, z: LIGHT_RADIUS },
isSpotlight: false,
diffuseColor: birds[whichBird].color,
ambientColor: { red: 0, green: 0, blue: 0 },
specularColor: { red: 255, green: 255, blue: 255 },
constantAttenuation: 0,
linearAttenuation: 4.0,
quadraticAttenuation: 2.0,
lifetime: 10
});
}
playing.push({ audioId: Audio.playSound(birds[whichBird].sound, options), entityId: entityId, lightId: lightId, color: birds[whichBird].color });
}
if (playing.length != numPlaying) {
numPlaying = playing.length;
//print("number playing = " + numPlaying);
}
for (var i = 0; i < playing.length; i++) {
if (!Audio.isInjectorPlaying(playing[i].audioId)) {
Entities.deleteEntity(playing[i].entityId);
if (useLights) {
Entities.deleteEntity(playing[i].lightId);
}
playing.splice(i, 1);
} else {
var loudness = Audio.getLoudness(playing[i].audioId);
var newColor = { red: playing[i].color.red, green: playing[i].color.green, blue: playing[i].color.blue };
if (loudness > 0.05) {
newColor.red *= (1.0 - loudness);
newColor.green *= (1.0 - loudness);
newColor.blue *= (1.0 - loudness);
}
var properties = Entities.getEntityProperties(playing[i].entityId);
var newPosition = Vec3.sum(properties.position, randomVector(BIRD_VELOCITY * deltaTime));
if (properties) {
properties.position = newPosition;
Entities.editEntity(playing[i].entityId, { position: properties.position, color: newColor });
}
if (useLights) {
var lightProperties = Entities.getEntityProperties(playing[i].lightId);
if (lightProperties) {
Entities.editEntity(playing[i].lightId, { position: newPosition, diffuseColor: newColor });
}
}
}
}
}
loadBirds();
// Connect a call back that happens every frame
Script.update.connect(maybePlaySound);
// Delete our little friends if script is stopped
Script.scriptEnding.connect(function() {
for (var i = 0; i < playing.length; i++) {
Entities.deleteEntity(playing[i].entityId);
if (useLights) {
Entities.deleteEntity(playing[i].lightId);
}
}
});
function loadBirds() {
var sound_filenames = ["bushtit_1.raw", "bushtit_2.raw", "bushtit_3.raw", "mexicanWhipoorwill.raw",
"rosyfacedlovebird.raw", "saysphoebe.raw", "westernscreechowl.raw", "bandtailedpigeon.wav", "bridledtitmouse.wav",
"browncrestedflycatcher.wav", "commonnighthawk.wav", "commonpoorwill.wav", "doublecrestedcormorant.wav",
"gambelsquail.wav", "goldcrownedkinglet.wav", "greaterroadrunner.wav","groovebilledani.wav","hairywoodpecker.wav",
"housewren.wav","hummingbird.wav", "mountainchickadee.wav", "nightjar.wav", "piebilledgrieb.wav", "pygmynuthatch.wav",
"whistlingduck.wav", "woodpecker.wav"];
var colors = [
{ red: 242, green: 207, blue: 013 },
{ red: 238, green: 94, blue: 11 },
{ red: 81, green: 30, blue: 7 },
{ red: 195, green: 176, blue: 81 },
{ red: 235, green: 190, blue: 152 },
{ red: 167, green: 99, blue: 52 },
{ red: 199, green: 122, blue: 108 },
{ red: 246, green: 220, blue: 189 },
{ red: 208, green: 145, blue: 65 },
{ red: 173, green: 120 , blue: 71 },
{ red: 132, green: 147, blue: 174 },
{ red: 164, green: 74, blue: 40 },
{ red: 131, green: 127, blue: 134 },
{ red: 209, green: 157, blue: 117 },
{ red: 205, green: 191, blue: 193 },
{ red: 193, green: 154, blue: 118 },
{ red: 205, green: 190, blue: 169 },
{ red: 199, green: 111, blue: 69 },
{ red: 221, green: 223, blue: 228 },
{ red: 115, green: 92, blue: 87 },
{ red: 214, green: 165, blue: 137 },
{ red: 160, green: 124, blue: 33 },
{ red: 117, green: 91, blue: 86 },
{ red: 113, green: 104, blue: 107 },
{ red: 216, green: 153, blue: 99 },
{ red: 242, green: 226, blue: 64 }
];
var SOUND_BASE_URL = "http://public.highfidelity.io/sounds/Animals/";
for (var i = 0; i < sound_filenames.length; i++) {
birds.push({ sound: new Sound(SOUND_BASE_URL + sound_filenames[i]),
color: colors[i]
} );
}
}

11
examples/guidedTour.js
View file

@ -12,6 +12,8 @@ var MIN_CHANGE = 2.0;
var LANDING_DISTANCE = 2.0;
var LANDING_RANDOM = 0.2;
var relativePosition;
function update(deltaTime) {
if (Math.random() < deltaTime) {
@ -26,20 +28,15 @@ function update(deltaTime) {
}
if (guide) {
relativePosition = Vec3.subtract(MyAvatar.position, lastGuidePosition);
// Check whether guide has moved, update if so
if (Vec3.length(lastGuidePosition) == 0.0) {
lastGuidePosition = guide.position;
} else {
if (Vec3.length(Vec3.subtract(lastGuidePosition, guide.position)) > MIN_CHANGE) {
var meToGuide = Vec3.multiply(Vec3.normalize(Vec3.subtract(guide.position, MyAvatar.position)), LANDING_DISTANCE);
var newPosition = Vec3.subtract(guide.position, meToGuide);
newPosition = Vec3.sum(newPosition, { x: Math.random() * LANDING_RANDOM - LANDING_RANDOM / 2.0,
y: 0,
z: Math.random() * LANDING_RANDOM - LANDING_RANDOM / 2.0 });
var newPosition = Vec3.sum(guide.position, relativePosition);
MyAvatar.position = newPosition;
lastGuidePosition = guide.position;
MyAvatar.orientation = guide.orientation;
}
}
}

46
examples/lightExample.js Normal file
View file

@ -0,0 +1,46 @@
//
// lightExample.js
// examples
//
// Created by Philip Rosedale on November 5, 2014
// Copyright 2014 High Fidelity, Inc.
//
// Makes a light right in front of your avatar, as well as a sphere at that location.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
var position = Vec3.sum(MyAvatar.position, Quat.getFront(Camera.getOrientation()));
var sphereID = Entities.addEntity({
type: "Sphere",
position: position,
dimensions: { x: 0.1, y: 0.1, z: 0.1 },
color: { red: 255, green: 255, blue: 0 }
});
var lightID = Entities.addEntity({
type: "Light",
position: position,
dimensions: { x: 1, y: 1, z: 1 },
angularVelocity: { x: 0, y: 0, z: 0 },
angularDamping: 0,
isSpotlight: false,
diffuseColor: { red: 255, green: 255, blue: 0 },
ambientColor: { red: 0, green: 0, blue: 0 },
specularColor: { red: 255, green: 255, blue: 255 },
constantAttenuation: 0,
linearAttenuation: 1,
quadraticAttenuation: 0,
exponent: 0,
cutoff: 180, // in degrees
});
Script.scriptEnding.connect(function() {
print("Deleted sphere and light");
Entities.deleteEntity(sphereID);
Entities.deleteEntity(lightID);
});

17
interface/src/Audio.cpp
View file

@ -77,6 +77,9 @@ Audio::Audio(QObject* parent) :
_isStereoInput(false),
_averagedLatency(0.0),
_lastInputLoudness(0),
_inputFrameCounter(0),
_quietestFrame(std::numeric_limits<float>::max()),
_loudestFrame(0.0f),
_timeSinceLastClip(-1.0),
_dcOffset(0),
_noiseGateMeasuredFloor(0),
@ -717,6 +720,20 @@ void Audio::handleAudioInput() {
}
_lastInputLoudness = fabs(loudness / NETWORK_BUFFER_LENGTH_SAMPLES_PER_CHANNEL);
if (_quietestFrame > _lastInputLoudness) {
_quietestFrame = _lastInputLoudness;
}
if (_loudestFrame < _lastInputLoudness) {
_loudestFrame = _lastInputLoudness;
}
const int FRAMES_FOR_NOISE_DETECTION = 400;
if (_inputFrameCounter++ > FRAMES_FOR_NOISE_DETECTION) {
_quietestFrame = std::numeric_limits<float>::max();
_loudestFrame = 0.0f;
_inputFrameCounter = 0;
}
// If Noise Gate is enabled, check and turn the gate on and off
if (!_audioSourceInjectEnabled && _noiseGateEnabled) {

3
interface/src/Audio.h
View file

@ -213,6 +213,9 @@ private:
QElapsedTimer _timeSinceLastReceived;
float _averagedLatency;
float _lastInputLoudness;
int _inputFrameCounter;
float _quietestFrame;
float _loudestFrame;
float _timeSinceLastClip;
float _dcOffset;
float _noiseGateMeasuredFloor;

21
interface/src/Hair.cpp
View file

@ -52,12 +52,10 @@ Hair::Hair(int strands,
glm::vec3 thisVertex;
for (int strand = 0; strand < _strands; strand++) {
float strandAngle = randFloat() * PI;
float azimuth;
float elevation = - (randFloat() * PI);
azimuth = PI_OVER_TWO;
if (randFloat() < 0.5f) {
azimuth *= -1.0f;
}
float azimuth = (float)strand / (float)_strands * PI * 2.0f;
float elevation = 0.0f;
glm::vec3 thisStrand(sinf(azimuth) * cosf(elevation), sinf(elevation), -cosf(azimuth) * cosf(elevation));
thisStrand *= _radius;
@ -115,11 +113,22 @@ void Hair::simulate(float deltaTime) {
glm::vec3 diff = thisPosition - _hairLastPosition[vertexIndex];
_hairPosition[vertexIndex] += diff * HAIR_DAMPING;
/*
// Resolve collisions with sphere
if (glm::length(_hairPosition[vertexIndex]) < _radius) {
_hairPosition[vertexIndex] += glm::normalize(_hairPosition[vertexIndex]) *
(_radius - glm::length(_hairPosition[vertexIndex]));
} */
// Collide with a conical body descending from the root of the hair
glm::vec3 thisVertex = _hairPosition[vertexIndex];
float depth = -thisVertex.y;
thisVertex.y = 0.0f;
const float BODY_CONE_ANGLE = 0.30;
if (glm::length(thisVertex) < depth * BODY_CONE_ANGLE) {
_hairPosition[vertexIndex] += glm::normalize(thisVertex) * (depth * BODY_CONE_ANGLE - glm::length(thisVertex));
}
// Add random thing driven by loudness
float loudnessFactor = (_loudness > SOUND_THRESHOLD) ? logf(_loudness - SOUND_THRESHOLD) / 2000.0f : 0.0f;

15
libraries/audio/src/AudioInjector.cpp
View file

@ -26,6 +26,7 @@ AudioInjector::AudioInjector(QObject* parent) :
_sound(NULL),
_options(),
_shouldStop(false),
_loudness(0.0f),
_isFinished(false),
_currentSendPosition(0)
{
@ -35,6 +36,7 @@ AudioInjector::AudioInjector(Sound* sound, const AudioInjectorOptions& injectorO
_sound(sound),
_options(injectorOptions),
_shouldStop(false),
_loudness(0.0f),
_isFinished(false),
_currentSendPosition(0)
{
@ -44,6 +46,10 @@ void AudioInjector::setOptions(AudioInjectorOptions& options) {
_options = options;
}
float AudioInjector::getLoudness() {
return _loudness;
}
const uchar MAX_INJECTOR_VOLUME = 0xFF;
void AudioInjector::injectAudio() {
@ -117,6 +123,15 @@ void AudioInjector::injectAudio() {
int bytesToCopy = std::min(((_options.isStereo()) ? 2 : 1) * NETWORK_BUFFER_LENGTH_BYTES_PER_CHANNEL,
soundByteArray.size() - _currentSendPosition);
// Measure the loudness of this frame
_loudness = 0.0f;
for (int i = 0; i < bytesToCopy; i += sizeof(int16_t)) {
_loudness += abs(*reinterpret_cast<int16_t*>(soundByteArray.data() + _currentSendPosition + i)) /
(MAX_SAMPLE_VALUE / 2.0f);
}
_loudness /= (float)(bytesToCopy / sizeof(int16_t));
memcpy(injectAudioPacket.data() + positionOptionOffset,
&_options.getPosition(),
sizeof(_options.getPosition()));

4
libraries/audio/src/AudioInjector.h
View file

@ -34,14 +34,18 @@ public slots:
void stop() { _shouldStop = true; }
void setOptions(AudioInjectorOptions& options);
void setCurrentSendPosition(int currentSendPosition) { _currentSendPosition = currentSendPosition; }
float getLoudness();
signals:
void finished();
private:
Sound* _sound;
AudioInjectorOptions _options;
bool _shouldStop;
float _loudness;
bool _isFinished;
int _currentSendPosition;
};
Q_DECLARE_METATYPE(AudioInjector*)

10
libraries/audio/src/AudioScriptingInterface.cpp
View file

@ -68,6 +68,14 @@ bool AudioScriptingInterface::isInjectorPlaying(AudioInjector* injector) {
return (injector != NULL);
}
float AudioScriptingInterface::getLoudness(AudioInjector* injector) {
if (injector) {
return injector->getLoudness();
} else {
return 0.0f;
}
}
void AudioScriptingInterface::injectorStopped() {
_activeInjectors.removeAll(QPointer<AudioInjector>(reinterpret_cast<AudioInjector*>(sender())));
}
}

4
libraries/audio/src/AudioScriptingInterface.h
View file

@ -26,6 +26,9 @@ public:
void stopAllInjectors();
public slots:
static float getLoudness(AudioInjector* injector);
AudioInjector* playSound(Sound* sound, const AudioInjectorOptions* injectorOptions = NULL);
void stopInjector(AudioInjector* injector);
bool isInjectorPlaying(AudioInjector* injector);
@ -36,5 +39,6 @@ private:
AudioScriptingInterface() {};
QList< QPointer<AudioInjector> > _activeInjectors;
};
#endif // hifi_AudioScriptingInterface_h

21
libraries/audio/src/PositionalAudioStream.cpp
View file

@ -31,7 +31,10 @@ PositionalAudioStream::PositionalAudioStream(PositionalAudioStream::Type type, b
_isStereo(isStereo),
_ignorePenumbra(false),
_lastPopOutputTrailingLoudness(0.0f),
_lastPopOutputLoudness(0.0f)
_lastPopOutputLoudness(0.0f),
_quietestTrailingFrameLoudness(std::numeric_limits<float>::max()),
_loudestTrailingFrameLoudness(0.0f),
_frameCounter(0)
{
}
@ -43,8 +46,9 @@ void PositionalAudioStream::resetStats() {
void PositionalAudioStream::updateLastPopOutputLoudnessAndTrailingLoudness() {
_lastPopOutputLoudness = _ringBuffer.getFrameLoudness(_lastPopOutput);
const int TRAILING_AVERAGE_FRAMES = 100;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES;
const int TRAILING_MUTE_THRESHOLD_FRAMES = 400;
const int TRAILING_LOUDNESS_FRAMES = 200;
const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_LOUDNESS_FRAMES;
const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO;
const float LOUDNESS_EPSILON = 0.000001f;
@ -57,6 +61,17 @@ void PositionalAudioStream::updateLastPopOutputLoudnessAndTrailingLoudness() {
_lastPopOutputTrailingLoudness = 0;
}
}
if (_frameCounter++ == TRAILING_MUTE_THRESHOLD_FRAMES) {
_frameCounter = 0;
_quietestTrailingFrameLoudness = std::numeric_limits<float>::max();
_loudestTrailingFrameLoudness = 0.0f;
}
if (_lastPopOutputLoudness < _quietestTrailingFrameLoudness) {
_quietestTrailingFrameLoudness = _lastPopOutputLoudness;
}
if (_lastPopOutputLoudness > _loudestTrailingFrameLoudness) {
_loudestTrailingFrameLoudness = _lastPopOutputLoudness;
}
}
int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteArray) {

6
libraries/audio/src/PositionalAudioStream.h
View file

@ -36,6 +36,8 @@ public:
void updateLastPopOutputLoudnessAndTrailingLoudness();
float getLastPopOutputTrailingLoudness() const { return _lastPopOutputTrailingLoudness; }
float getLastPopOutputLoudness() const { return _lastPopOutputLoudness; }
float getQuietestTrailingFrameLoudness() const { return _quietestTrailingFrameLoudness; }
float getLoudestTrailingFrameLoudness() const { return _loudestTrailingFrameLoudness; }
bool shouldLoopbackForNode() const { return _shouldLoopbackForNode; }
bool isStereo() const { return _isStereo; }
@ -43,6 +45,7 @@ public:
PositionalAudioStream::Type getType() const { return _type; }
const glm::vec3& getPosition() const { return _position; }
const glm::quat& getOrientation() const { return _orientation; }
protected:
// disallow copying of PositionalAudioStream objects
@ -63,6 +66,9 @@ protected:
float _lastPopOutputTrailingLoudness;
float _lastPopOutputLoudness;
float _quietestTrailingFrameLoudness;
float _loudestTrailingFrameLoudness;
int _frameCounter;
};
#endif // hifi_PositionalAudioStream_h