Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-23 16:14:01 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #2392 from birarda/audio-scaling

Browse source 
allow Agents to send microphone audio using Sound objects
This commit is contained in:
Philip Rosedale 2014-03-19 17:00:46 -07:00
commit 22048aaba5
5 changed files with 73 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
assignment-client/src/Agent.cpp
View file

@ -26,8 +26,7 @@
Agent::Agent(const QByteArray& packet) :
ThreadedAssignment(packet),
_voxelEditSender(),
_particleEditSender(),
_avatarAudioStream(NULL)
_particleEditSender()
{
// be the parent of the script engine so it gets moved when we do
_scriptEngine.setParent(this);
@ -36,30 +35,6 @@ Agent::Agent(const QByteArray& packet) :
_scriptEngine.getParticlesScriptingInterface()->setPacketSender(&_particleEditSender);
}
Agent::~Agent() {
delete _avatarAudioStream;
}
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
void Agent::setSendAvatarAudioStream(bool sendAvatarAudioStream) {
if (sendAvatarAudioStream) {
// the agentAudioStream number of samples is related to the ScriptEngine callback rate
_avatarAudioStream = new int16_t[SCRIPT_AUDIO_BUFFER_SAMPLES];
// fill the _audioStream with zeroes to start
memset(_avatarAudioStream, 0, SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t));
_scriptEngine.setNumAvatarAudioBufferSamples(SCRIPT_AUDIO_BUFFER_SAMPLES);
_scriptEngine.setAvatarAudioBuffer(_avatarAudioStream);
} else {
delete _avatarAudioStream;
_avatarAudioStream = NULL;
_scriptEngine.setAvatarAudioBuffer(NULL);
}
}
void Agent::readPendingDatagrams() {
QByteArray receivedPacket;
HifiSockAddr senderSockAddr;

14
assignment-client/src/Agent.h
View file

@ -28,20 +28,24 @@ class Agent : public ThreadedAssignment {
Q_OBJECT
Q_PROPERTY(bool isAvatar READ isAvatar WRITE setIsAvatar)
Q_PROPERTY(bool sendAvatarAudioStream READ isSendingAvatarAudioStream WRITE setSendAvatarAudioStream)
Q_PROPERTY(bool isPlayingAvatarSound READ isPlayingAvatarSound)
Q_PROPERTY(bool isListeningToAudioStream READ isListeningToAudioStream WRITE setIsListeningToAudioStream)
public:
Agent(const QByteArray& packet);
~Agent();
void setIsAvatar(bool isAvatar) { QMetaObject::invokeMethod(&_scriptEngine, "setIsAvatar", Q_ARG(bool, isAvatar)); }
bool isAvatar() const { return _scriptEngine.isAvatar(); }
void setSendAvatarAudioStream(bool sendAvatarAudioStream);
bool isSendingAvatarAudioStream() const { return (bool) _scriptEngine.sendsAvatarAudioStream(); }
bool isPlayingAvatarSound() const { return _scriptEngine.isPlayingAvatarSound(); }
bool isListeningToAudioStream() const { return _scriptEngine.isListeningToAudioStream(); }
void setIsListeningToAudioStream(bool isListeningToAudioStream)
{ _scriptEngine.setIsListeningToAudioStream(isListeningToAudioStream); }
public slots:
void run();
void readPendingDatagrams();
void playAvatarSound(Sound* avatarSound) { _scriptEngine.setAvatarSound(avatarSound); }
private:
ScriptEngine _scriptEngine;
@ -50,8 +54,6 @@ private:
ParticleTreeHeadlessViewer _particleViewer;
VoxelTreeHeadlessViewer _voxelViewer;
int16_t* _avatarAudioStream;
};
#endif /* defined(__hifi__Agent__) */

21
examples/bot.js
View file

@ -29,7 +29,6 @@ var CHANCE_OF_BIG_MOVE = 0.1;
var isMoving = false;
var isTurningHead = false;
var isPlayingAudio = false;
var X_MIN = 0.0;
var X_MAX = 5.0;
@ -60,20 +59,11 @@ function clamp(val, min, max){
}
// Play a random sound from a list of conversational audio clips
function audioDone() {
isPlayingAudio = false;
}
var AVERAGE_AUDIO_LENGTH = 8000;
function playRandomSound(position) {
if (!isPlayingAudio) {
function playRandomSound() {
if (!Agent.isPlayingAvatarSound) {
var whichSound = Math.floor((Math.random() * sounds.length) % sounds.length);
var audioOptions = new AudioInjectionOptions();
audioOptions.volume = 0.25 + (Math.random() * 0.75);
audioOptions.position = position;
Audio.playSound(sounds[whichSound], audioOptions);
isPlayingAudio = true;
Script.setTimeout(audioDone, AVERAGE_AUDIO_LENGTH);
Audio.playSound(sounds[whichSound]);
}
}
@ -104,6 +94,7 @@ Avatar.skeletonModelURL = "https://s3-us-west-1.amazonaws.com/highfidelity-publi
Avatar.billboardURL = "https://s3-us-west-1.amazonaws.com/highfidelity-public/meshes/billboards/bot" + botNumber + ".png";
Agent.isAvatar = true;
Agent.isListeningToAudioStream = true;
// change the avatar's position to the random one
Avatar.position = firstPosition;
@ -111,10 +102,10 @@ printVector("New bot, position = ", Avatar.position);
function updateBehavior(deltaTime) {
if (Math.random() < CHANCE_OF_SOUND) {
playRandomSound(Avatar.position);
playRandomSound();
}
if (isPlayingAudio) {
if (Agent.isPlayingAvatarSound) {
Avatar.handPosition = Vec3.sum(Avatar.position, Quat.getFront(Avatar.orientation));
}

65
libraries/script-engine/src/ScriptEngine.cpp
View file

@ -14,6 +14,7 @@
#include <QtNetwork/QNetworkRequest>
#include <QtNetwork/QNetworkReply>
#include <AudioRingBuffer.h>
#include <AvatarData.h>
#include <NodeList.h>
#include <PacketHeaders.h>
@ -52,7 +53,9 @@ ScriptEngine::ScriptEngine(const QString& scriptContents, bool wantMenuItems, co
_avatarIdentityTimer(NULL),
_avatarBillboardTimer(NULL),
_timerFunctionMap(),
_avatarAudioBuffer(NULL),
_isListeningToAudioStream(false),
_avatarSound(NULL),
_numAvatarSoundSentBytes(0),
_controllerScriptingInterface(controllerScriptingInterface),
_avatarData(NULL),
_wantMenuItems(wantMenuItems),
@ -260,27 +263,55 @@ void ScriptEngine::run() {
}
if (_isAvatar && _avatarData) {
const int SCRIPT_AUDIO_BUFFER_SAMPLES = floor(((SCRIPT_DATA_CALLBACK_USECS * SAMPLE_RATE) / (1000 * 1000)) + 0.5);
const int SCRIPT_AUDIO_BUFFER_BYTES = SCRIPT_AUDIO_BUFFER_SAMPLES * sizeof(int16_t);
QByteArray avatarPacket = byteArrayWithPopulatedHeader(PacketTypeAvatarData);
avatarPacket.append(_avatarData->toByteArray());
nodeList->broadcastToNodes(avatarPacket, NodeSet() << NodeType::AvatarMixer);
if (_avatarAudioBuffer && _numAvatarAudioBufferSamples > 0) {
// if have an avatar audio stream then send it out to our audio-mixer
if (_isListeningToAudioStream || _avatarSound) {
// if we have an avatar audio stream then send it out to our audio-mixer
bool silentFrame = true;
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < _numAvatarAudioBufferSamples; ++i) {
if (_avatarAudioBuffer[i] != 0) {
silentFrame = false;
break;
int16_t numAvailableSamples = SCRIPT_AUDIO_BUFFER_SAMPLES;
const int16_t* nextSoundOutput = NULL;
if (_avatarSound) {
const QByteArray& soundByteArray = _avatarSound->getByteArray();
nextSoundOutput = reinterpret_cast<const int16_t*>(soundByteArray.data()
+ _numAvatarSoundSentBytes);
int numAvailableBytes = (soundByteArray.size() - _numAvatarSoundSentBytes) > SCRIPT_AUDIO_BUFFER_BYTES
? SCRIPT_AUDIO_BUFFER_BYTES
: soundByteArray.size() - _numAvatarSoundSentBytes;
numAvailableSamples = numAvailableBytes / sizeof(int16_t);
// check if the all of the _numAvatarAudioBufferSamples to be sent are silence
for (int i = 0; i < numAvailableSamples; ++i) {
if (nextSoundOutput[i] != 0) {
silentFrame = false;
break;
}
}
_numAvatarSoundSentBytes += numAvailableBytes;
if (_numAvatarSoundSentBytes == soundByteArray.size()) {
// we're done with this sound object - so set our pointer back to NULL
// and our sent bytes back to zero
_avatarSound = NULL;
_numAvatarSoundSentBytes = 0;
}
}
QByteArray audioPacket = byteArrayWithPopulatedHeader(silentFrame
? PacketTypeSilentAudioFrame
: PacketTypeMicrophoneAudioNoEcho);
QDataStream packetStream(&audioPacket, QIODevice::Append);
// use the orientation and position of this avatar for the source of this audio
@ -289,13 +320,17 @@ void ScriptEngine::run() {
packetStream.writeRawData(reinterpret_cast<const char*>(&headOrientation), sizeof(glm::quat));
if (silentFrame) {
if (!_isListeningToAudioStream) {
// if we have a silent frame and we're not listening then just send nothing and break out of here
break;
}
// write the number of silent samples so the audio-mixer can uphold timing
int16_t numSilentSamples = _numAvatarAudioBufferSamples;
packetStream.writeRawData(reinterpret_cast<const char*>(&numSilentSamples), sizeof(int16_t));
} else {
packetStream.writeRawData(reinterpret_cast<const char*>(&SCRIPT_AUDIO_BUFFER_SAMPLES), sizeof(int16_t));
} else if (nextSoundOutput) {
// write the raw audio data
packetStream.writeRawData(reinterpret_cast<const char*>(_avatarAudioBuffer),
_numAvatarAudioBufferSamples * sizeof(int16_t));
packetStream.writeRawData(reinterpret_cast<const char*>(nextSoundOutput),
numAvailableSamples * sizeof(int16_t));
}
nodeList->broadcastToNodes(audioPacket, NodeSet() << NodeType::AudioMixer);
@ -303,7 +338,7 @@ void ScriptEngine::run() {
}
qint64 now = usecTimestampNow();
float deltaTime = (float)(now - lastUpdate)/(float)USECS_PER_SECOND;
float deltaTime = (float) (now - lastUpdate) / (float) USECS_PER_SECOND;
emit update(deltaTime);
lastUpdate = now;

14
libraries/script-engine/src/ScriptEngine.h
View file

@ -56,10 +56,11 @@ public:
void setAvatarData(AvatarData* avatarData, const QString& objectName);
void setAvatarAudioBuffer(int16_t* avatarAudioBuffer) { _avatarAudioBuffer = avatarAudioBuffer; }
bool sendsAvatarAudioStream() const { return (bool) _avatarAudioBuffer; }
void setNumAvatarAudioBufferSamples(int numAvatarAudioBufferSamples)
{ _numAvatarAudioBufferSamples = numAvatarAudioBufferSamples; }
bool isListeningToAudioStream() const { return _isListeningToAudioStream; }
void setIsListeningToAudioStream(bool isListeningToAudioStream) { _isListeningToAudioStream = isListeningToAudioStream; }
void setAvatarSound(Sound* avatarSound) { _avatarSound = avatarSound; }
bool isPlayingAvatarSound() const { return _avatarSound != NULL; }
void init();
void run(); /// runs continuously until Agent.stop() is called
@ -91,8 +92,9 @@ protected:
QTimer* _avatarIdentityTimer;
QTimer* _avatarBillboardTimer;
QHash<QTimer*, QScriptValue> _timerFunctionMap;
int16_t* _avatarAudioBuffer;
int _numAvatarAudioBufferSamples;
bool _isListeningToAudioStream;
Sound* _avatarSound;
int _numAvatarSoundSentBytes;
private:
void sendAvatarIdentityPacket();