From 21076af73907ca385e789b655fe72413f2117e0f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 5 Dec 2013 14:29:55 -0800 Subject: [PATCH 1/3] hook audio loudness back to QAudioInput --- interface/src/Application.cpp | 19 ++----------------- interface/src/Audio.cpp | 9 +++++++++ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 02160b926e..ba2b302902 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -10,15 +10,6 @@ #include #include -#ifdef _WIN32 -#include "Syssocket.h" -#include "Systime.h" -#else -#include -#include -#include -#endif - #include #include #include @@ -2426,10 +2417,8 @@ void Application::updateAudio(float deltaTime) { PerformanceWarning warn(showWarnings, "Application::updateAudio()"); // Update audio stats for procedural sounds - #ifndef _WIN32 _audio.setLastAcceleration(_myAvatar.getThrust()); _audio.setLastVelocity(_myAvatar.getVelocity()); - #endif } void Application::updateCursor(float deltaTime) { @@ -2569,10 +2558,8 @@ void Application::updateAvatar(float deltaTime) { } // Get audio loudness data from audio input device - #ifndef _WIN32 - _myAvatar.getHead().setAudioLoudness(_audio.getLastInputLoudness()); - #endif - + _myAvatar.getHead().setAudioLoudness(_audio.getLastInputLoudness()); + NodeList* nodeList = NodeList::getInstance(); // send head/hand data to the avatar mixer and voxel server @@ -3203,14 +3190,12 @@ void Application::displayOverlay() { } } - #ifndef _WIN32 if (Menu::getInstance()->isOptionChecked(MenuOption::Stats)) { _audio.render(_glWidget->width(), _glWidget->height()); if (Menu::getInstance()->isOptionChecked(MenuOption::Oscilloscope)) { _audioScope.render(45, _glWidget->height() - 200); } } - #endif //noiseTest(_glWidget->width(), _glWidget->height()); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index e4f0677787..2d7997a021 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -194,6 +194,15 @@ void Audio::handleAudioInput() { // copy samples from the inputByteArray to the stereoInputBuffer memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); + // Measure the loudness of the signal from the microphone and store in audio object + float loudness = 0; + for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; i += 2) { + loudness += abs(stereoInputBuffer[i]); + } + + loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL * SAMPLE_RATE_RATIO; + _lastInputLoudness = loudness; + } else { // take samples we have in this callback and store them in the first half of the static buffer // to send off in the next callback From 5e73e803e98edec0f0cbe71bb54ac6cbceef7a3f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 5 Dec 2013 14:49:28 -0800 Subject: [PATCH 2/3] fixes for procedural audio input --- interface/src/Audio.cpp | 29 ++++++++++++++--------------- interface/src/Audio.h | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 2d7997a021..c3565a4eb0 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -186,11 +186,6 @@ void Audio::handleAudioInput() { QByteArray inputByteArray = _inputDevice->read(CALLBACK_IO_BUFFER_SIZE); if (_isBufferSendCallback) { - // this is the second half of a full buffer of data - - // zero out the monoAudioSamples array - memset(monoAudioSamples, 0, BUFFER_LENGTH_BYTES_PER_CHANNEL); - // copy samples from the inputByteArray to the stereoInputBuffer memcpy((char*) (stereoInputBuffer + bufferSizeSamples), inputByteArray.data(), inputByteArray.size()); @@ -204,6 +199,10 @@ void Audio::handleAudioInput() { _lastInputLoudness = loudness; } else { + // this is the first half of a full buffer of data + // zero out the monoAudioSamples array + memset(monoAudioSamples, 0, BUFFER_LENGTH_BYTES_PER_CHANNEL); + // take samples we have in this callback and store them in the first half of the static buffer // to send off in the next callback memcpy((char*) stereoInputBuffer, inputByteArray.data(), inputByteArray.size()); @@ -516,7 +515,7 @@ void Audio::render(int screenWidth, int screenHeight) { } // Take a pointer to the acquired microphone input samples and add procedural sounds -void Audio::addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int numSamples) { +void Audio::addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples) { const float MAX_AUDIBLE_VELOCITY = 6.0; const float MIN_AUDIBLE_VELOCITY = 0.1; const int VOLUME_BASELINE = 400; @@ -531,8 +530,8 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int // Add a noise-modulated sinewave with volume that tapers off with speed increasing if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t)(sinf((float) (_proceduralEffectSample + i) / SOUND_PITCH ) - * volume * (1.f + randFloat() * 0.25f) * speed); + monoInput[i] += (int16_t)(sinf((float) (_proceduralEffectSample + i) / SOUND_PITCH ) + * volume * (1.f + randFloat() * 0.25f) * speed); } } const float COLLISION_SOUND_CUTOFF_LEVEL = 0.01f; @@ -545,17 +544,17 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int for (int i = 0; i < numSamples; i++) { t = (float) _proceduralEffectSample + (float) i; - sample = sinf(t * _collisionSoundFrequency) + - sinf(t * _collisionSoundFrequency / DOWN_TWO_OCTAVES) + - sinf(t * _collisionSoundFrequency / DOWN_FOUR_OCTAVES * UP_MAJOR_FIFTH); + sample = sinf(t * _collisionSoundFrequency) + + sinf(t * _collisionSoundFrequency / DOWN_TWO_OCTAVES) + + sinf(t * _collisionSoundFrequency / DOWN_FOUR_OCTAVES * UP_MAJOR_FIFTH); sample *= _collisionSoundMagnitude * COLLISION_SOUND_MAX_VOLUME; int16_t collisionSample = (int16_t) sample; - inputBuffer[i] += collisionSample; + monoInput[i] += collisionSample; for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoOutput[j] += collisionSample; + stereoUpsampledOutput[j] += collisionSample; } _collisionSoundMagnitude *= _collisionSoundDuration; @@ -578,10 +577,10 @@ void Audio::addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int int16_t collisionSample = (int16_t) sample; - inputBuffer[i] += collisionSample; + monoInput[i] += collisionSample; for (int j = (i * 4); j < (i * 4) + 4; j++) { - stereoOutput[j] += collisionSample; + stereoUpsampledOutput[j] += collisionSample; } _drumSoundVolume *= (1.f - _drumSoundDecay); diff --git a/interface/src/Audio.h b/interface/src/Audio.h index b92bc08860..60f563b221 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -113,7 +113,7 @@ private: inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal - void addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int numSamples); + void addProceduralSounds(int16_t* monoInput, int16_t* stereoUpsampledOutput, int numSamples); void renderToolIcon(int screenHeight); }; From 3f9e00d977bf0b18563a160de3396222f5ddd8cd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 5 Dec 2013 15:10:01 -0800 Subject: [PATCH 3/3] fix a boneheaded mistake in AvatarMixer broadcastAvatarData --- assignment-client/src/avatars/AvatarMixer.cpp | 10 ++++++---- interface/src/Application.cpp | 2 +- libraries/shared/src/NodeList.cpp | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index c1486a170d..0f72ae9680 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -58,9 +58,8 @@ void attachAvatarDataToNode(Node* newNode) { // determine which avatars are included in the packet stream // 4) we should optimize the avatar data format to be more compact (100 bytes is pretty wasteful). void broadcastAvatarData() { - static unsigned char broadcastPacketBuffer[MAX_PACKET_SIZE]; + static unsigned char broadcastPacket[MAX_PACKET_SIZE]; static unsigned char avatarDataBuffer[MAX_PACKET_SIZE]; - unsigned char* broadcastPacket = (unsigned char*)&broadcastPacketBuffer[0]; int numHeaderBytes = populateTypeAndVersion(broadcastPacket, PACKET_TYPE_BULK_AVATAR_DATA); unsigned char* currentBufferPosition = broadcastPacket + numHeaderBytes; int packetLength = currentBufferPosition - broadcastPacket; @@ -79,7 +78,9 @@ void broadcastAvatarData() { // send back a packet with other active node data to this node for (NodeList::iterator otherNode = nodeList->begin(); otherNode != nodeList->end(); otherNode++) { if (otherNode->getLinkedData() && otherNode->getUUID() != node->getUUID()) { - unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], &*node); + + unsigned char* avatarDataEndpoint = addNodeToBroadcastPacket((unsigned char*)&avatarDataBuffer[0], + &*otherNode); int avatarDataLength = avatarDataEndpoint - (unsigned char*)&avatarDataBuffer; if (avatarDataLength + packetLength <= MAX_PACKET_SIZE) { @@ -89,7 +90,8 @@ void broadcastAvatarData() { } else { packetsSent++; //printf("packetsSent=%d packetLength=%d\n", packetsSent, packetLength); - nodeList->getNodeSocket().writeDatagram((char*) broadcastPacket, currentBufferPosition - broadcastPacket, + nodeList->getNodeSocket().writeDatagram((char*) broadcastPacket, + currentBufferPosition - broadcastPacket, node->getActiveSocket()->getAddress(), node->getActiveSocket()->getPort()); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index ba2b302902..b48a622772 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1997,7 +1997,7 @@ void Application::updateAvatars(float deltaTime, glm::vec3 mouseRayOrigin, glm:: for(NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { node->lock(); - if (node->getLinkedData() != NULL) { + if (node->getLinkedData()) { Avatar *avatar = (Avatar *)node->getLinkedData(); if (!avatar->isInitialized()) { avatar->init(); diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 3df90d4377..7f0a26a364 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -175,6 +175,7 @@ void NodeList::processBulkNodeData(const HifiSockAddr& senderAddress, unsigned c Node* bulkSendNode = nodeWithAddress(senderAddress); if (bulkSendNode) { + bulkSendNode->setLastHeardMicrostamp(usecTimestampNow()); bulkSendNode->recordBytesReceived(numTotalBytes);