From 232f79489dcbb40f6033511a03d70dac5f5d3f16 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 22 Aug 2013 13:12:53 -0700 Subject: [PATCH] decouple AM logic from main.cpp to be assignable --- assignment-client/CMakeLists.txt | 6 +- assignment-client/src/main.cpp | 3 + audio-mixer/CMakeLists.txt | 12 - audio-mixer/src/main.cpp | 402 +---------------- libraries/audio/CMakeLists.txt | 9 +- libraries/audio/src/AudioMixer.cpp | 419 ++++++++++++++++++ libraries/audio/src/AudioMixer.h | 17 + .../audio}/src/AvatarAudioRingBuffer.cpp | 0 .../audio}/src/AvatarAudioRingBuffer.h | 0 .../audio}/src/InjectedAudioRingBuffer.cpp | 0 .../audio}/src/InjectedAudioRingBuffer.h | 2 +- .../audio}/src/PositionalAudioRingBuffer.cpp | 0 .../audio}/src/PositionalAudioRingBuffer.h | 2 +- libraries/shared/src/NodeList.cpp | 2 +- 14 files changed, 456 insertions(+), 418 deletions(-) create mode 100644 libraries/audio/src/AudioMixer.cpp create mode 100644 libraries/audio/src/AudioMixer.h rename {audio-mixer => libraries/audio}/src/AvatarAudioRingBuffer.cpp (100%) rename {audio-mixer => libraries/audio}/src/AvatarAudioRingBuffer.h (100%) rename {audio-mixer => libraries/audio}/src/InjectedAudioRingBuffer.cpp (100%) rename {audio-mixer => libraries/audio}/src/InjectedAudioRingBuffer.h (97%) rename {audio-mixer => libraries/audio}/src/PositionalAudioRingBuffer.cpp (100%) rename {audio-mixer => libraries/audio}/src/PositionalAudioRingBuffer.h (98%) diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index af7ae621d0..62df071de0 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -5,9 +5,13 @@ set(TARGET_NAME assignment-client) set(ROOT_DIR ..) set(MACRO_DIR ${ROOT_DIR}/cmake/macros) +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules/") + include(${MACRO_DIR}/SetupHifiProject.cmake) setup_hifi_project(${TARGET_NAME} TRUE) # link in the shared library include(${MACRO_DIR}/LinkHifiLibrary.cmake) -link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) \ No newline at end of file +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) +link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) \ No newline at end of file diff --git a/assignment-client/src/main.cpp b/assignment-client/src/main.cpp index 1217aa86ff..43d162f86a 100644 --- a/assignment-client/src/main.cpp +++ b/assignment-client/src/main.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,8 @@ int main(int argc, const char* argv[]) { qDebug() << "Received an assignment of type" << assignmentType << "\n"; + AudioMixer::run(); + // reset our NodeList by switching back to unassigned and clearing the list nodeList->setOwnerType(NODE_TYPE_UNASSIGNED); nodeList->clear(); diff --git a/audio-mixer/CMakeLists.txt b/audio-mixer/CMakeLists.txt index 472327de42..cd6d2850c5 100644 --- a/audio-mixer/CMakeLists.txt +++ b/audio-mixer/CMakeLists.txt @@ -3,25 +3,13 @@ cmake_minimum_required(VERSION 2.8) set(ROOT_DIR ..) set(MACRO_DIR ${ROOT_DIR}/cmake/macros) -# setup for find modules -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules/") - set(TARGET_NAME audio-mixer) include(${MACRO_DIR}/SetupHifiProject.cmake) setup_hifi_project(${TARGET_NAME} TRUE) -# set up the external glm library -include(${MACRO_DIR}/IncludeGLM.cmake) -include_glm(${TARGET_NAME} ${ROOT_DIR}) - # link the shared hifi library include(${MACRO_DIR}/LinkHifiLibrary.cmake) link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) -# link the stk library -set(STK_ROOT_DIR ${ROOT_DIR}/externals/stk) -find_package(STK REQUIRED) -target_link_libraries(${TARGET_NAME} ${STK_LIBRARIES}) -include_directories(${STK_INCLUDE_DIRS}) \ No newline at end of file diff --git a/audio-mixer/src/main.cpp b/audio-mixer/src/main.cpp index 6e912d7c21..fa5a0e5086 100644 --- a/audio-mixer/src/main.cpp +++ b/audio-mixer/src/main.cpp @@ -6,64 +6,7 @@ // Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include "Syssocket.h" -#include "Systime.h" -#include -#else -#include -#include -#include -#include -#endif //_WIN32 - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "AvatarAudioRingBuffer.h" -#include "InjectedAudioRingBuffer.h" - -const unsigned short MIXER_LISTEN_PORT = 55443; - -const short JITTER_BUFFER_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); - -const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / SAMPLE_RATE) * 1000000); - -const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); -const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); - -void attachNewBufferToNode(Node *newNode) { - if (!newNode->getLinkedData()) { - if (newNode->getType() == NODE_TYPE_AGENT) { - newNode->setLinkedData(new AvatarAudioRingBuffer()); - } else { - newNode->setLinkedData(new InjectedAudioRingBuffer()); - } - } -} +#include bool wantLocalDomain = false; @@ -85,350 +28,7 @@ int main(int argc, const char* argv[]) { NodeList::getInstance()->setDomainHostname(domainIP); } - ssize_t receivedBytes = 0; - - nodeList->linkedDataCreateCallback = attachNewBufferToNode; - nodeList->startSilentNodeRemovalThread(); - - unsigned char* packetData = new unsigned char[MAX_PACKET_SIZE]; - - sockaddr* nodeAddress = new sockaddr; - - // make sure our node socket is non-blocking - nodeList->getNodeSocket()->setBlocking(false); - - int nextFrame = 0; - timeval startTime; - - int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO); - unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; - populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO); - - int16_t clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2] = {}; - - gettimeofday(&startTime, NULL); - - timeval lastDomainServerCheckIn = {}; - - timeval beginSendTime, endSendTime; - float sumFrameTimePercentages = 0.0f; - int numStatCollections = 0; - - stk::StkFrames stkFrameBuffer(BUFFER_LENGTH_SAMPLES_PER_CHANNEL, 1); - - // if we'll be sending stats, call the Logstash::socket() method to make it load the logstash IP outside the loop - if (Logstash::shouldSendStats()) { - Logstash::socket(); - } - - while (true) { - if (Logstash::shouldSendStats()) { - gettimeofday(&beginSendTime, NULL); - } - - // send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed - if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) { - gettimeofday(&lastDomainServerCheckIn, NULL); - NodeList::getInstance()->sendDomainServerCheckIn(); - - if (Logstash::shouldSendStats() && numStatCollections > 0) { - // if we should be sending stats to Logstash send the appropriate average now - const char MIXER_LOGSTASH_METRIC_NAME[] = "audio-mixer-frame-time-usage"; - - float averageFrameTimePercentage = sumFrameTimePercentages / numStatCollections; - Logstash::stashValue(STAT_TYPE_TIMER, MIXER_LOGSTASH_METRIC_NAME, averageFrameTimePercentage); - - sumFrameTimePercentages = 0.0f; - numStatCollections = 0; - } - } - - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - PositionalAudioRingBuffer* positionalRingBuffer = (PositionalAudioRingBuffer*) node->getLinkedData(); - if (positionalRingBuffer && positionalRingBuffer->shouldBeAddedToMix(JITTER_BUFFER_SAMPLES)) { - // this is a ring buffer that is ready to go - // set its flag so we know to push its buffer when all is said and done - positionalRingBuffer->setWillBeAddedToMix(true); - } - } - - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - - const int PHASE_DELAY_AT_90 = 20; - - if (node->getType() == NODE_TYPE_AGENT) { - AvatarAudioRingBuffer* nodeRingBuffer = (AvatarAudioRingBuffer*) node->getLinkedData(); - - // zero out the client mix for this node - memset(clientSamples, 0, sizeof(clientSamples)); - - // loop through all other nodes that have sufficient audio to mix - for (NodeList::iterator otherNode = nodeList->begin(); otherNode != nodeList->end(); otherNode++) { - if (((PositionalAudioRingBuffer*) otherNode->getLinkedData())->willBeAddedToMix() - && (otherNode != node || (otherNode == node && nodeRingBuffer->shouldLoopbackForNode()))) { - PositionalAudioRingBuffer* otherNodeBuffer = (PositionalAudioRingBuffer*) otherNode->getLinkedData(); - // based on our listen mode we will do this mixing... - if (nodeRingBuffer->isListeningToNode(*otherNode)) { - float bearingRelativeAngleToSource = 0.0f; - float attenuationCoefficient = 1.0f; - int numSamplesDelay = 0; - float weakChannelAmplitudeRatio = 1.0f; - - stk::TwoPole* otherNodeTwoPole = NULL; - - // only do axis/distance attenuation when in normal mode - if (otherNode != node && nodeRingBuffer->getListeningMode() == AudioRingBuffer::NORMAL) { - - glm::vec3 listenerPosition = nodeRingBuffer->getPosition(); - glm::vec3 relativePosition = otherNodeBuffer->getPosition() - nodeRingBuffer->getPosition(); - glm::quat inverseOrientation = glm::inverse(nodeRingBuffer->getOrientation()); - - float distanceSquareToSource = glm::dot(relativePosition, relativePosition); - float radius = 0.0f; - - if (otherNode->getType() == NODE_TYPE_AUDIO_INJECTOR) { - InjectedAudioRingBuffer* injectedBuffer = (InjectedAudioRingBuffer*) otherNodeBuffer; - radius = injectedBuffer->getRadius(); - attenuationCoefficient *= injectedBuffer->getAttenuationRatio(); - } - - if (radius == 0 || (distanceSquareToSource > radius * radius)) { - // this is either not a spherical source, or the listener is outside the sphere - - if (radius > 0) { - // this is a spherical source - the distance used for the coefficient - // needs to be the closest point on the boundary to the source - - // ovveride the distance to the node with the distance to the point on the - // boundary of the sphere - distanceSquareToSource -= (radius * radius); - - } else { - // calculate the angle delivery for off-axis attenuation - glm::vec3 rotatedListenerPosition = glm::inverse(otherNodeBuffer->getOrientation()) - * relativePosition; - - float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedListenerPosition)); - - const float MAX_OFF_AXIS_ATTENUATION = 0.2f; - const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; - - float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + - (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / 90.0f)); - - // multiply the current attenuation coefficient by the calculated off axis coefficient - attenuationCoefficient *= offAxisCoefficient; - } - - glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; - - const float DISTANCE_SCALE = 2.5f; - const float GEOMETRIC_AMPLITUDE_SCALAR = 0.3f; - const float DISTANCE_LOG_BASE = 2.5f; - const float DISTANCE_SCALE_LOG = logf(DISTANCE_SCALE) / logf(DISTANCE_LOG_BASE); - - // calculate the distance coefficient using the distance to this node - float distanceCoefficient = powf(GEOMETRIC_AMPLITUDE_SCALAR, - DISTANCE_SCALE_LOG + - (0.5f * logf(distanceSquareToSource) / logf(DISTANCE_LOG_BASE)) - 1); - distanceCoefficient = std::min(1.0f, distanceCoefficient); - - // multiply the current attenuation coefficient by the distance coefficient - attenuationCoefficient *= distanceCoefficient; - - // project the rotated source position vector onto the XZ plane - rotatedSourcePosition.y = 0.0f; - - // produce an oriented angle about the y-axis - bearingRelativeAngleToSource = glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), - glm::normalize(rotatedSourcePosition), - glm::vec3(0.0f, 1.0f, 0.0f)); - - const float PHASE_AMPLITUDE_RATIO_AT_90 = 0.5; - - // figure out the number of samples of delay and the ratio of the amplitude - // in the weak channel for audio spatialization - float sinRatio = fabsf(sinf(glm::radians(bearingRelativeAngleToSource))); - numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; - weakChannelAmplitudeRatio = 1 - (PHASE_AMPLITUDE_RATIO_AT_90 * sinRatio); - - // grab the TwoPole object for this source, add it if it doesn't exist - TwoPoleNodeMap& nodeTwoPoles = nodeRingBuffer->getTwoPoles(); - TwoPoleNodeMap::iterator twoPoleIterator = nodeTwoPoles.find(otherNode->getNodeID()); - - if (twoPoleIterator == nodeTwoPoles.end()) { - // setup the freeVerb effect for this source for this client - otherNodeTwoPole = nodeTwoPoles[otherNode->getNodeID()] = new stk::TwoPole; - } else { - otherNodeTwoPole = twoPoleIterator->second; - } - - // calculate the reasonance for this TwoPole based on angle to source - float TWO_POLE_CUT_OFF_FREQUENCY = 800.0f; - float TWO_POLE_MAX_FILTER_STRENGTH = 0.4f; - - otherNodeTwoPole->setResonance(TWO_POLE_CUT_OFF_FREQUENCY, - TWO_POLE_MAX_FILTER_STRENGTH - * fabsf(bearingRelativeAngleToSource) / 180.0f, - true); - } - } - - int16_t* sourceBuffer = otherNodeBuffer->getNextOutput(); - - int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) - ? clientSamples - : clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) - ? clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL - : clientSamples; - - int16_t* delaySamplePointer = otherNodeBuffer->getNextOutput() == otherNodeBuffer->getBuffer() - ? otherNodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay - : otherNodeBuffer->getNextOutput() - numSamplesDelay; - - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { - // load up the stkFrameBuffer with this source's samples - stkFrameBuffer[s] = (stk::StkFloat) sourceBuffer[s]; - } - - // perform the TwoPole effect on the stkFrameBuffer - if (otherNodeTwoPole) { - otherNodeTwoPole->tick(stkFrameBuffer); - } - - for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { - if (s < numSamplesDelay) { - // pull the earlier sample for the delayed channel - int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; - - delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); - } - - int16_t currentSample = stkFrameBuffer[s] * attenuationCoefficient; - - goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); - - if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { - int sumSample = delayedChannel[s + numSamplesDelay] - + (currentSample * weakChannelAmplitudeRatio); - delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, - MIN_SAMPLE_VALUE, - MAX_SAMPLE_VALUE); - } - - if (s >= BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PHASE_DELAY_AT_90) { - // this could be a delayed sample on the next pass - // so store the affected back in the ARB - otherNodeBuffer->getNextOutput()[s] = (int16_t) stkFrameBuffer[s]; - } - } - } - } - } - - memcpy(clientPacket + numBytesPacketHeader, clientSamples, sizeof(clientSamples)); - nodeList->getNodeSocket()->send(node->getPublicSocket(), clientPacket, sizeof(clientPacket)); - } - } - - // push forward the next output pointers for any audio buffers we used - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - PositionalAudioRingBuffer* nodeBuffer = (PositionalAudioRingBuffer*) node->getLinkedData(); - if (nodeBuffer && nodeBuffer->willBeAddedToMix()) { - nodeBuffer->setNextOutput(nodeBuffer->getNextOutput() + BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - - if (nodeBuffer->getNextOutput() >= nodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { - nodeBuffer->setNextOutput(nodeBuffer->getBuffer()); - } - nodeBuffer->setWillBeAddedToMix(false); - } - } - - // pull any new audio data from nodes off of the network stack - while (nodeList->getNodeSocket()->receive(nodeAddress, packetData, &receivedBytes) && - packetVersionMatch(packetData)) { - if (packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO || - packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO) { - - unsigned char* currentBuffer = packetData + numBytesForPacketHeader(packetData); - uint16_t sourceID; - memcpy(&sourceID, currentBuffer, sizeof(sourceID)); - - Node* avatarNode = nodeList->addOrUpdateNode(nodeAddress, - nodeAddress, - NODE_TYPE_AGENT, - sourceID); - - nodeList->updateNodeWithData(nodeAddress, packetData, receivedBytes); - - if (std::isnan(((PositionalAudioRingBuffer *)avatarNode->getLinkedData())->getOrientation().x)) { - // kill off this node - temporary solution to mixer crash on mac sleep - avatarNode->setAlive(false); - } - } else if (packetData[0] == PACKET_TYPE_INJECT_AUDIO) { - Node* matchingInjector = NULL; - - for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { - if (node->getLinkedData()) { - - InjectedAudioRingBuffer* ringBuffer = (InjectedAudioRingBuffer*) node->getLinkedData(); - if (memcmp(ringBuffer->getStreamIdentifier(), - packetData + numBytesForPacketHeader(packetData), - STREAM_IDENTIFIER_NUM_BYTES) == 0) { - // this is the matching stream, assign to matchingInjector and stop looking - matchingInjector = &*node; - break; - } - } - } - - if (!matchingInjector) { - matchingInjector = nodeList->addOrUpdateNode(NULL, - NULL, - NODE_TYPE_AUDIO_INJECTOR, - nodeList->getLastNodeID()); - nodeList->increaseNodeID(); - - } - - // give the new audio data to the matching injector node - nodeList->updateNodeWithData(matchingInjector, packetData, receivedBytes); - } else if (packetData[0] == PACKET_TYPE_PING) { - - // If the packet is a ping, let processNodeData handle it. - nodeList->processNodeData(nodeAddress, packetData, receivedBytes); - } - } - - if (Logstash::shouldSendStats()) { - // send a packet to our logstash instance - - // calculate the percentage value for time elapsed for this send (of the max allowable time) - gettimeofday(&endSendTime, NULL); - - float percentageOfMaxElapsed = ((float) (usecTimestamp(&endSendTime) - usecTimestamp(&beginSendTime)) - / BUFFER_SEND_INTERVAL_USECS) * 100.0f; - - sumFrameTimePercentages += percentageOfMaxElapsed; - - numStatCollections++; - } - - int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); - - if (usecToSleep > 0) { - usleep(usecToSleep); - } else { - std::cout << "Took too much time, not sleeping!\n"; - } - } return 0; } diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt index 6070649060..6c458149bc 100644 --- a/libraries/audio/CMakeLists.txt +++ b/libraries/audio/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cm set(TARGET_NAME audio) +# set up the external glm library include(${MACRO_DIR}/SetupHifiLibrary.cmake) setup_hifi_library(${TARGET_NAME}) @@ -15,4 +16,10 @@ include(${MACRO_DIR}/IncludeGLM.cmake) include_glm(${TARGET_NAME} ${ROOT_DIR}) include(${MACRO_DIR}/LinkHifiLibrary.cmake) -link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) \ No newline at end of file +link_hifi_library(shared ${TARGET_NAME} ${ROOT_DIR}) + +# link the stk library +set(STK_ROOT_DIR ${ROOT_DIR}/externals/stk) +find_package(STK REQUIRED) +target_link_libraries(${TARGET_NAME} ${STK_LIBRARIES}) +include_directories(${STK_INCLUDE_DIRS}) \ No newline at end of file diff --git a/libraries/audio/src/AudioMixer.cpp b/libraries/audio/src/AudioMixer.cpp new file mode 100644 index 0000000000..0b6920b63c --- /dev/null +++ b/libraries/audio/src/AudioMixer.cpp @@ -0,0 +1,419 @@ +// +// AudioMixer.cpp +// hifi +// +// Created by Stephen Birarda on 8/22/13. +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include "Syssocket.h" +#include "Systime.h" +#include +#else +#include +#include +#include +#include +#endif //_WIN32 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "AudioRingBuffer.h" + +#include "AvatarAudioRingBuffer.h" +#include "InjectedAudioRingBuffer.h" + +#include "AudioMixer.h" + +const unsigned short MIXER_LISTEN_PORT = 55443; + +const short JITTER_BUFFER_MSECS = 12; +const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_MSECS * (SAMPLE_RATE / 1000.0); + +const unsigned int BUFFER_SEND_INTERVAL_USECS = floorf((BUFFER_LENGTH_SAMPLES_PER_CHANNEL / SAMPLE_RATE) * 1000000); + +const int MAX_SAMPLE_VALUE = std::numeric_limits::max(); +const int MIN_SAMPLE_VALUE = std::numeric_limits::min(); + +void attachNewBufferToNode(Node *newNode) { + if (!newNode->getLinkedData()) { + if (newNode->getType() == NODE_TYPE_AGENT) { + newNode->setLinkedData(new AvatarAudioRingBuffer()); + } else { + newNode->setLinkedData(new InjectedAudioRingBuffer()); + } + } +} + +void AudioMixer::run() { + + NodeList *nodeList = NodeList::getInstance(); + nodeList->setOwnerType(NODE_TYPE_AUDIO_MIXER); + + ssize_t receivedBytes = 0; + + nodeList->linkedDataCreateCallback = attachNewBufferToNode; + + nodeList->startSilentNodeRemovalThread(); + + unsigned char* packetData = new unsigned char[MAX_PACKET_SIZE]; + + sockaddr* nodeAddress = new sockaddr; + + // make sure our node socket is non-blocking + nodeList->getNodeSocket()->setBlocking(false); + + int nextFrame = 0; + timeval startTime; + + int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MIXED_AUDIO); + unsigned char clientPacket[BUFFER_LENGTH_BYTES_STEREO + numBytesPacketHeader]; + populateTypeAndVersion(clientPacket, PACKET_TYPE_MIXED_AUDIO); + + int16_t clientSamples[BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 2] = {}; + + gettimeofday(&startTime, NULL); + + timeval lastDomainServerCheckIn = {}; + + timeval beginSendTime, endSendTime; + float sumFrameTimePercentages = 0.0f; + int numStatCollections = 0; + + stk::StkFrames stkFrameBuffer(BUFFER_LENGTH_SAMPLES_PER_CHANNEL, 1); + + // if we'll be sending stats, call the Logstash::socket() method to make it load the logstash IP outside the loop + if (Logstash::shouldSendStats()) { + Logstash::socket(); + } + + while (true) { + if (Logstash::shouldSendStats()) { + gettimeofday(&beginSendTime, NULL); + } + + // send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed + if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) { + gettimeofday(&lastDomainServerCheckIn, NULL); + NodeList::getInstance()->sendDomainServerCheckIn(); + + if (Logstash::shouldSendStats() && numStatCollections > 0) { + // if we should be sending stats to Logstash send the appropriate average now + const char MIXER_LOGSTASH_METRIC_NAME[] = "audio-mixer-frame-time-usage"; + + float averageFrameTimePercentage = sumFrameTimePercentages / numStatCollections; + Logstash::stashValue(STAT_TYPE_TIMER, MIXER_LOGSTASH_METRIC_NAME, averageFrameTimePercentage); + + sumFrameTimePercentages = 0.0f; + numStatCollections = 0; + } + } + + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + PositionalAudioRingBuffer* positionalRingBuffer = (PositionalAudioRingBuffer*) node->getLinkedData(); + if (positionalRingBuffer && positionalRingBuffer->shouldBeAddedToMix(JITTER_BUFFER_SAMPLES)) { + // this is a ring buffer that is ready to go + // set its flag so we know to push its buffer when all is said and done + positionalRingBuffer->setWillBeAddedToMix(true); + } + } + + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + + const int PHASE_DELAY_AT_90 = 20; + + if (node->getType() == NODE_TYPE_AGENT) { + AvatarAudioRingBuffer* nodeRingBuffer = (AvatarAudioRingBuffer*) node->getLinkedData(); + + // zero out the client mix for this node + memset(clientSamples, 0, sizeof(clientSamples)); + + // loop through all other nodes that have sufficient audio to mix + for (NodeList::iterator otherNode = nodeList->begin(); otherNode != nodeList->end(); otherNode++) { + if (((PositionalAudioRingBuffer*) otherNode->getLinkedData())->willBeAddedToMix() + && (otherNode != node || (otherNode == node && nodeRingBuffer->shouldLoopbackForNode()))) { + PositionalAudioRingBuffer* otherNodeBuffer = (PositionalAudioRingBuffer*) otherNode->getLinkedData(); + // based on our listen mode we will do this mixing... + if (nodeRingBuffer->isListeningToNode(*otherNode)) { + float bearingRelativeAngleToSource = 0.0f; + float attenuationCoefficient = 1.0f; + int numSamplesDelay = 0; + float weakChannelAmplitudeRatio = 1.0f; + + stk::TwoPole* otherNodeTwoPole = NULL; + + // only do axis/distance attenuation when in normal mode + if (otherNode != node && nodeRingBuffer->getListeningMode() == AudioRingBuffer::NORMAL) { + + glm::vec3 listenerPosition = nodeRingBuffer->getPosition(); + glm::vec3 relativePosition = otherNodeBuffer->getPosition() - nodeRingBuffer->getPosition(); + glm::quat inverseOrientation = glm::inverse(nodeRingBuffer->getOrientation()); + + float distanceSquareToSource = glm::dot(relativePosition, relativePosition); + float radius = 0.0f; + + if (otherNode->getType() == NODE_TYPE_AUDIO_INJECTOR) { + InjectedAudioRingBuffer* injectedBuffer = (InjectedAudioRingBuffer*) otherNodeBuffer; + radius = injectedBuffer->getRadius(); + attenuationCoefficient *= injectedBuffer->getAttenuationRatio(); + } + + if (radius == 0 || (distanceSquareToSource > radius * radius)) { + // this is either not a spherical source, or the listener is outside the sphere + + if (radius > 0) { + // this is a spherical source - the distance used for the coefficient + // needs to be the closest point on the boundary to the source + + // ovveride the distance to the node with the distance to the point on the + // boundary of the sphere + distanceSquareToSource -= (radius * radius); + + } else { + // calculate the angle delivery for off-axis attenuation + glm::vec3 rotatedListenerPosition = glm::inverse(otherNodeBuffer->getOrientation()) + * relativePosition; + + float angleOfDelivery = glm::angle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedListenerPosition)); + + const float MAX_OFF_AXIS_ATTENUATION = 0.2f; + const float OFF_AXIS_ATTENUATION_FORMULA_STEP = (1 - MAX_OFF_AXIS_ATTENUATION) / 2.0f; + + float offAxisCoefficient = MAX_OFF_AXIS_ATTENUATION + + (OFF_AXIS_ATTENUATION_FORMULA_STEP * (angleOfDelivery / 90.0f)); + + // multiply the current attenuation coefficient by the calculated off axis coefficient + attenuationCoefficient *= offAxisCoefficient; + } + + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + const float DISTANCE_SCALE = 2.5f; + const float GEOMETRIC_AMPLITUDE_SCALAR = 0.3f; + const float DISTANCE_LOG_BASE = 2.5f; + const float DISTANCE_SCALE_LOG = logf(DISTANCE_SCALE) / logf(DISTANCE_LOG_BASE); + + // calculate the distance coefficient using the distance to this node + float distanceCoefficient = powf(GEOMETRIC_AMPLITUDE_SCALAR, + DISTANCE_SCALE_LOG + + (0.5f * logf(distanceSquareToSource) / logf(DISTANCE_LOG_BASE)) - 1); + distanceCoefficient = std::min(1.0f, distanceCoefficient); + + // multiply the current attenuation coefficient by the distance coefficient + attenuationCoefficient *= distanceCoefficient; + + // project the rotated source position vector onto the XZ plane + rotatedSourcePosition.y = 0.0f; + + // produce an oriented angle about the y-axis + bearingRelativeAngleToSource = glm::orientedAngle(glm::vec3(0.0f, 0.0f, -1.0f), + glm::normalize(rotatedSourcePosition), + glm::vec3(0.0f, 1.0f, 0.0f)); + + const float PHASE_AMPLITUDE_RATIO_AT_90 = 0.5; + + // figure out the number of samples of delay and the ratio of the amplitude + // in the weak channel for audio spatialization + float sinRatio = fabsf(sinf(glm::radians(bearingRelativeAngleToSource))); + numSamplesDelay = PHASE_DELAY_AT_90 * sinRatio; + weakChannelAmplitudeRatio = 1 - (PHASE_AMPLITUDE_RATIO_AT_90 * sinRatio); + + // grab the TwoPole object for this source, add it if it doesn't exist + TwoPoleNodeMap& nodeTwoPoles = nodeRingBuffer->getTwoPoles(); + TwoPoleNodeMap::iterator twoPoleIterator = nodeTwoPoles.find(otherNode->getNodeID()); + + if (twoPoleIterator == nodeTwoPoles.end()) { + // setup the freeVerb effect for this source for this client + otherNodeTwoPole = nodeTwoPoles[otherNode->getNodeID()] = new stk::TwoPole; + } else { + otherNodeTwoPole = twoPoleIterator->second; + } + + // calculate the reasonance for this TwoPole based on angle to source + float TWO_POLE_CUT_OFF_FREQUENCY = 800.0f; + float TWO_POLE_MAX_FILTER_STRENGTH = 0.4f; + + otherNodeTwoPole->setResonance(TWO_POLE_CUT_OFF_FREQUENCY, + TWO_POLE_MAX_FILTER_STRENGTH + * fabsf(bearingRelativeAngleToSource) / 180.0f, + true); + } + } + + int16_t* sourceBuffer = otherNodeBuffer->getNextOutput(); + + int16_t* goodChannel = (bearingRelativeAngleToSource > 0.0f) + ? clientSamples + : clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + int16_t* delayedChannel = (bearingRelativeAngleToSource > 0.0f) + ? clientSamples + BUFFER_LENGTH_SAMPLES_PER_CHANNEL + : clientSamples; + + int16_t* delaySamplePointer = otherNodeBuffer->getNextOutput() == otherNodeBuffer->getBuffer() + ? otherNodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES - numSamplesDelay + : otherNodeBuffer->getNextOutput() - numSamplesDelay; + + for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { + // load up the stkFrameBuffer with this source's samples + stkFrameBuffer[s] = (stk::StkFloat) sourceBuffer[s]; + } + + // perform the TwoPole effect on the stkFrameBuffer + if (otherNodeTwoPole) { + otherNodeTwoPole->tick(stkFrameBuffer); + } + + for (int s = 0; s < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; s++) { + if (s < numSamplesDelay) { + // pull the earlier sample for the delayed channel + int earlierSample = delaySamplePointer[s] * attenuationCoefficient * weakChannelAmplitudeRatio; + + delayedChannel[s] = glm::clamp(delayedChannel[s] + earlierSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); + } + + int16_t currentSample = stkFrameBuffer[s] * attenuationCoefficient; + + goodChannel[s] = glm::clamp(goodChannel[s] + currentSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); + + if (s + numSamplesDelay < BUFFER_LENGTH_SAMPLES_PER_CHANNEL) { + int sumSample = delayedChannel[s + numSamplesDelay] + + (currentSample * weakChannelAmplitudeRatio); + delayedChannel[s + numSamplesDelay] = glm::clamp(sumSample, + MIN_SAMPLE_VALUE, + MAX_SAMPLE_VALUE); + } + + if (s >= BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PHASE_DELAY_AT_90) { + // this could be a delayed sample on the next pass + // so store the affected back in the ARB + otherNodeBuffer->getNextOutput()[s] = (int16_t) stkFrameBuffer[s]; + } + } + } + } + } + + memcpy(clientPacket + numBytesPacketHeader, clientSamples, sizeof(clientSamples)); + nodeList->getNodeSocket()->send(node->getPublicSocket(), clientPacket, sizeof(clientPacket)); + } + } + + // push forward the next output pointers for any audio buffers we used + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + PositionalAudioRingBuffer* nodeBuffer = (PositionalAudioRingBuffer*) node->getLinkedData(); + if (nodeBuffer && nodeBuffer->willBeAddedToMix()) { + nodeBuffer->setNextOutput(nodeBuffer->getNextOutput() + BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + + if (nodeBuffer->getNextOutput() >= nodeBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) { + nodeBuffer->setNextOutput(nodeBuffer->getBuffer()); + } + nodeBuffer->setWillBeAddedToMix(false); + } + } + + // pull any new audio data from nodes off of the network stack + while (nodeList->getNodeSocket()->receive(nodeAddress, packetData, &receivedBytes) && + packetVersionMatch(packetData)) { + if (packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO || + packetData[0] == PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO) { + + unsigned char* currentBuffer = packetData + numBytesForPacketHeader(packetData); + uint16_t sourceID; + memcpy(&sourceID, currentBuffer, sizeof(sourceID)); + + Node* avatarNode = nodeList->addOrUpdateNode(nodeAddress, + nodeAddress, + NODE_TYPE_AGENT, + sourceID); + + nodeList->updateNodeWithData(nodeAddress, packetData, receivedBytes); + + if (std::isnan(((PositionalAudioRingBuffer *)avatarNode->getLinkedData())->getOrientation().x)) { + // kill off this node - temporary solution to mixer crash on mac sleep + avatarNode->setAlive(false); + } + } else if (packetData[0] == PACKET_TYPE_INJECT_AUDIO) { + Node* matchingInjector = NULL; + + for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) { + if (node->getLinkedData()) { + + InjectedAudioRingBuffer* ringBuffer = (InjectedAudioRingBuffer*) node->getLinkedData(); + if (memcmp(ringBuffer->getStreamIdentifier(), + packetData + numBytesForPacketHeader(packetData), + STREAM_IDENTIFIER_NUM_BYTES) == 0) { + // this is the matching stream, assign to matchingInjector and stop looking + matchingInjector = &*node; + break; + } + } + } + + if (!matchingInjector) { + matchingInjector = nodeList->addOrUpdateNode(NULL, + NULL, + NODE_TYPE_AUDIO_INJECTOR, + nodeList->getLastNodeID()); + nodeList->increaseNodeID(); + + } + + // give the new audio data to the matching injector node + nodeList->updateNodeWithData(matchingInjector, packetData, receivedBytes); + } else if (packetData[0] == PACKET_TYPE_PING) { + + // If the packet is a ping, let processNodeData handle it. + nodeList->processNodeData(nodeAddress, packetData, receivedBytes); + } + } + + if (Logstash::shouldSendStats()) { + // send a packet to our logstash instance + + // calculate the percentage value for time elapsed for this send (of the max allowable time) + gettimeofday(&endSendTime, NULL); + + float percentageOfMaxElapsed = ((float) (usecTimestamp(&endSendTime) - usecTimestamp(&beginSendTime)) + / BUFFER_SEND_INTERVAL_USECS) * 100.0f; + + sumFrameTimePercentages += percentageOfMaxElapsed; + + numStatCollections++; + } + + int usecToSleep = usecTimestamp(&startTime) + (++nextFrame * BUFFER_SEND_INTERVAL_USECS) - usecTimestampNow(); + + if (usecToSleep > 0) { + usleep(usecToSleep); + } else { + std::cout << "Took too much time, not sleeping!\n"; + } + } +} \ No newline at end of file diff --git a/libraries/audio/src/AudioMixer.h b/libraries/audio/src/AudioMixer.h new file mode 100644 index 0000000000..6318e756dc --- /dev/null +++ b/libraries/audio/src/AudioMixer.h @@ -0,0 +1,17 @@ +// +// AudioMixer.h +// hifi +// +// Created by Stephen Birarda on 8/22/13. +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. +// + +#ifndef __hifi__AudioMixer__ +#define __hifi__AudioMixer__ + +class AudioMixer { +public: + static void run(); +}; + +#endif /* defined(__hifi__AudioMixer__) */ diff --git a/audio-mixer/src/AvatarAudioRingBuffer.cpp b/libraries/audio/src/AvatarAudioRingBuffer.cpp similarity index 100% rename from audio-mixer/src/AvatarAudioRingBuffer.cpp rename to libraries/audio/src/AvatarAudioRingBuffer.cpp diff --git a/audio-mixer/src/AvatarAudioRingBuffer.h b/libraries/audio/src/AvatarAudioRingBuffer.h similarity index 100% rename from audio-mixer/src/AvatarAudioRingBuffer.h rename to libraries/audio/src/AvatarAudioRingBuffer.h diff --git a/audio-mixer/src/InjectedAudioRingBuffer.cpp b/libraries/audio/src/InjectedAudioRingBuffer.cpp similarity index 100% rename from audio-mixer/src/InjectedAudioRingBuffer.cpp rename to libraries/audio/src/InjectedAudioRingBuffer.cpp diff --git a/audio-mixer/src/InjectedAudioRingBuffer.h b/libraries/audio/src/InjectedAudioRingBuffer.h similarity index 97% rename from audio-mixer/src/InjectedAudioRingBuffer.h rename to libraries/audio/src/InjectedAudioRingBuffer.h index f5caef5f75..e1df9ac5b9 100644 --- a/audio-mixer/src/InjectedAudioRingBuffer.h +++ b/libraries/audio/src/InjectedAudioRingBuffer.h @@ -9,7 +9,7 @@ #ifndef __hifi__InjectedAudioRingBuffer__ #define __hifi__InjectedAudioRingBuffer__ -#include +#include "AudioInjector.h" #include "PositionalAudioRingBuffer.h" diff --git a/audio-mixer/src/PositionalAudioRingBuffer.cpp b/libraries/audio/src/PositionalAudioRingBuffer.cpp similarity index 100% rename from audio-mixer/src/PositionalAudioRingBuffer.cpp rename to libraries/audio/src/PositionalAudioRingBuffer.cpp diff --git a/audio-mixer/src/PositionalAudioRingBuffer.h b/libraries/audio/src/PositionalAudioRingBuffer.h similarity index 98% rename from audio-mixer/src/PositionalAudioRingBuffer.h rename to libraries/audio/src/PositionalAudioRingBuffer.h index 6c7ee9ce3f..b43cd60660 100644 --- a/audio-mixer/src/PositionalAudioRingBuffer.h +++ b/libraries/audio/src/PositionalAudioRingBuffer.h @@ -12,7 +12,7 @@ #include #include -#include +#include "AudioRingBuffer.h" class PositionalAudioRingBuffer : public AudioRingBuffer { public: diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 4d5b9e7c3b..985da1f3cf 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -31,7 +31,7 @@ const char SOLO_NODE_TYPES[2] = { }; const char DEFAULT_DOMAIN_HOSTNAME[MAX_HOSTNAME_BYTES] = "root.highfidelity.io"; -const char DEFAULT_DOMAIN_IP[INET_ADDRSTRLEN] = ""; // IP Address will be re-set by lookup on startup +const char DEFAULT_DOMAIN_IP[INET_ADDRSTRLEN] = "10.0.0.20"; // IP Address will be re-set by lookup on startup const int DEFAULT_DOMAINSERVER_PORT = 40102; bool silentNodeThreadStopFlag = false;