Merge pull request #1322 from birarda/qt-for-portaudio

replace PortAudio with QAudioInput and QAudioOutput

assignment-client/src/AssignmentClient.cpp

@@ -78,8 +78,8 @@ void AssignmentClient::readPendingDatagrams() {
             if (_currentAssignment) {
                 // have the threaded current assignment handle this datagram
                 QMetaObject::invokeMethod(_currentAssignment, "processDatagram", Qt::QueuedConnection,
-                                          Q_ARG(const QByteArray&, QByteArray((char*) packetData, receivedBytes)),
-                                          Q_ARG(const HifiSockAddr&, senderSockAddr));
+                                          Q_ARG(QByteArray, QByteArray((char*) packetData, receivedBytes)),
+                                          Q_ARG(HifiSockAddr, senderSockAddr));
             } else if (packetData[0] == PACKET_TYPE_DEPLOY_ASSIGNMENT || packetData[0] == PACKET_TYPE_CREATE_ASSIGNMENT) {
                 
                 if (_currentAssignment) {

cmake/modules/FindPortAudio.cmake

@@ -1,44 +0,0 @@
-#  Find the static PortAudio library
-#
-#  You must provide a PORTAUDIO_ROOT_DIR which contains lib and include directories
-#
-#  Once done this will define
-#
-#  PORTAUDIO_FOUND - system found PortAudio
-#  PORTAUDIO_INCLUDE_DIRS - the PortAudio include directory
-#  PORTAUDIO_LIBRARIES - Link this to use PortAudio
-#
-#  Created on 5/14/2013 by Stephen Birarda
-#  Copyright (c) 2013 High Fidelity
-#
-
-if (PORTAUDIO_LIBRARIES AND PORTAUDIO_INCLUDE_DIRS)
-  # in cache already
-  set(PORTAUDIO_FOUND TRUE)
-else (PORTAUDIO_LIBRARIES AND PORTAUDIO_INCLUDE_DIRS)
-  find_path(PORTAUDIO_INCLUDE_DIRS portaudio.h ${PORTAUDIO_ROOT_DIR}/include)
-
-  if (APPLE)
-    find_library(PORTAUDIO_LIBRARIES libportaudio.a ${PORTAUDIO_ROOT_DIR}/lib/MacOS/)
-  elseif (UNIX)
-    find_library(PORTAUDIO_LIBRARIES libportaudio.a ${PORTAUDIO_ROOT_DIR}/lib/UNIX/)
-  endif ()
-
-  if (PORTAUDIO_INCLUDE_DIRS AND PORTAUDIO_LIBRARIES)
-    set(PORTAUDIO_FOUND TRUE)
-  endif (PORTAUDIO_INCLUDE_DIRS AND PORTAUDIO_LIBRARIES)
- 
-  if (PORTAUDIO_FOUND)
-    if (NOT PortAudio_FIND_QUIETLY)
-      message(STATUS "Found PortAudio: ${PORTAUDIO_LIBRARIES}")
-    endif (NOT PortAudio_FIND_QUIETLY)
-  else (PORTAUDIO_FOUND)
-    if (PortAudio_FIND_REQUIRED)
-      message(FATAL_ERROR "Could not find PortAudio")
-    endif (PortAudio_FIND_REQUIRED)
-  endif (PORTAUDIO_FOUND)
-
-  # show the PORTAUDIO_INCLUDE_DIRS and PORTAUDIO_LIBRARIES variables only in the advanced view
-  mark_as_advanced(PORTAUDIO_INCLUDE_DIRS PORTAUDIO_LIBRARIES)
-
-endif (PORTAUDIO_LIBRARIES AND PORTAUDIO_INCLUDE_DIRS)

interface/CMakeLists.txt

@@ -13,7 +13,6 @@ set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR)
 set(LIBVPX_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibVPX)
 set(LEAP_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Leap)
 set(MOTIONDRIVER_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/MotionDriver)
-set(PORTAUDIO_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/PortAudio)
 set(OPENCV_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/OpenCV)
 set(SIXENSE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Sixense)
 set(UVCCAMERACONTROL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/UVCCameraControl)
@@ -210,11 +209,6 @@ if (WIN32)
         wsock32.lib
     )
 else (WIN32)
-  # link the PortAudio library
-  find_package(PortAudio REQUIRED)
-  include_directories(${PORTAUDIO_INCLUDE_DIRS})
-  target_link_libraries(${TARGET_NAME} ${PORTAUDIO_LIBRARIES})
-  
   # link required libraries on UNIX
   if (UNIX AND NOT APPLE) 
       find_package(Threads REQUIRED)

interface/src/Application.cpp

@@ -133,9 +133,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
         _lookatIndicatorScale(1.0f),
         _perfStatsOn(false),
         _chatEntryOn(false),
-#ifndef _WIN32
         _audio(&_audioScope, STARTUP_JITTER_SAMPLES),
-#endif
         _stopNetworkReceiveThread(false),  
         _voxelProcessor(),
         _voxelHideShowThread(&_voxels),
@@ -163,12 +161,19 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
     
     NodeList::createInstance(NODE_TYPE_AGENT, listenPort);
     
+    // put the audio processing on a separate thread
+    QThread* audioThread = new QThread(this);
+    
+    _audio.moveToThread(audioThread);
+    connect(audioThread, SIGNAL(started()), &_audio, SLOT(start()));
+    
+    audioThread->start();
+    
     NodeList::getInstance()->addHook(&_voxels);
     NodeList::getInstance()->addHook(this);
     NodeList::getInstance()->addDomainListener(this);
     NodeList::getInstance()->addDomainListener(&_voxels);
 
-    
     // network receive thread and voxel parsing thread are both controlled by the --nonblocking command line
     _enableProcessVoxelsThread = _enableNetworkThread = !cmdOptionExists(argc, constArgv, "--nonblocking");
     
@@ -236,6 +241,10 @@ Application::Application(int& argc, char** argv, timeval &startup_time) :
 }
 
 Application::~Application() {
+    // ask the audio thread to quit and wait until it is done
+    _audio.thread()->quit();
+    _audio.thread()->wait();
+    
     storeSizeAndPosition();
     NodeList::getInstance()->removeHook(&_voxels);
     NodeList::getInstance()->removeHook(this);
@@ -243,8 +252,6 @@ Application::~Application() {
 
     _sharedVoxelSystem.changeTree(new VoxelTree);
 
-    _audio.shutdown();
-
     VoxelTreeElement::removeDeleteHook(&_voxels); // we don't need to do this processing on shutdown
     delete Menu::getInstance();
     
@@ -637,9 +644,6 @@ void Application::keyPressEvent(QKeyEvent* event) {
             case Qt::Key_Period:
                 Menu::getInstance()->handleViewFrustumOffsetKeyModifier(event->key());
                 break;
-            case Qt::Key_Semicolon:
-                _audio.ping();
-                break;
             case Qt::Key_Apostrophe:
                 _audioScope.inputPaused = !_audioScope.inputPaused;
                 break; 
@@ -2425,7 +2429,6 @@ void Application::updateAudio(float deltaTime) {
     #ifndef _WIN32
     _audio.setLastAcceleration(_myAvatar.getThrust());
     _audio.setLastVelocity(_myAvatar.getVelocity());
-    _audio.eventuallyAnalyzePing();
     #endif
 }
 
@@ -4024,14 +4027,18 @@ void Application::resetSensors() {
     _webcam.reset();
     _faceshift.reset();
     LeapManager::reset();
-    OculusManager::reset();
+    
+    if (OculusManager::isConnected()) {
+        OculusManager::reset();
+    }
+    
     QCursor::setPos(_headMouseX, _headMouseY);
     _myAvatar.reset();
     _myTransmitter.resetLevels();
     _myAvatar.setVelocity(glm::vec3(0,0,0));
     _myAvatar.setThrust(glm::vec3(0,0,0));
     
-    _audio.reset();
+    QMetaObject::invokeMethod(&_audio, "reset", Qt::QueuedConnection);
 }
 
 static void setShortcutsEnabled(QWidget* widget, bool enabled) {

interface/src/Audio.h

@@ -14,9 +14,7 @@
 
 #include "InterfaceConfig.h"
 
-#include <QObject>
-
-#include <portaudio.h>
+#include <QtCore/QObject>
 
 #include <AudioRingBuffer.h>
 #include <StdDev.h>
@@ -32,19 +30,20 @@ static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2;
 static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t);
 static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2;
 
+class QAudioInput;
+class QAudioOutput;
+class QIODevice;
+
 class Audio : public QObject {
     Q_OBJECT
 public:
-    // initializes audio I/O
-    Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples);
+    // setup for audio I/O
+    Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples, QObject* parent = 0);
     
-    void shutdown();
-
-    void reset(); 
     void render(int screenWidth, int screenHeight);
     
     void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes);
-
+    
     float getLastInputLoudness() const { return _lastInputLoudness; }
     
     void setLastAcceleration(const glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }
@@ -56,26 +55,27 @@ public:
     void lowPassFilter(int16_t* inputBuffer);
     
     void startCollisionSound(float magnitude, float frequency, float noise, float duration, bool flashScreen);
-
     void startDrumSound(float volume, float frequency, float duration, float decay);
-
+    
     float getCollisionSoundMagnitude() { return _collisionSoundMagnitude; }
     
     bool getCollisionFlashesScreen() { return _collisionFlashesScreen; }
     
-    void ping();
-    
     void init(QGLWidget *parent = 0);
     bool mousePressEvent(int x, int y);
-
-    // Call periodically to eventually perform round trip time analysis,
-    // in which case 'true' is returned - otherwise the return value is 'false'.
-    // The results of the analysis are written to the log.
-    bool eventuallyAnalyzePing();
+    
+    public slots:
+    void start();
+    void handleAudioInput();
+    void reset();
     
 private:
-    
-    PaStream* _stream;
+    QAudioInput* _audioInput;
+    QIODevice* _inputDevice;
+    QAudioOutput* _audioOutput;
+    QIODevice* _outputDevice;
+    bool _isBufferSendCallback;
+    int16_t* _nextOutputSamples;
     AudioRingBuffer _ringBuffer;
     Oscilloscope* _scope;
     StDev _stdev;
@@ -84,33 +84,16 @@ private:
     float _averagedLatency;
     float _measuredJitter;
     int16_t _jitterBufferSamples;
-    int _wasStarved;
-    int _numStarves;
     float _lastInputLoudness;
     glm::vec3 _lastVelocity;
     glm::vec3 _lastAcceleration;
     int _totalPacketsReceived;
-    timeval _firstPacketReceivedTime;
-    int _packetsReceivedThisPlayback;
-    // Ping analysis
-    int16_t* _echoSamplesLeft;
-    volatile bool _isSendingEchoPing;
-    volatile bool _pingAnalysisPending;
-    int _pingFramesToRecord;
-    // Flange effect
-    int _samplesLeftForFlange;
-    int _lastYawMeasuredMaximum;
-    float _flangeIntensity;
-    float _flangeRate;
-    float _flangeWeight;
     
-    // Collision sound generator
     float _collisionSoundMagnitude;
     float _collisionSoundFrequency;
     float _collisionSoundNoise;
     float _collisionSoundDuration;
     bool _collisionFlashesScreen;
-    int _proceduralEffectSample;
     
     // Drum sound generator
     float _drumSoundVolume;
@@ -118,7 +101,9 @@ private:
     float _drumSoundDuration;
     float _drumSoundDecay;
     int _drumSoundSample;
-
+    
+    int _proceduralEffectSample;
+    int _numFramesDisplayStarve;
     bool _muted;
     bool _localEcho;
     GLuint _micTextureId;
@@ -127,28 +112,12 @@ private:
     
     // Audio callback in class context.
     inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight);
-
-    // When requested, sends/receives a signal for round trip time determination.
-    // Called from 'performIO'.
-    inline void eventuallySendRecvPing(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight);
-
-    // Determines round trip time of the audio system. Called from 'eventuallyAnalyzePing'.
-    inline void analyzePing();
-
+    
     // Add sounds that we want the user to not hear themselves, by adding on top of mic input signal
-    void addProceduralSounds(int16_t* inputBuffer, int16_t* outputLeft, int16_t* outputRight, int numSamples);
-
-
-    // Audio callback called by portaudio. Calls 'performIO'.
-    static int audioCallback(const void *inputBuffer,
-                             void *outputBuffer,
-                             unsigned long framesPerBuffer,
-                             const PaStreamCallbackTimeInfo *timeInfo,
-                             PaStreamCallbackFlags statusFlags,
-                             void *userData);
-
+    void addProceduralSounds(int16_t* inputBuffer, int16_t* stereoOutput, int numSamples);
+    
     void renderToolIcon(int screenHeight);
 };
 
 
-#endif /* defined(__interface__audio__) */
+#endif /* defined(__interface__audio__) */

interface/src/Oscilloscope.cpp

@@ -6,13 +6,16 @@
 //  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
 //
 
-#include "Oscilloscope.h"
-
-#include "InterfaceConfig.h"
 #include <limits>
 #include <cstring>
 #include <algorithm>
 
+#include <QtCore/QDebug>
+
+#include "InterfaceConfig.h"
+
+#include "Oscilloscope.h"
+
 //  Reimplemented 4/26/13 (tosh) - don't blame Philip for bugs
 
 using namespace std;
@@ -65,37 +68,50 @@ Oscilloscope::~Oscilloscope() {
     delete[] _samples;
 }
 
-void Oscilloscope::addSamples(unsigned ch, short const* data, unsigned n) {
+void Oscilloscope::addStereoSamples(const QByteArray& audioByteArray, bool isInput) {
 
     if (! enabled || inputPaused) {
         return;
     }
-
-    // determine start/end offset of this channel's region
-    unsigned baseOffs = MAX_SAMPLES_PER_CHANNEL * ch;
-    unsigned endOffs = baseOffs + MAX_SAMPLES_PER_CHANNEL;
-
-    // fetch write position for this channel
-    unsigned writePos = _writePos[ch];
-
-    // determine write position after adding the samples 
-    unsigned newWritePos = writePos + n;
-    unsigned n2 = 0;
-    if (newWritePos >= endOffs) {
-        // passed boundary of the circular buffer? -> we need to copy two blocks
-        n2 = newWritePos - endOffs;
-        newWritePos = baseOffs + n2;
-        n -= n2;
+    
+    unsigned int numSamplesPerChannel = audioByteArray.size() / (sizeof(int16_t) * 2);
+    int16_t samples[numSamplesPerChannel];
+    const int16_t* stereoSamples = (int16_t*) audioByteArray.constData();
+    
+    for (int channel = 0; channel < (isInput ? 1 : 2); channel++) {
+        // add samples for each of the channels
+        
+        // enumerate the interleaved stereoSamples array and pull out the samples for this channel
+        for (int i = 0; i < audioByteArray.size() / sizeof(int16_t); i += 2) {
+            samples[i / 2] = stereoSamples[i + channel];
+        }
+        
+        // determine start/end offset of this channel's region
+        unsigned baseOffs = MAX_SAMPLES_PER_CHANNEL * (channel + !isInput);
+        unsigned endOffs = baseOffs + MAX_SAMPLES_PER_CHANNEL;
+        
+        // fetch write position for this channel
+        unsigned writePos = _writePos[channel + !isInput];
+        
+        // determine write position after adding the samples
+        unsigned newWritePos = writePos + numSamplesPerChannel;
+        unsigned n2 = 0;
+        if (newWritePos >= endOffs) {
+            // passed boundary of the circular buffer? -> we need to copy two blocks
+            n2 = newWritePos - endOffs;
+            newWritePos = baseOffs + n2;
+            numSamplesPerChannel -= n2;
+        }
+        
+        // copy data
+        memcpy(_samples + writePos, samples, numSamplesPerChannel * sizeof(int16_t));
+        if (n2 > 0) {
+            memcpy(_samples + baseOffs, samples + numSamplesPerChannel, n2 * sizeof(int16_t));
+        }
+        
+        // set new write position for this channel 
+        _writePos[channel + !isInput] = newWritePos;
     }
-
-    // copy data 
-    memcpy(_samples + writePos, data, n * sizeof(short));
-    if (n2 > 0) {
-        memcpy(_samples + baseOffs, data + n, n2 * sizeof(short));
-    }
-
-    // set new write position for this channel 
-    _writePos[ch] = newWritePos;
 }
         
 void Oscilloscope::render(int x, int y) {

interface/src/Oscilloscope.h

@@ -11,13 +11,14 @@
 
 #include <cassert>
 
-class Oscilloscope {
+#include <QtCore/QObject>
+
+class Oscilloscope : public QObject {
+    Q_OBJECT
 public:
     Oscilloscope(int width, int height, bool isEnabled);
     ~Oscilloscope();
 
-    void addSamples(unsigned ch, short const* data, unsigned n);
-
     void render(int x, int y);
 
     // Switches: On/Off, Stop Time
@@ -57,7 +58,8 @@ public:
     // Sets the number of input samples per output sample. Without filtering
     // just uses every nTh sample.
     void setDownsampleRatio(unsigned n) { assert(n > 0); _downsampleRatio = n; }
-
+public slots:
+    void addStereoSamples(const QByteArray& audioByteArray, bool isInput);
 private:
     // don't copy/assign
     Oscilloscope(Oscilloscope const&); // = delete;

libraries/audio/src/AudioRingBuffer.h

@@ -16,7 +16,7 @@
 
 #include "NodeData.h"
 
-const float SAMPLE_RATE = 22050.0;
+const int SAMPLE_RATE = 22050;
 
 const int BUFFER_LENGTH_BYTES_STEREO = 1024;
 const int BUFFER_LENGTH_BYTES_PER_CHANNEL = 512;