diff --git a/cmake/modules/FindSpeexDSP.cmake b/cmake/modules/FindSpeexDSP.cmake new file mode 100644 index 0000000000..cd9eed44d5 --- /dev/null +++ b/cmake/modules/FindSpeexDSP.cmake @@ -0,0 +1,88 @@ +# Copyright (c) 2009, Whispersoft s.r.l. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Whispersoft s.r.l. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +############################# +# THIS IS A MODIFIED COPY. # +############################# + +# +# Changed 6/18/2013 by Tobias Schwinger +# Copyright (c) 2013 High Fidelity +# + +# +# Finds SPEEXDSP library +# +# SPEEXDSP_INCLUDE_DIRS - where to find speex.h, etc. +# SPEEXDSP_LIBRARIES - List of libraries when using SPEEXDSP. +# SPEEXDSP_FOUND - True if SPEEXDSP found. +# + +if (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARIES) + set(SPEEXDSP_FOUND TRUE) +else (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARIES) + + find_path(SPEEXDSP_INCLUDE_DIRS speex/speex.h + /usr/include + /usr/local/include + ${SPEEX_ROOT_DIR}/include + ) + + set(SPEEXDSP_NAMES speexdsp) + find_library(SPEEXDSP_LIBRARY NAMES ${SPEEXDSP_NAMES} PATHS /usr/lib usr/local/lib) + if (NOT SPEEXDSP_LIBRARY AND APPLE) + find_library(SPEEXDSP_LIBRARY NAMES ${SPEEXDSP_NAMES} PATHS ${SPEEX_ROOT_DIR}/lib/MacOS) + elseif (WIN32) + find_library(SPEEXDSP_LIBRARY NAMES ${SPEEXDSP_NAMES} PATHS ${SPEEX_ROOT_DIR}/lib/Win32) + endif () + + if (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARY) + set(SPEEXDSP_FOUND TRUE) + set(SPEEXDSP_LIBRARIES ${SPEEXDSP_LIBRARY}) + else (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARY) + set(SPEEXDSP_FOUND FALSE) + set(SPEEXDSP_LIBRARIES) + endif (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARY) + + if (SPEEXDSP_FOUND) + message(STATUS "Found SpeexDSP: ${SPEEXDSP_LIBRARY}") + else (SPEEXDSP_FOUND) + if (SPEEXDSP_FIND_REQUIRED) + message(STATUS "Looked for SpeexDSP libraries named ${SPEEXDSP_NAMES}.") + message(STATUS "Include file detected: [${SPEEXDSP_INCLUDE_DIRS}].") + message(STATUS "Lib file detected: [${SPEEXDSP_LIBRARY}].") + message(FATAL_ERROR "=========> Could NOT find SpeexDSP library") + endif (SPEEXDSP_FIND_REQUIRED) + endif (SPEEXDSP_FOUND) + + mark_as_advanced(SPEEXDSP_INCLUDE_DIRS SPEEXDSP_LIBRARIES) + +endif (SPEEXDSP_INCLUDE_DIRS AND SPEEXDSP_LIBRARIES) + diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 9a57a546f1..148e6819be 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -10,6 +10,7 @@ project(${TARGET_NAME}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules/") set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR) set(PORTAUDIO_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/PortAudio) +set(SPEEX_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Speex) set(OPENCV_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/OpenCV) set(UVCCAMERACONTROL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/UVCCameraControl) @@ -86,6 +87,7 @@ link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) # find required libraries find_package(GLM REQUIRED) find_package(LibOVR) +find_package(SpeexDSP REQUIRED) find_package(OpenCV) find_package(ZLIB) find_package(UVCCameraControl) @@ -103,9 +105,10 @@ include_directories( ${GLM_INCLUDE_DIRS} ${LIBOVR_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} + ${SPEEXDSP_INCLUDE_DIRS} ) -target_link_libraries(${TARGET_NAME} ${QT_LIBRARIES} ${OPENCV_LIBRARIES} ${ZLIB_LIBRARIES}) +target_link_libraries(${TARGET_NAME} ${QT_LIBRARIES} ${OPENCV_LIBRARIES} ${ZLIB_LIBRARIES} ${SPEEXDSP_LIBRARIES}) if (APPLE) # link in required OS X frameworks and include the right GL headers diff --git a/interface/external/Speex/include/speex/speex.h b/interface/external/Speex/include/speex/speex.h new file mode 100644 index 0000000000..82ba016237 --- /dev/null +++ b/interface/external/Speex/include/speex/speex.h @@ -0,0 +1,424 @@ +/* Copyright (C) 2002-2006 Jean-Marc Valin*/ +/** + @file speex.h + @brief Describes the different modes of the codec +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef SPEEX_H +#define SPEEX_H +/** @defgroup Codec Speex encoder and decoder + * This is the Speex codec itself. + * @{ + */ + +#include "speex/speex_bits.h" +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Values allowed for *ctl() requests */ + +/** Set enhancement on/off (decoder only) */ +#define SPEEX_SET_ENH 0 +/** Get enhancement state (decoder only) */ +#define SPEEX_GET_ENH 1 + +/*Would be SPEEX_SET_FRAME_SIZE, but it's (currently) invalid*/ +/** Obtain frame size used by encoder/decoder */ +#define SPEEX_GET_FRAME_SIZE 3 + +/** Set quality value */ +#define SPEEX_SET_QUALITY 4 +/** Get current quality setting */ +/* #define SPEEX_GET_QUALITY 5 -- Doesn't make much sense, does it? */ + +/** Set sub-mode to use */ +#define SPEEX_SET_MODE 6 +/** Get current sub-mode in use */ +#define SPEEX_GET_MODE 7 + +/** Set low-band sub-mode to use (wideband only)*/ +#define SPEEX_SET_LOW_MODE 8 +/** Get current low-band mode in use (wideband only)*/ +#define SPEEX_GET_LOW_MODE 9 + +/** Set high-band sub-mode to use (wideband only)*/ +#define SPEEX_SET_HIGH_MODE 10 +/** Get current high-band mode in use (wideband only)*/ +#define SPEEX_GET_HIGH_MODE 11 + +/** Set VBR on (1) or off (0) */ +#define SPEEX_SET_VBR 12 +/** Get VBR status (1 for on, 0 for off) */ +#define SPEEX_GET_VBR 13 + +/** Set quality value for VBR encoding (0-10) */ +#define SPEEX_SET_VBR_QUALITY 14 +/** Get current quality value for VBR encoding (0-10) */ +#define SPEEX_GET_VBR_QUALITY 15 + +/** Set complexity of the encoder (0-10) */ +#define SPEEX_SET_COMPLEXITY 16 +/** Get current complexity of the encoder (0-10) */ +#define SPEEX_GET_COMPLEXITY 17 + +/** Set bit-rate used by the encoder (or lower) */ +#define SPEEX_SET_BITRATE 18 +/** Get current bit-rate used by the encoder or decoder */ +#define SPEEX_GET_BITRATE 19 + +/** Define a handler function for in-band Speex request*/ +#define SPEEX_SET_HANDLER 20 + +/** Define a handler function for in-band user-defined request*/ +#define SPEEX_SET_USER_HANDLER 22 + +/** Set sampling rate used in bit-rate computation */ +#define SPEEX_SET_SAMPLING_RATE 24 +/** Get sampling rate used in bit-rate computation */ +#define SPEEX_GET_SAMPLING_RATE 25 + +/** Reset the encoder/decoder memories to zero*/ +#define SPEEX_RESET_STATE 26 + +/** Get VBR info (mostly used internally) */ +#define SPEEX_GET_RELATIVE_QUALITY 29 + +/** Set VAD status (1 for on, 0 for off) */ +#define SPEEX_SET_VAD 30 + +/** Get VAD status (1 for on, 0 for off) */ +#define SPEEX_GET_VAD 31 + +/** Set Average Bit-Rate (ABR) to n bits per seconds */ +#define SPEEX_SET_ABR 32 +/** Get Average Bit-Rate (ABR) setting (in bps) */ +#define SPEEX_GET_ABR 33 + +/** Set DTX status (1 for on, 0 for off) */ +#define SPEEX_SET_DTX 34 +/** Get DTX status (1 for on, 0 for off) */ +#define SPEEX_GET_DTX 35 + +/** Set submode encoding in each frame (1 for yes, 0 for no, setting to no breaks the standard) */ +#define SPEEX_SET_SUBMODE_ENCODING 36 +/** Get submode encoding in each frame */ +#define SPEEX_GET_SUBMODE_ENCODING 37 + +/*#define SPEEX_SET_LOOKAHEAD 38*/ +/** Returns the lookahead used by Speex */ +#define SPEEX_GET_LOOKAHEAD 39 + +/** Sets tuning for packet-loss concealment (expected loss rate) */ +#define SPEEX_SET_PLC_TUNING 40 +/** Gets tuning for PLC */ +#define SPEEX_GET_PLC_TUNING 41 + +/** Sets the max bit-rate allowed in VBR mode */ +#define SPEEX_SET_VBR_MAX_BITRATE 42 +/** Gets the max bit-rate allowed in VBR mode */ +#define SPEEX_GET_VBR_MAX_BITRATE 43 + +/** Turn on/off input/output high-pass filtering */ +#define SPEEX_SET_HIGHPASS 44 +/** Get status of input/output high-pass filtering */ +#define SPEEX_GET_HIGHPASS 45 + +/** Get "activity level" of the last decoded frame, i.e. + how much damage we cause if we remove the frame */ +#define SPEEX_GET_ACTIVITY 47 + + +/* Preserving compatibility:*/ +/** Equivalent to SPEEX_SET_ENH */ +#define SPEEX_SET_PF 0 +/** Equivalent to SPEEX_GET_ENH */ +#define SPEEX_GET_PF 1 + + + + +/* Values allowed for mode queries */ +/** Query the frame size of a mode */ +#define SPEEX_MODE_FRAME_SIZE 0 + +/** Query the size of an encoded frame for a particular sub-mode */ +#define SPEEX_SUBMODE_BITS_PER_FRAME 1 + + + +/** Get major Speex version */ +#define SPEEX_LIB_GET_MAJOR_VERSION 1 +/** Get minor Speex version */ +#define SPEEX_LIB_GET_MINOR_VERSION 3 +/** Get micro Speex version */ +#define SPEEX_LIB_GET_MICRO_VERSION 5 +/** Get extra Speex version */ +#define SPEEX_LIB_GET_EXTRA_VERSION 7 +/** Get Speex version string */ +#define SPEEX_LIB_GET_VERSION_STRING 9 + +/*#define SPEEX_LIB_SET_ALLOC_FUNC 10 +#define SPEEX_LIB_GET_ALLOC_FUNC 11 +#define SPEEX_LIB_SET_FREE_FUNC 12 +#define SPEEX_LIB_GET_FREE_FUNC 13 + +#define SPEEX_LIB_SET_WARNING_FUNC 14 +#define SPEEX_LIB_GET_WARNING_FUNC 15 +#define SPEEX_LIB_SET_ERROR_FUNC 16 +#define SPEEX_LIB_GET_ERROR_FUNC 17 +*/ + +/** Number of defined modes in Speex */ +#define SPEEX_NB_MODES 3 + +/** modeID for the defined narrowband mode */ +#define SPEEX_MODEID_NB 0 + +/** modeID for the defined wideband mode */ +#define SPEEX_MODEID_WB 1 + +/** modeID for the defined ultra-wideband mode */ +#define SPEEX_MODEID_UWB 2 + +struct SpeexMode; + + +/* Prototypes for mode function pointers */ + +/** Encoder state initialization function */ +typedef void *(*encoder_init_func)(const struct SpeexMode *mode); + +/** Encoder state destruction function */ +typedef void (*encoder_destroy_func)(void *st); + +/** Main encoding function */ +typedef int (*encode_func)(void *state, void *in, SpeexBits *bits); + +/** Function for controlling the encoder options */ +typedef int (*encoder_ctl_func)(void *state, int request, void *ptr); + +/** Decoder state initialization function */ +typedef void *(*decoder_init_func)(const struct SpeexMode *mode); + +/** Decoder state destruction function */ +typedef void (*decoder_destroy_func)(void *st); + +/** Main decoding function */ +typedef int (*decode_func)(void *state, SpeexBits *bits, void *out); + +/** Function for controlling the decoder options */ +typedef int (*decoder_ctl_func)(void *state, int request, void *ptr); + + +/** Query function for a mode */ +typedef int (*mode_query_func)(const void *mode, int request, void *ptr); + +/** Struct defining a Speex mode */ +typedef struct SpeexMode { + /** Pointer to the low-level mode data */ + const void *mode; + + /** Pointer to the mode query function */ + mode_query_func query; + + /** The name of the mode (you should not rely on this to identify the mode)*/ + const char *modeName; + + /**ID of the mode*/ + int modeID; + + /**Version number of the bitstream (incremented every time we break + bitstream compatibility*/ + int bitstream_version; + + /** Pointer to encoder initialization function */ + encoder_init_func enc_init; + + /** Pointer to encoder destruction function */ + encoder_destroy_func enc_destroy; + + /** Pointer to frame encoding function */ + encode_func enc; + + /** Pointer to decoder initialization function */ + decoder_init_func dec_init; + + /** Pointer to decoder destruction function */ + decoder_destroy_func dec_destroy; + + /** Pointer to frame decoding function */ + decode_func dec; + + /** ioctl-like requests for encoder */ + encoder_ctl_func enc_ctl; + + /** ioctl-like requests for decoder */ + decoder_ctl_func dec_ctl; + +} SpeexMode; + +/** + * Returns a handle to a newly created Speex encoder state structure. For now, + * the "mode" argument can be &nb_mode or &wb_mode . In the future, more modes + * may be added. Note that for now if you have more than one channels to + * encode, you need one state per channel. + * + * @param mode The mode to use (either speex_nb_mode or speex_wb.mode) + * @return A newly created encoder state or NULL if state allocation fails + */ +void *speex_encoder_init(const SpeexMode *mode); + +/** Frees all resources associated to an existing Speex encoder state. + * @param state Encoder state to be destroyed */ +void speex_encoder_destroy(void *state); + +/** Uses an existing encoder state to encode one frame of speech pointed to by + "in". The encoded bit-stream is saved in "bits". + @param state Encoder state + @param in Frame that will be encoded with a +-2^15 range. This data MAY be + overwritten by the encoder and should be considered uninitialised + after the call. + @param bits Bit-stream where the data will be written + @return 0 if frame needs not be transmitted (DTX only), 1 otherwise + */ +int speex_encode(void *state, float *in, SpeexBits *bits); + +/** Uses an existing encoder state to encode one frame of speech pointed to by + "in". The encoded bit-stream is saved in "bits". + @param state Encoder state + @param in Frame that will be encoded with a +-2^15 range + @param bits Bit-stream where the data will be written + @return 0 if frame needs not be transmitted (DTX only), 1 otherwise + */ +int speex_encode_int(void *state, spx_int16_t *in, SpeexBits *bits); + +/** Used like the ioctl function to control the encoder parameters + * + * @param state Encoder state + * @param request ioctl-type request (one of the SPEEX_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown, -2 for invalid parameter + */ +int speex_encoder_ctl(void *state, int request, void *ptr); + + +/** Returns a handle to a newly created decoder state structure. For now, + * the mode argument can be &nb_mode or &wb_mode . In the future, more modes + * may be added. Note that for now if you have more than one channels to + * decode, you need one state per channel. + * + * @param mode Speex mode (one of speex_nb_mode or speex_wb_mode) + * @return A newly created decoder state or NULL if state allocation fails + */ +void *speex_decoder_init(const SpeexMode *mode); + +/** Frees all resources associated to an existing decoder state. + * + * @param state State to be destroyed + */ +void speex_decoder_destroy(void *state); + +/** Uses an existing decoder state to decode one frame of speech from + * bit-stream bits. The output speech is saved written to out. + * + * @param state Decoder state + * @param bits Bit-stream from which to decode the frame (NULL if the packet was lost) + * @param out Where to write the decoded frame + * @return return status (0 for no error, -1 for end of stream, -2 corrupt stream) + */ +int speex_decode(void *state, SpeexBits *bits, float *out); + +/** Uses an existing decoder state to decode one frame of speech from + * bit-stream bits. The output speech is saved written to out. + * + * @param state Decoder state + * @param bits Bit-stream from which to decode the frame (NULL if the packet was lost) + * @param out Where to write the decoded frame + * @return return status (0 for no error, -1 for end of stream, -2 corrupt stream) + */ +int speex_decode_int(void *state, SpeexBits *bits, spx_int16_t *out); + +/** Used like the ioctl function to control the encoder parameters + * + * @param state Decoder state + * @param request ioctl-type request (one of the SPEEX_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown, -2 for invalid parameter + */ +int speex_decoder_ctl(void *state, int request, void *ptr); + + +/** Query function for mode information + * + * @param mode Speex mode + * @param request ioctl-type request (one of the SPEEX_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown, -2 for invalid parameter + */ +int speex_mode_query(const SpeexMode *mode, int request, void *ptr); + +/** Functions for controlling the behavior of libspeex + * @param request ioctl-type request (one of the SPEEX_LIB_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown, -2 for invalid parameter + */ +int speex_lib_ctl(int request, void *ptr); + +/** Default narrowband mode */ +extern const SpeexMode speex_nb_mode; + +/** Default wideband mode */ +extern const SpeexMode speex_wb_mode; + +/** Default "ultra-wideband" mode */ +extern const SpeexMode speex_uwb_mode; + +/** List of all modes available */ +extern const SpeexMode * const speex_mode_list[SPEEX_NB_MODES]; + +/** Obtain one of the modes available */ +const SpeexMode * speex_lib_get_mode (int mode); + +#ifndef WIN32 +/* We actually override the function in the narrowband case so that we can avoid linking in the wideband stuff */ +#define speex_lib_get_mode(mode) ((mode)==SPEEX_MODEID_NB ? &speex_nb_mode : speex_lib_get_mode (mode)) +#endif + +#ifdef __cplusplus +} +#endif + +/** @}*/ +#endif diff --git a/interface/external/Speex/include/speex/speex_bits.h b/interface/external/Speex/include/speex/speex_bits.h new file mode 100644 index 0000000000..a26fb4ce0c --- /dev/null +++ b/interface/external/Speex/include/speex/speex_bits.h @@ -0,0 +1,174 @@ +/* Copyright (C) 2002 Jean-Marc Valin */ +/** + @file speex_bits.h + @brief Handles bit packing/unpacking +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BITS_H +#define BITS_H +/** @defgroup SpeexBits SpeexBits: Bit-stream manipulations + * This is the structure that holds the bit-stream when encoding or decoding + * with Speex. It allows some manipulations as well. + * @{ + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** Bit-packing data structure representing (part of) a bit-stream. */ +typedef struct SpeexBits { + char *chars; /**< "raw" data */ + int nbBits; /**< Total number of bits stored in the stream*/ + int charPtr; /**< Position of the byte "cursor" */ + int bitPtr; /**< Position of the bit "cursor" within the current char */ + int owner; /**< Does the struct "own" the "raw" buffer (member "chars") */ + int overflow;/**< Set to one if we try to read past the valid data */ + int buf_size;/**< Allocated size for buffer */ + int reserved1; /**< Reserved for future use */ + void *reserved2; /**< Reserved for future use */ +} SpeexBits; + +/** Initializes and allocates resources for a SpeexBits struct */ +void speex_bits_init(SpeexBits *bits); + +/** Initializes SpeexBits struct using a pre-allocated buffer*/ +void speex_bits_init_buffer(SpeexBits *bits, void *buff, int buf_size); + +/** Sets the bits in a SpeexBits struct to use data from an existing buffer (for decoding without copying data) */ +void speex_bits_set_bit_buffer(SpeexBits *bits, void *buff, int buf_size); + +/** Frees all resources associated to a SpeexBits struct. Right now this does nothing since no resources are allocated, but this could change in the future.*/ +void speex_bits_destroy(SpeexBits *bits); + +/** Resets bits to initial value (just after initialization, erasing content)*/ +void speex_bits_reset(SpeexBits *bits); + +/** Rewind the bit-stream to the beginning (ready for read) without erasing the content */ +void speex_bits_rewind(SpeexBits *bits); + +/** Initializes the bit-stream from the data in an area of memory */ +void speex_bits_read_from(SpeexBits *bits, char *bytes, int len); + +/** Append bytes to the bit-stream + * + * @param bits Bit-stream to operate on + * @param bytes pointer to the bytes what will be appended + * @param len Number of bytes of append + */ +void speex_bits_read_whole_bytes(SpeexBits *bits, char *bytes, int len); + +/** Write the content of a bit-stream to an area of memory + * + * @param bits Bit-stream to operate on + * @param bytes Memory location where to write the bits + * @param max_len Maximum number of bytes to write (i.e. size of the "bytes" buffer) + * @return Number of bytes written to the "bytes" buffer +*/ +int speex_bits_write(SpeexBits *bits, char *bytes, int max_len); + +/** Like speex_bits_write, but writes only the complete bytes in the stream. Also removes the written bytes from the stream */ +int speex_bits_write_whole_bytes(SpeexBits *bits, char *bytes, int max_len); + +/** Append bits to the bit-stream + * @param bits Bit-stream to operate on + * @param data Value to append as integer + * @param nbBits number of bits to consider in "data" + */ +void speex_bits_pack(SpeexBits *bits, int data, int nbBits); + +/** Interpret the next bits in the bit-stream as a signed integer + * + * @param bits Bit-stream to operate on + * @param nbBits Number of bits to interpret + * @return A signed integer represented by the bits read + */ +int speex_bits_unpack_signed(SpeexBits *bits, int nbBits); + +/** Interpret the next bits in the bit-stream as an unsigned integer + * + * @param bits Bit-stream to operate on + * @param nbBits Number of bits to interpret + * @return An unsigned integer represented by the bits read + */ +unsigned int speex_bits_unpack_unsigned(SpeexBits *bits, int nbBits); + +/** Returns the number of bytes in the bit-stream, including the last one even if it is not "full" + * + * @param bits Bit-stream to operate on + * @return Number of bytes in the stream + */ +int speex_bits_nbytes(SpeexBits *bits); + +/** Same as speex_bits_unpack_unsigned, but without modifying the cursor position + * + * @param bits Bit-stream to operate on + * @param nbBits Number of bits to look for + * @return Value of the bits peeked, interpreted as unsigned + */ +unsigned int speex_bits_peek_unsigned(SpeexBits *bits, int nbBits); + +/** Get the value of the next bit in the stream, without modifying the + * "cursor" position + * + * @param bits Bit-stream to operate on + * @return Value of the bit peeked (one bit only) + */ +int speex_bits_peek(SpeexBits *bits); + +/** Advances the position of the "bit cursor" in the stream + * + * @param bits Bit-stream to operate on + * @param n Number of bits to advance + */ +void speex_bits_advance(SpeexBits *bits, int n); + +/** Returns the number of bits remaining to be read in a stream + * + * @param bits Bit-stream to operate on + * @return Number of bits that can still be read from the stream + */ +int speex_bits_remaining(SpeexBits *bits); + +/** Insert a terminator so that the data can be sent as a packet while auto-detecting + * the number of frames in each packet + * + * @param bits Bit-stream to operate on + */ +void speex_bits_insert_terminator(SpeexBits *bits); + +#ifdef __cplusplus +} +#endif + +/* @} */ +#endif diff --git a/interface/external/Speex/include/speex/speex_buffer.h b/interface/external/Speex/include/speex/speex_buffer.h new file mode 100644 index 0000000000..df56f5f18b --- /dev/null +++ b/interface/external/Speex/include/speex/speex_buffer.h @@ -0,0 +1,68 @@ +/* Copyright (C) 2007 Jean-Marc Valin + + File: speex_buffer.h + This is a very simple ring buffer implementation. It is not thread-safe + so you need to do your own locking. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SPEEX_BUFFER_H +#define SPEEX_BUFFER_H + +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct SpeexBuffer_; +typedef struct SpeexBuffer_ SpeexBuffer; + +SpeexBuffer *speex_buffer_init(int size); + +void speex_buffer_destroy(SpeexBuffer *st); + +int speex_buffer_write(SpeexBuffer *st, void *data, int len); + +int speex_buffer_writezeros(SpeexBuffer *st, int len); + +int speex_buffer_read(SpeexBuffer *st, void *data, int len); + +int speex_buffer_get_available(SpeexBuffer *st); + +int speex_buffer_resize(SpeexBuffer *st, int len); + +#ifdef __cplusplus +} +#endif + +#endif + + + + diff --git a/interface/external/Speex/include/speex/speex_callbacks.h b/interface/external/Speex/include/speex/speex_callbacks.h new file mode 100644 index 0000000000..6f450b3a3a --- /dev/null +++ b/interface/external/Speex/include/speex/speex_callbacks.h @@ -0,0 +1,134 @@ +/* Copyright (C) 2002 Jean-Marc Valin*/ +/** + @file speex_callbacks.h + @brief Describes callback handling and in-band signalling +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef SPEEX_CALLBACKS_H +#define SPEEX_CALLBACKS_H +/** @defgroup SpeexCallbacks Various definitions for Speex callbacks supported by the decoder. + * @{ + */ + +#include "speex.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Total number of callbacks */ +#define SPEEX_MAX_CALLBACKS 16 + +/* Describes all the in-band requests */ + +/*These are 1-bit requests*/ +/** Request for perceptual enhancement (1 for on, 0 for off) */ +#define SPEEX_INBAND_ENH_REQUEST 0 +/** Reserved */ +#define SPEEX_INBAND_RESERVED1 1 + +/*These are 4-bit requests*/ +/** Request for a mode change */ +#define SPEEX_INBAND_MODE_REQUEST 2 +/** Request for a low mode change */ +#define SPEEX_INBAND_LOW_MODE_REQUEST 3 +/** Request for a high mode change */ +#define SPEEX_INBAND_HIGH_MODE_REQUEST 4 +/** Request for VBR (1 on, 0 off) */ +#define SPEEX_INBAND_VBR_QUALITY_REQUEST 5 +/** Request to be sent acknowledge */ +#define SPEEX_INBAND_ACKNOWLEDGE_REQUEST 6 +/** Request for VBR (1 for on, 0 for off) */ +#define SPEEX_INBAND_VBR_REQUEST 7 + +/*These are 8-bit requests*/ +/** Send a character in-band */ +#define SPEEX_INBAND_CHAR 8 +/** Intensity stereo information */ +#define SPEEX_INBAND_STEREO 9 + +/*These are 16-bit requests*/ +/** Transmit max bit-rate allowed */ +#define SPEEX_INBAND_MAX_BITRATE 10 + +/*These are 32-bit requests*/ +/** Acknowledge packet reception */ +#define SPEEX_INBAND_ACKNOWLEDGE 12 + +/** Callback function type */ +typedef int (*speex_callback_func)(SpeexBits *bits, void *state, void *data); + +/** Callback information */ +typedef struct SpeexCallback { + int callback_id; /**< ID associated to the callback */ + speex_callback_func func; /**< Callback handler function */ + void *data; /**< Data that will be sent to the handler */ + void *reserved1; /**< Reserved for future use */ + int reserved2; /**< Reserved for future use */ +} SpeexCallback; + +/** Handle in-band request */ +int speex_inband_handler(SpeexBits *bits, SpeexCallback *callback_list, void *state); + +/** Standard handler for mode request (change mode, no questions asked) */ +int speex_std_mode_request_handler(SpeexBits *bits, void *state, void *data); + +/** Standard handler for high mode request (change high mode, no questions asked) */ +int speex_std_high_mode_request_handler(SpeexBits *bits, void *state, void *data); + +/** Standard handler for in-band characters (write to stderr) */ +int speex_std_char_handler(SpeexBits *bits, void *state, void *data); + +/** Default handler for user-defined requests: in this case, just ignore */ +int speex_default_user_handler(SpeexBits *bits, void *state, void *data); + + + +/** Standard handler for low mode request (change low mode, no questions asked) */ +int speex_std_low_mode_request_handler(SpeexBits *bits, void *state, void *data); + +/** Standard handler for VBR request (Set VBR, no questions asked) */ +int speex_std_vbr_request_handler(SpeexBits *bits, void *state, void *data); + +/** Standard handler for enhancer request (Turn enhancer on/off, no questions asked) */ +int speex_std_enh_request_handler(SpeexBits *bits, void *state, void *data); + +/** Standard handler for VBR quality request (Set VBR quality, no questions asked) */ +int speex_std_vbr_quality_request_handler(SpeexBits *bits, void *state, void *data); + + +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif diff --git a/interface/external/Speex/include/speex/speex_config_types.h b/interface/external/Speex/include/speex/speex_config_types.h new file mode 100644 index 0000000000..bd548546b0 --- /dev/null +++ b/interface/external/Speex/include/speex/speex_config_types.h @@ -0,0 +1,11 @@ +#ifndef __SPEEX_TYPES_H__ +#define __SPEEX_TYPES_H__ + +/* these are filled in by configure */ +typedef short spx_int16_t; +typedef unsigned short spx_uint16_t; +typedef int spx_int32_t; +typedef unsigned int spx_uint32_t; + +#endif + diff --git a/interface/external/Speex/include/speex/speex_echo.h b/interface/external/Speex/include/speex/speex_echo.h new file mode 100644 index 0000000000..53bcd28a1a --- /dev/null +++ b/interface/external/Speex/include/speex/speex_echo.h @@ -0,0 +1,170 @@ +/* Copyright (C) Jean-Marc Valin */ +/** + @file speex_echo.h + @brief Echo cancellation +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SPEEX_ECHO_H +#define SPEEX_ECHO_H +/** @defgroup SpeexEchoState SpeexEchoState: Acoustic echo canceller + * This is the acoustic echo canceller module. + * @{ + */ +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Obtain frame size used by the AEC */ +#define SPEEX_ECHO_GET_FRAME_SIZE 3 + +/** Set sampling rate */ +#define SPEEX_ECHO_SET_SAMPLING_RATE 24 +/** Get sampling rate */ +#define SPEEX_ECHO_GET_SAMPLING_RATE 25 + +/* Can't set window sizes */ +/** Get size of impulse response (int32) */ +#define SPEEX_ECHO_GET_IMPULSE_RESPONSE_SIZE 27 + +/* Can't set window content */ +/** Get impulse response (int32[]) */ +#define SPEEX_ECHO_GET_IMPULSE_RESPONSE 29 + +/** Internal echo canceller state. Should never be accessed directly. */ +struct SpeexEchoState_; + +/** @class SpeexEchoState + * This holds the state of the echo canceller. You need one per channel. +*/ + +/** Internal echo canceller state. Should never be accessed directly. */ +typedef struct SpeexEchoState_ SpeexEchoState; + +/** Creates a new echo canceller state + * @param frame_size Number of samples to process at one time (should correspond to 10-20 ms) + * @param filter_length Number of samples of echo to cancel (should generally correspond to 100-500 ms) + * @return Newly-created echo canceller state + */ +SpeexEchoState *speex_echo_state_init(int frame_size, int filter_length); + +/** Creates a new multi-channel echo canceller state + * @param frame_size Number of samples to process at one time (should correspond to 10-20 ms) + * @param filter_length Number of samples of echo to cancel (should generally correspond to 100-500 ms) + * @param nb_mic Number of microphone channels + * @param nb_speakers Number of speaker channels + * @return Newly-created echo canceller state + */ +SpeexEchoState *speex_echo_state_init_mc(int frame_size, int filter_length, int nb_mic, int nb_speakers); + +/** Destroys an echo canceller state + * @param st Echo canceller state +*/ +void speex_echo_state_destroy(SpeexEchoState *st); + +/** Performs echo cancellation a frame, based on the audio sent to the speaker (no delay is added + * to playback in this form) + * + * @param st Echo canceller state + * @param rec Signal from the microphone (near end + far end echo) + * @param play Signal played to the speaker (received from far end) + * @param out Returns near-end signal with echo removed + */ +void speex_echo_cancellation(SpeexEchoState *st, const spx_int16_t *rec, const spx_int16_t *play, spx_int16_t *out); + +/** Performs echo cancellation a frame (deprecated) */ +void speex_echo_cancel(SpeexEchoState *st, const spx_int16_t *rec, const spx_int16_t *play, spx_int16_t *out, spx_int32_t *Yout); + +/** Perform echo cancellation using internal playback buffer, which is delayed by two frames + * to account for the delay introduced by most soundcards (but it could be off!) + * @param st Echo canceller state + * @param rec Signal from the microphone (near end + far end echo) + * @param out Returns near-end signal with echo removed +*/ +void speex_echo_capture(SpeexEchoState *st, const spx_int16_t *rec, spx_int16_t *out); + +/** Let the echo canceller know that a frame was just queued to the soundcard + * @param st Echo canceller state + * @param play Signal played to the speaker (received from far end) +*/ +void speex_echo_playback(SpeexEchoState *st, const spx_int16_t *play); + +/** Reset the echo canceller to its original state + * @param st Echo canceller state + */ +void speex_echo_state_reset(SpeexEchoState *st); + +/** Used like the ioctl function to control the echo canceller parameters + * + * @param st Echo canceller state + * @param request ioctl-type request (one of the SPEEX_ECHO_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown + */ +int speex_echo_ctl(SpeexEchoState *st, int request, void *ptr); + + + +struct SpeexDecorrState_; + +typedef struct SpeexDecorrState_ SpeexDecorrState; + + +/** Create a state for the channel decorrelation algorithm + This is useful for multi-channel echo cancellation only + * @param rate Sampling rate + * @param channels Number of channels (it's a bit pointless if you don't have at least 2) + * @param frame_size Size of the frame to process at ones (counting samples *per* channel) +*/ +SpeexDecorrState *speex_decorrelate_new(int rate, int channels, int frame_size); + +/** Remove correlation between the channels by modifying the phase and possibly + adding noise in a way that is not (or little) perceptible. + * @param st Decorrelator state + * @param in Input audio in interleaved format + * @param out Result of the decorrelation (out *may* alias in) + * @param strength How much alteration of the audio to apply from 0 to 100. +*/ +void speex_decorrelate(SpeexDecorrState *st, const spx_int16_t *in, spx_int16_t *out, int strength); + +/** Destroy a Decorrelation state + * @param st State to destroy +*/ +void speex_decorrelate_destroy(SpeexDecorrState *st); + + +#ifdef __cplusplus +} +#endif + + +/** @}*/ +#endif diff --git a/interface/external/Speex/include/speex/speex_header.h b/interface/external/Speex/include/speex/speex_header.h new file mode 100644 index 0000000000..f85b2496ae --- /dev/null +++ b/interface/external/Speex/include/speex/speex_header.h @@ -0,0 +1,94 @@ +/* Copyright (C) 2002 Jean-Marc Valin */ +/** + @file speex_header.h + @brief Describes the Speex header +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + + +#ifndef SPEEX_HEADER_H +#define SPEEX_HEADER_H +/** @defgroup SpeexHeader SpeexHeader: Makes it easy to write/parse an Ogg/Speex header + * This is the Speex header for the Ogg encapsulation. You don't need that if you just use RTP. + * @{ + */ + +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct SpeexMode; + +/** Length of the Speex header identifier */ +#define SPEEX_HEADER_STRING_LENGTH 8 + +/** Maximum number of characters for encoding the Speex version number in the header */ +#define SPEEX_HEADER_VERSION_LENGTH 20 + +/** Speex header info for file-based formats */ +typedef struct SpeexHeader { + char speex_string[SPEEX_HEADER_STRING_LENGTH]; /**< Identifies a Speex bit-stream, always set to "Speex " */ + char speex_version[SPEEX_HEADER_VERSION_LENGTH]; /**< Speex version */ + spx_int32_t speex_version_id; /**< Version for Speex (for checking compatibility) */ + spx_int32_t header_size; /**< Total size of the header ( sizeof(SpeexHeader) ) */ + spx_int32_t rate; /**< Sampling rate used */ + spx_int32_t mode; /**< Mode used (0 for narrowband, 1 for wideband) */ + spx_int32_t mode_bitstream_version; /**< Version ID of the bit-stream */ + spx_int32_t nb_channels; /**< Number of channels encoded */ + spx_int32_t bitrate; /**< Bit-rate used */ + spx_int32_t frame_size; /**< Size of frames */ + spx_int32_t vbr; /**< 1 for a VBR encoding, 0 otherwise */ + spx_int32_t frames_per_packet; /**< Number of frames stored per Ogg packet */ + spx_int32_t extra_headers; /**< Number of additional headers after the comments */ + spx_int32_t reserved1; /**< Reserved for future use, must be zero */ + spx_int32_t reserved2; /**< Reserved for future use, must be zero */ +} SpeexHeader; + +/** Initializes a SpeexHeader using basic information */ +void speex_init_header(SpeexHeader *header, int rate, int nb_channels, const struct SpeexMode *m); + +/** Creates the header packet from the header itself (mostly involves endianness conversion) */ +char *speex_header_to_packet(SpeexHeader *header, int *size); + +/** Creates a SpeexHeader from a packet */ +SpeexHeader *speex_packet_to_header(char *packet, int size); + +/** Frees the memory allocated by either speex_header_to_packet() or speex_packet_to_header() */ +void speex_header_free(void *ptr); + +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif diff --git a/interface/external/Speex/include/speex/speex_jitter.h b/interface/external/Speex/include/speex/speex_jitter.h new file mode 100644 index 0000000000..d68674b13a --- /dev/null +++ b/interface/external/Speex/include/speex/speex_jitter.h @@ -0,0 +1,197 @@ +/* Copyright (C) 2002 Jean-Marc Valin */ +/** + @file speex_jitter.h + @brief Adaptive jitter buffer for Speex +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef SPEEX_JITTER_H +#define SPEEX_JITTER_H +/** @defgroup JitterBuffer JitterBuffer: Adaptive jitter buffer + * This is the jitter buffer that reorders UDP/RTP packets and adjusts the buffer size + * to maintain good quality and low latency. + * @{ + */ + +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Generic adaptive jitter buffer state */ +struct JitterBuffer_; + +/** Generic adaptive jitter buffer state */ +typedef struct JitterBuffer_ JitterBuffer; + +/** Definition of an incoming packet */ +typedef struct _JitterBufferPacket JitterBufferPacket; + +/** Definition of an incoming packet */ +struct _JitterBufferPacket { + char *data; /**< Data bytes contained in the packet */ + spx_uint32_t len; /**< Length of the packet in bytes */ + spx_uint32_t timestamp; /**< Timestamp for the packet */ + spx_uint32_t span; /**< Time covered by the packet (same units as timestamp) */ + spx_uint16_t sequence; /**< RTP Sequence number if available (0 otherwise) */ + spx_uint32_t user_data; /**< Put whatever data you like here (it's ignored by the jitter buffer) */ +}; + +/** Packet has been retrieved */ +#define JITTER_BUFFER_OK 0 +/** Packet is lost or is late */ +#define JITTER_BUFFER_MISSING 1 +/** A "fake" packet is meant to be inserted here to increase buffering */ +#define JITTER_BUFFER_INSERTION 2 +/** There was an error in the jitter buffer */ +#define JITTER_BUFFER_INTERNAL_ERROR -1 +/** Invalid argument */ +#define JITTER_BUFFER_BAD_ARGUMENT -2 + + +/** Set minimum amount of extra buffering required (margin) */ +#define JITTER_BUFFER_SET_MARGIN 0 +/** Get minimum amount of extra buffering required (margin) */ +#define JITTER_BUFFER_GET_MARGIN 1 +/* JITTER_BUFFER_SET_AVAILABLE_COUNT wouldn't make sense */ + +/** Get the amount of available packets currently buffered */ +#define JITTER_BUFFER_GET_AVAILABLE_COUNT 3 +/** Included because of an early misspelling (will remove in next release) */ +#define JITTER_BUFFER_GET_AVALIABLE_COUNT 3 + +/** Assign a function to destroy unused packet. When setting that, the jitter + buffer no longer copies packet data. */ +#define JITTER_BUFFER_SET_DESTROY_CALLBACK 4 +/** */ +#define JITTER_BUFFER_GET_DESTROY_CALLBACK 5 + +/** Tell the jitter buffer to only adjust the delay in multiples of the step parameter provided */ +#define JITTER_BUFFER_SET_DELAY_STEP 6 +/** */ +#define JITTER_BUFFER_GET_DELAY_STEP 7 + +/** Tell the jitter buffer to only do concealment in multiples of the size parameter provided */ +#define JITTER_BUFFER_SET_CONCEALMENT_SIZE 8 +#define JITTER_BUFFER_GET_CONCEALMENT_SIZE 9 + +/** Absolute max amount of loss that can be tolerated regardless of the delay. Typical loss + should be half of that or less. */ +#define JITTER_BUFFER_SET_MAX_LATE_RATE 10 +#define JITTER_BUFFER_GET_MAX_LATE_RATE 11 + +/** Equivalent cost of one percent late packet in timestamp units */ +#define JITTER_BUFFER_SET_LATE_COST 12 +#define JITTER_BUFFER_GET_LATE_COST 13 + + +/** Initialises jitter buffer + * + * @param step_size Starting value for the size of concleanment packets and delay + adjustment steps. Can be changed at any time using JITTER_BUFFER_SET_DELAY_STEP + and JITTER_BUFFER_GET_CONCEALMENT_SIZE. + * @return Newly created jitter buffer state + */ +JitterBuffer *jitter_buffer_init(int step_size); + +/** Restores jitter buffer to its original state + * + * @param jitter Jitter buffer state + */ +void jitter_buffer_reset(JitterBuffer *jitter); + +/** Destroys jitter buffer + * + * @param jitter Jitter buffer state + */ +void jitter_buffer_destroy(JitterBuffer *jitter); + +/** Put one packet into the jitter buffer + * + * @param jitter Jitter buffer state + * @param packet Incoming packet +*/ +void jitter_buffer_put(JitterBuffer *jitter, const JitterBufferPacket *packet); + +/** Get one packet from the jitter buffer + * + * @param jitter Jitter buffer state + * @param packet Returned packet + * @param desired_span Number of samples (or units) we wish to get from the buffer (no guarantee) + * @param current_timestamp Timestamp for the returned packet +*/ +int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int32_t desired_span, spx_int32_t *start_offset); + +/** Used right after jitter_buffer_get() to obtain another packet that would have the same timestamp. + * This is mainly useful for media where a single "frame" can be split into several packets. + * + * @param jitter Jitter buffer state + * @param packet Returned packet + */ +int jitter_buffer_get_another(JitterBuffer *jitter, JitterBufferPacket *packet); + +/** Get pointer timestamp of jitter buffer + * + * @param jitter Jitter buffer state +*/ +int jitter_buffer_get_pointer_timestamp(JitterBuffer *jitter); + +/** Advance by one tick + * + * @param jitter Jitter buffer state +*/ +void jitter_buffer_tick(JitterBuffer *jitter); + +/** Telling the jitter buffer about the remaining data in the application buffer + * @param jitter Jitter buffer state + * @param rem Amount of data buffered by the application (timestamp units) + */ +void jitter_buffer_remaining_span(JitterBuffer *jitter, spx_uint32_t rem); + +/** Used like the ioctl function to control the jitter buffer parameters + * + * @param jitter Jitter buffer state + * @param request ioctl-type request (one of the JITTER_BUFFER_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown +*/ +int jitter_buffer_ctl(JitterBuffer *jitter, int request, void *ptr); + +int jitter_buffer_update_delay(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int32_t *start_offset); + +/* @} */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/interface/external/Speex/include/speex/speex_preprocess.h b/interface/external/Speex/include/speex/speex_preprocess.h new file mode 100644 index 0000000000..f8eef2cd91 --- /dev/null +++ b/interface/external/Speex/include/speex/speex_preprocess.h @@ -0,0 +1,219 @@ +/* Copyright (C) 2003 Epic Games + Written by Jean-Marc Valin */ +/** + * @file speex_preprocess.h + * @brief Speex preprocessor. The preprocess can do noise suppression, + * residual echo suppression (after using the echo canceller), automatic + * gain control (AGC) and voice activity detection (VAD). +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SPEEX_PREPROCESS_H +#define SPEEX_PREPROCESS_H +/** @defgroup SpeexPreprocessState SpeexPreprocessState: The Speex preprocessor + * This is the Speex preprocessor. The preprocess can do noise suppression, + * residual echo suppression (after using the echo canceller), automatic + * gain control (AGC) and voice activity detection (VAD). + * @{ + */ + +#include "speex/speex_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** State of the preprocessor (one per channel). Should never be accessed directly. */ +struct SpeexPreprocessState_; + +/** State of the preprocessor (one per channel). Should never be accessed directly. */ +typedef struct SpeexPreprocessState_ SpeexPreprocessState; + + +/** Creates a new preprocessing state. You MUST create one state per channel processed. + * @param frame_size Number of samples to process at one time (should correspond to 10-20 ms). Must be + * the same value as that used for the echo canceller for residual echo cancellation to work. + * @param sampling_rate Sampling rate used for the input. + * @return Newly created preprocessor state +*/ +SpeexPreprocessState *speex_preprocess_state_init(int frame_size, int sampling_rate); + +/** Destroys a preprocessor state + * @param st Preprocessor state to destroy +*/ +void speex_preprocess_state_destroy(SpeexPreprocessState *st); + +/** Preprocess a frame + * @param st Preprocessor state + * @param x Audio sample vector (in and out). Must be same size as specified in speex_preprocess_state_init(). + * @return Bool value for voice activity (1 for speech, 0 for noise/silence), ONLY if VAD turned on. +*/ +int speex_preprocess_run(SpeexPreprocessState *st, spx_int16_t *x); + +/** Preprocess a frame (deprecated, use speex_preprocess_run() instead)*/ +int speex_preprocess(SpeexPreprocessState *st, spx_int16_t *x, spx_int32_t *echo); + +/** Update preprocessor state, but do not compute the output + * @param st Preprocessor state + * @param x Audio sample vector (in only). Must be same size as specified in speex_preprocess_state_init(). +*/ +void speex_preprocess_estimate_update(SpeexPreprocessState *st, spx_int16_t *x); + +/** Used like the ioctl function to control the preprocessor parameters + * @param st Preprocessor state + * @param request ioctl-type request (one of the SPEEX_PREPROCESS_* macros) + * @param ptr Data exchanged to-from function + * @return 0 if no error, -1 if request in unknown +*/ +int speex_preprocess_ctl(SpeexPreprocessState *st, int request, void *ptr); + + + +/** Set preprocessor denoiser state */ +#define SPEEX_PREPROCESS_SET_DENOISE 0 +/** Get preprocessor denoiser state */ +#define SPEEX_PREPROCESS_GET_DENOISE 1 + +/** Set preprocessor Automatic Gain Control state */ +#define SPEEX_PREPROCESS_SET_AGC 2 +/** Get preprocessor Automatic Gain Control state */ +#define SPEEX_PREPROCESS_GET_AGC 3 + +/** Set preprocessor Voice Activity Detection state */ +#define SPEEX_PREPROCESS_SET_VAD 4 +/** Get preprocessor Voice Activity Detection state */ +#define SPEEX_PREPROCESS_GET_VAD 5 + +/** Set preprocessor Automatic Gain Control level (float) */ +#define SPEEX_PREPROCESS_SET_AGC_LEVEL 6 +/** Get preprocessor Automatic Gain Control level (float) */ +#define SPEEX_PREPROCESS_GET_AGC_LEVEL 7 + +/** Set preprocessor dereverb state */ +#define SPEEX_PREPROCESS_SET_DEREVERB 8 +/** Get preprocessor dereverb state */ +#define SPEEX_PREPROCESS_GET_DEREVERB 9 + +/** Set preprocessor dereverb level */ +#define SPEEX_PREPROCESS_SET_DEREVERB_LEVEL 10 +/** Get preprocessor dereverb level */ +#define SPEEX_PREPROCESS_GET_DEREVERB_LEVEL 11 + +/** Set preprocessor dereverb decay */ +#define SPEEX_PREPROCESS_SET_DEREVERB_DECAY 12 +/** Get preprocessor dereverb decay */ +#define SPEEX_PREPROCESS_GET_DEREVERB_DECAY 13 + +/** Set probability required for the VAD to go from silence to voice */ +#define SPEEX_PREPROCESS_SET_PROB_START 14 +/** Get probability required for the VAD to go from silence to voice */ +#define SPEEX_PREPROCESS_GET_PROB_START 15 + +/** Set probability required for the VAD to stay in the voice state (integer percent) */ +#define SPEEX_PREPROCESS_SET_PROB_CONTINUE 16 +/** Get probability required for the VAD to stay in the voice state (integer percent) */ +#define SPEEX_PREPROCESS_GET_PROB_CONTINUE 17 + +/** Set maximum attenuation of the noise in dB (negative number) */ +#define SPEEX_PREPROCESS_SET_NOISE_SUPPRESS 18 +/** Get maximum attenuation of the noise in dB (negative number) */ +#define SPEEX_PREPROCESS_GET_NOISE_SUPPRESS 19 + +/** Set maximum attenuation of the residual echo in dB (negative number) */ +#define SPEEX_PREPROCESS_SET_ECHO_SUPPRESS 20 +/** Get maximum attenuation of the residual echo in dB (negative number) */ +#define SPEEX_PREPROCESS_GET_ECHO_SUPPRESS 21 + +/** Set maximum attenuation of the residual echo in dB when near end is active (negative number) */ +#define SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE 22 +/** Get maximum attenuation of the residual echo in dB when near end is active (negative number) */ +#define SPEEX_PREPROCESS_GET_ECHO_SUPPRESS_ACTIVE 23 + +/** Set the corresponding echo canceller state so that residual echo suppression can be performed (NULL for no residual echo suppression) */ +#define SPEEX_PREPROCESS_SET_ECHO_STATE 24 +/** Get the corresponding echo canceller state */ +#define SPEEX_PREPROCESS_GET_ECHO_STATE 25 + +/** Set maximal gain increase in dB/second (int32) */ +#define SPEEX_PREPROCESS_SET_AGC_INCREMENT 26 + +/** Get maximal gain increase in dB/second (int32) */ +#define SPEEX_PREPROCESS_GET_AGC_INCREMENT 27 + +/** Set maximal gain decrease in dB/second (int32) */ +#define SPEEX_PREPROCESS_SET_AGC_DECREMENT 28 + +/** Get maximal gain decrease in dB/second (int32) */ +#define SPEEX_PREPROCESS_GET_AGC_DECREMENT 29 + +/** Set maximal gain in dB (int32) */ +#define SPEEX_PREPROCESS_SET_AGC_MAX_GAIN 30 + +/** Get maximal gain in dB (int32) */ +#define SPEEX_PREPROCESS_GET_AGC_MAX_GAIN 31 + +/* Can't set loudness */ +/** Get loudness */ +#define SPEEX_PREPROCESS_GET_AGC_LOUDNESS 33 + +/* Can't set gain */ +/** Get current gain (int32 percent) */ +#define SPEEX_PREPROCESS_GET_AGC_GAIN 35 + +/* Can't set spectrum size */ +/** Get spectrum size for power spectrum (int32) */ +#define SPEEX_PREPROCESS_GET_PSD_SIZE 37 + +/* Can't set power spectrum */ +/** Get power spectrum (int32[] of squared values) */ +#define SPEEX_PREPROCESS_GET_PSD 39 + +/* Can't set noise size */ +/** Get spectrum size for noise estimate (int32) */ +#define SPEEX_PREPROCESS_GET_NOISE_PSD_SIZE 41 + +/* Can't set noise estimate */ +/** Get noise estimate (int32[] of squared values) */ +#define SPEEX_PREPROCESS_GET_NOISE_PSD 43 + +/* Can't set speech probability */ +/** Get speech probability in last frame (int32). */ +#define SPEEX_PREPROCESS_GET_PROB 45 + +/** Set preprocessor Automatic Gain Control level (int32) */ +#define SPEEX_PREPROCESS_SET_AGC_TARGET 46 +/** Get preprocessor Automatic Gain Control level (int32) */ +#define SPEEX_PREPROCESS_GET_AGC_TARGET 47 + +#ifdef __cplusplus +} +#endif + +/** @}*/ +#endif diff --git a/interface/external/Speex/include/speex/speex_resampler.h b/interface/external/Speex/include/speex/speex_resampler.h new file mode 100644 index 0000000000..54eef8d7b8 --- /dev/null +++ b/interface/external/Speex/include/speex/speex_resampler.h @@ -0,0 +1,340 @@ +/* Copyright (C) 2007 Jean-Marc Valin + + File: speex_resampler.h + Resampling code + + The design goals of this code are: + - Very fast algorithm + - Low memory requirement + - Good *perceptual* quality (and not best SNR) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +*/ + + +#ifndef SPEEX_RESAMPLER_H +#define SPEEX_RESAMPLER_H + +#ifdef OUTSIDE_SPEEX + +/********* WARNING: MENTAL SANITY ENDS HERE *************/ + +/* If the resampler is defined outside of Speex, we change the symbol names so that + there won't be any clash if linking with Speex later on. */ + +/* #define RANDOM_PREFIX your software name here */ +#ifndef RANDOM_PREFIX +#error "Please define RANDOM_PREFIX (above) to something specific to your project to prevent symbol name clashes" +#endif + +#define CAT_PREFIX2(a,b) a ## b +#define CAT_PREFIX(a,b) CAT_PREFIX2(a, b) + +#define speex_resampler_init CAT_PREFIX(RANDOM_PREFIX,_resampler_init) +#define speex_resampler_init_frac CAT_PREFIX(RANDOM_PREFIX,_resampler_init_frac) +#define speex_resampler_destroy CAT_PREFIX(RANDOM_PREFIX,_resampler_destroy) +#define speex_resampler_process_float CAT_PREFIX(RANDOM_PREFIX,_resampler_process_float) +#define speex_resampler_process_int CAT_PREFIX(RANDOM_PREFIX,_resampler_process_int) +#define speex_resampler_process_interleaved_float CAT_PREFIX(RANDOM_PREFIX,_resampler_process_interleaved_float) +#define speex_resampler_process_interleaved_int CAT_PREFIX(RANDOM_PREFIX,_resampler_process_interleaved_int) +#define speex_resampler_set_rate CAT_PREFIX(RANDOM_PREFIX,_resampler_set_rate) +#define speex_resampler_get_rate CAT_PREFIX(RANDOM_PREFIX,_resampler_get_rate) +#define speex_resampler_set_rate_frac CAT_PREFIX(RANDOM_PREFIX,_resampler_set_rate_frac) +#define speex_resampler_get_ratio CAT_PREFIX(RANDOM_PREFIX,_resampler_get_ratio) +#define speex_resampler_set_quality CAT_PREFIX(RANDOM_PREFIX,_resampler_set_quality) +#define speex_resampler_get_quality CAT_PREFIX(RANDOM_PREFIX,_resampler_get_quality) +#define speex_resampler_set_input_stride CAT_PREFIX(RANDOM_PREFIX,_resampler_set_input_stride) +#define speex_resampler_get_input_stride CAT_PREFIX(RANDOM_PREFIX,_resampler_get_input_stride) +#define speex_resampler_set_output_stride CAT_PREFIX(RANDOM_PREFIX,_resampler_set_output_stride) +#define speex_resampler_get_output_stride CAT_PREFIX(RANDOM_PREFIX,_resampler_get_output_stride) +#define speex_resampler_get_input_latency CAT_PREFIX(RANDOM_PREFIX,_resampler_get_input_latency) +#define speex_resampler_get_output_latency CAT_PREFIX(RANDOM_PREFIX,_resampler_get_output_latency) +#define speex_resampler_skip_zeros CAT_PREFIX(RANDOM_PREFIX,_resampler_skip_zeros) +#define speex_resampler_reset_mem CAT_PREFIX(RANDOM_PREFIX,_resampler_reset_mem) +#define speex_resampler_strerror CAT_PREFIX(RANDOM_PREFIX,_resampler_strerror) + +#define spx_int16_t short +#define spx_int32_t int +#define spx_uint16_t unsigned short +#define spx_uint32_t unsigned int + +#else /* OUTSIDE_SPEEX */ + +#include "speex/speex_types.h" + +#endif /* OUTSIDE_SPEEX */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SPEEX_RESAMPLER_QUALITY_MAX 10 +#define SPEEX_RESAMPLER_QUALITY_MIN 0 +#define SPEEX_RESAMPLER_QUALITY_DEFAULT 4 +#define SPEEX_RESAMPLER_QUALITY_VOIP 3 +#define SPEEX_RESAMPLER_QUALITY_DESKTOP 5 + +enum { + RESAMPLER_ERR_SUCCESS = 0, + RESAMPLER_ERR_ALLOC_FAILED = 1, + RESAMPLER_ERR_BAD_STATE = 2, + RESAMPLER_ERR_INVALID_ARG = 3, + RESAMPLER_ERR_PTR_OVERLAP = 4, + + RESAMPLER_ERR_MAX_ERROR +}; + +struct SpeexResamplerState_; +typedef struct SpeexResamplerState_ SpeexResamplerState; + +/** Create a new resampler with integer input and output rates. + * @param nb_channels Number of channels to be processed + * @param in_rate Input sampling rate (integer number of Hz). + * @param out_rate Output sampling rate (integer number of Hz). + * @param quality Resampling quality between 0 and 10, where 0 has poor quality + * and 10 has very high quality. + * @return Newly created resampler state + * @retval NULL Error: not enough memory + */ +SpeexResamplerState *speex_resampler_init(spx_uint32_t nb_channels, + spx_uint32_t in_rate, + spx_uint32_t out_rate, + int quality, + int *err); + +/** Create a new resampler with fractional input/output rates. The sampling + * rate ratio is an arbitrary rational number with both the numerator and + * denominator being 32-bit integers. + * @param nb_channels Number of channels to be processed + * @param ratio_num Numerator of the sampling rate ratio + * @param ratio_den Denominator of the sampling rate ratio + * @param in_rate Input sampling rate rounded to the nearest integer (in Hz). + * @param out_rate Output sampling rate rounded to the nearest integer (in Hz). + * @param quality Resampling quality between 0 and 10, where 0 has poor quality + * and 10 has very high quality. + * @return Newly created resampler state + * @retval NULL Error: not enough memory + */ +SpeexResamplerState *speex_resampler_init_frac(spx_uint32_t nb_channels, + spx_uint32_t ratio_num, + spx_uint32_t ratio_den, + spx_uint32_t in_rate, + spx_uint32_t out_rate, + int quality, + int *err); + +/** Destroy a resampler state. + * @param st Resampler state + */ +void speex_resampler_destroy(SpeexResamplerState *st); + +/** Resample a float array. The input and output buffers must *not* overlap. + * @param st Resampler state + * @param channel_index Index of the channel to process for the multi-channel + * base (0 otherwise) + * @param in Input buffer + * @param in_len Number of input samples in the input buffer. Returns the + * number of samples processed + * @param out Output buffer + * @param out_len Size of the output buffer. Returns the number of samples written + */ +int speex_resampler_process_float(SpeexResamplerState *st, + spx_uint32_t channel_index, + const float *in, + spx_uint32_t *in_len, + float *out, + spx_uint32_t *out_len); + +/** Resample an int array. The input and output buffers must *not* overlap. + * @param st Resampler state + * @param channel_index Index of the channel to process for the multi-channel + * base (0 otherwise) + * @param in Input buffer + * @param in_len Number of input samples in the input buffer. Returns the number + * of samples processed + * @param out Output buffer + * @param out_len Size of the output buffer. Returns the number of samples written + */ +int speex_resampler_process_int(SpeexResamplerState *st, + spx_uint32_t channel_index, + const spx_int16_t *in, + spx_uint32_t *in_len, + spx_int16_t *out, + spx_uint32_t *out_len); + +/** Resample an interleaved float array. The input and output buffers must *not* overlap. + * @param st Resampler state + * @param in Input buffer + * @param in_len Number of input samples in the input buffer. Returns the number + * of samples processed. This is all per-channel. + * @param out Output buffer + * @param out_len Size of the output buffer. Returns the number of samples written. + * This is all per-channel. + */ +int speex_resampler_process_interleaved_float(SpeexResamplerState *st, + const float *in, + spx_uint32_t *in_len, + float *out, + spx_uint32_t *out_len); + +/** Resample an interleaved int array. The input and output buffers must *not* overlap. + * @param st Resampler state + * @param in Input buffer + * @param in_len Number of input samples in the input buffer. Returns the number + * of samples processed. This is all per-channel. + * @param out Output buffer + * @param out_len Size of the output buffer. Returns the number of samples written. + * This is all per-channel. + */ +int speex_resampler_process_interleaved_int(SpeexResamplerState *st, + const spx_int16_t *in, + spx_uint32_t *in_len, + spx_int16_t *out, + spx_uint32_t *out_len); + +/** Set (change) the input/output sampling rates (integer value). + * @param st Resampler state + * @param in_rate Input sampling rate (integer number of Hz). + * @param out_rate Output sampling rate (integer number of Hz). + */ +int speex_resampler_set_rate(SpeexResamplerState *st, + spx_uint32_t in_rate, + spx_uint32_t out_rate); + +/** Get the current input/output sampling rates (integer value). + * @param st Resampler state + * @param in_rate Input sampling rate (integer number of Hz) copied. + * @param out_rate Output sampling rate (integer number of Hz) copied. + */ +void speex_resampler_get_rate(SpeexResamplerState *st, + spx_uint32_t *in_rate, + spx_uint32_t *out_rate); + +/** Set (change) the input/output sampling rates and resampling ratio + * (fractional values in Hz supported). + * @param st Resampler state + * @param ratio_num Numerator of the sampling rate ratio + * @param ratio_den Denominator of the sampling rate ratio + * @param in_rate Input sampling rate rounded to the nearest integer (in Hz). + * @param out_rate Output sampling rate rounded to the nearest integer (in Hz). + */ +int speex_resampler_set_rate_frac(SpeexResamplerState *st, + spx_uint32_t ratio_num, + spx_uint32_t ratio_den, + spx_uint32_t in_rate, + spx_uint32_t out_rate); + +/** Get the current resampling ratio. This will be reduced to the least + * common denominator. + * @param st Resampler state + * @param ratio_num Numerator of the sampling rate ratio copied + * @param ratio_den Denominator of the sampling rate ratio copied + */ +void speex_resampler_get_ratio(SpeexResamplerState *st, + spx_uint32_t *ratio_num, + spx_uint32_t *ratio_den); + +/** Set (change) the conversion quality. + * @param st Resampler state + * @param quality Resampling quality between 0 and 10, where 0 has poor + * quality and 10 has very high quality. + */ +int speex_resampler_set_quality(SpeexResamplerState *st, + int quality); + +/** Get the conversion quality. + * @param st Resampler state + * @param quality Resampling quality between 0 and 10, where 0 has poor + * quality and 10 has very high quality. + */ +void speex_resampler_get_quality(SpeexResamplerState *st, + int *quality); + +/** Set (change) the input stride. + * @param st Resampler state + * @param stride Input stride + */ +void speex_resampler_set_input_stride(SpeexResamplerState *st, + spx_uint32_t stride); + +/** Get the input stride. + * @param st Resampler state + * @param stride Input stride copied + */ +void speex_resampler_get_input_stride(SpeexResamplerState *st, + spx_uint32_t *stride); + +/** Set (change) the output stride. + * @param st Resampler state + * @param stride Output stride + */ +void speex_resampler_set_output_stride(SpeexResamplerState *st, + spx_uint32_t stride); + +/** Get the output stride. + * @param st Resampler state copied + * @param stride Output stride + */ +void speex_resampler_get_output_stride(SpeexResamplerState *st, + spx_uint32_t *stride); + +/** Get the latency in input samples introduced by the resampler. + * @param st Resampler state + */ +int speex_resampler_get_input_latency(SpeexResamplerState *st); + +/** Get the latency in output samples introduced by the resampler. + * @param st Resampler state + */ +int speex_resampler_get_output_latency(SpeexResamplerState *st); + +/** Make sure that the first samples to go out of the resamplers don't have + * leading zeros. This is only useful before starting to use a newly created + * resampler. It is recommended to use that when resampling an audio file, as + * it will generate a file with the same length. For real-time processing, + * it is probably easier not to use this call (so that the output duration + * is the same for the first frame). + * @param st Resampler state + */ +int speex_resampler_skip_zeros(SpeexResamplerState *st); + +/** Reset a resampler so a new (unrelated) stream can be processed. + * @param st Resampler state + */ +int speex_resampler_reset_mem(SpeexResamplerState *st); + +/** Returns the English meaning for an error code + * @param err Error code + * @return English string + */ +const char *speex_resampler_strerror(int err); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/interface/external/Speex/include/speex/speex_stereo.h b/interface/external/Speex/include/speex/speex_stereo.h new file mode 100644 index 0000000000..a259713b82 --- /dev/null +++ b/interface/external/Speex/include/speex/speex_stereo.h @@ -0,0 +1,91 @@ +/* Copyright (C) 2002 Jean-Marc Valin*/ +/** + @file speex_stereo.h + @brief Describes the handling for intensity stereo +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef STEREO_H +#define STEREO_H +/** @defgroup SpeexStereoState SpeexStereoState: Handling Speex stereo files + * This describes the Speex intensity stereo encoding/decoding + * @{ + */ + +#include "speex/speex_types.h" +#include "speex/speex_bits.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** If you access any of these fields directly, I'll personally come and bite you */ +typedef struct SpeexStereoState { + float balance; /**< Left/right balance info */ + float e_ratio; /**< Ratio of energies: E(left+right)/[E(left)+E(right)] */ + float smooth_left; /**< Smoothed left channel gain */ + float smooth_right; /**< Smoothed right channel gain */ + float reserved1; /**< Reserved for future use */ + float reserved2; /**< Reserved for future use */ +} SpeexStereoState; + +/** Deprecated. Use speex_stereo_state_init() instead. */ +#define SPEEX_STEREO_STATE_INIT {1,.5,1,1,0,0} + +/** Initialise/create a stereo stereo state */ +SpeexStereoState *speex_stereo_state_init(); + +/** Reset/re-initialise an already allocated stereo state */ +void speex_stereo_state_reset(SpeexStereoState *stereo); + +/** Destroy a stereo stereo state */ +void speex_stereo_state_destroy(SpeexStereoState *stereo); + +/** Transforms a stereo frame into a mono frame and stores intensity stereo info in 'bits' */ +void speex_encode_stereo(float *data, int frame_size, SpeexBits *bits); + +/** Transforms a stereo frame into a mono frame and stores intensity stereo info in 'bits' */ +void speex_encode_stereo_int(spx_int16_t *data, int frame_size, SpeexBits *bits); + +/** Transforms a mono frame into a stereo frame using intensity stereo info */ +void speex_decode_stereo(float *data, int frame_size, SpeexStereoState *stereo); + +/** Transforms a mono frame into a stereo frame using intensity stereo info */ +void speex_decode_stereo_int(spx_int16_t *data, int frame_size, SpeexStereoState *stereo); + +/** Callback handler for intensity stereo info */ +int speex_std_stereo_request_handler(SpeexBits *bits, void *state, void *data); + +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif diff --git a/interface/external/Speex/include/speex/speex_types.h b/interface/external/Speex/include/speex/speex_types.h new file mode 100644 index 0000000000..852fed801d --- /dev/null +++ b/interface/external/Speex/include/speex/speex_types.h @@ -0,0 +1,126 @@ +/* speex_types.h taken from libogg */ +/******************************************************************** + * * + * THIS FILE IS PART OF THE OggVorbis SOFTWARE CODEC SOURCE CODE. * + * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS * + * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * + * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * + * * + * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2002 * + * by the Xiph.Org Foundation http://www.xiph.org/ * + * * + ******************************************************************** + + function: #ifdef jail to whip a few platforms into the UNIX ideal. + last mod: $Id: os_types.h 7524 2004-08-11 04:20:36Z conrad $ + + ********************************************************************/ +/** + @file speex_types.h + @brief Speex types +*/ +#ifndef _SPEEX_TYPES_H +#define _SPEEX_TYPES_H + +#if defined(_WIN32) + +# if defined(__CYGWIN__) +# include <_G_config.h> + typedef _G_int32_t spx_int32_t; + typedef _G_uint32_t spx_uint32_t; + typedef _G_int16_t spx_int16_t; + typedef _G_uint16_t spx_uint16_t; +# elif defined(__MINGW32__) + typedef short spx_int16_t; + typedef unsigned short spx_uint16_t; + typedef int spx_int32_t; + typedef unsigned int spx_uint32_t; +# elif defined(__MWERKS__) + typedef int spx_int32_t; + typedef unsigned int spx_uint32_t; + typedef short spx_int16_t; + typedef unsigned short spx_uint16_t; +# else + /* MSVC/Borland */ + typedef __int32 spx_int32_t; + typedef unsigned __int32 spx_uint32_t; + typedef __int16 spx_int16_t; + typedef unsigned __int16 spx_uint16_t; +# endif + +#elif defined(__MACOS__) + +# include + typedef SInt16 spx_int16_t; + typedef UInt16 spx_uint16_t; + typedef SInt32 spx_int32_t; + typedef UInt32 spx_uint32_t; + +#elif (defined(__APPLE__) && defined(__MACH__)) /* MacOS X Framework build */ + +# include + typedef int16_t spx_int16_t; + typedef u_int16_t spx_uint16_t; + typedef int32_t spx_int32_t; + typedef u_int32_t spx_uint32_t; + +#elif defined(__BEOS__) + + /* Be */ +# include + typedef int16_t spx_int16_t; + typedef u_int16_t spx_uint16_t; + typedef int32_t spx_int32_t; + typedef u_int32_t spx_uint32_t; + +#elif defined (__EMX__) + + /* OS/2 GCC */ + typedef short spx_int16_t; + typedef unsigned short spx_uint16_t; + typedef int spx_int32_t; + typedef unsigned int spx_uint32_t; + +#elif defined (DJGPP) + + /* DJGPP */ + typedef short spx_int16_t; + typedef int spx_int32_t; + typedef unsigned int spx_uint32_t; + +#elif defined(R5900) + + /* PS2 EE */ + typedef int spx_int32_t; + typedef unsigned spx_uint32_t; + typedef short spx_int16_t; + +#elif defined(__SYMBIAN32__) + + /* Symbian GCC */ + typedef signed short spx_int16_t; + typedef unsigned short spx_uint16_t; + typedef signed int spx_int32_t; + typedef unsigned int spx_uint32_t; + +#elif defined(CONFIG_TI_C54X) || defined (CONFIG_TI_C55X) + + typedef short spx_int16_t; + typedef unsigned short spx_uint16_t; + typedef long spx_int32_t; + typedef unsigned long spx_uint32_t; + +#elif defined(CONFIG_TI_C6X) + + typedef short spx_int16_t; + typedef unsigned short spx_uint16_t; + typedef int spx_int32_t; + typedef unsigned int spx_uint32_t; + +#else + +# include + +#endif + +#endif /* _SPEEX_TYPES_H */ diff --git a/interface/external/Speex/lib/MacOS/libspeex.a b/interface/external/Speex/lib/MacOS/libspeex.a new file mode 100644 index 0000000000..81d053f238 Binary files /dev/null and b/interface/external/Speex/lib/MacOS/libspeex.a differ diff --git a/interface/external/Speex/lib/MacOS/libspeexdsp.a b/interface/external/Speex/lib/MacOS/libspeexdsp.a new file mode 100644 index 0000000000..08fe02a630 Binary files /dev/null and b/interface/external/Speex/lib/MacOS/libspeexdsp.a differ diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 45bcad281d..6954339cee 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -502,9 +503,11 @@ void Application::keyPressEvent(QKeyEvent* event) { break; case Qt::Key_Semicolon: - _audio.startEchoTest(); + _audio.ping(); break; - + case Qt::Key_Apostrophe: + _audioScope.inputPaused = !_audioScope.inputPaused; + break; case Qt::Key_L: _displayLevels = !_displayLevels; break; @@ -885,6 +888,10 @@ void Application::editPreferences() { QDoubleSpinBox* headCameraPitchYawScale = new QDoubleSpinBox(); headCameraPitchYawScale->setValue(_headCameraPitchYawScale); form->addRow("Head Camera Pitch/Yaw Scale:", headCameraPitchYawScale); + + QCheckBox* audioEchoCancellation = new QCheckBox(); + audioEchoCancellation->setChecked(_audio.isCancellingEcho()); + form->addRow("Audio Echo Cancellation", audioEchoCancellation); QDoubleSpinBox* leanScale = new QDoubleSpinBox(); leanScale->setValue(_myAvatar.getLeanScale()); @@ -901,7 +908,7 @@ void Application::editPreferences() { QUrl url(avatarURL->text()); _myAvatar.getVoxels()->setVoxelURL(url); sendAvatarVoxelURLMessage(url); - + _audio.setIsCancellingEcho( audioEchoCancellation->isChecked() ); _headCameraPitchYawScale = headCameraPitchYawScale->value(); _myAvatar.setLeanScale(leanScale->value()); } @@ -980,6 +987,7 @@ void Application::doFalseColorizeInView() { } void Application::doFalseColorizeOccluded() { + CoverageMap::wantDebugging = true; _voxels.falseColorizeOccluded(); } @@ -1640,6 +1648,7 @@ void Application::update(float deltaTime) { #ifndef _WIN32 _audio.setLastAcceleration(_myAvatar.getThrust()); _audio.setLastVelocity(_myAvatar.getVelocity()); + _audio.eventuallyAnalyzePing(); #endif } @@ -2674,6 +2683,9 @@ void Application::loadSettings(QSettings* settings) { _viewFrustumOffsetDistance = loadSetting(settings, "viewFrustumOffsetDistance", 0.0f); _viewFrustumOffsetUp = loadSetting(settings, "viewFrustumOffsetUp" , 0.0f); settings->endGroup(); + settings->beginGroup("Audio Echo Cancellation"); + _audio.setIsCancellingEcho(settings->value("enabled", false).toBool()); + settings->endGroup(); scanMenuBar(&Application::loadAction, settings); getAvatar()->loadData(settings); @@ -2693,6 +2705,9 @@ void Application::saveSettings(QSettings* settings) { settings->setValue("viewFrustumOffsetDistance", _viewFrustumOffsetDistance); settings->setValue("viewFrustumOffsetUp", _viewFrustumOffsetUp); settings->endGroup(); + settings->beginGroup("Audio"); + settings->setValue("echoCancellation", _audio.isCancellingEcho()); + settings->endGroup(); scanMenuBar(&Application::saveAction, settings); getAvatar()->saveData(settings); diff --git a/interface/src/Audio.cpp b/interface/src/Audio.cpp index 1f1f9cc335..533b6e9e3b 100644 --- a/interface/src/Audio.cpp +++ b/interface/src/Audio.cpp @@ -19,95 +19,82 @@ #include #include #include +#include #include "Application.h" #include "Audio.h" #include "Util.h" #include "Log.h" -const int NUM_AUDIO_CHANNELS = 2; +#define VISUALIZE_ECHO_CANCELLATION -const int PACKET_LENGTH_BYTES = 1024; -const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; -const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); -const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; +static const int NUM_AUDIO_CHANNELS = 2; -const int PHASE_DELAY_AT_90 = 20; -const float AMPLITUDE_RATIO_AT_90 = 0.5; +static const int PACKET_LENGTH_BYTES = 1024; +static const int PACKET_LENGTH_BYTES_PER_CHANNEL = PACKET_LENGTH_BYTES / 2; +static const int PACKET_LENGTH_SAMPLES = PACKET_LENGTH_BYTES / sizeof(int16_t); +static const int PACKET_LENGTH_SAMPLES_PER_CHANNEL = PACKET_LENGTH_SAMPLES / 2; -const int MIN_FLANGE_EFFECT_THRESHOLD = 600; -const int MAX_FLANGE_EFFECT_THRESHOLD = 1500; -const float FLANGE_BASE_RATE = 4; -const float MAX_FLANGE_SAMPLE_WEIGHT = 0.50; -const float MIN_FLANGE_INTENSITY = 0.25; +static const int PHASE_DELAY_AT_90 = 20; +static const float AMPLITUDE_RATIO_AT_90 = 0.5; -const float JITTER_BUFFER_LENGTH_MSECS = 12; -const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); +static const int MIN_FLANGE_EFFECT_THRESHOLD = 600; +static const int MAX_FLANGE_EFFECT_THRESHOLD = 1500; +static const float FLANGE_BASE_RATE = 4; +static const float MAX_FLANGE_SAMPLE_WEIGHT = 0.50; +static const float MIN_FLANGE_INTENSITY = 0.25; -const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; +static const float JITTER_BUFFER_LENGTH_MSECS = 12; +static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS * + NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); + +static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; + +static const int AGENT_LOOPBACK_MODIFIER = 307; + +// Speex preprocessor and echo canceller adaption +static const int AEC_N_CHANNELS_MIC = 1; // Number of microphone channels +static const int AEC_N_CHANNELS_PLAY = 2; // Number of speaker channels +static const int AEC_FILTER_LENGTH = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 20; // Width of the filter +static const int AEC_BUFFERED_FRAMES = 6; // Maximum number of frames to buffer +static const int AEC_BUFFERED_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * AEC_BUFFERED_FRAMES; +static const int AEC_BUFFERED_SAMPLES = AEC_BUFFERED_SAMPLES_PER_CHANNEL * AEC_N_CHANNELS_PLAY; +static const int AEC_TMP_BUFFER_SIZE = (AEC_N_CHANNELS_MIC + // Temporary space for processing a + AEC_N_CHANNELS_PLAY) * BUFFER_LENGTH_SAMPLES_PER_CHANNEL; // single frame + +// Speex preprocessor and echo canceller configuration +static const int AEC_NOISE_REDUCTION = -80; // Noise reduction (important) +static const int AEC_RESIDUAL_ECHO_REDUCTION = -60; // Residual echo reduction +static const int AEC_RESIDUAL_ECHO_REDUCTION_ACTIVE = -45; // ~on active side +static const bool AEC_USE_AGC = true; // Automatic gain control +static const int AEC_AGC_MAX_GAIN = -30; // Gain in db +static const int AEC_AGC_TARGET_LEVEL = 9000; // Target reference level +static const int AEC_AGC_MAX_INC = 6; // Max increase in db/s +static const int AEC_AGC_MAX_DEC = 200; // Max decrease in db/s +static const bool AEC_USE_VAD = false; // Voice activity determination + +// Ping test configuration +static const float PING_PITCH = 16.f; // Ping wavelength, # samples / radian +static const float PING_VOLUME = 32000.f; // Ping peak amplitude +static const int PING_MIN_AMPLI = 225; // Minimum amplitude +static const int PING_MAX_PERIOD_DIFFERENCE = 15; // Maximum # samples from expected period +static const int PING_PERIOD = int(Radians::twicePi() * PING_PITCH); // Sine period based on the given pitch +static const int PING_HALF_PERIOD = int(Radians::pi() * PING_PITCH); // Distance between extrema +static const int PING_FRAMES_TO_RECORD = AEC_BUFFERED_FRAMES; // Frames to record for analysis +static const int PING_SAMPLES_TO_ANALYZE = AEC_BUFFERED_SAMPLES_PER_CHANNEL; // Samples to analyze (reusing AEC buffer) +static const int PING_BUFFER_OFFSET = BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PING_PERIOD * 2.0f; // Signal start -const int AGENT_LOOPBACK_MODIFIER = 307; +inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight) { -int numStarves = 0; -StDev stdev; - -int samplesLeftForFlange = 0; -int lastYawMeasuredMaximum = 0; -float flangeIntensity = 0; -float flangeRate = 0; -float flangeWeight = 0; - -float usecsAtStartup = 0; - -// inputBuffer A pointer to an internal portaudio data buffer containing data read by portaudio. -// outputBuffer A pointer to an internal portaudio data buffer to be read by the configured output device. -// frames Number of frames that portaudio requests to be read/written. -// timeInfo Portaudio time info. Currently unused. -// statusFlags Portaudio status flags. Currently unused. -// userData Pointer to supplied user data (in this case, a pointer to the parent Audio object -int audioCallback (const void* inputBuffer, - void* outputBuffer, - unsigned long frames, - const PaStreamCallbackTimeInfo *timeInfo, - PaStreamCallbackFlags statusFlags, - void* userData) { - - Audio* parentAudio = (Audio*) userData; AgentList* agentList = AgentList::getInstance(); - Application* interface = Application::getInstance(); Avatar* interfaceAvatar = interface->getAvatar(); - - int16_t* inputLeft = ((int16_t**) inputBuffer)[0]; - int16_t* outputLeft = ((int16_t**) outputBuffer)[0]; - int16_t* outputRight = ((int16_t**) outputBuffer)[1]; + eventuallyCancelEcho(inputLeft); + // Add Procedural effects to input samples - parentAudio->addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - - // add output (@speakers) data to the scope - parentAudio->_scope->addSamples(1, outputLeft, PACKET_LENGTH_SAMPLES_PER_CHANNEL); - parentAudio->_scope->addSamples(2, outputRight, PACKET_LENGTH_SAMPLES_PER_CHANNEL); - - // if needed, add input/output data to echo analysis buffers - if (parentAudio->_echoInputFrameCountdown > 0) { - if (--parentAudio->_echoInputFrameCountdown == 0) { - memcpy(parentAudio->_echoInputSamples, inputLeft, - PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); - parentAudio->_echoInputFrameCountdown = 0; - printLog("got input\n"); - } - } - - if (parentAudio->_isGatheringEchoOutputFrames) { - memcpy(parentAudio->_echoOutputSamples, outputLeft, - PACKET_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); - parentAudio->_isGatheringEchoOutputFrames = false; - parentAudio->_echoInputFrameCountdown = 2; - printLog("got output\n"); - } + addProceduralSounds(inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); if (agentList && inputLeft) { @@ -118,11 +105,17 @@ int audioCallback (const void* inputBuffer, } loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL; - parentAudio->_lastInputLoudness = loudness; + _lastInputLoudness = loudness; // add input (@microphone) data to the scope - parentAudio->_scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); - +#ifdef VISUALIZE_ECHO_CANCELLATION + if (! isCancellingEcho()) { +#endif + _scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); +#ifdef VISUALIZE_ECHO_CANCELLATION + } +#endif + Agent* audioMixer = agentList->soloAgentOfType(AGENT_TYPE_AUDIO_MIXER); if (audioMixer) { @@ -161,7 +154,7 @@ int audioCallback (const void* inputBuffer, memset(outputLeft, 0, PACKET_LENGTH_BYTES_PER_CHANNEL); memset(outputRight, 0, PACKET_LENGTH_BYTES_PER_CHANNEL); - AudioRingBuffer* ringBuffer = &parentAudio->_ringBuffer; + AudioRingBuffer* ringBuffer = &_ringBuffer; // if we've been reset, and there isn't any new packets yet // just play some silence @@ -174,11 +167,11 @@ int audioCallback (const void* inputBuffer, } else if (ringBuffer->diffLastWriteNextOutput() < PACKET_LENGTH_SAMPLES) { ringBuffer->setStarted(false); - ::numStarves++; - parentAudio->_packetsReceivedThisPlayback = 0; + _numStarves++; + _packetsReceivedThisPlayback = 0; // printLog("Starved #%d\n", starve_counter); - parentAudio->_wasStarved = 10; // Frames to render the indication that the system was starved. + _wasStarved = 10; // Frames to render the indication that the system was starved. } else { if (!ringBuffer->isStarted()) { ringBuffer->setStarted(true); @@ -193,20 +186,20 @@ int audioCallback (const void* inputBuffer, int lastYawMeasured = fabsf(interfaceAvatar->getHeadYawRate()); - if (!::samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) { + if (!_samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) { // we should flange for one second - if ((::lastYawMeasuredMaximum = std::max(::lastYawMeasuredMaximum, lastYawMeasured)) != lastYawMeasured) { - ::lastYawMeasuredMaximum = std::min(::lastYawMeasuredMaximum, MIN_FLANGE_EFFECT_THRESHOLD); + if ((_lastYawMeasuredMaximum = std::max(_lastYawMeasuredMaximum, lastYawMeasured)) != lastYawMeasured) { + _lastYawMeasuredMaximum = std::min(_lastYawMeasuredMaximum, MIN_FLANGE_EFFECT_THRESHOLD); - ::samplesLeftForFlange = SAMPLE_RATE; + _samplesLeftForFlange = SAMPLE_RATE; - ::flangeIntensity = MIN_FLANGE_INTENSITY + - ((::lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / + _flangeIntensity = MIN_FLANGE_INTENSITY + + ((_lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) / (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) * (1 - MIN_FLANGE_INTENSITY); - ::flangeRate = FLANGE_BASE_RATE * ::flangeIntensity; - ::flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * ::flangeIntensity; + _flangeRate = FLANGE_BASE_RATE * _flangeIntensity; + _flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * _flangeIntensity; } } @@ -215,12 +208,12 @@ int audioCallback (const void* inputBuffer, int leftSample = ringBuffer->getNextOutput()[s]; int rightSample = ringBuffer->getNextOutput()[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - if (::samplesLeftForFlange > 0) { - float exponent = (SAMPLE_RATE - ::samplesLeftForFlange - (SAMPLE_RATE / ::flangeRate)) / - (SAMPLE_RATE / ::flangeRate); - int sampleFlangeDelay = (SAMPLE_RATE / (1000 * ::flangeIntensity)) * powf(2, exponent); + if (_samplesLeftForFlange > 0) { + float exponent = (SAMPLE_RATE - _samplesLeftForFlange - (SAMPLE_RATE / _flangeRate)) / + (SAMPLE_RATE / _flangeRate); + int sampleFlangeDelay = (SAMPLE_RATE / (1000 * _flangeIntensity)) * powf(2, exponent); - if (::samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) { + if (_samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) { // we have a delayed sample to add to this sample int16_t *flangeFrame = ringBuffer->getNextOutput(); @@ -238,13 +231,13 @@ int audioCallback (const void* inputBuffer, int16_t leftFlangeSample = flangeFrame[flangeIndex]; int16_t rightFlangeSample = flangeFrame[flangeIndex + PACKET_LENGTH_SAMPLES_PER_CHANNEL]; - leftSample = (1 - ::flangeWeight) * leftSample + (::flangeWeight * leftFlangeSample); - rightSample = (1 - ::flangeWeight) * rightSample + (::flangeWeight * rightFlangeSample); + leftSample = (1 - _flangeWeight) * leftSample + (_flangeWeight * leftFlangeSample); + rightSample = (1 - _flangeWeight) * rightSample + (_flangeWeight * rightFlangeSample); - ::samplesLeftForFlange--; + _samplesLeftForFlange--; - if (::samplesLeftForFlange == 0) { - ::lastYawMeasuredMaximum = 0; + if (_samplesLeftForFlange == 0) { + _lastYawMeasuredMaximum = 0; } } } @@ -259,23 +252,48 @@ int audioCallback (const void* inputBuffer, } } } - if (parentAudio->_isSendingEchoPing) { - const float PING_PITCH = 4.f; - const float PING_VOLUME = 32000.f; - for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL; s++) { - outputLeft[s] = outputRight[s] = (int16_t)(sinf((float) s / PING_PITCH) * PING_VOLUME); + + eventuallySendRecvPing(inputLeft, outputLeft, outputRight); + eventuallyRecordEcho(outputLeft, outputRight); + + + // add output (@speakers) data just written to the scope +#ifdef VISUALIZE_ECHO_CANCELLATION + if (! isCancellingEcho()) { + _scope->setColor(2, 0x00ffff); +#endif + _scope->addSamples(1, outputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + _scope->addSamples(2, outputRight, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); +#ifdef VISUALIZE_ECHO_CANCELLATION } - printLog("Send echo ping\n"); - parentAudio->_isSendingEchoPing = false; - parentAudio->_isGatheringEchoOutputFrames = true; +#endif + + gettimeofday(&_lastCallbackTime, NULL); +} + +// inputBuffer A pointer to an internal portaudio data buffer containing data read by portaudio. +// outputBuffer A pointer to an internal portaudio data buffer to be read by the configured output device. +// frames Number of frames that portaudio requests to be read/written. +// timeInfo Portaudio time info. Currently unused. +// statusFlags Portaudio status flags. Currently unused. +// userData Pointer to supplied user data (in this case, a pointer to the parent Audio object +int Audio::audioCallback (const void* inputBuffer, + void* outputBuffer, + unsigned long frames, + const PaStreamCallbackTimeInfo *timeInfo, + PaStreamCallbackFlags statusFlags, + void* userData) { - } - gettimeofday(&parentAudio->_lastCallbackTime, NULL); + int16_t* inputLeft = static_cast(inputBuffer)[0]; + int16_t* outputLeft = static_cast(outputBuffer)[0]; + int16_t* outputRight = static_cast(outputBuffer)[1]; + + static_cast(userData)->performIO(inputLeft, outputLeft, outputRight); return paContinue; } -void outputPortAudioError(PaError error) { +static void outputPortAudioError(PaError error) { if (error != paNoError) { printLog("-- portaudio termination error --\n"); printLog("PortAudio error (%d): %s\n", error, Pa_GetErrorText(error)); @@ -288,20 +306,30 @@ Audio::Audio(Oscilloscope* scope) : _scope(scope), _averagedLatency(0.0), _measuredJitter(0), - _jitterBufferLengthMsecs(12.0), - _jitterBufferSamples(_jitterBufferLengthMsecs * - NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0)), +// _jitterBufferLengthMsecs(12.0), +// _jitterBufferSamples(_jitterBufferLengthMsecs * +// NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0)), _wasStarved(0), + _numStarves(0), _lastInputLoudness(0), _lastVelocity(0), _lastAcceleration(0), _totalPacketsReceived(0), _firstPlaybackTime(), _packetsReceivedThisPlayback(0), - _shouldStartEcho(false), + _isCancellingEcho(false), + _echoDelay(0), + _echoSamplesLeft(0l), + _speexEchoState(NULL), + _speexPreprocessState(NULL), _isSendingEchoPing(false), - _echoInputFrameCountdown(0), - _isGatheringEchoOutputFrames(false) + _pingAnalysisPending(false), + _pingFramesToRecord(0), + _samplesLeftForFlange(0), + _lastYawMeasuredMaximum(0), + _flangeIntensity(0.0f), + _flangeRate(0.0f), + _flangeWeight(0.0f) { outputPortAudioError(Pa_Initialize()); outputPortAudioError(Pa_OpenDefaultStream(&_stream, @@ -312,15 +340,48 @@ Audio::Audio(Oscilloscope* scope) : BUFFER_LENGTH_SAMPLES_PER_CHANNEL, audioCallback, (void*) this)); - + + if (! _stream) { + return; + } + + _echoSamplesLeft = new int16_t[AEC_BUFFERED_SAMPLES + AEC_TMP_BUFFER_SIZE]; + if (! _echoSamplesLeft) { + return; + } + memset(_echoSamplesLeft, 0, AEC_BUFFERED_SAMPLES * sizeof(int16_t)); + _echoSamplesRight = _echoSamplesLeft + AEC_BUFFERED_SAMPLES_PER_CHANNEL; + _speexTmpBuf = _echoSamplesRight + AEC_BUFFERED_SAMPLES_PER_CHANNEL; + + _speexPreprocessState = speex_preprocess_state_init(BUFFER_LENGTH_SAMPLES_PER_CHANNEL, SAMPLE_RATE); + if (_speexPreprocessState) { + _speexEchoState = speex_echo_state_init_mc(BUFFER_LENGTH_SAMPLES_PER_CHANNEL, + AEC_FILTER_LENGTH, AEC_N_CHANNELS_MIC, AEC_N_CHANNELS_PLAY); + if (_speexEchoState) { + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_ECHO_STATE, _speexEchoState); + int tmp; + speex_echo_ctl(_speexEchoState, SPEEX_ECHO_SET_SAMPLING_RATE, &(tmp = SAMPLE_RATE)); + tmp = AEC_NOISE_REDUCTION; + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &tmp); + tmp = AEC_RESIDUAL_ECHO_REDUCTION; + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS, &tmp); + tmp = AEC_RESIDUAL_ECHO_REDUCTION_ACTIVE; + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE, &tmp); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_AGC, &(tmp = int(AEC_USE_AGC))); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_AGC_MAX_GAIN, &(tmp = AEC_AGC_MAX_GAIN)); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_AGC_TARGET, &(tmp = AEC_AGC_TARGET_LEVEL)); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &(tmp = AEC_AGC_MAX_INC)); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_AGC_DECREMENT, &(tmp = AEC_AGC_MAX_DEC)); + speex_preprocess_ctl(_speexPreprocessState, SPEEX_PREPROCESS_SET_VAD, &(tmp = int(AEC_USE_VAD))); + } else { + speex_preprocess_state_destroy(_speexPreprocessState); + _speexPreprocessState = NULL; + } + } + // start the stream now that sources are good to go outputPortAudioError(Pa_StartStream(_stream)); - - _echoInputSamples = new int16_t[BUFFER_LENGTH_BYTES_PER_CHANNEL]; - _echoOutputSamples = new int16_t[BUFFER_LENGTH_BYTES_PER_CHANNEL]; - memset(_echoInputSamples, 0, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int)); - memset(_echoOutputSamples, 0, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int)); - + gettimeofday(&_lastReceiveTime, NULL); } @@ -329,89 +390,13 @@ Audio::~Audio() { outputPortAudioError(Pa_CloseStream(_stream)); outputPortAudioError(Pa_Terminate()); } -} - -void Audio::renderEchoCompare() { - const int XPOS = 0; - const int YPOS = 500; - const int YSCALE = 500; - const int XSCALE = 2; - glPointSize(1.0); - glLineWidth(1.0); - glDisable(GL_LINE_SMOOTH); - glColor3f(1,1,1); - glBegin(GL_LINE_STRIP); - for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - glVertex2f(XPOS + i * XSCALE, YPOS + _echoInputSamples[i]/YSCALE); + if (_speexEchoState) { + speex_preprocess_state_destroy(_speexPreprocessState); + speex_echo_state_destroy(_speexEchoState); } - glEnd(); - glColor3f(0,1,1); - glBegin(GL_LINE_STRIP); - for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) { - glVertex2f(XPOS + i * XSCALE, YPOS + _echoOutputSamples[i]/YSCALE); - } - glEnd(); + delete[] _echoSamplesLeft; } -// Take a pointer to the acquired microphone input samples and add procedural sounds -void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { - const float MAX_AUDIBLE_VELOCITY = 6.0; - const float MIN_AUDIBLE_VELOCITY = 0.1; - const int VOLUME_BASELINE = 400; - const float SOUND_PITCH = 8.f; - - float speed = glm::length(_lastVelocity); - float volume = VOLUME_BASELINE * (1.f - speed / MAX_AUDIBLE_VELOCITY); - - // Add a noise-modulated sinewave with volume that tapers off with speed increasing - if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { - for (int i = 0; i < numSamples; i++) { - inputBuffer[i] += (int16_t)((sinf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed); - } - } -} - -void Audio::startEchoTest() { - _shouldStartEcho = true; - _isSendingEchoPing = true; - _isGatheringEchoOutputFrames = false; - -} - -void Audio::analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples) { - // Compare output and input streams, looking for evidence of correlation needing echo cancellation - // - // OFFSET_RANGE tells us how many samples to vary the analysis window when looking for correlation, - // and should be equal to the largest physical distance between speaker and microphone, where - // OFFSET_RANGE = 1 / (speedOfSound (meters / sec) / SamplingRate (samples / sec)) * distance - // - const int OFFSET_RANGE = 10; - const int SIGNAL_FLOOR = 1000; - float correlation[2 * OFFSET_RANGE + 1]; - int numChecked = 0; - bool foundSignal = false; - - memset(correlation, 0, sizeof(float) * (2 * OFFSET_RANGE + 1)); - - for (int offset = -OFFSET_RANGE; offset <= OFFSET_RANGE; offset++) { - for (int i = 0; i < numSamples; i++) { - if ((i + offset >= 0) && (i + offset < numSamples)) { - correlation[offset + OFFSET_RANGE] += - (float) abs(inputBuffer[i] - outputBuffer[i + offset]); - numChecked++; - foundSignal |= (inputBuffer[i] > SIGNAL_FLOOR); - } - } - correlation[offset + OFFSET_RANGE] /= numChecked; - numChecked = 0; - if (foundSignal) { - printLog("%4.2f, ", correlation[offset + OFFSET_RANGE]); - } - } - if (foundSignal) printLog("\n"); -} - - void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes) { const int NUM_INITIAL_PACKETS_DISCARD = 3; @@ -423,12 +408,12 @@ void Audio::addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBy // Discard first few received packets for computing jitter (often they pile up on start) if (_totalPacketsReceived > NUM_INITIAL_PACKETS_DISCARD) { - ::stdev.addValue(timeDiff); + _stdev.addValue(timeDiff); } - if (::stdev.getSamples() > 500) { - _measuredJitter = ::stdev.getStDev(); - ::stdev.reset(); + if (_stdev.getSamples() > 500) { + _measuredJitter = _stdev.getStDev(); + _stdev.reset(); } if (!_ringBuffer.isStarted()) { @@ -538,4 +523,258 @@ void Audio::render(int screenWidth, int screenHeight) { } } +// Take a pointer to the acquired microphone input samples and add procedural sounds +void Audio::addProceduralSounds(int16_t* inputBuffer, int numSamples) { + const float MAX_AUDIBLE_VELOCITY = 6.0; + const float MIN_AUDIBLE_VELOCITY = 0.1; + const int VOLUME_BASELINE = 400; + const float SOUND_PITCH = 8.f; + + float speed = glm::length(_lastVelocity); + float volume = VOLUME_BASELINE * (1.f - speed / MAX_AUDIBLE_VELOCITY); + + // Add a noise-modulated sinewave with volume that tapers off with speed increasing + if ((speed > MIN_AUDIBLE_VELOCITY) && (speed < MAX_AUDIBLE_VELOCITY)) { + for (int i = 0; i < numSamples; i++) { + inputBuffer[i] += (int16_t)((sinf((float) i / SOUND_PITCH * speed) * randFloat()) * volume * speed); + } + } +} + +// ----------------------------- +// Speex-based echo cancellation +// ----------------------------- + +bool Audio::isCancellingEcho() const { + return _isCancellingEcho && ! (_pingFramesToRecord != 0 || _pingAnalysisPending || ! _speexPreprocessState); +} + +void Audio::setIsCancellingEcho(bool enable) { + if (enable && _speexPreprocessState) { + speex_echo_state_reset(_speexEchoState); + _echoWritePos = 0; + memset(_echoSamplesLeft, 0, AEC_BUFFERED_SAMPLES * sizeof(int16_t)); + } + _isCancellingEcho = enable; +} + +inline void Audio::eventuallyCancelEcho(int16_t* inputLeft) { + if (! isCancellingEcho()) { + return; + } + + // Construct an artificial frame from the captured playback + // that contains the appropriately delayed output to cancel + unsigned n = BUFFER_LENGTH_SAMPLES_PER_CHANNEL, n2 = 0; + unsigned readPos = (_echoWritePos + AEC_BUFFERED_SAMPLES_PER_CHANNEL - _echoDelay) % AEC_BUFFERED_SAMPLES_PER_CHANNEL; + unsigned readEnd = readPos + n; + if (readEnd >= AEC_BUFFERED_SAMPLES_PER_CHANNEL) { + n2 = (readEnd -= AEC_BUFFERED_SAMPLES_PER_CHANNEL); + n -= n2; + } + // Use two subsequent buffers for the two stereo channels + int16_t* playBufferLeft = _speexTmpBuf + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + memcpy(playBufferLeft, _echoSamplesLeft + readPos, n * sizeof(int16_t)); + memcpy(playBufferLeft + n, _echoSamplesLeft, n2 * sizeof(int16_t)); + int16_t* playBufferRight = playBufferLeft + BUFFER_LENGTH_SAMPLES_PER_CHANNEL; + memcpy(playBufferRight, _echoSamplesRight + readPos, n * sizeof(int16_t)); + memcpy(playBufferRight + n, _echoSamplesLeft, n2 * sizeof(int16_t)); + +#ifdef VISUALIZE_ECHO_CANCELLATION + // Visualize the input + _scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); + _scope->addSamples(1, playBufferLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); +#endif + + // Have Speex perform echo cancellation + speex_echo_cancellation(_speexEchoState, inputLeft, playBufferLeft, _speexTmpBuf); + memcpy(inputLeft, _speexTmpBuf, BUFFER_LENGTH_BYTES_PER_CHANNEL); + speex_preprocess_run(_speexPreprocessState, inputLeft); + +#ifdef VISUALIZE_ECHO_CANCELLATION + // Visualize the result + _scope->setColor(2, 0x00ff00); + _scope->addSamples(2, inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL); +#endif +} + +inline void Audio::eventuallyRecordEcho(int16_t* outputLeft, int16_t* outputRight) { + if (! isCancellingEcho()) { + return; + } + + // Copy playback data to circular buffers + unsigned n = BUFFER_LENGTH_SAMPLES_PER_CHANNEL, n2 = 0; + unsigned writeEnd = _echoWritePos + n; + if (writeEnd >= AEC_BUFFERED_SAMPLES_PER_CHANNEL) { + n2 = (writeEnd -= AEC_BUFFERED_SAMPLES_PER_CHANNEL); + n -= n2; + } + memcpy(_echoSamplesLeft + _echoWritePos, outputLeft, n * sizeof(int16_t)); + memcpy(_echoSamplesLeft, outputLeft + n, n2 * sizeof(int16_t)); + memcpy(_echoSamplesRight + _echoWritePos, outputRight, n * sizeof(int16_t)); + memcpy(_echoSamplesRight, outputRight + n, n2 * sizeof(int16_t)); + + _echoWritePos = writeEnd; +} + +// ----------------------------------------------------------- +// Accoustic ping (audio system round trip time determination) +// ----------------------------------------------------------- + +void Audio::ping() { + + _pingFramesToRecord = PING_FRAMES_TO_RECORD; + _isSendingEchoPing = true; + _scope->setDownsampleRatio(8); + _scope->inputPaused = false; +} + +inline void Audio::eventuallySendRecvPing(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight) { + + if (_isSendingEchoPing) { + + // Overwrite output with ping signal. + // + // Using a signed variant of sinc because it's speaker-reproducible + // with a unique, characteristic point in time (its center), aligned + // to the right of the output buffer. + // + // | + // | | + // ...--- t --------+-+-+-+-+-------> + // | | : + // | : + // buffer :<- start of next buffer + // : : : + // :---: sine period + // :-: half sine period + // + memset(outputLeft, 0, PING_BUFFER_OFFSET * sizeof(int16_t)); + outputLeft += PING_BUFFER_OFFSET; + memset(outputRight, 0, PING_BUFFER_OFFSET * sizeof(int16_t)); + outputRight += PING_BUFFER_OFFSET; + for (int s = -PING_PERIOD; s < PING_PERIOD; ++s) { + float t = float(s) / PING_PITCH; + *outputLeft++ = *outputRight++ = int16_t(PING_VOLUME * + sinf(t) / fmaxf(1.0f, pow((abs(t)-1.5f) / 1.5f, 1.2f))); + } + + // As of the next frame, we'll be recoding PING_FRAMES_TO_RECORD from + // the mic (pointless to start now as we can't record unsent audio). + _isSendingEchoPing = false; + printLog("Send audio ping\n"); + + } else if (_pingFramesToRecord > 0) { + + // Store input samples + int offset = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * ( + PING_FRAMES_TO_RECORD - _pingFramesToRecord); + memcpy(_echoSamplesLeft + offset, + inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL * sizeof(int16_t)); + + --_pingFramesToRecord; + + if (_pingFramesToRecord == 0) { + _pingAnalysisPending = true; + printLog("Received ping echo\n"); + } + } +} + +static int findExtremum(int16_t const* samples, int length, int sign) { + + int x0 = -1; + int y0 = -PING_VOLUME; + for (int x = 0; x < length; ++samples, ++x) { + int y = *samples * sign; + if (y > y0) { + x0 = x; + y0 = y; + } + } + return x0; +} + +inline void Audio::analyzePing() { + + // Determine extrema + int botAt = findExtremum(_echoSamplesLeft, PING_SAMPLES_TO_ANALYZE, -1); + if (botAt == -1) { + printLog("Audio Ping: Minimum not found.\n"); + return; + } + int topAt = findExtremum(_echoSamplesLeft, PING_SAMPLES_TO_ANALYZE, 1); + if (topAt == -1) { + printLog("Audio Ping: Maximum not found.\n"); + return; + } + + // Determine peak amplitude - warn if low + int ampli = (_echoSamplesLeft[topAt] - _echoSamplesLeft[botAt]) / 2; + if (ampli < PING_MIN_AMPLI) { + printLog("Audio Ping unreliable - low amplitude %d.\n", ampli); + } + + // Determine period - warn if doesn't look like our signal + int halfPeriod = abs(topAt - botAt); + if (abs(halfPeriod-PING_HALF_PERIOD) > PING_MAX_PERIOD_DIFFERENCE) { + printLog("Audio Ping unreliable - peak distance %d vs. %d\n", halfPeriod, PING_HALF_PERIOD); + } + + // Ping is sent: + // + // ---[ record ]--[ play ]--- audio in space/time ---> + // : : : + // : : ping: ->X<- + // : : : + // : : |+| (buffer end - signal center = t1-t0) + // : |<----------+ + // : : : : + // : ->X<- (corresponding input buffer position t0) + // : : : : + // : : : : + // : : : : + // Next frame (we're recording from now on): + // : : : + // : - - --[ record ]--[ play ]------------------> + // : : : : + // : : |<-- (start of recording t1) + // : : : + // : : : + // At some frame, the signal is picked up: + // : : : : + // : : : : + // : : : V + // : : : - - --[ record ]--[ play ]----------> + // : V : : + // : |<--------->| + // |+|<------->| period + measured samples + // + // If we could pick up the signal at t0 we'd have zero round trip + // time - in this case we had recorded the output buffer instantly + // in its entirety (we can't - but there's the proper reference + // point). We know the number of samples from t1 and, knowing that + // data is streaming continuously, we know that t1-t0 is the distance + // of the characterisic point from the end of the buffer. + + int delay = (botAt + topAt) / 2 + PING_PERIOD; + + printLog("\n| Audio Ping results:\n+----- ---- --- - - - - -\n\n" + "Delay = %d samples (%d ms)\nPeak amplitude = %d\n\n", + delay, (delay * 1000) / int(SAMPLE_RATE), ampli); +} + +bool Audio::eventuallyAnalyzePing() { + + if (! _pingAnalysisPending) { + return false; + } + _scope->inputPaused = true; + analyzePing(); + setIsCancellingEcho(_isCancellingEcho); + _pingAnalysisPending = false; + return true; +} + #endif diff --git a/interface/src/Audio.h b/interface/src/Audio.h index 197daf0194..ad27b5e711 100644 --- a/interface/src/Audio.h +++ b/interface/src/Audio.h @@ -11,7 +11,11 @@ #include +#include +#include + #include +#include #include "Oscilloscope.h" #include "Avatar.h" @@ -24,57 +28,91 @@ public: void render(int screenWidth, int screenHeight); + void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); + float getLastInputLoudness() const { return _lastInputLoudness; }; void setLastAcceleration(glm::vec3 lastAcceleration) { _lastAcceleration = lastAcceleration; }; void setLastVelocity(glm::vec3 lastVelocity) { _lastVelocity = lastVelocity; }; - - void addProceduralSounds(int16_t* inputBuffer, int numSamples); - void analyzeEcho(int16_t* inputBuffer, int16_t* outputBuffer, int numSamples); - - void addReceivedAudioToBuffer(unsigned char* receivedData, int receivedBytes); - - void startEchoTest(); - void renderEchoCompare(); - + void setIsCancellingEcho(bool enabled); + bool isCancellingEcho() const; + + void ping(); + + // Call periodically to eventually perform round trip time analysis, + // in which case 'true' is returned - otherwise the return value is 'false'. + // The results of the analysis are written to the log. + bool eventuallyAnalyzePing(); + private: PaStream* _stream; AudioRingBuffer _ringBuffer; Oscilloscope* _scope; + StDev _stdev; timeval _lastCallbackTime; timeval _lastReceiveTime; float _averagedLatency; float _measuredJitter; - float _jitterBufferLengthMsecs; - short _jitterBufferSamples; +// float _jitterBufferLengthMsecs; // currently unused +// short _jitterBufferSamples; // currently unsused int _wasStarved; + int _numStarves; float _lastInputLoudness; glm::vec3 _lastVelocity; glm::vec3 _lastAcceleration; int _totalPacketsReceived; timeval _firstPlaybackTime; int _packetsReceivedThisPlayback; - // Echo Analysis - bool _shouldStartEcho; - bool _isSendingEchoPing; - int16_t* _echoInputSamples; - int16_t* _echoOutputSamples; - int _echoInputFrameCountdown; - bool _isGatheringEchoOutputFrames; + // Echo cancellation + volatile bool _isCancellingEcho; + unsigned _echoWritePos; + unsigned _echoDelay; + int16_t* _echoSamplesLeft; + int16_t* _echoSamplesRight; + int16_t* _speexTmpBuf; + SpeexEchoState* _speexEchoState; + SpeexPreprocessState* _speexPreprocessState; + // Ping analysis + volatile bool _isSendingEchoPing; + volatile bool _pingAnalysisPending; + int _pingFramesToRecord; + // Flange effect + int _samplesLeftForFlange; + int _lastYawMeasuredMaximum; + float _flangeIntensity; + float _flangeRate; + float _flangeWeight; + + // Audio callback in class context. + inline void performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); + + // When echo cancellation is enabled, subtract recorded echo from the input. + // Called from 'performIO' before the input has been processed. + inline void eventuallyCancelEcho(int16_t* inputLeft); + // When EC is enabled, record output samples. + // Called from 'performIO' after the output has been generated. + inline void eventuallyRecordEcho(int16_t* outputLeft, int16_t* outputRight); + + // When requested, sends/receives a signal for round trip time determination. + // Called from 'performIO'. + inline void eventuallySendRecvPing(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight); + + // Determines round trip time of the audio system. Called from 'eventuallyAnalyzePing'. + inline void analyzePing(); + + void addProceduralSounds(int16_t* inputBuffer, int numSamples); + + + // Audio callback called by portaudio. Calls 'performIO'. + static int audioCallback(const void *inputBuffer, + void *outputBuffer, + unsigned long framesPerBuffer, + const PaStreamCallbackTimeInfo *timeInfo, + PaStreamCallbackFlags statusFlags, + void *userData); - - - // give access to AudioData class from audioCallback - friend int audioCallback (const void*, void*, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*); }; -// Audio callback called by portaudio. -int audioCallback (const void *inputBuffer, - void *outputBuffer, - unsigned long framesPerBuffer, - const PaStreamCallbackTimeInfo *timeInfo, - PaStreamCallbackFlags statusFlags, - void *userData); #endif /* defined(__interface__audio__) */ diff --git a/interface/src/Oscilloscope.cpp b/interface/src/Oscilloscope.cpp index 5fbd747f0d..ebaddbbb04 100644 --- a/interface/src/Oscilloscope.cpp +++ b/interface/src/Oscilloscope.cpp @@ -54,6 +54,10 @@ Oscilloscope::Oscilloscope(int w, int h, bool isEnabled) : for (unsigned ch = 0; ch < MAX_CHANNELS; ++ch) { _writePos[ch] = MAX_SAMPLES_PER_CHANNEL * ch; } + + _colors[0] = 0xffffff; + _colors[1] = 0x00ffff; + _colors[2] = 0x00ffff; } Oscilloscope::~Oscilloscope() { @@ -138,17 +142,22 @@ void Oscilloscope::render(int x, int y) { glScaled(1.0f, _height / 32767.0f, 1.0f); glVertexPointer(2, GL_SHORT, 0, _vertices); glEnableClientState(GL_VERTEX_ARRAY); + glDisableClientState(GL_COLOR_ARRAY); + glDisableClientState(GL_INDEX_ARRAY); + glDisableClientState(GL_NORMAL_ARRAY); + glDisableClientState(GL_TEXTURE_COORD_ARRAY); // render channel 0 - glColor3f(1.0f, 1.0f, 1.0f); + glColor3ub(GLubyte(_colors[0] >> 16), GLubyte((_colors[0] >> 8) & 0xff), GLubyte(_colors[0] & 0xff)); glDrawArrays(GL_LINES, MAX_SAMPLES_PER_CHANNEL * 0, usedWidth); // render channel 1 glColor3f(0.0f, 1.0f ,1.0f); + glColor3ub(GLubyte(_colors[1] >> 16), GLubyte((_colors[1] >> 8) & 0xff), GLubyte(_colors[1] & 0xff)); glDrawArrays(GL_LINES, MAX_SAMPLES_PER_CHANNEL * 1, usedWidth); // render channel 2 - glColor3f(0.0f, 1.0f ,1.0f); + glColor3ub(GLubyte(_colors[2] >> 16), GLubyte((_colors[2] >> 8) & 0xff), GLubyte(_colors[2] & 0xff)); glDrawArrays(GL_LINES, MAX_SAMPLES_PER_CHANNEL * 2, usedWidth); // reset rendering state diff --git a/interface/src/Oscilloscope.h b/interface/src/Oscilloscope.h index 65616e701b..d81fc11358 100644 --- a/interface/src/Oscilloscope.h +++ b/interface/src/Oscilloscope.h @@ -28,6 +28,9 @@ public: static unsigned const MAX_CHANNELS = 3; static unsigned const MAX_SAMPLES_PER_CHANNEL = 4096; + // Sets the color for a specific channel. + void setColor(unsigned ch, unsigned rgb) { assert(ch < MAX_CHANNELS); if (! inputPaused) { _colors[ch] = rgb; } } + // Controls a simple one pole IIR low pass filter that is provided to // reduce high frequencies aliasing (to lower ones) when downsampling. // @@ -54,7 +57,7 @@ public: // Sets the number of input samples per output sample. Without filtering // just uses every nTh sample. void setDownsampleRatio(unsigned n) { assert(n > 0); _downsampleRatio = n; } - + private: // don't copy/assign Oscilloscope(Oscilloscope const&); // = delete; @@ -70,6 +73,7 @@ private: float _lowPassCoeff; unsigned _downsampleRatio; + unsigned _colors[MAX_CHANNELS]; }; #endif /* defined(__interface__oscilloscope__) */ diff --git a/interface/src/VoxelSystem.cpp b/interface/src/VoxelSystem.cpp index 4a81d3ec2d..69edf376ee 100644 --- a/interface/src/VoxelSystem.cpp +++ b/interface/src/VoxelSystem.cpp @@ -1202,19 +1202,19 @@ bool VoxelSystem::falseColorizeOccludedOperation(VoxelNode* node, void* extraDat AABox voxelBox = node->getAABox(); voxelBox.scale(TREE_SCALE); - VoxelProjectedPolygon* voxelShadow = new VoxelProjectedPolygon(args->viewFrustum->getProjectedShadow(voxelBox)); + VoxelProjectedPolygon* voxelPolygon = new VoxelProjectedPolygon(args->viewFrustum->getProjectedPolygon(voxelBox)); // If we're not all in view, then ignore it, and just return. But keep searching... - if (!voxelShadow->getAllInView()) { + if (!voxelPolygon->getAllInView()) { args->nonLeavesOutOfView++; - delete voxelShadow; + delete voxelPolygon; return true; } - CoverageMap::StorageResult result = args->map->checkMap(voxelShadow, false); - if (result == CoverageMap::OCCLUDED) { + CoverageMapStorageResult result = args->map->checkMap(voxelPolygon, false); + if (result == OCCLUDED) { args->nonLeavesOccluded++; - delete voxelShadow; + delete voxelPolygon; FalseColorizeSubTreeOperationArgs subArgs; subArgs.color[0] = 0; @@ -1230,7 +1230,7 @@ bool VoxelSystem::falseColorizeOccludedOperation(VoxelNode* node, void* extraDat return false; } - delete voxelShadow; + delete voxelPolygon; return true; // keep looking... } @@ -1239,23 +1239,23 @@ bool VoxelSystem::falseColorizeOccludedOperation(VoxelNode* node, void* extraDat AABox voxelBox = node->getAABox(); voxelBox.scale(TREE_SCALE); - VoxelProjectedPolygon* voxelShadow = new VoxelProjectedPolygon(args->viewFrustum->getProjectedShadow(voxelBox)); + VoxelProjectedPolygon* voxelPolygon = new VoxelProjectedPolygon(args->viewFrustum->getProjectedPolygon(voxelBox)); // If we're not all in view, then ignore it, and just return. But keep searching... - if (!voxelShadow->getAllInView()) { + if (!voxelPolygon->getAllInView()) { args->outOfView++; - delete voxelShadow; + delete voxelPolygon; return true; } - CoverageMap::StorageResult result = args->map->checkMap(voxelShadow, true); - if (result == CoverageMap::OCCLUDED) { + CoverageMapStorageResult result = args->map->checkMap(voxelPolygon, true); + if (result == OCCLUDED) { node->setFalseColor(255, 0, 0); args->occludedVoxels++; - } else if (result == CoverageMap::STORED) { + } else if (result == STORED) { args->notOccludedVoxels++; //printLog("***** falseColorizeOccludedOperation() NODE is STORED *****\n"); - } else if (result == CoverageMap::DOESNT_FIT) { + } else if (result == DOESNT_FIT) { //printLog("***** falseColorizeOccludedOperation() NODE DOESNT_FIT???? *****\n"); } } diff --git a/jenkins/jobs.groovy b/jenkins/jobs.groovy index 273ad6c674..45f2b19477 100644 --- a/jenkins/jobs.groovy +++ b/jenkins/jobs.groovy @@ -1,16 +1,18 @@ +JENKINS_URL = 'https://jenkins.below92.com/' +GITHUB_HOOK_URL = 'https://github.com/worklist/hifi/' +GIT_REPO_URL = 'git@github.com:worklist/hifi.git' +HIPCHAT_ROOM = 'High Fidelity' + def hifiJob(String targetName, Boolean deploy) { - def JENKINS_URL = 'https://jenkins.below92.com/' - def GITHUB_HOOK_URL = 'https://github.com/worklist/hifi/' - def GIT_REPO_URL = 'git@github.com:worklist/hifi.git' - def HIPCHAT_ROOM = 'High Fidelity' - job { name "hifi-${targetName}" logRotator(7, -1, -1, -1) scm { git(GIT_REPO_URL, 'master') { node -> - node << includedRegions << "${targetName}/.*\nlibraries/.*" + node / includedRegions << "${targetName}/.*\nlibraries/.*" + node / 'userRemoteConfigs' / 'hudson.plugins.git.UserRemoteConfig' / 'name' << '' + node / 'userRemoteConfigs' / 'hudson.plugins.git.UserRemoteConfig' / 'refspec' << '' } } @@ -22,29 +24,20 @@ def hifiJob(String targetName, Boolean deploy) { 'jenkins.plugins.hipchat.HipChatNotifier_-HipChatJobProperty' { room HIPCHAT_ROOM - } + } + + 'hudson.plugins.buildblocker.BuildBlockerProperty' { + useBuildBlocker true + blockingJobs 'hifi--seed' + } } project / 'triggers' << 'com.cloudbees.jenkins.GitHubPushTrigger' { spec '' } - - project / 'builders' << 'hudson.plugins.cmake.CmakeBuilder' { - sourceDir targetName - buildDir 'build' - installDir '' - buildType 'RelWithDebInfo' - generator 'Unix Makefiles' - makeCommand 'make' - installCommand 'make install' - preloadScript '' - cmakeArgs '' - projectCmakePath '/usr/bin/cmake' - cleanBuild 'false' - cleanInstallDir 'false' - builderImpl '' - } - } + } + + configure cmakeBuild(targetName, 'make install') if (deploy) { publishers { @@ -85,26 +78,44 @@ def hifiJob(String targetName, Boolean deploy) { } } -def deployTargets = [ - 'animation-server', - 'audio-mixer', - 'avatar-mixer', - 'domain-server', - 'eve', - 'pairing-server', - 'space-server', - 'voxel-server' -] - -/* setup all of the deploys jobs that use the above template */ -deployTargets.each { - hifiJob(it, true) +static Closure cmakeBuild(srcDir, instCommand) { + return { project -> + project / 'builders' / 'hudson.plugins.cmake.CmakeBuilder' { + sourceDir srcDir + buildDir 'build' + installDir '' + buildType 'RelWithDebInfo' + generator 'Unix Makefiles' + makeCommand 'make' + installCommand instCommand + preloadScript '' + cmakeArgs '' + projectCmakePath '/usr/bin/cmake' + cleanBuild 'false' + cleanInstallDir 'false' + builderImpl '' + } + } } -/* setup the interface job, doesn't deploy */ -hifiJob('interface', false) +def targets = [ + 'animation-server':true, + 'audio-mixer':true, + 'avatar-mixer':true, + 'domain-server':true, + 'eve':true, + 'pairing-server':true, + 'space-server':true, + 'voxel-server':true, + 'interface':false, +] -/* setup the parametrized-build job for builds from jenkins */ +/* setup all of the target jobs to use the above template */ +for (target in targets) { + queue hifiJob(target.key, target.value) +} + +/* setup the parametrized build job for builds from jenkins */ parameterizedJob = hifiJob('$TARGET', true) parameterizedJob.with { name 'hifi-branch-deploy' diff --git a/libraries/shared/src/OctalCode.cpp b/libraries/shared/src/OctalCode.cpp index d51f058459..e9eaa1c49a 100644 --- a/libraries/shared/src/OctalCode.cpp +++ b/libraries/shared/src/OctalCode.cpp @@ -3,7 +3,7 @@ // hifi // // Created by Stephen Birarda on 3/15/13. -// +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // #include diff --git a/libraries/shared/src/OctalCode.h b/libraries/shared/src/OctalCode.h index 761eac1953..228d5f72b5 100644 --- a/libraries/shared/src/OctalCode.h +++ b/libraries/shared/src/OctalCode.h @@ -3,7 +3,7 @@ // hifi // // Created by Stephen Birarda on 3/15/13. -// +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // #ifndef __hifi__OctalCode__ diff --git a/libraries/shared/src/SimpleMovingAverage.cpp b/libraries/shared/src/SimpleMovingAverage.cpp index db433c020f..df41433618 100644 --- a/libraries/shared/src/SimpleMovingAverage.cpp +++ b/libraries/shared/src/SimpleMovingAverage.cpp @@ -3,7 +3,7 @@ // hifi // // Created by Stephen Birarda on 4/18/13. -// +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // #include "SharedUtil.h" diff --git a/libraries/shared/src/SimpleMovingAverage.h b/libraries/shared/src/SimpleMovingAverage.h index b1d0709342..7c8605db5d 100644 --- a/libraries/shared/src/SimpleMovingAverage.h +++ b/libraries/shared/src/SimpleMovingAverage.h @@ -3,8 +3,9 @@ // hifi // // Created by Stephen Birarda on 4/18/13. -// Replaces Brad Hefta-Gaub's CounterStats class (RIP) +// Copyright (c) 2013 HighFidelity, Inc. All rights reserved. // +// Replaces Brad Hefta-Gaub's CounterStats class (RIP) // #ifndef __hifi__Stats__ diff --git a/libraries/voxels/src/AABox.cpp b/libraries/voxels/src/AABox.cpp index d4b7e1383c..36e65561c3 100644 --- a/libraries/voxels/src/AABox.cpp +++ b/libraries/voxels/src/AABox.cpp @@ -105,6 +105,17 @@ bool AABox::contains(const glm::vec3& point) const { isWithin(point.z, _corner.z, _size.z); } +bool AABox::contains(const AABox& otherBox) const { + for (int v = BOTTOM_LEFT_NEAR; v < TOP_LEFT_FAR; v++) { + glm::vec3 vertex = otherBox.getVertex((BoxVertex)v); + if (!contains(vertex)) { + return false; + } + } + return true; +} + + // determines whether a value is within the expanded extents static bool isWithinExpanded(float value, float corner, float size, float expansion) { return value >= corner - expansion && value <= corner + size + expansion; diff --git a/libraries/voxels/src/AABox.h b/libraries/voxels/src/AABox.h index d295f24aea..735799cd05 100644 --- a/libraries/voxels/src/AABox.h +++ b/libraries/voxels/src/AABox.h @@ -63,6 +63,7 @@ public: glm::vec3 getVertex(BoxVertex vertex) const; bool contains(const glm::vec3& point) const; + bool contains(const AABox& otherBox) const; bool expandedContains(const glm::vec3& point, float expansion) const; bool expandedIntersectsSegment(const glm::vec3& start, const glm::vec3& end, float expansion) const; bool findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, float& distance, BoxFace& face) const; diff --git a/libraries/voxels/src/CoverageMap.cpp b/libraries/voxels/src/CoverageMap.cpp index 76a04b2468..eae1262a59 100644 --- a/libraries/voxels/src/CoverageMap.cpp +++ b/libraries/voxels/src/CoverageMap.cpp @@ -3,6 +3,7 @@ // hifi // // Added by Brad Hefta-Gaub on 06/11/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // #include "CoverageMap.h" @@ -11,10 +12,49 @@ #include "Log.h" int CoverageMap::_mapCount = 0; +int CoverageMap::_checkMapRootCalls = 0; +bool CoverageMap::wantDebugging = false; + const BoundingBox CoverageMap::ROOT_BOUNDING_BOX = BoundingBox(glm::vec2(-2.f,-2.f), glm::vec2(4.f,4.f)); +// Coverage Map's polygon coordinates are from -1 to 1 in the following mapping to screen space. +// +// (0,0) (windowWidth, 0) +// -1,1 1,1 +// +-----------------------+ +// | | | +// | | | +// | -1,0 | | +// |-----------+-----------| +// | 0,0 | +// | | | +// | | | +// | | | +// +-----------------------+ +// -1,-1 1,-1 +// (0,windowHeight) (windowWidth,windowHeight) +// + +// Choosing a minimum sized polygon. Since we know a typical window is approximately 1500 pixels wide +// then a pixel on our screen will be ~ 2.0/1500 or 0.0013 "units" wide, similarly pixels are typically +// about that tall as well. If we say that polygons should be at least 10x10 pixels to be considered "big enough" +// then we can calculate a reasonable polygon area +const int TYPICAL_SCREEN_WIDTH_IN_PIXELS = 1500; +const int MINIMUM_POLYGON_AREA_SIDE_IN_PIXELS = 10; +const float TYPICAL_SCREEN_PIXEL_WIDTH = (2.0f / TYPICAL_SCREEN_WIDTH_IN_PIXELS); +const float CoverageMap::MINIMUM_POLYGON_AREA_TO_STORE = (TYPICAL_SCREEN_PIXEL_WIDTH * MINIMUM_POLYGON_AREA_SIDE_IN_PIXELS) * + (TYPICAL_SCREEN_PIXEL_WIDTH * MINIMUM_POLYGON_AREA_SIDE_IN_PIXELS); + CoverageMap::CoverageMap(BoundingBox boundingBox, bool isRoot, bool managePolygons) : - _isRoot(isRoot), _myBoundingBox(boundingBox), _managePolygons(managePolygons) { + _isRoot(isRoot), + _myBoundingBox(boundingBox), + _managePolygons(managePolygons), + _topHalf (boundingBox.topHalf() , false, managePolygons, TOP_HALF ), + _bottomHalf (boundingBox.bottomHalf(), false, managePolygons, BOTTOM_HALF ), + _leftHalf (boundingBox.leftHalf() , false, managePolygons, LEFT_HALF ), + _rightHalf (boundingBox.rightHalf() , false, managePolygons, RIGHT_HALF ), + _remainder (boundingBox, isRoot, managePolygons, REMAINDER ) +{ _mapCount++; init(); //printLog("CoverageMap created... _mapCount=%d\n",_mapCount); @@ -25,25 +65,13 @@ CoverageMap::~CoverageMap() { }; void CoverageMap::erase() { - // If we're in charge of managing the polygons, then clean them up first - if (_managePolygons) { - for (int i = 0; i < _polygonCount; i++) { - delete _polygons[i]; - _polygons[i] = NULL; // do we need to do this? - } - } - - // Now, clean up our local storage - _polygonCount = 0; - _polygonArraySize = 0; - if (_polygons) { - delete[] _polygons; - _polygons = NULL; - } - if (_polygonDistances) { - delete[] _polygonDistances; - _polygonDistances = NULL; - } + // tell our regions to erase() + _topHalf.erase(); + _bottomHalf.erase(); + _leftHalf.erase(); + _rightHalf.erase(); + _remainder.erase(); + for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { if (_childMaps[i]) { delete _childMaps[i]; @@ -51,26 +79,25 @@ void CoverageMap::erase() { } } -/** - if (_isRoot) { + if (_isRoot && wantDebugging) { printLog("CoverageMap last to be deleted...\n"); + printLog("MINIMUM_POLYGON_AREA_TO_STORE=%f\n",MINIMUM_POLYGON_AREA_TO_STORE); printLog("_mapCount=%d\n",_mapCount); - printLog("_maxPolygonsUsed=%d\n",_maxPolygonsUsed); - printLog("_totalPolygons=%d\n",_totalPolygons); - - _maxPolygonsUsed = 0; - _totalPolygons = 0; + printLog("_checkMapRootCalls=%d\n",_checkMapRootCalls); + printLog("_maxPolygonsUsed=%d\n",CoverageRegion::_maxPolygonsUsed); + printLog("_totalPolygons=%d\n",CoverageRegion::_totalPolygons); + printLog("_occlusionTests=%d\n",CoverageRegion::_occlusionTests); + printLog("_outOfOrderPolygon=%d\n",CoverageRegion::_outOfOrderPolygon); + CoverageRegion::_maxPolygonsUsed = 0; + CoverageRegion::_totalPolygons = 0; + CoverageRegion::_occlusionTests = 0; + CoverageRegion::_outOfOrderPolygon = 0; _mapCount = 0; + _checkMapRootCalls = 0; } -**/ - } void CoverageMap::init() { - _polygonCount = 0; - _polygonArraySize = 0; - _polygons = NULL; - _polygonDistances = NULL; memset(_childMaps,0,sizeof(_childMaps)); } @@ -94,79 +121,59 @@ BoundingBox CoverageMap::getChildBoundingBox(int childIndex) { return result; } - -void CoverageMap::growPolygonArray() { - VoxelProjectedPolygon** newPolygons = new VoxelProjectedPolygon*[_polygonArraySize + DEFAULT_GROW_SIZE]; - float* newDistances = new float[_polygonArraySize + DEFAULT_GROW_SIZE]; - - - if (_polygons) { - memcpy(newPolygons, _polygons, sizeof(VoxelProjectedPolygon*) * _polygonCount); - delete[] _polygons; - memcpy(newDistances, _polygonDistances, sizeof(float) * _polygonCount); - delete[] _polygonDistances; - } - _polygons = newPolygons; - _polygonDistances = newDistances; - _polygonArraySize = _polygonArraySize + DEFAULT_GROW_SIZE; - //printLog("CoverageMap::growPolygonArray() _polygonArraySize=%d...\n",_polygonArraySize); -} - -int CoverageMap::_maxPolygonsUsed = 0; -int CoverageMap::_totalPolygons = 0; - -// just handles storage in the array, doesn't test for occlusion or -// determining if this is the correct map to store in! -void CoverageMap::storeInArray(VoxelProjectedPolygon* polygon) { - - _totalPolygons++; - - if (_polygonArraySize < _polygonCount + 1) { - growPolygonArray(); - } - - // This old code assumes that polygons will always be added in z-buffer order, but that doesn't seem to - // be a good assumption. So instead, we will need to sort this by distance. Use a binary search to find the - // insertion point in this array, and shift the array accordingly - const int IGNORED = NULL; - _polygonCount = insertIntoSortedArrays((void*)polygon, polygon->getDistance(), IGNORED, - (void**)_polygons, _polygonDistances, IGNORED, - _polygonCount, _polygonArraySize); - - if (_polygonCount > _maxPolygonsUsed) { - _maxPolygonsUsed = _polygonCount; - //printLog("CoverageMap new _maxPolygonsUsed reached=%d\n",_maxPolygonsUsed); - //_myBoundingBox.printDebugDetails("map._myBoundingBox"); - } -} - - // possible results = STORED/NOT_STORED, OCCLUDED, DOESNT_FIT -CoverageMap::StorageResult CoverageMap::checkMap(VoxelProjectedPolygon* polygon, bool storeIt) { - if (_isRoot || _myBoundingBox.contains(polygon->getBoundingBox())) { - // check to make sure this polygon isn't occluded by something at this level - for (int i = 0; i < _polygonCount; i++) { - VoxelProjectedPolygon* polygonAtThisLevel = _polygons[i]; - // Check to make sure that the polygon in question is "behind" the polygon in the list - // otherwise, we don't need to test it's occlusion (although, it means we've potentially - // added an item previously that may be occluded??? Is that possible? Maybe not, because two - // voxels can't have the exact same outline. So one occludes the other, they can't both occlude - // each other. - if (polygonAtThisLevel->occludes(*polygon)) { - // if the polygonAtThisLevel is actually behind the one we're inserting, then we don't - // want to report our inserted one as occluded, but we do want to add our inserted one. - if (polygonAtThisLevel->getDistance() >= polygon->getDistance()) { - if (storeIt) { - storeInArray(polygon); - return STORED; - } else { - return NOT_STORED; - } - } - // this polygon is occluded by a closer polygon, so don't store it, and let the caller know - return OCCLUDED; - } +CoverageMapStorageResult CoverageMap::checkMap(VoxelProjectedPolygon* polygon, bool storeIt) { + + if (_isRoot) { + _checkMapRootCalls++; + } + + // short circuit: we don't handle polygons that aren't all in view, so, if the polygon in question is + // not in view, then we just discard it with a DOESNT_FIT, this saves us time checking values later. + if (!polygon->getAllInView()) { + return DOESNT_FIT; + } + + BoundingBox polygonBox(polygon->getBoundingBox()); + if (_isRoot || _myBoundingBox.contains(polygonBox)) { + + CoverageMapStorageResult result = NOT_STORED; + CoverageRegion* storeIn = &_remainder; + bool fitsInAHalf = false; + + // Check each half of the box independently + if (_topHalf.contains(polygonBox)) { + result = _topHalf.checkRegion(polygon, polygonBox, storeIt); + storeIn = &_topHalf; + fitsInAHalf = true; + } else if (_bottomHalf.contains(polygonBox)) { + result = _bottomHalf.checkRegion(polygon, polygonBox, storeIt); + storeIn = &_bottomHalf; + fitsInAHalf = true; + } else if (_leftHalf.contains(polygonBox)) { + result = _leftHalf.checkRegion(polygon, polygonBox, storeIt); + storeIn = &_leftHalf; + fitsInAHalf = true; + } else if (_rightHalf.contains(polygonBox)) { + result = _rightHalf.checkRegion(polygon, polygonBox, storeIt); + storeIn = &_rightHalf; + fitsInAHalf = true; } + + // if we got this far, there are one of two possibilities, either a polygon doesn't fit + // in one of the halves, or it did fit, but it wasn't occluded by anything only in that + // half. In either of these cases, we want to check our remainder region to see if its + // occluded by anything there + if (!(result == STORED || result == OCCLUDED)) { + result = _remainder.checkRegion(polygon, polygonBox, storeIt); + } + + // It's possible that this first set of checks might have resulted in an out of order polygon + // in which case we just return.. + if (result == STORED || result == OCCLUDED) { + return result; + } + // if we made it here, then it means the polygon being stored is not occluded // at this level of the quad tree, so we can continue to insert it into the map. // First we check to see if it fits in any of our sub maps @@ -183,11 +190,184 @@ CoverageMap::StorageResult CoverageMap::checkMap(VoxelProjectedPolygon* polygon, // if we got this far, then the polygon is in our bounding box, but doesn't fit in // any of our child bounding boxes, so we should add it here. if (storeIt) { - storeInArray(polygon); - return STORED; + if (polygon->getBoundingBox().area() > CoverageMap::MINIMUM_POLYGON_AREA_TO_STORE) { + //printLog("storing polygon of area: %f\n",polygon->getBoundingBox().area()); + storeIn->storeInArray(polygon); + return STORED; + } else { + return NOT_STORED; + } } else { return NOT_STORED; } } return DOESNT_FIT; } + + +CoverageRegion::CoverageRegion(BoundingBox boundingBox, bool isRoot, bool managePolygons, RegionName regionName) : + _isRoot(isRoot), + _myBoundingBox(boundingBox), + _managePolygons(managePolygons), + _regionName(regionName) +{ + init(); +}; + +CoverageRegion::~CoverageRegion() { + erase(); +}; + +void CoverageRegion::init() { + _polygonCount = 0; + _polygonArraySize = 0; + _polygons = NULL; + _polygonDistances = NULL; +} + + +void CoverageRegion::erase() { + +/* + if (_polygonCount) { + printLog("CoverageRegion::erase()...\n"); + printLog("_polygonCount=%d\n",_polygonCount); + _myBoundingBox.printDebugDetails(getRegionName()); + //for (int i = 0; i < _polygonCount; i++) { + // printLog("_polygons[%d]=",i); + // _polygons[i]->getBoundingBox().printDebugDetails(); + //} + } +*/ + // If we're in charge of managing the polygons, then clean them up first + if (_managePolygons) { + for (int i = 0; i < _polygonCount; i++) { + delete _polygons[i]; + _polygons[i] = NULL; // do we need to do this? + } + } + + // Now, clean up our local storage + _polygonCount = 0; + _polygonArraySize = 0; + if (_polygons) { + delete[] _polygons; + _polygons = NULL; + } + if (_polygonDistances) { + delete[] _polygonDistances; + _polygonDistances = NULL; + } +} + +void CoverageRegion::growPolygonArray() { + VoxelProjectedPolygon** newPolygons = new VoxelProjectedPolygon*[_polygonArraySize + DEFAULT_GROW_SIZE]; + float* newDistances = new float[_polygonArraySize + DEFAULT_GROW_SIZE]; + + + if (_polygons) { + memcpy(newPolygons, _polygons, sizeof(VoxelProjectedPolygon*) * _polygonCount); + delete[] _polygons; + memcpy(newDistances, _polygonDistances, sizeof(float) * _polygonCount); + delete[] _polygonDistances; + } + _polygons = newPolygons; + _polygonDistances = newDistances; + _polygonArraySize = _polygonArraySize + DEFAULT_GROW_SIZE; + //printLog("CoverageMap::growPolygonArray() _polygonArraySize=%d...\n",_polygonArraySize); +} + +const char* CoverageRegion::getRegionName() const { + switch (_regionName) { + case TOP_HALF: + return "TOP_HALF"; + case BOTTOM_HALF: + return "BOTTOM_HALF"; + case LEFT_HALF: + return "LEFT_HALF"; + case RIGHT_HALF: + return "RIGHT_HALF"; + default: + case REMAINDER: + return "REMAINDER"; + } + return "REMAINDER"; +} + +int CoverageRegion::_maxPolygonsUsed = 0; +int CoverageRegion::_totalPolygons = 0; +int CoverageRegion::_occlusionTests = 0; +int CoverageRegion::_outOfOrderPolygon = 0; + +// just handles storage in the array, doesn't test for occlusion or +// determining if this is the correct map to store in! +void CoverageRegion::storeInArray(VoxelProjectedPolygon* polygon) { + + _totalPolygons++; + + if (_polygonArraySize < _polygonCount + 1) { + growPolygonArray(); + } + + // This old code assumes that polygons will always be added in z-buffer order, but that doesn't seem to + // be a good assumption. So instead, we will need to sort this by distance. Use a binary search to find the + // insertion point in this array, and shift the array accordingly + const int IGNORED = NULL; + _polygonCount = insertIntoSortedArrays((void*)polygon, polygon->getDistance(), IGNORED, + (void**)_polygons, _polygonDistances, IGNORED, + _polygonCount, _polygonArraySize); + + // Debugging and Optimization Tuning code. + if (_polygonCount > _maxPolygonsUsed) { + _maxPolygonsUsed = _polygonCount; + //printLog("CoverageRegion new _maxPolygonsUsed reached=%d region=%s\n",_maxPolygonsUsed, getRegionName()); + //_myBoundingBox.printDebugDetails("map._myBoundingBox"); + } else { + //printLog("CoverageRegion::storeInArray() _polygonCount=%d region=%s\n",_polygonCount, getRegionName()); + } +} + + + +CoverageMapStorageResult CoverageRegion::checkRegion(VoxelProjectedPolygon* polygon, const BoundingBox& polygonBox, bool storeIt) { + + CoverageMapStorageResult result = DOESNT_FIT; + + if (_isRoot || _myBoundingBox.contains(polygonBox)) { + result = NOT_STORED; // if we got here, then we DO fit... + + // check to make sure this polygon isn't occluded by something at this level + for (int i = 0; i < _polygonCount; i++) { + VoxelProjectedPolygon* polygonAtThisLevel = _polygons[i]; + // Check to make sure that the polygon in question is "behind" the polygon in the list + // otherwise, we don't need to test it's occlusion (although, it means we've potentially + // added an item previously that may be occluded??? Is that possible? Maybe not, because two + // voxels can't have the exact same outline. So one occludes the other, they can't both occlude + // each other. + + + _occlusionTests++; + if (polygonAtThisLevel->occludes(*polygon)) { + // if the polygonAtThisLevel is actually behind the one we're inserting, then we don't + // want to report our inserted one as occluded, but we do want to add our inserted one. + if (polygonAtThisLevel->getDistance() >= polygon->getDistance()) { + _outOfOrderPolygon++; + if (storeIt) { + if (polygon->getBoundingBox().area() > CoverageMap::MINIMUM_POLYGON_AREA_TO_STORE) { + //printLog("storing polygon of area: %f\n",polygon->getBoundingBox().area()); + storeInArray(polygon); + return STORED; + } else { + return NOT_STORED; + } + } else { + return NOT_STORED; + } + } + // this polygon is occluded by a closer polygon, so don't store it, and let the caller know + return OCCLUDED; + } + } + } + return result; +} diff --git a/libraries/voxels/src/CoverageMap.h b/libraries/voxels/src/CoverageMap.h index 38c7dc6145..8b01eabeee 100644 --- a/libraries/voxels/src/CoverageMap.h +++ b/libraries/voxels/src/CoverageMap.h @@ -3,6 +3,7 @@ // hifi // // Added by Brad Hefta-Gaub on 06/11/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. // #ifndef _COVERAGE_MAP_ @@ -11,6 +12,45 @@ #include #include "VoxelProjectedPolygon.h" +typedef enum {STORED, OCCLUDED, DOESNT_FIT, NOT_STORED} CoverageMapStorageResult; +typedef enum {TOP_HALF, BOTTOM_HALF, LEFT_HALF, RIGHT_HALF, REMAINDER} RegionName; + +class CoverageRegion { + +public: + + CoverageRegion(BoundingBox boundingBox, bool isRoot, bool managePolygons = true, RegionName regionName = REMAINDER); + ~CoverageRegion(); + + CoverageMapStorageResult checkRegion(VoxelProjectedPolygon* polygon, const BoundingBox& polygonBox, bool storeIt); + void storeInArray(VoxelProjectedPolygon* polygon); + + bool contains(const BoundingBox& box) const { return _myBoundingBox.contains(box); }; + void erase(); // erase the coverage region + + static int _maxPolygonsUsed; + static int _totalPolygons; + static int _occlusionTests; + static int _outOfOrderPolygon; + + + const char* getRegionName() const; + +private: + void init(); + + bool _isRoot; // is this map the root, if so, it never returns DOESNT_FIT + BoundingBox _myBoundingBox; + bool _managePolygons; // will the coverage map delete the polygons on destruct + RegionName _regionName; + int _polygonCount; // how many polygons at this level + int _polygonArraySize; // how much room is there to store polygons at this level + VoxelProjectedPolygon** _polygons; + float* _polygonDistances; + void growPolygonArray(); + static const int DEFAULT_GROW_SIZE = 100; +}; + class CoverageMap { public: @@ -18,36 +58,37 @@ public: static const bool NOT_ROOT=false; static const bool IS_ROOT=true; static const BoundingBox ROOT_BOUNDING_BOX; + static const float MINIMUM_POLYGON_AREA_TO_STORE; CoverageMap(BoundingBox boundingBox = ROOT_BOUNDING_BOX, bool isRoot = IS_ROOT, bool managePolygons = true); ~CoverageMap(); - typedef enum {STORED, OCCLUDED, DOESNT_FIT, NOT_STORED} StorageResult; - StorageResult checkMap(VoxelProjectedPolygon* polygon, bool storeIt = true); + CoverageMapStorageResult checkMap(VoxelProjectedPolygon* polygon, bool storeIt = true); BoundingBox getChildBoundingBox(int childIndex); void erase(); // erase the coverage map + static bool wantDebugging; + private: void init(); - void growPolygonArray(); - void storeInArray(VoxelProjectedPolygon* polygon); bool _isRoot; // is this map the root, if so, it never returns DOESNT_FIT BoundingBox _myBoundingBox; - bool _managePolygons; // will the coverage map delete the polygons on destruct - int _polygonCount; // how many polygons at this level - int _polygonArraySize; // how much room is there to store polygons at this level - VoxelProjectedPolygon** _polygons; - float* _polygonDistances; CoverageMap* _childMaps[NUMBER_OF_CHILDREN]; + bool _managePolygons; // will the coverage map delete the polygons on destruct + + // We divide the map into 5 regions representing each possible half of the map, and the whole map + // this allows us to keep the list of polygons shorter + CoverageRegion _topHalf; + CoverageRegion _bottomHalf; + CoverageRegion _leftHalf; + CoverageRegion _rightHalf; + CoverageRegion _remainder; - static const int DEFAULT_GROW_SIZE = 100; - static int _mapCount; - static int _maxPolygonsUsed; - static int _totalPolygons; - + static int _mapCount; + static int _checkMapRootCalls; }; diff --git a/libraries/voxels/src/ViewFrustum.cpp b/libraries/voxels/src/ViewFrustum.cpp index 786e5a1af0..4ded457d44 100644 --- a/libraries/voxels/src/ViewFrustum.cpp +++ b/libraries/voxels/src/ViewFrustum.cpp @@ -115,6 +115,18 @@ void ViewFrustum::calculate() { _planes[NEAR_PLANE ].set3Points(_nearBottomRight,_nearBottomLeft,_nearTopLeft); _planes[FAR_PLANE ].set3Points(_farBottomLeft,_farBottomRight,_farTopRight); + // Also calculate our projection matrix in case people want to project points... + // Projection matrix : Field of View, ratio, display range : near to far + glm::mat4 projection = glm::perspective(_fieldOfView, _aspectRatio, _nearClip, _farClip); + glm::vec3 lookAt = _position + _direction; + glm::mat4 view = glm::lookAt(_position, lookAt, _up); + + // Our ModelViewProjection : multiplication of our 3 matrices (note: model is identity, so we can drop it) + _ourModelViewProjectionMatrix = projection * view; // Remember, matrix multiplication is the other way around + + // Set up our keyhole bounding box... + glm::vec3 corner = _position - _keyholeRadius; + _keyholeBoundingBox = AABox(corner,(_keyholeRadius * 2.0f)); } //enum { TOP_PLANE = 0, BOTTOM_PLANE, LEFT_PLANE, RIGHT_PLANE, NEAR_PLANE, FAR_PLANE }; @@ -130,14 +142,14 @@ const char* ViewFrustum::debugPlaneName (int plane) const { return "Unknown"; } -ViewFrustum::location ViewFrustum::pointInSphere(const glm::vec3& point, const glm::vec3& center, float radius ) const { +ViewFrustum::location ViewFrustum::pointInKeyhole(const glm::vec3& point) const { ViewFrustum::location result = INTERSECT; - float distance = glm::distance(point, center); - if (distance > radius) { + float distance = glm::distance(point, _position); + if (distance > _keyholeRadius) { result = OUTSIDE; - } else if (distance < radius) { + } else if (distance < _keyholeRadius) { result = INSIDE; } @@ -147,15 +159,13 @@ ViewFrustum::location ViewFrustum::pointInSphere(const glm::vec3& point, const g // To determine if two spheres intersect, simply calculate the distance between the centers of the two spheres. // If the distance is greater than the sum of the two sphere radii, they don’t intersect. Otherwise they intersect. // If the distance plus the radius of sphere A is less than the radius of sphere B then, sphere A is inside of sphere B -ViewFrustum::location ViewFrustum::sphereInSphere(const glm::vec3& centerA, float radiusA, - const glm::vec3& centerB, float radiusB ) const { - +ViewFrustum::location ViewFrustum::sphereInKeyhole(const glm::vec3& center, float radius) const { ViewFrustum::location result = INTERSECT; - float distanceFromAtoB = glm::distance(centerA, centerB); - if (distanceFromAtoB > (radiusA + radiusB)) { + float distance = glm::distance(center, _position); + if (distance > (radius + _keyholeRadius)) { result = OUTSIDE; - } else if ((distanceFromAtoB + radiusA) < radiusB) { + } else if ((distance + radius) < _keyholeRadius) { result = INSIDE; } @@ -166,9 +176,16 @@ ViewFrustum::location ViewFrustum::sphereInSphere(const glm::vec3& centerA, floa // A box is inside a sphere if all of its corners are inside the sphere // A box intersects a sphere if any of its edges (as rays) interesect the sphere // A box is outside a sphere if none of its edges (as rays) interesect the sphere -ViewFrustum::location ViewFrustum::boxInSphere(const AABox& box, const glm::vec3& center, float radius) const { +ViewFrustum::location ViewFrustum::boxInKeyhole(const AABox& box) const { + + // First check to see if the box is in the bounding box for the sphere, if it's not, then we can short circuit + // this and not check with sphere penetration which is more expensive + if (!_keyholeBoundingBox.contains(box)) { + return OUTSIDE; + } + glm::vec3 penetration; - bool intersects = box.findSpherePenetration(center, radius, penetration); + bool intersects = box.findSpherePenetration(_position, _keyholeRadius, penetration); ViewFrustum::location result = OUTSIDE; @@ -177,32 +194,18 @@ ViewFrustum::location ViewFrustum::boxInSphere(const AABox& box, const glm::vec3 result = INTERSECT; // test all the corners, if they are all inside the sphere, the entire box is in the sphere - glm::vec3 testPoint = box.getCorner(); - glm::vec3 size = box.getSize(); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(size.x, 0.0f, 0.0f); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(0.0f, 0.0f, size.z); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(size.x, 0.0f, size.z); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(0.0f, size.y, 0.0f); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(size.x, size.y, 0.0f); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(0.0f, size.y, size.z); - if (pointInSphere(testPoint, center, radius)) { - testPoint = box.getCorner() + glm::vec3(size.x, size.y, size.z); - if (pointInSphere(testPoint, center, radius)) { - result = INSIDE; - } - } - } - } - } - } + bool allPointsInside = true; // assume the best + for (int v = BOTTOM_LEFT_NEAR; v < TOP_LEFT_FAR; v++) { + glm::vec3 vertex = box.getVertex((BoxVertex)v); + if (!pointInKeyhole(vertex)) { + allPointsInside = false; + break; } } + + if (allPointsInside) { + result = INSIDE; + } } return result; @@ -214,7 +217,7 @@ ViewFrustum::location ViewFrustum::pointInFrustum(const glm::vec3& point) const // If we have a keyholeRadius, check that first, since it's cheaper if (_keyholeRadius >= 0.0f) { - keyholeResult = pointInSphere(point, _position, _keyholeRadius); + keyholeResult = pointInKeyhole(point); } if (keyholeResult == INSIDE) { return keyholeResult; @@ -237,7 +240,7 @@ ViewFrustum::location ViewFrustum::sphereInFrustum(const glm::vec3& center, floa // If we have a keyholeRadius, check that first, since it's cheaper if (_keyholeRadius >= 0.0f) { - keyholeResult = sphereInSphere(center, radius, _position, _keyholeRadius); + keyholeResult = sphereInKeyhole(center, radius); } if (keyholeResult == INSIDE) { return keyholeResult; @@ -264,7 +267,7 @@ ViewFrustum::location ViewFrustum::boxInFrustum(const AABox& box) const { // If we have a keyholeRadius, check that first, since it's cheaper if (_keyholeRadius >= 0.0f) { - keyholeResult = boxInSphere(box, _position, _keyholeRadius); + keyholeResult = boxInKeyhole(box); } if (keyholeResult == INSIDE) { return keyholeResult; @@ -425,18 +428,10 @@ void ViewFrustum::printDebugDetails() const { _eyeOffsetOrientation.w ); } - glm::vec2 ViewFrustum::projectPoint(glm::vec3 point, bool& pointInView) const { - // Projection matrix : Field of View, ratio, display range : near to far - glm::mat4 projection = glm::perspective(_fieldOfView, _aspectRatio, _nearClip, _farClip); - glm::vec3 lookAt = _position + _direction; - glm::mat4 view = glm::lookAt(_position, lookAt, _up); - // Our ModelViewProjection : multiplication of our 3 matrices (note: model is identity, so we can drop it) - glm::mat4 VP = projection * view; // Remember, matrix multiplication is the other way around - - glm::vec4 pointVec4 = glm::vec4(point,1); - glm::vec4 projectedPointVec4 = VP * pointVec4; + glm::vec4 pointVec4 = glm::vec4(point,1) ; + glm::vec4 projectedPointVec4 = _ourModelViewProjectionMatrix * pointVec4; pointInView = (projectedPointVec4.w > 0); // math! If the w result is negative then the point is behind the viewer // what happens with w is 0??? @@ -503,7 +498,7 @@ const int hullVertexLookup[MAX_POSSIBLE_COMBINATIONS][MAX_SHADOW_VERTEX_COUNT+1] {6, TOP_RIGHT_NEAR, TOP_RIGHT_FAR, BOTTOM_RIGHT_FAR, BOTTOM_LEFT_FAR, BOTTOM_LEFT_NEAR, TOP_LEFT_NEAR}, // back, top, left }; -VoxelProjectedPolygon ViewFrustum::getProjectedShadow(const AABox& box) const { +VoxelProjectedPolygon ViewFrustum::getProjectedPolygon(const AABox& box) const { glm::vec3 bottomNearRight = box.getCorner(); glm::vec3 topFarLeft = box.getCorner() + box.getSize(); int lookUp = ((_position.x < bottomNearRight.x) ) // 1 = right | compute 6-bit diff --git a/libraries/voxels/src/ViewFrustum.h b/libraries/voxels/src/ViewFrustum.h index f2b329ee30..6833eb6134 100644 --- a/libraries/voxels/src/ViewFrustum.h +++ b/libraries/voxels/src/ViewFrustum.h @@ -89,14 +89,14 @@ public: void printDebugDetails() const; glm::vec2 projectPoint(glm::vec3 point, bool& pointInView) const; - VoxelProjectedPolygon getProjectedShadow(const AABox& box) const; + VoxelProjectedPolygon getProjectedPolygon(const AABox& box) const; private: // Used for keyhole calculations - ViewFrustum::location pointInSphere(const glm::vec3& point, const glm::vec3& center, float radius) const; - ViewFrustum::location sphereInSphere(const glm::vec3& centerA, float radiusA, const glm::vec3& centerB, float radiusB) const; - ViewFrustum::location boxInSphere(const AABox& box, const glm::vec3& center, float radius) const; + ViewFrustum::location pointInKeyhole(const glm::vec3& point) const; + ViewFrustum::location sphereInKeyhole(const glm::vec3& center, float radius) const; + ViewFrustum::location boxInKeyhole(const AABox& box) const; // camera location/orientation attributes glm::vec3 _position; @@ -117,6 +117,7 @@ private: // keyhole attributes float _keyholeRadius; + AABox _keyholeBoundingBox; // Calculated values @@ -137,6 +138,8 @@ private: const char* debugPlaneName (int plane) const; + // Used to project points + glm::mat4 _ourModelViewProjectionMatrix; }; diff --git a/libraries/voxels/src/VoxelProjectedPolygon.cpp b/libraries/voxels/src/VoxelProjectedPolygon.cpp index baef436d08..c609b27df4 100644 --- a/libraries/voxels/src/VoxelProjectedPolygon.cpp +++ b/libraries/voxels/src/VoxelProjectedPolygon.cpp @@ -10,6 +10,30 @@ #include "Log.h" +BoundingBox BoundingBox::topHalf() const { + float halfY = size.y/2.0f; + BoundingBox result(glm::vec2(corner.x,corner.y + halfY), glm::vec2(size.x, halfY)); + return result; +} + +BoundingBox BoundingBox::bottomHalf() const { + float halfY = size.y/2.0f; + BoundingBox result(corner, glm::vec2(size.x, halfY)); + return result; +} + +BoundingBox BoundingBox::leftHalf() const { + float halfX = size.x/2.0f; + BoundingBox result(corner, glm::vec2(halfX, size.y)); + return result; +} + +BoundingBox BoundingBox::rightHalf() const { + float halfX = size.x/2.0f; + BoundingBox result(glm::vec2(corner.x + halfX , corner.y), glm::vec2(halfX, size.y)); + return result; +} + bool BoundingBox::contains(const BoundingBox& box) const { return ( (box.corner.x >= corner.x) && @@ -48,7 +72,7 @@ void VoxelProjectedPolygon::setVertex(int vertex, const glm::vec2& point) { }; -bool VoxelProjectedPolygon::occludes(const VoxelProjectedPolygon& occludee) const { +bool VoxelProjectedPolygon::occludes(const VoxelProjectedPolygon& occludee, bool checkAllInView) const { // if we are completely out of view, then we definitely don't occlude! // if the occludee is completely out of view, then we also don't occlude it @@ -56,7 +80,7 @@ bool VoxelProjectedPolygon::occludes(const VoxelProjectedPolygon& occludee) cons // this is true, but unfortunately, we're not quite handling projects in the // case when SOME points are in view and others are not. So, we will not consider // occlusion for any shadows that are partially in view. - if (!getAllInView() || !occludee.getAllInView() ) { + if (checkAllInView && (!getAllInView() || !occludee.getAllInView())) { return false; } diff --git a/libraries/voxels/src/VoxelProjectedPolygon.h b/libraries/voxels/src/VoxelProjectedPolygon.h index c6e4b665fa..8123ca6a85 100644 --- a/libraries/voxels/src/VoxelProjectedPolygon.h +++ b/libraries/voxels/src/VoxelProjectedPolygon.h @@ -20,6 +20,12 @@ public: glm::vec2 corner; glm::vec2 size; bool contains(const BoundingBox& box) const; + float area() const { return size.x * size.y; }; + + BoundingBox topHalf() const; + BoundingBox bottomHalf() const; + BoundingBox leftHalf() const; + BoundingBox rightHalf() const; void printDebugDetails(const char* label=NULL) const; }; @@ -48,7 +54,7 @@ public: bool getAllInView() const { return _allInView; }; void setAllInView(bool allInView) { _allInView = allInView; }; - bool occludes(const VoxelProjectedPolygon& occludee) const; + bool occludes(const VoxelProjectedPolygon& occludee, bool checkAllInView = false) const; bool pointInside(const glm::vec2& point) const; float getMaxX() const { return _maxX; } diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index ec9ae008e6..e2f66fa52b 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -1125,16 +1125,16 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp //node->printDebugDetails("upper section, params.wantOcclusionCulling... node="); AABox voxelBox = node->getAABox(); voxelBox.scale(TREE_SCALE); - VoxelProjectedPolygon* voxelShadow = new VoxelProjectedPolygon(params.viewFrustum->getProjectedShadow(voxelBox)); + VoxelProjectedPolygon* voxelPolygon = new VoxelProjectedPolygon(params.viewFrustum->getProjectedPolygon(voxelBox)); // In order to check occlusion culling, the shadow has to be "all in view" otherwise, we will ignore occlusion // culling and proceed as normal - if (voxelShadow->getAllInView()) { - //node->printDebugDetails("upper section, voxelShadow->getAllInView() node="); + if (voxelPolygon->getAllInView()) { + //node->printDebugDetails("upper section, voxelPolygon->getAllInView() node="); - CoverageMap::StorageResult result = params.map->checkMap(voxelShadow, false); - delete voxelShadow; // cleanup - if (result == CoverageMap::OCCLUDED) { + CoverageMapStorageResult result = params.map->checkMap(voxelPolygon, false); + delete voxelPolygon; // cleanup + if (result == OCCLUDED) { //node->printDebugDetails("upper section, non-Leaf is occluded!! node="); //args->nonLeavesOccluded++; @@ -1147,7 +1147,7 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp //node->printDebugDetails("upper section, shadow Not in view node="); // If this shadow wasn't "all in view" then we ignored it for occlusion culling, but // we do need to clean up memory and proceed as normal... - delete voxelShadow; + delete voxelPolygon; } } } @@ -1241,26 +1241,27 @@ int VoxelTree::encodeTreeBitstreamRecursion(VoxelNode* node, unsigned char* outp AABox voxelBox = childNode->getAABox(); voxelBox.scale(TREE_SCALE); - VoxelProjectedPolygon* voxelShadow = new VoxelProjectedPolygon(params.viewFrustum->getProjectedShadow(voxelBox)); + VoxelProjectedPolygon* voxelPolygon = new VoxelProjectedPolygon( + params.viewFrustum->getProjectedPolygon(voxelBox)); // In order to check occlusion culling, the shadow has to be "all in view" otherwise, we will ignore occlusion // culling and proceed as normal - if (voxelShadow->getAllInView()) { - CoverageMap::StorageResult result = params.map->checkMap(voxelShadow, true); + if (voxelPolygon->getAllInView()) { + CoverageMapStorageResult result = params.map->checkMap(voxelPolygon, true); // In all cases where the shadow wasn't stored, we need to free our own memory. // In the case where it is stored, the CoverageMap will free memory for us later. - if (result != CoverageMap::STORED) { - delete voxelShadow; + if (result != STORED) { + delete voxelPolygon; } // If while attempting to add this voxel's shadow, we determined it was occluded, then // we don't need to process it further and we can exit early. - if (result == CoverageMap::OCCLUDED) { + if (result == OCCLUDED) { childIsOccluded = true; } } else { - delete voxelShadow; + delete voxelPolygon; } } // wants occlusion culling & isLeaf()