diff --git a/avatar-mixer/src/main.cpp b/avatar-mixer/src/main.cpp index bff0b529f8..f8bc36f6e6 100644 --- a/avatar-mixer/src/main.cpp +++ b/avatar-mixer/src/main.cpp @@ -117,6 +117,7 @@ int main(int argc, const char* argv[]) { break; case PACKET_TYPE_AVATAR_VOXEL_URL: + case PACKET_TYPE_AVATAR_FACE_VIDEO: // grab the node ID from the packet unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID); diff --git a/cmake/modules/FindLibVPX.cmake b/cmake/modules/FindLibVPX.cmake new file mode 100644 index 0000000000..1d75dbab27 --- /dev/null +++ b/cmake/modules/FindLibVPX.cmake @@ -0,0 +1,46 @@ +# - Try to find the LibVPX library to perform VP8 video encoding/decoding +# +# You must provide a LIBVPX_ROOT_DIR which contains lib and include directories +# +# Once done this will define +# +# LIBVPX_FOUND - system found LibVPX +# LIBVPX_INCLUDE_DIRS - the LibVPX include directory +# LIBVPX_LIBRARIES - Link this to use LibVPX +# +# Created on 7/17/2013 by Andrzej Kapolka +# Copyright (c) 2013 High Fidelity +# + +if (LIBVPX_LIBRARIES AND LIBVPX_INCLUDE_DIRS) + # in cache already + set(LIBVPX_FOUND TRUE) +else (LIBVPX_LIBRARIES AND LIBVPX_INCLUDE_DIRS) + find_path(LIBVPX_INCLUDE_DIRS vpx_codec.h ${LIBVPX_ROOT_DIR}/include) + + if (APPLE) + find_library(LIBVPX_LIBRARIES libvpx.a ${LIBVPX_ROOT_DIR}/lib/MacOS/) + elseif (UNIX) + find_library(LIBVPX_LIBRARIES libvpx.a ${LIBVPX_ROOT_DIR}/lib/UNIX/) + elseif (WIN32) + find_library(LIBVPX_LIBRARIES libvpx.lib ${LIBVPX_ROOT_DIR}/lib/Win32/) + endif () + + if (LIBVPX_INCLUDE_DIRS AND LIBVPX_LIBRARIES) + set(LIBVPX_FOUND TRUE) + endif (LIBVPX_INCLUDE_DIRS AND LIBVPX_LIBRARIES) + + if (LIBVPX_FOUND) + if (NOT LibVPX_FIND_QUIETLY) + message(STATUS "Found LibVPX: ${LIBVPX_LIBRARIES}") + endif (NOT LibVPX_FIND_QUIETLY) + else (LIBVPX_FOUND) + if (LibVPX_FIND_REQUIRED) + message(FATAL_ERROR "Could not find LibVPX") + endif (LibVPX_FIND_REQUIRED) + endif (LIBVPX_FOUND) + + # show the LIBVPX_INCLUDE_DIRS and LIBVPX_LIBRARIES variables only in the advanced view + mark_as_advanced(LIBVPX_INCLUDE_DIRS LIBVPX_LIBRARIES) + +endif (LIBVPX_LIBRARIES AND LIBVPX_INCLUDE_DIRS) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 829ccbff06..9f8a02a12a 100755 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -9,6 +9,7 @@ project(${TARGET_NAME}) # setup for find modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules/") set(LIBOVR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibOVR) +set(LIBVPX_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/LibVPX) set(LEAP_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/Leap) set(MOTIONDRIVER_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/MotionDriver) set(PORTAUDIO_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/PortAudio) @@ -36,7 +37,7 @@ configure_file(InterfaceConfig.h.in ${PROJECT_BINARY_DIR}/includes/InterfaceConf # grab the implementation and header files from src dirs file(GLOB INTERFACE_SRCS src/*.cpp src/*.h) -foreach(SUBDIR ui renderer) +foreach(SUBDIR avatar ui renderer) file(GLOB SUBDIR_SRCS src/${SUBDIR}/*.cpp src/${SUBDIR}/*.h) set(INTERFACE_SRCS ${INTERFACE_SRCS} ${SUBDIR_SRCS}) endforeach(SUBDIR) @@ -75,7 +76,8 @@ add_subdirectory(external/fervor/) include_directories(external/fervor/) # run qt moc on qt-enabled headers -qt4_wrap_cpp(INTERFACE_SRCS src/Application.h src/AvatarVoxelSystem.h src/Webcam.h src/ui/BandwidthDialog.h) +qt4_wrap_cpp(INTERFACE_SRCS src/Application.h src/Webcam.h src/avatar/AvatarVoxelSystem.h + src/avatar/Face.h src/ui/BandwidthDialog.h) # create the executable, make it a bundle on OS X add_executable(${TARGET_NAME} MACOSX_BUNDLE ${INTERFACE_SRCS}) @@ -92,6 +94,7 @@ link_hifi_library(audio ${TARGET_NAME} ${ROOT_DIR}) # find required libraries find_package(GLM REQUIRED) find_package(LibOVR) +find_package(LibVPX) find_package(Leap) find_package(MotionDriver) find_package(OpenCV) @@ -119,6 +122,7 @@ include_directories( SYSTEM ${GLM_INCLUDE_DIRS} ${LIBOVR_INCLUDE_DIRS} + ${LIBVPX_INCLUDE_DIRS} ${LEAP_INCLUDE_DIRS} ${MOTIONDRIVER_INCLUDE_DIRS} ${OPENCV_INCLUDE_DIRS} @@ -128,6 +132,7 @@ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${OPENCV_INCLUDE_DIRS}") target_link_libraries( ${TARGET_NAME} ${QT_LIBRARIES} + ${LIBVPX_LIBRARIES} ${MOTIONDRIVER_LIBRARIES} ${OPENCV_LIBRARIES} ${ZLIB_LIBRARIES} diff --git a/interface/external/LibVPX/AUTHORS b/interface/external/LibVPX/AUTHORS new file mode 100644 index 0000000000..0937d5d1b3 --- /dev/null +++ b/interface/external/LibVPX/AUTHORS @@ -0,0 +1,65 @@ +# This file is automatically generated from the git commit history +# by tools/gen_authors.sh. + +Aaron Watry +Adrian Grange +Alex Converse +Alexis Ballier +Alok Ahuja +Alpha Lam +Andoni Morales Alastruey +Andres Mejia +Aron Rosenberg +Attila Nagy +Deb Mukherjee +Fabio Pedretti +Frank Galligan +Fredrik Söderquist +Fritz Koenig +Gaute Strokkenes +Giuseppe Scrivano +Guillermo Ballester Valor +Henrik Lundin +James Berry +James Zern +Jan Kratochvil +Jeff Faust +Jeff Muizelaar +Jim Bankoski +Johann Koenig +John Koleszar +Joshua Bleecher Snyder +Justin Clift +Justin Lebar +KO Myung-Hun +Lou Quillio +Luca Barbato +Makoto Kato +Marco Paniconi +Martin Ettl +Michael Kohler +Mike Hommey +Mikhal Shemer +Pascal Massimino +Patrik Westin +Paul Wilkins +Pavol Rusnak +Philip Jägenstedt +Priit Laes +Rafael Ávila de Espíndola +Rafaël Carré +Ralph Giles +Ronald S. Bultje +Scott LaVarnway +Stefan Holmer +Taekhyun Kim +Takanori MATSUURA +Tero Rintaluoma +Thijs Vermeir +Timothy B. Terriberry +Tom Finegan +Yaowu Xu +Yunqing Wang +Google Inc. +The Mozilla Foundation +The Xiph.Org Foundation diff --git a/interface/external/LibVPX/LICENSE b/interface/external/LibVPX/LICENSE new file mode 100644 index 0000000000..1ce44343c4 --- /dev/null +++ b/interface/external/LibVPX/LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2010, The WebM Project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google, nor the WebM Project, nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/interface/external/LibVPX/PATENTS b/interface/external/LibVPX/PATENTS new file mode 100644 index 0000000000..4414d83850 --- /dev/null +++ b/interface/external/LibVPX/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the WebM Project. + +Google hereby grants to you a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer, and otherwise run, modify and propagate the contents of this +implementation of VP8, where such license applies only to those patent +claims, both currently owned by Google and acquired in the future, +licensable by Google that are necessarily infringed by this +implementation of VP8. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of VP8 or any code incorporated within this +implementation of VP8 constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of VP8 +shall terminate as of the date such litigation is filed. diff --git a/interface/external/LibVPX/include/vp8.h b/interface/external/LibVPX/include/vp8.h new file mode 100644 index 0000000000..0b4cb1b9e6 --- /dev/null +++ b/interface/external/LibVPX/include/vp8.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\defgroup vp8 VP8 + * \ingroup codecs + * VP8 is vpx's newest video compression algorithm that uses motion + * compensated prediction, Discrete Cosine Transform (DCT) coding of the + * prediction error signal and context dependent entropy coding techniques + * based on arithmetic principles. It features: + * - YUV 4:2:0 image format + * - Macro-block based coding (16x16 luma plus two 8x8 chroma) + * - 1/4 (1/8) pixel accuracy motion compensated prediction + * - 4x4 DCT transform + * - 128 level linear quantizer + * - In loop deblocking filter + * - Context-based entropy coding + * + * @{ + */ +/*!\file + * \brief Provides controls common to both the VP8 encoder and decoder. + */ +#ifndef VP8_H +#define VP8_H +#include "vpx_codec_impl_top.h" + +/*!\brief Control functions + * + * The set of macros define the control functions of VP8 interface + */ +enum vp8_com_control_id { + VP8_SET_REFERENCE = 1, /**< pass in an external frame into decoder to be used as reference frame */ + VP8_COPY_REFERENCE = 2, /**< get a copy of reference frame from the decoder */ + VP8_SET_POSTPROC = 3, /**< set the decoder's post processing settings */ + VP8_SET_DBG_COLOR_REF_FRAME = 4, /**< set the reference frames to color for each macroblock */ + VP8_SET_DBG_COLOR_MB_MODES = 5, /**< set which macro block modes to color */ + VP8_SET_DBG_COLOR_B_MODES = 6, /**< set which blocks modes to color */ + VP8_SET_DBG_DISPLAY_MV = 7, /**< set which motion vector modes to draw */ + + /* TODO(jkoleszar): The encoder incorrectly reuses some of these values (5+) + * for its control ids. These should be migrated to something like the + * VP8_DECODER_CTRL_ID_START range next time we're ready to break the ABI. + */ + VP9_GET_REFERENCE = 128, /**< get a pointer to a reference frame */ + VP8_COMMON_CTRL_ID_MAX, + VP8_DECODER_CTRL_ID_START = 256 +}; + +/*!\brief post process flags + * + * The set of macros define VP8 decoder post processing flags + */ +enum vp8_postproc_level { + VP8_NOFILTERING = 0, + VP8_DEBLOCK = 1 << 0, + VP8_DEMACROBLOCK = 1 << 1, + VP8_ADDNOISE = 1 << 2, + VP8_DEBUG_TXT_FRAME_INFO = 1 << 3, /**< print frame information */ + VP8_DEBUG_TXT_MBLK_MODES = 1 << 4, /**< print macro block modes over each macro block */ + VP8_DEBUG_TXT_DC_DIFF = 1 << 5, /**< print dc diff for each macro block */ + VP8_DEBUG_TXT_RATE_INFO = 1 << 6, /**< print video rate info (encoder only) */ + VP8_MFQE = 1 << 10 +}; + +/*!\brief post process flags + * + * This define a structure that describe the post processing settings. For + * the best objective measure (using the PSNR metric) set post_proc_flag + * to VP8_DEBLOCK and deblocking_level to 1. + */ + +typedef struct vp8_postproc_cfg { + int post_proc_flag; /**< the types of post processing to be done, should be combination of "vp8_postproc_level" */ + int deblocking_level; /**< the strength of deblocking, valid range [0, 16] */ + int noise_level; /**< the strength of additive noise, valid range [0, 16] */ +} vp8_postproc_cfg_t; + +/*!\brief reference frame type + * + * The set of macros define the type of VP8 reference frames + */ +typedef enum vpx_ref_frame_type { + VP8_LAST_FRAME = 1, + VP8_GOLD_FRAME = 2, + VP8_ALTR_FRAME = 4 +} vpx_ref_frame_type_t; + +/*!\brief reference frame data struct + * + * define the data struct to access vp8 reference frames + */ + +typedef struct vpx_ref_frame { + vpx_ref_frame_type_t frame_type; /**< which reference frame */ + vpx_image_t img; /**< reference frame data in image format */ +} vpx_ref_frame_t; + +typedef struct vp9_ref_frame { + int idx; /**< frame index to get (input) */ + vpx_image_t img; /**< img structure to populate (output) */ +} vp9_ref_frame_t; + +/*!\brief vp8 decoder control function parameter type + * + * defines the data type for each of VP8 decoder control function requires + */ + +VPX_CTRL_USE_TYPE(VP8_SET_REFERENCE, vpx_ref_frame_t *) +VPX_CTRL_USE_TYPE(VP8_COPY_REFERENCE, vpx_ref_frame_t *) +VPX_CTRL_USE_TYPE(VP8_SET_POSTPROC, vp8_postproc_cfg_t *) +VPX_CTRL_USE_TYPE(VP8_SET_DBG_COLOR_REF_FRAME, int) +VPX_CTRL_USE_TYPE(VP8_SET_DBG_COLOR_MB_MODES, int) +VPX_CTRL_USE_TYPE(VP8_SET_DBG_COLOR_B_MODES, int) +VPX_CTRL_USE_TYPE(VP8_SET_DBG_DISPLAY_MV, int) +VPX_CTRL_USE_TYPE(VP9_GET_REFERENCE, vp9_ref_frame_t *) + + +/*! @} - end defgroup vp8 */ + +#include "vpx_codec_impl_bottom.h" +#endif diff --git a/interface/external/LibVPX/include/vp8cx.h b/interface/external/LibVPX/include/vp8cx.h new file mode 100644 index 0000000000..f8e2ef9c3e --- /dev/null +++ b/interface/external/LibVPX/include/vp8cx.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\defgroup vp8_encoder WebM VP8 Encoder + * \ingroup vp8 + * + * @{ + */ +#include "vp8.h" + +/*!\file + * \brief Provides definitions for using the VP8 encoder algorithm within the + * vpx Codec Interface. + */ +#ifndef VP8CX_H +#define VP8CX_H +#include "vpx_codec_impl_top.h" + +/*!\name Algorithm interface for VP8 + * + * This interface provides the capability to encode raw VP8 streams, as would + * be found in AVI files. + * @{ + */ +extern vpx_codec_iface_t vpx_codec_vp8_cx_algo; +extern vpx_codec_iface_t *vpx_codec_vp8_cx(void); + +/* TODO(jkoleszar): These move to VP9 in a later patch set. */ +extern vpx_codec_iface_t vpx_codec_vp9_cx_algo; +extern vpx_codec_iface_t *vpx_codec_vp9_cx(void); +extern vpx_codec_iface_t vpx_codec_vp9x_cx_algo; +extern vpx_codec_iface_t *vpx_codec_vp9x_cx(void); + +/*!@} - end algorithm interface member group*/ + + +/* + * Algorithm Flags + */ + +/*!\brief Don't reference the last frame + * + * When this flag is set, the encoder will not use the last frame as a + * predictor. When not set, the encoder will choose whether to use the + * last frame or not automatically. + */ +#define VP8_EFLAG_NO_REF_LAST (1<<16) + + +/*!\brief Don't reference the golden frame + * + * When this flag is set, the encoder will not use the golden frame as a + * predictor. When not set, the encoder will choose whether to use the + * golden frame or not automatically. + */ +#define VP8_EFLAG_NO_REF_GF (1<<17) + + +/*!\brief Don't reference the alternate reference frame + * + * When this flag is set, the encoder will not use the alt ref frame as a + * predictor. When not set, the encoder will choose whether to use the + * alt ref frame or not automatically. + */ +#define VP8_EFLAG_NO_REF_ARF (1<<21) + + +/*!\brief Don't update the last frame + * + * When this flag is set, the encoder will not update the last frame with + * the contents of the current frame. + */ +#define VP8_EFLAG_NO_UPD_LAST (1<<18) + + +/*!\brief Don't update the golden frame + * + * When this flag is set, the encoder will not update the golden frame with + * the contents of the current frame. + */ +#define VP8_EFLAG_NO_UPD_GF (1<<22) + + +/*!\brief Don't update the alternate reference frame + * + * When this flag is set, the encoder will not update the alt ref frame with + * the contents of the current frame. + */ +#define VP8_EFLAG_NO_UPD_ARF (1<<23) + + +/*!\brief Force golden frame update + * + * When this flag is set, the encoder copy the contents of the current frame + * to the golden frame buffer. + */ +#define VP8_EFLAG_FORCE_GF (1<<19) + + +/*!\brief Force alternate reference frame update + * + * When this flag is set, the encoder copy the contents of the current frame + * to the alternate reference frame buffer. + */ +#define VP8_EFLAG_FORCE_ARF (1<<24) + + +/*!\brief Disable entropy update + * + * When this flag is set, the encoder will not update its internal entropy + * model based on the entropy of this frame. + */ +#define VP8_EFLAG_NO_UPD_ENTROPY (1<<20) + + +/*!\brief VP8 encoder control functions + * + * This set of macros define the control functions available for the VP8 + * encoder interface. + * + * \sa #vpx_codec_control + */ +enum vp8e_enc_control_id { + VP8E_UPD_ENTROPY = 5, /**< control function to set mode of entropy update in encoder */ + VP8E_UPD_REFERENCE, /**< control function to set reference update mode in encoder */ + VP8E_USE_REFERENCE, /**< control function to set which reference frame encoder can use */ + VP8E_SET_ROI_MAP, /**< control function to pass an ROI map to encoder */ + VP8E_SET_ACTIVEMAP, /**< control function to pass an Active map to encoder */ + VP8E_SET_SCALEMODE = 11, /**< control function to set encoder scaling mode */ + /*!\brief control function to set vp8 encoder cpuused + * + * Changes in this value influences, among others, the encoder's selection + * of motion estimation methods. Values greater than 0 will increase encoder + * speed at the expense of quality. + * The full set of adjustments can be found in + * onyx_if.c:vp8_set_speed_features(). + * \todo List highlights of the changes at various levels. + * + * \note Valid range: -16..16 + */ + VP8E_SET_CPUUSED = 13, + VP8E_SET_ENABLEAUTOALTREF, /**< control function to enable vp8 to automatic set and use altref frame */ + VP8E_SET_NOISE_SENSITIVITY, /**< control function to set noise sensitivity */ + VP8E_SET_SHARPNESS, /**< control function to set sharpness */ + VP8E_SET_STATIC_THRESHOLD, /**< control function to set the threshold for macroblocks treated static */ + VP8E_SET_TOKEN_PARTITIONS, /**< control function to set the number of token partitions */ + VP8E_GET_LAST_QUANTIZER, /**< return the quantizer chosen by the + encoder for the last frame using the internal + scale */ + VP8E_GET_LAST_QUANTIZER_64, /**< return the quantizer chosen by the + encoder for the last frame, using the 0..63 + scale as used by the rc_*_quantizer config + parameters */ + VP8E_SET_ARNR_MAXFRAMES, /**< control function to set the max number of frames blurred creating arf*/ + VP8E_SET_ARNR_STRENGTH, /**< control function to set the filter strength for the arf */ + VP8E_SET_ARNR_TYPE, /**< control function to set the type of filter to use for the arf*/ + VP8E_SET_TUNING, /**< control function to set visual tuning */ + /*!\brief control function to set constrained quality level + * + * \attention For this value to be used vpx_codec_enc_cfg_t::g_usage must be + * set to #VPX_CQ. + * \note Valid range: 0..63 + */ + VP8E_SET_CQ_LEVEL, + + /*!\brief Max data rate for Intra frames + * + * This value controls additional clamping on the maximum size of a + * keyframe. It is expressed as a percentage of the average + * per-frame bitrate, with the special (and default) value 0 meaning + * unlimited, or no additional clamping beyond the codec's built-in + * algorithm. + * + * For example, to allocate no more than 4.5 frames worth of bitrate + * to a keyframe, set this to 450. + * + */ + VP8E_SET_MAX_INTRA_BITRATE_PCT, + + + /* TODO(jkoleszar): Move to vp9cx.h */ + VP9E_SET_LOSSLESS, + VP9E_SET_TILE_COLUMNS, + VP9E_SET_TILE_ROWS, + VP9E_SET_FRAME_PARALLEL_DECODING +}; + +/*!\brief vpx 1-D scaling mode + * + * This set of constants define 1-D vpx scaling modes + */ +typedef enum vpx_scaling_mode_1d { + VP8E_NORMAL = 0, + VP8E_FOURFIVE = 1, + VP8E_THREEFIVE = 2, + VP8E_ONETWO = 3 +} VPX_SCALING_MODE; + + +/*!\brief vpx region of interest map + * + * These defines the data structures for the region of interest map + * + */ + +typedef struct vpx_roi_map { + unsigned char *roi_map; /**< specify an id between 0 and 3 for each 16x16 region within a frame */ + unsigned int rows; /**< number of rows */ + unsigned int cols; /**< number of cols */ + // TODO(paulwilkins): broken for VP9 which has 8 segments + // q and loop filter deltas for each segment + // (see MAX_MB_SEGMENTS) + int delta_q[4]; + int delta_lf[4]; + // Static breakout threshold for each segment + unsigned int static_threshold[4]; +} vpx_roi_map_t; + +/*!\brief vpx active region map + * + * These defines the data structures for active region map + * + */ + + +typedef struct vpx_active_map { + unsigned char *active_map; /**< specify an on (1) or off (0) each 16x16 region within a frame */ + unsigned int rows; /**< number of rows */ + unsigned int cols; /**< number of cols */ +} vpx_active_map_t; + +/*!\brief vpx image scaling mode + * + * This defines the data structure for image scaling mode + * + */ +typedef struct vpx_scaling_mode { + VPX_SCALING_MODE h_scaling_mode; /**< horizontal scaling mode */ + VPX_SCALING_MODE v_scaling_mode; /**< vertical scaling mode */ +} vpx_scaling_mode_t; + +/*!\brief VP8 token partition mode + * + * This defines VP8 partitioning mode for compressed data, i.e., the number of + * sub-streams in the bitstream. Used for parallelized decoding. + * + */ + +typedef enum { + VP8_ONE_TOKENPARTITION = 0, + VP8_TWO_TOKENPARTITION = 1, + VP8_FOUR_TOKENPARTITION = 2, + VP8_EIGHT_TOKENPARTITION = 3 +} vp8e_token_partitions; + + +/*!\brief VP8 model tuning parameters + * + * Changes the encoder to tune for certain types of input material. + * + */ +typedef enum { + VP8_TUNE_PSNR, + VP8_TUNE_SSIM +} vp8e_tuning; + + +/*!\brief VP8 encoder control function parameter type + * + * Defines the data types that VP8E control functions take. Note that + * additional common controls are defined in vp8.h + * + */ + + +/* These controls have been deprecated in favor of the flags parameter to + * vpx_codec_encode(). See the definition of VP8_EFLAG_* above. + */ +VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_UPD_ENTROPY, int) +VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_UPD_REFERENCE, int) +VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_USE_REFERENCE, int) + +VPX_CTRL_USE_TYPE(VP8E_SET_ROI_MAP, vpx_roi_map_t *) +VPX_CTRL_USE_TYPE(VP8E_SET_ACTIVEMAP, vpx_active_map_t *) +VPX_CTRL_USE_TYPE(VP8E_SET_SCALEMODE, vpx_scaling_mode_t *) + +VPX_CTRL_USE_TYPE(VP8E_SET_CPUUSED, int) +VPX_CTRL_USE_TYPE(VP8E_SET_ENABLEAUTOALTREF, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_SHARPNESS, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_STATIC_THRESHOLD, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_TOKEN_PARTITIONS, int) /* vp8e_token_partitions */ + +VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_MAXFRAMES, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_STRENGTH, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_TYPE, unsigned int) +VPX_CTRL_USE_TYPE(VP8E_SET_TUNING, int) /* vp8e_tuning */ +VPX_CTRL_USE_TYPE(VP8E_SET_CQ_LEVEL, unsigned int) + +VPX_CTRL_USE_TYPE(VP9E_SET_TILE_COLUMNS, int) +VPX_CTRL_USE_TYPE(VP9E_SET_TILE_ROWS, int) + +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER, int *) +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER_64, int *) + +VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) + +VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) + +VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PARALLEL_DECODING, unsigned int) +/*! @} - end defgroup vp8_encoder */ +#include "vpx_codec_impl_bottom.h" +#endif diff --git a/interface/external/LibVPX/include/vp8dx.h b/interface/external/LibVPX/include/vp8dx.h new file mode 100644 index 0000000000..7d250ccae8 --- /dev/null +++ b/interface/external/LibVPX/include/vp8dx.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#include "vp8.h" + +/*!\defgroup vp8_decoder WebM VP8 Decoder + * \ingroup vp8 + * + * @{ + */ +/*!\file + * \brief Provides definitions for using the VP8 algorithm within the vpx Decoder + * interface. + */ +#ifndef VP8DX_H +#define VP8DX_H +#include "vpx_codec_impl_top.h" + +/*!\name Algorithm interface for VP8 + * + * This interface provides the capability to decode raw VP8 streams, as would + * be found in AVI files and other non-Flash uses. + * @{ + */ +extern vpx_codec_iface_t vpx_codec_vp8_dx_algo; +extern vpx_codec_iface_t *vpx_codec_vp8_dx(void); + +/* TODO(jkoleszar): These move to VP9 in a later patch set. */ +extern vpx_codec_iface_t vpx_codec_vp9_dx_algo; +extern vpx_codec_iface_t *vpx_codec_vp9_dx(void); +/*!@} - end algorithm interface member group*/ + +/* Include controls common to both the encoder and decoder */ +#include "vp8.h" + + +/*!\brief VP8 decoder control functions + * + * This set of macros define the control functions available for the VP8 + * decoder interface. + * + * \sa #vpx_codec_control + */ +enum vp8_dec_control_id { + /** control function to get info on which reference frames were updated + * by the last decode + */ + VP8D_GET_LAST_REF_UPDATES = VP8_DECODER_CTRL_ID_START, + + /** check if the indicated frame is corrupted */ + VP8D_GET_FRAME_CORRUPTED, + + /** control function to get info on which reference frames were used + * by the last decode + */ + VP8D_GET_LAST_REF_USED, + + /** decryption function to decrypt encoded buffer data immediately + * before decoding. Takes a vp8_decrypt_init, which contains + * a callback function and opaque context pointer. + */ + VP8D_SET_DECRYPTOR, + + /** For testing. */ + VP9_INVERT_TILE_DECODE_ORDER, + + VP8_DECODER_CTRL_ID_MAX +}; + +typedef struct vp8_decrypt_init { + /** Decrypt n bytes of data from input -> output, using the decrypt_state + * passed in VP8D_SET_DECRYPTOR. + */ + void (*decrypt_cb)(void *decrypt_state, const unsigned char *input, + unsigned char *output, int count); + void *decrypt_state; +} vp8_decrypt_init; + +/*!\brief VP8 decoder control function parameter type + * + * Defines the data types that VP8D control functions take. Note that + * additional common controls are defined in vp8.h + * + */ + + +VPX_CTRL_USE_TYPE(VP8D_GET_LAST_REF_UPDATES, int *) +VPX_CTRL_USE_TYPE(VP8D_GET_FRAME_CORRUPTED, int *) +VPX_CTRL_USE_TYPE(VP8D_GET_LAST_REF_USED, int *) +VPX_CTRL_USE_TYPE(VP8D_SET_DECRYPTOR, vp8_decrypt_init *) +VPX_CTRL_USE_TYPE(VP9_INVERT_TILE_DECODE_ORDER, int) + +/*! @} - end defgroup vp8_decoder */ + + +#include "vpx_codec_impl_bottom.h" +#endif diff --git a/interface/external/LibVPX/include/vpx_codec.h b/interface/external/LibVPX/include/vpx_codec.h new file mode 100644 index 0000000000..2e6f1e757a --- /dev/null +++ b/interface/external/LibVPX/include/vpx_codec.h @@ -0,0 +1,558 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\defgroup codec Common Algorithm Interface + * This abstraction allows applications to easily support multiple video + * formats with minimal code duplication. This section describes the interface + * common to all codecs (both encoders and decoders). + * @{ + */ + +/*!\file + * \brief Describes the codec algorithm interface to applications. + * + * This file describes the interface between an application and a + * video codec algorithm. + * + * An application instantiates a specific codec instance by using + * vpx_codec_init() and a pointer to the algorithm's interface structure: + * 
+ *     my_app.c:
+ *       extern vpx_codec_iface_t my_codec;
+ *       {
+ *           vpx_codec_ctx_t algo;
+ *           res = vpx_codec_init(&algo, &my_codec);
+ *       }
+ *
+ * + * Once initialized, the instance is manged using other functions from + * the vpx_codec_* family. + */ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef VPX_CODEC_H +#define VPX_CODEC_H +#include "vpx_integer.h" +#include "vpx_image.h" + + /*!\brief Decorator indicating a function is deprecated */ +#ifndef DEPRECATED +#if defined(__GNUC__) && __GNUC__ +#define DEPRECATED __attribute__ ((deprecated)) +#elif defined(_MSC_VER) +#define DEPRECATED +#else +#define DEPRECATED +#endif +#endif /* DEPRECATED */ + +#ifndef DECLSPEC_DEPRECATED +#if defined(__GNUC__) && __GNUC__ +#define DECLSPEC_DEPRECATED /**< \copydoc #DEPRECATED */ +#elif defined(_MSC_VER) +#define DECLSPEC_DEPRECATED __declspec(deprecated) /**< \copydoc #DEPRECATED */ +#else +#define DECLSPEC_DEPRECATED /**< \copydoc #DEPRECATED */ +#endif +#endif /* DECLSPEC_DEPRECATED */ + + /*!\brief Decorator indicating a function is potentially unused */ +#ifdef UNUSED +#elif __GNUC__ +#define UNUSED __attribute__ ((unused)) +#else +#define UNUSED +#endif + + /*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define VPX_CODEC_ABI_VERSION (2 + VPX_IMAGE_ABI_VERSION) /**<\hideinitializer*/ + + /*!\brief Algorithm return codes */ + typedef enum { + /*!\brief Operation completed without error */ + VPX_CODEC_OK, + + /*!\brief Unspecified error */ + VPX_CODEC_ERROR, + + /*!\brief Memory operation failed */ + VPX_CODEC_MEM_ERROR, + + /*!\brief ABI version mismatch */ + VPX_CODEC_ABI_MISMATCH, + + /*!\brief Algorithm does not have required capability */ + VPX_CODEC_INCAPABLE, + + /*!\brief The given bitstream is not supported. + * + * The bitstream was unable to be parsed at the highest level. The decoder + * is unable to proceed. This error \ref SHOULD be treated as fatal to the + * stream. */ + VPX_CODEC_UNSUP_BITSTREAM, + + /*!\brief Encoded bitstream uses an unsupported feature + * + * The decoder does not implement a feature required by the encoder. This + * return code should only be used for features that prevent future + * pictures from being properly decoded. This error \ref MAY be treated as + * fatal to the stream or \ref MAY be treated as fatal to the current GOP. + */ + VPX_CODEC_UNSUP_FEATURE, + + /*!\brief The coded data for this stream is corrupt or incomplete + * + * There was a problem decoding the current frame. This return code + * should only be used for failures that prevent future pictures from + * being properly decoded. This error \ref MAY be treated as fatal to the + * stream or \ref MAY be treated as fatal to the current GOP. If decoding + * is continued for the current GOP, artifacts may be present. + */ + VPX_CODEC_CORRUPT_FRAME, + + /*!\brief An application-supplied parameter is not valid. + * + */ + VPX_CODEC_INVALID_PARAM, + + /*!\brief An iterator reached the end of list. + * + */ + VPX_CODEC_LIST_END + + } + vpx_codec_err_t; + + + /*! \brief Codec capabilities bitfield + * + * Each codec advertises the capabilities it supports as part of its + * ::vpx_codec_iface_t interface structure. Capabilities are extra interfaces + * or functionality, and are not required to be supported. + * + * The available flags are specified by VPX_CODEC_CAP_* defines. + */ + typedef long vpx_codec_caps_t; +#define VPX_CODEC_CAP_DECODER 0x1 /**< Is a decoder */ +#define VPX_CODEC_CAP_ENCODER 0x2 /**< Is an encoder */ +#define VPX_CODEC_CAP_XMA 0x4 /**< Supports eXternal Memory Allocation */ + + + /*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow for + * proper memory allocation. + * + * The available flags are specified by VPX_CODEC_USE_* defines. + */ + typedef long vpx_codec_flags_t; +#define VPX_CODEC_USE_XMA 0x00000001 /**< Use eXternal Memory Allocation mode */ + + + /*!\brief Codec interface structure. + * + * Contains function pointers and other data private to the codec + * implementation. This structure is opaque to the application. + */ + typedef const struct vpx_codec_iface vpx_codec_iface_t; + + + /*!\brief Codec private data structure. + * + * Contains data private to the codec implementation. This structure is opaque + * to the application. + */ + typedef struct vpx_codec_priv vpx_codec_priv_t; + + + /*!\brief Iterator + * + * Opaque storage used for iterating over lists. + */ + typedef const void *vpx_codec_iter_t; + + + /*!\brief Codec context structure + * + * All codecs \ref MUST support this context structure fully. In general, + * this data should be considered private to the codec algorithm, and + * not be manipulated or examined by the calling application. Applications + * may reference the 'name' member to get a printable description of the + * algorithm. + */ + typedef struct vpx_codec_ctx { + const char *name; /**< Printable interface name */ + vpx_codec_iface_t *iface; /**< Interface pointers */ + vpx_codec_err_t err; /**< Last returned error */ + const char *err_detail; /**< Detailed info, if available */ + vpx_codec_flags_t init_flags; /**< Flags passed at init time */ + union { + struct vpx_codec_dec_cfg *dec; /**< Decoder Configuration Pointer */ + struct vpx_codec_enc_cfg *enc; /**< Encoder Configuration Pointer */ + void *raw; + } config; /**< Configuration pointer aliasing union */ + vpx_codec_priv_t *priv; /**< Algorithm private storage */ + } vpx_codec_ctx_t; + + + /* + * Library Version Number Interface + * + * For example, see the following sample return values: + * vpx_codec_version() (1<<16 | 2<<8 | 3) + * vpx_codec_version_str() "v1.2.3-rc1-16-gec6a1ba" + * vpx_codec_version_extra_str() "rc1-16-gec6a1ba" + */ + + /*!\brief Return the version information (as an integer) + * + * Returns a packed encoding of the library version number. This will only include + * the major.minor.patch component of the version number. Note that this encoded + * value should be accessed through the macros provided, as the encoding may change + * in the future. + * + */ + int vpx_codec_version(void); +#define VPX_VERSION_MAJOR(v) ((v>>16)&0xff) /**< extract major from packed version */ +#define VPX_VERSION_MINOR(v) ((v>>8)&0xff) /**< extract minor from packed version */ +#define VPX_VERSION_PATCH(v) ((v>>0)&0xff) /**< extract patch from packed version */ + + /*!\brief Return the version major number */ +#define vpx_codec_version_major() ((vpx_codec_version()>>16)&0xff) + + /*!\brief Return the version minor number */ +#define vpx_codec_version_minor() ((vpx_codec_version()>>8)&0xff) + + /*!\brief Return the version patch number */ +#define vpx_codec_version_patch() ((vpx_codec_version()>>0)&0xff) + + + /*!\brief Return the version information (as a string) + * + * Returns a printable string containing the full library version number. This may + * contain additional text following the three digit version number, as to indicate + * release candidates, prerelease versions, etc. + * + */ + const char *vpx_codec_version_str(void); + + + /*!\brief Return the version information (as a string) + * + * Returns a printable "extra string". This is the component of the string returned + * by vpx_codec_version_str() following the three digit version number. + * + */ + const char *vpx_codec_version_extra_str(void); + + + /*!\brief Return the build configuration + * + * Returns a printable string containing an encoded version of the build + * configuration. This may be useful to vpx support. + * + */ + const char *vpx_codec_build_config(void); + + + /*!\brief Return the name for a given interface + * + * Returns a human readable string for name of the given codec interface. + * + * \param[in] iface Interface pointer + * + */ + const char *vpx_codec_iface_name(vpx_codec_iface_t *iface); + + + /*!\brief Convert error number to printable string + * + * Returns a human readable string for the last error returned by the + * algorithm. The returned error will be one line and will not contain + * any newline characters. + * + * + * \param[in] err Error number. + * + */ + const char *vpx_codec_err_to_string(vpx_codec_err_t err); + + + /*!\brief Retrieve error synopsis for codec context + * + * Returns a human readable string for the last error returned by the + * algorithm. The returned error will be one line and will not contain + * any newline characters. + * + * + * \param[in] ctx Pointer to this instance's context. + * + */ + const char *vpx_codec_error(vpx_codec_ctx_t *ctx); + + + /*!\brief Retrieve detailed error information for codec context + * + * Returns a human readable string providing detailed information about + * the last error. + * + * \param[in] ctx Pointer to this instance's context. + * + * \retval NULL + * No detailed information is available. + */ + const char *vpx_codec_error_detail(vpx_codec_ctx_t *ctx); + + + /* REQUIRED FUNCTIONS + * + * The following functions are required to be implemented for all codecs. + * They represent the base case functionality expected of all codecs. + */ + + /*!\brief Destroy a codec instance + * + * Destroys a codec context, freeing any associated memory buffers. + * + * \param[in] ctx Pointer to this instance's context + * + * \retval #VPX_CODEC_OK + * The codec algorithm initialized. + * \retval #VPX_CODEC_MEM_ERROR + * Memory allocation failed. + */ + vpx_codec_err_t vpx_codec_destroy(vpx_codec_ctx_t *ctx); + + + /*!\brief Get the capabilities of an algorithm. + * + * Retrieves the capabilities bitfield from the algorithm's interface. + * + * \param[in] iface Pointer to the algorithm interface + * + */ + vpx_codec_caps_t vpx_codec_get_caps(vpx_codec_iface_t *iface); + + + /*!\brief Control algorithm + * + * This function is used to exchange algorithm specific data with the codec + * instance. This can be used to implement features specific to a particular + * algorithm. + * + * This wrapper function dispatches the request to the helper function + * associated with the given ctrl_id. It tries to call this function + * transparently, but will return #VPX_CODEC_ERROR if the request could not + * be dispatched. + * + * Note that this function should not be used directly. Call the + * #vpx_codec_control wrapper macro instead. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] ctrl_id Algorithm specific control identifier + * + * \retval #VPX_CODEC_OK + * The control request was processed. + * \retval #VPX_CODEC_ERROR + * The control request was not processed. + * \retval #VPX_CODEC_INVALID_PARAM + * The data was not valid. + */ + vpx_codec_err_t vpx_codec_control_(vpx_codec_ctx_t *ctx, + int ctrl_id, + ...); +#if defined(VPX_DISABLE_CTRL_TYPECHECKS) && VPX_DISABLE_CTRL_TYPECHECKS +# define vpx_codec_control(ctx,id,data) vpx_codec_control_(ctx,id,data) +# define VPX_CTRL_USE_TYPE(id, typ) +# define VPX_CTRL_USE_TYPE_DEPRECATED(id, typ) +# define VPX_CTRL_VOID(id, typ) + +#else + /*!\brief vpx_codec_control wrapper macro + * + * This macro allows for type safe conversions across the variadic parameter + * to vpx_codec_control_(). + * + * \internal + * It works by dispatching the call to the control function through a wrapper + * function named with the id parameter. + */ +# define vpx_codec_control(ctx,id,data) vpx_codec_control_##id(ctx,id,data)\ + /**<\hideinitializer*/ + + + /*!\brief vpx_codec_control type definition macro + * + * This macro allows for type safe conversions across the variadic parameter + * to vpx_codec_control_(). It defines the type of the argument for a given + * control identifier. + * + * \internal + * It defines a static function with + * the correctly typed arguments as a wrapper to the type-unsafe internal + * function. + */ +# define VPX_CTRL_USE_TYPE(id, typ) \ + static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t*, int, typ) UNUSED;\ + \ + static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t *ctx, int ctrl_id, typ data) {\ + return vpx_codec_control_(ctx, ctrl_id, data);\ + } /**<\hideinitializer*/ + + + /*!\brief vpx_codec_control deprecated type definition macro + * + * Like #VPX_CTRL_USE_TYPE, but indicates that the specified control is + * deprecated and should not be used. Consult the documentation for your + * codec for more information. + * + * \internal + * It defines a static function with the correctly typed arguments as a + * wrapper to the type-unsafe internal function. + */ +# define VPX_CTRL_USE_TYPE_DEPRECATED(id, typ) \ + DECLSPEC_DEPRECATED static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t*, int, typ) DEPRECATED UNUSED;\ + \ + DECLSPEC_DEPRECATED static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t *ctx, int ctrl_id, typ data) {\ + return vpx_codec_control_(ctx, ctrl_id, data);\ + } /**<\hideinitializer*/ + + + /*!\brief vpx_codec_control void type definition macro + * + * This macro allows for type safe conversions across the variadic parameter + * to vpx_codec_control_(). It indicates that a given control identifier takes + * no argument. + * + * \internal + * It defines a static function without a data argument as a wrapper to the + * type-unsafe internal function. + */ +# define VPX_CTRL_VOID(id) \ + static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t*, int) UNUSED;\ + \ + static vpx_codec_err_t \ + vpx_codec_control_##id(vpx_codec_ctx_t *ctx, int ctrl_id) {\ + return vpx_codec_control_(ctx, ctrl_id);\ + } /**<\hideinitializer*/ + + +#endif + + + /*!\defgroup cap_xma External Memory Allocation Functions + * + * The following functions are required to be implemented for all codecs + * that advertise the VPX_CODEC_CAP_XMA capability. Calling these functions + * for codecs that don't advertise this capability will result in an error + * code being returned, usually VPX_CODEC_INCAPABLE + * @{ + */ + + + /*!\brief Memory Map Entry + * + * This structure is used to contain the properties of a memory segment. It + * is populated by the codec in the request phase, and by the calling + * application once the requested allocation has been performed. + */ + typedef struct vpx_codec_mmap { + /* + * The following members are set by the codec when requesting a segment + */ + unsigned int id; /**< identifier for the segment's contents */ + unsigned long sz; /**< size of the segment, in bytes */ + unsigned int align; /**< required alignment of the segment, in bytes */ + unsigned int flags; /**< bitfield containing segment properties */ +#define VPX_CODEC_MEM_ZERO 0x1 /**< Segment must be zeroed by allocation */ +#define VPX_CODEC_MEM_WRONLY 0x2 /**< Segment need not be readable */ +#define VPX_CODEC_MEM_FAST 0x4 /**< Place in fast memory, if available */ + + /* The following members are to be filled in by the allocation function */ + void *base; /**< pointer to the allocated segment */ + void (*dtor)(struct vpx_codec_mmap *map); /**< destructor to call */ + void *priv; /**< allocator private storage */ + } vpx_codec_mmap_t; /**< alias for struct vpx_codec_mmap */ + + + /*!\brief Iterate over the list of segments to allocate. + * + * Iterates over a list of the segments to allocate. The iterator storage + * should be initialized to NULL to start the iteration. Iteration is complete + * when this function returns VPX_CODEC_LIST_END. The amount of memory needed to + * allocate is dependent upon the size of the encoded stream. In cases where the + * stream is not available at allocation time, a fixed size must be requested. + * The codec will not be able to operate on streams larger than the size used at + * allocation time. + * + * \param[in] ctx Pointer to this instance's context. + * \param[out] mmap Pointer to the memory map entry to populate. + * \param[in,out] iter Iterator storage, initialized to NULL + * + * \retval #VPX_CODEC_OK + * The memory map entry was populated. + * \retval #VPX_CODEC_ERROR + * Codec does not support XMA mode. + * \retval #VPX_CODEC_MEM_ERROR + * Unable to determine segment size from stream info. + */ + vpx_codec_err_t vpx_codec_get_mem_map(vpx_codec_ctx_t *ctx, + vpx_codec_mmap_t *mmap, + vpx_codec_iter_t *iter); + + + /*!\brief Identify allocated segments to codec instance + * + * Stores a list of allocated segments in the codec. Segments \ref MUST be + * passed in the order they are read from vpx_codec_get_mem_map(), but may be + * passed in groups of any size. Segments \ref MUST be set only once. The + * allocation function \ref MUST ensure that the vpx_codec_mmap_t::base member + * is non-NULL. If the segment requires cleanup handling (e.g., calling free() + * or close()) then the vpx_codec_mmap_t::dtor member \ref MUST be populated. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] mmaps Pointer to the first memory map entry in the list. + * \param[in] num_maps Number of entries being set at this time + * + * \retval #VPX_CODEC_OK + * The segment was stored in the codec context. + * \retval #VPX_CODEC_INCAPABLE + * Codec does not support XMA mode. + * \retval #VPX_CODEC_MEM_ERROR + * Segment base address was not set, or segment was already stored. + + */ + vpx_codec_err_t vpx_codec_set_mem_map(vpx_codec_ctx_t *ctx, + vpx_codec_mmap_t *mmaps, + unsigned int num_maps); + + /*!@} - end defgroup cap_xma*/ + /*!@} - end defgroup codec*/ + + +#endif +#ifdef __cplusplus +} +#endif diff --git a/interface/external/LibVPX/include/vpx_codec_impl_bottom.h b/interface/external/LibVPX/include/vpx_codec_impl_bottom.h new file mode 100644 index 0000000000..6eb79a88a4 --- /dev/null +++ b/interface/external/LibVPX/include/vpx_codec_impl_bottom.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/* + * This file is to be included at the bottom of the header files defining the + * interface to individual codecs and contains matching blocks to those defined + * in vpx_codec_impl_top.h + */ +#ifdef __cplusplus +} +#endif diff --git a/interface/external/LibVPX/include/vpx_codec_impl_top.h b/interface/external/LibVPX/include/vpx_codec_impl_top.h new file mode 100644 index 0000000000..c9b8cfab29 --- /dev/null +++ b/interface/external/LibVPX/include/vpx_codec_impl_top.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/* + * This file is to be included at the top of the header files defining the + * interface to individual codecs and contains various workarounds common + * to all codec implementations. + */ +#ifdef __cplusplus +extern "C" { +#endif diff --git a/interface/external/LibVPX/include/vpx_decoder.h b/interface/external/LibVPX/include/vpx_decoder.h new file mode 100644 index 0000000000..e7701e5123 --- /dev/null +++ b/interface/external/LibVPX/include/vpx_decoder.h @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\defgroup decoder Decoder Algorithm Interface + * \ingroup codec + * This abstraction allows applications using this decoder to easily support + * multiple video formats with minimal code duplication. This section describes + * the interface common to all decoders. + * @{ + */ + +/*!\file + * \brief Describes the decoder algorithm interface to applications. + * + * This file describes the interface between an application and a + * video decoder algorithm. + * + */ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef VPX_DECODER_H +#define VPX_DECODER_H +#include "vpx_codec.h" + + /*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define VPX_DECODER_ABI_VERSION (2 + VPX_CODEC_ABI_VERSION) /**<\hideinitializer*/ + + /*! \brief Decoder capabilities bitfield + * + * Each decoder advertises the capabilities it supports as part of its + * ::vpx_codec_iface_t interface structure. Capabilities are extra interfaces + * or functionality, and are not required to be supported by a decoder. + * + * The available flags are specified by VPX_CODEC_CAP_* defines. + */ +#define VPX_CODEC_CAP_PUT_SLICE 0x10000 /**< Will issue put_slice callbacks */ +#define VPX_CODEC_CAP_PUT_FRAME 0x20000 /**< Will issue put_frame callbacks */ +#define VPX_CODEC_CAP_POSTPROC 0x40000 /**< Can postprocess decoded frame */ +#define VPX_CODEC_CAP_ERROR_CONCEALMENT 0x80000 /**< Can conceal errors due to + packet loss */ +#define VPX_CODEC_CAP_INPUT_FRAGMENTS 0x100000 /**< Can receive encoded frames + one fragment at a time */ + + /*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow for + * proper memory allocation. + * + * The available flags are specified by VPX_CODEC_USE_* defines. + */ +#define VPX_CODEC_CAP_FRAME_THREADING 0x200000 /**< Can support frame-based + multi-threading */ + +#define VPX_CODEC_USE_POSTPROC 0x10000 /**< Postprocess decoded frame */ +#define VPX_CODEC_USE_ERROR_CONCEALMENT 0x20000 /**< Conceal errors in decoded + frames */ +#define VPX_CODEC_USE_INPUT_FRAGMENTS 0x40000 /**< The input frame should be + passed to the decoder one + fragment at a time */ +#define VPX_CODEC_USE_FRAME_THREADING 0x80000 /**< Enable frame-based + multi-threading */ + + /*!\brief Stream properties + * + * This structure is used to query or set properties of the decoded + * stream. Algorithms may extend this structure with data specific + * to their bitstream by setting the sz member appropriately. + */ + typedef struct vpx_codec_stream_info { + unsigned int sz; /**< Size of this structure */ + unsigned int w; /**< Width (or 0 for unknown/default) */ + unsigned int h; /**< Height (or 0 for unknown/default) */ + unsigned int is_kf; /**< Current frame is a keyframe */ + } vpx_codec_stream_info_t; + + /* REQUIRED FUNCTIONS + * + * The following functions are required to be implemented for all decoders. + * They represent the base case functionality expected of all decoders. + */ + + + /*!\brief Initialization Configurations + * + * This structure is used to pass init time configuration options to the + * decoder. + */ + typedef struct vpx_codec_dec_cfg { + unsigned int threads; /**< Maximum number of threads to use, default 1 */ + unsigned int w; /**< Width */ + unsigned int h; /**< Height */ + } vpx_codec_dec_cfg_t; /**< alias for struct vpx_codec_dec_cfg */ + + + /*!\brief Initialize a decoder instance + * + * Initializes a decoder context using the given interface. Applications + * should call the vpx_codec_dec_init convenience macro instead of this + * function directly, to ensure that the ABI version number parameter + * is properly initialized. + * + * If the library was configured with --disable-multithread, this call + * is not thread safe and should be guarded with a lock if being used + * in a multithreaded context. + * + * In XMA mode (activated by setting VPX_CODEC_USE_XMA in the flags + * parameter), the storage pointed to by the cfg parameter must be + * kept readable and stable until all memory maps have been set. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] flags Bitfield of VPX_CODEC_USE_* flags + * \param[in] ver ABI version number. Must be set to + * VPX_DECODER_ABI_VERSION + * \retval #VPX_CODEC_OK + * The decoder algorithm initialized. + * \retval #VPX_CODEC_MEM_ERROR + * Memory allocation failed. + */ + vpx_codec_err_t vpx_codec_dec_init_ver(vpx_codec_ctx_t *ctx, + vpx_codec_iface_t *iface, + vpx_codec_dec_cfg_t *cfg, + vpx_codec_flags_t flags, + int ver); + + /*!\brief Convenience macro for vpx_codec_dec_init_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define vpx_codec_dec_init(ctx, iface, cfg, flags) \ + vpx_codec_dec_init_ver(ctx, iface, cfg, flags, VPX_DECODER_ABI_VERSION) + + + /*!\brief Parse stream info from a buffer + * + * Performs high level parsing of the bitstream. Construction of a decoder + * context is not necessary. Can be used to determine if the bitstream is + * of the proper format, and to extract information from the stream. + * + * \param[in] iface Pointer to the algorithm interface + * \param[in] data Pointer to a block of data to parse + * \param[in] data_sz Size of the data buffer + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #VPX_CODEC_OK + * Bitstream is parsable and stream information updated + */ + vpx_codec_err_t vpx_codec_peek_stream_info(vpx_codec_iface_t *iface, + const uint8_t *data, + unsigned int data_sz, + vpx_codec_stream_info_t *si); + + + /*!\brief Return information about the current stream. + * + * Returns information about the stream that has been parsed during decoding. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] si Pointer to stream info to update. The size member + * \ref MUST be properly initialized, but \ref MAY be + * clobbered by the algorithm. This parameter \ref MAY + * be NULL. + * + * \retval #VPX_CODEC_OK + * Bitstream is parsable and stream information updated + */ + vpx_codec_err_t vpx_codec_get_stream_info(vpx_codec_ctx_t *ctx, + vpx_codec_stream_info_t *si); + + + /*!\brief Decode data + * + * Processes a buffer of coded data. If the processing results in a new + * decoded frame becoming available, PUT_SLICE and PUT_FRAME events may be + * generated, as appropriate. Encoded data \ref MUST be passed in DTS (decode + * time stamp) order. Frames produced will always be in PTS (presentation + * time stamp) order. + * If the decoder is configured with VPX_CODEC_USE_INPUT_FRAGMENTS enabled, + * data and data_sz can contain a fragment of the encoded frame. Fragment + * \#n must contain at least partition \#n, but can also contain subsequent + * partitions (\#n+1 - \#n+i), and if so, fragments \#n+1, .., \#n+i must + * be empty. When no more data is available, this function should be called + * with NULL as data and 0 as data_sz. The memory passed to this function + * must be available until the frame has been decoded. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] data Pointer to this block of new coded data. If + * NULL, a VPX_CODEC_CB_PUT_FRAME event is posted + * for the previously decoded frame. + * \param[in] data_sz Size of the coded data, in bytes. + * \param[in] user_priv Application specific data to associate with + * this frame. + * \param[in] deadline Soft deadline the decoder should attempt to meet, + * in us. Set to zero for unlimited. + * + * \return Returns #VPX_CODEC_OK if the coded data was processed completely + * and future pictures can be decoded without error. Otherwise, + * see the descriptions of the other error codes in ::vpx_codec_err_t + * for recoverability capabilities. + */ + vpx_codec_err_t vpx_codec_decode(vpx_codec_ctx_t *ctx, + const uint8_t *data, + unsigned int data_sz, + void *user_priv, + long deadline); + + + /*!\brief Decoded frames iterator + * + * Iterates over a list of the frames available for display. The iterator + * storage should be initialized to NULL to start the iteration. Iteration is + * complete when this function returns NULL. + * + * The list of available frames becomes valid upon completion of the + * vpx_codec_decode call, and remains valid until the next call to vpx_codec_decode. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] iter Iterator storage, initialized to NULL + * + * \return Returns a pointer to an image, if one is ready for display. Frames + * produced will always be in PTS (presentation time stamp) order. + */ + vpx_image_t *vpx_codec_get_frame(vpx_codec_ctx_t *ctx, + vpx_codec_iter_t *iter); + + + /*!\defgroup cap_put_frame Frame-Based Decoding Functions + * + * The following functions are required to be implemented for all decoders + * that advertise the VPX_CODEC_CAP_PUT_FRAME capability. Calling these functions + * for codecs that don't advertise this capability will result in an error + * code being returned, usually VPX_CODEC_ERROR + * @{ + */ + + /*!\brief put frame callback prototype + * + * This callback is invoked by the decoder to notify the application of + * the availability of decoded image data. + */ + typedef void (*vpx_codec_put_frame_cb_fn_t)(void *user_priv, + const vpx_image_t *img); + + + /*!\brief Register for notification of frame completion. + * + * Registers a given function to be called when a decoded frame is + * available. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb Pointer to the callback function + * \param[in] user_priv User's private data + * + * \retval #VPX_CODEC_OK + * Callback successfully registered. + * \retval #VPX_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * posting slice completion. + */ + vpx_codec_err_t vpx_codec_register_put_frame_cb(vpx_codec_ctx_t *ctx, + vpx_codec_put_frame_cb_fn_t cb, + void *user_priv); + + + /*!@} - end defgroup cap_put_frame */ + + /*!\defgroup cap_put_slice Slice-Based Decoding Functions + * + * The following functions are required to be implemented for all decoders + * that advertise the VPX_CODEC_CAP_PUT_SLICE capability. Calling these functions + * for codecs that don't advertise this capability will result in an error + * code being returned, usually VPX_CODEC_ERROR + * @{ + */ + + /*!\brief put slice callback prototype + * + * This callback is invoked by the decoder to notify the application of + * the availability of partially decoded image data. The + */ + typedef void (*vpx_codec_put_slice_cb_fn_t)(void *user_priv, + const vpx_image_t *img, + const vpx_image_rect_t *valid, + const vpx_image_rect_t *update); + + + /*!\brief Register for notification of slice completion. + * + * Registers a given function to be called when a decoded slice is + * available. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cb Pointer to the callback function + * \param[in] user_priv User's private data + * + * \retval #VPX_CODEC_OK + * Callback successfully registered. + * \retval #VPX_CODEC_ERROR + * Decoder context not initialized, or algorithm not capable of + * posting slice completion. + */ + vpx_codec_err_t vpx_codec_register_put_slice_cb(vpx_codec_ctx_t *ctx, + vpx_codec_put_slice_cb_fn_t cb, + void *user_priv); + + + /*!@} - end defgroup cap_put_slice*/ + + /*!@} - end defgroup decoder*/ + +#endif + +#ifdef __cplusplus +} +#endif diff --git a/interface/external/LibVPX/include/vpx_encoder.h b/interface/external/LibVPX/include/vpx_encoder.h new file mode 100644 index 0000000000..ffdbc0644f --- /dev/null +++ b/interface/external/LibVPX/include/vpx_encoder.h @@ -0,0 +1,924 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\defgroup encoder Encoder Algorithm Interface + * \ingroup codec + * This abstraction allows applications using this encoder to easily support + * multiple video formats with minimal code duplication. This section describes + * the interface common to all encoders. + * @{ + */ + +/*!\file + * \brief Describes the encoder algorithm interface to applications. + * + * This file describes the interface between an application and a + * video encoder algorithm. + * + */ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef VPX_ENCODER_H +#define VPX_ENCODER_H +#include "vpx_codec.h" + + /*! Temporal Scalability: Maximum length of the sequence defining frame + * layer membership + */ +#define VPX_TS_MAX_PERIODICITY 16 + + /*! Temporal Scalability: Maximum number of coding layers */ +#define VPX_TS_MAX_LAYERS 5 + + /*!\deprecated Use #VPX_TS_MAX_PERIODICITY instead. */ +#define MAX_PERIODICITY VPX_TS_MAX_PERIODICITY + + /*!\deprecated Use #VPX_TS_MAX_LAYERS instead. */ +#define MAX_LAYERS VPX_TS_MAX_LAYERS + + /*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define VPX_ENCODER_ABI_VERSION (3 + VPX_CODEC_ABI_VERSION) /**<\hideinitializer*/ + + + /*! \brief Encoder capabilities bitfield + * + * Each encoder advertises the capabilities it supports as part of its + * ::vpx_codec_iface_t interface structure. Capabilities are extra + * interfaces or functionality, and are not required to be supported + * by an encoder. + * + * The available flags are specified by VPX_CODEC_CAP_* defines. + */ +#define VPX_CODEC_CAP_PSNR 0x10000 /**< Can issue PSNR packets */ + + /*! Can output one partition at a time. Each partition is returned in its + * own VPX_CODEC_CX_FRAME_PKT, with the FRAME_IS_FRAGMENT flag set for + * every partition but the last. In this mode all frames are always + * returned partition by partition. + */ +#define VPX_CODEC_CAP_OUTPUT_PARTITION 0x20000 + + + /*! \brief Initialization-time Feature Enabling + * + * Certain codec features must be known at initialization time, to allow + * for proper memory allocation. + * + * The available flags are specified by VPX_CODEC_USE_* defines. + */ +#define VPX_CODEC_USE_PSNR 0x10000 /**< Calculate PSNR on each frame */ +#define VPX_CODEC_USE_OUTPUT_PARTITION 0x20000 /**< Make the encoder output one + partition at a time. */ + + + /*!\brief Generic fixed size buffer structure + * + * This structure is able to hold a reference to any fixed size buffer. + */ + typedef struct vpx_fixed_buf { + void *buf; /**< Pointer to the data */ + size_t sz; /**< Length of the buffer, in chars */ + } vpx_fixed_buf_t; /**< alias for struct vpx_fixed_buf */ + + + /*!\brief Time Stamp Type + * + * An integer, which when multiplied by the stream's time base, provides + * the absolute time of a sample. + */ + typedef int64_t vpx_codec_pts_t; + + + /*!\brief Compressed Frame Flags + * + * This type represents a bitfield containing information about a compressed + * frame that may be useful to an application. The most significant 16 bits + * can be used by an algorithm to provide additional detail, for example to + * support frame types that are codec specific (MPEG-1 D-frames for example) + */ + typedef uint32_t vpx_codec_frame_flags_t; +#define VPX_FRAME_IS_KEY 0x1 /**< frame is the start of a GOP */ +#define VPX_FRAME_IS_DROPPABLE 0x2 /**< frame can be dropped without affecting + the stream (no future frame depends on + this one) */ +#define VPX_FRAME_IS_INVISIBLE 0x4 /**< frame should be decoded but will not + be shown */ +#define VPX_FRAME_IS_FRAGMENT 0x8 /**< this is a fragment of the encoded + frame */ + + /*!\brief Error Resilient flags + * + * These flags define which error resilient features to enable in the + * encoder. The flags are specified through the + * vpx_codec_enc_cfg::g_error_resilient variable. + */ + typedef uint32_t vpx_codec_er_flags_t; +#define VPX_ERROR_RESILIENT_DEFAULT 0x1 /**< Improve resiliency against + losses of whole frames */ +#define VPX_ERROR_RESILIENT_PARTITIONS 0x2 /**< The frame partitions are + independently decodable by the + bool decoder, meaning that + partitions can be decoded even + though earlier partitions have + been lost. Note that intra + predicition is still done over + the partition boundary. */ + + /*!\brief Encoder output packet variants + * + * This enumeration lists the different kinds of data packets that can be + * returned by calls to vpx_codec_get_cx_data(). Algorithms \ref MAY + * extend this list to provide additional functionality. + */ + enum vpx_codec_cx_pkt_kind { + VPX_CODEC_CX_FRAME_PKT, /**< Compressed video frame */ + VPX_CODEC_STATS_PKT, /**< Two-pass statistics for this frame */ + VPX_CODEC_PSNR_PKT, /**< PSNR statistics for this frame */ + VPX_CODEC_CUSTOM_PKT = 256 /**< Algorithm extensions */ + }; + + + /*!\brief Encoder output packet + * + * This structure contains the different kinds of output data the encoder + * may produce while compressing a frame. + */ + typedef struct vpx_codec_cx_pkt { + enum vpx_codec_cx_pkt_kind kind; /**< packet variant */ + union { + struct { + void *buf; /**< compressed data buffer */ + size_t sz; /**< length of compressed data */ + vpx_codec_pts_t pts; /**< time stamp to show frame + (in timebase units) */ + unsigned long duration; /**< duration to show frame + (in timebase units) */ + vpx_codec_frame_flags_t flags; /**< flags for this frame */ + int partition_id; /**< the partition id + defines the decoding order + of the partitions. Only + applicable when "output partition" + mode is enabled. First partition + has id 0.*/ + + } frame; /**< data for compressed frame packet */ + struct vpx_fixed_buf twopass_stats; /**< data for two-pass packet */ + struct vpx_psnr_pkt { + unsigned int samples[4]; /**< Number of samples, total/y/u/v */ + uint64_t sse[4]; /**< sum squared error, total/y/u/v */ + double psnr[4]; /**< PSNR, total/y/u/v */ + } psnr; /**< data for PSNR packet */ + struct vpx_fixed_buf raw; /**< data for arbitrary packets */ + + /* This packet size is fixed to allow codecs to extend this + * interface without having to manage storage for raw packets, + * i.e., if it's smaller than 128 bytes, you can store in the + * packet list directly. + */ + char pad[128 - sizeof(enum vpx_codec_cx_pkt_kind)]; /**< fixed sz */ + } data; /**< packet data */ + } vpx_codec_cx_pkt_t; /**< alias for struct vpx_codec_cx_pkt */ + + + /*!\brief Rational Number + * + * This structure holds a fractional value. + */ + typedef struct vpx_rational { + int num; /**< fraction numerator */ + int den; /**< fraction denominator */ + } vpx_rational_t; /**< alias for struct vpx_rational */ + + + /*!\brief Multi-pass Encoding Pass */ + enum vpx_enc_pass { + VPX_RC_ONE_PASS, /**< Single pass mode */ + VPX_RC_FIRST_PASS, /**< First pass of multi-pass mode */ + VPX_RC_LAST_PASS /**< Final pass of multi-pass mode */ + }; + + + /*!\brief Rate control mode */ + enum vpx_rc_mode { + VPX_VBR, /**< Variable Bit Rate (VBR) mode */ + VPX_CBR, /**< Constant Bit Rate (CBR) mode */ + VPX_CQ /**< Constant Quality (CQ) mode */ + }; + + + /*!\brief Keyframe placement mode. + * + * This enumeration determines whether keyframes are placed automatically by + * the encoder or whether this behavior is disabled. Older releases of this + * SDK were implemented such that VPX_KF_FIXED meant keyframes were disabled. + * This name is confusing for this behavior, so the new symbols to be used + * are VPX_KF_AUTO and VPX_KF_DISABLED. + */ + enum vpx_kf_mode { + VPX_KF_FIXED, /**< deprecated, implies VPX_KF_DISABLED */ + VPX_KF_AUTO, /**< Encoder determines optimal placement automatically */ + VPX_KF_DISABLED = 0 /**< Encoder does not place keyframes. */ + }; + + + /*!\brief Encoded Frame Flags + * + * This type indicates a bitfield to be passed to vpx_codec_encode(), defining + * per-frame boolean values. By convention, bits common to all codecs will be + * named VPX_EFLAG_*, and bits specific to an algorithm will be named + * /algo/_eflag_*. The lower order 16 bits are reserved for common use. + */ + typedef long vpx_enc_frame_flags_t; +#define VPX_EFLAG_FORCE_KF (1<<0) /**< Force this frame to be a keyframe */ + + + /*!\brief Encoder configuration structure + * + * This structure contains the encoder settings that have common representations + * across all codecs. This doesn't imply that all codecs support all features, + * however. + */ + typedef struct vpx_codec_enc_cfg { + /* + * generic settings (g) + */ + + /*!\brief Algorithm specific "usage" value + * + * Algorithms may define multiple values for usage, which may convey the + * intent of how the application intends to use the stream. If this value + * is non-zero, consult the documentation for the codec to determine its + * meaning. + */ + unsigned int g_usage; + + + /*!\brief Maximum number of threads to use + * + * For multi-threaded implementations, use no more than this number of + * threads. The codec may use fewer threads than allowed. The value + * 0 is equivalent to the value 1. + */ + unsigned int g_threads; + + + /*!\brief Bitstream profile to use + * + * Some codecs support a notion of multiple bitstream profiles. Typically + * this maps to a set of features that are turned on or off. Often the + * profile to use is determined by the features of the intended decoder. + * Consult the documentation for the codec to determine the valid values + * for this parameter, or set to zero for a sane default. + */ + unsigned int g_profile; /**< profile of bitstream to use */ + + + + /*!\brief Width of the frame + * + * This value identifies the presentation resolution of the frame, + * in pixels. Note that the frames passed as input to the encoder must + * have this resolution. Frames will be presented by the decoder in this + * resolution, independent of any spatial resampling the encoder may do. + */ + unsigned int g_w; + + + /*!\brief Height of the frame + * + * This value identifies the presentation resolution of the frame, + * in pixels. Note that the frames passed as input to the encoder must + * have this resolution. Frames will be presented by the decoder in this + * resolution, independent of any spatial resampling the encoder may do. + */ + unsigned int g_h; + + + /*!\brief Stream timebase units + * + * Indicates the smallest interval of time, in seconds, used by the stream. + * For fixed frame rate material, or variable frame rate material where + * frames are timed at a multiple of a given clock (ex: video capture), + * the \ref RECOMMENDED method is to set the timebase to the reciprocal + * of the frame rate (ex: 1001/30000 for 29.970 Hz NTSC). This allows the + * pts to correspond to the frame number, which can be handy. For + * re-encoding video from containers with absolute time timestamps, the + * \ref RECOMMENDED method is to set the timebase to that of the parent + * container or multimedia framework (ex: 1/1000 for ms, as in FLV). + */ + struct vpx_rational g_timebase; + + + /*!\brief Enable error resilient modes. + * + * The error resilient bitfield indicates to the encoder which features + * it should enable to take measures for streaming over lossy or noisy + * links. + */ + vpx_codec_er_flags_t g_error_resilient; + + + /*!\brief Multi-pass Encoding Mode + * + * This value should be set to the current phase for multi-pass encoding. + * For single pass, set to #VPX_RC_ONE_PASS. + */ + enum vpx_enc_pass g_pass; + + + /*!\brief Allow lagged encoding + * + * If set, this value allows the encoder to consume a number of input + * frames before producing output frames. This allows the encoder to + * base decisions for the current frame on future frames. This does + * increase the latency of the encoding pipeline, so it is not appropriate + * in all situations (ex: realtime encoding). + * + * Note that this is a maximum value -- the encoder may produce frames + * sooner than the given limit. Set this value to 0 to disable this + * feature. + */ + unsigned int g_lag_in_frames; + + + /* + * rate control settings (rc) + */ + + /*!\brief Temporal resampling configuration, if supported by the codec. + * + * Temporal resampling allows the codec to "drop" frames as a strategy to + * meet its target data rate. This can cause temporal discontinuities in + * the encoded video, which may appear as stuttering during playback. This + * trade-off is often acceptable, but for many applications is not. It can + * be disabled in these cases. + * + * Note that not all codecs support this feature. All vpx VPx codecs do. + * For other codecs, consult the documentation for that algorithm. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer falls below this percentage of fullness, a + * dropped frame is indicated. Set the threshold to zero (0) to disable + * this feature. + */ + unsigned int rc_dropframe_thresh; + + + /*!\brief Enable/disable spatial resampling, if supported by the codec. + * + * Spatial resampling allows the codec to compress a lower resolution + * version of the frame, which is then upscaled by the encoder to the + * correct presentation resolution. This increases visual quality at + * low data rates, at the expense of CPU time on the encoder/decoder. + */ + unsigned int rc_resize_allowed; + + + /*!\brief Spatial resampling up watermark. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer rises above this percentage of fullness, the + * encoder will step up to a higher resolution version of the frame. + */ + unsigned int rc_resize_up_thresh; + + + /*!\brief Spatial resampling down watermark. + * + * This threshold is described as a percentage of the target data buffer. + * When the data buffer falls below this percentage of fullness, the + * encoder will step down to a lower resolution version of the frame. + */ + unsigned int rc_resize_down_thresh; + + + /*!\brief Rate control algorithm to use. + * + * Indicates whether the end usage of this stream is to be streamed over + * a bandwidth constrained link, indicating that Constant Bit Rate (CBR) + * mode should be used, or whether it will be played back on a high + * bandwidth link, as from a local disk, where higher variations in + * bitrate are acceptable. + */ + enum vpx_rc_mode rc_end_usage; + + + /*!\brief Two-pass stats buffer. + * + * A buffer containing all of the stats packets produced in the first + * pass, concatenated. + */ + struct vpx_fixed_buf rc_twopass_stats_in; + + + /*!\brief Target data rate + * + * Target bandwidth to use for this stream, in kilobits per second. + */ + unsigned int rc_target_bitrate; + + + /* + * quantizer settings + */ + + + /*!\brief Minimum (Best Quality) Quantizer + * + * The quantizer is the most direct control over the quality of the + * encoded image. The range of valid values for the quantizer is codec + * specific. Consult the documentation for the codec to determine the + * values to use. To determine the range programmatically, call + * vpx_codec_enc_config_default() with a usage value of 0. + */ + unsigned int rc_min_quantizer; + + + /*!\brief Maximum (Worst Quality) Quantizer + * + * The quantizer is the most direct control over the quality of the + * encoded image. The range of valid values for the quantizer is codec + * specific. Consult the documentation for the codec to determine the + * values to use. To determine the range programmatically, call + * vpx_codec_enc_config_default() with a usage value of 0. + */ + unsigned int rc_max_quantizer; + + + /* + * bitrate tolerance + */ + + + /*!\brief Rate control adaptation undershoot control + * + * This value, expressed as a percentage of the target bitrate, + * controls the maximum allowed adaptation speed of the codec. + * This factor controls the maximum amount of bits that can + * be subtracted from the target bitrate in order to compensate + * for prior overshoot. + * + * Valid values in the range 0-1000. + */ + unsigned int rc_undershoot_pct; + + + /*!\brief Rate control adaptation overshoot control + * + * This value, expressed as a percentage of the target bitrate, + * controls the maximum allowed adaptation speed of the codec. + * This factor controls the maximum amount of bits that can + * be added to the target bitrate in order to compensate for + * prior undershoot. + * + * Valid values in the range 0-1000. + */ + unsigned int rc_overshoot_pct; + + + /* + * decoder buffer model parameters + */ + + + /*!\brief Decoder Buffer Size + * + * This value indicates the amount of data that may be buffered by the + * decoding application. Note that this value is expressed in units of + * time (milliseconds). For example, a value of 5000 indicates that the + * client will buffer (at least) 5000ms worth of encoded data. Use the + * target bitrate (#rc_target_bitrate) to convert to bits/bytes, if + * necessary. + */ + unsigned int rc_buf_sz; + + + /*!\brief Decoder Buffer Initial Size + * + * This value indicates the amount of data that will be buffered by the + * decoding application prior to beginning playback. This value is + * expressed in units of time (milliseconds). Use the target bitrate + * (#rc_target_bitrate) to convert to bits/bytes, if necessary. + */ + unsigned int rc_buf_initial_sz; + + + /*!\brief Decoder Buffer Optimal Size + * + * This value indicates the amount of data that the encoder should try + * to maintain in the decoder's buffer. This value is expressed in units + * of time (milliseconds). Use the target bitrate (#rc_target_bitrate) + * to convert to bits/bytes, if necessary. + */ + unsigned int rc_buf_optimal_sz; + + + /* + * 2 pass rate control parameters + */ + + + /*!\brief Two-pass mode CBR/VBR bias + * + * Bias, expressed on a scale of 0 to 100, for determining target size + * for the current frame. The value 0 indicates the optimal CBR mode + * value should be used. The value 100 indicates the optimal VBR mode + * value should be used. Values in between indicate which way the + * encoder should "lean." + */ + unsigned int rc_2pass_vbr_bias_pct; /**< RC mode bias between CBR and VBR(0-100: 0->CBR, 100->VBR) */ + + + /*!\brief Two-pass mode per-GOP minimum bitrate + * + * This value, expressed as a percentage of the target bitrate, indicates + * the minimum bitrate to be used for a single GOP (aka "section") + */ + unsigned int rc_2pass_vbr_minsection_pct; + + + /*!\brief Two-pass mode per-GOP maximum bitrate + * + * This value, expressed as a percentage of the target bitrate, indicates + * the maximum bitrate to be used for a single GOP (aka "section") + */ + unsigned int rc_2pass_vbr_maxsection_pct; + + + /* + * keyframing settings (kf) + */ + + /*!\brief Keyframe placement mode + * + * This value indicates whether the encoder should place keyframes at a + * fixed interval, or determine the optimal placement automatically + * (as governed by the #kf_min_dist and #kf_max_dist parameters) + */ + enum vpx_kf_mode kf_mode; + + + /*!\brief Keyframe minimum interval + * + * This value, expressed as a number of frames, prevents the encoder from + * placing a keyframe nearer than kf_min_dist to the previous keyframe. At + * least kf_min_dist frames non-keyframes will be coded before the next + * keyframe. Set kf_min_dist equal to kf_max_dist for a fixed interval. + */ + unsigned int kf_min_dist; + + + /*!\brief Keyframe maximum interval + * + * This value, expressed as a number of frames, forces the encoder to code + * a keyframe if one has not been coded in the last kf_max_dist frames. + * A value of 0 implies all frames will be keyframes. Set kf_min_dist + * equal to kf_max_dist for a fixed interval. + */ + unsigned int kf_max_dist; + + /* + * Temporal scalability settings (ts) + */ + + /*!\brief Number of coding layers + * + * This value specifies the number of coding layers to be used. + */ + unsigned int ts_number_layers; + + /*!\brief Target bitrate for each layer + * + * These values specify the target coding bitrate for each coding layer. + */ + unsigned int ts_target_bitrate[VPX_TS_MAX_LAYERS]; + + /*!\brief Frame rate decimation factor for each layer + * + * These values specify the frame rate decimation factors to apply + * to each layer. + */ + unsigned int ts_rate_decimator[VPX_TS_MAX_LAYERS]; + + /*!\brief Length of the sequence defining frame layer membership + * + * This value specifies the length of the sequence that defines the + * membership of frames to layers. For example, if ts_periodicity=8 then + * frames are assigned to coding layers with a repeated sequence of + * length 8. + */ + unsigned int ts_periodicity; + + /*!\brief Template defining the membership of frames to coding layers + * + * This array defines the membership of frames to coding layers. For a + * 2-layer encoding that assigns even numbered frames to one layer (0) + * and odd numbered frames to a second layer (1) with ts_periodicity=8, + * then ts_layer_id = (0,1,0,1,0,1,0,1). + */ + unsigned int ts_layer_id[VPX_TS_MAX_PERIODICITY]; + } vpx_codec_enc_cfg_t; /**< alias for struct vpx_codec_enc_cfg */ + + + /*!\brief Initialize an encoder instance + * + * Initializes a encoder context using the given interface. Applications + * should call the vpx_codec_enc_init convenience macro instead of this + * function directly, to ensure that the ABI version number parameter + * is properly initialized. + * + * If the library was configured with --disable-multithread, this call + * is not thread safe and should be guarded with a lock if being used + * in a multithreaded context. + * + * In XMA mode (activated by setting VPX_CODEC_USE_XMA in the flags + * parameter), the storage pointed to by the cfg parameter must be + * kept readable and stable until all memory maps have been set. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] flags Bitfield of VPX_CODEC_USE_* flags + * \param[in] ver ABI version number. Must be set to + * VPX_ENCODER_ABI_VERSION + * \retval #VPX_CODEC_OK + * The decoder algorithm initialized. + * \retval #VPX_CODEC_MEM_ERROR + * Memory allocation failed. + */ + vpx_codec_err_t vpx_codec_enc_init_ver(vpx_codec_ctx_t *ctx, + vpx_codec_iface_t *iface, + vpx_codec_enc_cfg_t *cfg, + vpx_codec_flags_t flags, + int ver); + + + /*!\brief Convenience macro for vpx_codec_enc_init_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define vpx_codec_enc_init(ctx, iface, cfg, flags) \ + vpx_codec_enc_init_ver(ctx, iface, cfg, flags, VPX_ENCODER_ABI_VERSION) + + + /*!\brief Initialize multi-encoder instance + * + * Initializes multi-encoder context using the given interface. + * Applications should call the vpx_codec_enc_init_multi convenience macro + * instead of this function directly, to ensure that the ABI version number + * parameter is properly initialized. + * + * In XMA mode (activated by setting VPX_CODEC_USE_XMA in the flags + * parameter), the storage pointed to by the cfg parameter must be + * kept readable and stable until all memory maps have been set. + * + * \param[in] ctx Pointer to this instance's context. + * \param[in] iface Pointer to the algorithm interface to use. + * \param[in] cfg Configuration to use, if known. May be NULL. + * \param[in] num_enc Total number of encoders. + * \param[in] flags Bitfield of VPX_CODEC_USE_* flags + * \param[in] dsf Pointer to down-sampling factors. + * \param[in] ver ABI version number. Must be set to + * VPX_ENCODER_ABI_VERSION + * \retval #VPX_CODEC_OK + * The decoder algorithm initialized. + * \retval #VPX_CODEC_MEM_ERROR + * Memory allocation failed. + */ + vpx_codec_err_t vpx_codec_enc_init_multi_ver(vpx_codec_ctx_t *ctx, + vpx_codec_iface_t *iface, + vpx_codec_enc_cfg_t *cfg, + int num_enc, + vpx_codec_flags_t flags, + vpx_rational_t *dsf, + int ver); + + + /*!\brief Convenience macro for vpx_codec_enc_init_multi_ver() + * + * Ensures the ABI version parameter is properly set. + */ +#define vpx_codec_enc_init_multi(ctx, iface, cfg, num_enc, flags, dsf) \ + vpx_codec_enc_init_multi_ver(ctx, iface, cfg, num_enc, flags, dsf, \ + VPX_ENCODER_ABI_VERSION) + + + /*!\brief Get a default configuration + * + * Initializes a encoder configuration structure with default values. Supports + * the notion of "usages" so that an algorithm may offer different default + * settings depending on the user's intended goal. This function \ref SHOULD + * be called by all applications to initialize the configuration structure + * before specializing the configuration with application specific values. + * + * \param[in] iface Pointer to the algorithm interface to use. + * \param[out] cfg Configuration buffer to populate + * \param[in] usage End usage. Set to 0 or use codec specific values. + * + * \retval #VPX_CODEC_OK + * The configuration was populated. + * \retval #VPX_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #VPX_CODEC_INVALID_PARAM + * A parameter was NULL, or the usage value was not recognized. + */ + vpx_codec_err_t vpx_codec_enc_config_default(vpx_codec_iface_t *iface, + vpx_codec_enc_cfg_t *cfg, + unsigned int usage); + + + /*!\brief Set or change configuration + * + * Reconfigures an encoder instance according to the given configuration. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] cfg Configuration buffer to use + * + * \retval #VPX_CODEC_OK + * The configuration was populated. + * \retval #VPX_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #VPX_CODEC_INVALID_PARAM + * A parameter was NULL, or the usage value was not recognized. + */ + vpx_codec_err_t vpx_codec_enc_config_set(vpx_codec_ctx_t *ctx, + const vpx_codec_enc_cfg_t *cfg); + + + /*!\brief Get global stream headers + * + * Retrieves a stream level global header packet, if supported by the codec. + * + * \param[in] ctx Pointer to this instance's context + * + * \retval NULL + * Encoder does not support global header + * \retval Non-NULL + * Pointer to buffer containing global header packet + */ + vpx_fixed_buf_t *vpx_codec_get_global_headers(vpx_codec_ctx_t *ctx); + + +#define VPX_DL_REALTIME (1) /**< deadline parameter analogous to + * VPx REALTIME mode. */ +#define VPX_DL_GOOD_QUALITY (1000000) /**< deadline parameter analogous to + * VPx GOOD QUALITY mode. */ +#define VPX_DL_BEST_QUALITY (0) /**< deadline parameter analogous to + * VPx BEST QUALITY mode. */ + /*!\brief Encode a frame + * + * Encodes a video frame at the given "presentation time." The presentation + * time stamp (PTS) \ref MUST be strictly increasing. + * + * The encoder supports the notion of a soft real-time deadline. Given a + * non-zero value to the deadline parameter, the encoder will make a "best + * effort" guarantee to return before the given time slice expires. It is + * implicit that limiting the available time to encode will degrade the + * output quality. The encoder can be given an unlimited time to produce the + * best possible frame by specifying a deadline of '0'. This deadline + * supercedes the VPx notion of "best quality, good quality, realtime". + * Applications that wish to map these former settings to the new deadline + * based system can use the symbols #VPX_DL_REALTIME, #VPX_DL_GOOD_QUALITY, + * and #VPX_DL_BEST_QUALITY. + * + * When the last frame has been passed to the encoder, this function should + * continue to be called, with the img parameter set to NULL. This will + * signal the end-of-stream condition to the encoder and allow it to encode + * any held buffers. Encoding is complete when vpx_codec_encode() is called + * and vpx_codec_get_cx_data() returns no data. + * + * \param[in] ctx Pointer to this instance's context + * \param[in] img Image data to encode, NULL to flush. + * \param[in] pts Presentation time stamp, in timebase units. + * \param[in] duration Duration to show frame, in timebase units. + * \param[in] flags Flags to use for encoding this frame. + * \param[in] deadline Time to spend encoding, in microseconds. (0=infinite) + * + * \retval #VPX_CODEC_OK + * The configuration was populated. + * \retval #VPX_CODEC_INCAPABLE + * Interface is not an encoder interface. + * \retval #VPX_CODEC_INVALID_PARAM + * A parameter was NULL, the image format is unsupported, etc. + */ + vpx_codec_err_t vpx_codec_encode(vpx_codec_ctx_t *ctx, + const vpx_image_t *img, + vpx_codec_pts_t pts, + unsigned long duration, + vpx_enc_frame_flags_t flags, + unsigned long deadline); + + /*!\brief Set compressed data output buffer + * + * Sets the buffer that the codec should output the compressed data + * into. This call effectively sets the buffer pointer returned in the + * next VPX_CODEC_CX_FRAME_PKT packet. Subsequent packets will be + * appended into this buffer. The buffer is preserved across frames, + * so applications must periodically call this function after flushing + * the accumulated compressed data to disk or to the network to reset + * the pointer to the buffer's head. + * + * `pad_before` bytes will be skipped before writing the compressed + * data, and `pad_after` bytes will be appended to the packet. The size + * of the packet will be the sum of the size of the actual compressed + * data, pad_before, and pad_after. The padding bytes will be preserved + * (not overwritten). + * + * Note that calling this function does not guarantee that the returned + * compressed data will be placed into the specified buffer. In the + * event that the encoded data will not fit into the buffer provided, + * the returned packet \ref MAY point to an internal buffer, as it would + * if this call were never used. In this event, the output packet will + * NOT have any padding, and the application must free space and copy it + * to the proper place. This is of particular note in configurations + * that may output multiple packets for a single encoded frame (e.g., lagged + * encoding) or if the application does not reset the buffer periodically. + * + * Applications may restore the default behavior of the codec providing + * the compressed data buffer by calling this function with a NULL + * buffer. + * + * Applications \ref MUSTNOT call this function during iteration of + * vpx_codec_get_cx_data(). + * + * \param[in] ctx Pointer to this instance's context + * \param[in] buf Buffer to store compressed data into + * \param[in] pad_before Bytes to skip before writing compressed data + * \param[in] pad_after Bytes to skip after writing compressed data + * + * \retval #VPX_CODEC_OK + * The buffer was set successfully. + * \retval #VPX_CODEC_INVALID_PARAM + * A parameter was NULL, the image format is unsupported, etc. + */ + vpx_codec_err_t vpx_codec_set_cx_data_buf(vpx_codec_ctx_t *ctx, + const vpx_fixed_buf_t *buf, + unsigned int pad_before, + unsigned int pad_after); + + + /*!\brief Encoded data iterator + * + * Iterates over a list of data packets to be passed from the encoder to the + * application. The different kinds of packets available are enumerated in + * #vpx_codec_cx_pkt_kind. + * + * #VPX_CODEC_CX_FRAME_PKT packets should be passed to the application's + * muxer. Multiple compressed frames may be in the list. + * #VPX_CODEC_STATS_PKT packets should be appended to a global buffer. + * + * The application \ref MUST silently ignore any packet kinds that it does + * not recognize or support. + * + * The data buffers returned from this function are only guaranteed to be + * valid until the application makes another call to any vpx_codec_* function. + * + * \param[in] ctx Pointer to this instance's context + * \param[in,out] iter Iterator storage, initialized to NULL + * + * \return Returns a pointer to an output data packet (compressed frame data, + * two-pass statistics, etc.) or NULL to signal end-of-list. + * + */ + const vpx_codec_cx_pkt_t *vpx_codec_get_cx_data(vpx_codec_ctx_t *ctx, + vpx_codec_iter_t *iter); + + + /*!\brief Get Preview Frame + * + * Returns an image that can be used as a preview. Shows the image as it would + * exist at the decompressor. The application \ref MUST NOT write into this + * image buffer. + * + * \param[in] ctx Pointer to this instance's context + * + * \return Returns a pointer to a preview image, or NULL if no image is + * available. + * + */ + const vpx_image_t *vpx_codec_get_preview_frame(vpx_codec_ctx_t *ctx); + + + /*!@} - end defgroup encoder*/ + +#endif +#ifdef __cplusplus +} +#endif diff --git a/interface/external/LibVPX/include/vpx_image.h b/interface/external/LibVPX/include/vpx_image.h new file mode 100644 index 0000000000..c304bacd07 --- /dev/null +++ b/interface/external/LibVPX/include/vpx_image.h @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +/*!\file + * \brief Describes the vpx image descriptor and associated operations + * + */ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef VPX_IMAGE_H +#define VPX_IMAGE_H + + /*!\brief Current ABI version number + * + * \internal + * If this file is altered in any way that changes the ABI, this value + * must be bumped. Examples include, but are not limited to, changing + * types, removing or reassigning enums, adding/removing/rearranging + * fields to structures + */ +#define VPX_IMAGE_ABI_VERSION (1) /**<\hideinitializer*/ + + +#define VPX_IMG_FMT_PLANAR 0x100 /**< Image is a planar format */ +#define VPX_IMG_FMT_UV_FLIP 0x200 /**< V plane precedes U plane in memory */ +#define VPX_IMG_FMT_HAS_ALPHA 0x400 /**< Image has an alpha channel component */ + + + /*!\brief List of supported image formats */ + typedef enum vpx_img_fmt { + VPX_IMG_FMT_NONE, + VPX_IMG_FMT_RGB24, /**< 24 bit per pixel packed RGB */ + VPX_IMG_FMT_RGB32, /**< 32 bit per pixel packed 0RGB */ + VPX_IMG_FMT_RGB565, /**< 16 bit per pixel, 565 */ + VPX_IMG_FMT_RGB555, /**< 16 bit per pixel, 555 */ + VPX_IMG_FMT_UYVY, /**< UYVY packed YUV */ + VPX_IMG_FMT_YUY2, /**< YUYV packed YUV */ + VPX_IMG_FMT_YVYU, /**< YVYU packed YUV */ + VPX_IMG_FMT_BGR24, /**< 24 bit per pixel packed BGR */ + VPX_IMG_FMT_RGB32_LE, /**< 32 bit packed BGR0 */ + VPX_IMG_FMT_ARGB, /**< 32 bit packed ARGB, alpha=255 */ + VPX_IMG_FMT_ARGB_LE, /**< 32 bit packed BGRA, alpha=255 */ + VPX_IMG_FMT_RGB565_LE, /**< 16 bit per pixel, gggbbbbb rrrrrggg */ + VPX_IMG_FMT_RGB555_LE, /**< 16 bit per pixel, gggbbbbb 0rrrrrgg */ + VPX_IMG_FMT_YV12 = VPX_IMG_FMT_PLANAR | VPX_IMG_FMT_UV_FLIP | 1, /**< planar YVU */ + VPX_IMG_FMT_I420 = VPX_IMG_FMT_PLANAR | 2, + VPX_IMG_FMT_VPXYV12 = VPX_IMG_FMT_PLANAR | VPX_IMG_FMT_UV_FLIP | 3, /** < planar 4:2:0 format with vpx color space */ + VPX_IMG_FMT_VPXI420 = VPX_IMG_FMT_PLANAR | 4, + VPX_IMG_FMT_I422 = VPX_IMG_FMT_PLANAR | 5, + VPX_IMG_FMT_I444 = VPX_IMG_FMT_PLANAR | 6, + VPX_IMG_FMT_444A = VPX_IMG_FMT_PLANAR | VPX_IMG_FMT_HAS_ALPHA | 7 + } vpx_img_fmt_t; /**< alias for enum vpx_img_fmt */ + +#if !defined(VPX_CODEC_DISABLE_COMPAT) || !VPX_CODEC_DISABLE_COMPAT +#define IMG_FMT_PLANAR VPX_IMG_FMT_PLANAR /**< \deprecated Use #VPX_IMG_FMT_PLANAR */ +#define IMG_FMT_UV_FLIP VPX_IMG_FMT_UV_FLIP /**< \deprecated Use #VPX_IMG_FMT_UV_FLIP */ +#define IMG_FMT_HAS_ALPHA VPX_IMG_FMT_HAS_ALPHA /**< \deprecated Use #VPX_IMG_FMT_HAS_ALPHA */ + + /*!\brief Deprecated list of supported image formats + * \deprecated New code should use #vpx_img_fmt + */ +#define img_fmt vpx_img_fmt + /*!\brief alias for enum img_fmt. + * \deprecated New code should use #vpx_img_fmt_t + */ +#define img_fmt_t vpx_img_fmt_t + +#define IMG_FMT_NONE VPX_IMG_FMT_NONE /**< \deprecated Use #VPX_IMG_FMT_NONE */ +#define IMG_FMT_RGB24 VPX_IMG_FMT_RGB24 /**< \deprecated Use #VPX_IMG_FMT_RGB24 */ +#define IMG_FMT_RGB32 VPX_IMG_FMT_RGB32 /**< \deprecated Use #VPX_IMG_FMT_RGB32 */ +#define IMG_FMT_RGB565 VPX_IMG_FMT_RGB565 /**< \deprecated Use #VPX_IMG_FMT_RGB565 */ +#define IMG_FMT_RGB555 VPX_IMG_FMT_RGB555 /**< \deprecated Use #VPX_IMG_FMT_RGB555 */ +#define IMG_FMT_UYVY VPX_IMG_FMT_UYVY /**< \deprecated Use #VPX_IMG_FMT_UYVY */ +#define IMG_FMT_YUY2 VPX_IMG_FMT_YUY2 /**< \deprecated Use #VPX_IMG_FMT_YUY2 */ +#define IMG_FMT_YVYU VPX_IMG_FMT_YVYU /**< \deprecated Use #VPX_IMG_FMT_YVYU */ +#define IMG_FMT_BGR24 VPX_IMG_FMT_BGR24 /**< \deprecated Use #VPX_IMG_FMT_BGR24 */ +#define IMG_FMT_RGB32_LE VPX_IMG_FMT_RGB32_LE /**< \deprecated Use #VPX_IMG_FMT_RGB32_LE */ +#define IMG_FMT_ARGB VPX_IMG_FMT_ARGB /**< \deprecated Use #VPX_IMG_FMT_ARGB */ +#define IMG_FMT_ARGB_LE VPX_IMG_FMT_ARGB_LE /**< \deprecated Use #VPX_IMG_FMT_ARGB_LE */ +#define IMG_FMT_RGB565_LE VPX_IMG_FMT_RGB565_LE /**< \deprecated Use #VPX_IMG_FMT_RGB565_LE */ +#define IMG_FMT_RGB555_LE VPX_IMG_FMT_RGB555_LE /**< \deprecated Use #VPX_IMG_FMT_RGB555_LE */ +#define IMG_FMT_YV12 VPX_IMG_FMT_YV12 /**< \deprecated Use #VPX_IMG_FMT_YV12 */ +#define IMG_FMT_I420 VPX_IMG_FMT_I420 /**< \deprecated Use #VPX_IMG_FMT_I420 */ +#define IMG_FMT_VPXYV12 VPX_IMG_FMT_VPXYV12 /**< \deprecated Use #VPX_IMG_FMT_VPXYV12 */ +#define IMG_FMT_VPXI420 VPX_IMG_FMT_VPXI420 /**< \deprecated Use #VPX_IMG_FMT_VPXI420 */ +#endif /* VPX_CODEC_DISABLE_COMPAT */ + + /**\brief Image Descriptor */ + typedef struct vpx_image { + vpx_img_fmt_t fmt; /**< Image Format */ + + /* Image storage dimensions */ + unsigned int w; /**< Stored image width */ + unsigned int h; /**< Stored image height */ + + /* Image display dimensions */ + unsigned int d_w; /**< Displayed image width */ + unsigned int d_h; /**< Displayed image height */ + + /* Chroma subsampling info */ + unsigned int x_chroma_shift; /**< subsampling order, X */ + unsigned int y_chroma_shift; /**< subsampling order, Y */ + + /* Image data pointers. */ +#define VPX_PLANE_PACKED 0 /**< To be used for all packed formats */ +#define VPX_PLANE_Y 0 /**< Y (Luminance) plane */ +#define VPX_PLANE_U 1 /**< U (Chroma) plane */ +#define VPX_PLANE_V 2 /**< V (Chroma) plane */ +#define VPX_PLANE_ALPHA 3 /**< A (Transparency) plane */ +#if !defined(VPX_CODEC_DISABLE_COMPAT) || !VPX_CODEC_DISABLE_COMPAT +#define PLANE_PACKED VPX_PLANE_PACKED +#define PLANE_Y VPX_PLANE_Y +#define PLANE_U VPX_PLANE_U +#define PLANE_V VPX_PLANE_V +#define PLANE_ALPHA VPX_PLANE_ALPHA +#endif + unsigned char *planes[4]; /**< pointer to the top left pixel for each plane */ + int stride[4]; /**< stride between rows for each plane */ + + int bps; /**< bits per sample (for packed formats) */ + + /* The following member may be set by the application to associate data + * with this image. + */ + void *user_priv; /**< may be set by the application to associate data + * with this image. */ + + /* The following members should be treated as private. */ + unsigned char *img_data; /**< private */ + int img_data_owner; /**< private */ + int self_allocd; /**< private */ + } vpx_image_t; /**< alias for struct vpx_image */ + + /**\brief Representation of a rectangle on a surface */ + typedef struct vpx_image_rect { + unsigned int x; /**< leftmost column */ + unsigned int y; /**< topmost row */ + unsigned int w; /**< width */ + unsigned int h; /**< height */ + } vpx_image_rect_t; /**< alias for struct vpx_image_rect */ + + /*!\brief Open a descriptor, allocating storage for the underlying image + * + * Returns a descriptor for storing an image of the given format. The + * storage for the descriptor is allocated on the heap. + * + * \param[in] img Pointer to storage for descriptor. If this parameter + * is NULL, the storage for the descriptor will be + * allocated on the heap. + * \param[in] fmt Format for the image + * \param[in] d_w Width of the image + * \param[in] d_h Height of the image + * \param[in] align Alignment, in bytes, of the image buffer and + * each row in the image(stride). + * + * \return Returns a pointer to the initialized image descriptor. If the img + * parameter is non-null, the value of the img parameter will be + * returned. + */ + vpx_image_t *vpx_img_alloc(vpx_image_t *img, + vpx_img_fmt_t fmt, + unsigned int d_w, + unsigned int d_h, + unsigned int align); + + /*!\brief Open a descriptor, using existing storage for the underlying image + * + * Returns a descriptor for storing an image of the given format. The + * storage for descriptor has been allocated elsewhere, and a descriptor is + * desired to "wrap" that storage. + * + * \param[in] img Pointer to storage for descriptor. If this parameter + * is NULL, the storage for the descriptor will be + * allocated on the heap. + * \param[in] fmt Format for the image + * \param[in] d_w Width of the image + * \param[in] d_h Height of the image + * \param[in] align Alignment, in bytes, of each row in the image. + * \param[in] img_data Storage to use for the image + * + * \return Returns a pointer to the initialized image descriptor. If the img + * parameter is non-null, the value of the img parameter will be + * returned. + */ + vpx_image_t *vpx_img_wrap(vpx_image_t *img, + vpx_img_fmt_t fmt, + unsigned int d_w, + unsigned int d_h, + unsigned int align, + unsigned char *img_data); + + + /*!\brief Set the rectangle identifying the displayed portion of the image + * + * Updates the displayed rectangle (aka viewport) on the image surface to + * match the specified coordinates and size. + * + * \param[in] img Image descriptor + * \param[in] x leftmost column + * \param[in] y topmost row + * \param[in] w width + * \param[in] h height + * + * \return 0 if the requested rectangle is valid, nonzero otherwise. + */ + int vpx_img_set_rect(vpx_image_t *img, + unsigned int x, + unsigned int y, + unsigned int w, + unsigned int h); + + + /*!\brief Flip the image vertically (top for bottom) + * + * Adjusts the image descriptor's pointers and strides to make the image + * be referenced upside-down. + * + * \param[in] img Image descriptor + */ + void vpx_img_flip(vpx_image_t *img); + + /*!\brief Close an image descriptor + * + * Frees all allocated storage associated with an image descriptor. + * + * \param[in] img Image descriptor + */ + void vpx_img_free(vpx_image_t *img); + +#endif +#ifdef __cplusplus +} +#endif diff --git a/interface/external/LibVPX/include/vpx_integer.h b/interface/external/LibVPX/include/vpx_integer.h new file mode 100644 index 0000000000..0ccc96cd12 --- /dev/null +++ b/interface/external/LibVPX/include/vpx_integer.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2010 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#ifndef VPX_INTEGER_H +#define VPX_INTEGER_H + +/* get ptrdiff_t, size_t, wchar_t, NULL */ +#include + +#if (defined(_MSC_VER) && (_MSC_VER < 1600)) || defined(VPX_EMULATE_INTTYPES) +typedef signed char int8_t; +typedef signed short int16_t; +typedef signed int int32_t; + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; + +#if (defined(_MSC_VER) && (_MSC_VER < 1600)) +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; +#define INT64_MAX _I64_MAX +#define INT16_MAX _I16_MAX +#define INT16_MIN _I16_MIN +#endif + +#ifndef _UINTPTR_T_DEFINED +typedef size_t uintptr_t; +#endif + +#else + +/* Most platforms have the C99 standard integer types. */ + +#if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS) +#define __STDC_FORMAT_MACROS +#endif +#include + +#endif + +/* VS2010 defines stdint.h, but not inttypes.h */ +#if defined(_MSC_VER) +#define PRId64 "I64d" +#else +#include +#endif + +#endif diff --git a/interface/external/LibVPX/lib/MacOS/libvpx.a b/interface/external/LibVPX/lib/MacOS/libvpx.a new file mode 100644 index 0000000000..de2f2fe0ac Binary files /dev/null and b/interface/external/LibVPX/lib/MacOS/libvpx.a differ diff --git a/interface/external/LibVPX/lib/UNIX/libvpx.a b/interface/external/LibVPX/lib/UNIX/libvpx.a new file mode 100644 index 0000000000..4d3e32abbc Binary files /dev/null and b/interface/external/LibVPX/lib/UNIX/libvpx.a differ diff --git a/interface/resources/shaders/face.frag b/interface/resources/shaders/face.frag new file mode 100644 index 0000000000..dbabc1250c --- /dev/null +++ b/interface/resources/shaders/face.frag @@ -0,0 +1,17 @@ +#version 120 + +// +// face.frag +// fragment shader +// +// Created by Andrzej Kapolka on 7/12/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +// the color texture +uniform sampler2D colorTexture; + +void main(void) { + // for now, just modulate color + gl_FragColor = gl_Color * texture2D(colorTexture, gl_TexCoord[0].st); +} diff --git a/interface/resources/shaders/face.vert b/interface/resources/shaders/face.vert new file mode 100644 index 0000000000..358a8902fe --- /dev/null +++ b/interface/resources/shaders/face.vert @@ -0,0 +1,36 @@ +#version 120 + +// +// face.vert +// vertex shader +// +// Created by Andrzej Kapolka on 7/12/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +// the lower left texture coordinate +uniform vec2 texCoordCorner; + +// the texture coordinate vector from left to right +uniform vec2 texCoordRight; + +// the texture coordinate vector from bottom to the top +uniform vec2 texCoordUp; + +// the aspect ratio of the image +uniform float aspectRatio; + +// the depth texture +uniform sampler2D depthTexture; + +void main(void) { + gl_TexCoord[0] = vec4(texCoordCorner + gl_Vertex.x * texCoordRight + gl_Vertex.y * texCoordUp, 0.0, 1.0); + float depth = texture2D(depthTexture, gl_TexCoord[0].st).r; + + // set alpha to zero for invalid depth values + const float MIN_VISIBLE_DEPTH = 1.0 / 255.0; + const float MAX_VISIBLE_DEPTH = 254.0 / 255.0; + gl_FrontColor = vec4(1.0, 1.0, 1.0, step(MIN_VISIBLE_DEPTH, depth) * (1.0 - step(MAX_VISIBLE_DEPTH, depth))); + gl_Position = gl_ModelViewProjectionMatrix * vec4(0.5 - gl_Vertex.x, + (gl_Vertex.y - 0.5) / aspectRatio, depth * 2.0 - 2.0, 1.0); +} diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index adb88147e9..9571b44839 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -199,6 +199,7 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _mousePressed(false), _mouseVoxelScale(1.0f / 1024.0f), _justEditedVoxel(false), + _isLookingAtOtherAvatar(false), _paintOn(false), _dominantColor(0), _perfStatsOn(false), @@ -235,6 +236,31 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : NodeList::getInstance()->getNodeSocket()->setBlocking(false); } + // setup QSettings +#ifdef Q_WS_MAC + QString resourcesPath = QCoreApplication::applicationDirPath() + "/../Resources"; +#else + QString resourcesPath = QCoreApplication::applicationDirPath() + "/resources"; +#endif + + // read the ApplicationInfo.ini file for Name/Version/Domain information + QSettings applicationInfo(resourcesPath + "/info/ApplicationInfo.ini", QSettings::IniFormat); + + // set the associated application properties + applicationInfo.beginGroup("INFO"); + + setApplicationName(applicationInfo.value("name").toString()); + setApplicationVersion(applicationInfo.value("version").toString()); + setOrganizationName(applicationInfo.value("organizationName").toString()); + setOrganizationDomain(applicationInfo.value("organizationDomain").toString()); + + _settings = new QSettings(this); + + // check if there is a saved domain server hostname + // this must be done now instead of with the other setting checks to allow manual override with + // --domain or --local options + NodeList::getInstance()->loadData(_settings); + const char* domainIP = getCmdOption(argc, constArgv, "--domain"); if (domainIP) { NodeList::getInstance()->setDomainIP(domainIP); @@ -268,23 +294,6 @@ Application::Application(int& argc, char** argv, timeval &startup_time) : _window->setCentralWidget(_glWidget); -#ifdef Q_WS_MAC - QString resourcesPath = QCoreApplication::applicationDirPath() + "/../Resources"; -#else - QString resourcesPath = QCoreApplication::applicationDirPath() + "/resources"; -#endif - - // read the ApplicationInfo.ini file for Name/Version/Domain information - QSettings applicationInfo(resourcesPath + "/info/ApplicationInfo.ini", QSettings::IniFormat); - - // set the associated application properties - applicationInfo.beginGroup("INFO"); - - setApplicationName(applicationInfo.value("name").toString()); - setApplicationVersion(applicationInfo.value("version").toString()); - setOrganizationName(applicationInfo.value("organizationName").toString()); - setOrganizationDomain(applicationInfo.value("organizationDomain").toString()); - #if defined(Q_WS_MAC) && defined(QT_NO_DEBUG) // if this is a release OS X build use fervor to check for an update FvUpdater::sharedUpdater()->SetFeedURL("https://s3-us-west-1.amazonaws.com/highfidelity/appcast.xml"); @@ -929,6 +938,36 @@ void Application::sendPingPackets() { nodesToPing, sizeof(nodesToPing)); } +void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data) { + unsigned char packet[MAX_PACKET_SIZE]; + unsigned char* packetPosition = packet; + + packetPosition += populateTypeAndVersion(packetPosition, PACKET_TYPE_AVATAR_FACE_VIDEO); + + *(uint16_t*)packetPosition = NodeList::getInstance()->getOwnerID(); + packetPosition += sizeof(uint16_t); + + *(uint32_t*)packetPosition = frameCount; + packetPosition += sizeof(uint32_t); + + *(uint32_t*)packetPosition = data.size(); + packetPosition += sizeof(uint32_t); + + uint32_t* offsetPosition = (uint32_t*)packetPosition; + packetPosition += sizeof(uint32_t); + + int headerSize = packetPosition - packet; + + // break the data up into submessages of the maximum size + *offsetPosition = 0; + while (*offsetPosition < data.size()) { + int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize); + memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize); + getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1); + *offsetPosition += payloadSize; + } +} + // Every second, check the frame rates and other stuff void Application::timer() { gettimeofday(&_timerEnd, NULL); @@ -1038,7 +1077,7 @@ void Application::sendAvatarVoxelURLMessage(const QUrl& url) { controlledBroadcastToNodes((unsigned char*)message.data(), message.size(), & NODE_TYPE_AVATAR_MIXER, 1); } -void Application::processAvatarVoxelURLMessage(unsigned char *packetData, size_t dataBytes) { +static Avatar* processAvatarMessageHeader(unsigned char*& packetData, size_t& dataBytes) { // skip the header int numBytesPacketHeader = numBytesForPacketHeader(packetData); packetData += numBytesPacketHeader; @@ -1052,18 +1091,31 @@ void Application::processAvatarVoxelURLMessage(unsigned char *packetData, size_t // make sure the node exists Node* node = NodeList::getInstance()->nodeWithID(nodeID); if (!node || !node->getLinkedData()) { - return; + return NULL; } Avatar* avatar = static_cast(node->getLinkedData()); - if (!avatar->isInitialized()) { - return; // wait until initialized - } + return avatar->isInitialized() ? avatar : NULL; +} + +void Application::processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes) { + Avatar* avatar = processAvatarMessageHeader(packetData, dataBytes); + if (!avatar) { + return; + } QUrl url = QUrl::fromEncoded(QByteArray((char*)packetData, dataBytes)); // invoke the set URL function on the simulate/render thread QMetaObject::invokeMethod(avatar->getVoxels(), "setVoxelURL", Q_ARG(QUrl, url)); } +void Application::processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes) { + Avatar* avatar = processAvatarMessageHeader(packetData, dataBytes); + if (!avatar) { + return; + } + avatar->getHead().getFace().processVideoMessage(packetData, dataBytes); +} + void Application::checkBandwidthMeterClick() { // ... to be called upon button release @@ -1143,26 +1195,29 @@ void Application::editPreferences() { return; } + QByteArray newHostname; - const char* newHostname = domainServerHostname->text().toLocal8Bit().data(); - + if (domainServerHostname->text().size() > 0) { + // the user input a new hostname, use that + newHostname = domainServerHostname->text().toAscii(); + } else { + // the user left the field blank, use the default hostname + newHostname = QByteArray(DEFAULT_DOMAIN_HOSTNAME); + } + // check if the domain server hostname is new - if (memcmp(NodeList::getInstance()->getDomainHostname(), newHostname, sizeof(&newHostname)) != 0) { - // if so we need to clear the nodelist and delete the local voxels - Node *voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); - - if (voxelServer) { - voxelServer->lock(); - } - - _voxels.killLocalVoxels(); - - if (voxelServer) { - voxelServer->unlock(); - } + if (memcmp(NodeList::getInstance()->getDomainHostname(), newHostname.constData(), newHostname.size()) != 0) { NodeList::getInstance()->clear(); - NodeList::getInstance()->setDomainHostname(newHostname); + + // kill the local voxels + _voxels.killLocalVoxels(); + + // reset the environment to default + _environment.resetToDefault(); + + // set the new hostname + NodeList::getInstance()->setDomainHostname(newHostname.constData()); } QUrl url(avatarURL->text()); @@ -1670,10 +1725,13 @@ void Application::initMenu() { (_renderAvatarBalls = renderMenu->addAction("Avatar as Balls"))->setCheckable(true); _renderAvatarBalls->setChecked(false); renderMenu->addAction("Cycle Voxel Mode", _myAvatar.getVoxels(), SLOT(cycleMode())); + renderMenu->addAction("Cycle Face Mode", &_myAvatar.getHead().getFace(), SLOT(cycleRenderMode())); (_renderFrameTimerOn = renderMenu->addAction("Show Timer"))->setCheckable(true); _renderFrameTimerOn->setChecked(false); (_renderLookatOn = renderMenu->addAction("Lookat Vectors"))->setCheckable(true); _renderLookatOn->setChecked(false); + (_renderLookatIndicatorOn = renderMenu->addAction("Lookat Indicator"))->setCheckable(true); + _renderLookatIndicatorOn->setChecked(true); (_manualFirstPerson = renderMenu->addAction( "First Person", this, SLOT(setRenderFirstPerson(bool)), Qt::Key_P))->setCheckable(true); (_manualThirdPerson = renderMenu->addAction( @@ -1786,7 +1844,6 @@ void Application::initMenu() { settingsMenu->addAction("Export settings", this, SLOT(exportSettings())); _networkAccessManager = new QNetworkAccessManager(this); - _settings = new QSettings(this); } void Application::updateFrustumRenderModeAction() { @@ -1885,6 +1942,7 @@ bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& m glm::vec3 headPosition = avatar->getHead().getPosition(); if (rayIntersectsSphere(mouseRayOrigin, mouseRayDirection, headPosition, HEAD_SPHERE_RADIUS)) { eyePosition = avatar->getHead().getEyeLevelPosition(); + _lookatOtherPosition = headPosition; return true; } } @@ -1892,6 +1950,15 @@ bool Application::isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& m return false; } +void Application::renderLookatIndicator(glm::vec3 pointOfInterest, Camera& whichCamera) { + + const float DISTANCE_FROM_HEAD_SPHERE = 0.1f; + const float YELLOW[] = { 1.0f, 1.0f, 0.0f }; + glm::vec3 haloOrigin(pointOfInterest.x, pointOfInterest.y + DISTANCE_FROM_HEAD_SPHERE, pointOfInterest.z); + glColor3f(YELLOW[0], YELLOW[1], YELLOW[2]); + renderCircle(haloOrigin, 0.1f, glm::vec3(0.0f, 1.0f, 0.0f), 30); +} + void Application::update(float deltaTime) { // Use Transmitter Hand to move hand if connected, else use mouse if (_myTransmitter.isConnected()) { @@ -1919,7 +1986,7 @@ void Application::update(float deltaTime) { // Set where I am looking based on my mouse ray (so that other people can see) glm::vec3 eyePosition; - if (isLookingAtOtherAvatar(mouseRayOrigin, mouseRayDirection, eyePosition)) { + if (_isLookingAtOtherAvatar = isLookingAtOtherAvatar(mouseRayOrigin, mouseRayDirection, eyePosition)) { // If the mouse is over another avatar's head... glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); @@ -1936,7 +2003,7 @@ void Application::update(float deltaTime) { glm::vec3 front = orientation * IDENTITY_FRONT; glm::vec3 up = orientation * IDENTITY_UP; glm::vec3 towardVoxel = getMouseVoxelWorldCoordinates(_mouseVoxelDragging) - - _myAvatar.getCameraPosition(); + - _myAvatar.getCameraPosition(); // is this an error? getCameraPosition dne towardVoxel = front * glm::length(towardVoxel); glm::vec3 lateralToVoxel = glm::cross(up, glm::normalize(towardVoxel)) * glm::length(towardVoxel); _voxelThrust = glm::vec3(0, 0, 0); @@ -2036,10 +2103,8 @@ void Application::update(float deltaTime) { // Leap finger-sensing device LeapManager::enableFakeFingers(_simulateLeapHand->isChecked() || _testRaveGlove->isChecked()); - LeapManager::nextFrame(); _myAvatar.getHand().setRaveGloveActive(_testRaveGlove->isChecked()); - _myAvatar.getHand().setLeapFingers(LeapManager::getFingerTips(), LeapManager::getFingerRoots()); - _myAvatar.getHand().setLeapHands(LeapManager::getHandPositions(), LeapManager::getHandNormals()); + LeapManager::nextFrame(_myAvatar); // Read serial port interface devices if (_serialHeadSensor.isActive()) { @@ -2183,7 +2248,7 @@ void Application::updateAvatar(float deltaTime) { _viewFrustum.computePickRay(MIDPOINT_OF_SCREEN, MIDPOINT_OF_SCREEN, screenCenterRayOrigin, screenCenterRayDirection); glm::vec3 eyePosition; - if (isLookingAtOtherAvatar(screenCenterRayOrigin, screenCenterRayDirection, eyePosition)) { + if (_isLookingAtOtherAvatar = isLookingAtOtherAvatar(screenCenterRayOrigin, screenCenterRayDirection, eyePosition)) { glm::vec3 myLookAtFromMouse(eyePosition); _myAvatar.getHead().setLookAtPosition(myLookAtFromMouse); } @@ -2211,7 +2276,7 @@ void Application::updateAvatar(float deltaTime) { // actually need to calculate the view frustum planes to send these details // to the server. loadViewFrustum(_myCamera, _viewFrustum); - _myAvatar.setCameraPosition(_viewFrustum.getPosition()); + _myAvatar.setCameraPosition(_viewFrustum.getPosition()); // setCameraPosition() dne _myAvatar.setCameraOrientation(_viewFrustum.getOrientation()); _myAvatar.setCameraFov(_viewFrustum.getFieldOfView()); _myAvatar.setCameraAspectRatio(_viewFrustum.getAspectRatio()); @@ -2582,6 +2647,10 @@ void Application::displaySide(Camera& whichCamera) { } _myAvatar.render(_lookingInMirror->isChecked(), _renderAvatarBalls->isChecked()); _myAvatar.setDisplayingLookatVectors(_renderLookatOn->isChecked()); + + if (_renderLookatIndicatorOn->isChecked() && _isLookingAtOtherAvatar) { + renderLookatIndicator(_lookatOtherPosition, whichCamera); + } } if (TESTING_PARTICLE_SYSTEM) { @@ -3352,7 +3421,7 @@ void* Application::networkReceive(void* args) { case PACKET_TYPE_ENVIRONMENT_DATA: { if (app->_renderVoxels->isChecked()) { Node* voxelServer = NodeList::getInstance()->soloNodeOfType(NODE_TYPE_VOXEL_SERVER); - if (voxelServer) { + if (voxelServer && socketMatch(voxelServer->getActiveSocket(), &senderAddress)) { voxelServer->lock(); if (app->_incomingPacket[0] == PACKET_TYPE_ENVIRONMENT_DATA) { @@ -3375,6 +3444,9 @@ void* Application::networkReceive(void* args) { case PACKET_TYPE_AVATAR_VOXEL_URL: processAvatarVoxelURLMessage(app->_incomingPacket, bytesReceived); break; + case PACKET_TYPE_AVATAR_FACE_VIDEO: + processAvatarFaceVideoMessage(app->_incomingPacket, bytesReceived); + break; default: NodeList::getInstance()->processNodeData(&senderAddress, app->_incomingPacket, bytesReceived); break; @@ -3456,7 +3528,7 @@ void Application::saveSettings(QSettings* settings) { if (!settings) { settings = getSettings(); } - + settings->setValue("headCameraPitchYawScale", _headCameraPitchYawScale); settings->setValue("audioJitterBufferSamples", _audioJitterBufferSamples); settings->setValue("horizontalFieldOfView", _horizontalFieldOfView); @@ -3471,6 +3543,9 @@ void Application::saveSettings(QSettings* settings) { scanMenuBar(&Application::saveAction, settings); getAvatar()->saveData(settings); _swatch.saveData(settings); + + // ask the NodeList to save its data + NodeList::getInstance()->saveData(settings); } void Application::importSettings() { diff --git a/interface/src/Application.h b/interface/src/Application.h index 9333808a27..d29328c0b5 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -20,16 +20,15 @@ #include #include +#include #ifndef _WIN32 #include "Audio.h" #endif -#include "Avatar.h" #include "BandwidthMeter.h" #include "Camera.h" #include "Environment.h" -#include "HandControl.h" #include "PacketHeaders.h" #include "ParticleSystem.h" #include "renderer/GeometryCache.h" @@ -37,12 +36,13 @@ #include "Stars.h" #include "Swatch.h" #include "ToolsPalette.h" -#include "ui/ChatEntry.h" -#include "ui/BandwidthDialog.h" #include "ViewFrustum.h" #include "VoxelSystem.h" #include "Webcam.h" - +#include "avatar/Avatar.h" +#include "avatar/HandControl.h" +#include "ui/BandwidthDialog.h" +#include "ui/ChatEntry.h" class QAction; class QActionGroup; @@ -103,6 +103,10 @@ public: QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; } GeometryCache* getGeometryCache() { return &_geometryCache; } + +public slots: + + void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data); void setGroundPlaneImpact(float groundPlaneImpact) { _groundPlaneImpact = groundPlaneImpact; } @@ -185,7 +189,8 @@ private: static bool sendVoxelsOperation(VoxelNode* node, void* extraData); static void sendVoxelEditMessage(PACKET_TYPE type, VoxelDetail& detail); static void sendAvatarVoxelURLMessage(const QUrl& url); - static void processAvatarVoxelURLMessage(unsigned char *packetData, size_t dataBytes); + static void processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes); + static void processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes); static void sendPingPackets(); void initMenu(); @@ -195,6 +200,7 @@ private: void update(float deltaTime); bool isLookingAtOtherAvatar(glm::vec3& mouseRayOrigin, glm::vec3& mouseRayDirection, glm::vec3& eyePosition); + void renderLookatIndicator(glm::vec3 pointOfInterest, Camera& whichCamera); void updateAvatar(float deltaTime); void loadViewFrustum(Camera& camera, ViewFrustum& viewFrustum); @@ -250,6 +256,7 @@ private: QAction* _renderStatsOn; // Whether to show onscreen text overlay with stats QAction* _renderFrameTimerOn; // Whether to show onscreen text overlay with stats QAction* _renderLookatOn; // Whether to show lookat vectors from avatar eyes if looking at something + QAction* _renderLookatIndicatorOn; QAction* _manualFirstPerson; // Whether to force first-person mode QAction* _manualThirdPerson; // Whether to force third-person mode QAction* _logOn; // Whether to show on-screen log @@ -363,6 +370,9 @@ private: float _mouseVoxelScale; // the scale for adding/removing voxels glm::vec3 _lastMouseVoxelPos; // the position of the last mouse voxel edit bool _justEditedVoxel; // set when we've just added/deleted/colored a voxel + + bool _isLookingAtOtherAvatar; + glm::vec3 _lookatOtherPosition; bool _paintOn; // Whether to paint voxels as you fly around unsigned char _dominantColor; // The dominant color of the voxel we're painting diff --git a/interface/src/Environment.cpp b/interface/src/Environment.cpp index 35ed121e38..adfb1cfb3b 100644 --- a/interface/src/Environment.cpp +++ b/interface/src/Environment.cpp @@ -47,6 +47,11 @@ void Environment::init() { _data[getZeroAddress()][0]; } +void Environment::resetToDefault() { + _data.clear(); + _data[getZeroAddress()][0]; +} + void Environment::renderAtmospheres(Camera& camera) { // get the lock for the duration of the call QMutexLocker locker(&_mutex); diff --git a/interface/src/Environment.h b/interface/src/Environment.h index f711f0c38c..7bdbfa600b 100644 --- a/interface/src/Environment.h +++ b/interface/src/Environment.h @@ -24,6 +24,7 @@ class Environment { public: void init(); + void resetToDefault(); void renderAtmospheres(Camera& camera); glm::vec3 getGravity (const glm::vec3& position); diff --git a/interface/src/LeapManager.cpp b/interface/src/LeapManager.cpp index fe47e527fa..a941d0a221 100755 --- a/interface/src/LeapManager.cpp +++ b/interface/src/LeapManager.cpp @@ -7,6 +7,7 @@ // #include "LeapManager.h" +#include "avatar/Avatar.h" #include #include // needed for RTLD_LAZY #include @@ -16,48 +17,15 @@ bool LeapManager::_doFakeFingers = false; Leap::Controller* LeapManager::_controller = NULL; HifiLeapListener* LeapManager::_listener = NULL; -namespace { -glm::vec3 fakeHandOffset(0.0f, 50.0f, 50.0f); -} // end anonymous namespace - class HifiLeapListener : public Leap::Listener { public: HifiLeapListener() {} virtual ~HifiLeapListener() {} Leap::Frame lastFrame; - std::vector fingerTips; - std::vector fingerRoots; - std::vector handPositions; - std::vector handNormals; - virtual void onFrame(const Leap::Controller& controller) { #ifndef LEAP_STUBS - Leap::Frame frame = controller.frame(); - int numFingers = frame.fingers().count(); - fingerTips.resize(numFingers); - fingerRoots.resize(numFingers); - for (int i = 0; i < numFingers; ++i) { - const Leap::Finger& thisFinger = frame.fingers()[i]; - const Leap::Vector pos = thisFinger.stabilizedTipPosition(); - fingerTips[i] = glm::vec3(pos.x, pos.y, pos.z); - - const Leap::Vector root = pos - thisFinger.direction() * thisFinger.length(); - fingerRoots[i] = glm::vec3(root.x, root.y, root.z); - } - - int numHands = frame.hands().count(); - handPositions.resize(numHands); - handNormals.resize(numHands); - for (int i = 0; i < numHands; ++i) { - const Leap::Hand& thisHand = frame.hands()[i]; - const Leap::Vector pos = thisHand.palmPosition(); - handPositions[i] = glm::vec3(pos.x, pos.y, pos.z); - - const Leap::Vector norm = thisHand.palmNormal(); - handNormals[i] = glm::vec3(norm.x, norm.y, norm.z); - } - lastFrame = frame; + lastFrame = controller.frame(); #endif } @@ -80,10 +48,195 @@ void LeapManager::terminate() { _controller = NULL; } -void LeapManager::nextFrame() { +void LeapManager::nextFrame(Avatar& avatar) { + // Apply the frame data directly to the avatar. + Hand& hand = avatar.getHand(); + + // If we actually get valid Leap data, this will be set to true; + bool gotRealData = false; + if (controllersExist()) { _listener->onFrame(*_controller); } + +#ifndef LEAP_STUBS + if (controllersExist()) { + gotRealData = true; + // First, see which palms and fingers are still valid. + Leap::Frame& frame = _listener->lastFrame; + + // Note that this is O(n^2) at worst, but n is very small. + + // After this many frames of no data, assume the digit is lost. + const int assumeLostAfterFrameCount = 10; + + // Increment our frame data counters + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + palm.incrementFramesWithoutData(); + if (palm.getFramesWithoutData() > assumeLostAfterFrameCount) { + palm.setActive(false); + } + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.incrementFramesWithoutData(); + if (finger.getFramesWithoutData() > assumeLostAfterFrameCount) { + finger.setActive(false); + } + } + } + + size_t numLeapHands = frame.hands().count(); + std::vector palmAssignment(numLeapHands); + + // Look for matches + for (size_t index = 0; index < numLeapHands; ++index) { + PalmData* takeoverCandidate = NULL; + palmAssignment[index] = NULL; + Leap::Hand leapHand = frame.hands()[index]; + int id = leapHand.id(); + if (leapHand.isValid()) { + for (size_t i = 0; i < hand.getNumPalms() && palmAssignment[index] == NULL; ++i) { + PalmData& palm = hand.getPalms()[i]; + if (palm.getLeapID() == id) { + // Found hand with the same ID. We're set! + palmAssignment[index] = &palm; + palm.resetFramesWithoutData(); + } + else if (palm.getFramesWithoutData() > assumeLostAfterFrameCount) { + takeoverCandidate = &palm; + } + } + if (palmAssignment[index] == NULL) { + palmAssignment[index] = takeoverCandidate; + } + if (palmAssignment[index] == NULL) { + palmAssignment[index] = &hand.addNewPalm(); + } + } + } + + // Apply the assignments + for (size_t index = 0; index < numLeapHands; ++index) { + if (palmAssignment[index]) { + Leap::Hand leapHand = frame.hands()[index]; + PalmData& palm = *(palmAssignment[index]); + + palm.resetFramesWithoutData(); + palm.setLeapID(leapHand.id()); + palm.setActive(true); + const Leap::Vector pos = leapHand.palmPosition(); + const Leap::Vector normal = leapHand.palmNormal(); + palm.setRawPosition(glm::vec3(pos.x, pos.y, pos.z)); + palm.setRawNormal(glm::vec3(normal.x, normal.y, normal.z)); + } + } + + // Look for fingers per palm + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + if (palm.isActive()) { + Leap::Hand leapHand = frame.hand(palm.getLeapID()); + if (leapHand.isValid()) { + int numLeapFingers = leapHand.fingers().count(); + std::vector fingerAssignment(numLeapFingers); + + + // Look for matches + for (size_t index = 0; index < numLeapFingers; ++index) { + FingerData* takeoverCandidate = NULL; + fingerAssignment[index] = NULL; + Leap::Finger leapFinger = leapHand.fingers()[index]; + int id = leapFinger.id(); + if (leapFinger.isValid()) { + for (size_t f = 0; f < palm.getNumFingers() && fingerAssignment[index] == NULL; ++f) { + FingerData& finger = palm.getFingers()[f]; + if (finger.getLeapID() == id) { + // Found hand with the same ID. We're set! + fingerAssignment[index] = &finger; + } + else if (finger.getFramesWithoutData() > assumeLostAfterFrameCount) { + takeoverCandidate = &finger; + } + } + // If we didn't find a match, but we found an unused finger, us it. + if (fingerAssignment[index] == NULL) { + fingerAssignment[index] = takeoverCandidate; + } + } + } + + // Apply the assignments + for (size_t index = 0; index < numLeapFingers; ++index) { + if (fingerAssignment[index]) { + Leap::Finger leapFinger = leapHand.fingers()[index]; + FingerData& finger = *(fingerAssignment[index]); + + finger.resetFramesWithoutData(); + finger.setLeapID(leapFinger.id()); + finger.setActive(true); + const Leap::Vector tip = leapFinger.stabilizedTipPosition(); + const Leap::Vector root = tip - leapFinger.direction() * leapFinger.length(); + finger.setRawTipPosition(glm::vec3(tip.x, tip.y, tip.z)); + finger.setRawRootPosition(glm::vec3(root.x, root.y, root.z)); + } + } + } + } + } + } +#endif + if (!gotRealData) { + if (_doFakeFingers) { + // There's no real Leap data and we need to fake it. + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + static const glm::vec3 fakeHandOffsets[] = { + glm::vec3( -500.0f, 50.0f, 50.0f), + glm::vec3( 0.0f, 50.0f, 50.0f) + }; + static const glm::vec3 fakeHandFingerMirrors[] = { + glm::vec3( -1.0f, 1.0f, 1.0f), + glm::vec3( 1.0f, 1.0f, 1.0f) + }; + static const glm::vec3 fakeFingerPositions[] = { + glm::vec3( -60.0f, 0.0f, -40.0f), + glm::vec3( -20.0f, 0.0f, -60.0f), + glm::vec3( 20.0f, 0.0f, -60.0f), + glm::vec3( 60.0f, 0.0f, -40.0f), + glm::vec3( -50.0f, 0.0f, 30.0f) + }; + + PalmData& palm = hand.getPalms()[i]; + palm.setActive(true); + // Simulated data + + palm.setRawPosition(glm::vec3( 0.0f, 0.0f, 0.0f) + fakeHandOffsets[i]); + palm.setRawNormal(glm::vec3(0.0f, 1.0f, 0.0f)); + + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setActive(true); + const float tipScale = 1.5f; + const float rootScale = 0.75f; + glm::vec3 fingerPos = fakeFingerPositions[f] * fakeHandFingerMirrors[i]; + finger.setRawTipPosition(fingerPos * tipScale + fakeHandOffsets[i]); + finger.setRawRootPosition(fingerPos * rootScale + fakeHandOffsets[i]); + } + } + } + else { + // Just deactivate everything. + for (size_t i = 0; i < hand.getNumPalms(); ++i) { + PalmData& palm = hand.getPalms()[i]; + palm.setActive(false); + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setActive(false); + } + } + } + } + hand.updateFingerTrails(); } void LeapManager::enableFakeFingers(bool enable) { @@ -98,77 +251,6 @@ bool LeapManager::controllersExist() { #endif } -const std::vector& LeapManager::getFingerTips() { - if (controllersExist()) { - return _listener->fingerTips; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - float scale = 1.5f; - stubData.push_back(glm::vec3( -60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -50.0f, 0.0f, 30.0f) * scale + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getFingerRoots() { - if (controllersExist()) { - return _listener->fingerRoots; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - float scale = 0.75f; - stubData.push_back(glm::vec3( -60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 20.0f, 0.0f, -60.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( 60.0f, 0.0f, -40.0f) * scale + fakeHandOffset); - stubData.push_back(glm::vec3( -50.0f, 0.0f, 30.0f) * scale + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getHandPositions() { - if (controllersExist()) { - return _listener->handPositions; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - glm::vec3 handOffset(0.0f, 50.0f, 50.0f); - stubData.push_back(glm::vec3( 0.0f, 0.0f, 0.0f) + fakeHandOffset); - } - return stubData; - } -} - -const std::vector& LeapManager::getHandNormals() { - if (controllersExist()) { - return _listener->handNormals; - } - else { - static std::vector stubData; - stubData.clear(); - if (_doFakeFingers) { - // Simulated data - stubData.push_back(glm::vec3(0.0f, 1.0f, 0.0f)); - } - return stubData; - } -} - std::string LeapManager::statusString() { std::stringstream leapString; #ifndef LEAP_STUBS diff --git a/interface/src/LeapManager.h b/interface/src/LeapManager.h index e6ac304677..11dbefe849 100755 --- a/interface/src/LeapManager.h +++ b/interface/src/LeapManager.h @@ -13,6 +13,7 @@ #include #include +class Avatar; class HifiLeapListener; namespace Leap { class Controller; @@ -20,7 +21,7 @@ namespace Leap { class LeapManager { public: - static void nextFrame(); // called once per frame to get new Leap data + static void nextFrame(Avatar& avatar); // called once per frame to get new Leap data static bool controllersExist(); // Returns true if there's at least one active Leap plugged in static void enableFakeFingers(bool enable); // put fake data in if there's no Leap plugged in static const std::vector& getFingerTips(); diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 6aff15240e..0f2c3a8955 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -541,8 +541,6 @@ void runTimingTests() { gettimeofday(&endTime, NULL); elapsedMsecs = diffclock(&startTime, &endTime); qDebug("vec3 assign and dot() usecs: %f\n", 1000.0f * elapsedMsecs / (float) numTests); - - } float loadSetting(QSettings* settings, const char* name, float defaultValue) { diff --git a/interface/src/Util.h b/interface/src/Util.h index 8bc77a98ea..fe59637a42 100644 --- a/interface/src/Util.h +++ b/interface/src/Util.h @@ -71,7 +71,6 @@ void renderCircle(glm::vec3 position, float radius, glm::vec3 surfaceNormal, int void runTimingTests(); - float loadSetting(QSettings* settings, const char* name, float defaultValue); bool rayIntersectsSphere(glm::vec3& rayStarting, glm::vec3& rayNormalizedDirection, glm::vec3& sphereCenter, double sphereRadius); diff --git a/interface/src/Webcam.cpp b/interface/src/Webcam.cpp index f9dd5ffe2c..e37ca6c70e 100644 --- a/interface/src/Webcam.cpp +++ b/interface/src/Webcam.cpp @@ -8,12 +8,15 @@ #include #include -#include +#include +#include #ifdef __APPLE__ #include #endif +#include + #include "Application.h" #include "Webcam.h" @@ -25,11 +28,11 @@ using namespace xn; #endif // register types with Qt metatype system -int jointVectorMetaType = qRegisterMetaType("JointVector"); +int jointVectorMetaType = qRegisterMetaType("JointVector"); int matMetaType = qRegisterMetaType("cv::Mat"); int rotatedRectMetaType = qRegisterMetaType("cv::RotatedRect"); -Webcam::Webcam() : _enabled(false), _active(false), _frameTextureID(0), _depthTextureID(0) { +Webcam::Webcam() : _enabled(false), _active(false), _colorTextureID(0), _depthTextureID(0) { // the grabber simply runs as fast as possible _grabber = new FrameGrabber(); _grabber->moveToThread(&_grabberThread); @@ -64,13 +67,13 @@ void Webcam::reset() { } void Webcam::renderPreview(int screenWidth, int screenHeight) { - if (_enabled && _frameTextureID != 0) { - glBindTexture(GL_TEXTURE_2D, _frameTextureID); + if (_enabled && _colorTextureID != 0) { + glBindTexture(GL_TEXTURE_2D, _colorTextureID); glEnable(GL_TEXTURE_2D); glColor3f(1.0f, 1.0f, 1.0f); glBegin(GL_QUADS); const int PREVIEW_HEIGHT = 200; - int previewWidth = _frameWidth * PREVIEW_HEIGHT / _frameHeight; + int previewWidth = _textureSize.width * PREVIEW_HEIGHT / _textureSize.height; int top = screenHeight - 600; int left = screenWidth - previewWidth - 10; @@ -87,16 +90,14 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) { if (_depthTextureID != 0) { glBindTexture(GL_TEXTURE_2D, _depthTextureID); glBegin(GL_QUADS); - int depthPreviewWidth = _depthWidth * PREVIEW_HEIGHT / _depthHeight; - int depthLeft = screenWidth - depthPreviewWidth - 10; glTexCoord2f(0, 0); - glVertex2f(depthLeft, top - PREVIEW_HEIGHT); + glVertex2f(left, top - PREVIEW_HEIGHT); glTexCoord2f(1, 0); - glVertex2f(depthLeft + depthPreviewWidth, top - PREVIEW_HEIGHT); + glVertex2f(left + previewWidth, top - PREVIEW_HEIGHT); glTexCoord2f(1, 1); - glVertex2f(depthLeft + depthPreviewWidth, top); + glVertex2f(left + previewWidth, top); glTexCoord2f(0, 1); - glVertex2f(depthLeft, top); + glVertex2f(left, top); glEnd(); glBindTexture(GL_TEXTURE_2D, 0); @@ -106,10 +107,10 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) { glColor3f(1.0f, 0.0f, 0.0f); glPointSize(4.0f); glBegin(GL_POINTS); - float projectedScale = PREVIEW_HEIGHT / (float)_depthHeight; + float projectedScale = PREVIEW_HEIGHT / _textureSize.height; foreach (const Joint& joint, _joints) { if (joint.isValid) { - glVertex2f(depthLeft + joint.projected.x * projectedScale, + glVertex2f(left + joint.projected.x * projectedScale, top - PREVIEW_HEIGHT + joint.projected.y * projectedScale); } } @@ -125,15 +126,16 @@ void Webcam::renderPreview(int screenWidth, int screenHeight) { glBegin(GL_LINE_LOOP); Point2f facePoints[4]; _faceRect.points(facePoints); - float xScale = previewWidth / (float)_frameWidth; - float yScale = PREVIEW_HEIGHT / (float)_frameHeight; + float xScale = previewWidth / _textureSize.width; + float yScale = PREVIEW_HEIGHT / _textureSize.height; glVertex2f(left + facePoints[0].x * xScale, top + facePoints[0].y * yScale); glVertex2f(left + facePoints[1].x * xScale, top + facePoints[1].y * yScale); glVertex2f(left + facePoints[2].x * xScale, top + facePoints[2].y * yScale); glVertex2f(left + facePoints[3].x * xScale, top + facePoints[3].y * yScale); glEnd(); - char fps[20]; + const int MAX_FPS_CHARACTERS = 30; + char fps[MAX_FPS_CHARACTERS]; sprintf(fps, "FPS: %d", (int)(roundf(_frameCount * 1000000.0f / (usecTimestampNow() - _startTimestamp)))); drawtext(left, top + PREVIEW_HEIGHT + 20, 0.10, 0, 1, 0, fps); } @@ -147,20 +149,21 @@ Webcam::~Webcam() { delete _grabber; } -void Webcam::setFrame(const Mat& frame, int format, const Mat& depth, const RotatedRect& faceRect, const JointVector& joints) { - IplImage image = frame; - glPixelStorei(GL_UNPACK_ROW_LENGTH, image.widthStep / 3); - if (_frameTextureID == 0) { - glGenTextures(1, &_frameTextureID); - glBindTexture(GL_TEXTURE_2D, _frameTextureID); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, _frameWidth = image.width, _frameHeight = image.height, 0, format, - GL_UNSIGNED_BYTE, image.imageData); +void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const RotatedRect& faceRect, const JointVector& joints) { + IplImage colorImage = color; + glPixelStorei(GL_UNPACK_ROW_LENGTH, colorImage.widthStep / 3); + if (_colorTextureID == 0) { + glGenTextures(1, &_colorTextureID); + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, _textureSize.width = colorImage.width, _textureSize.height = colorImage.height, + 0, format, GL_UNSIGNED_BYTE, colorImage.imageData); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - qDebug("Capturing video at %dx%d.\n", _frameWidth, _frameHeight); + qDebug("Capturing video at %gx%g.\n", _textureSize.width, _textureSize.height); } else { - glBindTexture(GL_TEXTURE_2D, _frameTextureID); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _frameWidth, _frameHeight, format, GL_UNSIGNED_BYTE, image.imageData); + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, format, + GL_UNSIGNED_BYTE, colorImage.imageData); } if (!depth.empty()) { @@ -169,14 +172,14 @@ void Webcam::setFrame(const Mat& frame, int format, const Mat& depth, const Rota if (_depthTextureID == 0) { glGenTextures(1, &_depthTextureID); glBindTexture(GL_TEXTURE_2D, _depthTextureID); - glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, _depthWidth = depthImage.width, _depthHeight = depthImage.height, 0, + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depthImage.width, depthImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, depthImage.imageData); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - qDebug("Capturing depth at %dx%d.\n", _depthWidth, _depthHeight); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); } else { glBindTexture(GL_TEXTURE_2D, _depthTextureID); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _depthWidth, _depthHeight, GL_LUMINANCE, + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, _textureSize.width, _textureSize.height, GL_LUMINANCE, GL_UNSIGNED_BYTE, depthImage.imageData); } } @@ -225,30 +228,23 @@ void Webcam::setFrame(const Mat& frame, int format, const Mat& depth, const Rota _estimatedPosition = _estimatedJoints[AVATAR_JOINT_HEAD_BASE].position; } else { - // roll is just the angle of the face rect (correcting for 180 degree rotations) - float roll = faceRect.angle; - if (roll < -90.0f) { - roll += 180.0f; - - } else if (roll > 90.0f) { - roll -= 180.0f; - } + // roll is just the angle of the face rect const float ROTATION_SMOOTHING = 0.95f; - _estimatedRotation.z = glm::mix(roll, _estimatedRotation.z, ROTATION_SMOOTHING); + _estimatedRotation.z = glm::mix(_faceRect.angle, _estimatedRotation.z, ROTATION_SMOOTHING); // determine position based on translation and scaling of the face rect if (_initialFaceRect.size.area() == 0) { - _initialFaceRect = faceRect; + _initialFaceRect = _faceRect; _estimatedPosition = glm::vec3(); } else { - float proportion = sqrtf(_initialFaceRect.size.area() / (float)faceRect.size.area()); + float proportion = sqrtf(_initialFaceRect.size.area() / (float)_faceRect.size.area()); const float DISTANCE_TO_CAMERA = 0.333f; const float POSITION_SCALE = 0.5f; float z = DISTANCE_TO_CAMERA * proportion - DISTANCE_TO_CAMERA; glm::vec3 position = glm::vec3( - (faceRect.center.x - _initialFaceRect.center.x) * proportion * POSITION_SCALE / _frameWidth, - (faceRect.center.y - _initialFaceRect.center.y) * proportion * POSITION_SCALE / _frameWidth, + (_faceRect.center.x - _initialFaceRect.center.x) * proportion * POSITION_SCALE / _textureSize.width, + (_faceRect.center.y - _initialFaceRect.center.y) * proportion * POSITION_SCALE / _textureSize.width, z); const float POSITION_SMOOTHING = 0.95f; _estimatedPosition = glm::mix(position, _estimatedPosition, POSITION_SMOOTHING); @@ -262,7 +258,8 @@ void Webcam::setFrame(const Mat& frame, int format, const Mat& depth, const Rota QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame())); } -FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0) { +FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), + _depthOffset(0.0), _codec(), _frameCount(0) { } FrameGrabber::~FrameGrabber() { @@ -370,29 +367,35 @@ void FrameGrabber::shutdown() { cvReleaseCapture(&_capture); _capture = 0; } + if (_codec.name != 0) { + vpx_codec_destroy(&_codec); + _codec.name = 0; + } _initialized = false; thread()->quit(); } +static Point clip(const Point& point, const Rect& bounds) { + return Point(glm::clamp(point.x, bounds.x, bounds.x + bounds.width - 1), + glm::clamp(point.y, bounds.y, bounds.y + bounds.height - 1)); +} + void FrameGrabber::grabFrame() { if (!(_initialized || init())) { return; } int format = GL_BGR; - Mat frame; + Mat color, depth; JointVector joints; #ifdef HAVE_OPENNI if (_depthGenerator.IsValid()) { _xnContext.WaitAnyUpdateAll(); - frame = Mat(_imageMetaData.YRes(), _imageMetaData.XRes(), CV_8UC3, (void*)_imageGenerator.GetImageMap()); + color = Mat(_imageMetaData.YRes(), _imageMetaData.XRes(), CV_8UC3, (void*)_imageGenerator.GetImageMap()); format = GL_RGB; - Mat depth = Mat(_depthMetaData.YRes(), _depthMetaData.XRes(), CV_16UC1, (void*)_depthGenerator.GetDepthMap()); - const double EIGHT_BIT_MAX = 255; - const double ELEVEN_BIT_MAX = 2047; - depth.convertTo(_grayDepthFrame, CV_8UC1, EIGHT_BIT_MAX / ELEVEN_BIT_MAX); + depth = Mat(_depthMetaData.YRes(), _depthMetaData.XRes(), CV_16UC1, (void*)_depthGenerator.GetDepthMap()); _userID = 0; XnUInt16 userCount = 1; @@ -428,7 +431,7 @@ void FrameGrabber::grabFrame() { } #endif - if (frame.empty()) { + if (color.empty()) { IplImage* image = cvQueryFrame(_capture); if (image == 0) { // try again later @@ -441,7 +444,7 @@ void FrameGrabber::grabFrame() { qDebug("Invalid webcam image format.\n"); return; } - frame = image; + color = image; } // if we don't have a search window (yet), try using the face cascade @@ -449,11 +452,11 @@ void FrameGrabber::grabFrame() { float ranges[] = { 0, 180 }; const float* range = ranges; if (_searchWindow.area() == 0) { - vector faces; - _faceCascade.detectMultiScale(frame, faces, 1.1, 6); + vector faces; + _faceCascade.detectMultiScale(color, faces, 1.1, 6); if (!faces.empty()) { _searchWindow = faces.front(); - updateHSVFrame(frame, format); + updateHSVFrame(color, format); Mat faceHsv(_hsvFrame, _searchWindow); Mat faceMask(_mask, _searchWindow); @@ -466,17 +469,159 @@ void FrameGrabber::grabFrame() { } RotatedRect faceRect; if (_searchWindow.area() > 0) { - updateHSVFrame(frame, format); + updateHSVFrame(color, format); calcBackProject(&_hsvFrame, 1, &channels, _histogram, _backProject, &range); bitwise_and(_backProject, _mask, _backProject); faceRect = CamShift(_backProject, _searchWindow, TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1)); - _searchWindow = faceRect.boundingRect(); - } + Rect faceBounds = faceRect.boundingRect(); + Rect imageBounds(0, 0, color.cols, color.rows); + _searchWindow = Rect(clip(faceBounds.tl(), imageBounds), clip(faceBounds.br(), imageBounds)); + } + +#ifdef HAVE_OPENNI + if (_depthGenerator.IsValid()) { + // convert from 11 to 8 bits, centered about the mean face depth (if possible) + if (_searchWindow.area() > 0) { + const double DEPTH_OFFSET_SMOOTHING = 0.95; + const double EIGHT_BIT_MIDPOINT = 128.0; + double meanOffset = EIGHT_BIT_MIDPOINT - mean(depth(_searchWindow))[0]; + _depthOffset = (_depthOffset == 0.0) ? meanOffset : glm::mix(meanOffset, _depthOffset, DEPTH_OFFSET_SMOOTHING); + } + depth.convertTo(_grayDepthFrame, CV_8UC1, 1.0, _depthOffset); + } +#endif + + const int ENCODED_FACE_WIDTH = 128; + const int ENCODED_FACE_HEIGHT = 128; + int combinedFaceHeight = ENCODED_FACE_HEIGHT * (depth.empty() ? 1 : 2); + if (_codec.name == 0) { + // initialize encoder context + vpx_codec_enc_cfg_t codecConfig; + vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0); + codecConfig.rc_target_bitrate = ENCODED_FACE_WIDTH * combinedFaceHeight * codecConfig.rc_target_bitrate / + codecConfig.g_w / codecConfig.g_h; + codecConfig.g_w = ENCODED_FACE_WIDTH; + codecConfig.g_h = combinedFaceHeight; + vpx_codec_enc_init(&_codec, vpx_codec_vp8_cx(), &codecConfig, 0); + } + + // correct for 180 degree rotations + if (faceRect.angle < -90.0f) { + faceRect.angle += 180.0f; + + } else if (faceRect.angle > 90.0f) { + faceRect.angle -= 180.0f; + } + + // compute the smoothed face rect + if (_smoothedFaceRect.size.area() == 0) { + _smoothedFaceRect = faceRect; + + } else { + const float FACE_RECT_SMOOTHING = 0.9f; + _smoothedFaceRect.center.x = glm::mix(faceRect.center.x, _smoothedFaceRect.center.x, FACE_RECT_SMOOTHING); + _smoothedFaceRect.center.y = glm::mix(faceRect.center.y, _smoothedFaceRect.center.y, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.width = glm::mix(faceRect.size.width, _smoothedFaceRect.size.width, FACE_RECT_SMOOTHING); + _smoothedFaceRect.size.height = glm::mix(faceRect.size.height, _smoothedFaceRect.size.height, FACE_RECT_SMOOTHING); + _smoothedFaceRect.angle = glm::mix(faceRect.angle, _smoothedFaceRect.angle, FACE_RECT_SMOOTHING); + } + + // resize/rotate face into encoding rectangle + _faceColor.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC3); + Point2f sourcePoints[4]; + _smoothedFaceRect.points(sourcePoints); + Point2f destPoints[] = { Point2f(0, ENCODED_FACE_HEIGHT), Point2f(0, 0), Point2f(ENCODED_FACE_WIDTH, 0) }; + Mat transform = getAffineTransform(sourcePoints, destPoints); + warpAffine(color, _faceColor, transform, _faceColor.size()); + + // convert from RGB to YV12 + const int ENCODED_BITS_PER_Y = 8; + const int ENCODED_BITS_PER_VU = 2; + const int ENCODED_BITS_PER_PIXEL = ENCODED_BITS_PER_Y + 2 * ENCODED_BITS_PER_VU; + const int BITS_PER_BYTE = 8; + _encodedFace.fill(128, ENCODED_FACE_WIDTH * combinedFaceHeight * ENCODED_BITS_PER_PIXEL / BITS_PER_BYTE); + vpx_image_t vpxImage; + vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, ENCODED_FACE_WIDTH, combinedFaceHeight, 1, (unsigned char*)_encodedFace.data()); + uchar* yline = vpxImage.planes[0]; + uchar* vline = vpxImage.planes[1]; + uchar* uline = vpxImage.planes[2]; + const int Y_RED_WEIGHT = (int)(0.299 * 256); + const int Y_GREEN_WEIGHT = (int)(0.587 * 256); + const int Y_BLUE_WEIGHT = (int)(0.114 * 256); + const int V_RED_WEIGHT = (int)(0.713 * 256); + const int U_BLUE_WEIGHT = (int)(0.564 * 256); + int redIndex = 0; + int greenIndex = 1; + int blueIndex = 2; + if (format == GL_BGR) { + redIndex = 2; + blueIndex = 0; + } + for (int i = 0; i < ENCODED_FACE_HEIGHT; i += 2) { + uchar* ydest = yline; + uchar* vdest = vline; + uchar* udest = uline; + for (int j = 0; j < ENCODED_FACE_WIDTH; j += 2) { + uchar* tl = _faceColor.ptr(i, j); + uchar* tr = _faceColor.ptr(i, j + 1); + uchar* bl = _faceColor.ptr(i + 1, j); + uchar* br = _faceColor.ptr(i + 1, j + 1); + + ydest[0] = (tl[redIndex] * Y_RED_WEIGHT + tl[1] * Y_GREEN_WEIGHT + tl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[1] = (tr[redIndex] * Y_RED_WEIGHT + tr[1] * Y_GREEN_WEIGHT + tr[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[ENCODED_FACE_WIDTH] = (bl[redIndex] * Y_RED_WEIGHT + bl[greenIndex] * + Y_GREEN_WEIGHT + bl[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest[ENCODED_FACE_WIDTH + 1] = (br[redIndex] * Y_RED_WEIGHT + br[greenIndex] * + Y_GREEN_WEIGHT + br[blueIndex] * Y_BLUE_WEIGHT) >> 8; + ydest += 2; + + int totalRed = tl[redIndex] + tr[redIndex] + bl[redIndex] + br[redIndex]; + int totalGreen = tl[greenIndex] + tr[greenIndex] + bl[greenIndex] + br[greenIndex]; + int totalBlue = tl[blueIndex] + tr[blueIndex] + bl[blueIndex] + br[blueIndex]; + int totalY = (totalRed * Y_RED_WEIGHT + totalGreen * Y_GREEN_WEIGHT + totalBlue * Y_BLUE_WEIGHT) >> 8; + + *vdest++ = (((totalRed - totalY) * V_RED_WEIGHT) >> 10) + 128; + *udest++ = (((totalBlue - totalY) * U_BLUE_WEIGHT) >> 10) + 128; + } + yline += vpxImage.stride[0] * 2; + vline += vpxImage.stride[1]; + uline += vpxImage.stride[2]; + } + + // if we have depth data, warp that and just copy it in + if (!depth.empty()) { + _faceDepth.create(ENCODED_FACE_WIDTH, ENCODED_FACE_HEIGHT, CV_8UC1); + warpAffine(_grayDepthFrame, _faceDepth, transform, _faceDepth.size()); + + uchar* dest = (uchar*)_encodedFace.data() + vpxImage.stride[0] * ENCODED_FACE_HEIGHT; + for (int i = 0; i < ENCODED_FACE_HEIGHT; i++) { + memcpy(dest, _faceDepth.ptr(i), ENCODED_FACE_WIDTH); + dest += vpxImage.stride[0]; + } + } + + // encode the frame + vpx_codec_encode(&_codec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME); + + // extract the encoded frame + vpx_codec_iter_t iterator = 0; + const vpx_codec_cx_pkt_t* packet; + while ((packet = vpx_codec_get_cx_data(&_codec, &iterator)) != 0) { + if (packet->kind == VPX_CODEC_CX_FRAME_PKT) { + // prepend the aspect ratio + QByteArray payload(sizeof(float), 0); + *(float*)payload.data() = _smoothedFaceRect.size.width / _smoothedFaceRect.size.height; + payload.append((const char*)packet->data.frame.buf, packet->data.frame.sz); + QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", Q_ARG(int, _frameCount), + Q_ARG(QByteArray, payload)); + } + } + QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame", - Q_ARG(cv::Mat, frame), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), - Q_ARG(cv::RotatedRect, faceRect), Q_ARG(JointVector, joints)); + Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame), + Q_ARG(cv::RotatedRect, _smoothedFaceRect), Q_ARG(JointVector, joints)); } bool FrameGrabber::init() { @@ -506,6 +651,11 @@ bool FrameGrabber::init() { _userGenerator.GetSkeletonCap().SetSkeletonProfile(XN_SKEL_PROFILE_UPPER); + // make the depth viewpoint match that of the video image + if (_depthGenerator.IsCapabilitySupported(XN_CAPABILITY_ALTERNATIVE_VIEW_POINT)) { + _depthGenerator.GetAlternativeViewPointCap().SetViewPoint(_imageGenerator); + } + _xnContext.StartGeneratingAll(); return true; } diff --git a/interface/src/Webcam.h b/interface/src/Webcam.h index 4bc557f4fb..260eda0897 100644 --- a/interface/src/Webcam.h +++ b/interface/src/Webcam.h @@ -23,6 +23,8 @@ #include #endif +#include + #include "InterfaceConfig.h" class QImage; @@ -42,7 +44,14 @@ public: Webcam(); ~Webcam(); - const bool isActive() const { return _active; } + bool isActive() const { return _active; } + + GLuint getColorTextureID() const { return _colorTextureID; } + GLuint getDepthTextureID() const { return _depthTextureID; } + const cv::Size2f& getTextureSize() const { return _textureSize; } + + const cv::RotatedRect& getFaceRect() const { return _faceRect; } + const glm::vec3& getEstimatedPosition() const { return _estimatedPosition; } const glm::vec3& getEstimatedRotation() const { return _estimatedRotation; } const JointVector& getEstimatedJoints() const { return _estimatedJoints; } @@ -53,9 +62,9 @@ public: public slots: void setEnabled(bool enabled); - void setFrame(const cv::Mat& video, int format, const cv::Mat& depth, + void setFrame(const cv::Mat& color, int format, const cv::Mat& depth, const cv::RotatedRect& faceRect, const JointVector& joints); - + private: QThread _grabberThread; @@ -63,12 +72,9 @@ private: bool _enabled; bool _active; - int _frameWidth; - int _frameHeight; - int _depthWidth; - int _depthHeight; - GLuint _frameTextureID; + GLuint _colorTextureID; GLuint _depthTextureID; + cv::Size2f _textureSize; cv::RotatedRect _faceRect; cv::RotatedRect _initialFaceRect; JointVector _joints; @@ -111,7 +117,15 @@ private: cv::Mat _backProject; cv::Rect _searchWindow; cv::Mat _grayDepthFrame; - + double _depthOffset; + + vpx_codec_ctx_t _codec; + int _frameCount; + cv::Mat _faceColor; + cv::Mat _faceDepth; + QByteArray _encodedFace; + cv::RotatedRect _smoothedFaceRect; + #ifdef HAVE_OPENNI xn::Context _xnContext; xn::DepthGenerator _depthGenerator; diff --git a/interface/src/Avatar.cpp b/interface/src/avatar/Avatar.cpp similarity index 99% rename from interface/src/Avatar.cpp rename to interface/src/avatar/Avatar.cpp index 9d1e4da0fb..ddb78f595b 100755 --- a/interface/src/Avatar.cpp +++ b/interface/src/avatar/Avatar.cpp @@ -313,6 +313,12 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, if (webcam->isActive()) { estimatedPosition = webcam->getEstimatedPosition(); + // apply face data + _head.getFace().setColorTextureID(webcam->getColorTextureID()); + _head.getFace().setDepthTextureID(webcam->getDepthTextureID()); + _head.getFace().setTextureSize(webcam->getTextureSize()); + _head.getFace().setTextureRect(webcam->getFaceRect()); + // compute and store the joint rotations const JointVector& joints = webcam->getEstimatedJoints(); _joints.clear(); @@ -327,6 +333,8 @@ void Avatar::updateFromGyrosAndOrWebcam(bool gyroLook, } } } + } else { + _head.getFace().setColorTextureID(0); } _head.setPitch(estimatedRotation.x * amplifyAngle.x + pitchFromTouch); _head.setYaw(estimatedRotation.y * amplifyAngle.y + yawFromTouch); diff --git a/interface/src/Avatar.h b/interface/src/avatar/Avatar.h similarity index 100% rename from interface/src/Avatar.h rename to interface/src/avatar/Avatar.h diff --git a/interface/src/AvatarTouch.cpp b/interface/src/avatar/AvatarTouch.cpp similarity index 100% rename from interface/src/AvatarTouch.cpp rename to interface/src/avatar/AvatarTouch.cpp diff --git a/interface/src/AvatarTouch.h b/interface/src/avatar/AvatarTouch.h similarity index 100% rename from interface/src/AvatarTouch.h rename to interface/src/avatar/AvatarTouch.h diff --git a/interface/src/AvatarVoxelSystem.cpp b/interface/src/avatar/AvatarVoxelSystem.cpp similarity index 100% rename from interface/src/AvatarVoxelSystem.cpp rename to interface/src/avatar/AvatarVoxelSystem.cpp diff --git a/interface/src/AvatarVoxelSystem.h b/interface/src/avatar/AvatarVoxelSystem.h similarity index 100% rename from interface/src/AvatarVoxelSystem.h rename to interface/src/avatar/AvatarVoxelSystem.h diff --git a/interface/src/avatar/Face.cpp b/interface/src/avatar/Face.cpp new file mode 100644 index 0000000000..d8a8d6c2a6 --- /dev/null +++ b/interface/src/avatar/Face.cpp @@ -0,0 +1,345 @@ +// +// Face.cpp +// interface +// +// Created by Andrzej Kapolka on 7/11/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#include + +#include +#include + +#include + +#include "Application.h" +#include "Avatar.h" +#include "Head.h" +#include "Face.h" +#include "renderer/ProgramObject.h" + +using namespace cv; + +ProgramObject* Face::_program = 0; +int Face::_texCoordCornerLocation; +int Face::_texCoordRightLocation; +int Face::_texCoordUpLocation; +int Face::_aspectRatioLocation; +GLuint Face::_vboID; +GLuint Face::_iboID; + +Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH), + _colorTextureID(0), _depthTextureID(0), _codec(), _frameCount(0) { + // we may have been created in the network thread, but we live in the main thread + moveToThread(Application::getInstance()->thread()); +} + +Face::~Face() { + if (_codec.name != 0) { + vpx_codec_destroy(&_codec); + + // delete our textures, since we know that we own them + if (_colorTextureID != 0) { + glDeleteTextures(1, &_colorTextureID); + } + if (_depthTextureID != 0) { + glDeleteTextures(1, &_depthTextureID); + } + } +} + +void Face::setTextureRect(const cv::RotatedRect& textureRect) { + _textureRect = textureRect; + _aspectRatio = _textureRect.size.width / _textureRect.size.height; +} + +int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) { + if (_codec.name == 0) { + // initialize decoder context + vpx_codec_dec_init(&_codec, vpx_codec_vp8_dx(), 0, 0); + } + // skip the header + unsigned char* packetPosition = packetData; + + int frameCount = *(uint32_t*)packetPosition; + packetPosition += sizeof(uint32_t); + + int frameSize = *(uint32_t*)packetPosition; + packetPosition += sizeof(uint32_t); + + int frameOffset = *(uint32_t*)packetPosition; + packetPosition += sizeof(uint32_t); + + if (frameCount < _frameCount) { // old frame; ignore + return dataBytes; + + } else if (frameCount > _frameCount) { // new frame; reset + _frameCount = frameCount; + _frameBytesRemaining = frameSize; + _arrivingFrame.resize(frameSize); + } + + int payloadSize = dataBytes - (packetPosition - packetData); + memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize); + + if ((_frameBytesRemaining -= payloadSize) <= 0) { + float aspectRatio = *(const float*)_arrivingFrame.constData(); + vpx_codec_decode(&_codec, (const uint8_t*)_arrivingFrame.constData() + sizeof(float), + _arrivingFrame.size() - sizeof(float), 0, 0); + vpx_codec_iter_t iterator = 0; + vpx_image_t* image; + while ((image = vpx_codec_get_frame(&_codec, &iterator)) != 0) { + // convert from YV12 to RGB + const int imageHeight = image->d_w; + Mat color(imageHeight, image->d_w, CV_8UC3); + uchar* yline = image->planes[0]; + uchar* vline = image->planes[1]; + uchar* uline = image->planes[2]; + const int RED_V_WEIGHT = (int)(1.403 * 256); + const int GREEN_V_WEIGHT = (int)(0.714 * 256); + const int GREEN_U_WEIGHT = (int)(0.344 * 256); + const int BLUE_U_WEIGHT = (int)(1.773 * 256); + for (int i = 0; i < imageHeight; i += 2) { + uchar* ysrc = yline; + uchar* vsrc = vline; + uchar* usrc = uline; + for (int j = 0; j < image->d_w; j += 2) { + uchar* tl = color.ptr(i, j); + uchar* tr = color.ptr(i, j + 1); + uchar* bl = color.ptr(i + 1, j); + uchar* br = color.ptr(i + 1, j + 1); + + int v = *vsrc++ - 128; + int u = *usrc++ - 128; + + int redOffset = (RED_V_WEIGHT * v) >> 8; + int greenOffset = (GREEN_V_WEIGHT * v + GREEN_U_WEIGHT * u) >> 8; + int blueOffset = (BLUE_U_WEIGHT * u) >> 8; + + int ytl = ysrc[0]; + int ytr = ysrc[1]; + int ybl = ysrc[image->w]; + int ybr = ysrc[image->w + 1]; + ysrc += 2; + + tl[0] = ytl + redOffset; + tl[1] = ytl - greenOffset; + tl[2] = ytl + blueOffset; + + tr[0] = ytr + redOffset; + tr[1] = ytr - greenOffset; + tr[2] = ytr + blueOffset; + + bl[0] = ybl + redOffset; + bl[1] = ybl - greenOffset; + bl[2] = ybl + blueOffset; + + br[0] = ybr + redOffset; + br[1] = ybr - greenOffset; + br[2] = ybr + blueOffset; + } + yline += image->stride[0] * 2; + vline += image->stride[1]; + uline += image->stride[2]; + } + Mat depth; + if (image->d_h > imageHeight) { + // if the height is greater than the width, we have depth data + depth.create(imageHeight, image->d_w, CV_8UC1); + uchar* src = image->planes[0] + image->stride[0] * imageHeight; + for (int i = 0; i < imageHeight; i++) { + memcpy(depth.ptr(i), src, image->d_w); + src += image->stride[0]; + } + } + QMetaObject::invokeMethod(this, "setFrame", Q_ARG(cv::Mat, color), + Q_ARG(cv::Mat, depth), Q_ARG(float, aspectRatio)); + } + } + + return dataBytes; +} + +bool Face::render(float alpha) { + if (_colorTextureID == 0 || _textureRect.size.area() == 0) { + return false; + } + glPushMatrix(); + + glTranslatef(_owningHead->getPosition().x, _owningHead->getPosition().y, _owningHead->getPosition().z); + glm::quat orientation = _owningHead->getOrientation(); + glm::vec3 axis = glm::axis(orientation); + glRotatef(glm::angle(orientation), axis.x, axis.y, axis.z); + float scale = BODY_BALL_RADIUS_HEAD_BASE * _owningHead->getScale(); + glScalef(scale, scale, scale); + + glColor4f(1.0f, 1.0f, 1.0f, alpha); + + Point2f points[4]; + _textureRect.points(points); + + if (_depthTextureID != 0) { + const int VERTEX_WIDTH = 100; + const int VERTEX_HEIGHT = 100; + const int VERTEX_COUNT = VERTEX_WIDTH * VERTEX_HEIGHT; + const int ELEMENTS_PER_VERTEX = 2; + const int BUFFER_ELEMENTS = VERTEX_COUNT * ELEMENTS_PER_VERTEX; + const int QUAD_WIDTH = VERTEX_WIDTH - 1; + const int QUAD_HEIGHT = VERTEX_HEIGHT - 1; + const int QUAD_COUNT = QUAD_WIDTH * QUAD_HEIGHT; + const int TRIANGLES_PER_QUAD = 2; + const int INDICES_PER_TRIANGLE = 3; + const int INDEX_COUNT = QUAD_COUNT * TRIANGLES_PER_QUAD * INDICES_PER_TRIANGLE; + + if (_program == 0) { + _program = new ProgramObject(); + _program->addShaderFromSourceFile(QGLShader::Vertex, "resources/shaders/face.vert"); + _program->addShaderFromSourceFile(QGLShader::Fragment, "resources/shaders/face.frag"); + _program->link(); + + _program->bind(); + _program->setUniformValue("depthTexture", 0); + _program->setUniformValue("colorTexture", 1); + _program->release(); + + _texCoordCornerLocation = _program->uniformLocation("texCoordCorner"); + _texCoordRightLocation = _program->uniformLocation("texCoordRight"); + _texCoordUpLocation = _program->uniformLocation("texCoordUp"); + _aspectRatioLocation = _program->uniformLocation("aspectRatio"); + + glGenBuffers(1, &_vboID); + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + float* vertices = new float[BUFFER_ELEMENTS]; + float* vertexPosition = vertices; + for (int i = 0; i < VERTEX_HEIGHT; i++) { + for (int j = 0; j < VERTEX_WIDTH; j++) { + *vertexPosition++ = j / (float)(VERTEX_WIDTH - 1); + *vertexPosition++ = i / (float)(VERTEX_HEIGHT - 1); + } + } + glBufferData(GL_ARRAY_BUFFER, BUFFER_ELEMENTS * sizeof(float), vertices, GL_STATIC_DRAW); + delete[] vertices; + + glGenBuffers(1, &_iboID); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + int* indices = new int[INDEX_COUNT]; + int* indexPosition = indices; + for (int i = 0; i < QUAD_HEIGHT; i++) { + for (int j = 0; j < QUAD_WIDTH; j++) { + *indexPosition++ = i * VERTEX_WIDTH + j; + *indexPosition++ = (i + 1) * VERTEX_WIDTH + j; + *indexPosition++ = i * VERTEX_WIDTH + j + 1; + + *indexPosition++ = i * VERTEX_WIDTH + j + 1; + *indexPosition++ = (i + 1) * VERTEX_WIDTH + j; + *indexPosition++ = (i + 1) * VERTEX_WIDTH + j + 1; + } + } + glBufferData(GL_ELEMENT_ARRAY_BUFFER, INDEX_COUNT * sizeof(int), indices, GL_STATIC_DRAW); + delete[] indices; + + } else { + glBindBuffer(GL_ARRAY_BUFFER, _vboID); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _iboID); + } + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + + _program->bind(); + _program->setUniformValue(_texCoordCornerLocation, + points[0].x / _textureSize.width, points[0].y / _textureSize.height); + _program->setUniformValue(_texCoordRightLocation, + (points[3].x - points[0].x) / _textureSize.width, (points[3].y - points[0].y) / _textureSize.height); + _program->setUniformValue(_texCoordUpLocation, + (points[1].x - points[0].x) / _textureSize.width, (points[1].y - points[0].y) / _textureSize.height); + _program->setUniformValue(_aspectRatioLocation, _aspectRatio); + glEnableClientState(GL_VERTEX_ARRAY); + glVertexPointer(2, GL_FLOAT, 0, 0); + + glEnable(GL_ALPHA_TEST); + glAlphaFunc(GL_EQUAL, 1.0f); + + if (_renderMode == MESH) { + glDrawRangeElementsEXT(GL_TRIANGLES, 0, VERTEX_COUNT - 1, INDEX_COUNT, GL_UNSIGNED_INT, 0); + glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); + + } else { // _renderMode == POINTS + glPointSize(5.0f); + glDrawArrays(GL_POINTS, 0, VERTEX_COUNT); + glPointSize(1.0f); + } + + glDisable(GL_ALPHA_TEST); + + glDisableClientState(GL_VERTEX_ARRAY); + glBindBuffer(GL_ARRAY_BUFFER, 0); + _program->release(); + + glBindTexture(GL_TEXTURE_2D, 0); + glActiveTexture(GL_TEXTURE0); + + } else { + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + glEnable(GL_TEXTURE_2D); + + glBegin(GL_QUADS); + glTexCoord2f(points[0].x / _textureSize.width, points[0].y / _textureSize.height); + glVertex3f(0.5f, -0.5f / _aspectRatio, -0.5f); + glTexCoord2f(points[1].x / _textureSize.width, points[1].y / _textureSize.height); + glVertex3f(0.5f, 0.5f / _aspectRatio, -0.5f); + glTexCoord2f(points[2].x / _textureSize.width, points[2].y / _textureSize.height); + glVertex3f(-0.5f, 0.5f / _aspectRatio, -0.5f); + glTexCoord2f(points[3].x / _textureSize.width, points[3].y / _textureSize.height); + glVertex3f(-0.5f, -0.5f / _aspectRatio, -0.5f); + glEnd(); + + glDisable(GL_TEXTURE_2D); + } + + glBindTexture(GL_TEXTURE_2D, 0); + + glPopMatrix(); + + return true; +} + +void Face::cycleRenderMode() { + _renderMode = (RenderMode)((_renderMode + 1) % RENDER_MODE_COUNT); +} + +void Face::setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio) { + if (_colorTextureID == 0) { + glGenTextures(1, &_colorTextureID); + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, color.cols, color.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + _textureSize = color.size(); + _textureRect = RotatedRect(Point2f(color.cols * 0.5f, color.rows * 0.5f), _textureSize, 0.0f); + + } else { + glBindTexture(GL_TEXTURE_2D, _colorTextureID); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, color.cols, color.rows, GL_RGB, GL_UNSIGNED_BYTE, color.ptr()); + } + + if (!depth.empty()) { + if (_depthTextureID == 0) { + glGenTextures(1, &_depthTextureID); + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, depth.cols, depth.rows, 0, + GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + + } else { + glBindTexture(GL_TEXTURE_2D, _depthTextureID); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, depth.cols, depth.rows, GL_LUMINANCE, GL_UNSIGNED_BYTE, depth.ptr()); + } + } + glBindTexture(GL_TEXTURE_2D, 0); + + _aspectRatio = aspectRatio; +} + diff --git a/interface/src/avatar/Face.h b/interface/src/avatar/Face.h new file mode 100644 index 0000000000..1f9d41a1b3 --- /dev/null +++ b/interface/src/avatar/Face.h @@ -0,0 +1,75 @@ +// +// Face.h +// interface +// +// Created by Andrzej Kapolka on 7/11/13. +// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. +// + +#ifndef __interface__Face__ +#define __interface__Face__ + +#include + +#include + +#include + +#include "InterfaceConfig.h" + +class Head; +class ProgramObject; + +class Face : public QObject { + Q_OBJECT + +public: + + Face(Head* owningHead); + ~Face(); + + void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; } + void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; } + void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; } + void setTextureRect(const cv::RotatedRect& textureRect); + + int processVideoMessage(unsigned char* packetData, size_t dataBytes); + + bool render(float alpha); + +public slots: + + void cycleRenderMode(); + +private slots: + + void setFrame(const cv::Mat& color, const cv::Mat& depth, float aspectRatio); + +private: + + enum RenderMode { MESH, POINTS, RENDER_MODE_COUNT }; + + Head* _owningHead; + RenderMode _renderMode; + GLuint _colorTextureID; + GLuint _depthTextureID; + cv::Size2f _textureSize; + cv::RotatedRect _textureRect; + float _aspectRatio; + + vpx_codec_ctx_t _codec; + + QByteArray _arrivingFrame; + int _frameCount; + int _frameBytesRemaining; + + static ProgramObject* _program; + static int _texCoordCornerLocation; + static int _texCoordRightLocation; + static int _texCoordUpLocation; + static int _aspectRatioLocation; + static GLuint _vboID; + static GLuint _iboID; +}; + +#endif /* defined(__interface__Face__) */ diff --git a/interface/src/Hand.cpp b/interface/src/avatar/Hand.cpp similarity index 89% rename from interface/src/Hand.cpp rename to interface/src/avatar/Hand.cpp index 69d8ddfaac..97594d0f50 100755 --- a/interface/src/Hand.cpp +++ b/interface/src/avatar/Hand.cpp @@ -95,6 +95,7 @@ void Hand::render(bool lookingInMirror) { glEnable(GL_DEPTH_TEST); glEnable(GL_RESCALE_NORMAL); + renderFingerTrails(); renderHandSpheres(); } @@ -173,44 +174,35 @@ void Hand::renderHandSpheres() { glPopMatrix(); } -void Hand::setLeapFingers(const std::vector& fingerTips, - const std::vector& fingerRoots) { - // TODO: add id-checking here to increase finger stability - - size_t fingerIndex = 0; +void Hand::renderFingerTrails() { + // Draw the finger root cones for (size_t i = 0; i < getNumPalms(); ++i) { PalmData& palm = getPalms()[i]; - for (size_t f = 0; f < palm.getNumFingers(); ++f) { - FingerData& finger = palm.getFingers()[f]; - if (fingerIndex < fingerTips.size()) { - finger.setActive(true); - finger.setRawTipPosition(fingerTips[fingerIndex]); - finger.setRawRootPosition(fingerRoots[fingerIndex]); - fingerIndex++; - } - else { - finger.setActive(false); + if (palm.isActive()) { + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + int numPositions = finger.getTrailNumPositions(); + if (numPositions > 0) { + glBegin(GL_TRIANGLE_STRIP); + for (int t = 0; t < numPositions; ++t) + { + const glm::vec3& center = finger.getTrailPosition(t); + const float halfWidth = 0.001f; + const glm::vec3 edgeDirection(1.0f, 0.0f, 0.0f); + glm::vec3 edge0 = center + edgeDirection * halfWidth; + glm::vec3 edge1 = center - edgeDirection * halfWidth; + float alpha = 1.0f - ((float)t / (float)(numPositions - 1)); + glColor4f(1.0f, 0.0f, 0.0f, alpha); + glVertex3fv((float*)&edge0); + glVertex3fv((float*)&edge1); + } + glEnd(); + } } } } } -void Hand::setLeapHands(const std::vector& handPositions, - const std::vector& handNormals) { - for (size_t i = 0; i < getNumPalms(); ++i) { - PalmData& palm = getPalms()[i]; - if (i < handPositions.size()) { - palm.setActive(true); - palm.setRawPosition(handPositions[i]); - palm.setRawNormal(handNormals[i]); - } - else { - palm.setActive(false); - } - } -} - - void Hand::updateFingerParticles(float deltaTime) { if (!_particleSystemInitialized) { diff --git a/interface/src/Hand.h b/interface/src/avatar/Hand.h similarity index 88% rename from interface/src/Hand.h rename to interface/src/avatar/Hand.h index fb6b863458..f37046ed3a 100755 --- a/interface/src/Hand.h +++ b/interface/src/avatar/Hand.h @@ -42,10 +42,6 @@ public: void render(bool lookingInMirror); void setBallColor (glm::vec3 ballColor ) { _ballColor = ballColor; } - void setLeapFingers (const std::vector& fingerTips, - const std::vector& fingerRoots); - void setLeapHands (const std::vector& handPositions, - const std::vector& handNormals); void updateFingerParticles(float deltaTime); void setRaveGloveActive(bool active) { _isRaveGloveActive = active; } @@ -74,6 +70,7 @@ private: // private methods void renderRaveGloveStage(); void renderHandSpheres(); + void renderFingerTrails(); void calculateGeometry(); }; diff --git a/interface/src/HandControl.cpp b/interface/src/avatar/HandControl.cpp similarity index 100% rename from interface/src/HandControl.cpp rename to interface/src/avatar/HandControl.cpp diff --git a/interface/src/HandControl.h b/interface/src/avatar/HandControl.h similarity index 100% rename from interface/src/HandControl.h rename to interface/src/avatar/HandControl.h diff --git a/interface/src/Head.cpp b/interface/src/avatar/Head.cpp similarity index 98% rename from interface/src/Head.cpp rename to interface/src/avatar/Head.cpp index a477910d67..b8393e26ab 100644 --- a/interface/src/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -82,7 +82,8 @@ Head::Head(Avatar* owningAvatar) : _cameraYaw(_yaw), _isCameraMoving(false), _cameraFollowsHead(false), - _cameraFollowHeadRate(0.0f) + _cameraFollowHeadRate(0.0f), + _face(this) { if (USING_PHYSICAL_MOHAWK) { resetHairPhysics(); @@ -271,7 +272,7 @@ void Head::calculateGeometry() { + up * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_UP_OFFSET + front * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_FRONT_OFFSET; - _eyeLevelPosition = _position + up * _scale * EYE_UP_OFFSET; + _eyeLevelPosition = _position + up * _scale * BODY_BALL_RADIUS_HEAD_BASE * EYE_UP_OFFSET; //calculate the eyebrow positions _leftEyeBrowPosition = _leftEyePosition; @@ -291,18 +292,20 @@ void Head::render(float alpha) { _renderAlpha = alpha; - calculateGeometry(); + if (!_face.render(alpha)) { + calculateGeometry(); - glEnable(GL_DEPTH_TEST); - glEnable(GL_RESCALE_NORMAL); - - renderMohawk(); - renderHeadSphere(); - renderEyeBalls(); - renderEars(); - renderMouth(); - renderEyeBrows(); + glEnable(GL_DEPTH_TEST); + glEnable(GL_RESCALE_NORMAL); + renderMohawk(); + renderHeadSphere(); + renderEyeBalls(); + renderEars(); + renderMouth(); + renderEyeBrows(); + } + if (_renderLookatVectors) { renderLookatVectors(_leftEyePosition, _rightEyePosition, _lookAtPosition); } @@ -592,7 +595,7 @@ void Head::renderEyeBalls() { glm::quat rotation = rotationBetween(front, targetLookatVector) * orientation; glm::vec3 rotationAxis = glm::axis(rotation); glRotatef(glm::angle(rotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); - glTranslatef(0.0f, 0.0f, -IRIS_PROTRUSION); + glTranslatef(0.0f, 0.0f, -_scale * IRIS_PROTRUSION); glScalef(_scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS); // flatten the iris @@ -616,7 +619,7 @@ void Head::renderEyeBalls() { glm::quat rotation = rotationBetween(front, targetLookatVector) * orientation; glm::vec3 rotationAxis = glm::axis(rotation); glRotatef(glm::angle(rotation), rotationAxis.x, rotationAxis.y, rotationAxis.z); - glTranslatef(0.0f, 0.0f, -IRIS_PROTRUSION); + glTranslatef(0.0f, 0.0f, -_scale * IRIS_PROTRUSION); glScalef(_scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS * 2.0f, _scale * IRIS_RADIUS); // flatten the iris diff --git a/interface/src/Head.h b/interface/src/avatar/Head.h similarity index 96% rename from interface/src/Head.h rename to interface/src/avatar/Head.h index 1f49854cd7..4c1429e1d6 100644 --- a/interface/src/Head.h +++ b/interface/src/avatar/Head.h @@ -10,11 +10,17 @@ #include #include + +#include + #include -#include "world.h" + +#include + +#include "Face.h" #include "InterfaceConfig.h" #include "SerialInterface.h" -#include +#include "world.h" enum eyeContactTargets { @@ -53,12 +59,15 @@ public: glm::quat getOrientation() const; glm::quat getCameraOrientation () const; + float getScale() const { return _scale; } glm::vec3 getPosition() const { return _position; } const glm::vec3& getEyeLevelPosition() const { return _eyeLevelPosition; } glm::vec3 getRightDirection() const { return getOrientation() * IDENTITY_RIGHT; } glm::vec3 getUpDirection () const { return getOrientation() * IDENTITY_UP; } glm::vec3 getFrontDirection() const { return getOrientation() * IDENTITY_FRONT; } + Face& getFace() { return _face; } + const bool getReturnToCenter() const { return _returnHeadToCenter; } // Do you want head to try to return to center (depends on interface detected) float getAverageLoudness() {return _averageLoudness;}; glm::vec3 calculateAverageEyePosition() { return _leftEyePosition + (_rightEyePosition - _leftEyePosition ) * ONE_HALF; } @@ -120,6 +129,7 @@ private: bool _isCameraMoving; bool _cameraFollowsHead; float _cameraFollowHeadRate; + Face _face; static ProgramObject* _irisProgram; static GLuint _irisTextureID; diff --git a/interface/src/Skeleton.cpp b/interface/src/avatar/Skeleton.cpp similarity index 100% rename from interface/src/Skeleton.cpp rename to interface/src/avatar/Skeleton.cpp diff --git a/interface/src/Skeleton.h b/interface/src/avatar/Skeleton.h similarity index 100% rename from interface/src/Skeleton.h rename to interface/src/avatar/Skeleton.h diff --git a/libraries/avatars/src/HandData.cpp b/libraries/avatars/src/HandData.cpp index 32e83b352d..7bb6eebdc0 100755 --- a/libraries/avatars/src/HandData.cpp +++ b/libraries/avatars/src/HandData.cpp @@ -13,15 +13,22 @@ HandData::HandData(AvatarData* owningAvatar) : _baseOrientation(0.0f, 0.0f, 0.0f, 1.0f), _owningAvatarData(owningAvatar) { - for (int i = 0; i < 2; ++i) { - _palms.push_back(PalmData(this)); - } + // Start with two palms + addNewPalm(); + addNewPalm(); +} + +PalmData& HandData::addNewPalm() { + _palms.push_back(PalmData(this)); + return _palms.back(); } PalmData::PalmData(HandData* owningHandData) : _rawPosition(0, 0, 0), _rawNormal(0, 1, 0), _isActive(false), +_leapID(LEAPID_INVALID), +_numFramesWithoutData(0), _owningHandData(owningHandData) { for (int i = 0; i < NUM_FINGERS_PER_HAND; ++i) { @@ -33,9 +40,13 @@ FingerData::FingerData(PalmData* owningPalmData, HandData* owningHandData) : _tipRawPosition(0, 0, 0), _rootRawPosition(0, 0, 0), _isActive(false), +_leapID(LEAPID_INVALID), +_numFramesWithoutData(0), _owningPalmData(owningPalmData), _owningHandData(owningHandData) { + const int standardTrailLength = 30; + setTrailLength(standardTrailLength); } void HandData::encodeRemoteData(std::vector& fingerVectors) { @@ -80,3 +91,66 @@ void HandData::decodeRemoteData(const std::vector& fingerVectors) { } } +void HandData::setFingerTrailLength(unsigned int length) { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.setTrailLength(length); + } + } +} + +void HandData::updateFingerTrails() { + for (size_t i = 0; i < getNumPalms(); ++i) { + PalmData& palm = getPalms()[i]; + for (size_t f = 0; f < palm.getNumFingers(); ++f) { + FingerData& finger = palm.getFingers()[f]; + finger.updateTrail(); + } + } +} + +void FingerData::setTrailLength(unsigned int length) { + _tipTrailPositions.resize(length); + _tipTrailCurrentStartIndex = 0; + _tipTrailCurrentValidLength = 0; +} + +void FingerData::updateTrail() { + if (_tipTrailPositions.size() == 0) + return; + + if (_isActive) { + // Add the next point in the trail. + _tipTrailCurrentStartIndex--; + if (_tipTrailCurrentStartIndex < 0) + _tipTrailCurrentStartIndex = _tipTrailPositions.size() - 1; + + _tipTrailPositions[_tipTrailCurrentStartIndex] = getTipPosition(); + + if (_tipTrailCurrentValidLength < _tipTrailPositions.size()) + _tipTrailCurrentValidLength++; + } + else { + // It's not active, so just shorten the trail. + if (_tipTrailCurrentValidLength > 0) + _tipTrailCurrentValidLength--; + } +} + +int FingerData::getTrailNumPositions() { + return _tipTrailCurrentValidLength; +} + +const glm::vec3& FingerData::getTrailPosition(int index) { + if (index >= _tipTrailCurrentValidLength) { + static glm::vec3 zero(0,0,0); + return zero; + } + int posIndex = (index + _tipTrailCurrentStartIndex) % _tipTrailCurrentValidLength; + return _tipTrailPositions[posIndex]; +} + + + diff --git a/libraries/avatars/src/HandData.h b/libraries/avatars/src/HandData.h index 65b32ff5c6..d2b5cae90d 100755 --- a/libraries/avatars/src/HandData.h +++ b/libraries/avatars/src/HandData.h @@ -20,6 +20,7 @@ class FingerData; class PalmData; const int NUM_FINGERS_PER_HAND = 5; +const int LEAPID_INVALID = -1; class HandData { public: @@ -39,6 +40,10 @@ public: std::vector& getPalms() { return _palms; } size_t getNumPalms() { return _palms.size(); } + PalmData& addNewPalm(); + + void setFingerTrailLength(unsigned int length); + void updateFingerTrails(); // Use these for sending and receiving hand data void encodeRemoteData(std::vector& fingerVectors); @@ -65,15 +70,31 @@ public: const glm::vec3& getTipRawPosition() const { return _tipRawPosition; } const glm::vec3& getRootRawPosition() const { return _rootRawPosition; } bool isActive() const { return _isActive; } + int getLeapID() const { return _leapID; } void setActive(bool active) { _isActive = active; } + void setLeapID(int id) { _leapID = id; } void setRawTipPosition(const glm::vec3& pos) { _tipRawPosition = pos; } void setRawRootPosition(const glm::vec3& pos) { _rootRawPosition = pos; } + void setTrailLength(unsigned int length); + void updateTrail(); + + int getTrailNumPositions(); + const glm::vec3& getTrailPosition(int index); + + void incrementFramesWithoutData() { _numFramesWithoutData++; } + void resetFramesWithoutData() { _numFramesWithoutData = 0; } + int getFramesWithoutData() const { return _numFramesWithoutData; } private: glm::vec3 _tipRawPosition; glm::vec3 _rootRawPosition; - bool _isActive; // This has current valid data + bool _isActive; // This has current valid data + int _leapID; // the Leap's serial id for this tracked object + int _numFramesWithoutData; // after too many frames without data, this tracked object assumed lost. + std::vector _tipTrailPositions; + int _tipTrailCurrentStartIndex; + int _tipTrailCurrentValidLength; PalmData* _owningPalmData; HandData* _owningHandData; }; @@ -86,19 +107,27 @@ public: const glm::vec3& getRawPosition() const { return _rawPosition; } const glm::vec3& getRawNormal() const { return _rawNormal; } bool isActive() const { return _isActive; } + int getLeapID() const { return _leapID; } std::vector& getFingers() { return _fingers; } size_t getNumFingers() { return _fingers.size(); } void setActive(bool active) { _isActive = active; } + void setLeapID(int id) { _leapID = id; } void setRawPosition(const glm::vec3& pos) { _rawPosition = pos; } void setRawNormal(const glm::vec3& normal) { _rawNormal = normal; } + void incrementFramesWithoutData() { _numFramesWithoutData++; } + void resetFramesWithoutData() { _numFramesWithoutData = 0; } + int getFramesWithoutData() const { return _numFramesWithoutData; } + private: std::vector _fingers; glm::vec3 _rawPosition; glm::vec3 _rawNormal; - bool _isActive; // This has current valid data + bool _isActive; // This has current valid data + int _leapID; // the Leap's serial id for this tracked object + int _numFramesWithoutData; // after too many frames without data, this tracked object assumed lost. HandData* _owningHandData; }; diff --git a/libraries/shared/src/NodeList.cpp b/libraries/shared/src/NodeList.cpp index 91fe1f4a21..9406c6b14d 100644 --- a/libraries/shared/src/NodeList.cpp +++ b/libraries/shared/src/NodeList.cpp @@ -499,6 +499,37 @@ void NodeList::startSilentNodeRemovalThread() { void NodeList::stopSilentNodeRemovalThread() { silentNodeThreadStopFlag = true; pthread_join(removeSilentNodesThread, NULL); + +} + +const QString QSETTINGS_GROUP_NAME = "NodeList"; +const QString DOMAIN_SERVER_SETTING_KEY = "domainServerHostname"; + +void NodeList::loadData(QSettings *settings) { + settings->beginGroup(DOMAIN_SERVER_SETTING_KEY); + + QString domainServerHostname = settings->value(DOMAIN_SERVER_SETTING_KEY).toString(); + + if (domainServerHostname.size() > 0) { + memset(_domainHostname, 0, MAX_HOSTNAME_BYTES); + memcpy(_domainHostname, domainServerHostname.toAscii().constData(), domainServerHostname.size()); + } + + settings->endGroup(); +} + +void NodeList::saveData(QSettings* settings) { + settings->beginGroup(DOMAIN_SERVER_SETTING_KEY); + + if (memcmp(_domainHostname, DEFAULT_DOMAIN_HOSTNAME, strlen(DEFAULT_DOMAIN_HOSTNAME)) != 0) { + // the user is using a different hostname, store it + settings->setValue(DOMAIN_SERVER_SETTING_KEY, QVariant(_domainHostname)); + } else { + // the user has switched back to default, remove the current setting + settings->remove(DOMAIN_SERVER_SETTING_KEY); + } + + settings->endGroup(); } NodeList::iterator NodeList::begin() const { diff --git a/libraries/shared/src/NodeList.h b/libraries/shared/src/NodeList.h index 20eaddc738..2a66fc7374 100644 --- a/libraries/shared/src/NodeList.h +++ b/libraries/shared/src/NodeList.h @@ -14,6 +14,8 @@ #include #include +#include + #include "Node.h" #include "UDPSocket.h" @@ -32,7 +34,7 @@ const int DOMAIN_SERVER_CHECK_IN_USECS = 1 * 1000000; extern const char SOLO_NODE_TYPES[3]; -const int MAX_HOSTNAME_BYTES = 255; +const int MAX_HOSTNAME_BYTES = 256; extern const char DEFAULT_DOMAIN_HOSTNAME[MAX_HOSTNAME_BYTES]; extern const char DEFAULT_DOMAIN_IP[INET_ADDRSTRLEN]; // IP Address will be re-set by lookup on startup @@ -99,6 +101,9 @@ public: void startSilentNodeRemovalThread(); void stopSilentNodeRemovalThread(); + void loadData(QSettings* settings); + void saveData(QSettings* settings); + friend class NodeListIterator; private: static NodeList* _sharedInstance; diff --git a/libraries/shared/src/PacketHeaders.h b/libraries/shared/src/PacketHeaders.h index 00c19e657a..c62abb85e9 100644 --- a/libraries/shared/src/PacketHeaders.h +++ b/libraries/shared/src/PacketHeaders.h @@ -29,6 +29,7 @@ const PACKET_TYPE PACKET_TYPE_VOXEL_DATA = 'V'; const PACKET_TYPE PACKET_TYPE_VOXEL_DATA_MONOCHROME = 'v'; const PACKET_TYPE PACKET_TYPE_BULK_AVATAR_DATA = 'X'; const PACKET_TYPE PACKET_TYPE_AVATAR_VOXEL_URL = 'U'; +const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_VIDEO = 'F'; const PACKET_TYPE PACKET_TYPE_TRANSMITTER_DATA_V2 = 'T'; const PACKET_TYPE PACKET_TYPE_ENVIRONMENT_DATA = 'e'; const PACKET_TYPE PACKET_TYPE_DOMAIN_LIST_REQUEST = 'L'; diff --git a/libraries/voxels/src/ViewFrustum.cpp b/libraries/voxels/src/ViewFrustum.cpp index c13da815f8..ae48dbbe7d 100644 --- a/libraries/voxels/src/ViewFrustum.cpp +++ b/libraries/voxels/src/ViewFrustum.cpp @@ -364,8 +364,6 @@ bool ViewFrustum::matches(const ViewFrustum& compareTo, bool debug) const { return result; } - - void ViewFrustum::computePickRay(float x, float y, glm::vec3& origin, glm::vec3& direction) const { origin = _nearTopLeft + x*(_nearTopRight - _nearTopLeft) + y*(_nearBottomLeft - _nearTopLeft); direction = glm::normalize(origin - _position); diff --git a/voxel-server/src/main.cpp b/voxel-server/src/main.cpp index 5b1ad57861..8a8048035b 100644 --- a/voxel-server/src/main.cpp +++ b/voxel-server/src/main.cpp @@ -44,9 +44,9 @@ const int MIN_BRIGHTNESS = 64; const float DEATH_STAR_RADIUS = 4.0; const float MAX_CUBE = 0.05f; -const int VOXEL_SEND_INTERVAL_USECS = 100 * 1000; -int PACKETS_PER_CLIENT_PER_INTERVAL = 30; -const int SENDING_TIME_TO_SPARE = 20 * 1000; // usec of sending interval to spare for calculating voxels +const int VOXEL_SEND_INTERVAL_USECS = 17 * 1000; // approximately 60fps +int PACKETS_PER_CLIENT_PER_INTERVAL = 20; +const int SENDING_TIME_TO_SPARE = 5 * 1000; // usec of sending interval to spare for calculating voxels const int MAX_VOXEL_TREE_DEPTH_LEVELS = 4;