Working on transmitting and receiving the face video.

avatar-mixer/src/main.cpp

@@ -117,6 +117,7 @@ int main(int argc, const char* argv[]) {
                     
                     break;
                 case PACKET_TYPE_AVATAR_VOXEL_URL:
+                case PACKET_TYPE_AVATAR_FACE_VIDEO:
                     // grab the node ID from the packet
                     unpackNodeId(packetData + numBytesForPacketHeader(packetData), &nodeID);
                     

interface/external/LibVPX/AUTHORS

@@ -0,0 +1,65 @@
+# This file is automatically generated from the git commit history
+# by tools/gen_authors.sh.
+
+Aaron Watry <awatry@gmail.com>
+Adrian Grange <agrange@google.com>
+Alex Converse <alex.converse@gmail.com>
+Alexis Ballier <aballier@gentoo.org>
+Alok Ahuja <waveletcoeff@gmail.com>
+Alpha Lam <hclam@google.com>
+Andoni Morales Alastruey <ylatuya@gmail.com>
+Andres Mejia <mcitadel@gmail.com>
+Aron Rosenberg <arosenberg@logitech.com>
+Attila Nagy <attilanagy@google.com>
+Deb Mukherjee <debargha@google.com>
+Fabio Pedretti <fabio.ped@libero.it>
+Frank Galligan <fgalligan@google.com>
+Fredrik Söderquist <fs@opera.com>
+Fritz Koenig <frkoenig@google.com>
+Gaute Strokkenes <gaute.strokkenes@broadcom.com>
+Giuseppe Scrivano <gscrivano@gnu.org>
+Guillermo Ballester Valor <gbvalor@gmail.com>
+Henrik Lundin <hlundin@google.com>
+James Berry <jamesberry@google.com>
+James Zern <jzern@google.com>
+Jan Kratochvil <jan.kratochvil@redhat.com>
+Jeff Faust <jfaust@google.com>
+Jeff Muizelaar <jmuizelaar@mozilla.com>
+Jim Bankoski <jimbankoski@google.com>
+Johann Koenig <johannkoenig@google.com>
+John Koleszar <jkoleszar@google.com>
+Joshua Bleecher Snyder <josh@treelinelabs.com>
+Justin Clift <justin@salasaga.org>
+Justin Lebar <justin.lebar@gmail.com>
+KO Myung-Hun <komh@chollian.net>
+Lou Quillio <louquillio@google.com>
+Luca Barbato <lu_zero@gentoo.org>
+Makoto Kato <makoto.kt@gmail.com>
+Marco Paniconi <marpan@google.com>
+Martin Ettl <ettl.martin78@googlemail.com>
+Michael Kohler <michaelkohler@live.com>
+Mike Hommey <mhommey@mozilla.com>
+Mikhal Shemer <mikhal@google.com>
+Pascal Massimino <pascal.massimino@gmail.com>
+Patrik Westin <patrik.westin@gmail.com>
+Paul Wilkins <paulwilkins@google.com>
+Pavol Rusnak <stick@gk2.sk>
+Philip Jägenstedt <philipj@opera.com>
+Priit Laes <plaes@plaes.org>
+Rafael Ávila de Espíndola <rafael.espindola@gmail.com>
+Rafaël Carré <funman@videolan.org>
+Ralph Giles <giles@xiph.org>
+Ronald S. Bultje <rbultje@google.com>
+Scott LaVarnway <slavarnway@google.com>
+Stefan Holmer <holmer@google.com>
+Taekhyun Kim <takim@nvidia.com>
+Takanori MATSUURA <t.matsuu@gmail.com>
+Tero Rintaluoma <teror@google.com>
+Thijs Vermeir <thijsvermeir@gmail.com>
+Timothy B. Terriberry <tterribe@xiph.org>
+Tom Finegan <tomfinegan@google.com>
+Yaowu Xu <yaowu@google.com>
+Yunqing Wang <yunqingwang@google.com>
+Google Inc.
+The Mozilla Foundation
+The Xiph.Org Foundation

interface/external/LibVPX/LICENSE

@@ -0,0 +1,31 @@
+Copyright (c) 2010, The WebM Project authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+  * Neither the name of Google, nor the WebM Project, nor the names
+    of its contributors may be used to endorse or promote products
+    derived from this software without specific prior written
+    permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

interface/external/LibVPX/PATENTS

@@ -0,0 +1,22 @@
+Additional IP Rights Grant (Patents)
+
+"This implementation" means the copyrightable works distributed by
+Google as part of the WebM Project.
+
+Google hereby grants to you a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer, and otherwise run, modify and propagate the contents of this
+implementation of VP8, where such license applies only to those patent
+claims, both currently owned by Google and acquired in the future,
+licensable by Google that are necessarily infringed by this
+implementation of VP8. This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation. If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of VP8 or any code incorporated within this
+implementation of VP8 constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of VP8
+shall terminate as of the date such litigation is filed.

interface/src/Application.cpp

@@ -920,6 +920,36 @@ void Application::sendPingPackets() {
                                               nodesToPing, sizeof(nodesToPing));
 }
 
+void Application::sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data) {
+    unsigned char packet[MAX_PACKET_SIZE];
+    unsigned char* packetPosition = packet;
+    
+    packetPosition += populateTypeAndVersion(packetPosition, PACKET_TYPE_AVATAR_FACE_VIDEO);
+    
+    *(uint16_t*)packetPosition = NodeList::getInstance()->getOwnerID();
+    packetPosition += sizeof(uint16_t);
+    
+    *(uint32_t*)packetPosition = frameCount;
+    packetPosition += sizeof(uint32_t);
+    
+    *(uint32_t*)packetPosition = data.size();
+    packetPosition += sizeof(uint32_t);
+    
+    uint32_t* offsetPosition = (uint32_t*)packetPosition;
+    packetPosition += sizeof(uint32_t);
+    
+    int headerSize = packetPosition - packet;
+    
+    // break the data up into submessages of the maximum size
+    *offsetPosition = 0;
+    while (*offsetPosition < data.size()) {
+        int payloadSize = min(data.size() - (int)*offsetPosition, MAX_PACKET_SIZE - headerSize);
+        memcpy(packetPosition, data.constData() + *offsetPosition, payloadSize);
+        getInstance()->controlledBroadcastToNodes(packet, headerSize + payloadSize, &NODE_TYPE_AVATAR_MIXER, 1);
+        *offsetPosition += payloadSize;
+    }
+}
+
 //  Every second, check the frame rates and other stuff
 void Application::timer() {
     gettimeofday(&_timerEnd, NULL);
@@ -1029,7 +1059,7 @@ void Application::sendAvatarVoxelURLMessage(const QUrl& url) {
     controlledBroadcastToNodes((unsigned char*)message.data(), message.size(), & NODE_TYPE_AVATAR_MIXER, 1);
 }
 
-void Application::processAvatarVoxelURLMessage(unsigned char *packetData, size_t dataBytes) {
+static Avatar* processAvatarMessageHeader(unsigned char*& packetData, size_t& dataBytes) {
     // skip the header
     int numBytesPacketHeader = numBytesForPacketHeader(packetData);
     packetData += numBytesPacketHeader;
@@ -1043,18 +1073,31 @@ void Application::processAvatarVoxelURLMessage(unsigned char *packetData, size_t
     // make sure the node exists
     Node* node = NodeList::getInstance()->nodeWithID(nodeID);
     if (!node || !node->getLinkedData()) {
-        return;
+        return 0;
     }
     Avatar* avatar = static_cast<Avatar*>(node->getLinkedData());
-    if (!avatar->isInitialized()) {
-        return; // wait until initialized
-    }
+    return avatar->isInitialized() ? avatar : 0;
+}
+
+void Application::processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes) {
+    Avatar* avatar = processAvatarMessageHeader(packetData, dataBytes);
+    if (avatar == 0) {
+        return;
+    } 
     QUrl url = QUrl::fromEncoded(QByteArray((char*)packetData, dataBytes));
     
     // invoke the set URL function on the simulate/render thread
     QMetaObject::invokeMethod(avatar->getVoxels(), "setVoxelURL", Q_ARG(QUrl, url));
 }
 
+void Application::processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes) {
+    Avatar* avatar = processAvatarMessageHeader(packetData, dataBytes);
+    if (avatar == 0) {
+        return;
+    }
+    avatar->getHead().getFace().processVideoMessage(packetData, dataBytes);
+}
+
 void Application::checkBandwidthMeterClick() {
     // ... to be called upon button release
 
@@ -3358,6 +3401,9 @@ void* Application::networkReceive(void* args) {
                     case PACKET_TYPE_AVATAR_VOXEL_URL:
                         processAvatarVoxelURLMessage(app->_incomingPacket, bytesReceived);
                         break;
+                    case PACKET_TYPE_AVATAR_FACE_VIDEO:
+                        processAvatarFaceVideoMessage(app->_incomingPacket, bytesReceived);
+                        break;
                     default:
                         NodeList::getInstance()->processNodeData(&senderAddress, app->_incomingPacket, bytesReceived);
                         break;

interface/src/Application.h

@@ -99,6 +99,10 @@ public:
     
     QNetworkAccessManager* getNetworkAccessManager() { return _networkAccessManager; }
     GeometryCache* getGeometryCache() { return &_geometryCache; }
+
+public slots:
+
+    void sendAvatarFaceVideoMessage(int frameCount, const QByteArray& data);    
     
 private slots:
     
@@ -178,7 +182,8 @@ private:
     static bool sendVoxelsOperation(VoxelNode* node, void* extraData);
     static void sendVoxelEditMessage(PACKET_TYPE type, VoxelDetail& detail);
     static void sendAvatarVoxelURLMessage(const QUrl& url);
-    static void processAvatarVoxelURLMessage(unsigned char *packetData, size_t dataBytes);
+    static void processAvatarVoxelURLMessage(unsigned char* packetData, size_t dataBytes);
+    static void processAvatarFaceVideoMessage(unsigned char* packetData, size_t dataBytes);
     static void sendPingPackets();
     
     void initMenu();

interface/src/Webcam.cpp

@@ -279,7 +279,8 @@ void Webcam::setFrame(const Mat& color, int format, const Mat& depth, const Rota
     QTimer::singleShot(qMax((int)remaining / 1000, 0), _grabber, SLOT(grabFrame()));
 }
 
-FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0), _depthOffset(0.0) {
+FrameGrabber::FrameGrabber() : _initialized(false), _capture(0), _searchWindow(0, 0, 0, 0),
+    _depthOffset(0.0), _codec(), _frameCount(0) {
 }
 
 FrameGrabber::~FrameGrabber() {
@@ -387,6 +388,10 @@ void FrameGrabber::shutdown() {
         cvReleaseCapture(&_capture);
         _capture = 0;
     }
+    if (_codec.name != 0) {
+        vpx_codec_destroy(&_codec);
+        _codec.name = 0;
+    }
     _initialized = false;
     
     thread()->quit();
@@ -507,7 +512,28 @@ void FrameGrabber::grabFrame() {
     }
 #endif
 
-    //vpx_codec_encode(&_encoderContext, img, pts, duration, 0, VPX_DL_REALTIME);
+    if (_codec.name == 0) {
+        // initialize encoder context
+        vpx_codec_enc_cfg_t codecConfig;
+        vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &codecConfig, 0);
+        codecConfig.rc_target_bitrate = color.cols * color.rows * codecConfig.rc_target_bitrate /
+            codecConfig.g_w / codecConfig.g_h;
+        codecConfig.g_w = color.cols;
+        codecConfig.g_h = color.rows;
+        vpx_codec_enc_init(&_codec, vpx_codec_vp8_cx(), &codecConfig, 0); 
+    }
+    vpx_image_t vpxImage;
+    vpx_img_wrap(&vpxImage, VPX_IMG_FMT_YV12, color.cols, color.rows, 1, color.ptr());    
+    vpx_codec_encode(&_codec, &vpxImage, ++_frameCount, 1, 0, VPX_DL_REALTIME);
+
+    vpx_codec_iter_t iterator = 0;
+    const vpx_codec_cx_pkt_t* packet;
+    while ((packet = vpx_codec_get_cx_data(&_codec, &iterator)) != 0) {
+        if (packet->kind == VPX_CODEC_CX_FRAME_PKT) {
+            QMetaObject::invokeMethod(Application::getInstance(), "sendAvatarFaceVideoMessage", Q_ARG(int, _frameCount),
+                Q_ARG(QByteArray, QByteArray((const char*)packet->data.frame.buf, packet->data.frame.sz)));
+        }
+    }
 
     QMetaObject::invokeMethod(Application::getInstance()->getWebcam(), "setFrame",
         Q_ARG(cv::Mat, color), Q_ARG(int, format), Q_ARG(cv::Mat, _grayDepthFrame),
@@ -524,9 +550,6 @@ bool FrameGrabber::init() {
         return false;
     }
 
-    // initialize encoder context
-    vpx_codec_enc_init(&_encoderContext, vpx_codec_vp8_cx(), 0, 0);
-
     // first try for a Kinect
 #ifdef HAVE_OPENNI
     _xnContext.Init();

interface/src/Webcam.h

@@ -119,7 +119,8 @@ private:
     cv::Mat _grayDepthFrame;
     double _depthOffset;
     
-    vpx_codec_ctx_t _encoderContext;
+    vpx_codec_ctx_t _codec;
+    int _frameCount;
     
 #ifdef HAVE_OPENNI
     xn::Context _xnContext;

interface/src/avatar/Face.cpp

@@ -8,6 +8,11 @@
 
 #include <glm/gtx/quaternion.hpp>
 
+#include <vpx_decoder.h>
+#include <vp8dx.h>
+
+#include <PacketHeaders.h>
+
 #include "Avatar.h"
 #include "Head.h"
 #include "Face.h"
@@ -22,7 +27,56 @@ int Face::_texCoordUpLocation;
 GLuint Face::_vboID;
 GLuint Face::_iboID;
 
-Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH), _colorTextureID(0), _depthTextureID(0) {
+Face::Face(Head* owningHead) : _owningHead(owningHead), _renderMode(MESH),
+    _colorTextureID(0), _depthTextureID(0), _codec(), _frameCount(0) {
+}
+
+Face::~Face() {
+    if (_codec.name != 0) {
+        vpx_codec_destroy(&_codec);
+        _codec.name = 0;
+    }
+}
+
+int Face::processVideoMessage(unsigned char* packetData, size_t dataBytes) {
+    if (_codec.name == 0) {
+        // initialize decoder context
+        vpx_codec_dec_init(&_codec, vpx_codec_vp8_dx(), 0, 0);
+    }
+    // skip the header
+    unsigned char* packetPosition = packetData;
+
+    int frameCount = *(uint32_t*)packetPosition;
+    packetPosition += sizeof(uint32_t);
+    
+    int frameSize = *(uint32_t*)packetPosition;
+    packetPosition += sizeof(uint32_t);
+    
+    int frameOffset = *(uint32_t*)packetPosition;
+    packetPosition += sizeof(uint32_t);
+    
+    if (frameCount < _frameCount) { // old frame; ignore
+        return dataBytes; 
+    
+    } else if (frameCount > _frameCount) { // new frame; reset
+        _frameCount = frameCount;
+        _frameBytesRemaining = frameSize;
+        _arrivingFrame.resize(frameSize);
+    }
+    
+    int payloadSize = dataBytes - (packetPosition - packetData);
+    memcpy(_arrivingFrame.data() + frameOffset, packetPosition, payloadSize);
+    
+    if ((_frameBytesRemaining -= payloadSize) <= 0) {
+        int result = vpx_codec_decode(&_codec, (const uint8_t*)_arrivingFrame.constData(), _arrivingFrame.size(), 0, 0);
+        vpx_codec_iter_t iterator = 0;
+        vpx_image_t* image;
+        while ((image = vpx_codec_get_frame(&_codec, &iterator)) != 0) {
+            
+        }
+    }
+    
+    return dataBytes;
 }
 
 bool Face::render(float alpha) {

interface/src/avatar/Face.h

@@ -13,6 +13,8 @@
 
 #include <opencv2/opencv.hpp>
 
+#include <vpx_codec.h>
+
 #include "InterfaceConfig.h"
 
 class Head;
@@ -24,12 +26,15 @@ class Face : public QObject {
 public:
     
     Face(Head* owningHead);
+    ~Face();
 
     void setColorTextureID(GLuint colorTextureID) { _colorTextureID = colorTextureID; }
     void setDepthTextureID(GLuint depthTextureID) { _depthTextureID = depthTextureID; }
     void setTextureSize(const cv::Size2f& textureSize) { _textureSize = textureSize; }
     void setTextureRect(const cv::RotatedRect& textureRect) { _textureRect = textureRect; }
     
+    int processVideoMessage(unsigned char* packetData, size_t dataBytes);
+    
     bool render(float alpha);
     
 public slots:
@@ -47,6 +52,12 @@ private:
     cv::Size2f _textureSize;
     cv::RotatedRect _textureRect;
     
+    vpx_codec_ctx_t _codec;
+    
+    QByteArray _arrivingFrame;
+    int _frameCount;
+    int _frameBytesRemaining;
+    
     static ProgramObject* _program;
     static int _texCoordCornerLocation;
     static int _texCoordRightLocation;

libraries/shared/src/PacketHeaders.h

@@ -29,6 +29,7 @@ const PACKET_TYPE PACKET_TYPE_VOXEL_DATA = 'V';
 const PACKET_TYPE PACKET_TYPE_VOXEL_DATA_MONOCHROME = 'v';
 const PACKET_TYPE PACKET_TYPE_BULK_AVATAR_DATA = 'X';
 const PACKET_TYPE PACKET_TYPE_AVATAR_VOXEL_URL = 'U';
+const PACKET_TYPE PACKET_TYPE_AVATAR_FACE_VIDEO = 'F';
 const PACKET_TYPE PACKET_TYPE_TRANSMITTER_DATA_V2 = 'T';
 const PACKET_TYPE PACKET_TYPE_ENVIRONMENT_DATA = 'e';
 const PACKET_TYPE PACKET_TYPE_DOMAIN_LIST_REQUEST = 'L';