diff --git a/BUILD.md b/BUILD.md index 487bcb0eb1..b8bc1cd14c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -4,7 +4,7 @@ Dependencies * [Qt](http://qt-project.org/downloads) ~> 5.2.0 * [zLib](http://www.zlib.net/) ~> 1.2.8 * [glm](http://glm.g-truc.net/0.9.5/index.html) ~> 0.9.5.2 -* [qxmpp](https://code.google.com/p/qxmpp/) ~> 0.7.6 +* [qxmpp](https://github.com/qxmpp-project/qxmpp/) ~> 0.7.6 * [GnuTLS](http://gnutls.org/download.html) ~> 3.2.12 * IMPORTANT: GnuTLS 3.2.12 is critical to avoid a security vulnerability. @@ -83,17 +83,41 @@ If the build completes successfully, you will have built targets for all compone Windows === -Currently building on Windows has only been tested on Windows SDK 7.1 with Visual Studio C++ 2010 Express. +####Visual Studio + +Currently building on Windows has been tested using the following compilers: +* Visual Studio C++ 2010 Express +* Visual Studio 2013 + +(If anyone can test using Visual Studio 2013 Express then please update this document) + +#####Windows SDK 7.1 + +Whichever version of Visual Studio you use, first install [Microsoft Windows SDK for Windows 7 and .NET Framework 4](http://www.microsoft.com/en-us/download/details.aspx?id=8279). + +#####Visual Studio C++ 2010 Express Visual Studio C++ 2010 Express can be downloaded [here](http://www.visualstudio.com/en-us/downloads#d-2010-express). The following patches/service packs are also required: -* [Windows SDK 7.1/.NET 4 Framework](http://www.microsoft.com/en-us/download/details.aspx?id=8279) * [VS2010 SP1](http://www.microsoft.com/en-us/download/details.aspx?id=23691) * [VS2010 SP1 Compiler Update](http://www.microsoft.com/en-us/download/details.aspx?id=4422) +Some of the build instructions will ask you to start a Visual Studio Command Prompt. You should have a shortcut in your Start menu called "Open Visual Studio Command Prompt (2010)" which will do so. + +#####Visual Studio 2013 + +This product must be purchased separately. + +Visual Studio 2013 doesn't have a shortcut to start a Visual Studio Command Prompt. Instead, start a regular command prompt and then run: + + "%VS120COMNTOOLS%\vsvars32.bat" + ####Qt -You can use the online installer, or the offline installer. If you use the offline installer, be sure to select the "OpenGL" version. +You can use the online installer or the offline installer. If you use the offline installer, be sure to select the "OpenGL" version. + +NOTE: Qt does not support 64-bit builds on Windows 7, so you must use the 32-bit version of libraries for interface.exe to run. The 32-bit version of the static library is the one linked by our CMake find modules. + * Download the online installer [here](http://qt-project.org/downloads) * When it asks you to select components, ONLY select the following: * Qt > Qt 5.2.0 > **msvc2010 32-bit OpenGL** @@ -101,67 +125,135 @@ You can use the online installer, or the offline installer. If you use the offli * Download the offline installer [here](http://download.qt-project.org/official_releases/qt/5.2/5.2.0/qt-windows-opensource-5.2.0-msvc2010_opengl-x86-offline.exe) Once Qt is installed, you need to manually configure the following: -* Make sure the Qt runtime DLLs are loadable (You could add the Qt\5.2.0\msvc2010_opengl\bin\ directory to your path.) - You must do this before you attempt to build because some tools for the build depend on Qt. -* Set the QT_CMAKE_PREFIX_PATH environment variable to your Qt\5.2.0\msvc2010_opengl directory - -####zLib -NOTE: zLib should configure itself correctly on install. However, sometimes zLib doesn't properly detect system components and fails to configure itself correctly. When it fails, it will not correctly set the #if HAVE_UNISTD_H at line 287 of zconf.h to #if 0... if it fails, you're build will have errors in the voxels target. You can correct this by setting the #if to 0 instead of 1, since Windows does not have unistd.h. +* Make sure the Qt runtime DLLs are loadable. You must do this before you attempt to build because some tools for the build depend on Qt. E.g., add to the PATH: `Qt\5.2.0\msvc2010_opengl\bin\`. +* Set the QT_CMAKE_PREFIX_PATH environment variable to your `Qt\5.2.0\msvc2010_opengl` directory. ####External Libraries CMake will need to know where the headers and libraries for required external dependencies are. -If you installed zLib using the installer, the Cmake find module for zLib should locate it on your system. - -The recommended route for CMake to find the other external dependencies is to place all of the dependencies in one folder and set one ENV variable - HIFI_LIB_DIR. That ENV variable should point to a directory with the following structure: +The recommended route for CMake to find the external dependencies is to place all of the dependencies in one folder and set one ENV variable - HIFI_LIB_DIR. That ENV variable should point to a directory with the following structure: root_lib_dir - -> glm - -> glm - -> glm.hpp + -> freeglut + -> bin + -> include + -> lib -> glew -> bin -> include -> lib - -> freeglut + -> glm + -> glm + -> glm.hpp + -> gnutls -> bin -> include -> lib -> qxmpp -> include -> lib - -> gnutls - -> bin - -> include - -> lib + -> zlib + -> include + -> lib + -> test -For many of the external libraries where precompiled binaries are readily available you should be able to simply copy the extracted folder that you get from the download links provided at the top of the guide. Otherwise you may need to build from source and install the built product to this directory. The `root_lib_dir` in the above example can be wherever you choose on your system - as long as the environment variable HIFI_LIB_DIR is set to it. +For many of the external libraries where precompiled binaries are readily available you should be able to simply copy the extracted folder that you get from the download links provided at the top of the guide. Otherwise you may need to build from source and install the built product to this directory. The `root_lib_dir` in the above example can be wherever you choose on your system - as long as the environment variable HIFI_LIB_DIR is set to it. From here on, whenever you see %HIFI_LIB_DIR% you should substitute the directory that you chose. -*NOTE: Be careful with glm. For the folder other libraries would normally call 'include', the folder containing the headers, glm opts to use 'glm'. You will have a glm folder nested inside the top-level glm folder.* +As with the Qt libraries, you will need to make sure that directories containing DLL'S are in your path. Where possible, you can use static builds of the external dependencies to avoid this requirement. -*NOTE: Qt does not support 64-bit builds on Windows 7, so you must use the 32-bit version of libraries for interface.exe to run. The 32-bit version of the static library is the one linked by our CMake find modules* +#### OpenSSL -##### DLLs -As with the Qt libraries, you will need to make sure the directories containing dynamically-linked libraries is in your path. +QT will use OpenSSL if it's available, but it doesn't install it, so you must install it separately. -For example, for a dynamically linked build of freeglut, the directory to add to your path in which the DLL is found is `FREEGLUT_DIR/bin`. Where possible, you can use static builds of the external dependencies to avoid this requirement. +Your system may already have several versions of the OpenSSL DLL's (ssleay32.dll, libeay32.dll) lying around, but they may be the wrong version. If these DLL's are in the PATH then QT will try to use them, and if they're the wrong version then you will see the following errors in the console: + + QSslSocket: cannot resolve TLSv1_1_client_method + QSslSocket: cannot resolve TLSv1_2_client_method + QSslSocket: cannot resolve TLSv1_1_server_method + QSslSocket: cannot resolve TLSv1_2_server_method + QSslSocket: cannot resolve SSL_select_next_proto + QSslSocket: cannot resolve SSL_CTX_set_next_proto_select_cb + QSslSocket: cannot resolve SSL_get0_next_proto_negotiated + +To prevent these problems, install OpenSSL yourself. Download the following binary packages [from this website](http://slproweb.com/products/Win32OpenSSL.html): +* Visual C++ 2008 Redistributables +* Win32 OpenSSL v1.0.1h + +Install OpenSSL into the Windows system directory, to make sure that QT uses the version that you've just installed, and not some other version. + +#### Zlib + +Download the compiled DLL from the [zlib website](http://www.zlib.net/). Extract to %HIFI_LIB_DIR%\zlib. + +Add the following environment variables (remember to substitute your own directory for %HIFI_LIB_DIR%): + + ZLIB_LIBRARY=%HIFI_LIB_DIR%\zlib\lib\zdll.lib + ZLIB_INCLUDE_DIR=%HIFI_LIB_DIR%\zlib\include + +Add to the PATH: `%HIFI_LIB_DIR%\zlib` + +Important! This should be added at the beginning of the path, not the end. That's because your +system likely has many copies of zlib1.dll, and you want High Fidelity to use the correct version. If High Fidelity picks up the wrong zlib1.dll then it might be unable to use it, and that would cause it to fail to start, showing only the cryptic error "The application was unable to start correctly: 0xc0000022". + +#### freeglut + +Download the binary package: `freeglut-MSVC-2.8.1-1.mp.zip`. Extract to %HIFI_LIB_DIR%\freeglut. + +Add to the PATH: `%HIFI_LIB_DIR%\freeglut\bin` + +#### GLEW + +Download the binary package: `glew-1.10.0-win32.zip`. Extract to %HIFI_LIB_DIR%\glew (you'll need to rename the default directory name). + +Add to the PATH: `%HIFI_LIB_DIR%\glew\bin\Release\Win32` + +#### GLM + +This package contains only headers, so there's nothing to add to the PATH. + +Be careful with glm. For the folder other libraries would normally call 'include', the folder containing the headers, glm opts to use 'glm'. You will have a glm folder nested inside the top-level glm folder. + +#### GnuTLS -#####GnuTLS You can get a precompiled version of GnuTLS for Windows [here](http://gnutls.org/download.html). -To use GnuTLS with Visual Studio, you will need to create `libgnutls-28.lib`, the import library for Visual Studio projects. this is done using the `lib` command in the `bin` folder of your GnuTLS download. Run the following in a Visual Studio Command Prompt (found in the tools menu of Visual Studio). +To use GnuTLS with Visual Studio, you will need to create `libgnutls-28.lib`, the import library for Visual Studio projects. This is done using the `lib` command in the `bin` folder of your GnuTLS download. Start a Visual Studio Command Prompt, and then run: - $GNUTLS_DIR\bin> lib /def:libgnutls-28.def + cd %HIFI_LIB_DIR%\gnutls\bin + lib /def:libgnutls-28.def + copy libgnutls-28.lib ..\lib -This will create `libgnutls-28.lib` in the `bin` folder. Copy that file to the `lib` sub-folder of your GnuTLS folder, and the Cmake FindGnuTLS module in this repo will find it during the Cmake run. +The Cmake FindGnuTLS module will now find libgnutls-28.lib during the Cmake run. -####Building in Visual Studio +Add to the PATH: `%HIFI_LIB_DIR%\gnutls\bin` + +#### qxmpp + +Download a source-code release from the [qxmpp GitHub page](https://github.com/qxmpp-project/qxmpp/releases). + +Start a Visual Studio Command Prompt. + + mkdir %HIFI_LIB_DIR%\build + tar xfz qxmpp-0.7.6.tar.gz -C %HIFI_LIB_DIR%\build + cd %HIFI_LIB_DIR%\build\qxmpp-0.7.6 + qmake PREFIX=%HIFI_LIB_DIR%\qxmpp # This creates "Makefile" + nmake + nmake install + +Add to the PATH: `%HIFI_LIB_DIR%\qxmpp\lib` + +#### Build High Fidelity using Visual Studio Follow the same build steps from the CMake section, but pass a different generator to CMake. - cmake .. -G "Visual Studio 10" + cmake .. -DZLIB_LIBRARY=%ZLIB_LIBRARY% -DZLIB_INCLUDE_DIR=%ZLIB_INCLUDE_DIR% -G "Visual Studio 10" + +If you're using Visual Studio 2013 then pass "Visual Studio 12" instead of "Visual Studio 10" (yes, 12, not 13). + +Open %HIFI_DIR%\build\hifi.sln and compile. ####Running Interface -If you need to debug Interface, you can run interface from within Visual Studio (see the section below). You can also run Interface by launching it from command line or File Explorer from $YOUR_HIFI_PATH\build\interface\Debug\interface.exe +If you need to debug Interface, you can run interface from within Visual Studio (see the section below). You can also run Interface by launching it from command line or File Explorer from %HIFI_DIR%\build\interface\Debug\interface.exe ####Debugging Interface * In the Solution Explorer, right click interface and click Set as StartUp Project diff --git a/CMakeLists.txt b/CMakeLists.txt index cb1e4224cf..a399e11168 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,4 +49,5 @@ add_subdirectory(assignment-client) add_subdirectory(domain-server) add_subdirectory(interface) add_subdirectory(tests) +add_subdirectory(tools) add_subdirectory(voxel-edit) diff --git a/assignment-client/src/models/ModelServer.h b/assignment-client/src/models/ModelServer.h index 38acc7f1e1..8a26f07f11 100644 --- a/assignment-client/src/models/ModelServer.h +++ b/assignment-client/src/models/ModelServer.h @@ -33,6 +33,7 @@ public: virtual const char* getMyServerName() const { return MODEL_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return MODEL_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_MODELS_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeModelEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 545c502036..76a6845342 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include #include @@ -17,6 +18,7 @@ #include "OctreeInboundPacketProcessor.h" static QUuid DEFAULT_NODE_ID_REF; +const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServer) : _myServer(myServer), @@ -25,7 +27,8 @@ OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServe _totalProcessTime(0), _totalLockWaitTime(0), _totalElementsInPacket(0), - _totalPackets(0) + _totalPackets(0), + _lastNackTime(usecTimestampNow()) { } @@ -35,10 +38,38 @@ void OctreeInboundPacketProcessor::resetStats() { _totalLockWaitTime = 0; _totalElementsInPacket = 0; _totalPackets = 0; + _lastNackTime = usecTimestampNow(); _singleSenderStats.clear(); } +unsigned long OctreeInboundPacketProcessor::getMaxWait() const { + // calculate time until next sendNackPackets() + quint64 nextNackTime = _lastNackTime + TOO_LONG_SINCE_LAST_NACK; + quint64 now = usecTimestampNow(); + if (now >= nextNackTime) { + return 0; + } + return (nextNackTime - now) / USECS_PER_MSEC + 1; +} + +void OctreeInboundPacketProcessor::preProcess() { + // check if it's time to send a nack. If yes, do so + quint64 now = usecTimestampNow(); + if (now - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + sendNackPackets(); + } +} + +void OctreeInboundPacketProcessor::midProcess() { + // check if it's time to send a nack. If yes, do so + quint64 now = usecTimestampNow(); + if (now - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + sendNackPackets(); + } +} void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet) { @@ -123,13 +154,13 @@ void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendin qDebug() << "sender has no known nodeUUID."; } } - trackInboundPackets(nodeUUID, sequence, transitTime, editsInPacket, processTime, lockWaitTime); + trackInboundPacket(nodeUUID, sequence, transitTime, editsInPacket, processTime, lockWaitTime); } else { qDebug("unknown packet ignored... packetType=%d", packetType); } } -void OctreeInboundPacketProcessor::trackInboundPackets(const QUuid& nodeUUID, int sequence, quint64 transitTime, +void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, unsigned short int sequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { _totalTransitTime += transitTime; @@ -142,31 +173,169 @@ void OctreeInboundPacketProcessor::trackInboundPackets(const QUuid& nodeUUID, in // see if this is the first we've heard of this node... if (_singleSenderStats.find(nodeUUID) == _singleSenderStats.end()) { SingleSenderStats stats; - - stats._totalTransitTime += transitTime; - stats._totalProcessTime += processTime; - stats._totalLockWaitTime += lockWaitTime; - stats._totalElementsInPacket += editsInPacket; - stats._totalPackets++; - + stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); _singleSenderStats[nodeUUID] = stats; } else { SingleSenderStats& stats = _singleSenderStats[nodeUUID]; - stats._totalTransitTime += transitTime; - stats._totalProcessTime += processTime; - stats._totalLockWaitTime += lockWaitTime; - stats._totalElementsInPacket += editsInPacket; - stats._totalPackets++; + stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); } } +int OctreeInboundPacketProcessor::sendNackPackets() { -SingleSenderStats::SingleSenderStats() { - _totalTransitTime = 0; - _totalProcessTime = 0; - _totalLockWaitTime = 0; - _totalElementsInPacket = 0; - _totalPackets = 0; + int packetsSent = 0; + char packet[MAX_PACKET_SIZE]; + + NodeToSenderStatsMapIterator i = _singleSenderStats.begin(); + while (i != _singleSenderStats.end()) { + + QUuid nodeUUID = i.key(); + SingleSenderStats nodeStats = i.value(); + + // check if this node is still alive. Remove its stats if it's dead. + if (!isAlive(nodeUUID)) { + i = _singleSenderStats.erase(i); + continue; + } + + // if there are packets from _node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. + if (hasPacketsToProcessFrom(nodeUUID)) { + i++; + continue; + } + + const SharedNodePointer& destinationNode = NodeList::getInstance()->getNodeHash().value(nodeUUID); + const QSet& missingSequenceNumbers = nodeStats.getMissingSequenceNumbers(); + + // construct nack packet(s) for this node + int numSequenceNumbersAvailable = missingSequenceNumbers.size(); + QSet::const_iterator missingSequenceNumberIterator = missingSequenceNumbers.constBegin(); + while (numSequenceNumbersAvailable > 0) { + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, _myServer->getMyEditNackType()); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + // calculate and pack the number of sequence numbers to nack + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(unsigned short int); + uint16_t numSequenceNumbers = std::min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers to nack + for (uint16_t i = 0; i < numSequenceNumbers; i++) { + unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt; + *sequenceNumberAt = *missingSequenceNumberIterator; + dataAt += sizeof(unsigned short int); + + missingSequenceNumberIterator++; + } + numSequenceNumbersAvailable -= numSequenceNumbers; + + // send it + NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); + packetsSent++; + } + i++; + } + return packetsSent; } +SingleSenderStats::SingleSenderStats() + : _totalTransitTime(0), + _totalProcessTime(0), + _totalLockWaitTime(0), + _totalElementsInPacket(0), + _totalPackets(0), + _incomingLastSequence(0), + _missingSequenceNumbers() +{ + +} + +void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, + int editsInPacket, quint64 processTime, quint64 lockWaitTime) { + + const int UINT16_RANGE = std::numeric_limits::max() + 1; + const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work + const int MAX_MISSING_SEQUENCE_SIZE = 100; + + unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + (unsigned short int)1; + + if (incomingSequence == expectedSequence) { // on time + _incomingLastSequence = incomingSequence; + } else { // out of order + int incoming = (int)incomingSequence; + int expected = (int)expectedSequence; + + // check if the gap between incoming and expected is reasonable, taking possible rollover into consideration + int absGap = std::abs(incoming - expected); + if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) { + // rollover likely occurred between incoming and expected. + // correct the larger of the two so that it's within [-UINT16_RANGE, -1] while the other remains within [0, UINT16_RANGE-1] + if (incoming > expected) { + incoming -= UINT16_RANGE; + } else { + expected -= UINT16_RANGE; + } + } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { + // ignore packet if gap is unreasonable + qDebug() << "ignoring unreasonable packet... sequence:" << incomingSequence + << "_incomingLastSequence:" << _incomingLastSequence; + return; + } + + // now that rollover has been corrected for (if it occurred), incoming and expected can be + // compared to each other directly, though one of them might be negative + if (incoming > expected) { // early + // add all sequence numbers that were skipped to the missing sequence numbers list + for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { + _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); + } + _incomingLastSequence = incomingSequence; + } else { // late + // remove this from missing sequence number if it's in there + _missingSequenceNumbers.remove(incomingSequence); + + // do not update _incomingLastSequence; it shouldn't become smaller + } + } + + // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP + // will be removed. + if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { + // some older sequence numbers may be from before a rollover point; this must be handled. + // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received + // before the most recent rollover. + int cutoff = (int)_incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP; + if (cutoff >= 0) { + foreach(unsigned short int missingSequence, _missingSequenceNumbers) { + unsigned short int nonRolloverCutoff = (unsigned short int)cutoff; + if (missingSequence > _incomingLastSequence || missingSequence <= nonRolloverCutoff) { + _missingSequenceNumbers.remove(missingSequence); + } + } + } else { + unsigned short int rolloverCutoff = (unsigned short int)(cutoff + UINT16_RANGE); + foreach(unsigned short int missingSequence, _missingSequenceNumbers) { + if (missingSequence > _incomingLastSequence && missingSequence <= rolloverCutoff) { + _missingSequenceNumbers.remove(missingSequence); + } + } + } + } + + // update other stats + _totalTransitTime += transitTime; + _totalProcessTime += processTime; + _totalLockWaitTime += lockWaitTime; + _totalElementsInPacket += editsInPacket; + _totalPackets++; +} diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index f637a9e7c9..46a57205cb 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -32,16 +32,24 @@ public: { return _totalElementsInPacket == 0 ? 0 : _totalProcessTime / _totalElementsInPacket; } quint64 getAverageLockWaitTimePerElement() const { return _totalElementsInPacket == 0 ? 0 : _totalLockWaitTime / _totalElementsInPacket; } - + const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } + + void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, + int editsInPacket, quint64 processTime, quint64 lockWaitTime); + quint64 _totalTransitTime; quint64 _totalProcessTime; quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; + + unsigned short int _incomingLastSequence; + QSet _missingSequenceNumbers; }; -typedef std::map NodeToSenderStatsMap; -typedef std::map::iterator NodeToSenderStatsMapIterator; +typedef QHash NodeToSenderStatsMap; +typedef QHash::iterator NodeToSenderStatsMapIterator; +typedef QHash::const_iterator NodeToSenderStatsMapConstIterator; /// Handles processing of incoming network packets for the voxel-server. As with other ReceivedPacketProcessor classes @@ -66,10 +74,18 @@ public: NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } protected: + virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); + virtual unsigned long getMaxWait() const; + virtual void preProcess(); + virtual void midProcess(); + private: - void trackInboundPackets(const QUuid& nodeUUID, int sequence, quint64 transitTime, + int sendNackPackets(); + +private: + void trackInboundPacket(const QUuid& nodeUUID, unsigned short int sequence, quint64 transitTime, int voxelsInPacket, quint64 processTime, quint64 lockWaitTime); OctreeServer* _myServer; @@ -82,5 +98,7 @@ private: quint64 _totalPackets; NodeToSenderStatsMap _singleSenderStats; + + quint64 _lastNackTime; }; #endif // hifi_OctreeInboundPacketProcessor_h diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index cb60f0816e..48c8674c03 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -648,10 +648,10 @@ bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url int senderNumber = 0; NodeToSenderStatsMap& allSenderStats = _octreeInboundPacketProcessor->getSingleSenderStats(); - for (NodeToSenderStatsMapIterator i = allSenderStats.begin(); i != allSenderStats.end(); i++) { + for (NodeToSenderStatsMapConstIterator i = allSenderStats.begin(); i != allSenderStats.end(); i++) { senderNumber++; - QUuid senderID = i->first; - SingleSenderStats& senderStats = i->second; + QUuid senderID = i.key(); + const SingleSenderStats& senderStats = i.value(); statsString += QString("\r\n Stats for sender %1 uuid: %2\r\n") .arg(senderNumber).arg(senderID.toString()); @@ -851,7 +851,6 @@ void OctreeServer::readPendingDatagrams() { // If we got a nack packet, then we're talking to an agent, and we // need to make sure we have it in our nodeList. if (matchingNode) { - nodeList->updateNodeWithDataFromPacket(matchingNode, receivedPacket); OctreeQueryNode* nodeData = (OctreeQueryNode*)matchingNode->getLinkedData(); if (nodeData) { nodeData->parseNackPacket(receivedPacket); @@ -1060,6 +1059,9 @@ void OctreeServer::nodeAdded(SharedNodePointer node) { void OctreeServer::nodeKilled(SharedNodePointer node) { quint64 start = usecTimestampNow(); + // calling this here since nodeKilled slot in ReceivedPacketProcessor can't be triggered by signals yet!! + _octreeInboundPacketProcessor->nodeKilled(node); + qDebug() << qPrintable(_safeServerName) << "server killed node:" << *node; OctreeQueryNode* nodeData = static_cast(node->getLinkedData()); if (nodeData) { diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 5595d139be..76b39c5771 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -68,6 +68,7 @@ public: virtual const char* getMyServerName() const = 0; virtual const char* getMyLoggingServerTargetName() const = 0; virtual const char* getMyDefaultPersistFilename() const = 0; + virtual PacketType getMyEditNackType() const = 0; // subclass may implement these method virtual void beforeRun() { }; diff --git a/assignment-client/src/octree/SentPacketHistory.cpp b/assignment-client/src/octree/SentPacketHistory.cpp deleted file mode 100644 index 0ea7fd8b69..0000000000 --- a/assignment-client/src/octree/SentPacketHistory.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// -// SentPacketHistory.cpp -// assignement-client/src/octree -// -// Created by Yixin Wang on 6/5/2014 -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "SentPacketHistory.h" - -SentPacketHistory::SentPacketHistory(int size) - : _sentPackets(size), - _newestPacketAt(0), - _numExistingPackets(0), - _newestSequenceNumber(0) -{ -} - -void SentPacketHistory::packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet) { - _newestSequenceNumber = sequenceNumber; - - // increment _newestPacketAt cyclically, insert new packet there. - // this will overwrite the oldest packet in the buffer - _newestPacketAt = (_newestPacketAt == _sentPackets.size() - 1) ? 0 : _newestPacketAt + 1; - _sentPackets[_newestPacketAt] = packet; - if (_numExistingPackets < _sentPackets.size()) { - _numExistingPackets++; - } -} - - -const QByteArray* SentPacketHistory::getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const { - OCTREE_PACKET_SEQUENCE seqDiff = _newestSequenceNumber - sequenceNumber; - if (!(seqDiff >= 0 && seqDiff < _numExistingPackets)) { - return NULL; - } - int packetAt = _newestPacketAt - seqDiff; - if (packetAt < 0) { - packetAt += _sentPackets.size(); - } - return &_sentPackets.at(packetAt); -} diff --git a/assignment-client/src/particles/ParticleServer.h b/assignment-client/src/particles/ParticleServer.h index d444368a9d..0b379a903e 100644 --- a/assignment-client/src/particles/ParticleServer.h +++ b/assignment-client/src/particles/ParticleServer.h @@ -33,6 +33,7 @@ public: virtual const char* getMyServerName() const { return PARTICLE_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return PARTICLE_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_PARTICLES_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeParticleEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/assignment-client/src/voxels/VoxelServer.h b/assignment-client/src/voxels/VoxelServer.h index b13f83b65f..fadcca2d19 100644 --- a/assignment-client/src/voxels/VoxelServer.h +++ b/assignment-client/src/voxels/VoxelServer.h @@ -42,6 +42,7 @@ public: virtual const char* getMyServerName() const { return VOXEL_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return VOXEL_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_VOXELS_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeVoxelEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/cmake/macros/AutoMTC.cmake b/cmake/macros/AutoMTC.cmake index 1682b9cd56..6f0216de7f 100644 --- a/cmake/macros/AutoMTC.cmake +++ b/cmake/macros/AutoMTC.cmake @@ -9,13 +9,9 @@ # macro(AUTO_MTC TARGET ROOT_DIR) - if (NOT TARGET mtc) - add_subdirectory("${ROOT_DIR}/tools/mtc" "${ROOT_DIR}/tools/mtc") - endif () - set(AUTOMTC_SRC ${TARGET}_automtc.cpp) file(GLOB INCLUDE_FILES src/*.h) add_custom_command(OUTPUT ${AUTOMTC_SRC} COMMAND mtc -o ${AUTOMTC_SRC} ${INCLUDE_FILES} DEPENDS mtc ${INCLUDE_FILES}) -endmacro() \ No newline at end of file +endmacro() diff --git a/examples/editModels.js b/examples/editModels.js index 1ab6ce0008..eebcd075fa 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -717,8 +717,8 @@ function rayPlaneIntersection(pickRay, point, normal) { function Tooltip() { this.x = 285; this.y = 115; - this.width = 110; - this.height = 115 ; + this.width = 500; + this.height = 145 ; this.margin = 5; this.decimals = 3; @@ -746,6 +746,9 @@ function Tooltip() { text += "yaw: " + angles.y.toFixed(this.decimals) + "\n" text += "roll: " + angles.z.toFixed(this.decimals) + "\n" text += "Scale: " + 2 * properties.radius.toFixed(this.decimals) + "\n" + text += "ID: " + properties.id + "\n" + text += "model url: " + properties.modelURL + "\n" + text += "animation url: " + properties.animationURL + "\n" Overlays.editOverlay(this.textOverlay, { text: text }); } @@ -971,17 +974,27 @@ function mouseMoveEvent(event) { var pixelPerDegrees = windowDimensions.y / (1 * 360); // the entire height of the window allow you to make 2 full rotations - var STEP = 15; - var delta = Math.floor((event.x - mouseLastPosition.x) / pixelPerDegrees); + //compute delta in pixel + var cameraForward = Quat.getFront(Camera.getOrientation()); + var rotationAxis = (!zIsPressed && xIsPressed) ? { x: 1, y: 0, z: 0 } : + (!zIsPressed && !xIsPressed) ? { x: 0, y: 1, z: 0 } : + { x: 0, y: 0, z: 1 }; + rotationAxis = Vec3.multiplyQbyV(selectedModelProperties.modelRotation, rotationAxis); + var orthogonalAxis = Vec3.cross(cameraForward, rotationAxis); + var mouseDelta = { x: event.x - mouseLastPosition + .x, y: mouseLastPosition.y - event.y, z: 0 }; + var transformedMouseDelta = Vec3.multiplyQbyV(Camera.getOrientation(), mouseDelta); + var delta = Math.floor(Vec3.dot(transformedMouseDelta, Vec3.normalize(orthogonalAxis)) / pixelPerDegrees); + var STEP = 15; if (!event.isShifted) { - delta = Math.floor(delta / STEP) * STEP; + delta = Math.round(delta / STEP) * STEP; } var rotation = Quat.fromVec3Degrees({ - x: (!zIsPressed && xIsPressed) ? delta : 0, // z is pressed - y: (!zIsPressed && !xIsPressed) ? delta : 0, // x is pressed - z: (zIsPressed && !xIsPressed) ? delta : 0 // neither is pressed + x: (!zIsPressed && xIsPressed) ? delta : 0, // x is pressed + y: (!zIsPressed && !xIsPressed) ? delta : 0, // neither is pressed + z: (zIsPressed && !xIsPressed) ? delta : 0 // z is pressed }); rotation = Quat.multiply(selectedModelProperties.oldRotation, rotation); @@ -1019,9 +1032,11 @@ var modelMenuAddedDelete = false; function setupModelMenus() { print("setupModelMenus()"); // add our menuitems + Menu.addMenuItem({ menuName: "Edit", menuItemName: "Models", isSeparator: true, beforeItem: "Physics" }); + Menu.addMenuItem({ menuName: "Edit", menuItemName: "Edit Properties...", + shortcutKeyEvent: { text: "`" }, afterItem: "Models" }); if (!Menu.menuItemExists("Edit","Delete")) { print("no delete... adding ours"); - Menu.addMenuItem({ menuName: "Edit", menuItemName: "Models", isSeparator: true, beforeItem: "Physics" }); Menu.addMenuItem({ menuName: "Edit", menuItemName: "Delete", shortcutKeyEvent: { text: "backspace" }, afterItem: "Models" }); modelMenuAddedDelete = true; @@ -1031,9 +1046,10 @@ function setupModelMenus() { } function cleanupModelMenus() { + Menu.removeSeparator("Edit", "Models"); + Menu.removeMenuItem("Edit", "Edit Properties..."); if (modelMenuAddedDelete) { // delete our menuitems - Menu.removeSeparator("Edit", "Models"); Menu.removeMenuItem("Edit", "Delete"); } } @@ -1054,7 +1070,8 @@ Controller.mouseMoveEvent.connect(mouseMoveEvent); Controller.mouseReleaseEvent.connect(mouseReleaseEvent); setupModelMenus(); -Menu.menuItemEvent.connect(function(menuItem){ + +function handeMenuEvent(menuItem){ print("menuItemEvent() in JS... menuItem=" + menuItem); if (menuItem == "Delete") { if (leftController.grabbing) { @@ -1072,8 +1089,36 @@ Menu.menuItemEvent.connect(function(menuItem){ } else { print(" Delete Model.... not holding..."); } + } else if (menuItem == "Edit Properties...") { + var editModelID = -1; + if (leftController.grabbing) { + print(" Edit Properties.... leftController.modelID="+ leftController.modelID); + editModelID = leftController.modelID; + } else if (rightController.grabbing) { + print(" Edit Properties.... rightController.modelID="+ rightController.modelID); + editModelID = rightController.modelID; + } else if (modelSelected) { + print(" Edit Properties.... selectedModelID="+ selectedModelID); + editModelID = selectedModelID; + } else { + print(" Edit Properties.... not holding..."); + } + if (editModelID != -1) { + print(" Edit Properties.... about to edit properties..."); + var propertyName = Window.prompt("Which property would you like to change?", "modelURL"); + var properties = Models.getModelProperties(editModelID); + var oldValue = properties[propertyName]; + var newValue = Window.prompt("New value for: " + propertyName, oldValue); + if (newValue != "") { + properties[propertyName] = newValue; + Models.editModel(editModelID, properties); + } + } + tooltip.show(false); } -}); +} +Menu.menuItemEvent.connect(handeMenuEvent); + // handling of inspect.js concurrence @@ -1107,4 +1152,11 @@ Controller.keyReleaseEvent.connect(function(event) { xIsPressed = false; somethingChanged = true; } + // since sometimes our menu shortcut keys don't work, trap our menu items here also and fire the appropriate menu items + if (event.text == "`") { + handeMenuEvent("Edit Properties..."); + } + if (event.text == "BACKSPACE") { + handeMenuEvent("Delete"); + } }); \ No newline at end of file diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 5507089798..75c3f442ca 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2123,19 +2123,19 @@ void Application::updateMyAvatar(float deltaTime) { const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { _lastNackTime = now; - sendNack(); + sendNackPackets(); } } } -void Application::sendNack() { +int Application::sendNackPackets() { if (Menu::getInstance()->isOptionChecked(MenuOption::DisableNackPackets)) { - return; + return 0; } + int packetsSent = 0; char packet[MAX_PACKET_SIZE]; - NodeList* nodeList = NodeList::getInstance(); // iterates thru all nodes in NodeList foreach(const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { @@ -2146,14 +2146,14 @@ void Application::sendNack() { || node->getType() == NodeType::ModelServer) ) { - // if there are octree packets from this node that are waiting to be processed, - // don't send a NACK since the missing packets may be among those waiting packets. - if (_octreeProcessor.hasPacketsToProcessFrom(node)) { - continue; - } - QUuid nodeUUID = node->getUUID(); + // if there are octree packets from this node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. + if (_octreeProcessor.hasPacketsToProcessFrom(nodeUUID)) { + continue; + } + _octreeSceneStatsLock.lockForRead(); // retreive octree scene stats of this node @@ -2163,40 +2163,48 @@ void Application::sendNack() { } OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; - // check if there are any sequence numbers that need to be nacked - int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); - if (numSequenceNumbersAvailable == 0) { - _octreeSceneStatsLock.unlock(); - continue; - } + // make copy of missing sequence numbers from stats + const QSet missingSequenceNumbers = stats.getMissingSequenceNumbers(); - char* dataAt = packet; - int bytesRemaining = MAX_PACKET_SIZE; - - // pack header - int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); - dataAt += numBytesPacketHeader; - bytesRemaining -= numBytesPacketHeader; - int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); - - // calculate and pack the number of sequence numbers - uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); - uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; - *numSequenceNumbersAt = numSequenceNumbers; - dataAt += sizeof(uint16_t); - - // pack sequence numbers - for (int i = 0; i < numSequenceNumbers; i++) { - OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; - *sequenceNumberAt = stats.getNextSequenceNumberToNack(); - dataAt += sizeof(OCTREE_PACKET_SEQUENCE); - } - _octreeSceneStatsLock.unlock(); - nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); + // construct nack packet(s) for this node + int numSequenceNumbersAvailable = missingSequenceNumbers.size(); + QSet::const_iterator missingSequenceNumbersIterator = missingSequenceNumbers.constBegin(); + while (numSequenceNumbersAvailable > 0) { + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + // calculate and pack the number of sequence numbers + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); + uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; + *sequenceNumberAt = *missingSequenceNumbersIterator; + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + + missingSequenceNumbersIterator++; + } + numSequenceNumbersAvailable -= numSequenceNumbers; + + // send it + NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, node); + packetsSent++; + } } } + return packetsSent; } void Application::queryOctree(NodeType_t serverType, PacketType packetType, NodeToJurisdictionMap& jurisdictions) { @@ -3270,10 +3278,16 @@ void Application::nodeAdded(SharedNodePointer node) { void Application::nodeKilled(SharedNodePointer node) { - // this is here because connecting NodeList::nodeKilled to OctreePacketProcessor::nodeKilled doesn't work: - // OctreePacketProcessor::nodeKilled is not called when NodeList::nodeKilled is emitted for some reason. + // These are here because connecting NodeList::nodeKilled to OctreePacketProcessor::nodeKilled doesn't work: + // OctreePacketProcessor::nodeKilled is not being called when NodeList::nodeKilled is emitted. + // This may have to do with GenericThread::threadRoutine() blocking the QThread event loop + _octreeProcessor.nodeKilled(node); + _voxelEditSender.nodeKilled(node); + _particleEditSender.nodeKilled(node); + _modelEditSender.nodeKilled(node); + if (node->getType() == NodeType::VoxelServer) { QUuid nodeUUID = node->getUUID(); // see if this is the first we've heard of this node... diff --git a/interface/src/Application.h b/interface/src/Application.h index 31e1f2291b..af0dfe9d15 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -413,7 +413,7 @@ private: static void attachNewHeadToNode(Node *newNode); static void* networkReceive(void* args); // network receive thread - void sendNack(); + int sendNackPackets(); MainWindow* _window; GLCanvas* _glWidget; // our GLCanvas has a couple extra features diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index cdd7e7ef0f..29528da126 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -145,6 +145,15 @@ void DatagramProcessor::processDatagrams() { } break; } + case PacketTypeVoxelEditNack: + application->_voxelEditSender.processNackPacket(incomingPacket); + break; + case PacketTypeParticleEditNack: + application->_particleEditSender.processNackPacket(incomingPacket); + break; + case PacketTypeModelEditNack: + application->_modelEditSender.processNackPacket(incomingPacket); + break; default: nodeList->processNodeData(senderSockAddr, incomingPacket); break; diff --git a/interface/src/models/ModelTreeRenderer.cpp b/interface/src/models/ModelTreeRenderer.cpp index b20bdce8f5..9c4b08fd99 100644 --- a/interface/src/models/ModelTreeRenderer.cpp +++ b/interface/src/models/ModelTreeRenderer.cpp @@ -22,6 +22,17 @@ ModelTreeRenderer::ModelTreeRenderer() : } ModelTreeRenderer::~ModelTreeRenderer() { + clearModelsCache(); +} + +void ModelTreeRenderer::clear() { + OctreeRenderer::clear(); + clearModelsCache(); +} + +void ModelTreeRenderer::clearModelsCache() { + qDebug() << "ModelTreeRenderer::clearModelsCache()..."; + // delete the models in _knownModelsItemModels foreach(Model* model, _knownModelsItemModels) { delete model; @@ -71,24 +82,39 @@ Model* ModelTreeRenderer::getModel(const ModelItem& modelItem) { if (modelItem.isKnownID()) { if (_knownModelsItemModels.find(modelItem.getID()) != _knownModelsItemModels.end()) { model = _knownModelsItemModels[modelItem.getID()]; - } else { - + if (QUrl(modelItem.getModelURL()) != model->getURL()) { + delete model; // delete the old model... + model = NULL; + _knownModelsItemModels.remove(modelItem.getID()); + } + } + + // if we don't have a model... + if (!model) { // Make sure we only create new models on the thread that owns the ModelTreeRenderer if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "getModel", Qt::BlockingQueuedConnection, Q_RETURN_ARG(Model*, model), Q_ARG(const ModelItem&, modelItem)); return model; } - + model = new Model(); model->init(); model->setURL(QUrl(modelItem.getModelURL())); _knownModelsItemModels[modelItem.getID()] = model; } + } else { if (_unknownModelsItemModels.find(modelItem.getCreatorTokenID()) != _unknownModelsItemModels.end()) { model = _unknownModelsItemModels[modelItem.getCreatorTokenID()]; - } else { + if (QUrl(modelItem.getModelURL()) != model->getURL()) { + delete model; // delete the old model... + model = NULL; + _unknownModelsItemModels.remove(modelItem.getID()); + } + } + + if (!model) { // Make sure we only create new models on the thread that owns the ModelTreeRenderer if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "getModel", Qt::BlockingQueuedConnection, diff --git a/interface/src/models/ModelTreeRenderer.h b/interface/src/models/ModelTreeRenderer.h index 2d418b1a25..b6df71565d 100644 --- a/interface/src/models/ModelTreeRenderer.h +++ b/interface/src/models/ModelTreeRenderer.h @@ -51,7 +51,11 @@ public: virtual const FBXGeometry* getGeometryForModel(const ModelItem& modelItem); + /// clears the tree + virtual void clear(); + protected: + void clearModelsCache(); Model* getModel(const ModelItem& modelItem); QMap _knownModelsItemModels; QMap _unknownModelsItemModels; diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 9b50ed0bcb..2101fcb9cd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -10,6 +10,7 @@ // #include +#include #include #include diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 80aa07b026..44342abe33 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -33,6 +33,7 @@ REGISTER_SIMPLE_TYPE_STREAMER(QByteArray) REGISTER_SIMPLE_TYPE_STREAMER(QColor) REGISTER_SIMPLE_TYPE_STREAMER(QScriptValue) REGISTER_SIMPLE_TYPE_STREAMER(QString) +REGISTER_SIMPLE_TYPE_STREAMER(QVariant) REGISTER_SIMPLE_TYPE_STREAMER(QUrl) REGISTER_SIMPLE_TYPE_STREAMER(QVariantList) REGISTER_SIMPLE_TYPE_STREAMER(QVariantHash) @@ -1178,7 +1179,9 @@ Bitstream& Bitstream::operator>(ObjectStreamerPointer& streamer) { } for (int i = 0; i < localProperties.size(); i++) { const StreamerPropertyPair& property = properties.at(i); - if (localProperties.at(i).first != property.first || property.second.propertyIndex() != i) { + const StreamerPropertyPair& localProperty = localProperties.at(i); + if (property.first != localProperty.first || + property.second.propertyIndex() != localProperty.second.propertyIndex()) { streamer = ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties)); return *this; } @@ -1618,6 +1621,10 @@ const QHash& Bitstream::getEnumStreamers() { return enumStreamers; } +static QByteArray getEnumName(const char* scope, const char* name) { + return QByteArray(scope) + "::" + name; +} + QHash Bitstream::createEnumStreamers() { QHash enumStreamers; foreach (const QMetaObject* metaObject, getMetaObjects()) { @@ -1625,7 +1632,11 @@ QHash Bitstream::createEnumStreamers() { QMetaEnum metaEnum = metaObject->enumerator(i); const TypeStreamer*& streamer = enumStreamers[ScopeNamePair(metaEnum.scope(), metaEnum.name())]; if (!streamer) { - streamer = new EnumTypeStreamer(metaEnum); + // look for a streamer registered by name + streamer = getTypeStreamers().value(QMetaType::type(getEnumName(metaEnum.scope(), metaEnum.name()))); + if (!streamer) { + streamer = new EnumTypeStreamer(metaEnum); + } } } } @@ -1657,10 +1668,572 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +QJsonValue JSONWriter::getData(bool value) { + return value; +} + +QJsonValue JSONWriter::getData(int value) { + return value; +} + +QJsonValue JSONWriter::getData(uint value) { + return (int)value; +} + +QJsonValue JSONWriter::getData(float value) { + return (double)value; +} + +QJsonValue JSONWriter::getData(const QByteArray& value) { + return QString(value.toPercentEncoding()); +} + +QJsonValue JSONWriter::getData(const QColor& value) { + return value.name(); +} + +QJsonValue JSONWriter::getData(const QScriptValue& value) { + QJsonObject object; + if (value.isUndefined()) { + object.insert("type", QString("UNDEFINED")); + + } else if (value.isNull()) { + object.insert("type", QString("NULL")); + + } else if (value.isBool()) { + object.insert("type", QString("BOOL")); + object.insert("value", value.toBool()); + + } else if (value.isNumber()) { + object.insert("type", QString("NUMBER")); + object.insert("value", value.toNumber()); + + } else if (value.isString()) { + object.insert("type", QString("STRING")); + object.insert("value", value.toString()); + + } else if (value.isVariant()) { + object.insert("type", QString("VARIANT")); + object.insert("value", getData(value.toVariant())); + + } else if (value.isQObject()) { + object.insert("type", QString("QOBJECT")); + object.insert("value", getData(value.toQObject())); + + } else if (value.isQMetaObject()) { + object.insert("type", QString("QMETAOBJECT")); + object.insert("value", getData(value.toQMetaObject())); + + } else if (value.isDate()) { + object.insert("type", QString("DATE")); + object.insert("value", getData(value.toDateTime())); + + } else if (value.isRegExp()) { + object.insert("type", QString("REGEXP")); + object.insert("value", getData(value.toRegExp())); + + } else if (value.isArray()) { + object.insert("type", QString("ARRAY")); + QJsonArray array; + int length = value.property(ScriptCache::getInstance()->getLengthString()).toInt32(); + for (int i = 0; i < length; i++) { + array.append(getData(value.property(i))); + } + object.insert("value", array); + + } else if (value.isObject()) { + object.insert("type", QString("OBJECT")); + QJsonObject valueObject; + for (QScriptValueIterator it(value); it.hasNext(); ) { + it.next(); + valueObject.insert(it.name(), getData(it.value())); + } + object.insert("value", valueObject); + + } else { + object.insert("type", QString("INVALID")); + } + return object; +} + +QJsonValue JSONWriter::getData(const QString& value) { + return value; +} + +QJsonValue JSONWriter::getData(const QUrl& value) { + return value.toString(); +} + +QJsonValue JSONWriter::getData(const QDateTime& value) { + return (qsreal)value.toMSecsSinceEpoch(); +} + +QJsonValue JSONWriter::getData(const QRegExp& value) { + QJsonObject object; + object.insert("pattern", value.pattern()); + object.insert("caseSensitivity", (int)value.caseSensitivity()); + object.insert("patternSyntax", (int)value.patternSyntax()); + object.insert("minimal", value.isMinimal()); + return object; +} + +QJsonValue JSONWriter::getData(const glm::vec3& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + return array; +} + +QJsonValue JSONWriter::getData(const glm::quat& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + array.append(value.w); + return array; +} + +QJsonValue JSONWriter::getData(const QMetaObject* metaObject) { + if (!metaObject) { + return QJsonValue(); + } + const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); + addObjectStreamer(streamer); + return QString(streamer->getName()); +} + +QJsonValue JSONWriter::getData(const QVariant& value) { + if (!value.isValid()) { + return QJsonValue(); + } + const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); + if (streamer) { + return streamer->getJSONVariantData(*this, value); + } else { + qWarning() << "Non-streamable type:" << value.typeName(); + return QJsonValue(); + } +} + +QJsonValue JSONWriter::getData(const SharedObjectPointer& object) { + if (object) { + addSharedObject(object); + return object->getID(); + } else { + return 0; + } +} + +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); +} + +QJsonValue JSONWriter::getData(const GenericValue& value) { + return value.getStreamer()->getJSONData(*this, value.getValue()); +} + +void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { + if (!_typeStreamerNames.contains(streamer->getName())) { + _typeStreamerNames.insert(streamer->getName()); + + QJsonValue metadata = streamer->getJSONMetadata(*this); + if (!metadata.isNull()) { + _typeStreamers.append(metadata); + } + } +} + +void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { + if (!_objectStreamerNames.contains(streamer->getName())) { + _objectStreamerNames.insert(streamer->getName()); + _objectStreamers.append(streamer->getJSONMetadata(*this)); + } +} + +void JSONWriter::addSharedObject(const SharedObjectPointer& object) { + if (!_sharedObjectIDs.contains(object->getID())) { + _sharedObjectIDs.insert(object->getID()); + + QJsonObject sharedObject; + sharedObject.insert("id", object->getID()); + sharedObject.insert("data", getData(static_cast(object.data()))); + _sharedObjects.append(sharedObject); + } +} + +QJsonDocument JSONWriter::getDocument() const { + QJsonObject top; + top.insert("contents", _contents); + top.insert("objects", _sharedObjects); + top.insert("classes", _objectStreamers); + top.insert("types", _typeStreamers); + return QJsonDocument(top); +} + +JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { + // create and map the type streamers in order + QJsonObject top = document.object(); + foreach (const QJsonValue& element, top.value("types").toArray()) { + QJsonObject type = element.toObject(); + QString name = type.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); + if (!baseStreamer) { + baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); + } + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; // no built-in type and no generics allowed; we give up + } + QString category = type.value("category").toString(); + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer + TypeStreamerPointer streamer; + if (category == "ENUM") { + QVector values; + int highestValue = 0; + foreach (const QJsonValue& value, type.value("values").toArray()) { + QJsonObject object = value.toObject(); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), intValue)); + } + streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, + values, getBitsForHighestValue(highestValue), QByteArray())); + + } else if (category == "STREAMABLE") { + QVector fields; + foreach (const QJsonValue& field, type.value("fields").toArray()) { + QJsonObject object = field.toObject(); + fields.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + streamer = TypeStreamerPointer(new GenericStreamableTypeStreamer(latinName, + fields, QByteArray())); + + } else if (category == "LIST") { + streamer = TypeStreamerPointer(new GenericListTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "SET") { + streamer = TypeStreamerPointer(new GenericSetTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "MAP") { + streamer = TypeStreamerPointer(new GenericMapTypeStreamer(latinName, + getTypeStreamer(type.value("keyType").toString()), + getTypeStreamer(type.value("valueType").toString()))); + } + _typeStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + // create a mapped streamer, determining along the way whether it matches our base + if (category == "ENUM") { + QHash mappings; + int highestValue = 0; + QMetaEnum metaEnum = baseStreamer->getMetaEnum(); + QJsonArray array = type.value("values").toArray(); + bool matches = (array.size() == metaEnum.keyCount()); + foreach (const QJsonValue& value, array) { + QJsonObject object = value.toObject(); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); + if (mapping != -1) { + mappings.insert(intValue, mapping); + } + matches &= (intValue == mapping); + } + // if everything matches our built-in enum, we can use that, which will be faster + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedEnumTypeStreamer(baseStreamer, + getBitsForHighestValue(highestValue), mappings))); + } + } else if (category == "STREAMABLE") { + QVector fields; + QJsonArray array = type.value("fields").toArray(); + const QVector& metaFields = baseStreamer->getMetaFields(); + bool matches = (array.size() == metaFields.size()); + for (int i = 0; i < array.size(); i++) { + QJsonObject object = array.at(i).toObject(); + TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); + int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); + fields.append(StreamerIndexPair(streamer, index)); + matches &= (index == i && streamer == metaFields.at(i).getStreamer()); + } + // if everything matches our built-in streamable, we can use that, which will be faster + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedStreamableTypeStreamer(baseStreamer, fields))); + } + } else if (category == "LIST") { + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedListTypeStreamer(baseStreamer, valueStreamer))); + } + } else if (category == "SET") { + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedSetTypeStreamer(baseStreamer, valueStreamer))); + } + } else if (category == "MAP") { + TypeStreamerPointer keyStreamer = getTypeStreamer(type.value("keyType").toString()); + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (keyStreamer == baseStreamer->getKeyStreamer() && valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedMapTypeStreamer( + baseStreamer, keyStreamer, valueStreamer))); + } + } + } + + // create and map the object streamers in order + foreach (const QJsonValue& element, top.value("classes").toArray()) { + QJsonObject clazz = element.toObject(); + QString name = clazz.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( + Bitstream::getMetaObjects().value(latinName)); + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; // no built-in class and no generics allowed; we give up + } + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer + QVector properties; + foreach (const QJsonValue& property, clazz.value("properties").toArray()) { + QJsonObject object = property.toObject(); + properties.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + ObjectStreamerPointer streamer = ObjectStreamerPointer(new GenericObjectStreamer( + latinName, properties, QByteArray())); + _objectStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + // create a mapped streamer, determining along the way whether it matches our base + const QMetaObject* metaObject = baseStreamer->getMetaObject(); + const QVector& baseProperties = baseStreamer->getProperties(); + QVector properties; + QJsonArray propertyArray = clazz.value("properties").toArray(); + bool matches = (baseProperties.size() == propertyArray.size()); + for (int i = 0; i < propertyArray.size(); i++) { + QJsonObject object = propertyArray.at(i).toObject(); + TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); + QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( + object.value("name").toString().toLatin1())); + properties.append(StreamerPropertyPair(typeStreamer, metaProperty)); + + const StreamerPropertyPair& baseProperty = baseProperties.at(i); + matches &= (typeStreamer == baseProperty.first && + metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); + } + // if everything matches our built-in type, we can use that directly, which will be faster + if (matches) { + _objectStreamers.insert(name, baseStreamer->getSelf()); + } else { + _objectStreamers.insert(name, ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties))); + } + } + + // create and map the objects in order + foreach (const QJsonValue& element, top.value("objects").toArray()) { + QJsonObject object = element.toObject(); + int id = object.value("id").toInt(); + QObject* qObject; + putData(object.value("data"), qObject); + if (qObject) { + _sharedObjects.insert(id, static_cast(qObject)); + } + } + + // prepare the contents for extraction + _contents = top.value("contents").toArray(); + _contentsIterator = _contents.constBegin(); +} + +void JSONReader::putData(const QJsonValue& data, bool& value) { + value = data.toBool(); +} + +void JSONReader::putData(const QJsonValue& data, int& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, uint& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, float& value) { + value = data.toDouble(); +} + +void JSONReader::putData(const QJsonValue& data, QByteArray& value) { + value = QByteArray::fromPercentEncoding(data.toString().toLatin1()); +} + +void JSONReader::putData(const QJsonValue& data, QColor& value) { + value.setNamedColor(data.toString()); +} + +void JSONReader::putData(const QJsonValue& data, QScriptValue& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + if (type == "UNDEFINED") { + value = QScriptValue(QScriptValue::UndefinedValue); + + } else if (type == "NULL") { + value = QScriptValue(QScriptValue::NullValue); + + } else if (type == "BOOL") { + value = QScriptValue(object.value("value").toBool()); + + } else if (type == "NUMBER") { + value = QScriptValue(object.value("value").toDouble()); + + } else if (type == "STRING") { + value = QScriptValue(object.value("value").toString()); + + } else if (type == "VARIANT") { + QVariant variant; + putData(object.value("value"), variant); + value = ScriptCache::getInstance()->getEngine()->newVariant(variant); + + } else if (type == "QOBJECT") { + QObject* qObject; + putData(object.value("value"), qObject); + value = ScriptCache::getInstance()->getEngine()->newQObject(qObject, QScriptEngine::ScriptOwnership); + + } else if (type == "QMETAOBJECT") { + const QMetaObject* metaObject; + putData(object.value("value"), metaObject); + value = ScriptCache::getInstance()->getEngine()->newQMetaObject(metaObject); + + } else if (type == "DATE") { + QDateTime dateTime; + putData(object.value("value"), dateTime); + value = ScriptCache::getInstance()->getEngine()->newDate(dateTime); + + } else if (type == "REGEXP") { + QRegExp regExp; + putData(object.value("value"), regExp); + value = ScriptCache::getInstance()->getEngine()->newRegExp(regExp); + + } else if (type == "ARRAY") { + QJsonArray array = object.value("value").toArray(); + value = ScriptCache::getInstance()->getEngine()->newArray(array.size()); + for (int i = 0; i < array.size(); i++) { + QScriptValue element; + putData(array.at(i), element); + value.setProperty(i, element); + } + } else if (type == "OBJECT") { + QJsonObject jsonObject = object.value("value").toObject(); + value = ScriptCache::getInstance()->getEngine()->newObject(); + for (QJsonObject::const_iterator it = jsonObject.constBegin(); it != jsonObject.constEnd(); it++) { + QScriptValue element; + putData(it.value(), element); + value.setProperty(it.key(), element); + } + } else { + value = QScriptValue(); + } +} + +void JSONReader::putData(const QJsonValue& data, QString& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QUrl& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QDateTime& value) { + value.setMSecsSinceEpoch((qint64)data.toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, QRegExp& value) { + QJsonObject object = data.toObject(); + value = QRegExp(object.value("pattern").toString(), (Qt::CaseSensitivity)object.value("caseSensitivity").toInt(), + (QRegExp::PatternSyntax)object.value("patternSyntax").toInt()); + value.setMinimal(object.value("minimal").toBool()); +} + +void JSONReader::putData(const QJsonValue& data, glm::vec3& value) { + QJsonArray array = data.toArray(); + value = glm::vec3(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, glm::quat& value) { + QJsonArray array = data.toArray(); + value = glm::quat(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble(), array.at(3).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { + ObjectStreamerPointer streamer = _objectStreamers.value(data.toString()); + value = streamer ? streamer->getMetaObject() : NULL; +} + +void JSONReader::putData(const QJsonValue& data, QVariant& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + TypeStreamerPointer streamer = getTypeStreamer(type); + if (streamer) { + streamer->putJSONVariantData(*this, object.value("value"), value); + } else { + value = QVariant(); + } +} + +void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { + value = _sharedObjects.value(data.toInt()); +} + +void JSONReader::putData(const QJsonValue& data, QObject*& value) { + QJsonObject object = data.toObject(); + ObjectStreamerPointer streamer = _objectStreamers.value(object.value("class").toString()); + value = streamer ? streamer->putJSONData(*this, object) : NULL; +} + +TypeStreamerPointer JSONReader::getTypeStreamer(const QString& name) const { + TypeStreamerPointer streamer = _typeStreamers.value(name); + if (!streamer) { + const TypeStreamer* defaultStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(name.toLatin1())); + if (defaultStreamer) { + streamer = defaultStreamer->getSelf(); + } else { + qWarning() << "Unknown type:" << name; + } + } + return streamer; +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } +ObjectStreamer::~ObjectStreamer() { +} + const QVector& ObjectStreamer::getProperties() const { static QVector emptyProperties; return emptyProperties; @@ -1699,6 +2272,50 @@ void MappedObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject MappedObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second.name())); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + properties.append(property.first->getJSONData(writer, property.second.read(object))); + } + data.insert("properties", properties); + return data; +} + +QObject* MappedObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + if (!_metaObject) { + return NULL; + } + QObject* object = _metaObject->newInstance(); + QJsonArray properties = jsonObject.value("properties").toArray(); + for (int i = 0; i < _properties.size(); i++) { + const StreamerPropertyPair& property = _properties.at(i); + if (property.second.isValid()) { + QVariant value; + property.first->putJSONData(reader, properties.at(i), value); + property.second.write(object, value); + } + } + return object; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -1775,6 +2392,47 @@ void GenericObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject GenericObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_name)); + QJsonArray properties; + foreach (const StreamerNamePair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second)); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_name)); + QJsonArray properties; + const QVariantList& values = static_cast(object)->getValues(); + for (int i = 0; i < _properties.size(); i++) { + properties.append(_properties.at(i).first->getJSONData(writer, values.at(i))); + } + data.insert("properties", properties); + return data; +} + +QObject* GenericObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + GenericSharedObject* object = new GenericSharedObject(_weakSelf); + QJsonArray properties = jsonObject.value("properties").toArray(); + QVariantList values; + for (int i = 0; i < _properties.size(); i++) { + QVariant value; + _properties.at(i).first->putJSONData(reader, properties.at(i), value); + values.append(value); + } + object->setValues(values); + return object; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -1876,6 +2534,71 @@ void TypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue TypeStreamer::getJSONMetadata(JSONWriter& writer) const { + Category category = getCategory(); + switch (category) { + case STREAMABLE_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const MetaField& metaField, getMetaFields()) { + QJsonObject field; + writer.addTypeStreamer(metaField.getStreamer()); + field.insert("type", QString(metaField.getStreamer()->getName())); + field.insert("name", QString(metaField.getName())); + fields.append(field); + } + metadata.insert("fields", fields); + return metadata; + } + case LIST_CATEGORY: + case SET_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString(category == LIST_CATEGORY ? "LIST" : "SET")); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + case MAP_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + const TypeStreamer* keyStreamer = getKeyStreamer(); + writer.addTypeStreamer(keyStreamer); + metadata.insert("keyType", QString(keyStreamer->getName())); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + default: + return QJsonValue(); + } +} + +QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return QJsonValue(); +} + +QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + writer.addTypeStreamer(this); + QJsonObject data; + data.insert("type", QString(getName())); + data.insert("value", getJSONData(writer, value)); + return data; +} + +void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(); +} + +void TypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + putJSONData(reader, data, value); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2004,7 +2727,7 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject) { EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* name) : _metaObject(metaObject), _enumName(name), - _name(QByteArray(metaObject->className()) + "::" + name), + _name(getEnumName(metaObject->className(), name)), _bits(-1) { _type = QMetaType::Int; @@ -2012,7 +2735,7 @@ EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* na } EnumTypeStreamer::EnumTypeStreamer(const QMetaEnum& metaEnum) : - _name(QByteArray(metaEnum.scope()) + "::" + metaEnum.name()), + _name(getEnumName(metaEnum.scope(), metaEnum.name())), _metaEnum(metaEnum), _bits(-1) { @@ -2045,6 +2768,30 @@ void EnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + QMetaEnum metaEnum = getMetaEnum(); + for (int i = 0; i < metaEnum.keyCount(); i++) { + QJsonObject value; + value.insert("key", QString(metaEnum.key(i))); + value.insert("value", metaEnum.value(i)); + values.append(value); + } + metadata.insert("values", values); + return metadata; +} + +QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + +void EnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2137,6 +2884,11 @@ MappedEnumTypeStreamer::MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, _mappings(mappings) { } +void MappedEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + _baseStreamer->setEnumValue(value, data.toInt(), _mappings); +} + QVariant MappedEnumTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int value = 0; @@ -2159,6 +2911,12 @@ const char* GenericTypeStreamer::getName() const { return _name.constData(); } +void GenericTypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QVariant containedValue; + putJSONData(reader, data, containedValue); + value = QVariant::fromValue(GenericValue(_weakSelf, containedValue)); +} + QVariant GenericTypeStreamer::readVariant(Bitstream& in) const { return QVariant::fromValue(GenericValue(_weakSelf, read(in))); } @@ -2194,6 +2952,29 @@ void GenericEnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + foreach (const NameIntPair& value, _values) { + QJsonObject object; + object.insert("key", QString(value.first)); + object.insert("value", value.second); + values.append(object); + } + metadata.insert("values", values); + return metadata; +} + +QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + +void GenericEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2215,6 +2996,19 @@ MappedStreamableTypeStreamer::MappedStreamableTypeStreamer(const TypeStreamer* b _fields(fields) { } +void MappedStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + QJsonArray array = data.toArray(); + for (int i = 0; i < _fields.size(); i++) { + const StreamerIndexPair& pair = _fields.at(i); + if (pair.second != -1) { + QVariant element; + pair.first->putJSONData(reader, array.at(i), element); + _baseStreamer->setField(value, pair.second, element); + } + } +} + QVariant MappedStreamableTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); foreach (const StreamerIndexPair& pair, _fields) { @@ -2270,6 +3064,42 @@ void GenericStreamableTypeStreamer::writeMetadata(Bitstream& out, bool full) con } } +QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const StreamerNamePair& field, _fields) { + QJsonObject object; + writer.addTypeStreamer(field.first.data()); + object.insert("type", QString(field.first->getName())); + object.insert("name", QString(field.second)); + fields.append(object); + } + metadata.insert("fields", fields); + return metadata; +} + +QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + for (int i = 0; i < _fields.size(); i++) { + array.append(_fields.at(i).first->getJSONData(writer, values.at(i))); + } + return array; +} + +void GenericStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + for (int i = 0; i < _fields.size(); i++) { + QVariant element; + _fields.at(i).first->putJSONData(reader, array.at(i), element); + values.append(element); + } + value = values; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2294,6 +3124,15 @@ MappedListTypeStreamer::MappedListTypeStreamer(const TypeStreamer* baseStreamer, _valueStreamer(valueStreamer) { } +void MappedListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + _baseStreamer->insert(value, elementValue); + } +} + QVariant MappedListTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2334,6 +3173,35 @@ void GenericListTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _valueStreamer.data(); } +QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("LIST")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + +QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + foreach (const QVariant& element, values) { + array.append(_valueStreamer->getJSONData(writer, element)); + } + return array; +} + +void GenericListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + foreach (const QJsonValue& element, array) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + values.append(elementValue); + } + value = values; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2376,6 +3244,15 @@ GenericSetTypeStreamer::GenericSetTypeStreamer(const QByteArray& name, const Typ GenericListTypeStreamer(name, valueStreamer) { } +QJsonValue GenericSetTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("SET")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + TypeStreamer::Category GenericSetTypeStreamer::getCategory() const { return SET_CATEGORY; } @@ -2387,6 +3264,18 @@ MappedMapTypeStreamer::MappedMapTypeStreamer(const TypeStreamer* baseStreamer, c _valueStreamer(valueStreamer) { } +void MappedMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + _baseStreamer->insert(value, elementKey, elementValue); + } +} + QVariant MappedMapTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2437,6 +3326,43 @@ void GenericMapTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _keyStreamer.data() << _valueStreamer.data(); } +QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + writer.addTypeStreamer(_keyStreamer.data()); + metadata.insert("keyType", QString(_keyStreamer->getName())); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + +QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantPairList values = value.value(); + QJsonArray array; + foreach (const QVariantPair& pair, values) { + QJsonArray pairArray; + pairArray.append(_keyStreamer->getJSONData(writer, pair.first)); + pairArray.append(_valueStreamer->getJSONData(writer, pair.second)); + array.append(pairArray); + } + return array; +} + +void GenericMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantPairList values; + foreach (const QJsonValue& element, array) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + values.append(QVariantPair(elementKey, elementValue)); + } + value = QVariant::fromValue(values); +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); @@ -2462,6 +3388,15 @@ TypeStreamer::Category GenericMapTypeStreamer::getCategory() const { return MAP_CATEGORY; } +QJsonValue GenericValueStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + GenericValue genericValue = value.value(); + writer.addTypeStreamer(genericValue.getStreamer().data()); + QJsonObject data; + data.insert("type", QString(genericValue.getStreamer()->getName())); + data.insert("value", genericValue.getStreamer()->getJSONData(writer, genericValue.getValue())); + return data; +} + void GenericValueStreamer::writeVariant(Bitstream& out, const QVariant& value) const { GenericValue genericValue = value.value(); out << genericValue.getStreamer().data(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 1589473b0e..0d9e516640 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -13,6 +13,9 @@ #define hifi_Bitstream_h #include +#include +#include +#include #include #include #include @@ -36,6 +39,7 @@ class Attribute; class AttributeValue; class Bitstream; class GenericValue; +class JSONWriter; class ObjectReader; class ObjectStreamer; class OwnedAttributeValue; @@ -195,12 +199,50 @@ template inline RepeatedValueStreamer& return *this; } -/// A stream for bit-aligned data. +/// A stream for bit-aligned data. Through a combination of code generation, reflection, macros, and templates, provides a +/// serialization mechanism that may be used for both networking and persistent storage. For unreliable networking, the +/// class provides a mapping system that resends mappings for ids until they are acknowledged (and thus persisted). For +/// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to +/// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously +/// stored data. +/// +/// The basic usage requires one to create a Bitstream that wraps an underlying QDataStream, specifying the metadata type +/// desired and (for readers) the generics mode. Then, one uses the << or >> operators to write or read values to/from +/// the stream (a stream instance may be used for reading or writing, but not both). For write streams, the flush +/// function should be called on completion to write any partial data. +/// +/// Polymorphic types are supported via the QVariant and QObject*/SharedObjectPointer types. When you write a QVariant or +/// QObject, the type or class name (at minimum) is written to the stream. When you read a QVariant or QObject, the default +/// behavior is to look for a corresponding registered type with the written name. With hash metadata, Bitstream can verify +/// that the local type matches the stream type, throwing the streamed version away if not. With full metadata, Bitstream can +/// create a mapping from the streamed type to the local type that applies the intersection of the two types' fields. +/// +/// To register types for streaming, select from the provided templates and macros. To register a QObject (or SharedObject) +/// subclass, use REGISTER_META_OBJECT in the class's source file. To register a streamable class for use in QVariant, use +/// the STREAMABLE/STREAM/DECLARE_STREAMABLE_METATYPE macros and use the mtc tool to generate the associated implementation +/// code. To register a QObject enum for use outside the QObject's direct properties, use the +/// DECLARE_ENUM_METATYPE/IMPLEMENT_ENUM_METATYPE macro pair. To register a collection type (QList, QVector, QSet, QMap) of +/// streamable types, use the registerCollectionMetaType template function. +/// +/// Delta-streaming is supported through the writeDelta/readDelta functions (analogous to <>), which accept a reference +/// value and stream only the information necessary to turn the reference into the target value. This assumes that the +/// reference value provided when reading is identical to the one used when writing. +/// +/// Special delta handling is provided for objects tracked by SharedObjectPointers. SharedObjects have IDs (representing their +/// unique identity on one system) as well as origin IDs (representing their identity as preserved over the course of +/// mutation). Once a shared object with a given ID is written (and its mapping persisted), that object's state will not be +/// written again; only its ID will be sent. However, if an object with a new ID but the same origin ID is written, that +/// object's state will be encoded as a delta between the previous object and the new one. So, to transmit delta-encoded +/// objects, one should treat each local SharedObject instance as immutable, replacing it when mutated with a cloned instance +/// generated by calling SharedObject::clone(true) and applying the desired changes. class Bitstream : public QObject { Q_OBJECT public: + /// Stores a set of mappings from values to ids written. Typically, one would store these mappings along with the send + /// record of the packet that contained them, persisting the mappings if/when the packet is acknowledged by the remote + /// party. class WriteMappings { public: QHash objectStreamerOffsets; @@ -210,6 +252,9 @@ public: QHash sharedObjectOffsets; }; + /// Stores a set of mappings from ids to values read. Typically, one would store these mappings along with the receive + /// record of the packet that contained them, persisting the mappings if/when the remote party indicates that it + /// has received the local party's acknowledgement of the packet. class ReadMappings { public: QHash objectStreamerValues; @@ -219,11 +264,14 @@ public: QHash sharedObjectValues; }; - /// Registers a metaobject under its name so that instances of it can be streamed. + /// Registers a metaobject under its name so that instances of it can be streamed. Consider using the REGISTER_META_OBJECT + /// at the top level of the source file associated with the class rather than calling this function directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerMetaObject(const char* className, const QMetaObject* metaObject); - /// Registers a streamer for the specified Qt-registered type. + /// Registers a streamer for the specified Qt-registered type. Consider using one of the registration macros (such as + /// REGISTER_SIMPLE_TYPE_STREAMER) at the top level of the associated source file rather than calling this function + /// directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerTypeStreamer(int type, TypeStreamer* streamer); @@ -233,7 +281,9 @@ public: /// Returns the meta-object registered under the supplied class name, if any. static const QMetaObject* getMetaObject(const QByteArray& className); - /// Returns the list of registered subclasses for the supplied meta-object. + /// Returns the list of registered subclasses for the supplied meta-object. When registered, metaobjects register + /// themselves as subclasses of all of their parents, mostly in order to allow editors to provide lists of available + /// subclasses. static QList getMetaObjectSubClasses(const QMetaObject* metaObject); enum MetadataType { NO_METADATA, HASH_METADATA, FULL_METADATA }; @@ -250,7 +300,8 @@ public: Bitstream(QDataStream& underlying, MetadataType metadataType = NO_METADATA, GenericsMode = NO_GENERICS, QObject* parent = NULL); - /// Substitutes the supplied metaobject for the given class name's default mapping. + /// Substitutes the supplied metaobject for the given class name's default mapping. This is mostly useful for testing the + /// process of mapping between different types, but may in the future be used for permanently renaming classes. void addMetaObjectSubstitution(const QByteArray& className, const QMetaObject* metaObject); /// Substitutes the supplied type for the given type name's default mapping. @@ -445,6 +496,9 @@ private slots: private: + friend class JSONReader; + friend class JSONWriter; + ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); TypeStreamerPointer readGenericTypeStreamer(const QByteArray& name, int category); @@ -471,7 +525,7 @@ private: static QHash& getMetaObjects(); static QMultiHash& getMetaObjectSubClasses(); static QHash& getTypeStreamers(); - + static const QHash& getObjectStreamers(); static QHash createObjectStreamers(); @@ -766,6 +820,189 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } +/// Provides a means of writing Bitstream-able data to JSON rather than the usual binary format in a manner that allows it to +/// be manipulated and re-read, converted to binary, etc. To use, create a JSONWriter, stream values in using the << operator, +/// and call getDocument to obtain the JSON data. +class JSONWriter { +public: + + QJsonValue getData(bool value); + QJsonValue getData(int value); + QJsonValue getData(uint value); + QJsonValue getData(float value); + QJsonValue getData(const QByteArray& value); + QJsonValue getData(const QColor& value); + QJsonValue getData(const QScriptValue& value); + QJsonValue getData(const QString& value); + QJsonValue getData(const QUrl& value); + QJsonValue getData(const QDateTime& value); + QJsonValue getData(const QRegExp& value); + QJsonValue getData(const glm::vec3& value); + QJsonValue getData(const glm::quat& value); + QJsonValue getData(const QMetaObject* value); + QJsonValue getData(const QVariant& value); + QJsonValue getData(const SharedObjectPointer& value); + QJsonValue getData(const QObject* value); + QJsonValue getData(const GenericValue& value); + + template QJsonValue getData(const T& value) { return QJsonValue(); } + + template QJsonValue getData(const QList& list); + template QJsonValue getData(const QVector& list); + template QJsonValue getData(const QSet& set); + template QJsonValue getData(const QHash& hash); + + template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } + + void appendToContents(const QJsonValue& value) { _contents.append(value); } + + void addSharedObject(const SharedObjectPointer& object); + void addObjectStreamer(const ObjectStreamer* streamer); + void addTypeStreamer(const TypeStreamer* streamer); + + QJsonDocument getDocument() const; + +private: + + QJsonArray _contents; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; + + QSet _objectStreamerNames; + QJsonArray _objectStreamers; + + QSet _typeStreamerNames; + QJsonArray _typeStreamers; +}; + +template inline QJsonValue JSONWriter::getData(const QList& list) { + QJsonArray array; + foreach (const T& value, list) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QVector& vector) { + QJsonArray array; + foreach (const T& value, vector) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QSet& set) { + QJsonArray array; + foreach (const T& value, set) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QHash& hash) { + QJsonArray array; + for (typename QHash::const_iterator it = hash.constBegin(); it != hash.constEnd(); it++) { + QJsonArray pair; + pair.append(getData(it.key())); + pair.append(getData(it.value())); + array.append(pair); + } + return array; +} + +/// Reads a document written by JSONWriter. To use, create a JSONReader and stream values out using the >> operator. +class JSONReader { +public: + + /// Creates a reader to read from the supplied document. + /// \param genericsMode the generics mode to use: NO_GENERICS to map all types in the document to built-in types, + /// FALLBACK_GENERICS to use generic containers where no matching built-in type is found, or ALL_GENERICS to + /// read all types to generic containers + JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode = Bitstream::NO_GENERICS); + + void putData(const QJsonValue& data, bool& value); + void putData(const QJsonValue& data, int& value); + void putData(const QJsonValue& data, uint& value); + void putData(const QJsonValue& data, float& value); + void putData(const QJsonValue& data, QByteArray& value); + void putData(const QJsonValue& data, QColor& value); + void putData(const QJsonValue& data, QScriptValue& value); + void putData(const QJsonValue& data, QString& value); + void putData(const QJsonValue& data, QUrl& value); + void putData(const QJsonValue& data, QDateTime& value); + void putData(const QJsonValue& data, QRegExp& value); + void putData(const QJsonValue& data, glm::vec3& value); + void putData(const QJsonValue& data, glm::quat& value); + void putData(const QJsonValue& data, const QMetaObject*& value); + void putData(const QJsonValue& data, QVariant& value); + void putData(const QJsonValue& data, SharedObjectPointer& value); + void putData(const QJsonValue& data, QObject*& value); + + template void putData(const QJsonValue& data, T& value) { value = T(); } + + template void putData(const QJsonValue& data, QList& list); + template void putData(const QJsonValue& data, QVector& list); + template void putData(const QJsonValue& data, QSet& set); + template void putData(const QJsonValue& data, QHash& hash); + + template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + + QJsonValue retrieveNextFromContents() { return *_contentsIterator++; } + + TypeStreamerPointer getTypeStreamer(const QString& name) const; + ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } + SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } + +private: + + QJsonArray _contents; + QJsonArray::const_iterator _contentsIterator; + + QHash _typeStreamers; + QHash _objectStreamers; + QHash _sharedObjects; +}; + +template inline void JSONReader::putData(const QJsonValue& data, QList& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QVector& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QSet& set) { + set.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + set.insert(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QHash& hash) { + hash.clear(); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + K key; + putData(pair.at(0), key); + V value; + putData(pair.at(1), value); + hash.insert(key, value); + } +} + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -773,6 +1010,7 @@ class ObjectStreamer { public: ObjectStreamer(const QMetaObject* metaObject); + virtual ~ObjectStreamer(); const QMetaObject* getMetaObject() const { return _metaObject; } const ObjectStreamerPointer& getSelf() const { return _self; } @@ -780,6 +1018,10 @@ public: virtual const char* getName() const = 0; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const = 0; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const = 0; + virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -790,7 +1032,7 @@ protected: friend class Bitstream; const QMetaObject* _metaObject; - ObjectStreamerPointer _self; + ObjectStreamerPointer _self; ///< set/used for built-in classes (never deleted), to obtain shared pointers }; /// A streamer that maps to a local class. @@ -802,6 +1044,9 @@ public: virtual const char* getName() const; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -822,6 +1067,9 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -830,9 +1078,10 @@ public: private: friend class Bitstream; + friend class JSONReader; QByteArray _name; - WeakObjectStreamerPointer _weakSelf; + WeakObjectStreamerPointer _weakSelf; ///< promoted to strong references in GenericSharedObject when reading QVector _properties; QByteArray _hash; }; @@ -854,10 +1103,12 @@ private: Q_DECLARE_METATYPE(const QMetaObject*) -/// Macro for registering streamable meta-objects. +/// Macro for registering streamable meta-objects. Typically, one would use this macro at the top level of the source file +/// associated with the class. #define REGISTER_META_OBJECT(x) static int x##Registration = Bitstream::registerMetaObject(#x, &x::staticMetaObject); -/// Contains a value along with a pointer to its streamer. +/// Contains a value along with a pointer to its streamer. This is stored in QVariants when using fallback generics and +/// no mapping to a built-in type can be found, or when using all generics and the value is any non-simple type. class GenericValue { public: @@ -876,7 +1127,8 @@ private: Q_DECLARE_METATYPE(GenericValue) -/// Contains a list of property values along with a pointer to their metadata. +/// Contains a list of property values along with a pointer to their metadata. This is stored when using fallback generics +/// and no mapping to a built-in class can be found, or for all QObjects when using all generics. class GenericSharedObject : public SharedObject { Q_OBJECT @@ -913,6 +1165,12 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual bool equal(const QVariant& first, const QVariant& second) const; virtual void write(Bitstream& out, const QVariant& value) const; @@ -957,7 +1215,7 @@ protected: friend class Bitstream; int _type; - TypeStreamerPointer _self; + TypeStreamerPointer _self; ///< set/used for built-in types (never deleted), to obtain shared pointers }; QDebug& operator<<(QDebug& debug, const TypeStreamer* typeStreamer); @@ -968,6 +1226,10 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject); template class SimpleTypeStreamer : public TypeStreamer { public: + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { + return writer.getData(value.value()); } + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + T rawValue; reader.putData(data, rawValue); value = QVariant::fromValue(rawValue); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } @@ -990,6 +1252,9 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1017,6 +1282,7 @@ public: MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, int bits, const QHash& mappings); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1034,14 +1300,16 @@ public: GenericTypeStreamer(const QByteArray& name); virtual const char* getName() const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant readVariant(Bitstream& in) const; protected: friend class Bitstream; + friend class JSONReader; QByteArray _name; - WeakTypeStreamerPointer _weakSelf; + WeakTypeStreamerPointer _weakSelf; ///< promoted to strong references in GenericValue when reading }; /// A streamer for generic enums. @@ -1051,6 +1319,9 @@ public: GenericEnumTypeStreamer(const QByteArray& name, const QVector& values, int bits, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1083,6 +1354,7 @@ public: MappedStreamableTypeStreamer(const TypeStreamer* baseStreamer, const QVector& fields); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1099,6 +1371,9 @@ public: GenericStreamableTypeStreamer(const QByteArray& name, const QVector& fields, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1153,6 +1428,7 @@ public: MappedListTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1169,11 +1445,14 @@ public: GenericListTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; -private: +protected: TypeStreamerPointer _valueStreamer; }; @@ -1206,6 +1485,7 @@ public: GenericSetTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; }; @@ -1232,6 +1512,7 @@ public: MappedMapTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& keyStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1250,6 +1531,9 @@ public: const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1264,25 +1548,31 @@ private: class GenericValueStreamer : public SimpleTypeStreamer { public: + QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; -/// Macro for registering simple type streamers. +/// Macro for registering simple type streamers. Typically, one would use this at the top level of the source file +/// associated with the type. #define REGISTER_SIMPLE_TYPE_STREAMER(X) static int X##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new SimpleTypeStreamer()); -/// Macro for registering collection type streamers. +/// Macro for registering collection type (QList, QVector, QSet, QMap) streamers. Typically, one would use this at the top +/// level of the source file associated with the type. #define REGISTER_COLLECTION_TYPE_STREAMER(X) static int x##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new CollectionTypeStreamer()); -/// Declares the metatype and the streaming operators. The last lines -/// ensure that the generated file will be included in the link phase. +/// Declares the metatype and the streaming operators. Typically, one would use this immediately after the definition of a +/// type flagged as STREAMABLE in its header file. The last lines ensure that the generated file will be included in the link +/// phase. #ifdef _WIN32 #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1292,6 +1582,8 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1302,18 +1594,27 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ _Pragma(STRINGIFY(unused(_TypePtr##X))) #endif +/// Declares an enum metatype. This is used when one desires to use an enum defined in a QObject outside of that class's +/// direct properties. Typically, one would use this immediately after the definition of the QObject containing the enum +/// in its header file. #define DECLARE_ENUM_METATYPE(S, N) Q_DECLARE_METATYPE(S::N) \ Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ - template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } + template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ + template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } +/// Implements an enum metatype. This performs the implementation of the previous macro, and would normally be used at the +/// top level of the source file associated with the QObject containing the enum. #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ Bitstream& operator<<(Bitstream& out, const S::N& obj) { \ @@ -1326,7 +1627,9 @@ public: return in.read(&obj, bits); \ } -/// Registers a simple type and its streamer. +/// Registers a simple type and its streamer. This would typically be used at the top level of the source file associated with +/// the type, and combines the registration of the type with the Qt metatype system with the registration of a simple type +/// streamer. /// \return the metatype id template int registerSimpleMetaType() { int type = qRegisterMetaType(); @@ -1334,7 +1637,8 @@ template int registerSimpleMetaType() { return type; } -/// Registers an enum type and its streamer. +/// Registers an enum type and its streamer. Rather than using this directly, consider using the IMPLEMENT_ENUM_METATYPE +/// macro. /// \return the metatype id template int registerEnumMetaType(const QMetaObject* metaObject, const char* name) { int type = qRegisterMetaType(); @@ -1342,7 +1646,8 @@ template int registerEnumMetaType(const QMetaObject* metaObject, const return type; } -/// Registers a streamable type and its streamer. +/// Registers a streamable type and its streamer. Rather than using this directly, consider using the +/// DECLARE_STREAMABLE_METATYPE macro and the mtc code generation tool. /// \return the metatype id template int registerStreamableMetaType() { int type = qRegisterMetaType(); @@ -1350,7 +1655,9 @@ template int registerStreamableMetaType() { return type; } -/// Registers a collection type and its streamer. +/// Registers a collection type and its streamer. This would typically be used at the top level of the source file associated +/// with the type, and combines the registration of the type with the Qt metatype system with the registration of a collection +/// type streamer. /// \return the metatype id template int registerCollectionMetaType() { int type = qRegisterMetaType(); @@ -1358,7 +1665,9 @@ template int registerCollectionMetaType() { return type; } -/// Flags a class as streamable (use as you would Q_OBJECT). +/// Flags a class as streamable. Use as you would Q_OBJECT: as the first line of a class definition, before any members or +/// access qualifiers. Typically, one would follow the definition with DECLARE_STREAMABLE_METATYPE. The mtc tool looks for +/// this macro in order to generate streaming (etc.) code for types. #define STREAMABLE public: \ static const int Type; \ static const QVector& getMetaFields(); \ @@ -1368,7 +1677,8 @@ template int registerCollectionMetaType() { private: \ static QHash createFieldIndices(); -/// Flags a field or base class as streaming. +/// Flags a (public) field within or base class of a streamable type as streaming. Use before all base classes or fields that +/// you want to stream. #define STREAM #endif // hifi_Bitstream_h diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index cf6ded74da..5ac88556f0 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -23,18 +23,50 @@ class ReliableChannel; -/// Performs simple datagram sequencing, packet fragmentation and reassembly. +/// Performs datagram sequencing, packet fragmentation and reassembly. Works with Bitstream to provide methods to send and +/// receive data over UDP with varying reliability and latency characteristics. To use, create a DatagramSequencer with the +/// fixed-size header that will be included with all outgoing datagrams and expected in all incoming ones (the contents of the +/// header are not checked on receive, only skipped over, and may be modified by the party that actually send the +/// datagram--this means that the header may include dynamically generated data, as long as its size remains fixed). Connect +/// the readyToWrite signal to a slot that will actually transmit the datagram to the remote party. When a datagram is +/// received from that party, call receivedDatagram with its contents. +/// +/// A "packet" represents a batch of data sent at one time (split into one or more datagrams sized below the MTU). Packets are +/// received in full and in order or not at all (that is, a packet being assembled is dropped as soon as a fragment from the +/// next packet is received). Packets can be any size, but the larger a packet is, the more likely it is to be dropped--so, +/// it's better to keep packet sizes close to the MTU. To write a packet, call startPacket, write data to the returned +/// Bitstream, then call endPacket (which will result in one or more firings of readyToWrite). Data written in this way is not +/// guaranteed to be received, but if it is received, it will arrive in order. This is a good way to transmit delta state: +/// state that represents the change between the last acknowledged state and the current state (which, if not received, will +/// not be resent as-is; instead, it will be replaced by up-to-date new deltas). +/// +/// There are two methods for sending reliable data. The first, for small messages that require minimum-latency processing, is +/// the high priority messaging system. When you call sendHighPriorityMessage, the message that you send will be included with +/// every outgoing packet until it is acknowledged. When the receiving party first sees the message, it will fire a +/// receivedHighPriorityMessage signal. +/// +/// The second method employs a set of independent reliable channels multiplexed onto the packet stream. These channels are +/// created lazily through the getReliableOutputChannel/getReliableInputChannel functions. Output channels contain buffers +/// to which one may write either arbitrary data (as a QIODevice) or messages (as QVariants), or switch between the two. +/// Each time a packet is sent, data pending for reliable output channels is added, in proportion to their relative priorities, +/// until the packet size limit set by setMaxPacketSize is reached. On the receive side, the streams are reconstructed and +/// (again, depending on whether messages are enabled) either the QIODevice reports that data is available, or, when a complete +/// message is decoded, the receivedMessage signal is fired. class DatagramSequencer : public QObject { Q_OBJECT public: + /// Contains the content of a high-priority message along with the number of the first packet in which it was sent. class HighPriorityMessage { public: QVariant data; int firstPacketNumber; }; + /// Creates a new datagram sequencer. + /// \param datagramHeader the content of the header that will be prepended to each outgoing datagram and whose length + /// will be skipped over in each incoming datagram DatagramSequencer(const QByteArray& datagramHeader = QByteArray(), QObject* parent = NULL); /// Returns a reference to the weak hash mapping remote ids to shared objects. @@ -267,15 +299,24 @@ class ReliableChannel : public QObject { public: + /// Returns the channel's index in the sequencer's channel map. int getIndex() const { return _index; } + /// Returns a reference to the buffer used to write/read data to/from this channel. CircularBuffer& getBuffer() { return _buffer; } + + /// Returns a reference to the data stream created on this channel's buffer. QDataStream& getDataStream() { return _dataStream; } + + /// Returns a reference to the bitstream created on this channel's data stream. Bitstream& getBitstream() { return _bitstream; } + /// Sets the channel priority, which determines how much of this channel's data (in proportion to the other channels) to + /// include in each outgoing packet. void setPriority(float priority) { _priority = priority; } float getPriority() const { return _priority; } + /// Returns the number of bytes available to read from this channel. int getBytesAvailable() const; /// Sets whether we expect to write/read framed messages. @@ -287,6 +328,7 @@ public: signals: + /// Fired when a framed message has been received on this channel. void receivedMessage(const QVariant& message); private slots: diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 8ac5333d10..0f87b0e607 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -66,7 +66,10 @@ enum PacketType { PacketTypeModelAddOrEdit, PacketTypeModelErase, PacketTypeModelAddResponse, - PacketTypeOctreeDataNack + PacketTypeOctreeDataNack, // 45 + PacketTypeVoxelEditNack, + PacketTypeParticleEditNack, + PacketTypeModelEditNack, }; typedef char PacketVersion; @@ -76,7 +79,7 @@ const QSet NON_VERIFIED_PACKETS = QSet() << PacketTypeDomainList << PacketTypeDomainListRequest << PacketTypeDomainOAuthRequest << PacketTypeCreateAssignment << PacketTypeRequestAssignment << PacketTypeStunResponse << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery - << PacketTypeOctreeDataNack; + << PacketTypeOctreeDataNack << PacketTypeVoxelEditNack << PacketTypeParticleEditNack << PacketTypeModelEditNack; const int NUM_BYTES_MD5_HASH = 16; const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; diff --git a/libraries/networking/src/ReceivedPacketProcessor.cpp b/libraries/networking/src/ReceivedPacketProcessor.cpp index 3ef518bbc2..59e1ecd456 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.cpp +++ b/libraries/networking/src/ReceivedPacketProcessor.cpp @@ -35,9 +35,10 @@ bool ReceivedPacketProcessor::process() { if (_packets.size() == 0) { _waitingOnPacketsMutex.lock(); - _hasPackets.wait(&_waitingOnPacketsMutex); + _hasPackets.wait(&_waitingOnPacketsMutex, getMaxWait()); _waitingOnPacketsMutex.unlock(); } + preProcess(); while (_packets.size() > 0) { lock(); // lock to make sure nothing changes on us NetworkPacket& packet = _packets.front(); // get the oldest packet @@ -46,7 +47,9 @@ bool ReceivedPacketProcessor::process() { _nodePacketCounts[temporary.getNode()->getUUID()]--; unlock(); // let others add to the packets processPacket(temporary.getNode(), temporary.getByteArray()); // process our temporary copy + midProcess(); } + postProcess(); return isStillRunning(); // keep running till they terminate us } diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 94ad2d9c41..607f9e54c2 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -33,9 +33,19 @@ public: /// Are there received packets waiting to be processed bool hasPacketsToProcess() const { return _packets.size() > 0; } - /// Are there received packets waiting to be processed from a certain node + /// Is a specified node still alive? + bool isAlive(const QUuid& nodeUUID) const { + return _nodePacketCounts.contains(nodeUUID); + } + + /// Are there received packets waiting to be processed from a specified node bool hasPacketsToProcessFrom(const SharedNodePointer& sendingNode) const { - return _nodePacketCounts[sendingNode->getUUID()] > 0; + return hasPacketsToProcessFrom(sendingNode->getUUID()); + } + + /// Are there received packets waiting to be processed from a specified node + bool hasPacketsToProcessFrom(const QUuid& nodeUUID) const { + return _nodePacketCounts[nodeUUID] > 0; } /// How many received packets waiting are to be processed @@ -53,9 +63,21 @@ protected: /// Implements generic processing behavior for this thread. virtual bool process(); + /// Determines the timeout of the wait when there are no packets to process. Default value means no timeout + virtual unsigned long getMaxWait() const { return ULONG_MAX; } + + /// Override to do work before the packets processing loop. Default does nothing. + virtual void preProcess() { } + + /// Override to do work inside the packet processing loop after a packet is processed. Default does nothing. + virtual void midProcess() { } + + /// Override to do work after the packets processing loop. Default does nothing. + virtual void postProcess() { } + virtual void terminating(); -private: +protected: QVector _packets; QHash _nodePacketCounts; diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp new file mode 100644 index 0000000000..841b5e909c --- /dev/null +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -0,0 +1,63 @@ +// +// SentPacketHistory.cpp +// libraries/networking/src +// +// Created by Yixin Wang on 6/5/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include "SentPacketHistory.h" +#include + +SentPacketHistory::SentPacketHistory(int size) + : _sentPackets(size), + _newestPacketAt(0), + _numExistingPackets(0), + _newestSequenceNumber(std::numeric_limits::max()) +{ +} + +void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& packet) { + + // check if given seq number has the expected value. if not, something's wrong with + // the code calling this function + uint16_t expectedSequenceNumber = _newestSequenceNumber + (uint16_t)1; + if (sequenceNumber != expectedSequenceNumber) { + qDebug() << "Unexpected sequence number passed to SentPacketHistory::packetSent()!" + << "Expected:" << expectedSequenceNumber << "Actual:" << sequenceNumber; + } + + _newestSequenceNumber = sequenceNumber; + + // increment _newestPacketAt cyclically, insert new packet there. + // this will overwrite the oldest packet in the buffer + _newestPacketAt = (_newestPacketAt == _sentPackets.size() - 1) ? 0 : _newestPacketAt + 1; + _sentPackets[_newestPacketAt] = packet; + if (_numExistingPackets < _sentPackets.size()) { + _numExistingPackets++; + } +} + +const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { + + const int UINT16_RANGE = std::numeric_limits::max() + 1; + + // if sequenceNumber > _newestSequenceNumber, assume sequenceNumber is from before the most recent rollover + // correct the diff so that it correctly represents how far back in the history sequenceNumber is + int seqDiff = (int)_newestSequenceNumber - (int)sequenceNumber; + if (seqDiff < 0) { + seqDiff += UINT16_RANGE; + } + // if desired sequence number is too old to be found in the history, return null + if (seqDiff >= _numExistingPackets) { + return NULL; + } + int packetAt = _newestPacketAt - seqDiff; + if (packetAt < 0) { + packetAt += _sentPackets.size(); + } + return &_sentPackets.at(packetAt); +} diff --git a/assignment-client/src/octree/SentPacketHistory.h b/libraries/networking/src/SentPacketHistory.h similarity index 61% rename from assignment-client/src/octree/SentPacketHistory.h rename to libraries/networking/src/SentPacketHistory.h index 4231400ac1..53a6919c42 100644 --- a/assignment-client/src/octree/SentPacketHistory.h +++ b/libraries/networking/src/SentPacketHistory.h @@ -1,6 +1,6 @@ // // SentPacketHistory.h -// assignement-client/src/octree +// libraries/networking/src // // Created by Yixin Wang on 6/5/2014 // @@ -11,25 +11,24 @@ #ifndef hifi_SentPacketHistory_h #define hifi_SentPacketHistory_h +#include #include #include -#include "OctreePacketData.h" - class SentPacketHistory { public: - SentPacketHistory(int size); + SentPacketHistory(int size = 1000); - void packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet); - const QByteArray* getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const; + void packetSent(uint16_t sequenceNumber, const QByteArray& packet); + const QByteArray* getPacket(uint16_t sequenceNumber) const; private: QVector _sentPackets; // circular buffer int _newestPacketAt; int _numExistingPackets; - OCTREE_PACKET_SEQUENCE _newestSequenceNumber; + uint16_t _newestSequenceNumber; }; #endif diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 898c41de08..43e253b2da 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -17,7 +17,6 @@ #include #include "OctreeEditPacketSender.h" - EditPacketBuffer::EditPacketBuffer(PacketType type, unsigned char* buffer, ssize_t length, QUuid nodeUUID) : _nodeUUID(nodeUUID), _currentType(type), @@ -89,7 +88,7 @@ bool OctreeEditPacketSender::serversExist() const { // This method is called when the edit packet layer has determined that it has a fully formed packet destined for // a known nodeID. -void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, ssize_t length) { +void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsigned char* buffer, ssize_t length) { NodeList* nodeList = NodeList::getInstance(); foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { @@ -97,12 +96,19 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned c if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { - queuePacketForSending(node, QByteArray(reinterpret_cast(buffer), length)); + QByteArray packet(reinterpret_cast(buffer), length); + queuePacketForSending(node, packet); + + // extract sequence number and add packet to history + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; + unsigned short int sequence = *((unsigned short int*)dataAt); + _sentPacketHistories[nodeUUID].packetSent(sequence, packet); // debugging output... bool wantDebugging = false; if (wantDebugging) { - int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast(buffer)); + int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast(buffer)); unsigned short int sequence = (*((unsigned short int*)(buffer + numBytesPacketHeader))); quint64 createdAt = (*((quint64*)(buffer + numBytesPacketHeader + sizeof(sequence)))); quint64 queuedAt = usecTimestampNow(); @@ -287,18 +293,20 @@ void OctreeEditPacketSender::releaseQueuedMessages() { if (!serversExist()) { _releaseQueuedMessagesPending = true; } else { - for (std::map::iterator i = _pendingEditPackets.begin(); i != _pendingEditPackets.end(); i++) { - releaseQueuedPacket(i->second); + for (QHash::iterator i = _pendingEditPackets.begin(); i != _pendingEditPackets.end(); i++) { + releaseQueuedPacket(i.value()); } } } void OctreeEditPacketSender::releaseQueuedPacket(EditPacketBuffer& packetBuffer) { + _releaseQueuedPacketMutex.lock(); if (packetBuffer._currentSize > 0 && packetBuffer._currentType != PacketTypeUnknown) { queuePacketToNode(packetBuffer._nodeUUID, &packetBuffer._currentBuffer[0], packetBuffer._currentSize); + packetBuffer._currentSize = 0; + packetBuffer._currentType = PacketTypeUnknown; } - packetBuffer._currentSize = 0; - packetBuffer._currentType = PacketTypeUnknown; + _releaseQueuedPacketMutex.unlock(); } void OctreeEditPacketSender::initializePacket(EditPacketBuffer& packetBuffer, PacketType type) { @@ -329,3 +337,41 @@ bool OctreeEditPacketSender::process() { // base class does most of the work. return PacketSender::process(); } + +void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { + // parse sending node from packet, retrieve packet history for that node + QUuid sendingNodeUUID = uuidFromPacketHeader(packet); + + // if packet history doesn't exist for the sender node (somehow), bail + if (!_sentPacketHistories.contains(sendingNodeUUID)) { + return; + } + const SentPacketHistory& sentPacketHistory = _sentPacketHistories.value(sendingNodeUUID); + + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; + + // read number of sequence numbers + uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); + dataAt += sizeof(uint16_t); + + // read sequence numbers and queue packets for resend + for (int i = 0; i < numSequenceNumbers; i++) { + unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); + dataAt += sizeof(unsigned short int); + + // retrieve packet from history + const QByteArray* packet = sentPacketHistory.getPacket(sequenceNumber); + if (packet) { + const SharedNodePointer& node = NodeList::getInstance()->getNodeHash().value(sendingNodeUUID); + queuePacketForSending(node, *packet); + } + } +} + +void OctreeEditPacketSender::nodeKilled(SharedNodePointer node) { + // TODO: add locks + QUuid nodeUUID = node->getUUID(); + _pendingEditPackets.remove(nodeUUID); + _sentPacketHistories.remove(nodeUUID); +} diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 0dc628c433..c16c0a2d4b 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -12,9 +12,11 @@ #ifndef hifi_OctreeEditPacketSender_h #define hifi_OctreeEditPacketSender_h +#include #include #include #include "JurisdictionMap.h" +#include "SentPacketHistory.h" /// Used for construction of edit packets class EditPacketBuffer { @@ -89,10 +91,16 @@ public: // you must override these... virtual char getMyNodeType() const = 0; virtual void adjustEditPacketForClockSkew(unsigned char* codeColorBuffer, ssize_t length, int clockSkew) { }; - + +public slots: + void nodeKilled(SharedNodePointer node); + +public: + void processNackPacket(const QByteArray& packet); + protected: bool _shouldSend; - void queuePacketToNode(const QUuid& nodeID, unsigned char* buffer, ssize_t length); + void queuePacketToNode(const QUuid& nodeID, const unsigned char* buffer, ssize_t length); void queuePendingPacketToNodes(PacketType type, unsigned char* buffer, ssize_t length); void queuePacketToNodes(unsigned char* buffer, ssize_t length); void initializePacket(EditPacketBuffer& packetBuffer, PacketType type); @@ -101,7 +109,7 @@ protected: void processPreServerExistsPackets(); // These are packets which are destined from know servers but haven't been released because they're still too small - std::map _pendingEditPackets; + QHash _pendingEditPackets; // These are packets that are waiting to be processed because we don't yet know if there are servers int _maxPendingMessages; @@ -114,5 +122,10 @@ protected: unsigned short int _sequenceNumber; int _maxPacketSize; + + QMutex _releaseQueuedPacketMutex; + + // TODO: add locks for this and _pendingEditPackets + QHash _sentPacketHistories; }; #endif // hifi_OctreeEditPacketSender_h diff --git a/libraries/octree/src/OctreeRenderer.h b/libraries/octree/src/OctreeRenderer.h index 18e68e26aa..d61ed3afce 100644 --- a/libraries/octree/src/OctreeRenderer.h +++ b/libraries/octree/src/OctreeRenderer.h @@ -62,7 +62,7 @@ public: static bool renderOperation(OctreeElement* element, void* extraData); /// clears the tree - void clear(); + virtual void clear(); protected: Octree* _tree; bool _managedTree; diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 868ef29886..28445ec327 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include #include @@ -46,7 +47,6 @@ OctreeSceneStats::OctreeSceneStats() : _incomingReallyLate(0), _incomingPossibleDuplicate(0), _missingSequenceNumbers(), - _sequenceNumbersToNack(), _incomingFlightTimeAverage(samples), _jurisdictionRoot(NULL) { @@ -159,7 +159,6 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingPossibleDuplicate = other._incomingPossibleDuplicate; _missingSequenceNumbers = other._missingSequenceNumbers; - _sequenceNumbersToNack = other._sequenceNumbersToNack; } @@ -842,7 +841,7 @@ const char* OctreeSceneStats::getItemValue(Item item) { } void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, - bool wasStatsPacket, int nodeClockSkewUsec) { + bool wasStatsPacket, int nodeClockSkewUsec) { const bool wantExtraDebugging = false; int numBytesPacketHeader = numBytesForPacketHeader(packet); @@ -852,10 +851,10 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, dataAt += sizeof(OCTREE_PACKET_FLAGS); OCTREE_PACKET_SEQUENCE sequence = (*(OCTREE_PACKET_SEQUENCE*)dataAt); dataAt += sizeof(OCTREE_PACKET_SEQUENCE); - + OCTREE_PACKET_SENT_TIME sentAt = (*(OCTREE_PACKET_SENT_TIME*)dataAt); dataAt += sizeof(OCTREE_PACKET_SENT_TIME); - + //bool packetIsColored = oneAtBit(flags, PACKET_IS_COLOR_BIT); //bool packetIsCompressed = oneAtBit(flags, PACKET_IS_COMPRESSED_BIT); @@ -877,29 +876,15 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, return; // ignore any packets that are unreasonable } + const int UINT16_RANGE = std::numeric_limits::max() + 1; + // determine our expected sequence number... handle rollover appropriately - OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + 1 : sequence; - - // Guard against possible corrupted packets... with bad sequence numbers - const int MAX_RESONABLE_SEQUENCE_OFFSET = 2000; - const int MIN_RESONABLE_SEQUENCE_OFFSET = -2000; - int sequenceOffset = (sequence - expected); - if (sequenceOffset > MAX_RESONABLE_SEQUENCE_OFFSET || sequenceOffset < MIN_RESONABLE_SEQUENCE_OFFSET) { - qDebug() << "ignoring unreasonable packet... sequence:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; - return; // ignore any packets that are unreasonable - } - - // track packets here... - _incomingPacket++; - _incomingBytes += packet.size(); - if (!wasStatsPacket) { - _incomingWastedBytes += (MAX_PACKET_SIZE - packet.size()); - } + OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + (quint16)1 : sequence; const int USECS_PER_MSEC = 1000; float flightTimeMsecs = flightTime / USECS_PER_MSEC; _incomingFlightTimeAverage.updateAverage(flightTimeMsecs); - + // track out of order and possibly lost packets... if (sequence == _incomingLastSequence) { if (wantExtraDebugging) { @@ -911,15 +896,46 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "out of order... got:" << sequence << "expected:" << expected; } + int sequenceInt = (int)sequence; + int expectedInt = (int)expected; + + // if distance between sequence and expected are more than half of the total range of possible seq numbers, + // assume that a rollover occurred between the two. + // correct the larger one so it's in the range [-UINT16_RANGE, -1] while the other remains in [0, UINT16_RANGE-1] + // after doing so, sequenceInt and expectedInt can be correctly compared to each other, though one may be negative + if (std::abs(sequenceInt - expectedInt) > UINT16_RANGE / 2) { + if (sequenceInt > expectedInt) { + sequenceInt -= UINT16_RANGE; + } + else { + expectedInt -= UINT16_RANGE; + } + } + + // Guard against possible corrupted packets... with bad sequence numbers + const int MAX_RESONABLE_SEQUENCE_OFFSET = 2000; + const int MIN_RESONABLE_SEQUENCE_OFFSET = -2000; + + int sequenceOffset = (sequenceInt - expectedInt); + if (sequenceOffset > MAX_RESONABLE_SEQUENCE_OFFSET || sequenceOffset < MIN_RESONABLE_SEQUENCE_OFFSET) { + qDebug() << "ignoring unreasonable packet... sequence:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; + return; // ignore any packets that are unreasonable + } + // if the sequence is less than our expected, then this might be a packet // that was delayed and so we should find it in our lostSequence list - if (sequence < expected) { + if (sequenceInt < expectedInt) { + + // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] + // if rollover between them: sequenceInt in [-UINT16_RANGE, -1], expectedInt in [0, UINT16_RANGE-1] + if (wantExtraDebugging) { qDebug() << "this packet is later than expected..."; } - if (sequence < std::max(0, (expected - MAX_MISSING_SEQUENCE_OLD_AGE))) { + if (sequenceInt < expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { _incomingReallyLate++; - } else { + } + else { _incomingLate++; } @@ -928,78 +944,91 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "found it in _missingSequenceNumbers"; } _missingSequenceNumbers.remove(sequence); - _sequenceNumbersToNack.remove(sequence); _incomingLikelyLost--; _incomingRecovered++; - } else { + } + else { // if we're still in our pruning window, and we didn't find it in our missing list, // than this is really unexpected and can probably only happen if the packet was a // duplicate - if (sequence >= std::max(0, (expected - MAX_MISSING_SEQUENCE_OLD_AGE))) { + if (sequenceInt >= expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { if (wantExtraDebugging) { - qDebug() << "sequence:" << sequence << "WAS NOT found in _missingSequenceNumbers, and not that old... (expected - MAX_MISSING_SEQUENCE_OLD_AGE):" << (expected - MAX_MISSING_SEQUENCE_OLD_AGE); + qDebug() << "sequence:" << sequence << "WAS NOT found in _missingSequenceNumbers, and not that old... (expected - MAX_MISSING_SEQUENCE_OLD_AGE):" + << (uint16_t)(expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE); } _incomingPossibleDuplicate++; } } - } - - if (sequence > expected) { + + // don't update _incomingLastSequence in this case. + // only bump the last sequence if it was greater than our expected sequence, this will keep us from + // accidentally going backwards when an out of order (recovered) packet comes in + + } else { // sequenceInt > expectedInt + + // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] + // if rollover between them: sequenceInt in [0, UINT16_RANGE-1], expectedInt in [-UINT16_RANGE, -1] + if (wantExtraDebugging) { qDebug() << "this packet is earlier than expected..."; } _incomingEarly++; // hmm... so, we either didn't get some packets, or this guy came early... - unsigned int missing = sequence - expected; + int missing = sequenceInt - expectedInt; if (wantExtraDebugging) { qDebug() << ">>>>>>>> missing gap=" << missing; } _incomingLikelyLost += missing; - for(unsigned int missingSequence = expected; missingSequence < sequence; missingSequence++) { + for (int missingSequenceInt = expectedInt; missingSequenceInt < sequenceInt; missingSequenceInt++) { + OCTREE_PACKET_SEQUENCE missingSequence = missingSequenceInt >= 0 ? missingSequenceInt : missingSequenceInt + UINT16_RANGE; _missingSequenceNumbers << missingSequence; - _sequenceNumbersToNack << missingSequence; } + + _incomingLastSequence = sequence; } + } else { // sequence = expected + + _incomingLastSequence = sequence; } } - // only bump the last sequence if it was greater than our expected sequence, this will keep us from - // accidentally going backwards when an out of order (recovered) packet comes in - if (sequence >= expected) { - _incomingLastSequence = sequence; - } - // do some garbage collecting on our _missingSequenceNumbers if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE) { if (wantExtraDebugging) { qDebug() << "too many _missingSequenceNumbers:" << _missingSequenceNumbers.size(); } + + int oldAgeCutoff = (int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE; + foreach(uint16_t missingItem, _missingSequenceNumbers) { if (wantExtraDebugging) { qDebug() << "checking item:" << missingItem << "is it in need of pruning?"; - qDebug() << "(_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE):" - << (_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE); + qDebug() << "(_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE):" + << (uint16_t)((int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE); } - if (missingItem <= std::max(0, _incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE)) { + + bool prune; + if (oldAgeCutoff >= 0) { + prune = (missingItem <= oldAgeCutoff || missingItem > _incomingLastSequence); + } + else { + prune = (missingItem <= oldAgeCutoff + UINT16_RANGE && missingItem > _incomingLastSequence); + } + + if (prune) { if (wantExtraDebugging) { qDebug() << "pruning really old missing sequence:" << missingItem; } _missingSequenceNumbers.remove(missingItem); - _sequenceNumbersToNack.remove(missingItem); } } } -} - -int OctreeSceneStats::getNumSequenceNumbersToNack() const { - return _sequenceNumbersToNack.size(); -} - -uint16_t OctreeSceneStats::getNextSequenceNumberToNack() { - QSet::Iterator it = _sequenceNumbersToNack.begin(); - uint16_t sequenceNumber = *it; - _sequenceNumbersToNack.remove(sequenceNumber); - return sequenceNumber; + // track packets here... + _incomingPacket++; + _incomingBytes += packet.size(); + if (!wasStatsPacket) { + _incomingWastedBytes += (MAX_PACKET_SIZE - packet.size()); + } } diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index 182bd6c86c..1c468a8dc6 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -172,9 +172,8 @@ public: quint32 getIncomingReallyLate() const { return _incomingReallyLate; } quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } - - int getNumSequenceNumbersToNack() const; - OCTREE_PACKET_SEQUENCE getNextSequenceNumberToNack(); + + const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } private: @@ -277,7 +276,6 @@ private: quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate QSet _missingSequenceNumbers; - QSet _sequenceNumbersToNack; SimpleMovingAverage _incomingFlightTimeAverage; // features related items diff --git a/libraries/shared/src/GenericThread.h b/libraries/shared/src/GenericThread.h index bbb01894ed..b2c0eb13db 100644 --- a/libraries/shared/src/GenericThread.h +++ b/libraries/shared/src/GenericThread.h @@ -57,7 +57,7 @@ protected: bool isStillRunning() const { return !_stopThread; } -private: +protected: QMutex _mutex; bool _stopThread; diff --git a/libraries/voxels/src/Tags.cpp b/libraries/voxels/src/Tags.cpp index d3c534735e..003e9ae80d 100644 --- a/libraries/voxels/src/Tags.cpp +++ b/libraries/voxels/src/Tags.cpp @@ -177,7 +177,9 @@ int retrieveData(std::string filename, std::stringstream &ss) { int type = file.peek(); if (type == 0x0A) { ss.flush(); - ss << file; + std::copy(std::istreambuf_iterator(file), + std::istreambuf_iterator(), + std::ostreambuf_iterator(ss)); return 0; } diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 6bd99a6c82..c9bce27cc3 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -83,7 +83,7 @@ static TestSharedObjectA::TestFlags getRandomTestFlags() { return flags; } -static QScriptValue createRandomScriptValue(bool complex = false) { +static QScriptValue createRandomScriptValue(bool complex = false, bool ensureHashOrder = false) { scriptObjectsCreated++; switch (randIntInRange(0, complex ? 5 : 3)) { case 0: @@ -108,31 +108,37 @@ static QScriptValue createRandomScriptValue(bool complex = false) { } default: { QScriptValue value = ScriptCache::getInstance()->getEngine()->newObject(); - if (randomBoolean()) { + if (ensureHashOrder) { + // we can't depend on the iteration order, so if we need it to be the same (as when comparing bytes), we + // can only have one property value.setProperty("foo", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bar", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("baz", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bong", createRandomScriptValue()); + } else { + if (randomBoolean()) { + value.setProperty("foo", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bar", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("baz", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bong", createRandomScriptValue()); + } } return value; } } } -static TestMessageC createRandomMessageC() { +static TestMessageC createRandomMessageC(bool ensureHashOrder = false) { TestMessageC message; message.foo = randomBoolean(); message.bar = rand(); message.baz = randFloat(); message.bong.foo = createRandomBytes(); message.bong.baz = getRandomTestEnum(); - message.bizzle = createRandomScriptValue(true); + message.bizzle = createRandomScriptValue(true, ensureHashOrder); return message; } @@ -146,7 +152,7 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { SharedObjectPointer testObjectWrittenB = new TestSharedObjectB(randFloat(), createRandomBytes(), TestSharedObjectB::THIRD_TEST_ENUM, TestSharedObjectB::SECOND_TEST_FLAG); out << testObjectWrittenB; - TestMessageC messageWritten = createRandomMessageC(); + TestMessageC messageWritten = createRandomMessageC(true); out << QVariant::fromValue(messageWritten); QByteArray endWritten = "end"; out << endWritten; @@ -243,6 +249,74 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { return true; } + if (metadataType != Bitstream::FULL_METADATA) { + return false; + } + + // now write to JSON + JSONWriter jsonWriter; + jsonWriter << testObjectReadA; + jsonWriter << testObjectReadB; + jsonWriter << messageRead; + jsonWriter << endRead; + QByteArray encodedJson = jsonWriter.getDocument().toJson(); + + // and read from JSON + JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); + jsonReader >> testObjectReadA; + jsonReader >> testObjectReadB; + jsonReader >> messageRead; + jsonReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray secondCompareArray; + QDataStream secondCompareOutStream(&secondCompareArray, QIODevice::WriteOnly); + Bitstream secondCompareOut(secondCompareOutStream, Bitstream::FULL_METADATA); + secondCompareOut << testObjectReadA; + secondCompareOut << testObjectReadB; + secondCompareOut << messageRead; + secondCompareOut << endRead; + secondCompareOut.flush(); + + if (compareArray != secondCompareArray) { + qDebug() << "Mismatch between written/JSON streams (generics)."; + return true; + } + + // once more, with mapping! + JSONReader secondJSONReader(QJsonDocument::fromJson(encodedJson)); + secondJSONReader >> testObjectReadA; + secondJSONReader >> testObjectReadB; + secondJSONReader >> messageRead; + secondJSONReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray thirdCompareArray; + QDataStream thirdCompareOutStream(&thirdCompareArray, QIODevice::WriteOnly); + Bitstream thirdCompareOut(thirdCompareOutStream, Bitstream::FULL_METADATA); + thirdCompareOut << testObjectReadA; + thirdCompareOut << testObjectReadB; + thirdCompareOut << messageRead; + thirdCompareOut << endRead; + thirdCompareOut.flush(); + + if (compareArray != thirdCompareArray) { + qDebug() << "Mismatch between written/JSON streams (mapped)."; + return true; + } + return false; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..79db82e90f --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 2.8) + +# add the tool directories +add_subdirectory(bitstream2json) +add_subdirectory(json2bitstream) +add_subdirectory(mtc) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt new file mode 100644 index 0000000000..fde80b4d33 --- /dev/null +++ b/tools/bitstream2json/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME bitstream2json) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/bitstream2json/src/main.cpp b/tools/bitstream2json/src/main.cpp new file mode 100644 index 0000000000..0f299527b0 --- /dev/null +++ b/tools/bitstream2json/src/main.cpp @@ -0,0 +1,70 @@ +// +// main.cpp +// tools/bitstream2json/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream inputData(&inputFile); + Bitstream input(inputData, Bitstream::FULL_METADATA, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + JSONWriter output; + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value = streamer->read(input); + output.appendToContents(streamer->getJSONData(output, value)); + } + } + + outputFile.write(output.getDocument().toJson()); + + return 0; +} diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt new file mode 100644 index 0000000000..51382d5858 --- /dev/null +++ b/tools/json2bitstream/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME json2bitstream) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/src/main.cpp b/tools/json2bitstream/src/main.cpp new file mode 100644 index 0000000000..ff09bcfdc6 --- /dev/null +++ b/tools/json2bitstream/src/main.cpp @@ -0,0 +1,76 @@ +// +// main.cpp +// tools/json2bitstream/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QJsonParseError error; + QJsonDocument document = QJsonDocument::fromJson(inputFile.readAll(), &error); + if (error.error != QJsonParseError::NoError) { + cerr << "Failed to read input file: " << error.errorString().toLatin1().constData() << endl; + return 1; + } + JSONReader input(document, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream outputData(&outputFile); + Bitstream output(outputData, Bitstream::FULL_METADATA); + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value; + streamer->putJSONData(input, input.retrieveNextFromContents(), value); + streamer->write(output, value); + } + } + output.flush(); + + return 0; +} diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 096ade4625..dd0ab837a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -123,11 +123,6 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "QHash " << name << "::createFieldIndices() {\n"; out << " QHash indices;\n"; out << " int index = 1;\n"; - foreach (const QString& base, str.clazz.bases) { - out << " foreach (const MetaField& field, " << base << "::getMetaFields()) {\n"; - out << " indices.insert(field.getName(), index++);\n"; - out << " }\n"; - } out << " foreach (const MetaField& field, getMetaFields()) {\n"; out << " indices.insert(field.getName(), index++);\n"; out << " }\n"; @@ -217,6 +212,36 @@ void generateOutput (QTextStream& out, const QList& streamables) { } out << "}\n"; + out << "template<> QJsonValue JSONWriter::getData(const " << name << "& value) {\n"; + out << " QJsonArray array;\n"; + foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } + foreach (const Field& field, str.fields) { + out << " array.append(getData(value." << field.name << "));\n"; + } + out << " return array;\n"; + out << "}\n"; + + out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; + if (!(str.clazz.bases.isEmpty() && str.fields.isEmpty())) { + out << " QJsonArray array = data.toArray(), subarray;\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + foreach (const QString& base, str.clazz.bases) { + out << " subarray = QJsonArray();\n"; + out << " for (int i = 0; i < " << base << "::getMetaFields().size(); i++) {\n"; + out << " subarray.append(*it++);\n"; + out << " }\n"; + out << " putData(subarray, static_cast<" << base << "&>(value));\n"; + } + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } + } + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true";