Merge branch 'master' of https://github.com/highfidelity/hifi into typed_arrays

assignment-client/src/metavoxels/MetavoxelServer.cpp

@@ -10,6 +10,9 @@
 //
 
 #include <QDateTime>
+#include <QFile>
+#include <QSaveFile>
+#include <QThread>
 
 #include <PacketHeaders.h>
 
@@ -44,6 +47,18 @@ void MetavoxelServer::run() {
     
     _lastSend = QDateTime::currentMSecsSinceEpoch();
     _sendTimer.start(SEND_INTERVAL);
+    
+    // initialize Bitstream before using it in multiple threads
+    Bitstream::preThreadingInit();
+    
+    // create the persister and start it in its own thread
+    _persister = new MetavoxelPersister(this);
+    QThread* persistenceThread = new QThread(this);
+    _persister->moveToThread(persistenceThread);
+    persistenceThread->start();
+    
+    // queue up the load
+    QMetaObject::invokeMethod(_persister, "load");
 }
 
 void MetavoxelServer::readPendingDatagrams() {
@@ -67,6 +82,12 @@ void MetavoxelServer::readPendingDatagrams() {
     }
 }
 
+void MetavoxelServer::aboutToFinish() {
+    QMetaObject::invokeMethod(_persister, "save", Q_ARG(const MetavoxelData&, _data));
+    _persister->thread()->quit();
+    _persister->thread()->wait();
+}
+
 void MetavoxelServer::maybeAttachSession(const SharedNodePointer& node) {
     if (node->getType() == NodeType::Agent) {
         QMutexLocker locker(&node->getMutex());
@@ -193,3 +214,44 @@ void MetavoxelSession::sendPacketGroup(int alreadySent) {
         _sequencer.endPacket();
     }
 }
+
+MetavoxelPersister::MetavoxelPersister(MetavoxelServer* server) :
+    _server(server) {
+}
+
+const char* SAVE_FILE = "metavoxels.dat";
+
+void MetavoxelPersister::load() {
+    QFile file(SAVE_FILE);
+    if (!file.exists()) {
+        return;
+    }
+    MetavoxelData data;
+    {
+        QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "...";
+        file.open(QIODevice::ReadOnly);
+        QDataStream inStream(&file);
+        Bitstream in(inStream);
+        try {
+            in >> data;
+        } catch (const BitstreamException& e) {
+            debug << "failed, " << e.getDescription();
+            return;
+        }
+        QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data));
+        debug << "done.";
+    }
+    data.dumpStats();
+}
+
+void MetavoxelPersister::save(const MetavoxelData& data) {
+    QDebug debug = qDebug() << "Writing to" << SAVE_FILE << "...";
+    QSaveFile file(SAVE_FILE);
+    file.open(QIODevice::WriteOnly);
+    QDataStream outStream(&file);
+    Bitstream out(outStream);
+    out << data;
+    out.flush();
+    file.commit();
+    debug << "done.";
+}

assignment-client/src/metavoxels/MetavoxelServer.h

@@ -20,6 +20,7 @@
 #include <Endpoint.h>
 
 class MetavoxelEditMessage;
+class MetavoxelPersister;
 class MetavoxelSession;
 
 /// Maintains a shared metavoxel system, accepting change requests and broadcasting updates.
@@ -34,10 +35,14 @@ public:
 
     const MetavoxelData& getData() const { return _data; }
     
+    Q_INVOKABLE void setData(const MetavoxelData& data) { _data = data; }
+
     virtual void run();
     
     virtual void readPendingDatagrams();
     
+    virtual void aboutToFinish();
+    
 private slots:
 
     void maybeAttachSession(const SharedNodePointer& node);
@@ -45,6 +50,8 @@ private slots:
     
 private:
     
+    MetavoxelPersister* _persister;
+    
     QTimer _sendTimer;
     qint64 _lastSend;
     
@@ -88,4 +95,20 @@ private:
     int _reliableDeltaID;
 };
 
+/// Handles persistence in a separate thread.
+class MetavoxelPersister : public QObject {
+    Q_OBJECT
+
+public:
+    
+    MetavoxelPersister(MetavoxelServer* server);
+
+    Q_INVOKABLE void load();
+    Q_INVOKABLE void save(const MetavoxelData& data);
+
+private:
+    
+    MetavoxelServer* _server;
+};
+
 #endif // hifi_MetavoxelServer_h

interface/src/Menu.cpp

@@ -441,13 +441,11 @@ Menu::Menu() :
     QMenu* timingMenu = developerMenu->addMenu("Timing and Statistics Tools");
     QMenu* perfTimerMenu = timingMenu->addMenu("Performance Timer");
     addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::DisplayTimingDetails, 0, true);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandDisplaySideTiming, 0, false);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandAvatarSimulateTiming, 0, false);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandAvatarUpdateTiming, 0, false);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMiscAvatarTiming, 0, false);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandIdleTiming, 0, false);
-    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandPaintGLTiming, 0, false);
     addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandUpdateTiming, 0, false);
+    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMyAvatarTiming, 0, false);
+    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandMyAvatarSimulateTiming, 0, false);
+    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandOtherAvatarTiming, 0, false);
+    addCheckableActionToQMenuAndActionHash(perfTimerMenu, MenuOption::ExpandPaintGLTiming, 0, false);
 
     addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::TestPing, 0, true);
     addCheckableActionToQMenuAndActionHash(timingMenu, MenuOption::FrameTimer);

interface/src/Menu.h

@@ -357,13 +357,11 @@ namespace MenuOption {
     const QString EnableGlowEffect = "Enable Glow Effect (Warning: Poor Oculus Performance)";
     const QString Enable3DTVMode = "Enable 3DTV Mode";
     const QString EnableVRMode = "Enable VR Mode";
-    const QString ExpandMiscAvatarTiming = "Expand Misc MyAvatar Timing";
+    const QString ExpandMyAvatarSimulateTiming = "Expand /myAvatar/simulation";
-    const QString ExpandAvatarUpdateTiming = "Expand MyAvatar update Timing";
+    const QString ExpandMyAvatarTiming = "Expand /myAvatar";
-    const QString ExpandAvatarSimulateTiming = "Expand MyAvatar simulate Timing";
+    const QString ExpandOtherAvatarTiming = "Expand /otherAvatar";
-    const QString ExpandDisplaySideTiming = "Expand Display Side Timing";
+    const QString ExpandPaintGLTiming = "Expand /paintGL";
-    const QString ExpandIdleTiming = "Expand Idle Timing";
+    const QString ExpandUpdateTiming = "Expand /update";
-    const QString ExpandPaintGLTiming = "Expand PaintGL Timing";
-    const QString ExpandUpdateTiming = "Expand Update Timing";
     const QString Faceplus = "Faceplus";
     const QString Faceshift = "Faceshift";
     const QString FilterSixense = "Smooth Sixense Movement";

interface/src/MetavoxelSystem.cpp

@@ -48,8 +48,10 @@ void MetavoxelSystem::init() {
 }
 
 MetavoxelLOD MetavoxelSystem::getLOD() const {
-    const float FIXED_LOD_THRESHOLD = 0.01f;
+    // the LOD threshold is temporarily tied to the avatar LOD parameter
-    return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD);
+    const float BASE_LOD_THRESHOLD = 0.01f;
+    return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(),
+        BASE_LOD_THRESHOLD * Menu::getInstance()->getAvatarLODDistanceMultiplier());
 }
 
 void MetavoxelSystem::simulate(float deltaTime) {

interface/src/ui/Stats.cpp

@@ -162,36 +162,25 @@ void Stats::drawBackground(unsigned int rgba, int x, int y, int width, int heigh
 }
 
 bool Stats::includeTimingRecord(const QString& name) {
-    bool included = false;
     if (Menu::getInstance()->isOptionChecked(MenuOption::DisplayTimingDetails)) {
-
+        if (name.startsWith("/idle/update/")) {
-        if (name == "idle/update") {
+            if (name.startsWith("/idle/update/myAvatar/")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming) ||
+                if (name.startsWith("/idle/update/myAvatar/simulate/")) {
-                       Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming);
+                    return Menu::getInstance()->isOptionChecked(MenuOption::ExpandMyAvatarSimulateTiming);
-        } else if (name == "idle/updateGL") {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming);
-        } else if (name.startsWith("idle/update")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming);
-        } else if (name.startsWith("idle/")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandIdleTiming);
-        } else if (name.startsWith("MyAvatar::simulate")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandAvatarSimulateTiming);
-        } else if (name.startsWith("MyAvatar::update/") || name.startsWith("updateMyAvatar")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandAvatarUpdateTiming);
-        } else if (name.startsWith("MyAvatar::")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandMiscAvatarTiming);
-        } else if (name == "paintGL/displaySide") {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandDisplaySideTiming) ||
-                       Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming);
-        } else if (name.startsWith("paintGL/displaySide/")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandDisplaySideTiming);
-        } else if (name.startsWith("paintGL/")) {
-            included = Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming);
-        } else {
-            included = true; // include everything else
                 }
+                return Menu::getInstance()->isOptionChecked(MenuOption::ExpandMyAvatarTiming);
+            } else if (name.startsWith("/idle/update/otherAvatars/")) {
+                return Menu::getInstance()->isOptionChecked(MenuOption::ExpandOtherAvatarTiming);
             }
-    return included;
+            return Menu::getInstance()->isOptionChecked(MenuOption::ExpandUpdateTiming);
+        } else if (name.startsWith("/idle/updateGL/paintGL/")) {
+            return Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming);
+        } else if (name.startsWith("/paintGL/")) {
+            return Menu::getInstance()->isOptionChecked(MenuOption::ExpandPaintGLTiming);
+        }
+        return true;
+    }
+    return false;
 }
 
 // display expanded or contracted stats
@@ -435,7 +424,7 @@ void Stats::display(
     MyAvatar* myAvatar = Application::getInstance()->getAvatar();
     glm::vec3 avatarPos = myAvatar->getPosition();
 
-    lines = _expanded ? 5 : 3;
+    lines = _expanded ? 8 : 3;
 
     drawBackground(backgroundColor, horizontalOffset, 0, _geoStatsWidth, lines * STATS_PELS_PER_LINE + 10);
     horizontalOffset += 5;
@@ -477,6 +466,41 @@ void Stats::display(
         
         verticalOffset += STATS_PELS_PER_LINE;
         drawText(horizontalOffset, verticalOffset, scale, rotation, font, downloads.str().c_str(), color);
+        
+        int internal = 0, leaves = 0;
+        int sendProgress = 0, sendTotal = 0;
+        int receiveProgress = 0, receiveTotal = 0;
+        foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) {
+            if (node->getType() == NodeType::MetavoxelServer) {
+                QMutexLocker locker(&node->getMutex());
+                MetavoxelClient* client = static_cast<MetavoxelSystemClient*>(node->getLinkedData());
+                if (client) {
+                    client->getData().countNodes(internal, leaves, Application::getInstance()->getMetavoxels()->getLOD());
+                    client->getSequencer().addReliableChannelStats(sendProgress, sendTotal, receiveProgress, receiveTotal);
+                }
+            }
+        }
+        stringstream nodes;
+        nodes << "Metavoxels: " << (internal + leaves);
+        verticalOffset += STATS_PELS_PER_LINE;
+        drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodes.str().c_str(), color);
+        
+        stringstream nodeTypes;
+        nodeTypes << "Internal: " << internal << "  Leaves: " << leaves;
+        verticalOffset += STATS_PELS_PER_LINE;
+        drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodeTypes.str().c_str(), color);
+        
+        if (sendTotal > 0 || receiveTotal > 0) {
+            stringstream reliableStats;
+            if (sendTotal > 0) {
+                reliableStats << "Upload: " << (sendProgress * 100 / sendTotal) << "%  ";
+            }
+            if (receiveTotal > 0) {
+                reliableStats << "Download: " << (receiveProgress * 100 / receiveTotal) << "%";
+            }
+            verticalOffset += STATS_PELS_PER_LINE;
+            drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableStats.str().c_str(), color);
+        }
     }
 
     verticalOffset = 0;

libraries/metavoxels/src/AttributeRegistry.cpp

@@ -9,7 +9,9 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <QReadLocker>
 #include <QScriptEngine>
+#include <QWriteLocker>
 
 #include "AttributeRegistry.h"
 #include "MetavoxelData.h"
@@ -69,6 +71,7 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute
     if (!attribute) {
         return attribute;
     }
+    QWriteLocker locker(&_attributesLock);
     AttributePointer& pointer = _attributes[attribute->getName()];
     if (!pointer) {
         pointer = attribute;
@@ -77,9 +80,15 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute
 }
 
 void AttributeRegistry::deregisterAttribute(const QString& name) {
+    QWriteLocker locker(&_attributesLock);
     _attributes.remove(name);
 }
 
+AttributePointer AttributeRegistry::getAttribute(const QString& name) {
+    QReadLocker locker(&_attributesLock);
+    return _attributes.value(name);
+}
+
 QScriptValue AttributeRegistry::getAttribute(QScriptContext* context, QScriptEngine* engine) {
     return engine->newQObject(getInstance()->getAttribute(context->argument(0).toString()).data(), QScriptEngine::QtOwnership,
         QScriptEngine::PreferExistingWrapperObject);
@@ -559,6 +568,10 @@ void SpannerSetAttribute::readMetavoxelRoot(MetavoxelData& data, MetavoxelStream
         }
         data.insert(state.attribute, object);
     }
+    // even if the root is empty, it should still exist
+    if (!data.getRoot(state.attribute)) {
+        data.createRoot(state.attribute);
+    }
 }
 
 void SpannerSetAttribute::writeMetavoxelRoot(const MetavoxelNode& root, MetavoxelStreamState& state) {
@@ -577,6 +590,10 @@ void SpannerSetAttribute::readMetavoxelDelta(MetavoxelData& data,
         }
         data.toggle(state.attribute, object);
     }
+    // even if the root is empty, it should still exist
+    if (!data.getRoot(state.attribute)) {
+        data.createRoot(state.attribute);
+    }
 }
 
 void SpannerSetAttribute::writeMetavoxelDelta(const MetavoxelNode& root,

libraries/metavoxels/src/AttributeRegistry.h

@@ -14,6 +14,7 @@
 
 #include <QHash>
 #include <QObject>
+#include <QReadWriteLock>
 #include <QSharedPointer>
 #include <QString>
 #include <QWidget>
@@ -61,11 +62,14 @@ public:
     void deregisterAttribute(const QString& name);
     
     /// Retrieves an attribute by name.
-    AttributePointer getAttribute(const QString& name) const { return _attributes.value(name); }
+    AttributePointer getAttribute(const QString& name);
     
     /// Returns a reference to the attribute hash.
     const QHash<QString, AttributePointer>& getAttributes() const { return _attributes; }
     
+    /// Returns a reference to the attributes lock.
+    QReadWriteLock& getAttributesLock() { return _attributesLock; }    
+    
     /// Returns a reference to the standard SharedObjectPointer "guide" attribute.
     const AttributePointer& getGuideAttribute() const { return _guideAttribute; }
     
@@ -92,6 +96,8 @@ private:
     static QScriptValue getAttribute(QScriptContext* context, QScriptEngine* engine);
 
     QHash<QString, AttributePointer> _attributes;
+    QReadWriteLock _attributesLock;
+    
     AttributePointer _guideAttribute;
     AttributePointer _spannersAttribute;
     AttributePointer _colorAttribute;

libraries/metavoxels/src/Bitstream.cpp

@@ -87,6 +87,12 @@ IDStreamer& IDStreamer::operator>>(int& value) {
     return *this;
 }
 
+void Bitstream::preThreadingInit() {
+    getObjectStreamers();
+    getEnumStreamers();
+    getEnumStreamersByName();    
+}
+
 int Bitstream::registerMetaObject(const char* className, const QMetaObject* metaObject) {
     getMetaObjects().insert(className, metaObject);
     

libraries/metavoxels/src/Bitstream.h

@@ -290,6 +290,11 @@ public:
         QHash<int, SharedObjectPointer> sharedObjectValues;
     };
 
+    /// Performs all of the various lazily initializations (of object streamers, etc.)  If multiple threads need to use
+    /// Bitstream instances, call this beforehand to prevent errors from occurring when multiple threads attempt lazy
+    /// initialization simultaneously.
+    static void preThreadingInit();
+
     /// Registers a metaobject under its name so that instances of it can be streamed.  Consider using the REGISTER_META_OBJECT
     /// at the top level of the source file associated with the class rather than calling this function directly.
     /// \return zero; the function only returns a value so that it can be used in static initialization

libraries/metavoxels/src/DatagramSequencer.cpp

@@ -79,6 +79,24 @@ ReliableChannel* DatagramSequencer::getReliableInputChannel(int index) {
     return channel;
 }
 
+void DatagramSequencer::addReliableChannelStats(int& sendProgress, int& sendTotal,
+        int& receiveProgress, int& receiveTotal) const {
+    foreach (ReliableChannel* channel, _reliableOutputChannels) {
+        int sent, total;
+        if (channel->getMessageSendProgress(sent, total)) {
+            sendProgress += sent;
+            sendTotal += total;
+        }
+    }
+    foreach (ReliableChannel* channel, _reliableInputChannels) {
+        int received, total;
+        if (channel->getMessageReceiveProgress(received, total)) {
+            receiveProgress += received;
+            receiveTotal += total;
+        }
+    }
+}
+
 int DatagramSequencer::notePacketGroup(int desiredPackets) {
     // figure out how much data we have enqueued and increase the number of packets desired
     int totalAvailable = 0;
@@ -684,6 +702,8 @@ void ReliableChannel::endMessage() {
     
     quint32 length = _buffer.pos() - _messageLengthPlaceholder;
     _buffer.writeBytes(_messageLengthPlaceholder, sizeof(quint32), (const char*)&length);
+    _messageReceivedOffset = getBytesWritten();
+    _messageSize = length;
 }
 
 void ReliableChannel::sendMessage(const QVariant& message) {
@@ -692,6 +712,26 @@ void ReliableChannel::sendMessage(const QVariant& message) {
     endMessage();
 }
 
+bool ReliableChannel::getMessageSendProgress(int& sent, int& total) const {
+    if (!_messagesEnabled || _offset >= _messageReceivedOffset) {
+        return false;
+    }
+    sent = qMax(0, _messageSize - (_messageReceivedOffset - _offset));
+    total = _messageSize;
+    return true;
+}
+
+bool ReliableChannel::getMessageReceiveProgress(int& received, int& total) const {
+    if (!_messagesEnabled || _buffer.bytesAvailable() < (int)sizeof(quint32)) {
+        return false;
+    }
+    quint32 length;
+    _buffer.readBytes(_buffer.pos(), sizeof(quint32), (char*)&length);
+    total = length;
+    received = _buffer.bytesAvailable();
+    return true;
+}
+
 void ReliableChannel::sendClearSharedObjectMessage(int id) {
     ClearSharedObjectMessage message = { id };
     sendMessage(QVariant::fromValue(message));
@@ -717,7 +757,8 @@ ReliableChannel::ReliableChannel(DatagramSequencer* sequencer, int index, bool o
     _offset(0),
     _writePosition(0),
     _writePositionResetPacketNumber(0),
-    _messagesEnabled(true) {
+    _messagesEnabled(true),
+    _messageReceivedOffset(0) {
     
     _buffer.open(output ? QIODevice::WriteOnly : QIODevice::ReadOnly);
     _dataStream.setByteOrder(QDataStream::LittleEndian);

libraries/metavoxels/src/DatagramSequencer.h

@@ -108,6 +108,9 @@ public:
     /// Returns the intput channel at the specified index, creating it if necessary.
     ReliableChannel* getReliableInputChannel(int index = 0);
     
+    /// Adds stats for all reliable channels to the referenced variables.
+    void addReliableChannelStats(int& sendProgress, int& sendTotal, int& receiveProgress, int& receiveTotal) const;
+    
     /// Notes that we're sending a group of packets.
     /// \param desiredPackets the number of packets we'd like to write in the group
     /// \return the number of packets to write in the group
@@ -376,6 +379,14 @@ public:
     /// writes the message to the bitstream, then calls endMessage).
     void sendMessage(const QVariant& message);
 
+    /// Determines the number of bytes uploaded towards the currently pending message.
+    /// \return true if there is a message pending, in which case the sent and total arguments will be set
+    bool getMessageSendProgress(int& sent, int& total) const;
+
+    /// Determines the number of bytes downloaded towards the currently pending message.
+    /// \return true if there is a message pending, in which case the received and total arguments will be set
+    bool getMessageReceiveProgress(int& received, int& total) const;
+
 signals:
 
     /// Fired when a framed message has been received on this channel.
@@ -416,6 +427,8 @@ private:
     SpanList _acknowledged;
     bool _messagesEnabled;
     int _messageLengthPlaceholder;
+    int _messageReceivedOffset;
+    int _messageSize;
 };
 
 #endif // hifi_DatagramSequencer_h

libraries/metavoxels/src/Endpoint.h

@@ -32,6 +32,8 @@ public:
         PacketRecord* baselineReceiveRecord = NULL);
     virtual ~Endpoint();
     
+    const DatagramSequencer& getSequencer() const { return _sequencer; }
+    
     virtual void update();
     
     virtual int parseData(const QByteArray& packet);

libraries/metavoxels/src/MetavoxelData.cpp

@@ -10,6 +10,7 @@
 //
 
 #include <QDateTime>
+#include <QDebugStateSaver>
 #include <QScriptEngine>
 #include <QtDebug>
 
@@ -627,6 +628,33 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l
     return true;
 }
 
+void MetavoxelData::countNodes(int& internal, int& leaves, const MetavoxelLOD& lod) const {
+    glm::vec3 minimum = getMinimum();
+    for (QHash<AttributePointer, MetavoxelNode*>::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) {
+        it.value()->countNodes(it.key(), minimum, _size, lod, internal, leaves);
+    }
+}
+
+void MetavoxelData::dumpStats(QDebug debug) const {
+    QDebugStateSaver saver(debug);
+    debug.nospace() << "[size=" << _size << ", roots=[";
+    int totalInternal = 0, totalLeaves = 0;
+    glm::vec3 minimum = getMinimum();
+    for (QHash<AttributePointer, MetavoxelNode*>::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) {
+        if (it != _roots.constBegin()) {
+            debug << ", ";
+        }
+        debug << it.key()->getName() << " (" << it.key()->metaObject()->className() << "): ";
+        int internal = 0, leaves = 0;
+        it.value()->countNodes(it.key(), minimum, _size, MetavoxelLOD(), internal, leaves);
+        debug << internal << " internal, " << leaves << " leaves, " << (internal + leaves) << " total";
+        totalInternal += internal;
+        totalLeaves += leaves;
+    }
+    debug << "], totalInternal=" << totalInternal << ", totalLeaves=" << totalLeaves <<
+        ", grandTotal=" << (totalInternal + totalLeaves) << "]";
+}
+
 bool MetavoxelData::operator==(const MetavoxelData& other) const {
     return _size == other._size && _roots == other._roots;
 }
@@ -1056,8 +1084,20 @@ void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::ve
     }
     float nextSize = size * 0.5f;
     for (int i = 0; i < CHILD_COUNT; i++) {
-        glm::vec3 nextMinimum = getNextMinimum(minimum, nextSize, i);
+        _children[i]->getSpanners(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, results);
-        _children[i]->getSpanners(attribute, nextMinimum, nextSize, lod, results);
+    }
+}
+
+void MetavoxelNode::countNodes(const AttributePointer& attribute, const glm::vec3& minimum,
+        float size, const MetavoxelLOD& lod, int& internal, int& leaves) const {
+    if (isLeaf() || !lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) {
+        leaves++;
+        return;
+    }
+    internal++;
+    float nextSize = size * 0.5f;
+    for (int i = 0; i < CHILD_COUNT; i++) {
+        _children[i]->countNodes(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, internal, leaves);
     }
 }
 

libraries/metavoxels/src/MetavoxelData.h

@@ -125,6 +125,11 @@ public:
     /// shallow comparison).
     bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const;
 
+    /// Counts the nodes in the data.
+    void countNodes(int& internalNodes, int& leaves, const MetavoxelLOD& lod = MetavoxelLOD()) const;
+
+    void dumpStats(QDebug debug = QDebug(QtDebugMsg)) const;
+
     bool operator==(const MetavoxelData& other) const;
     bool operator!=(const MetavoxelData& other) const;
 
@@ -221,6 +226,9 @@ public:
     void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum,
         float size, const MetavoxelLOD& lod, SharedObjectSet& results) const;
     
+    void countNodes(const AttributePointer& attribute, const glm::vec3& minimum,
+        float size, const MetavoxelLOD& lod, int& internalNodes, int& leaves) const;
+    
 private:
     Q_DISABLE_COPY(MetavoxelNode)
     

libraries/particles/src/Particle.cpp

@@ -357,7 +357,7 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     processedBytes = 0;
 
     // the first part of the data is our octcode...
-    int octets = numberOfThreeBitSectionsInCode(data);
+    int octets = numberOfThreeBitSectionsInCode(data, length);
     int lengthOfOctcode = bytesRequiredForCodeLength(octets);
 
     // we don't actually do anything with this octcode...
@@ -366,6 +366,14 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     
     // id
     uint32_t editID;
+
+    // check to make sure we have enough content to keep reading...
+    if (processedBytes + sizeof(editID) > length) {
+        valid = false;
+        processedBytes = length;
+        return newParticle; // fail as if we read the entire buffer
+    }
+
     memcpy(&editID, dataAt, sizeof(editID));
     dataAt += sizeof(editID);
     processedBytes += sizeof(editID);
@@ -377,6 +385,14 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
         // If this is a NEW_PARTICLE, then we assume that there's an additional uint32_t creatorToken, that
         // we want to send back to the creator as an map to the actual id
         uint32_t creatorTokenID;
+
+        // check to make sure we have enough content to keep reading...
+        if (processedBytes + sizeof(creatorTokenID) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
+
         memcpy(&creatorTokenID, dataAt, sizeof(creatorTokenID));
         dataAt += sizeof(creatorTokenID);
         processedBytes += sizeof(creatorTokenID);
@@ -409,6 +425,12 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     }
     
     // lastEdited
+    // check to make sure we have enough content to keep reading...
+    if (processedBytes + sizeof(newParticle._lastEdited) > length) {
+        valid = false;
+        processedBytes = length;
+        return newParticle; // fail as if we read the entire buffer
+    }
     memcpy(&newParticle._lastEdited, dataAt, sizeof(newParticle._lastEdited));
     dataAt += sizeof(newParticle._lastEdited);
     processedBytes += sizeof(newParticle._lastEdited);
@@ -417,6 +439,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     // properties included bits
     uint16_t packetContainsBits = 0;
     if (!isNewParticle) {
+        if (processedBytes + sizeof(packetContainsBits) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&packetContainsBits, dataAt, sizeof(packetContainsBits));
         dataAt += sizeof(packetContainsBits);
         processedBytes += sizeof(packetContainsBits);
@@ -425,6 +452,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // radius
     if (isNewParticle || ((packetContainsBits & CONTAINS_RADIUS) == CONTAINS_RADIUS)) {
+        if (processedBytes + sizeof(newParticle._radius) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._radius, dataAt, sizeof(newParticle._radius));
         dataAt += sizeof(newParticle._radius);
         processedBytes += sizeof(newParticle._radius);
@@ -432,6 +464,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // position
     if (isNewParticle || ((packetContainsBits & CONTAINS_POSITION) == CONTAINS_POSITION)) {
+        if (processedBytes + sizeof(newParticle._position) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._position, dataAt, sizeof(newParticle._position));
         dataAt += sizeof(newParticle._position);
         processedBytes += sizeof(newParticle._position);
@@ -439,6 +476,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // color
     if (isNewParticle || ((packetContainsBits & CONTAINS_COLOR) == CONTAINS_COLOR)) {
+        if (processedBytes + sizeof(newParticle._color) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(newParticle._color, dataAt, sizeof(newParticle._color));
         dataAt += sizeof(newParticle._color);
         processedBytes += sizeof(newParticle._color);
@@ -446,6 +488,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // velocity
     if (isNewParticle || ((packetContainsBits & CONTAINS_VELOCITY) == CONTAINS_VELOCITY)) {
+        if (processedBytes + sizeof(newParticle._velocity) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._velocity, dataAt, sizeof(newParticle._velocity));
         dataAt += sizeof(newParticle._velocity);
         processedBytes += sizeof(newParticle._velocity);
@@ -453,6 +500,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // gravity
     if (isNewParticle || ((packetContainsBits & CONTAINS_GRAVITY) == CONTAINS_GRAVITY)) {
+        if (processedBytes + sizeof(newParticle._gravity) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._gravity, dataAt, sizeof(newParticle._gravity));
         dataAt += sizeof(newParticle._gravity);
         processedBytes += sizeof(newParticle._gravity);
@@ -460,6 +512,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // damping
     if (isNewParticle || ((packetContainsBits & CONTAINS_DAMPING) == CONTAINS_DAMPING)) {
+        if (processedBytes + sizeof(newParticle._damping) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._damping, dataAt, sizeof(newParticle._damping));
         dataAt += sizeof(newParticle._damping);
         processedBytes += sizeof(newParticle._damping);
@@ -467,6 +524,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // lifetime
     if (isNewParticle || ((packetContainsBits & CONTAINS_LIFETIME) == CONTAINS_LIFETIME)) {
+        if (processedBytes + sizeof(newParticle._lifetime) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._lifetime, dataAt, sizeof(newParticle._lifetime));
         dataAt += sizeof(newParticle._lifetime);
         processedBytes += sizeof(newParticle._lifetime);
@@ -475,6 +537,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     // TODO: make inHand and shouldDie into single bits
     // inHand
     if (isNewParticle || ((packetContainsBits & CONTAINS_INHAND) == CONTAINS_INHAND)) {
+        if (processedBytes + sizeof(newParticle._inHand) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._inHand, dataAt, sizeof(newParticle._inHand));
         dataAt += sizeof(newParticle._inHand);
         processedBytes += sizeof(newParticle._inHand);
@@ -482,6 +549,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // shouldDie
     if (isNewParticle || ((packetContainsBits & CONTAINS_SHOULDDIE) == CONTAINS_SHOULDDIE)) {
+        if (processedBytes + sizeof(newParticle._shouldDie) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._shouldDie, dataAt, sizeof(newParticle._shouldDie));
         dataAt += sizeof(newParticle._shouldDie);
         processedBytes += sizeof(newParticle._shouldDie);
@@ -490,9 +562,20 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     // script
     if (isNewParticle || ((packetContainsBits & CONTAINS_SCRIPT) == CONTAINS_SCRIPT)) {
         uint16_t scriptLength;
+        if (processedBytes + sizeof(scriptLength) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&scriptLength, dataAt, sizeof(scriptLength));
         dataAt += sizeof(scriptLength);
         processedBytes += sizeof(scriptLength);
+
+        if (processedBytes + scriptLength > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         QString tempString((const char*)dataAt);
         newParticle._script = tempString;
         dataAt += scriptLength;
@@ -502,9 +585,20 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
     // modelURL
     if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_URL) == CONTAINS_MODEL_URL)) {
         uint16_t modelURLLength;
+        if (processedBytes + sizeof(modelURLLength) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&modelURLLength, dataAt, sizeof(modelURLLength));
         dataAt += sizeof(modelURLLength);
         processedBytes += sizeof(modelURLLength);
+
+        if (processedBytes + modelURLLength > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         QString tempString((const char*)dataAt);
         newParticle._modelURL = tempString;
         dataAt += modelURLLength;
@@ -513,6 +607,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // modelScale
     if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_SCALE) == CONTAINS_MODEL_SCALE)) {
+        if (processedBytes + sizeof(newParticle._modelScale) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._modelScale, dataAt, sizeof(newParticle._modelScale));
         dataAt += sizeof(newParticle._modelScale);
         processedBytes += sizeof(newParticle._modelScale);
@@ -520,6 +619,11 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // modelTranslation
     if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_TRANSLATION) == CONTAINS_MODEL_TRANSLATION)) {
+        if (processedBytes + sizeof(newParticle._modelTranslation) > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         memcpy(&newParticle._modelTranslation, dataAt, sizeof(newParticle._modelTranslation));
         dataAt += sizeof(newParticle._modelTranslation);
         processedBytes += sizeof(newParticle._modelTranslation);
@@ -527,6 +631,12 @@ Particle Particle::fromEditPacket(const unsigned char* data, int length, int& pr
 
     // modelRotation
     if (isNewParticle || ((packetContainsBits & CONTAINS_MODEL_ROTATION) == CONTAINS_MODEL_ROTATION)) {
+        const int expectedBytesForPackedQuat = sizeof(uint16_t) * 4; // this is how we pack the quats
+        if (processedBytes + expectedBytesForPackedQuat > length) {
+            valid = false;
+            processedBytes = length;
+            return newParticle; // fail as if we read the entire buffer
+        }
         int bytes = unpackOrientationQuatFromBytes(dataAt, newParticle._modelRotation);
         dataAt += bytes;
         processedBytes += bytes;