From efe443e2ff5b930b10bc8c075152af8e7b286b96 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Wed, 9 Jul 2014 14:01:46 -0700
Subject: [PATCH 1/8] Very basic metavoxel save/load.

---
 .../src/metavoxels/MetavoxelServer.cpp        | 59 +++++++++++++++++++
 .../src/metavoxels/MetavoxelServer.h          | 23 ++++++++
 libraries/metavoxels/src/Bitstream.cpp        |  6 ++
 libraries/metavoxels/src/Bitstream.h          |  5 ++
 4 files changed, 93 insertions(+)

diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp
index 14765e2ddc..928a693527 100644
--- a/assignment-client/src/metavoxels/MetavoxelServer.cpp
+++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp
@@ -10,6 +10,9 @@
 //
 
 #include <QDateTime>
+#include <QFile>
+#include <QSaveFile>
+#include <QThread>
 
 #include <PacketHeaders.h>
 
@@ -44,6 +47,18 @@ void MetavoxelServer::run() {
     
     _lastSend = QDateTime::currentMSecsSinceEpoch();
     _sendTimer.start(SEND_INTERVAL);
+    
+    // initialize Bitstream before using it in multiple threads
+    Bitstream::preThreadingInit();
+    
+    // create the persister and start the it in its own thread
+    _persister = new MetavoxelPersister(this);
+    QThread* persistenceThread = new QThread(this);
+    _persister->moveToThread(persistenceThread);
+    persistenceThread->start();
+    
+    // queue up the load
+    QMetaObject::invokeMethod(_persister, "load");
 }
 
 void MetavoxelServer::readPendingDatagrams() {
@@ -67,6 +82,12 @@ void MetavoxelServer::readPendingDatagrams() {
     }
 }
 
+void MetavoxelServer::aboutToFinish() {
+    QMetaObject::invokeMethod(_persister, "save", Q_ARG(const MetavoxelData&, _data));
+    _persister->thread()->quit();
+    _persister->thread()->wait();
+}
+
 void MetavoxelServer::maybeAttachSession(const SharedNodePointer& node) {
     if (node->getType() == NodeType::Agent) {
         QMutexLocker locker(&node->getMutex());
@@ -193,3 +214,41 @@ void MetavoxelSession::sendPacketGroup(int alreadySent) {
         _sequencer.endPacket();
     }
 }
+
+MetavoxelPersister::MetavoxelPersister(MetavoxelServer* server) :
+    _server(server) {
+}
+
+const char* SAVE_FILE = "metavoxels.dat";
+
+void MetavoxelPersister::load() {
+    QFile file(SAVE_FILE);
+    if (!file.exists()) {
+        return;
+    }
+    QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "...";
+    file.open(QIODevice::ReadOnly);
+    QDataStream inStream(&file);
+    Bitstream in(inStream);
+    MetavoxelData data;
+    try {
+        in >> data;
+    } catch (const BitstreamException& e) {
+        debug << "failed, " << e.getDescription();
+        return;
+    }
+    QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data));
+    debug << "done.";
+}
+
+void MetavoxelPersister::save(const MetavoxelData& data) {
+    QDebug debug = qDebug() << "Writing to" << SAVE_FILE << "...";
+    QSaveFile file(SAVE_FILE);
+    file.open(QIODevice::WriteOnly);
+    QDataStream outStream(&file);
+    Bitstream out(outStream);
+    out << data;
+    out.flush();
+    file.commit();
+    debug << "done.";
+}
diff --git a/assignment-client/src/metavoxels/MetavoxelServer.h b/assignment-client/src/metavoxels/MetavoxelServer.h
index 6df769227c..9ed765f65f 100644
--- a/assignment-client/src/metavoxels/MetavoxelServer.h
+++ b/assignment-client/src/metavoxels/MetavoxelServer.h
@@ -20,6 +20,7 @@
 #include <Endpoint.h>
 
 class MetavoxelEditMessage;
+class MetavoxelPersister;
 class MetavoxelSession;
 
 /// Maintains a shared metavoxel system, accepting change requests and broadcasting updates.
@@ -33,11 +34,15 @@ public:
     void applyEdit(const MetavoxelEditMessage& edit);
 
     const MetavoxelData& getData() const { return _data; }
+    
+    Q_INVOKABLE void setData(const MetavoxelData& data) { _data = data; }
 
     virtual void run();
     
     virtual void readPendingDatagrams();
     
+    virtual void aboutToFinish();
+    
 private slots:
 
     void maybeAttachSession(const SharedNodePointer& node);
@@ -45,6 +50,8 @@ private slots:
     
 private:
     
+    MetavoxelPersister* _persister;
+    
     QTimer _sendTimer;
     qint64 _lastSend;
     
@@ -88,4 +95,20 @@ private:
     int _reliableDeltaID;
 };
 
+/// Handles persistence in a separate thread.
+class MetavoxelPersister : public QObject {
+    Q_OBJECT
+
+public:
+    
+    MetavoxelPersister(MetavoxelServer* server);
+
+    Q_INVOKABLE void load();
+    Q_INVOKABLE void save(const MetavoxelData& data);
+
+private:
+    
+    MetavoxelServer* _server;
+};
+
 #endif // hifi_MetavoxelServer_h
diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp
index bc662aa890..0d4f8a52b4 100644
--- a/libraries/metavoxels/src/Bitstream.cpp
+++ b/libraries/metavoxels/src/Bitstream.cpp
@@ -87,6 +87,12 @@ IDStreamer& IDStreamer::operator>>(int& value) {
     return *this;
 }
 
+void Bitstream::preThreadingInit() {
+    getObjectStreamers();
+    getEnumStreamers();
+    getEnumStreamersByName();    
+}
+
 int Bitstream::registerMetaObject(const char* className, const QMetaObject* metaObject) {
     getMetaObjects().insert(className, metaObject);
     
diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h
index 70fde94b79..dacfeb2ee9 100644
--- a/libraries/metavoxels/src/Bitstream.h
+++ b/libraries/metavoxels/src/Bitstream.h
@@ -290,6 +290,11 @@ public:
         QHash<int, SharedObjectPointer> sharedObjectValues;
     };
 
+    /// Performs all of the various lazily initializations (of object streamers, etc.)  If multiple threads need to use
+    /// Bitstream instances, call this beforehand to prevent errors from occurring when multiple threads attempt lazy
+    /// initialization simultaneously.
+    static void preThreadingInit();
+
     /// Registers a metaobject under its name so that instances of it can be streamed.  Consider using the REGISTER_META_OBJECT
     /// at the top level of the source file associated with the class rather than calling this function directly.
     /// \return zero; the function only returns a value so that it can be used in static initialization

From f9036dbaecbff5d7a4eceb018d40e51902febb80 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Wed, 9 Jul 2014 14:22:33 -0700
Subject: [PATCH 2/8] Lock the attribute mapping, since it can be accessed from
 multiple threads.

---
 libraries/metavoxels/src/AttributeRegistry.cpp | 9 +++++++++
 libraries/metavoxels/src/AttributeRegistry.h   | 8 +++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp
index 33ce298859..b43759fa4f 100644
--- a/libraries/metavoxels/src/AttributeRegistry.cpp
+++ b/libraries/metavoxels/src/AttributeRegistry.cpp
@@ -9,7 +9,9 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <QReadLocker>
 #include <QScriptEngine>
+#include <QWriteLocker>
 
 #include "AttributeRegistry.h"
 #include "MetavoxelData.h"
@@ -69,6 +71,7 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute
     if (!attribute) {
         return attribute;
     }
+    QWriteLocker locker(&_attributesLock);
     AttributePointer& pointer = _attributes[attribute->getName()];
     if (!pointer) {
         pointer = attribute;
@@ -77,9 +80,15 @@ AttributePointer AttributeRegistry::registerAttribute(AttributePointer attribute
 }
 
 void AttributeRegistry::deregisterAttribute(const QString& name) {
+    QWriteLocker locker(&_attributesLock);
     _attributes.remove(name);
 }
 
+AttributePointer AttributeRegistry::getAttribute(const QString& name) {
+    QReadLocker locker(&_attributesLock);
+    return _attributes.value(name);
+}
+
 QScriptValue AttributeRegistry::getAttribute(QScriptContext* context, QScriptEngine* engine) {
     return engine->newQObject(getInstance()->getAttribute(context->argument(0).toString()).data(), QScriptEngine::QtOwnership,
         QScriptEngine::PreferExistingWrapperObject);
diff --git a/libraries/metavoxels/src/AttributeRegistry.h b/libraries/metavoxels/src/AttributeRegistry.h
index 7dc2e110b8..23e3c1fa97 100644
--- a/libraries/metavoxels/src/AttributeRegistry.h
+++ b/libraries/metavoxels/src/AttributeRegistry.h
@@ -14,6 +14,7 @@
 
 #include <QHash>
 #include <QObject>
+#include <QReadWriteLock>
 #include <QSharedPointer>
 #include <QString>
 #include <QWidget>
@@ -61,11 +62,14 @@ public:
     void deregisterAttribute(const QString& name);
     
     /// Retrieves an attribute by name.
-    AttributePointer getAttribute(const QString& name) const { return _attributes.value(name); }
+    AttributePointer getAttribute(const QString& name);
     
     /// Returns a reference to the attribute hash.
     const QHash<QString, AttributePointer>& getAttributes() const { return _attributes; }
     
+    /// Returns a reference to the attributes lock.
+    QReadWriteLock& getAttributesLock() { return _attributesLock; }    
+    
     /// Returns a reference to the standard SharedObjectPointer "guide" attribute.
     const AttributePointer& getGuideAttribute() const { return _guideAttribute; }
     
@@ -92,6 +96,8 @@ private:
     static QScriptValue getAttribute(QScriptContext* context, QScriptEngine* engine);
 
     QHash<QString, AttributePointer> _attributes;
+    QReadWriteLock _attributesLock;
+    
     AttributePointer _guideAttribute;
     AttributePointer _spannersAttribute;
     AttributePointer _colorAttribute;

From 3b7945fc14a83c9bf87a2484651be44d66175a50 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Wed, 9 Jul 2014 15:52:23 -0700
Subject: [PATCH 3/8] Gather and print out some stats after loading the data.

---
 .../src/metavoxels/MetavoxelServer.cpp        | 25 ++++++++-------
 libraries/metavoxels/src/MetavoxelData.cpp    | 31 +++++++++++++++++++
 libraries/metavoxels/src/MetavoxelData.h      |  4 +++
 3 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp
index 928a693527..956458404a 100644
--- a/assignment-client/src/metavoxels/MetavoxelServer.cpp
+++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp
@@ -226,19 +226,22 @@ void MetavoxelPersister::load() {
     if (!file.exists()) {
         return;
     }
-    QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "...";
-    file.open(QIODevice::ReadOnly);
-    QDataStream inStream(&file);
-    Bitstream in(inStream);
     MetavoxelData data;
-    try {
-        in >> data;
-    } catch (const BitstreamException& e) {
-        debug << "failed, " << e.getDescription();
-        return;
+    {
+        QDebug debug = qDebug() << "Reading from" << SAVE_FILE << "...";
+        file.open(QIODevice::ReadOnly);
+        QDataStream inStream(&file);
+        Bitstream in(inStream);
+        try {
+            in >> data;
+        } catch (const BitstreamException& e) {
+            debug << "failed, " << e.getDescription();
+            return;
+        }
+        QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data));
+        debug << "done.";
     }
-    QMetaObject::invokeMethod(_server, "setData", Q_ARG(const MetavoxelData&, data));
-    debug << "done.";
+    data.dumpStats();
 }
 
 void MetavoxelPersister::save(const MetavoxelData& data) {
diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp
index 2d61ede796..4b142d0f5e 100644
--- a/libraries/metavoxels/src/MetavoxelData.cpp
+++ b/libraries/metavoxels/src/MetavoxelData.cpp
@@ -10,6 +10,7 @@
 //
 
 #include <QDateTime>
+#include <QDebugStateSaver>
 #include <QScriptEngine>
 #include <QtDebug>
 
@@ -627,6 +628,25 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l
     return true;
 }
 
+void MetavoxelData::dumpStats(QDebug debug) const {
+    QDebugStateSaver saver(debug);
+    debug.nospace() << "[size=" << _size << ", roots=[";
+    int totalInternal = 0, totalLeaves = 0;
+    for (QHash<AttributePointer, MetavoxelNode*>::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) {
+        if (it != _roots.constBegin()) {
+            debug << ", ";
+        }
+        debug << it.key()->getName() << " (" << it.key()->metaObject()->className() << "): ";
+        int internal = 0, leaves = 0;
+        it.value()->countDescendants(internal, leaves);
+        debug << internal << " internal, " << leaves << " leaves, " << (internal + leaves) << " total";
+        totalInternal += internal;
+        totalLeaves += leaves;
+    }
+    debug << "], totalInternal=" << totalInternal << ", totalLeaves=" << totalLeaves <<
+        ", grandTotal=" << (totalInternal + totalLeaves) << "]";
+}
+
 bool MetavoxelData::operator==(const MetavoxelData& other) const {
     return _size == other._size && _roots == other._roots;
 }
@@ -1061,6 +1081,17 @@ void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::ve
     }
 }
 
+void MetavoxelNode::countDescendants(int& internalNodes, int& leaves) const {
+    if (isLeaf()) {
+        leaves++;
+        return;
+    }
+    internalNodes++;
+    for (int i = 0; i < CHILD_COUNT; i++) {
+        _children[i]->countDescendants(internalNodes, leaves);
+    }
+}
+
 int MetavoxelVisitor::encodeOrder(int first, int second, int third, int fourth,
         int fifth, int sixth, int seventh, int eighth) {
     return first | (second << 3) | (third << 6) | (fourth << 9) |
diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h
index 6a7ba33eb5..347d5a1f0b 100644
--- a/libraries/metavoxels/src/MetavoxelData.h
+++ b/libraries/metavoxels/src/MetavoxelData.h
@@ -125,6 +125,8 @@ public:
     /// shallow comparison).
     bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const;
 
+    void dumpStats(QDebug debug = QDebug(QtDebugMsg)) const;
+
     bool operator==(const MetavoxelData& other) const;
     bool operator!=(const MetavoxelData& other) const;
 
@@ -221,6 +223,8 @@ public:
     void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum,
         float size, const MetavoxelLOD& lod, SharedObjectSet& results) const;
     
+    void countDescendants(int& internalNodes, int& leaves) const;
+    
 private:
     Q_DISABLE_COPY(MetavoxelNode)
     

From dbd7ec18b0355fb3c157e0e7c4812830c781e025 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Wed, 9 Jul 2014 17:15:05 -0700
Subject: [PATCH 4/8] Show some stats on the client (node counts under LOD,
 reliable delta download progress).

---
 interface/src/ui/Stats.cpp                    | 36 ++++++++++++++++++-
 .../metavoxels/src/DatagramSequencer.cpp      | 11 ++++++
 libraries/metavoxels/src/DatagramSequencer.h  |  4 +++
 .../metavoxels/src/MetavoxelClientManager.cpp |  4 +++
 .../metavoxels/src/MetavoxelClientManager.h   |  2 ++
 libraries/metavoxels/src/MetavoxelData.cpp    | 23 ++++++++----
 libraries/metavoxels/src/MetavoxelData.h      |  6 +++-
 7 files changed, 77 insertions(+), 9 deletions(-)

diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp
index 3de21b449b..b0cbea14a9 100644
--- a/interface/src/ui/Stats.cpp
+++ b/interface/src/ui/Stats.cpp
@@ -377,7 +377,7 @@ void Stats::display(
     MyAvatar* myAvatar = Application::getInstance()->getAvatar();
     glm::vec3 avatarPos = myAvatar->getPosition();
 
-    lines = _expanded ? 5 : 3;
+    lines = _expanded ? 8 : 3;
 
     drawBackground(backgroundColor, horizontalOffset, 0, _geoStatsWidth, lines * STATS_PELS_PER_LINE + 10);
     horizontalOffset += 5;
@@ -419,6 +419,40 @@ void Stats::display(
         
         verticalOffset += STATS_PELS_PER_LINE;
         drawText(horizontalOffset, verticalOffset, scale, rotation, font, downloads.str().c_str(), color);
+        
+        
+        int internal = 0, leaves = 0;
+        int received = 0, total = 0;
+        foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) {
+            if (node->getType() == NodeType::MetavoxelServer) {
+                QMutexLocker locker(&node->getMutex());
+                MetavoxelClient* client = static_cast<MetavoxelSystemClient*>(node->getLinkedData());
+                if (client) {
+                    client->getData().countNodes(internal, leaves, Application::getInstance()->getMetavoxels()->getLOD());
+                    int clientReceived = 0, clientTotal = 0;
+                    if (client->getReliableDeltaProgress(clientReceived, clientTotal)) {
+                        received += clientReceived;
+                        total += clientTotal;
+                    }
+                }
+            }
+        }
+        stringstream nodes;
+        nodes << "Metavoxels: " << (internal + leaves);
+        verticalOffset += STATS_PELS_PER_LINE;
+        drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodes.str().c_str(), color);
+        
+        stringstream nodeTypes;
+        nodeTypes << "Internal: " << internal << "  Leaves: " << leaves;
+        verticalOffset += STATS_PELS_PER_LINE;
+        drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodeTypes.str().c_str(), color);
+        
+        if (total > 0) {
+            stringstream reliableDelta;
+            reliableDelta << "Reliable Delta: " << (received * 100 / total) << "%";
+            verticalOffset += STATS_PELS_PER_LINE;
+            drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableDelta.str().c_str(), color);
+        }
     }
 
     verticalOffset = 0;
diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp
index 536cfc9dfb..8c58233eb9 100644
--- a/libraries/metavoxels/src/DatagramSequencer.cpp
+++ b/libraries/metavoxels/src/DatagramSequencer.cpp
@@ -692,6 +692,17 @@ void ReliableChannel::sendMessage(const QVariant& message) {
     endMessage();
 }
 
+bool ReliableChannel::getMessageReceiveProgress(int& received, int& total) const {
+    if (!_messagesEnabled || _buffer.bytesAvailable() < (int)sizeof(quint32)) {
+        return false;
+    }
+    quint32 length;
+    _buffer.readBytes(_buffer.pos(), sizeof(quint32), (char*)&length);
+    total = length;
+    received = _buffer.bytesAvailable();
+    return true;
+}
+
 void ReliableChannel::sendClearSharedObjectMessage(int id) {
     ClearSharedObjectMessage message = { id };
     sendMessage(QVariant::fromValue(message));
diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h
index b6dce464f7..998e196a05 100644
--- a/libraries/metavoxels/src/DatagramSequencer.h
+++ b/libraries/metavoxels/src/DatagramSequencer.h
@@ -376,6 +376,10 @@ public:
     /// writes the message to the bitstream, then calls endMessage).
     void sendMessage(const QVariant& message);
 
+    /// Determines the number of bytes downloaded towards the currently pending message.
+    /// \return true if there is a message pending, in which case the received and total arguments will be set
+    bool getMessageReceiveProgress(int& received, int& total) const;
+
 signals:
 
     /// Fired when a framed message has been received on this channel.
diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp
index f3ea1ae8c5..94d9116794 100644
--- a/libraries/metavoxels/src/MetavoxelClientManager.cpp
+++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp
@@ -94,6 +94,10 @@ MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientM
         SIGNAL(receivedMessage(const QVariant&, Bitstream&)), SLOT(handleMessage(const QVariant&, Bitstream&)));
 }
 
+bool MetavoxelClient::getReliableDeltaProgress(int& received, int& total) const {
+    return _reliableDeltaChannel && _reliableDeltaChannel->getMessageReceiveProgress(received, total);
+}
+
 void MetavoxelClient::guide(MetavoxelVisitor& visitor) {
     visitor.setLOD(_manager->getLOD());
     _data.guide(visitor);
diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h
index ad6c86c8fc..809718aa01 100644
--- a/libraries/metavoxels/src/MetavoxelClientManager.h
+++ b/libraries/metavoxels/src/MetavoxelClientManager.h
@@ -53,6 +53,8 @@ public:
 
     MetavoxelData& getData() { return _data; }
 
+    bool getReliableDeltaProgress(int& received, int& total) const;
+
     void guide(MetavoxelVisitor& visitor);
     
     void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false);
diff --git a/libraries/metavoxels/src/MetavoxelData.cpp b/libraries/metavoxels/src/MetavoxelData.cpp
index 4b142d0f5e..1362731a8a 100644
--- a/libraries/metavoxels/src/MetavoxelData.cpp
+++ b/libraries/metavoxels/src/MetavoxelData.cpp
@@ -628,17 +628,25 @@ bool MetavoxelData::deepEquals(const MetavoxelData& other, const MetavoxelLOD& l
     return true;
 }
 
+void MetavoxelData::countNodes(int& internal, int& leaves, const MetavoxelLOD& lod) const {
+    glm::vec3 minimum = getMinimum();
+    for (QHash<AttributePointer, MetavoxelNode*>::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) {
+        it.value()->countNodes(it.key(), minimum, _size, lod, internal, leaves);
+    }
+}
+
 void MetavoxelData::dumpStats(QDebug debug) const {
     QDebugStateSaver saver(debug);
     debug.nospace() << "[size=" << _size << ", roots=[";
     int totalInternal = 0, totalLeaves = 0;
+    glm::vec3 minimum = getMinimum();
     for (QHash<AttributePointer, MetavoxelNode*>::const_iterator it = _roots.constBegin(); it != _roots.constEnd(); it++) {
         if (it != _roots.constBegin()) {
             debug << ", ";
         }
         debug << it.key()->getName() << " (" << it.key()->metaObject()->className() << "): ";
         int internal = 0, leaves = 0;
-        it.value()->countDescendants(internal, leaves);
+        it.value()->countNodes(it.key(), minimum, _size, MetavoxelLOD(), internal, leaves);
         debug << internal << " internal, " << leaves << " leaves, " << (internal + leaves) << " total";
         totalInternal += internal;
         totalLeaves += leaves;
@@ -1076,19 +1084,20 @@ void MetavoxelNode::getSpanners(const AttributePointer& attribute, const glm::ve
     }
     float nextSize = size * 0.5f;
     for (int i = 0; i < CHILD_COUNT; i++) {
-        glm::vec3 nextMinimum = getNextMinimum(minimum, nextSize, i);
-        _children[i]->getSpanners(attribute, nextMinimum, nextSize, lod, results);
+        _children[i]->getSpanners(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, results);
     }
 }
 
-void MetavoxelNode::countDescendants(int& internalNodes, int& leaves) const {
-    if (isLeaf()) {
+void MetavoxelNode::countNodes(const AttributePointer& attribute, const glm::vec3& minimum,
+        float size, const MetavoxelLOD& lod, int& internal, int& leaves) const {
+    if (isLeaf() || !lod.shouldSubdivide(minimum, size, attribute->getLODThresholdMultiplier())) {
         leaves++;
         return;
     }
-    internalNodes++;
+    internal++;
+    float nextSize = size * 0.5f;
     for (int i = 0; i < CHILD_COUNT; i++) {
-        _children[i]->countDescendants(internalNodes, leaves);
+        _children[i]->countNodes(attribute, getNextMinimum(minimum, nextSize, i), nextSize, lod, internal, leaves);
     }
 }
 
diff --git a/libraries/metavoxels/src/MetavoxelData.h b/libraries/metavoxels/src/MetavoxelData.h
index 347d5a1f0b..f558bf8e80 100644
--- a/libraries/metavoxels/src/MetavoxelData.h
+++ b/libraries/metavoxels/src/MetavoxelData.h
@@ -125,6 +125,9 @@ public:
     /// shallow comparison).
     bool deepEquals(const MetavoxelData& other, const MetavoxelLOD& lod = MetavoxelLOD()) const;
 
+    /// Counts the nodes in the data.
+    void countNodes(int& internalNodes, int& leaves, const MetavoxelLOD& lod = MetavoxelLOD()) const;
+
     void dumpStats(QDebug debug = QDebug(QtDebugMsg)) const;
 
     bool operator==(const MetavoxelData& other) const;
@@ -223,7 +226,8 @@ public:
     void getSpanners(const AttributePointer& attribute, const glm::vec3& minimum,
         float size, const MetavoxelLOD& lod, SharedObjectSet& results) const;
     
-    void countDescendants(int& internalNodes, int& leaves) const;
+    void countNodes(const AttributePointer& attribute, const glm::vec3& minimum,
+        float size, const MetavoxelLOD& lod, int& internalNodes, int& leaves) const;
     
 private:
     Q_DISABLE_COPY(MetavoxelNode)

From 817b50c7b8fd9a068a9ef841cb752ff3840f29c7 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Thu, 10 Jul 2014 15:34:45 -0700
Subject: [PATCH 5/8] Show upload/download stats for all reliable messages, not
 just reliable delta downloads.

---
 interface/src/ui/Stats.cpp                    | 22 +++++++------
 .../metavoxels/src/DatagramSequencer.cpp      | 32 ++++++++++++++++++-
 libraries/metavoxels/src/DatagramSequencer.h  |  9 ++++++
 libraries/metavoxels/src/Endpoint.h           |  2 ++
 .../metavoxels/src/MetavoxelClientManager.cpp |  4 ---
 .../metavoxels/src/MetavoxelClientManager.h   |  2 --
 6 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp
index b0cbea14a9..0d2762f923 100644
--- a/interface/src/ui/Stats.cpp
+++ b/interface/src/ui/Stats.cpp
@@ -422,18 +422,15 @@ void Stats::display(
         
         
         int internal = 0, leaves = 0;
-        int received = 0, total = 0;
+        int sendProgress = 0, sendTotal = 0;
+        int receiveProgress = 0, receiveTotal = 0;
         foreach (const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) {
             if (node->getType() == NodeType::MetavoxelServer) {
                 QMutexLocker locker(&node->getMutex());
                 MetavoxelClient* client = static_cast<MetavoxelSystemClient*>(node->getLinkedData());
                 if (client) {
                     client->getData().countNodes(internal, leaves, Application::getInstance()->getMetavoxels()->getLOD());
-                    int clientReceived = 0, clientTotal = 0;
-                    if (client->getReliableDeltaProgress(clientReceived, clientTotal)) {
-                        received += clientReceived;
-                        total += clientTotal;
-                    }
+                    client->getSequencer().addReliableChannelStats(sendProgress, sendTotal, receiveProgress, receiveTotal);
                 }
             }
         }
@@ -447,11 +444,16 @@ void Stats::display(
         verticalOffset += STATS_PELS_PER_LINE;
         drawText(horizontalOffset, verticalOffset, scale, rotation, font, nodeTypes.str().c_str(), color);
         
-        if (total > 0) {
-            stringstream reliableDelta;
-            reliableDelta << "Reliable Delta: " << (received * 100 / total) << "%";
+        if (sendTotal > 0 || receiveTotal > 0) {
+            stringstream reliableStats;
+            if (sendTotal > 0) {
+                reliableStats << "Upload: " << (sendProgress * 100 / sendTotal) << "%  ";
+            }
+            if (receiveTotal > 0) {
+                reliableStats << "Download: " << (receiveProgress * 100 / receiveTotal) << "%";
+            }
             verticalOffset += STATS_PELS_PER_LINE;
-            drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableDelta.str().c_str(), color);
+            drawText(horizontalOffset, verticalOffset, scale, rotation, font, reliableStats.str().c_str(), color);
         }
     }
 
diff --git a/libraries/metavoxels/src/DatagramSequencer.cpp b/libraries/metavoxels/src/DatagramSequencer.cpp
index 8c58233eb9..c757e131bb 100644
--- a/libraries/metavoxels/src/DatagramSequencer.cpp
+++ b/libraries/metavoxels/src/DatagramSequencer.cpp
@@ -79,6 +79,24 @@ ReliableChannel* DatagramSequencer::getReliableInputChannel(int index) {
     return channel;
 }
 
+void DatagramSequencer::addReliableChannelStats(int& sendProgress, int& sendTotal,
+        int& receiveProgress, int& receiveTotal) const {
+    foreach (ReliableChannel* channel, _reliableOutputChannels) {
+        int sent, total;
+        if (channel->getMessageSendProgress(sent, total)) {
+            sendProgress += sent;
+            sendTotal += total;
+        }
+    }
+    foreach (ReliableChannel* channel, _reliableInputChannels) {
+        int received, total;
+        if (channel->getMessageReceiveProgress(received, total)) {
+            receiveProgress += received;
+            receiveTotal += total;
+        }
+    }
+}
+
 int DatagramSequencer::notePacketGroup(int desiredPackets) {
     // figure out how much data we have enqueued and increase the number of packets desired
     int totalAvailable = 0;
@@ -684,6 +702,8 @@ void ReliableChannel::endMessage() {
     
     quint32 length = _buffer.pos() - _messageLengthPlaceholder;
     _buffer.writeBytes(_messageLengthPlaceholder, sizeof(quint32), (const char*)&length);
+    _messageReceivedOffset = getBytesWritten();
+    _messageSize = length;
 }
 
 void ReliableChannel::sendMessage(const QVariant& message) {
@@ -692,6 +712,15 @@ void ReliableChannel::sendMessage(const QVariant& message) {
     endMessage();
 }
 
+bool ReliableChannel::getMessageSendProgress(int& sent, int& total) const {
+    if (!_messagesEnabled || _offset >= _messageReceivedOffset) {
+        return false;
+    }
+    sent = qMax(0, _messageSize - (_messageReceivedOffset - _offset));
+    total = _messageSize;
+    return true;
+}
+
 bool ReliableChannel::getMessageReceiveProgress(int& received, int& total) const {
     if (!_messagesEnabled || _buffer.bytesAvailable() < (int)sizeof(quint32)) {
         return false;
@@ -728,7 +757,8 @@ ReliableChannel::ReliableChannel(DatagramSequencer* sequencer, int index, bool o
     _offset(0),
     _writePosition(0),
     _writePositionResetPacketNumber(0),
-    _messagesEnabled(true) {
+    _messagesEnabled(true),
+    _messageReceivedOffset(0) {
     
     _buffer.open(output ? QIODevice::WriteOnly : QIODevice::ReadOnly);
     _dataStream.setByteOrder(QDataStream::LittleEndian);
diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h
index 998e196a05..4a01679c68 100644
--- a/libraries/metavoxels/src/DatagramSequencer.h
+++ b/libraries/metavoxels/src/DatagramSequencer.h
@@ -108,6 +108,9 @@ public:
     /// Returns the intput channel at the specified index, creating it if necessary.
     ReliableChannel* getReliableInputChannel(int index = 0);
     
+    /// Adds stats for all reliable channels to the referenced variables.
+    void addReliableChannelStats(int& sendProgress, int& sendTotal, int& receiveProgress, int& receiveTotal) const;
+    
     /// Notes that we're sending a group of packets.
     /// \param desiredPackets the number of packets we'd like to write in the group
     /// \return the number of packets to write in the group
@@ -376,6 +379,10 @@ public:
     /// writes the message to the bitstream, then calls endMessage).
     void sendMessage(const QVariant& message);
 
+    /// Determines the number of bytes uploaded towards the currently pending message.
+    /// \return true if there is a message pending, in which case the sent and total arguments will be set
+    bool getMessageSendProgress(int& sent, int& total) const;
+
     /// Determines the number of bytes downloaded towards the currently pending message.
     /// \return true if there is a message pending, in which case the received and total arguments will be set
     bool getMessageReceiveProgress(int& received, int& total) const;
@@ -420,6 +427,8 @@ private:
     SpanList _acknowledged;
     bool _messagesEnabled;
     int _messageLengthPlaceholder;
+    int _messageReceivedOffset;
+    int _messageSize;
 };
 
 #endif // hifi_DatagramSequencer_h
diff --git a/libraries/metavoxels/src/Endpoint.h b/libraries/metavoxels/src/Endpoint.h
index 3c681a7b98..d6999196d8 100644
--- a/libraries/metavoxels/src/Endpoint.h
+++ b/libraries/metavoxels/src/Endpoint.h
@@ -32,6 +32,8 @@ public:
         PacketRecord* baselineReceiveRecord = NULL);
     virtual ~Endpoint();
     
+    const DatagramSequencer& getSequencer() const { return _sequencer; }
+    
     virtual void update();
     
     virtual int parseData(const QByteArray& packet);
diff --git a/libraries/metavoxels/src/MetavoxelClientManager.cpp b/libraries/metavoxels/src/MetavoxelClientManager.cpp
index 94d9116794..f3ea1ae8c5 100644
--- a/libraries/metavoxels/src/MetavoxelClientManager.cpp
+++ b/libraries/metavoxels/src/MetavoxelClientManager.cpp
@@ -94,10 +94,6 @@ MetavoxelClient::MetavoxelClient(const SharedNodePointer& node, MetavoxelClientM
         SIGNAL(receivedMessage(const QVariant&, Bitstream&)), SLOT(handleMessage(const QVariant&, Bitstream&)));
 }
 
-bool MetavoxelClient::getReliableDeltaProgress(int& received, int& total) const {
-    return _reliableDeltaChannel && _reliableDeltaChannel->getMessageReceiveProgress(received, total);
-}
-
 void MetavoxelClient::guide(MetavoxelVisitor& visitor) {
     visitor.setLOD(_manager->getLOD());
     _data.guide(visitor);
diff --git a/libraries/metavoxels/src/MetavoxelClientManager.h b/libraries/metavoxels/src/MetavoxelClientManager.h
index 809718aa01..ad6c86c8fc 100644
--- a/libraries/metavoxels/src/MetavoxelClientManager.h
+++ b/libraries/metavoxels/src/MetavoxelClientManager.h
@@ -53,8 +53,6 @@ public:
 
     MetavoxelData& getData() { return _data; }
 
-    bool getReliableDeltaProgress(int& received, int& total) const;
-
     void guide(MetavoxelVisitor& visitor);
     
     void applyEdit(const MetavoxelEditMessage& edit, bool reliable = false);

From d858e20af25b2a72d595c2b23c2eaddb7a826fbf Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Thu, 10 Jul 2014 16:02:53 -0700
Subject: [PATCH 6/8] For now, tie the metavoxel LOD threshold to the avatar
 LOD parameter.

---
 interface/src/MetavoxelSystem.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/interface/src/MetavoxelSystem.cpp b/interface/src/MetavoxelSystem.cpp
index 7a5119a62d..66933643ae 100644
--- a/interface/src/MetavoxelSystem.cpp
+++ b/interface/src/MetavoxelSystem.cpp
@@ -48,8 +48,10 @@ void MetavoxelSystem::init() {
 }
 
 MetavoxelLOD MetavoxelSystem::getLOD() const {
-    const float FIXED_LOD_THRESHOLD = 0.01f;
-    return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(), FIXED_LOD_THRESHOLD);
+    // the LOD threshold is temporarily tied to the avatar LOD parameter
+    const float BASE_LOD_THRESHOLD = 0.01f;
+    return MetavoxelLOD(Application::getInstance()->getCamera()->getPosition(),
+        BASE_LOD_THRESHOLD * Menu::getInstance()->getAvatarLODDistanceMultiplier());
 }
 
 void MetavoxelSystem::simulate(float deltaTime) {

From d121c8e07af33811b5ac636a00a2b4635c38c5de Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Thu, 10 Jul 2014 17:21:05 -0700
Subject: [PATCH 7/8] Fix for starting with empty spanners (due to LOD), then
 adding.

---
 libraries/metavoxels/src/AttributeRegistry.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libraries/metavoxels/src/AttributeRegistry.cpp b/libraries/metavoxels/src/AttributeRegistry.cpp
index b43759fa4f..64b2646261 100644
--- a/libraries/metavoxels/src/AttributeRegistry.cpp
+++ b/libraries/metavoxels/src/AttributeRegistry.cpp
@@ -568,6 +568,10 @@ void SpannerSetAttribute::readMetavoxelRoot(MetavoxelData& data, MetavoxelStream
         }
         data.insert(state.attribute, object);
     }
+    // even if the root is empty, it should still exist
+    if (!data.getRoot(state.attribute)) {
+        data.createRoot(state.attribute);
+    }
 }
 
 void SpannerSetAttribute::writeMetavoxelRoot(const MetavoxelNode& root, MetavoxelStreamState& state) {
@@ -586,6 +590,10 @@ void SpannerSetAttribute::readMetavoxelDelta(MetavoxelData& data,
         }
         data.toggle(state.attribute, object);
     }
+    // even if the root is empty, it should still exist
+    if (!data.getRoot(state.attribute)) {
+        data.createRoot(state.attribute);
+    }
 }
 
 void SpannerSetAttribute::writeMetavoxelDelta(const MetavoxelNode& root,

From af3af15084ea77f3dfe90509220b2a3d3ce75c58 Mon Sep 17 00:00:00 2001
From: Andrzej Kapolka <drzej.k@gmail.com>
Date: Thu, 10 Jul 2014 18:11:51 -0700
Subject: [PATCH 8/8] Comment, formatting fix.

---
 assignment-client/src/metavoxels/MetavoxelServer.cpp | 2 +-
 interface/src/ui/Stats.cpp                           | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/assignment-client/src/metavoxels/MetavoxelServer.cpp b/assignment-client/src/metavoxels/MetavoxelServer.cpp
index 956458404a..4d9c45ed6c 100644
--- a/assignment-client/src/metavoxels/MetavoxelServer.cpp
+++ b/assignment-client/src/metavoxels/MetavoxelServer.cpp
@@ -51,7 +51,7 @@ void MetavoxelServer::run() {
     // initialize Bitstream before using it in multiple threads
     Bitstream::preThreadingInit();
     
-    // create the persister and start the it in its own thread
+    // create the persister and start it in its own thread
     _persister = new MetavoxelPersister(this);
     QThread* persistenceThread = new QThread(this);
     _persister->moveToThread(persistenceThread);
diff --git a/interface/src/ui/Stats.cpp b/interface/src/ui/Stats.cpp
index 0d2762f923..eeff76395b 100644
--- a/interface/src/ui/Stats.cpp
+++ b/interface/src/ui/Stats.cpp
@@ -420,7 +420,6 @@ void Stats::display(
         verticalOffset += STATS_PELS_PER_LINE;
         drawText(horizontalOffset, verticalOffset, scale, rotation, font, downloads.str().c_str(), color);
         
-        
         int internal = 0, leaves = 0;
         int sendProgress = 0, sendTotal = 0;
         int receiveProgress = 0, receiveTotal = 0;