Merge pull request #880 from ZappoMan/threaded_voxel_server

Threaded voxel server- Fix "freezing" on hq

libraries/voxels/src/VoxelTree.cpp

@@ -52,6 +52,10 @@ VoxelTree::VoxelTree(bool shouldReaverage) :
     _shouldReaverage(shouldReaverage),
     _stopImport(false) {
     rootNode = new VoxelNode();
+    
+    pthread_mutex_init(&_encodeSetLock, NULL);
+    pthread_mutex_init(&_deleteSetLock, NULL);
+    pthread_mutex_init(&_deletePendingSetLock, NULL);
 }
 
 VoxelTree::~VoxelTree() {
@@ -60,6 +64,10 @@ VoxelTree::~VoxelTree() {
     for (int i = 0; i < NUMBER_OF_CHILDREN; i++) {
         delete rootNode->getChildAtIndex(i);
     }
+
+    pthread_mutex_destroy(&_encodeSetLock);
+    pthread_mutex_destroy(&_deleteSetLock);
+    pthread_mutex_destroy(&_deletePendingSetLock);
 }
 
 
@@ -397,7 +405,16 @@ void VoxelTree::deleteVoxelCodeFromTree(unsigned char* codeBuffer, bool collapse
     args.pathChanged        = false;
 
     VoxelNode* node = rootNode;
-    deleteVoxelCodeFromTreeRecursion(node, &args);
+    
+    // We can't encode and delete nodes at the same time, so we guard against deleting any node that is actively
+    // being encoded. And we stick that code on our pendingDelete list.
+    if (isEncoding(codeBuffer)) {
+        queueForLaterDelete(codeBuffer);
+    } else {
+        startDeleting(codeBuffer);
+        deleteVoxelCodeFromTreeRecursion(node, &args);
+        doneDeleting(codeBuffer);
+    }
 }
 
 void VoxelTree::deleteVoxelCodeFromTreeRecursion(VoxelNode* node, void* extraData) {
@@ -1002,15 +1019,16 @@ bool VoxelTree::findCapsulePenetration(const glm::vec3& start, const glm::vec3&
     return args.found;
 }
 
-
 int VoxelTree::encodeTreeBitstream(VoxelNode* node, unsigned char* outputBuffer, int availableBytes, VoxelNodeBag& bag,
-                                   EncodeBitstreamParams& params) const {
+                                   EncodeBitstreamParams& params) {
 
+    startEncoding(node);
     // How many bytes have we written so far at this level;
     int bytesWritten = 0;
     
     // If we're at a node that is out of view, then we can return, because no nodes below us will be in view!
     if (params.viewFrustum && !node->isInView(*params.viewFrustum)) {
+        doneEncoding(node);
         return bytesWritten;
     }
     
@@ -1061,6 +1079,8 @@ int VoxelTree::encodeTreeBitstream(VoxelNode* node, unsigned char* outputBuffer,
         // otherwise... if we didn't write any child bytes, then pretend like we also didn't write our octal code
         bytesWritten = 0;
     }
+    
+    doneEncoding(node);
     return bytesWritten;
 }
 
@@ -1743,7 +1763,7 @@ bool VoxelTree::readFromSchematicFile(const char *fileName) {
     return true;
 }
 
-void VoxelTree::writeToSVOFile(const char* fileName, VoxelNode* node) const {
+void VoxelTree::writeToSVOFile(const char* fileName, VoxelNode* node) {
 
     std::ofstream file(fileName, std::ios::out|std::ios::binary);
 
@@ -1829,6 +1849,65 @@ void VoxelTree::copyFromTreeIntoSubTree(VoxelTree* sourceTree, VoxelNode* destin
     }
 }
 
+void dumpSetContents(const char* name, std::set<unsigned char*> set) {
+    printf("set %s has %ld elements\n", name, set.size());
+    /*
+    for (std::set<unsigned char*>::iterator i = set.begin(); i != set.end(); ++i) {
+        printOctalCode(*i);
+    }
+    */
+}
+
+void VoxelTree::startEncoding(VoxelNode* node) {
+    pthread_mutex_lock(&_encodeSetLock);
+    _codesBeingEncoded.insert(node->getOctalCode());
+    pthread_mutex_unlock(&_encodeSetLock);
+}
+
+void VoxelTree::doneEncoding(VoxelNode* node) {
+    pthread_mutex_lock(&_encodeSetLock);
+    _codesBeingEncoded.erase(node->getOctalCode());
+    pthread_mutex_unlock(&_encodeSetLock);
+    
+    // if we have any pending delete codes, then delete them now.
+    emptyDeleteQueue();
+}
+
+void VoxelTree::startDeleting(unsigned char* code) {
+    pthread_mutex_lock(&_deleteSetLock);
+    _codesBeingDeleted.insert(code);
+    pthread_mutex_unlock(&_deleteSetLock);
+}
+
+void VoxelTree::doneDeleting(unsigned char* code) {
+    pthread_mutex_lock(&_deleteSetLock);
+    _codesBeingDeleted.erase(code);
+    pthread_mutex_unlock(&_deleteSetLock);
+}
+
+bool VoxelTree::isEncoding(unsigned char* codeBuffer) {
+    pthread_mutex_lock(&_encodeSetLock);
+    bool isEncoding = (_codesBeingEncoded.find(codeBuffer) != _codesBeingEncoded.end());
+    pthread_mutex_unlock(&_encodeSetLock);
+    return isEncoding;
+}
+
+void VoxelTree::queueForLaterDelete(unsigned char* codeBuffer) {
+    pthread_mutex_lock(&_deletePendingSetLock);
+    _codesPendingDelete.insert(codeBuffer);
+    pthread_mutex_unlock(&_deletePendingSetLock);
+}
+
+void VoxelTree::emptyDeleteQueue() {
+    pthread_mutex_lock(&_deletePendingSetLock);
+    for (std::set<unsigned char*>::iterator i = _codesPendingDelete.begin(); i != _codesPendingDelete.end(); ++i) {
+        unsigned char* codeToDelete = *i;
+        _codesBeingDeleted.erase(codeToDelete);
+        deleteVoxelCodeFromTree(codeToDelete, COLLAPSE_EMPTY_TREE);
+    }
+    pthread_mutex_unlock(&_deletePendingSetLock);
+}
+
 void VoxelTree::cancelImport() {
     _stopImport = true;
 }

libraries/voxels/src/VoxelTree.h

@@ -9,6 +9,7 @@
 #ifndef __hifi__VoxelTree__
 #define __hifi__VoxelTree__
 
+#include <set>
 #include <PointerStack.h>
 #include <SimpleMovingAverage.h>
 
@@ -155,7 +156,7 @@ public:
                                                 const glm::vec3& point, void* extraData=NULL);
 
     int encodeTreeBitstream(VoxelNode* node, unsigned char* outputBuffer, int availableBytes, VoxelNodeBag& bag, 
-                            EncodeBitstreamParams& params) const;
+                            EncodeBitstreamParams& params) ;
 
     bool isDirty() const { return _isDirty; };
     void clearDirtyBit() { _isDirty = false; };
@@ -172,7 +173,7 @@ public:
     void loadVoxelsFile(const char* fileName, bool wantColorRandomizer);
 
     // these will read/write files that match the wireformat, excluding the 'V' leading
-    void writeToSVOFile(const char* filename, VoxelNode* node = NULL) const;
+    void writeToSVOFile(const char* filename, VoxelNode* node = NULL);
     bool readFromSVOFile(const char* filename);
     // reads voxels from square image with alpha as a Y-axis
     bool readFromSquareARGB32Pixels(const char *filename);
@@ -219,6 +220,40 @@ private:
     unsigned long int _nodesChangedFromBitstream;
     bool _shouldReaverage;
     bool _stopImport;
+
+    /// Octal Codes of any subtrees currently being encoded. While any of these codes is being encoded, ancestors and 
+    /// descendants of them can not be deleted.
+    std::set<unsigned char*>  _codesBeingEncoded;
+    /// mutex lock to protect the encoding set
+    pthread_mutex_t _encodeSetLock;
+
+    /// Called to indicate that a VoxelNode is in the process of being encoded.
+    void startEncoding(VoxelNode* node);
+    /// Called to indicate that a VoxelNode is done being encoded.
+    void doneEncoding(VoxelNode* node);
+    /// Is the Octal Code currently being deleted?
+    bool isEncoding(unsigned char* codeBuffer);
+
+    /// Octal Codes of any subtrees currently being deleted. While any of these codes is being deleted, ancestors and 
+    /// descendants of them can not be encoded.
+    std::set<unsigned char*>  _codesBeingDeleted;
+    /// mutex lock to protect the deleting set
+    pthread_mutex_t _deleteSetLock;
+
+    /// Called to indicate that an octal code is in the process of being deleted.
+    void startDeleting(unsigned char* code);
+    /// Called to indicate that an octal code is done being deleted.
+    void doneDeleting(unsigned char* code);
+    /// Octal Codes that were attempted to be deleted but couldn't be because they were actively being encoded, and were
+    /// instead queued for later delete
+    std::set<unsigned char*>  _codesPendingDelete;
+    /// mutex lock to protect the deleting set
+    pthread_mutex_t _deletePendingSetLock;
+
+    /// Adds an Octal Code to the set of codes that needs to be deleted
+    void queueForLaterDelete(unsigned char* codeBuffer);
+    /// flushes out any Octal Codes that had to be queued
+    void emptyDeleteQueue();
 };
 
 float boundaryDistanceForRenderLevel(unsigned int renderLevel);

voxel-server/src/VoxelPersistThread.cpp

@@ -0,0 +1,37 @@
+//
+//  VoxelPersistThread.cpp
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/21/13
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded voxel persistance
+//
+
+#include <NodeList.h>
+#include <SharedUtil.h>
+
+#include "VoxelPersistThread.h"
+#include "VoxelServer.h"
+
+VoxelPersistThread::VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval) :
+    _tree(tree),
+    _filename(filename),
+    _persistInterval(persistInterval) {
+}
+
+bool VoxelPersistThread::process() {
+    uint64_t MSECS_TO_USECS = 1000;
+    usleep(_persistInterval * MSECS_TO_USECS);
+
+
+    // check the dirty bit and persist here...
+    if (_tree->isDirty()) {
+        printf("saving voxels to file %s...\n",_filename);
+        _tree->writeToSVOFile(_filename);
+        _tree->clearDirtyBit(); // tree is clean after saving
+        printf("DONE saving voxels to file...\n");
+    }
+
+    return isStillRunning();  // keep running till they terminate us
+}

voxel-server/src/VoxelPersistThread.h

@@ -0,0 +1,33 @@
+//
+//  VoxelPersistThread.h
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/21/13
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded voxel persistance
+//
+
+#ifndef __voxel_server__VoxelPersistThread__
+#define __voxel_server__VoxelPersistThread__
+
+#include <GenericThread.h>
+#include <NetworkPacket.h>
+#include <VoxelTree.h>
+
+/// Generalized threaded processor for handling received inbound packets. 
+class VoxelPersistThread : public virtual GenericThread {
+public:
+    static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+
+    VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval = DEFAULT_PERSIST_INTERVAL);
+protected:
+    /// Implements generic processing behavior for this thread.
+    virtual bool process();
+private:
+    VoxelTree* _tree;
+    const char* _filename;
+    int _persistInterval;
+};
+
+#endif // __voxel_server__VoxelPersistThread__

voxel-server/src/VoxelServer.h

@@ -0,0 +1,58 @@
+//  VoxelServer.h
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/21/13
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//
+
+#ifndef __voxel_server__VoxelServer__
+#define __voxel_server__VoxelServer__
+
+#include <SharedUtil.h>
+#include <NodeList.h> // for MAX_PACKET_SIZE
+#include <EnvironmentData.h>
+#include <JurisdictionSender.h>
+#include <VoxelTree.h>
+
+#include "VoxelServerPacketProcessor.h"
+
+
+const int MAX_FILENAME_LENGTH = 1024;
+const int VOXEL_LISTEN_PORT = 40106;
+const int VOXEL_SIZE_BYTES = 3 + (3 * sizeof(float));
+const int VOXELS_PER_PACKET = (MAX_PACKET_SIZE - 1) / VOXEL_SIZE_BYTES;
+const int MIN_BRIGHTNESS = 64;
+const float DEATH_STAR_RADIUS = 4.0;
+const float MAX_CUBE = 0.05f;
+const int VOXEL_SEND_INTERVAL_USECS = 17 * 1000; // approximately 60fps
+const int SENDING_TIME_TO_SPARE = 5 * 1000; // usec of sending interval to spare for calculating voxels
+const int INTERVALS_PER_SECOND = 1000 * 1000 / VOXEL_SEND_INTERVAL_USECS;
+const int MAX_VOXEL_TREE_DEPTH_LEVELS = 4;
+const int ENVIRONMENT_SEND_INTERVAL_USECS = 1000000;
+
+extern const char* LOCAL_VOXELS_PERSIST_FILE;
+extern const char* VOXELS_PERSIST_FILE;
+extern char voxelPersistFilename[MAX_FILENAME_LENGTH];
+extern int PACKETS_PER_CLIENT_PER_INTERVAL;
+
+extern VoxelTree serverTree; // this IS a reaveraging tree 
+extern bool wantVoxelPersist;
+extern bool wantLocalDomain;
+extern bool debugVoxelSending;
+extern bool shouldShowAnimationDebug;
+extern bool displayVoxelStats;
+extern bool debugVoxelReceiving;
+extern bool sendEnvironments;
+extern bool sendMinimalEnvironment;
+extern bool dumpVoxelsOnMove;
+extern EnvironmentData environmentData[3];
+extern int receivedPacketCount;
+extern JurisdictionMap* jurisdiction;
+extern JurisdictionSender* jurisdictionSender;
+extern VoxelServerPacketProcessor* voxelServerPacketProcessor;
+extern pthread_mutex_t treeLock;
+
+
+
+#endif // __voxel_server__VoxelServer__

voxel-server/src/VoxelServerPacketProcessor.cpp

@@ -0,0 +1,117 @@
+//
+//  VoxelServerPacketProcessor.cpp
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/21/13
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded network packet processor for the voxel-server
+//
+
+#include <PacketHeaders.h>
+#include <PerfStat.h>
+
+#include "VoxelServer.h"
+#include "VoxelServerPacketProcessor.h"
+
+
+void VoxelServerPacketProcessor::processPacket(sockaddr& senderAddress, unsigned char* packetData, ssize_t packetLength) {
+
+    int numBytesPacketHeader = numBytesForPacketHeader(packetData);
+    
+    if (packetData[0] == PACKET_TYPE_SET_VOXEL || packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE) {
+        bool destructive = (packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
+        PerformanceWarning warn(::shouldShowAnimationDebug,
+                                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                                ::shouldShowAnimationDebug);
+        
+        ::receivedPacketCount++;
+        
+        unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
+        if (::shouldShowAnimationDebug) {
+            printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
+                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                packetLength, itemNumber);
+        }
+        
+        if (::debugVoxelReceiving) {
+            printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
+                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                ::receivedPacketCount, packetLength, itemNumber);
+        }
+        int atByte = numBytesPacketHeader + sizeof(itemNumber);
+        unsigned char* voxelData = (unsigned char*)&packetData[atByte];
+        while (atByte < packetLength) {
+            unsigned char octets = (unsigned char)*voxelData;
+            const int COLOR_SIZE_IN_BYTES = 3;
+            int voxelDataSize = bytesRequiredForCodeLength(octets) + COLOR_SIZE_IN_BYTES;
+            int voxelCodeSize = bytesRequiredForCodeLength(octets);
+
+            if (::shouldShowAnimationDebug) {
+                int red   = voxelData[voxelCodeSize + 0];
+                int green = voxelData[voxelCodeSize + 1];
+                int blue  = voxelData[voxelCodeSize + 2];
+
+                float* vertices = firstVertexForCode(voxelData);
+                printf("inserting voxel: %f,%f,%f r=%d,g=%d,b=%d\n", vertices[0], vertices[1], vertices[2], red, green, blue);
+                delete[] vertices;
+            }
+        
+            serverTree.readCodeColorBufferToTree(voxelData, destructive);
+            // skip to next
+            voxelData += voxelDataSize;
+            atByte += voxelDataSize;
+        }
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+
+    } else if (packetData[0] == PACKET_TYPE_ERASE_VOXEL) {
+
+        // Send these bits off to the VoxelTree class to process them
+        pthread_mutex_lock(&::treeLock);
+        ::serverTree.processRemoveVoxelBitstream((unsigned char*)packetData, packetLength);
+        pthread_mutex_unlock(&::treeLock);
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+    } else if (packetData[0] == PACKET_TYPE_Z_COMMAND) {
+
+        // the Z command is a special command that allows the sender to send the voxel server high level semantic
+        // requests, like erase all, or add sphere scene
+        
+        char* command = (char*) &packetData[numBytesPacketHeader]; // start of the command
+        int commandLength = strlen(command); // commands are null terminated strings
+        int totalLength = numBytesPacketHeader + commandLength + 1; // 1 for null termination
+        printf("got Z message len(%ld)= %s\n", packetLength, command);
+        bool rebroadcast = true; // by default rebroadcast
+
+        while (totalLength <= packetLength) {
+            if (strcmp(command, TEST_COMMAND) == 0) {
+                printf("got Z message == a message, nothing to do, just report\n");
+            }
+            totalLength += commandLength + 1; // 1 for null termination
+        }
+
+        if (rebroadcast) {
+            // Now send this to the connected nodes so they can also process these messages
+            printf("rebroadcasting Z message to connected nodes... nodeList.broadcastToNodes()\n");
+            NodeList::getInstance()->broadcastToNodes(packetData, packetLength, &NODE_TYPE_AGENT, 1);
+        }
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+    } else {
+        printf("unknown packet ignored... packetData[0]=%c\n", packetData[0]);
+    }
+}
+

voxel-server/src/VoxelServerPacketProcessor.h

@@ -0,0 +1,22 @@
+//
+//  VoxelServerPacketProcessor.h
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/21/13
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded network packet processor for the voxel-server
+//
+
+#ifndef __voxel_server__VoxelServerPacketProcessor__
+#define __voxel_server__VoxelServerPacketProcessor__
+
+#include <ReceivedPacketProcessor.h>
+
+/// Handles processing of incoming network packets for the voxel-server. As with other ReceivedPacketProcessor classes 
+/// the user is responsible for reading inbound packets and adding them to the processing queue by calling queueReceivedPacket()
+class VoxelServerPacketProcessor : public ReceivedPacketProcessor {
+protected:
+    virtual void processPacket(sockaddr& senderAddress, unsigned char*  packetData, ssize_t packetLength);
+};
+#endif // __voxel_server__VoxelServerPacketProcessor__

voxel-server/src/main.cpp

@@ -23,6 +23,9 @@
 
 #include <JurisdictionSender.h>
 
+#include "VoxelPersistThread.h"
+#include "VoxelServerPacketProcessor.h"
+
 #ifdef _WIN32
 #include "Syssocket.h"
 #include "Systime.h"
@@ -32,37 +35,15 @@
 #include <ifaddrs.h>
 #endif
 
+#include "VoxelServer.h"
+
 const char* LOCAL_VOXELS_PERSIST_FILE = "resources/voxels.svo";
 const char* VOXELS_PERSIST_FILE = "/etc/highfidelity/voxel-server/resources/voxels.svo";
-const int MAX_FILENAME_LENGTH = 1024;
 char voxelPersistFilename[MAX_FILENAME_LENGTH];
-const int VOXEL_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
-
-const int VOXEL_LISTEN_PORT = 40106;
-
-
-const int VOXEL_SIZE_BYTES = 3 + (3 * sizeof(float));
-const int VOXELS_PER_PACKET = (MAX_PACKET_SIZE - 1) / VOXEL_SIZE_BYTES;
-
-const int MIN_BRIGHTNESS = 64;
-const float DEATH_STAR_RADIUS = 4.0;
-const float MAX_CUBE = 0.05f;
-
-const int VOXEL_SEND_INTERVAL_USECS = 17 * 1000; // approximately 60fps
 int PACKETS_PER_CLIENT_PER_INTERVAL = 10;
-const int SENDING_TIME_TO_SPARE = 5 * 1000; // usec of sending interval to spare for calculating voxels
-const int INTERVALS_PER_SECOND = 1000 * 1000 / VOXEL_SEND_INTERVAL_USECS;
-
-const int MAX_VOXEL_TREE_DEPTH_LEVELS = 4;
-
-const int ENVIRONMENT_SEND_INTERVAL_USECS = 1000000;
-
 VoxelTree serverTree(true); // this IS a reaveraging tree 
 bool wantVoxelPersist = true;
 bool wantLocalDomain = false;
-
-
-bool wantColorRandomizer = false;
 bool debugVoxelSending = false;
 bool shouldShowAnimationDebug = false;
 bool displayVoxelStats = false;
@@ -70,60 +51,15 @@ bool debugVoxelReceiving = false;
 bool sendEnvironments = true;
 bool sendMinimalEnvironment = false;
 bool dumpVoxelsOnMove = false;
-
 EnvironmentData environmentData[3];
-
 int receivedPacketCount = 0;
 JurisdictionMap* jurisdiction = NULL;
 JurisdictionSender* jurisdictionSender = NULL;
-
-void randomlyFillVoxelTree(int levelsToGo, VoxelNode *currentRootNode) {
-    // randomly generate children for this node
-    // the first level of the tree (where levelsToGo = MAX_VOXEL_TREE_DEPTH_LEVELS) has all 8
-    if (levelsToGo > 0) {
-
-        bool createdChildren = false;
-        createdChildren = false;
-        
-        for (int i = 0; i < 8; i++) {
-            if (true) {
-                // create a new VoxelNode to put here
-                currentRootNode->addChildAtIndex(i);
-                randomlyFillVoxelTree(levelsToGo - 1, currentRootNode->getChildAtIndex(i));
-                createdChildren = true;
-            }
-        }
-        
-        if (!createdChildren) {
-            // we didn't create any children for this node, making it a leaf
-            // give it a random color
-            currentRootNode->setRandomColor(MIN_BRIGHTNESS);
-        } else {
-            // set the color value for this node
-            currentRootNode->setColorFromAverageOfChildren();
-        }
-    } else {
-        // this is a leaf node, just give it a color
-        currentRootNode->setRandomColor(MIN_BRIGHTNESS);
-    }
-}
-
-void eraseVoxelTreeAndCleanupNodeVisitData() {
-
-    // As our tree to erase all it's voxels
-    ::serverTree.eraseAllVoxels();
-    // enumerate the nodes clean up their marker nodes
-    for (NodeList::iterator node = NodeList::getInstance()->begin(); node != NodeList::getInstance()->end(); node++) {
-        VoxelNodeData* nodeData = (VoxelNodeData*) node->getLinkedData();
-        if (nodeData) {
-            // clean up the node visit data
-            nodeData->nodeBag.deleteAll();
-        }
-    }
-}
-
+VoxelServerPacketProcessor* voxelServerPacketProcessor = NULL;
+VoxelPersistThread* voxelPersistThread = NULL;
 pthread_mutex_t treeLock;
 
+
 void handlePacketSend(NodeList* nodeList, 
                       NodeList::iterator& node,
                       VoxelNodeData* nodeData, 
@@ -161,14 +97,12 @@ void handlePacketSend(NodeList* nodeList,
     nodeData->resetVoxelPacket();
 }
 
-
 // Version of voxel distributor that sends the deepest LOD level at once
 void deepestLevelVoxelDistributor(NodeList* nodeList, 
                                   NodeList::iterator& node,
                                   VoxelNodeData* nodeData,
                                   bool viewFrustumChanged) {
 
-
     pthread_mutex_lock(&::treeLock);
 
     int truePacketsSent = 0;
@@ -380,29 +314,6 @@ void deepestLevelVoxelDistributor(NodeList* nodeList,
     pthread_mutex_unlock(&::treeLock);
 }
 
-uint64_t lastPersistVoxels = 0;
-void persistVoxelsWhenDirty() {
-    uint64_t now = usecTimestampNow();
-    if (::lastPersistVoxels == 0) {
-        ::lastPersistVoxels = now;
-    }
-    int sinceLastTime = (now - ::lastPersistVoxels) / 1000;
-
-    // check the dirty bit and persist here...
-    if (::wantVoxelPersist && ::serverTree.isDirty() && sinceLastTime > VOXEL_PERSIST_INTERVAL) {
-        {
-            PerformanceWarning warn(::shouldShowAnimationDebug, 
-                                    "persistVoxelsWhenDirty() - writeToSVOFile()", ::shouldShowAnimationDebug);
-
-            printf("saving voxels to file...\n");
-            serverTree.writeToSVOFile(::voxelPersistFilename);
-            serverTree.clearDirtyBit(); // tree is clean after saving
-            printf("DONE saving voxels to file...\n");
-        }
-        ::lastPersistVoxels = usecTimestampNow();
-    }
-}
-
 void* distributeVoxelsToListeners(void* args) {
     
     NodeList* nodeList = NodeList::getInstance();
@@ -547,10 +458,6 @@ int main(int argc, const char * argv[]) {
     ::shouldShowAnimationDebug = cmdOptionExists(argc, argv, WANT_ANIMATION_DEBUG);
     printf("shouldShowAnimationDebug=%s\n", debug::valueOf(::shouldShowAnimationDebug));
 
-    const char* WANT_COLOR_RANDOMIZER = "--wantColorRandomizer";
-    ::wantColorRandomizer = cmdOptionExists(argc, argv, WANT_COLOR_RANDOMIZER);
-    printf("wantColorRandomizer=%s\n", debug::valueOf(::wantColorRandomizer));
-
     // By default we will voxel persist, if you want to disable this, then pass in this parameter
     const char* NO_VOXEL_PERSIST = "--NoVoxelPersist";
     if (cmdOptionExists(argc, argv, NO_VOXEL_PERSIST)) {
@@ -589,6 +496,12 @@ int main(int argc, const char * argv[]) {
         unsigned long internalNodeCount = ::serverTree.rootNode->getSubTreeInternalNodeCount();
         unsigned long leafNodeCount     = ::serverTree.rootNode->getSubTreeLeafNodeCount();
         printf("Nodes after loading scene %lu nodes %lu internal %lu leaves\n", nodeCount, internalNodeCount, leafNodeCount);
+        
+        // now set up VoxelPersistThread
+        ::voxelPersistThread = new VoxelPersistThread(&::serverTree, ::voxelPersistFilename);
+        if (::voxelPersistThread) {
+            ::voxelPersistThread->initialize(true);
+        }
     }
 
     // Check to see if the user passed in a command line option for loading an old style local
@@ -610,32 +523,6 @@ int main(int argc, const char * argv[]) {
         printf("packetsPerSecond=%s PACKETS_PER_CLIENT_PER_INTERVAL=%d\n", packetsPerSecond, PACKETS_PER_CLIENT_PER_INTERVAL);
     }
     
-    const char* ADD_RANDOM_VOXELS = "--AddRandomVoxels";
-    if (cmdOptionExists(argc, argv, ADD_RANDOM_VOXELS)) {
-        // create an octal code buffer and load it with 0 so that the recursive tree fill can give
-        // octal codes to the tree nodes that it is creating
-        randomlyFillVoxelTree(MAX_VOXEL_TREE_DEPTH_LEVELS, serverTree.rootNode);
-    }
-
-    const char* ADD_SCENE = "--AddScene";
-    bool addScene = cmdOptionExists(argc, argv, ADD_SCENE);
-    const char* NO_ADD_SCENE = "--NoAddScene";
-    bool noAddScene = cmdOptionExists(argc, argv, NO_ADD_SCENE);
-    if (addScene && noAddScene) {
-        printf("WARNING! --AddScene and --NoAddScene are mutually exclusive. We will honor --NoAddScene\n");
-    }
-
-    // We will add a scene if...
-    //      1) we attempted to load a persistant file and it wasn't there
-    //      2) you asked us to add a scene
-    // HOWEVER -- we will NEVER add a scene if you explicitly tell us not to!
-    //
-    // TEMPORARILY DISABLED!!!
-    bool actuallyAddScene = false; // !noAddScene && (addScene || (::wantVoxelPersist && !persistantFileRead));
-    if (actuallyAddScene) {
-        addSphereScene(&serverTree);
-    }
-    
     // for now, initialize the environments with fixed values
     environmentData[1].setID(1);
     environmentData[1].setGravity(1.0f);
@@ -652,10 +539,10 @@ int main(int argc, const char * argv[]) {
     pthread_t sendVoxelThread;
     pthread_create(&sendVoxelThread, NULL, distributeVoxelsToListeners, NULL);
 
-    sockaddr nodePublicAddress;
+    sockaddr senderAddress;
     
     unsigned char *packetData = new unsigned char[MAX_PACKET_SIZE];
-    ssize_t receivedBytes;
+    ssize_t packetLength;
     
     timeval lastDomainServerCheckIn = {};
 
@@ -664,6 +551,12 @@ int main(int argc, const char * argv[]) {
     if (::jurisdictionSender) {
         ::jurisdictionSender->initialize(true);
     }
+    
+    // set up our VoxelServerPacketProcessor
+    ::voxelServerPacketProcessor = new VoxelServerPacketProcessor();
+    if (::voxelServerPacketProcessor) {
+        ::voxelServerPacketProcessor->initialize(true);
+    }
 
     // loop to send to nodes requesting data
     while (true) {
@@ -674,155 +567,34 @@ int main(int argc, const char * argv[]) {
             NodeList::getInstance()->sendDomainServerCheckIn();
         }
         
-        // check to see if we need to persist our voxel state
-        persistVoxelsWhenDirty();
-    
-        if (nodeList->getNodeSocket()->receive(&nodePublicAddress, packetData, &receivedBytes) &&
+        if (nodeList->getNodeSocket()->receive(&senderAddress, packetData, &packetLength) &&
             packetVersionMatch(packetData)) {
-            
+
             int numBytesPacketHeader = numBytesForPacketHeader(packetData);
-            
-            if (packetData[0] == PACKET_TYPE_SET_VOXEL || packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE) {
-                bool destructive = (packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
-                PerformanceWarning warn(::shouldShowAnimationDebug,
-                                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                                        ::shouldShowAnimationDebug);
-                
-                ::receivedPacketCount++;
-                
-                unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
-                if (::shouldShowAnimationDebug) {
-                    printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
-                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                        receivedBytes,itemNumber);
-                }
-                
-                if (::debugVoxelReceiving) {
-                    printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
-                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                        ::receivedPacketCount, receivedBytes,itemNumber);
-                }
-                int atByte = numBytesPacketHeader + sizeof(itemNumber);
-                unsigned char* voxelData = (unsigned char*)&packetData[atByte];
-                while (atByte < receivedBytes) {
-                    unsigned char octets = (unsigned char)*voxelData;
-                    const int COLOR_SIZE_IN_BYTES = 3;
-                    int voxelDataSize = bytesRequiredForCodeLength(octets) + COLOR_SIZE_IN_BYTES;
-                    int voxelCodeSize = bytesRequiredForCodeLength(octets);
 
-                    // color randomization on insert
-                    int colorRandomizer = ::wantColorRandomizer ? randIntInRange (-50, 50) : 0;
-                    int red   = voxelData[voxelCodeSize + 0];
-                    int green = voxelData[voxelCodeSize + 1];
-                    int blue  = voxelData[voxelCodeSize + 2];
-
-                    if (::shouldShowAnimationDebug) {
-                        printf("insert voxels - wantColorRandomizer=%s old r=%d,g=%d,b=%d \n",
-                            (::wantColorRandomizer?"yes":"no"),red,green,blue);
-                    }
-                
-                    red   = std::max(0, std::min(255, red   + colorRandomizer));
-                    green = std::max(0, std::min(255, green + colorRandomizer));
-                    blue  = std::max(0, std::min(255, blue  + colorRandomizer));
-
-                    if (::shouldShowAnimationDebug) {
-                        printf("insert voxels - wantColorRandomizer=%s NEW r=%d,g=%d,b=%d \n",
-                            (::wantColorRandomizer?"yes":"no"),red,green,blue);
-                    }
-                    voxelData[voxelCodeSize + 0] = red;
-                    voxelData[voxelCodeSize + 1] = green;
-                    voxelData[voxelCodeSize + 2] = blue;
-
-                    if (::shouldShowAnimationDebug) {
-                        float* vertices = firstVertexForCode(voxelData);
-                        printf("inserting voxel at: %f,%f,%f\n", vertices[0], vertices[1], vertices[2]);
-                        delete []vertices;
-                    }
-                
-                    serverTree.readCodeColorBufferToTree(voxelData, destructive);
-                    // skip to next
-                    voxelData += voxelDataSize;
-                    atByte += voxelDataSize;
-                }
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-
-            } else if (packetData[0] == PACKET_TYPE_ERASE_VOXEL) {
-
-                // Send these bits off to the VoxelTree class to process them
-                pthread_mutex_lock(&::treeLock);
-                serverTree.processRemoveVoxelBitstream((unsigned char*)packetData, receivedBytes);
-                pthread_mutex_unlock(&::treeLock);
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-            } else if (packetData[0] == PACKET_TYPE_Z_COMMAND) {
-
-                // the Z command is a special command that allows the sender to send the voxel server high level semantic
-                // requests, like erase all, or add sphere scene
-                
-                char* command = (char*) &packetData[numBytesPacketHeader]; // start of the command
-                int commandLength = strlen(command); // commands are null terminated strings
-                int totalLength = numBytesPacketHeader + commandLength + 1; // 1 for null termination
-                printf("got Z message len(%ld)= %s\n", receivedBytes, command);
-                bool rebroadcast = true; // by default rebroadcast
-
-                while (totalLength <= receivedBytes) {
-                    if (strcmp(command, ERASE_ALL_COMMAND) == 0) {
-                        printf("got Z message == erase all\n");
-                        eraseVoxelTreeAndCleanupNodeVisitData();
-                        rebroadcast = false;
-                    }
-                    if (strcmp(command, ADD_SCENE_COMMAND) == 0) {
-                        printf("got Z message == add scene\n");
-                        addSphereScene(&serverTree);
-                        rebroadcast = false;
-                    }
-                    if (strcmp(command, TEST_COMMAND) == 0) {
-                        printf("got Z message == a message, nothing to do, just report\n");
-                    }
-                    totalLength += commandLength + 1; // 1 for null termination
-                }
-
-                if (rebroadcast) {
-                    // Now send this to the connected nodes so they can also process these messages
-                    printf("rebroadcasting Z message to connected nodes... nodeList.broadcastToNodes()\n");
-                    nodeList->broadcastToNodes(packetData, receivedBytes, &NODE_TYPE_AGENT, 1);
-                }
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-            } else if (packetData[0] == PACKET_TYPE_HEAD_DATA) {
+            if (packetData[0] == PACKET_TYPE_HEAD_DATA) {
                 // If we got a PACKET_TYPE_HEAD_DATA, then we're talking to an NODE_TYPE_AVATAR, and we
                 // need to make sure we have it in our nodeList.
-                
+        
                 uint16_t nodeID = 0;
                 unpackNodeId(packetData + numBytesPacketHeader, &nodeID);
-                Node* node = nodeList->addOrUpdateNode(&nodePublicAddress,
-                                                       &nodePublicAddress,
+                Node* node = NodeList::getInstance()->addOrUpdateNode(&senderAddress,
+                                                       &senderAddress,
                                                        NODE_TYPE_AGENT,
                                                        nodeID);
-                
-                nodeList->updateNodeWithData(node, packetData, receivedBytes);
+        
+                NodeList::getInstance()->updateNodeWithData(node, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_PING) {
                 // If the packet is a ping, let processNodeData handle it.
-                nodeList->processNodeData(&nodePublicAddress, packetData, receivedBytes);
+                NodeList::getInstance()->processNodeData(&senderAddress, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_DOMAIN) {
-                nodeList->processNodeData(&nodePublicAddress, packetData, receivedBytes);
+                NodeList::getInstance()->processNodeData(&senderAddress, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_VOXEL_JURISDICTION_REQUEST) {
                 if (::jurisdictionSender) {
-                    jurisdictionSender->queueReceivedPacket(nodePublicAddress, packetData, receivedBytes);
+                    ::jurisdictionSender->queueReceivedPacket(senderAddress, packetData, packetLength);
                 }
+            } else if (::voxelServerPacketProcessor) {
+                ::voxelServerPacketProcessor->queueReceivedPacket(senderAddress, packetData, packetLength);
             } else {
                 printf("unknown packet ignored... packetData[0]=%c\n", packetData[0]);
             }
@@ -837,10 +609,20 @@ int main(int argc, const char * argv[]) {
     }
     
     if (::jurisdictionSender) {
-        jurisdictionSender->terminate();
+        ::jurisdictionSender->terminate();
         delete ::jurisdictionSender;
     }
 
+    if (::voxelServerPacketProcessor) {
+        ::voxelServerPacketProcessor->terminate();
+        delete ::voxelServerPacketProcessor;
+    }
+
+    if (::voxelPersistThread) {
+        ::voxelPersistThread->terminate();
+        delete ::voxelPersistThread;
+    }
+
     return 0;
 }