threads for persist and packet processing

voxel-server/src/VoxelPersistThread.cpp

@@ -0,0 +1,37 @@
+//
+//  VoxelPersistThread.cpp
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/12/13.
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded voxel persistance
+//
+
+#include <NodeList.h>
+#include <SharedUtil.h>
+
+#include "VoxelPersistThread.h"
+#include "VoxelServer.h"
+
+VoxelPersistThread::VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval) :
+    _tree(tree),
+    _filename(filename),
+    _persistInterval(persistInterval) {
+}
+
+bool VoxelPersistThread::process() {
+    uint64_t MSECS_TO_USECS = 1000;
+    usleep(_persistInterval * MSECS_TO_USECS);
+
+
+    // check the dirty bit and persist here...
+    if (_tree->isDirty()) {
+        printf("saving voxels to file %s...\n",_filename);
+        _tree->writeToSVOFile(_filename);
+        _tree->clearDirtyBit(); // tree is clean after saving
+        printf("DONE saving voxels to file...\n");
+    }
+
+    return isStillRunning();  // keep running till they terminate us
+}

voxel-server/src/VoxelPersistThread.h

@@ -0,0 +1,33 @@
+//
+//  VoxelPersistThread.h
+//  shared
+//
+//  Created by Brad Hefta-Gaub on 8/12/13.
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded received packet processor.
+//
+
+#ifndef __shared__VoxelPersistThread__
+#define __shared__VoxelPersistThread__
+
+#include <GenericThread.h>
+#include <NetworkPacket.h>
+#include <VoxelTree.h>
+
+/// Generalized threaded processor for handling received inbound packets. 
+class VoxelPersistThread : public virtual GenericThread {
+public:
+    static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+
+    VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval = DEFAULT_PERSIST_INTERVAL);
+protected:
+    /// Implements generic processing behavior for this thread.
+    virtual bool process();
+private:
+    VoxelTree* _tree;
+    const char* _filename;
+    int _persistInterval;
+};
+
+#endif // __shared__PacketReceiver__

voxel-server/src/VoxelServer.h

@@ -0,0 +1,58 @@
+//  VoxelServer.h
+//  voxel-server
+//
+//  Created by Stephen Birarda on 3/13/13.
+//
+//
+
+#ifndef __voxel_server__VoxelServer__
+#define __voxel_server__VoxelServer__
+
+#include <SharedUtil.h>
+#include <NodeList.h> // for MAX_PACKET_SIZE
+#include <EnvironmentData.h>
+#include <JurisdictionSender.h>
+#include <VoxelTree.h>
+
+#include "VoxelServerPacketProcessor.h"
+
+
+const int MAX_FILENAME_LENGTH = 1024;
+const int VOXEL_LISTEN_PORT = 40106;
+const int VOXEL_SIZE_BYTES = 3 + (3 * sizeof(float));
+const int VOXELS_PER_PACKET = (MAX_PACKET_SIZE - 1) / VOXEL_SIZE_BYTES;
+const int MIN_BRIGHTNESS = 64;
+const float DEATH_STAR_RADIUS = 4.0;
+const float MAX_CUBE = 0.05f;
+const int VOXEL_SEND_INTERVAL_USECS = 17 * 1000; // approximately 60fps
+const int SENDING_TIME_TO_SPARE = 5 * 1000; // usec of sending interval to spare for calculating voxels
+const int INTERVALS_PER_SECOND = 1000 * 1000 / VOXEL_SEND_INTERVAL_USECS;
+const int MAX_VOXEL_TREE_DEPTH_LEVELS = 4;
+const int ENVIRONMENT_SEND_INTERVAL_USECS = 1000000;
+
+extern const char* LOCAL_VOXELS_PERSIST_FILE;
+extern const char* VOXELS_PERSIST_FILE;
+extern char voxelPersistFilename[MAX_FILENAME_LENGTH];
+extern int PACKETS_PER_CLIENT_PER_INTERVAL;
+
+extern VoxelTree serverTree; // this IS a reaveraging tree 
+extern bool wantVoxelPersist;
+extern bool wantLocalDomain;
+extern bool wantColorRandomizer;
+extern bool debugVoxelSending;
+extern bool shouldShowAnimationDebug;
+extern bool displayVoxelStats;
+extern bool debugVoxelReceiving;
+extern bool sendEnvironments;
+extern bool sendMinimalEnvironment;
+extern bool dumpVoxelsOnMove;
+extern EnvironmentData environmentData[3];
+extern int receivedPacketCount;
+extern JurisdictionMap* jurisdiction;
+extern JurisdictionSender* jurisdictionSender;
+extern VoxelServerPacketProcessor* voxelServerPacketProcessor;
+extern pthread_mutex_t treeLock;
+
+
+
+#endif // __voxel_server__VoxelServer__

voxel-server/src/VoxelServerPacketProcessor.cpp

@@ -0,0 +1,136 @@
+//
+//  VoxelServerPacketProcessor.cpp
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/12/13.
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded network packet processor for the voxel-server
+//
+
+#include <PacketHeaders.h>
+#include <PerfStat.h>
+
+#include "VoxelServer.h"
+#include "VoxelServerPacketProcessor.h"
+
+
+void VoxelServerPacketProcessor::processPacket(sockaddr& senderAddress, unsigned char* packetData, ssize_t packetLength) {
+
+    int numBytesPacketHeader = numBytesForPacketHeader(packetData);
+    
+    if (packetData[0] == PACKET_TYPE_SET_VOXEL || packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE) {
+        bool destructive = (packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
+        PerformanceWarning warn(::shouldShowAnimationDebug,
+                                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                                ::shouldShowAnimationDebug);
+        
+        ::receivedPacketCount++;
+        
+        unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
+        if (::shouldShowAnimationDebug) {
+            printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
+                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                packetLength, itemNumber);
+        }
+        
+        if (::debugVoxelReceiving) {
+            printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
+                destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
+                ::receivedPacketCount, packetLength, itemNumber);
+        }
+        int atByte = numBytesPacketHeader + sizeof(itemNumber);
+        unsigned char* voxelData = (unsigned char*)&packetData[atByte];
+        while (atByte < packetLength) {
+            unsigned char octets = (unsigned char)*voxelData;
+            const int COLOR_SIZE_IN_BYTES = 3;
+            int voxelDataSize = bytesRequiredForCodeLength(octets) + COLOR_SIZE_IN_BYTES;
+            int voxelCodeSize = bytesRequiredForCodeLength(octets);
+
+            // color randomization on insert
+            int colorRandomizer = ::wantColorRandomizer ? randIntInRange (-50, 50) : 0;
+            int red   = voxelData[voxelCodeSize + 0];
+            int green = voxelData[voxelCodeSize + 1];
+            int blue  = voxelData[voxelCodeSize + 2];
+
+            if (::shouldShowAnimationDebug) {
+                printf("insert voxels - wantColorRandomizer=%s old r=%d,g=%d,b=%d \n",
+                    (::wantColorRandomizer?"yes":"no"),red,green,blue);
+            }
+        
+            red   = std::max(0, std::min(255, red   + colorRandomizer));
+            green = std::max(0, std::min(255, green + colorRandomizer));
+            blue  = std::max(0, std::min(255, blue  + colorRandomizer));
+
+            if (::shouldShowAnimationDebug) {
+                printf("insert voxels - wantColorRandomizer=%s NEW r=%d,g=%d,b=%d \n",
+                    (::wantColorRandomizer?"yes":"no"),red,green,blue);
+            }
+            voxelData[voxelCodeSize + 0] = red;
+            voxelData[voxelCodeSize + 1] = green;
+            voxelData[voxelCodeSize + 2] = blue;
+
+            if (::shouldShowAnimationDebug) {
+                float* vertices = firstVertexForCode(voxelData);
+                printf("inserting voxel at: %f,%f,%f\n", vertices[0], vertices[1], vertices[2]);
+                delete []vertices;
+            }
+        
+            serverTree.readCodeColorBufferToTree(voxelData, destructive);
+            // skip to next
+            voxelData += voxelDataSize;
+            atByte += voxelDataSize;
+        }
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+
+    } else if (packetData[0] == PACKET_TYPE_ERASE_VOXEL) {
+
+        // Send these bits off to the VoxelTree class to process them
+        pthread_mutex_lock(&::treeLock);
+        ::serverTree.processRemoveVoxelBitstream((unsigned char*)packetData, packetLength);
+        pthread_mutex_unlock(&::treeLock);
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+    } else if (packetData[0] == PACKET_TYPE_Z_COMMAND) {
+
+        // the Z command is a special command that allows the sender to send the voxel server high level semantic
+        // requests, like erase all, or add sphere scene
+        
+        char* command = (char*) &packetData[numBytesPacketHeader]; // start of the command
+        int commandLength = strlen(command); // commands are null terminated strings
+        int totalLength = numBytesPacketHeader + commandLength + 1; // 1 for null termination
+        printf("got Z message len(%ld)= %s\n", packetLength, command);
+        bool rebroadcast = true; // by default rebroadcast
+
+        while (totalLength <= packetLength) {
+            if (strcmp(command, TEST_COMMAND) == 0) {
+                printf("got Z message == a message, nothing to do, just report\n");
+            }
+            totalLength += commandLength + 1; // 1 for null termination
+        }
+
+        if (rebroadcast) {
+            // Now send this to the connected nodes so they can also process these messages
+            printf("rebroadcasting Z message to connected nodes... nodeList.broadcastToNodes()\n");
+            NodeList::getInstance()->broadcastToNodes(packetData, packetLength, &NODE_TYPE_AGENT, 1);
+        }
+
+        // Make sure our Node and NodeList knows we've heard from this node.
+        Node* node = NodeList::getInstance()->nodeWithAddress(&senderAddress);
+        if (node) {
+            node->setLastHeardMicrostamp(usecTimestampNow());
+        }
+    } else {
+        printf("unknown packet ignored... packetData[0]=%c\n", packetData[0]);
+    }
+}
+

voxel-server/src/VoxelServerPacketProcessor.h

@@ -0,0 +1,22 @@
+//
+//  VoxelServerPacketProcessor.h
+//  voxel-server
+//
+//  Created by Brad Hefta-Gaub on 8/12/13.
+//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
+//
+//  Threaded or non-threaded network packet processor for the voxel-server
+//
+
+#ifndef __voxel_server__VoxelServerPacketProcessor__
+#define __voxel_server__VoxelServerPacketProcessor__
+
+#include <ReceivedPacketProcessor.h>
+
+/// Handles processing of incoming network packets for the voxel-server. As with other ReceivedPacketProcessor classes 
+/// the user is responsible for reading inbound packets and adding them to the processing queue by calling queueReceivedPacket()
+class VoxelServerPacketProcessor : public ReceivedPacketProcessor {
+protected:
+    virtual void processPacket(sockaddr& senderAddress, unsigned char*  packetData, ssize_t packetLength);
+};
+#endif // __voxel_server__VoxelServerPacketProcessor__

voxel-server/src/main.cpp

@@ -23,6 +23,9 @@
 
 #include <JurisdictionSender.h>
 
+#include "VoxelPersistThread.h"
+#include "VoxelServerPacketProcessor.h"
+
 #ifdef _WIN32
 #include "Syssocket.h"
 #include "Systime.h"
@@ -32,36 +35,15 @@
 #include <ifaddrs.h>
 #endif
 
+#include "VoxelServer.h"
+
 const char* LOCAL_VOXELS_PERSIST_FILE = "resources/voxels.svo";
 const char* VOXELS_PERSIST_FILE = "/etc/highfidelity/voxel-server/resources/voxels.svo";
-const int MAX_FILENAME_LENGTH = 1024;
 char voxelPersistFilename[MAX_FILENAME_LENGTH];
-const int VOXEL_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
-
-const int VOXEL_LISTEN_PORT = 40106;
-
-
-const int VOXEL_SIZE_BYTES = 3 + (3 * sizeof(float));
-const int VOXELS_PER_PACKET = (MAX_PACKET_SIZE - 1) / VOXEL_SIZE_BYTES;
-
-const int MIN_BRIGHTNESS = 64;
-const float DEATH_STAR_RADIUS = 4.0;
-const float MAX_CUBE = 0.05f;
-
-const int VOXEL_SEND_INTERVAL_USECS = 17 * 1000; // approximately 60fps
 int PACKETS_PER_CLIENT_PER_INTERVAL = 10;
-const int SENDING_TIME_TO_SPARE = 5 * 1000; // usec of sending interval to spare for calculating voxels
-const int INTERVALS_PER_SECOND = 1000 * 1000 / VOXEL_SEND_INTERVAL_USECS;
-
-const int MAX_VOXEL_TREE_DEPTH_LEVELS = 4;
-
-const int ENVIRONMENT_SEND_INTERVAL_USECS = 1000000;
-
 VoxelTree serverTree(true); // this IS a reaveraging tree 
 bool wantVoxelPersist = true;
 bool wantLocalDomain = false;
-
-
 bool wantColorRandomizer = false;
 bool debugVoxelSending = false;
 bool shouldShowAnimationDebug = false;
@@ -70,43 +52,14 @@ bool debugVoxelReceiving = false;
 bool sendEnvironments = true;
 bool sendMinimalEnvironment = false;
 bool dumpVoxelsOnMove = false;
-
 EnvironmentData environmentData[3];
-
 int receivedPacketCount = 0;
 JurisdictionMap* jurisdiction = NULL;
 JurisdictionSender* jurisdictionSender = NULL;
+VoxelServerPacketProcessor* voxelServerPacketProcessor = NULL;
+VoxelPersistThread* voxelPersistThread = NULL;
+pthread_mutex_t treeLock;
 
-void randomlyFillVoxelTree(int levelsToGo, VoxelNode *currentRootNode) {
-    // randomly generate children for this node
-    // the first level of the tree (where levelsToGo = MAX_VOXEL_TREE_DEPTH_LEVELS) has all 8
-    if (levelsToGo > 0) {
-
-        bool createdChildren = false;
-        createdChildren = false;
-        
-        for (int i = 0; i < 8; i++) {
-            if (true) {
-                // create a new VoxelNode to put here
-                currentRootNode->addChildAtIndex(i);
-                randomlyFillVoxelTree(levelsToGo - 1, currentRootNode->getChildAtIndex(i));
-                createdChildren = true;
-            }
-        }
-        
-        if (!createdChildren) {
-            // we didn't create any children for this node, making it a leaf
-            // give it a random color
-            currentRootNode->setRandomColor(MIN_BRIGHTNESS);
-        } else {
-            // set the color value for this node
-            currentRootNode->setColorFromAverageOfChildren();
-        }
-    } else {
-        // this is a leaf node, just give it a color
-        currentRootNode->setRandomColor(MIN_BRIGHTNESS);
-    }
-}
 
 void eraseVoxelTreeAndCleanupNodeVisitData() {
 
@@ -122,8 +75,6 @@ void eraseVoxelTreeAndCleanupNodeVisitData() {
     }
 }
 
-pthread_mutex_t treeLock;
-
 void handlePacketSend(NodeList* nodeList, 
                       NodeList::iterator& node,
                       VoxelNodeData* nodeData, 
@@ -380,29 +331,6 @@ void deepestLevelVoxelDistributor(NodeList* nodeList,
     pthread_mutex_unlock(&::treeLock);
 }
 
-uint64_t lastPersistVoxels = 0;
-void persistVoxelsWhenDirty() {
-    uint64_t now = usecTimestampNow();
-    if (::lastPersistVoxels == 0) {
-        ::lastPersistVoxels = now;
-    }
-    int sinceLastTime = (now - ::lastPersistVoxels) / 1000;
-
-    // check the dirty bit and persist here...
-    if (::wantVoxelPersist && ::serverTree.isDirty() && sinceLastTime > VOXEL_PERSIST_INTERVAL) {
-        {
-            PerformanceWarning warn(::shouldShowAnimationDebug, 
-                                    "persistVoxelsWhenDirty() - writeToSVOFile()", ::shouldShowAnimationDebug);
-
-            printf("saving voxels to file...\n");
-            serverTree.writeToSVOFile(::voxelPersistFilename);
-            serverTree.clearDirtyBit(); // tree is clean after saving
-            printf("DONE saving voxels to file...\n");
-        }
-        ::lastPersistVoxels = usecTimestampNow();
-    }
-}
-
 void* distributeVoxelsToListeners(void* args) {
     
     NodeList* nodeList = NodeList::getInstance();
@@ -589,6 +517,12 @@ int main(int argc, const char * argv[]) {
         unsigned long internalNodeCount = ::serverTree.rootNode->getSubTreeInternalNodeCount();
         unsigned long leafNodeCount     = ::serverTree.rootNode->getSubTreeLeafNodeCount();
         printf("Nodes after loading scene %lu nodes %lu internal %lu leaves\n", nodeCount, internalNodeCount, leafNodeCount);
+        
+        // now set up VoxelPersistThread
+        ::voxelPersistThread = new VoxelPersistThread(&::serverTree, ::voxelPersistFilename);
+        if (::voxelPersistThread) {
+            ::voxelPersistThread->initialize(true);
+        }
     }
 
     // Check to see if the user passed in a command line option for loading an old style local
@@ -610,32 +544,6 @@ int main(int argc, const char * argv[]) {
         printf("packetsPerSecond=%s PACKETS_PER_CLIENT_PER_INTERVAL=%d\n", packetsPerSecond, PACKETS_PER_CLIENT_PER_INTERVAL);
     }
     
-    const char* ADD_RANDOM_VOXELS = "--AddRandomVoxels";
-    if (cmdOptionExists(argc, argv, ADD_RANDOM_VOXELS)) {
-        // create an octal code buffer and load it with 0 so that the recursive tree fill can give
-        // octal codes to the tree nodes that it is creating
-        randomlyFillVoxelTree(MAX_VOXEL_TREE_DEPTH_LEVELS, serverTree.rootNode);
-    }
-
-    const char* ADD_SCENE = "--AddScene";
-    bool addScene = cmdOptionExists(argc, argv, ADD_SCENE);
-    const char* NO_ADD_SCENE = "--NoAddScene";
-    bool noAddScene = cmdOptionExists(argc, argv, NO_ADD_SCENE);
-    if (addScene && noAddScene) {
-        printf("WARNING! --AddScene and --NoAddScene are mutually exclusive. We will honor --NoAddScene\n");
-    }
-
-    // We will add a scene if...
-    //      1) we attempted to load a persistant file and it wasn't there
-    //      2) you asked us to add a scene
-    // HOWEVER -- we will NEVER add a scene if you explicitly tell us not to!
-    //
-    // TEMPORARILY DISABLED!!!
-    bool actuallyAddScene = false; // !noAddScene && (addScene || (::wantVoxelPersist && !persistantFileRead));
-    if (actuallyAddScene) {
-        addSphereScene(&serverTree);
-    }
-    
     // for now, initialize the environments with fixed values
     environmentData[1].setID(1);
     environmentData[1].setGravity(1.0f);
@@ -652,10 +560,10 @@ int main(int argc, const char * argv[]) {
     pthread_t sendVoxelThread;
     pthread_create(&sendVoxelThread, NULL, distributeVoxelsToListeners, NULL);
 
-    sockaddr nodePublicAddress;
+    sockaddr senderAddress;
     
     unsigned char *packetData = new unsigned char[MAX_PACKET_SIZE];
-    ssize_t receivedBytes;
+    ssize_t packetLength;
     
     timeval lastDomainServerCheckIn = {};
 
@@ -664,6 +572,12 @@ int main(int argc, const char * argv[]) {
     if (::jurisdictionSender) {
         ::jurisdictionSender->initialize(true);
     }
+    
+    // set up our VoxelServerPacketProcessor
+    ::voxelServerPacketProcessor = new VoxelServerPacketProcessor();
+    if (::voxelServerPacketProcessor) {
+        ::voxelServerPacketProcessor->initialize(true);
+    }
 
     // loop to send to nodes requesting data
     while (true) {
@@ -674,155 +588,34 @@ int main(int argc, const char * argv[]) {
             NodeList::getInstance()->sendDomainServerCheckIn();
         }
         
-        // check to see if we need to persist our voxel state
+        if (nodeList->getNodeSocket()->receive(&senderAddress, packetData, &packetLength) &&
-        persistVoxelsWhenDirty();
-    
-        if (nodeList->getNodeSocket()->receive(&nodePublicAddress, packetData, &receivedBytes) &&
             packetVersionMatch(packetData)) {
-            
+
             int numBytesPacketHeader = numBytesForPacketHeader(packetData);
-            
-            if (packetData[0] == PACKET_TYPE_SET_VOXEL || packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE) {
-                bool destructive = (packetData[0] == PACKET_TYPE_SET_VOXEL_DESTRUCTIVE);
-                PerformanceWarning warn(::shouldShowAnimationDebug,
-                                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                                        ::shouldShowAnimationDebug);
-                
-                ::receivedPacketCount++;
-                
-                unsigned short int itemNumber = (*((unsigned short int*)(packetData + numBytesPacketHeader)));
-                if (::shouldShowAnimationDebug) {
-                    printf("got %s - command from client receivedBytes=%ld itemNumber=%d\n",
-                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                        receivedBytes,itemNumber);
-                }
-                
-                if (::debugVoxelReceiving) {
-                    printf("got %s - %d command from client receivedBytes=%ld itemNumber=%d\n",
-                        destructive ? "PACKET_TYPE_SET_VOXEL_DESTRUCTIVE" : "PACKET_TYPE_SET_VOXEL",
-                        ::receivedPacketCount, receivedBytes,itemNumber);
-                }
-                int atByte = numBytesPacketHeader + sizeof(itemNumber);
-                unsigned char* voxelData = (unsigned char*)&packetData[atByte];
-                while (atByte < receivedBytes) {
-                    unsigned char octets = (unsigned char)*voxelData;
-                    const int COLOR_SIZE_IN_BYTES = 3;
-                    int voxelDataSize = bytesRequiredForCodeLength(octets) + COLOR_SIZE_IN_BYTES;
-                    int voxelCodeSize = bytesRequiredForCodeLength(octets);
 
-                    // color randomization on insert
+            if (packetData[0] == PACKET_TYPE_HEAD_DATA) {
-                    int colorRandomizer = ::wantColorRandomizer ? randIntInRange (-50, 50) : 0;
-                    int red   = voxelData[voxelCodeSize + 0];
-                    int green = voxelData[voxelCodeSize + 1];
-                    int blue  = voxelData[voxelCodeSize + 2];
-
-                    if (::shouldShowAnimationDebug) {
-                        printf("insert voxels - wantColorRandomizer=%s old r=%d,g=%d,b=%d \n",
-                            (::wantColorRandomizer?"yes":"no"),red,green,blue);
-                    }
-                
-                    red   = std::max(0, std::min(255, red   + colorRandomizer));
-                    green = std::max(0, std::min(255, green + colorRandomizer));
-                    blue  = std::max(0, std::min(255, blue  + colorRandomizer));
-
-                    if (::shouldShowAnimationDebug) {
-                        printf("insert voxels - wantColorRandomizer=%s NEW r=%d,g=%d,b=%d \n",
-                            (::wantColorRandomizer?"yes":"no"),red,green,blue);
-                    }
-                    voxelData[voxelCodeSize + 0] = red;
-                    voxelData[voxelCodeSize + 1] = green;
-                    voxelData[voxelCodeSize + 2] = blue;
-
-                    if (::shouldShowAnimationDebug) {
-                        float* vertices = firstVertexForCode(voxelData);
-                        printf("inserting voxel at: %f,%f,%f\n", vertices[0], vertices[1], vertices[2]);
-                        delete []vertices;
-                    }
-                
-                    serverTree.readCodeColorBufferToTree(voxelData, destructive);
-                    // skip to next
-                    voxelData += voxelDataSize;
-                    atByte += voxelDataSize;
-                }
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-
-            } else if (packetData[0] == PACKET_TYPE_ERASE_VOXEL) {
-
-                // Send these bits off to the VoxelTree class to process them
-                pthread_mutex_lock(&::treeLock);
-                serverTree.processRemoveVoxelBitstream((unsigned char*)packetData, receivedBytes);
-                pthread_mutex_unlock(&::treeLock);
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-            } else if (packetData[0] == PACKET_TYPE_Z_COMMAND) {
-
-                // the Z command is a special command that allows the sender to send the voxel server high level semantic
-                // requests, like erase all, or add sphere scene
-                
-                char* command = (char*) &packetData[numBytesPacketHeader]; // start of the command
-                int commandLength = strlen(command); // commands are null terminated strings
-                int totalLength = numBytesPacketHeader + commandLength + 1; // 1 for null termination
-                printf("got Z message len(%ld)= %s\n", receivedBytes, command);
-                bool rebroadcast = true; // by default rebroadcast
-
-                while (totalLength <= receivedBytes) {
-                    if (strcmp(command, ERASE_ALL_COMMAND) == 0) {
-                        printf("got Z message == erase all\n");
-                        eraseVoxelTreeAndCleanupNodeVisitData();
-                        rebroadcast = false;
-                    }
-                    if (strcmp(command, ADD_SCENE_COMMAND) == 0) {
-                        printf("got Z message == add scene\n");
-                        addSphereScene(&serverTree);
-                        rebroadcast = false;
-                    }
-                    if (strcmp(command, TEST_COMMAND) == 0) {
-                        printf("got Z message == a message, nothing to do, just report\n");
-                    }
-                    totalLength += commandLength + 1; // 1 for null termination
-                }
-
-                if (rebroadcast) {
-                    // Now send this to the connected nodes so they can also process these messages
-                    printf("rebroadcasting Z message to connected nodes... nodeList.broadcastToNodes()\n");
-                    nodeList->broadcastToNodes(packetData, receivedBytes, &NODE_TYPE_AGENT, 1);
-                }
-
-                // Make sure our Node and NodeList knows we've heard from this node.
-                Node* node = NodeList::getInstance()->nodeWithAddress(&nodePublicAddress);
-                if (node) {
-                    node->setLastHeardMicrostamp(usecTimestampNow());
-                }
-            } else if (packetData[0] == PACKET_TYPE_HEAD_DATA) {
                 // If we got a PACKET_TYPE_HEAD_DATA, then we're talking to an NODE_TYPE_AVATAR, and we
                 // need to make sure we have it in our nodeList.
-                
+        
                 uint16_t nodeID = 0;
                 unpackNodeId(packetData + numBytesPacketHeader, &nodeID);
-                Node* node = nodeList->addOrUpdateNode(&nodePublicAddress,
+                Node* node = NodeList::getInstance()->addOrUpdateNode(&senderAddress,
-                                                       &nodePublicAddress,
+                                                       &senderAddress,
                                                        NODE_TYPE_AGENT,
                                                        nodeID);
-                
+        
-                nodeList->updateNodeWithData(node, packetData, receivedBytes);
+                NodeList::getInstance()->updateNodeWithData(node, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_PING) {
                 // If the packet is a ping, let processNodeData handle it.
-                nodeList->processNodeData(&nodePublicAddress, packetData, receivedBytes);
+                NodeList::getInstance()->processNodeData(&senderAddress, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_DOMAIN) {
-                nodeList->processNodeData(&nodePublicAddress, packetData, receivedBytes);
+                NodeList::getInstance()->processNodeData(&senderAddress, packetData, packetLength);
             } else if (packetData[0] == PACKET_TYPE_VOXEL_JURISDICTION_REQUEST) {
                 if (::jurisdictionSender) {
-                    jurisdictionSender->queueReceivedPacket(nodePublicAddress, packetData, receivedBytes);
+                    ::jurisdictionSender->queueReceivedPacket(senderAddress, packetData, packetLength);
                 }
+            } else if (::voxelServerPacketProcessor) {
+                ::voxelServerPacketProcessor->queueReceivedPacket(senderAddress, packetData, packetLength);
             } else {
                 printf("unknown packet ignored... packetData[0]=%c\n", packetData[0]);
             }
@@ -837,10 +630,20 @@ int main(int argc, const char * argv[]) {
     }
     
     if (::jurisdictionSender) {
-        jurisdictionSender->terminate();
+        ::jurisdictionSender->terminate();
         delete ::jurisdictionSender;
     }
 
+    if (::voxelServerPacketProcessor) {
+        ::voxelServerPacketProcessor->terminate();
+        delete ::voxelServerPacketProcessor;
+    }
+
+    if (::voxelPersistThread) {
+        ::voxelPersistThread->terminate();
+        delete ::voxelPersistThread;
+    }
+
     return 0;
 }