Mirrors/overte-HifiExperiments
3
0
Fork 0
mirror of https://github.com/HifiExperiments/overte.git synced 2025-08-13 11:14:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #887 from ZappoMan/threaded_voxel_server

Browse source 
Make separate per client threads for sending voxels to each client
This commit is contained in:
Andrzej Kapolka 2013-08-23 16:39:44 -07:00
commit 193b37eaa5
9 changed files with 418 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
voxel-server/src/NodeWatcher.cpp Normal file
View file

@ -0,0 +1,26 @@
//
// NodeWatcher.h
// voxel-server
//
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Node List Hook that watches for Node's being killed in order to clean up node specific memory and threads
//
#include <NodeList.h>
#include "NodeWatcher.h"
#include "VoxelNodeData.h"
void NodeWatcher::nodeAdded(Node* node) {
// do nothing
}
void NodeWatcher::nodeKilled(Node* node) {
// Use this to cleanup our node
if (node->getType() == NODE_TYPE_AGENT) {
VoxelNodeData* nodeData = (VoxelNodeData*)node->getLinkedData();
node->setLinkedData(NULL);
delete nodeData;
}
};

23
voxel-server/src/NodeWatcher.h Normal file
View file

@ -0,0 +1,23 @@
//
// NodeWatcher.h
// voxel-server
//
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Node List Hook that watches for Node's being killed in order to clean up node specific memory and threads
//
#ifndef __voxel_server__NodeWatcher__
#define __voxel_server__NodeWatcher__
#include <NodeList.h>
/// Voxel server's node watcher, which watches for nodes being killed and cleans up their data and threads
class NodeWatcher : public virtual NodeListHook {
public:
virtual void nodeAdded(Node* node);
virtual void nodeKilled(Node* node);
};
#endif // __voxel_server__NodeWatcher__

12
voxel-server/src/VoxelNodeData.cpp
View file

@ -11,6 +11,7 @@
#include "VoxelNodeData.h"
#include <cstring>
#include <cstdio>
#include "VoxelSendThread.h"
VoxelNodeData::VoxelNodeData(Node* owningNode) :
AvatarData(owningNode),
@ -21,11 +22,17 @@ VoxelNodeData::VoxelNodeData(Node* owningNode) :
_lastTimeBagEmpty(0),
_viewFrustumChanging(false),
_viewFrustumJustStoppedChanging(true),
_currentPacketIsColor(true)
_currentPacketIsColor(true),
_voxelSendThread(NULL)
{
_voxelPacket = new unsigned char[MAX_VOXEL_PACKET_SIZE];
_voxelPacketAt = _voxelPacket;
resetVoxelPacket();
// Create voxel sending thread...
uint16_t nodeID = getOwningNode()->getNodeID();
_voxelSendThread = new VoxelSendThread(nodeID);
_voxelSendThread->initialize(true);
}
@ -49,6 +56,9 @@ void VoxelNodeData::writeToPacket(unsigned char* buffer, int bytes) {
VoxelNodeData::~VoxelNodeData() {
delete[] _voxelPacket;
_voxelSendThread->terminate();
delete _voxelSendThread;
}
bool VoxelNodeData::updateCurrentViewFrustum() {

4
voxel-server/src/VoxelNodeData.h
View file

@ -18,6 +18,8 @@
#include <VoxelNodeBag.h>
#include <VoxelSceneStats.h>
class VoxelSendThread;
class VoxelNodeData : public AvatarData {
public:
VoxelNodeData(Node* owningNode);
@ -80,6 +82,8 @@ private:
bool _viewFrustumChanging;
bool _viewFrustumJustStoppedChanging;
bool _currentPacketIsColor;
VoxelSendThread* _voxelSendThread;
};
#endif /* defined(__hifi__VoxelNodeData__) */

2
voxel-server/src/VoxelPersistThread.cpp
View file

@ -5,7 +5,7 @@
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Threaded or non-threaded voxel persistance
// Threaded or non-threaded voxel persistence
//
#include <NodeList.h>

2
voxel-server/src/VoxelPersistThread.h
View file

@ -5,7 +5,7 @@
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Threaded or non-threaded voxel persistance
// Threaded or non-threaded voxel persistence
//
#ifndef __voxel_server__VoxelPersistThread__

303
voxel-server/src/VoxelSendThread.cpp Normal file
View file

@ -0,0 +1,303 @@
//
// VoxelSendThread.cpp
// voxel-server
//
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Threaded or non-threaded voxel packet sender
//
#include <NodeList.h>
#include <SharedUtil.h>
#include <PacketHeaders.h>
#include "VoxelSendThread.h"
#include "VoxelServer.h"
VoxelSendThread::VoxelSendThread(uint16_t nodeID) :
_nodeID(nodeID) {
}
bool VoxelSendThread::process() {
uint64_t lastSendTime = usecTimestampNow();
Node* node = NodeList::getInstance()->nodeWithID(_nodeID);
VoxelNodeData* nodeData = NULL;
if (node) {
nodeData = (VoxelNodeData*) node->getLinkedData();
}
// Sometimes the node data has not yet been linked, in which case we can't really do anything
if (nodeData) {
bool viewFrustumChanged = nodeData->updateCurrentViewFrustum();
if (::debugVoxelSending) {
printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged));
}
deepestLevelVoxelDistributor(node, nodeData, viewFrustumChanged);
}
// dynamically sleep until we need to fire off the next set of voxels
int usecToSleep = VOXEL_SEND_INTERVAL_USECS - (usecTimestampNow() - lastSendTime);
if (usecToSleep > 0) {
usleep(usecToSleep);
} else {
if (::debugVoxelSending) {
std::cout << "Last send took too much time, not sleeping!\n";
}
}
return isStillRunning(); // keep running till they terminate us
}
void VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int& trueBytesSent, int& truePacketsSent) {
// If we've got a stats message ready to send, then see if we can piggyback them together
if (nodeData->stats.isReadyToSend()) {
// Send the stats message to the client
unsigned char* statsMessage = nodeData->stats.getStatsMessage();
int statsMessageLength = nodeData->stats.getStatsMessageLength();
// If the size of the stats message and the voxel message will fit in a packet, then piggyback them
if (nodeData->getPacketLength() + statsMessageLength < MAX_PACKET_SIZE) {
// copy voxel message to back of stats message
memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength());
statsMessageLength += nodeData->getPacketLength();
// actually send it
NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength);
} else {
// not enough room in the packet, send two packets
NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength);
NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(),
nodeData->getPacket(), nodeData->getPacketLength());
}
} else {
// just send the voxel packet
NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(),
nodeData->getPacket(), nodeData->getPacketLength());
}
// remember to track our stats
nodeData->stats.packetSent(nodeData->getPacketLength());
trueBytesSent += nodeData->getPacketLength();
truePacketsSent++;
nodeData->resetVoxelPacket();
}
/// Version of voxel distributor that sends the deepest LOD level at once
void VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged) {
pthread_mutex_lock(&::treeLock);
int truePacketsSent = 0;
int trueBytesSent = 0;
// FOR NOW... node tells us if it wants to receive only view frustum deltas
bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
// If our packet already has content in it, then we must use the color choice of the waiting packet.
// If we're starting a fresh packet, then...
// If we're moving, and the client asked for low res, then we force monochrome, otherwise, use
// the clients requested color state.
bool wantColor = LOW_RES_MONO && nodeData->getWantLowResMoving() && viewFrustumChanged ? false : nodeData->getWantColor();
// If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color
// then let's just send that waiting packet.
if (wantColor != nodeData->getCurrentPacketIsColor()) {
if (nodeData->isPacketWaiting()) {
if (::debugVoxelSending) {
printf("wantColor=%s --- SENDING PARTIAL PACKET! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
} else {
if (::debugVoxelSending) {
printf("wantColor=%s --- FIXING HEADER! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
nodeData->resetVoxelPacket();
}
}
if (::debugVoxelSending) {
printf("wantColor=%s getCurrentPacketIsColor()=%s, viewFrustumChanged=%s, getWantLowResMoving()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving()));
}
const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL;
if (::debugVoxelSending) {
printf("deepestLevelVoxelDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()),
debug::valueOf(nodeData->getViewSent())
);
}
// If the current view frustum has changed OR we have nothing to send, then search against
// the current view frustum for things to send.
if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) {
uint64_t now = usecTimestampNow();
if (::debugVoxelSending) {
printf("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()));
if (nodeData->getLastTimeBagEmpty() > 0) {
float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f;
if (viewFrustumChanged) {
printf("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend);
} else {
printf("elapsed time to send scene = %f seconds", elapsedSceneSend);
}
printf(" [occlusionCulling:%s, wantDelta:%s, wantColor:%s ]\n",
debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta),
debug::valueOf(wantColor));
}
}
// if our view has changed, we need to reset these things...
if (viewFrustumChanged) {
if (::dumpVoxelsOnMove) {
nodeData->nodeBag.deleteAll();
}
nodeData->map.erase();
}
if (!viewFrustumChanged && !nodeData->getWantDelta()) {
// only set our last sent time if we weren't resetting due to frustum change
uint64_t now = usecTimestampNow();
nodeData->setLastTimeBagEmpty(now);
}
nodeData->stats.sceneCompleted();
if (::displayVoxelStats) {
nodeData->stats.printDebugDetails();
}
// start tracking our stats
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging();
// If we're starting a full scene, then definitely we want to empty the nodeBag
if (isFullScene) {
nodeData->nodeBag.deleteAll();
}
nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, ::serverTree.rootNode, ::jurisdiction);
// This is the start of "resending" the scene.
nodeData->nodeBag.insert(serverTree.rootNode);
}
// If we have something in our nodeBag, then turn them into packets and send them out...
if (!nodeData->nodeBag.isEmpty()) {
int bytesWritten = 0;
int packetsSentThisInterval = 0;
uint64_t start = usecTimestampNow();
bool shouldSendEnvironments = ::sendEnvironments && shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS);
while (packetsSentThisInterval < PACKETS_PER_CLIENT_PER_INTERVAL - (shouldSendEnvironments ? 1 : 0)) {
// Check to see if we're taking too long, and if so bail early...
uint64_t now = usecTimestampNow();
long elapsedUsec = (now - start);
long elapsedUsecPerPacket = (truePacketsSent == 0) ? 0 : (elapsedUsec / truePacketsSent);
long usecRemaining = (VOXEL_SEND_INTERVAL_USECS - elapsedUsec);
if (elapsedUsecPerPacket + SENDING_TIME_TO_SPARE > usecRemaining) {
if (::debugVoxelSending) {
printf("packetLoop() usecRemaining=%ld bailing early took %ld usecs to generate %d bytes in %d packets (%ld usec avg), %d nodes still to send\n",
usecRemaining, elapsedUsec, trueBytesSent, truePacketsSent, elapsedUsecPerPacket,
nodeData->nodeBag.count());
}
break;
}
if (!nodeData->nodeBag.isEmpty()) {
VoxelNode* subTree = nodeData->nodeBag.extract();
bool wantOcclusionCulling = nodeData->getWantOcclusionCulling();
CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP;
int boundaryLevelAdjust = viewFrustumChanged && nodeData->getWantLowResMoving()
? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST;
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) &&
nodeData->getViewFrustumJustStoppedChanging();
EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor,
WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
wantOcclusionCulling, coverageMap, boundaryLevelAdjust,
nodeData->getLastTimeBagEmpty(),
isFullScene, &nodeData->stats, ::jurisdiction);
nodeData->stats.encodeStarted();
bytesWritten = serverTree.encodeTreeBitstream(subTree, _tempOutputBuffer, MAX_VOXEL_PACKET_SIZE - 1,
nodeData->nodeBag, params);
nodeData->stats.encodeStopped();
if (nodeData->getAvailable() >= bytesWritten) {
nodeData->writeToPacket(_tempOutputBuffer, bytesWritten);
} else {
handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
packetsSentThisInterval++;
nodeData->resetVoxelPacket();
nodeData->writeToPacket(_tempOutputBuffer, bytesWritten);
}
} else {
if (nodeData->isPacketWaiting()) {
handlePacketSend(node, nodeData, trueBytesSent, truePacketsSent);
nodeData->resetVoxelPacket();
}
packetsSentThisInterval = PACKETS_PER_CLIENT_PER_INTERVAL; // done for now, no nodes left
}
}
// send the environment packet
if (shouldSendEnvironments) {
int numBytesPacketHeader = populateTypeAndVersion(_tempOutputBuffer, PACKET_TYPE_ENVIRONMENT_DATA);
int envPacketLength = numBytesPacketHeader;
int environmentsToSend = ::sendMinimalEnvironment ? 1 : sizeof(environmentData) / sizeof(EnvironmentData);
for (int i = 0; i < environmentsToSend; i++) {
envPacketLength += environmentData[i].getBroadcastData(_tempOutputBuffer + envPacketLength);
}
NodeList::getInstance()->getNodeSocket()->send(node->getActiveSocket(), _tempOutputBuffer, envPacketLength);
trueBytesSent += envPacketLength;
truePacketsSent++;
}
uint64_t end = usecTimestampNow();
int elapsedmsec = (end - start)/1000;
if (elapsedmsec > 100) {
if (elapsedmsec > 1000) {
int elapsedsec = (end - start)/1000000;
printf("WARNING! packetLoop() took %d seconds to generate %d bytes in %d packets %d nodes still to send\n",
elapsedsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
} else {
printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
} else if (::debugVoxelSending) {
printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
// if after sending packets we've emptied our bag, then we want to remember that we've sent all
// the voxels from the current view frustum
if (nodeData->nodeBag.isEmpty()) {
nodeData->updateLastKnownViewFrustum();
nodeData->setViewSent(true);
if (::debugVoxelSending) {
nodeData->map.printStats();
}
nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes
}
} // end if bag wasn't empty, and so we sent stuff...
pthread_mutex_unlock(&::treeLock);
}

37
voxel-server/src/VoxelSendThread.h Normal file
View file

@ -0,0 +1,37 @@
//
// VoxelSendThread.h
// voxel-server
//
// Created by Brad Hefta-Gaub on 8/21/13
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
//
// Threaded or non-threaded object for sending voxels to a client
//
#ifndef __voxel_server__VoxelSendThread__
#define __voxel_server__VoxelSendThread__
#include <GenericThread.h>
#include <NetworkPacket.h>
#include <VoxelTree.h>
#include <VoxelNodeBag.h>
#include "VoxelNodeData.h"
/// Threaded processor for sending voxel packets to a single client
class VoxelSendThread : public virtual GenericThread {
public:
VoxelSendThread(uint16_t nodeID);
protected:
/// Implements generic processing behavior for this thread.
virtual bool process();
private:
uint16_t _nodeID;
void handlePacketSend(Node* node, VoxelNodeData* nodeData, int& trueBytesSent, int& truePacketsSent);
void deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged);
unsigned char _tempOutputBuffer[MAX_VOXEL_PACKET_SIZE];
};
#endif // __voxel_server__VoxelSendThread__

310
voxel-server/src/main.cpp
View file

@ -23,7 +23,9 @@
#include <PerfStat.h>
#include <JurisdictionSender.h>
#include "NodeWatcher.h"
#include "VoxelPersistThread.h"
#include "VoxelSendThread.h"
#include "VoxelServerPacketProcessor.h"
#ifdef _WIN32
@ -58,297 +60,7 @@ JurisdictionSender* jurisdictionSender = NULL;
VoxelServerPacketProcessor* voxelServerPacketProcessor = NULL;
VoxelPersistThread* voxelPersistThread = NULL;
pthread_mutex_t treeLock;
void handlePacketSend(NodeList* nodeList,
NodeList::iterator& node,
VoxelNodeData* nodeData,
int& trueBytesSent, int& truePacketsSent) {
// If we've got a stats message ready to send, then see if we can piggyback them together
if (nodeData->stats.isReadyToSend()) {
// Send the stats message to the client
unsigned char* statsMessage = nodeData->stats.getStatsMessage();
int statsMessageLength = nodeData->stats.getStatsMessageLength();
// If the size of the stats message and the voxel message will fit in a packet, then piggyback them
if (nodeData->getPacketLength() + statsMessageLength < MAX_PACKET_SIZE) {
// copy voxel message to back of stats message
memcpy(statsMessage + statsMessageLength, nodeData->getPacket(), nodeData->getPacketLength());
statsMessageLength += nodeData->getPacketLength();
// actually send it
nodeList->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength);
} else {
// not enough room in the packet, send two packets
nodeList->getNodeSocket()->send(node->getActiveSocket(), statsMessage, statsMessageLength);
nodeList->getNodeSocket()->send(node->getActiveSocket(),
nodeData->getPacket(), nodeData->getPacketLength());
}
} else {
// just send the voxel packet
nodeList->getNodeSocket()->send(node->getActiveSocket(),
nodeData->getPacket(), nodeData->getPacketLength());
}
// remember to track our stats
nodeData->stats.packetSent(nodeData->getPacketLength());
trueBytesSent += nodeData->getPacketLength();
truePacketsSent++;
nodeData->resetVoxelPacket();
}
// Version of voxel distributor that sends the deepest LOD level at once
void deepestLevelVoxelDistributor(NodeList* nodeList,
NodeList::iterator& node,
VoxelNodeData* nodeData,
bool viewFrustumChanged) {
pthread_mutex_lock(&::treeLock);
int truePacketsSent = 0;
int trueBytesSent = 0;
// FOR NOW... node tells us if it wants to receive only view frustum deltas
bool wantDelta = viewFrustumChanged && nodeData->getWantDelta();
// If our packet already has content in it, then we must use the color choice of the waiting packet.
// If we're starting a fresh packet, then...
// If we're moving, and the client asked for low res, then we force monochrome, otherwise, use
// the clients requested color state.
bool wantColor = LOW_RES_MONO && nodeData->getWantLowResMoving() && viewFrustumChanged ? false : nodeData->getWantColor();
// If we have a packet waiting, and our desired want color, doesn't match the current waiting packets color
// then let's just send that waiting packet.
if (wantColor != nodeData->getCurrentPacketIsColor()) {
if (nodeData->isPacketWaiting()) {
if (::debugVoxelSending) {
printf("wantColor=%s --- SENDING PARTIAL PACKET! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
} else {
if (::debugVoxelSending) {
printf("wantColor=%s --- FIXING HEADER! nodeData->getCurrentPacketIsColor()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()));
}
nodeData->resetVoxelPacket();
}
}
if (::debugVoxelSending) {
printf("wantColor=%s getCurrentPacketIsColor()=%s, viewFrustumChanged=%s, getWantLowResMoving()=%s\n",
debug::valueOf(wantColor), debug::valueOf(nodeData->getCurrentPacketIsColor()),
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->getWantLowResMoving()));
}
const ViewFrustum* lastViewFrustum = wantDelta ? &nodeData->getLastKnownViewFrustum() : NULL;
if (::debugVoxelSending) {
printf("deepestLevelVoxelDistributor() viewFrustumChanged=%s, nodeBag.isEmpty=%s, viewSent=%s\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()),
debug::valueOf(nodeData->getViewSent())
);
}
// If the current view frustum has changed OR we have nothing to send, then search against
// the current view frustum for things to send.
if (viewFrustumChanged || nodeData->nodeBag.isEmpty()) {
uint64_t now = usecTimestampNow();
if (::debugVoxelSending) {
printf("(viewFrustumChanged=%s || nodeData->nodeBag.isEmpty() =%s)...\n",
debug::valueOf(viewFrustumChanged), debug::valueOf(nodeData->nodeBag.isEmpty()));
if (nodeData->getLastTimeBagEmpty() > 0) {
float elapsedSceneSend = (now - nodeData->getLastTimeBagEmpty()) / 1000000.0f;
if (viewFrustumChanged) {
printf("viewFrustumChanged resetting after elapsed time to send scene = %f seconds", elapsedSceneSend);
} else {
printf("elapsed time to send scene = %f seconds", elapsedSceneSend);
}
printf(" [occlusionCulling:%s, wantDelta:%s, wantColor:%s ]\n",
debug::valueOf(nodeData->getWantOcclusionCulling()), debug::valueOf(wantDelta),
debug::valueOf(wantColor));
}
}
// if our view has changed, we need to reset these things...
if (viewFrustumChanged) {
if (::dumpVoxelsOnMove) {
nodeData->nodeBag.deleteAll();
}
nodeData->map.erase();
}
if (!viewFrustumChanged && !nodeData->getWantDelta()) {
// only set our last sent time if we weren't resetting due to frustum change
uint64_t now = usecTimestampNow();
nodeData->setLastTimeBagEmpty(now);
}
nodeData->stats.sceneCompleted();
if (::displayVoxelStats) {
nodeData->stats.printDebugDetails();
}
// start tracking our stats
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) && nodeData->getViewFrustumJustStoppedChanging();
// If we're starting a full scene, then definitely we want to empty the nodeBag
if (isFullScene) {
nodeData->nodeBag.deleteAll();
}
nodeData->stats.sceneStarted(isFullScene, viewFrustumChanged, ::serverTree.rootNode, ::jurisdiction);
// This is the start of "resending" the scene.
nodeData->nodeBag.insert(serverTree.rootNode);
}
// If we have something in our nodeBag, then turn them into packets and send them out...
if (!nodeData->nodeBag.isEmpty()) {
static unsigned char tempOutputBuffer[MAX_VOXEL_PACKET_SIZE - 1]; // save on allocs by making this static
int bytesWritten = 0;
int packetsSentThisInterval = 0;
uint64_t start = usecTimestampNow();
bool shouldSendEnvironments = ::sendEnvironments && shouldDo(ENVIRONMENT_SEND_INTERVAL_USECS, VOXEL_SEND_INTERVAL_USECS);
while (packetsSentThisInterval < PACKETS_PER_CLIENT_PER_INTERVAL - (shouldSendEnvironments ? 1 : 0)) {
// Check to see if we're taking too long, and if so bail early...
uint64_t now = usecTimestampNow();
long elapsedUsec = (now - start);
long elapsedUsecPerPacket = (truePacketsSent == 0) ? 0 : (elapsedUsec / truePacketsSent);
long usecRemaining = (VOXEL_SEND_INTERVAL_USECS - elapsedUsec);
if (elapsedUsecPerPacket + SENDING_TIME_TO_SPARE > usecRemaining) {
if (::debugVoxelSending) {
printf("packetLoop() usecRemaining=%ld bailing early took %ld usecs to generate %d bytes in %d packets (%ld usec avg), %d nodes still to send\n",
usecRemaining, elapsedUsec, trueBytesSent, truePacketsSent, elapsedUsecPerPacket,
nodeData->nodeBag.count());
}
break;
}
if (!nodeData->nodeBag.isEmpty()) {
VoxelNode* subTree = nodeData->nodeBag.extract();
bool wantOcclusionCulling = nodeData->getWantOcclusionCulling();
CoverageMap* coverageMap = wantOcclusionCulling ? &nodeData->map : IGNORE_COVERAGE_MAP;
int boundaryLevelAdjust = viewFrustumChanged && nodeData->getWantLowResMoving()
? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST;
bool isFullScene = (!viewFrustumChanged || !nodeData->getWantDelta()) &&
nodeData->getViewFrustumJustStoppedChanging();
EncodeBitstreamParams params(INT_MAX, &nodeData->getCurrentViewFrustum(), wantColor,
WANT_EXISTS_BITS, DONT_CHOP, wantDelta, lastViewFrustum,
wantOcclusionCulling, coverageMap, boundaryLevelAdjust,
nodeData->getLastTimeBagEmpty(),
isFullScene, &nodeData->stats, ::jurisdiction);
nodeData->stats.encodeStarted();
bytesWritten = serverTree.encodeTreeBitstream(subTree, &tempOutputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1,
nodeData->nodeBag, params);
nodeData->stats.encodeStopped();
if (nodeData->getAvailable() >= bytesWritten) {
nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten);
} else {
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
packetsSentThisInterval++;
nodeData->resetVoxelPacket();
nodeData->writeToPacket(&tempOutputBuffer[0], bytesWritten);
}
} else {
if (nodeData->isPacketWaiting()) {
handlePacketSend(nodeList, node, nodeData, trueBytesSent, truePacketsSent);
nodeData->resetVoxelPacket();
}
packetsSentThisInterval = PACKETS_PER_CLIENT_PER_INTERVAL; // done for now, no nodes left
}
}
// send the environment packet
if (shouldSendEnvironments) {
int numBytesPacketHeader = populateTypeAndVersion(tempOutputBuffer, PACKET_TYPE_ENVIRONMENT_DATA);
int envPacketLength = numBytesPacketHeader;
int environmentsToSend = ::sendMinimalEnvironment ? 1 : sizeof(environmentData) / sizeof(EnvironmentData);
for (int i = 0; i < environmentsToSend; i++) {
envPacketLength += environmentData[i].getBroadcastData(tempOutputBuffer + envPacketLength);
}
nodeList->getNodeSocket()->send(node->getActiveSocket(), tempOutputBuffer, envPacketLength);
trueBytesSent += envPacketLength;
truePacketsSent++;
}
uint64_t end = usecTimestampNow();
int elapsedmsec = (end - start)/1000;
if (elapsedmsec > 100) {
if (elapsedmsec > 1000) {
int elapsedsec = (end - start)/1000000;
printf("WARNING! packetLoop() took %d seconds to generate %d bytes in %d packets %d nodes still to send\n",
elapsedsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
} else {
printf("WARNING! packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
} else if (::debugVoxelSending) {
printf("packetLoop() took %d milliseconds to generate %d bytes in %d packets, %d nodes still to send\n",
elapsedmsec, trueBytesSent, truePacketsSent, nodeData->nodeBag.count());
}
// if after sending packets we've emptied our bag, then we want to remember that we've sent all
// the voxels from the current view frustum
if (nodeData->nodeBag.isEmpty()) {
nodeData->updateLastKnownViewFrustum();
nodeData->setViewSent(true);
if (::debugVoxelSending) {
nodeData->map.printStats();
}
nodeData->map.erase(); // It would be nice if we could save this, and only reset it when the view frustum changes
}
} // end if bag wasn't empty, and so we sent stuff...
pthread_mutex_unlock(&::treeLock);
}
void* distributeVoxelsToListeners(void* args) {
NodeList* nodeList = NodeList::getInstance();
timeval lastSendTime;
while (true) {
gettimeofday(&lastSendTime, NULL);
for (NodeList::iterator node = nodeList->begin(); node != nodeList->end(); node++) {
VoxelNodeData* nodeData = (VoxelNodeData*) node->getLinkedData();
// Sometimes the node data has not yet been linked, in which case we can't really do anything
if (nodeData) {
bool viewFrustumChanged = nodeData->updateCurrentViewFrustum();
if (::debugVoxelSending) {
printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged));
}
deepestLevelVoxelDistributor(nodeList, node, nodeData, viewFrustumChanged);
}
}
// dynamically sleep until we need to fire off the next set of voxels
int usecToSleep = VOXEL_SEND_INTERVAL_USECS - (usecTimestampNow() - usecTimestamp(&lastSendTime));
if (usecToSleep > 0) {
usleep(usecToSleep);
} else {
if (::debugVoxelSending) {
std::cout << "Last send took too much time, not sleeping!\n";
}
}
}
pthread_exit(0);
}
NodeWatcher nodeWatcher; // used to cleanup AGENT data when agents are killed
void attachVoxelNodeDataToNode(Node* newNode) {
if (newNode->getLinkedData() == NULL) {
@ -420,6 +132,9 @@ int main(int argc, const char * argv[]) {
NodeList* nodeList = NodeList::createInstance(NODE_TYPE_VOXEL_SERVER, listenPort);
setvbuf(stdout, NULL, _IOLBF, 0);
// tell our NodeList about our desire to get notifications
nodeList->addHook(&nodeWatcher);
// Handle Local Domain testing with the --local command line
const char* local = "--local";
@ -462,7 +177,7 @@ int main(int argc, const char * argv[]) {
}
printf("wantVoxelPersist=%s\n", debug::valueOf(::wantVoxelPersist));
// if we want Voxel Persistance, load the local file now...
// if we want Voxel Persistence, load the local file now...
bool persistantFileRead = false;
if (::wantVoxelPersist) {
@ -532,9 +247,6 @@ int main(int argc, const char * argv[]) {
environmentData[2].setAtmosphereInnerRadius(0.1875f * TREE_SCALE);
environmentData[2].setAtmosphereOuterRadius(0.1875f * TREE_SCALE * 1.05f);
environmentData[2].setScatteringWavelengths(glm::vec3(0.475f, 0.570f, 0.650f)); // swaps red and blue
pthread_t sendVoxelThread;
pthread_create(&sendVoxelThread, NULL, distributeVoxelsToListeners, NULL);
sockaddr senderAddress;
@ -597,9 +309,6 @@ int main(int argc, const char * argv[]) {
}
}
pthread_join(sendVoxelThread, NULL);
pthread_mutex_destroy(&::treeLock);
if (::jurisdiction) {
delete ::jurisdiction;
}
@ -618,6 +327,11 @@ int main(int argc, const char * argv[]) {
::voxelPersistThread->terminate();
delete ::voxelPersistThread;
}
// tell our NodeList we're done with notifications
nodeList->removeHook(&nodeWatcher);
pthread_mutex_destroy(&::treeLock);
return 0;
}