Merge pull request #8934 from howard-stearns/more-reliable-crowds

More reliable crowds

scripts/developer/tests/performance/crowd-agent.js

@@ -16,7 +16,7 @@
 
 var MESSAGE_CHANNEL = "io.highfidelity.summon-crowd";
 
-print('crowd-agent version 3');
+print('crowd-agent version 4');
 
 /* Observations:
 - File urls for AC scripts silently fail. Use a local server (e.g., python SimpleHTTPServer) for development.
@@ -34,19 +34,56 @@ function getSound(data, callback) { // callback(sound) when downloaded (which ma
     if (sound.downloaded) {
         return callback(sound);
     }
-    sound.ready.connect(function () { callback(sound); });
+    function onDownloaded() {
+        sound.ready.disconnect(onDownloaded);
+        callback(sound);
+    }
+    sound.ready.connect(onDownloaded);
 }
 function onFinishedPlaying() {
     messageSend({key: 'finishedSound'});
 }
 
 var attachment;
+var stopper;
+function clearStopper() {
+    if (!stopper) {
+        return;
+    }
+    Script.clearTimeout(stopper);
+    stopper = null;
+}
+function stopAgent(parameters) {
+    function stop() {
+        clearStopper();
+        if (attachment) {
+            Avatar.detachOne(attachment.modelURL, attachment.jointName);
+            attachment = undefined;
+        }
+        Agent.isListeningToAudioStream = false;
+        Agent.isAvatar = false;
+        print('crowd-agent stopped', JSON.stringify(parameters), JSON.stringify(Agent));
+    }
+    // Shutting down lots of agents at once can be hard on other parts of the system. (See fogbugz 2095.)
+    // For now, accept a parameter to delay for the given number of milliseconds before stopping.
+    // (We cannot count on summoning scripts to spread out the STOP messages, because they might be doing so
+    // on scriptEnding, in which case they are not allowed to create new delays.)
+    if (parameters.delay) {
+        if (!stopper) { // Let the first stopper do the deed.
+            stopper = Script.setTimeout(stop, parameters.delay);
+        }
+    } else {
+        stop();
+    }
+}
+
 
 var MILLISECONDS_IN_SECOND = 1000;
 function startAgent(parameters) { // Can also be used to update.
     print('crowd-agent starting params', JSON.stringify(parameters), JSON.stringify(Agent));
+    clearStopper();
+    var wasOff = !Agent.isAvatar;
     Agent.isAvatar = true;
-    Agent.isListeningToAudioStream = true; // Send silence when not chattering.
     if (parameters.position) {
         Avatar.position = parameters.position;
     }
@@ -56,6 +93,11 @@ function startAgent(parameters) { // Can also be used to update.
     if (parameters.skeletonModelURL) {
         Avatar.skeletonModelURL = parameters.skeletonModelURL;
     }
+    if (parameters.listen != undefined) {
+        Agent.isListeningToAudioStream = parameters.listen; // Send silence when not chattering.
+    } else if (wasOff) {
+        Agent.isListeningToAudioStream = true;
+    }
     if (parameters.soundData) {
         getSound(parameters.soundData, function (sound) {
             Script.setTimeout(onFinishedPlaying, sound.duration * MILLISECONDS_IN_SECOND);
@@ -74,14 +116,6 @@ function startAgent(parameters) { // Can also be used to update.
     }
     print('crowd-agent avatars started');
 }
-function stopAgent(parameters) {
-    if (attachment) {
-        Avatar.detachOne(attachment.modelURL, attachment.jointName);
-        attachment = undefined;
-    }
-    Agent.isAvatar = false;
-    print('crowd-agent stopped', JSON.stringify(parameters), JSON.stringify(Agent));
-}
 
 function messageHandler(channel, messageString, senderID) {
     if (channel !== MESSAGE_CHANNEL) {

scripts/developer/tests/performance/domain-check.js

@@ -21,6 +21,10 @@ var NOMINAL_LOAD_TIME = 30;         // seconds
 var MAXIMUM_LOAD_TIME = NOMINAL_LOAD_TIME * 2;
 var MINIMUM_AVATARS = 25; // changeable by prompt
 
+// If we add or remove things too quickly, we get problems (e.g., audio, fogbugz 2095).
+// For now, spread them out this timing apart.
+var SPREAD_TIME_MS = 500;
+
 var DENSITY = 0.3; // square meters per person. Some say 10 sq ft is arm's length (0.9m^2), 4.5 is crowd (0.4m^2), 2.5 is mosh pit (0.2m^2).
 var SOUND_DATA = {url: "http://hifi-content.s3.amazonaws.com/howard/sounds/piano1.wav"};
 var AVATARS_CHATTERING_AT_ONCE = 4; // How many of the agents should we request to play SOUND at once.
@@ -34,7 +38,7 @@ var ANIMATION_DATA = {
     "loopFlag": true
 };
 
-var version = 3;
+var version = 4;
 function debug() {
     print.apply(null, [].concat.apply(['hrs fixme', version], [].map.call(arguments, JSON.stringify)));
 }
@@ -86,6 +90,7 @@ function nextAfter(array, id) { // Wrapping next element in array after id.
 
 var summonedAgents = [];
 var chattering = [];
+var accumulatedDelay = 0;
 var MESSAGE_CHANNEL = "io.highfidelity.summon-crowd";
 function messageSend(message) {
     Messages.sendMessage(MESSAGE_CHANNEL, JSON.stringify(message));
@@ -106,6 +111,7 @@ function messageHandler(channel, messageString, senderID) {
     }
     switch (message.key) {
     case "hello":
+        Script.setTimeout(function () {
             // There can be avatars we've summoned that do not yet appear in the AvatarList.
             avatarIdentifiers = without(AvatarList.getAvatarIdentifiers(), summonedAgents);
             debug('present', avatarIdentifiers, summonedAgents);
@@ -121,10 +127,13 @@ function messageHandler(channel, messageString, senderID) {
                     position: Vec3.sum(MyAvatar.position, {x: coord(), y: 0, z: coord()}),
                     orientation: Quat.fromPitchYawRollDegrees(0, Quat.safeEulerAngles(MyAvatar.orientation).y + (turnSpread * (Math.random() - 0.5)), 0),
                     soundData: chatter && SOUND_DATA,
+                    listen: true,
                     skeletonModelURL: "http://hifi-content.s3.amazonaws.com/howard/resources/meshes/defaultAvatar_full.fst",
                     animationData: ANIMATION_DATA
                 });
             }
+        }, accumulatedDelay);
+        accumulatedDelay += SPREAD_TIME_MS; // assume we'll get all the hello respsponses more or less together.
         break;
     case "finishedSound": // Give someone else a chance.
         chattering = without(chattering, [senderID]);
@@ -147,13 +156,15 @@ Messages.subscribe(MESSAGE_CHANNEL);
 Messages.messageReceived.connect(messageHandler);
 Script.scriptEnding.connect(function () {
     debug('stopping agents', summonedAgents);
-    summonedAgents.forEach(function (id) { messageSend({key: 'STOP', rcpt: id}); });
+    Messages.messageReceived.disconnect(messageHandler); // don't respond to any messages during shutdown
+    accumulatedDelay = 0;
+    summonedAgents.forEach(function (id) {
+        messageSend({key: 'STOP', rcpt: id, delay: accumulatedDelay});
+        accumulatedDelay += SPREAD_TIME_MS;
+    });
     debug('agents stopped');
-    Script.setTimeout(function () {
-        Messages.messageReceived.disconnect(messageHandler);
     Messages.unsubscribe(MESSAGE_CHANNEL);
     debug('unsubscribed');
-    }, 500);
 });
 
 var fail = false, results = "";
@@ -257,7 +268,7 @@ function doRender(continuation) {
     }
 
     config.newStats.connect(onNewStats);
-    startTwirl(720, 1, 15, 0.08, function () {
+    startTwirl(720, 1, 20, 0.08, function () {
         var end = Date.now();
         config.newStats.disconnect(onNewStats);
         addResult('frame rate', 1000 * frames / (end - start),
@@ -270,7 +281,7 @@ function doRender(continuation) {
 http://hifi-content.s3.amazonaws.com/howard/scripts/tests/performance/crowd-agent.js?v=3\n\
 on your domain server.";
             } else if (total < MINIMUM_AVATARS) {
-                fail = "FAIL: Only " + summonedAgents.length + " avatars reported. Missing " + (MINIMUM_AVATARS - total) + ".";
+                fail = "FAIL: Only " + summonedAgents.length + " agents reported. Now missing " + (MINIMUM_AVATARS - total) + " avatars, total.";
             }
         }
         continuation();

scripts/developer/tests/performance/summon.js

@@ -13,19 +13,26 @@
 //
 // See crowd-agent.js
 
-var version = 1;
+var version = 2;
 var label = "summon";
 function debug() {
     print.apply(null, [].concat.apply([label, version], [].map.call(arguments, JSON.stringify)));
 }
+
 var MINIMUM_AVATARS = 25; // We will summon agents to produce this many total. (Of course, there might not be enough agents.)
+var N_LISTENING = MINIMUM_AVATARS - 1;
+var AVATARS_CHATTERING_AT_ONCE = 4; // How many of the agents should we request to play SOUND_DATA at once.
+
+// If we add or remove things too quickly, we get problems (e.g., audio, fogbugz 2095).
+// For now, spread them out this timing apart.
+var SPREAD_TIME_MS = 500;
+
 var DENSITY = 0.3; // square meters per person. Some say 10 sq ft is arm's length (0.9m^2), 4.5 is crowd (0.4m^2), 2.5 is mosh pit (0.2m^2).
-var SOUND_DATA = {url: "http://howard-stearns.github.io/models/sounds/piano1.wav"};
+var SOUND_DATA = {url: "http://hifi-content.s3.amazonaws.com/howard/sounds/piano1.wav"};
-var AVATARS_CHATTERING_AT_ONCE = 4; // How many of the agents should we request to play SOUND at once.
 var NEXT_SOUND_SPREAD = 500; // millisecond range of how long to wait after one sound finishes, before playing the next
 var ANIMATION_DATA = {
-    "url": "http://howard-stearns.github.io/models/resources/avatar/animations/idle.fbx",
+    "url": "http://hifi-content.s3.amazonaws.com/howard/resources/avatar/animations/idle.fbx",
-    // "url": "http://howard-stearns.github.io/models/resources/avatar/animations/walk_fwd.fbx", // alternative example
+    // "url": "http://hifi-content.s3.amazonaws.com/howard/resources/avatar/animations/walk_fwd.fbx", // alternative example
     "startFrame": 0.0,
     "endFrame": 300.0,
     "timeScale": 1.0,
@@ -45,6 +52,8 @@ function nextAfter(array, id) { // Wrapping next element in array after id.
 
 var summonedAgents = [];
 var chattering = [];
+var nListening = 0;
+var accumulatedDelay = 0;
 var MESSAGE_CHANNEL = "io.highfidelity.summon-crowd";
 function messageSend(message) {
     Messages.sendMessage(MESSAGE_CHANNEL, JSON.stringify(message));
@@ -65,14 +74,19 @@ function messageHandler(channel, messageString, senderID) {
     }
     switch (message.key) {
     case "hello":
+        Script.setTimeout(function () {
             // There can be avatars we've summoned that do not yet appear in the AvatarList.
             avatarIdentifiers = without(AvatarList.getAvatarIdentifiers(), summonedAgents);
             debug('present', avatarIdentifiers, summonedAgents);
             if ((summonedAgents.length + avatarIdentifiers.length) < MINIMUM_AVATARS ) {
                 var chatter = chattering.length < AVATARS_CHATTERING_AT_ONCE;
+                var listen = nListening < N_LISTENING;
                 if (chatter) {
                     chattering.push(senderID);
                 }
+                if (listen) {
+                    nListening++;
+                }
                 summonedAgents.push(senderID);
                 messageSend({
                     key: 'SUMMON',
@@ -80,10 +94,13 @@ function messageHandler(channel, messageString, senderID) {
                     position: Vec3.sum(MyAvatar.position, {x: coord(), y: 0, z: coord()}),
                     orientation: Quat.fromPitchYawRollDegrees(0, Quat.safeEulerAngles(MyAvatar.orientation).y + (turnSpread * (Math.random() - 0.5)), 0),
                     soundData: chatter && SOUND_DATA,
-                skeletonModelURL: "http://howard-stearns.github.io/models/resources/meshes/defaultAvatar_full.fst",
+                    listen: listen,
+                    skeletonModelURL: "http://hifi-content.s3.amazonaws.com/howard/resources/meshes/defaultAvatar_full.fst",
                     animationData: ANIMATION_DATA
                 });
             }
+        }, accumulatedDelay);
+        accumulatedDelay += SPREAD_TIME_MS; // assume we'll get all the hello respsponses more or less together.
         break;
     case "finishedSound": // Give someone else a chance.
         chattering = without(chattering, [senderID]);
@@ -99,20 +116,22 @@ function messageHandler(channel, messageString, senderID) {
         Window.alert("Someone else is summoning avatars.");
         break;
     default:
-        print("crowd-agent received unrecognized message:", messageString);
+        print("crowd summon.js received unrecognized message:", messageString);
     }
 }
 Messages.subscribe(MESSAGE_CHANNEL);
 Messages.messageReceived.connect(messageHandler);
 Script.scriptEnding.connect(function () {
     debug('stopping agents', summonedAgents);
-    summonedAgents.forEach(function (id) { messageSend({key: 'STOP', rcpt: id}); });
+    Messages.messageReceived.disconnect(messageHandler); // don't respond to any messages during shutdown
+    accumulatedDelay = 0;
+    summonedAgents.forEach(function (id) {
+        messageSend({key: 'STOP', rcpt: id, delay: accumulatedDelay});
+        accumulatedDelay += SPREAD_TIME_MS;
+    }); 
     debug('agents stopped');
-    Script.setTimeout(function () {
-        Messages.messageReceived.disconnect(messageHandler);
     Messages.unsubscribe(MESSAGE_CHANNEL);
     debug('unsubscribed');
-    }, 500);
 });
 
 messageSend({key: 'HELO'}); // Ask agents to report in now.
@@ -120,9 +139,9 @@ Script.setTimeout(function () {
     var total = AvatarList.getAvatarIdentifiers().length;
     if (0 === summonedAgents.length) {
         Window.alert("No agents reported.\n\Please run " + MINIMUM_AVATARS + " instances of\n\
-http://cdn.highfidelity.com/davidkelly/production/scripts/tests/performance/crowd-agent.js\n\
+http://hifi-content.s3.amazonaws.com/howard/scripts/tests/performance/crowd-agent.js\n\
 on your domain server.");
     } else if (total < MINIMUM_AVATARS) {
-        Window.alert("Only " + summonedAgents.length + " of the expected " + (MINIMUM_AVATARS - total) + " agents reported in.");
+        Window.alert("Only " + summonedAgents.length + " agents reported. Now missing " + (MINIMUM_AVATARS - total) + " avatars, total.");
     }
-}, 5000);
+}, MINIMUM_AVATARS * SPREAD_TIME_MS )