From c8349dda6eaa64dfac5f4bb27ca5aaf118aa4ba4 Mon Sep 17 00:00:00 2001 From: AlessandroSigna Date: Thu, 12 Nov 2015 18:54:10 -0800 Subject: [PATCH 01/36] added examples for messages synchronization --- .../entityScripts/synchronizerEntityScript.js | 87 +++++++++++++ examples/entityScripts/synchronizerMaster.js | 119 ++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 examples/entityScripts/synchronizerEntityScript.js create mode 100644 examples/entityScripts/synchronizerMaster.js diff --git a/examples/entityScripts/synchronizerEntityScript.js b/examples/entityScripts/synchronizerEntityScript.js new file mode 100644 index 0000000000..733cb19eb3 --- /dev/null +++ b/examples/entityScripts/synchronizerEntityScript.js @@ -0,0 +1,87 @@ +// +// synchronizerEntityScript.js +// examples/entityScripts +// +// Created by Alessandro Signa on 11/12/15. +// Copyright 2015 High Fidelity, Inc. +// + +// This script shows how to create a synchronized event between avatars trhough an entity. +// It works using the entity's userData: the master change its value and every client checks it every frame +// This entity prints a message when the event starts and when it ends. +// The client running synchronizerMaster.js is the event master and it decides when the event starts/ends by pressing a button. +// All the avatars in the area when the master presses the button will receive a message. +// + +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + + + + +(function() { + var insideArea = false; + var isJoiningTheEvent = false; + var _this; + + function update(){ + var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); + var valueToCheck = userData.myKey.valueToCheck; + if(valueToCheck && !isJoiningTheEvent){ + _this.sendMessage(); + }else if((!valueToCheck && isJoiningTheEvent) || (isJoiningTheEvent && !insideArea)){ + _this.stopMessage(); + } + + } + + function ParamsEntity() { + _this = this; + return; + } + + + ParamsEntity.prototype = { + preload: function(entityID) { + print('entity loaded') + this.entityID = entityID; + Script.update.connect(update); + }, + enterEntity: function(entityID) { + print("enterEntity("+entityID+")"); + var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); + var valueToCheck = userData.myKey.valueToCheck; + if(!valueToCheck){ + //i'm in the area in time (before the event starts) + insideArea = true; + } + change(entityID); + }, + leaveEntity: function(entityID) { + print("leaveEntity("+entityID+")"); + Entities.editEntity(entityID, { color: { red: 255, green: 190, blue: 20} }); + insideArea = false; + }, + + sendMessage: function(myID){ + if(insideArea && !isJoiningTheEvent){ + print("The event started"); + isJoiningTheEvent = true; + } + }, + + stopMessage: function(myID){ + if(isJoiningTheEvent){ + print("The event ended"); + isJoiningTheEvent = false; + } + } + } + + function change(entityID) { + Entities.editEntity(entityID, { color: { red: 255, green: 100, blue: 220} }); + } + + + return new ParamsEntity(); +}); diff --git a/examples/entityScripts/synchronizerMaster.js b/examples/entityScripts/synchronizerMaster.js new file mode 100644 index 0000000000..fd4001ff3d --- /dev/null +++ b/examples/entityScripts/synchronizerMaster.js @@ -0,0 +1,119 @@ +// +// synchronizerMaster.js +// examples/entityScripts +// +// Created by Alessandro Signa on 11/12/15. +// Copyright 2015 High Fidelity, Inc. +// +// Run this script to spawn a box (synchronizer) and drive the start/end of the event for anyone who is inside the box +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +var PARAMS_SCRIPT_URL = 'https://raw.githubusercontent.com/AlessandroSigna/hifi/27fbef3e873d11648faf0a592bb2314a90c71624/examples/entityScripts/synchronizerEntityScript.js'; + + +HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; +Script.include("../libraries/toolBars.js"); +Script.include("../libraries/utils.js"); + + + +var rotation = Quat.safeEulerAngles(Camera.getOrientation()); +rotation = Quat.fromPitchYawRollDegrees(0, rotation.y, 0); +var center = Vec3.sum(MyAvatar.position, Vec3.multiply(1, Quat.getFront(rotation))); + +var TOOL_ICON_URL = HIFI_PUBLIC_BUCKET + "images/tools/"; +var ALPHA_ON = 1.0; +var ALPHA_OFF = 0.7; +var COLOR_TOOL_BAR = { red: 0, green: 0, blue: 0 }; + +var toolBar = null; +var recordIcon; + + + +var isHappening = false; + +var testEntity = Entities.addEntity({ + name: 'paramsTestEntity', + dimensions: { + x: 2, + y: 1, + z: 2 + }, + type: 'Box', + position: center, + color: { + red: 255, + green: 255, + blue: 255 + }, + visible: true, + ignoreForCollisions: true, + script: PARAMS_SCRIPT_URL, + + userData: JSON.stringify({ + myKey: { + valueToCheck: false + } + }) +}); + + +setupToolBar(); + +function setupToolBar() { + if (toolBar != null) { + print("Multiple calls to setupToolBar()"); + return; + } + Tool.IMAGE_HEIGHT /= 2; + Tool.IMAGE_WIDTH /= 2; + + toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL); //put the button in the up-left corner + + toolBar.setBack(COLOR_TOOL_BAR, ALPHA_OFF); + + recordIcon = toolBar.addTool({ + imageURL: TOOL_ICON_URL + "recording-record.svg", + subImage: { x: 0, y: 0, width: Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT }, + x: 0, y: 0, + width: Tool.IMAGE_WIDTH, + height: Tool.IMAGE_HEIGHT, + alpha: MyAvatar.isPlaying() ? ALPHA_OFF : ALPHA_ON, + visible: true + }, true, isHappening); + +} + +function mousePressEvent(event) { + clickedOverlay = Overlays.getOverlayAtPoint({ x: event.x, y: event.y }); + if (recordIcon === toolBar.clicked(clickedOverlay, false)) { + if (!isHappening) { + print("I'm the event master. I want the event starts"); + isHappening = true; + setEntityCustomData("myKey", testEntity, {valueToCheck: true}); + + } else { + print("I want the event stops"); + isHappening = false; + setEntityCustomData("myKey", testEntity, {valueToCheck: false}); + + } + } +} + +Script.setTimeout(function() { + print('sending data to entity'); + Entities.callEntityMethod(testEntity, 'testParams', data); +}, 1500) + +function cleanup() { + Entities.deleteEntity(testEntity); +} + + + + Script.scriptEnding.connect(cleanup); + Controller.mousePressEvent.connect(mousePressEvent); \ No newline at end of file From 5b66416b8c1d1e88272e8d63a8bf1ded9a25be59 Mon Sep 17 00:00:00 2001 From: AlessandroSigna Date: Fri, 13 Nov 2015 15:03:36 -0800 Subject: [PATCH 02/36] added cleanup before delete --- .../entityScripts/synchronizerEntityScript.js | 25 +++++++++++-------- examples/entityScripts/synchronizerMaster.js | 8 +++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/examples/entityScripts/synchronizerEntityScript.js b/examples/entityScripts/synchronizerEntityScript.js index 733cb19eb3..82dd954381 100644 --- a/examples/entityScripts/synchronizerEntityScript.js +++ b/examples/entityScripts/synchronizerEntityScript.js @@ -24,16 +24,7 @@ var isJoiningTheEvent = false; var _this; - function update(){ - var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); - var valueToCheck = userData.myKey.valueToCheck; - if(valueToCheck && !isJoiningTheEvent){ - _this.sendMessage(); - }else if((!valueToCheck && isJoiningTheEvent) || (isJoiningTheEvent && !insideArea)){ - _this.stopMessage(); - } - - } + function ParamsEntity() { _this = this; @@ -42,10 +33,19 @@ ParamsEntity.prototype = { + update: function(){ + var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); + var valueToCheck = userData.myKey.valueToCheck; + if(valueToCheck && !isJoiningTheEvent){ + _this.sendMessage(); + }else if((!valueToCheck && isJoiningTheEvent) || (isJoiningTheEvent && !insideArea)){ + _this.stopMessage(); + } + }, preload: function(entityID) { print('entity loaded') this.entityID = entityID; - Script.update.connect(update); + Script.update.connect(_this.update); }, enterEntity: function(entityID) { print("enterEntity("+entityID+")"); @@ -75,6 +75,9 @@ print("The event ended"); isJoiningTheEvent = false; } + }, + clean: function(entityID) { + Script.update.disconnect(_this.update); } } diff --git a/examples/entityScripts/synchronizerMaster.js b/examples/entityScripts/synchronizerMaster.js index fd4001ff3d..e5af46c461 100644 --- a/examples/entityScripts/synchronizerMaster.js +++ b/examples/entityScripts/synchronizerMaster.js @@ -10,7 +10,7 @@ // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -var PARAMS_SCRIPT_URL = 'https://raw.githubusercontent.com/AlessandroSigna/hifi/27fbef3e873d11648faf0a592bb2314a90c71624/examples/entityScripts/synchronizerEntityScript.js'; +var PARAMS_SCRIPT_URL = 'https://raw.githubusercontent.com/AlessandroSigna/hifi/05aa1d4ce49c719353007c245ae77ef2d2a8fc36/examples/entityScripts/synchronizerEntityScript.js'; HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; @@ -104,12 +104,10 @@ function mousePressEvent(event) { } } -Script.setTimeout(function() { - print('sending data to entity'); - Entities.callEntityMethod(testEntity, 'testParams', data); -}, 1500) function cleanup() { + toolBar.cleanup(); + Entities.callEntityMethod(testEntity, 'clean'); //have to call this before deleting to avoid the JSON warnings Entities.deleteEntity(testEntity); } From 5e395713a677a791133255c939c02e7eeeda6c8f Mon Sep 17 00:00:00 2001 From: AlessandroSigna Date: Fri, 13 Nov 2015 16:33:54 -0800 Subject: [PATCH 03/36] fix script url --- examples/entityScripts/synchronizerMaster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/entityScripts/synchronizerMaster.js b/examples/entityScripts/synchronizerMaster.js index e5af46c461..8b6c8c2b8b 100644 --- a/examples/entityScripts/synchronizerMaster.js +++ b/examples/entityScripts/synchronizerMaster.js @@ -10,7 +10,7 @@ // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -var PARAMS_SCRIPT_URL = 'https://raw.githubusercontent.com/AlessandroSigna/hifi/05aa1d4ce49c719353007c245ae77ef2d2a8fc36/examples/entityScripts/synchronizerEntityScript.js'; +var PARAMS_SCRIPT_URL = Script.resolvePath('synchronizerEntityScript.js'); HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; From cb26fc67fc348071fe22b99069f3843dd6152410 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Fri, 13 Nov 2015 17:56:14 -0800 Subject: [PATCH 04/36] Move recording interface back to float/seconds --- examples/utilities/record/recorder.js | 100 +++--- interface/resources/qml/RecorderDialog.qml | 105 ++++++ .../resources/qml/controls/ButtonAwesome.qml | 21 ++ interface/resources/qml/controls/Player.qml | 89 +++++ interface/src/Application.cpp | 15 +- interface/src/avatar/Head.cpp | 5 +- interface/src/avatar/MyAvatar.cpp | 123 +------ interface/src/avatar/MyAvatar.h | 12 - interface/src/avatar/SkeletonModel.cpp | 5 +- .../scripting/RecordingScriptingInterface.cpp | 328 ++++++++++++++++++ .../scripting/RecordingScriptingInterface.h | 80 +++++ interface/src/ui/RecorderDialog.cpp | 22 ++ interface/src/ui/RecorderDialog.h | 28 ++ libraries/avatars/src/AvatarData.cpp | 212 ++--------- libraries/avatars/src/AvatarData.h | 32 +- libraries/gl/src/gl/OffscreenQmlSurface.cpp | 4 +- libraries/recording/src/recording/Clip.cpp | 23 +- libraries/recording/src/recording/Clip.h | 18 +- libraries/recording/src/recording/Deck.cpp | 136 ++++++-- libraries/recording/src/recording/Deck.h | 44 ++- libraries/recording/src/recording/Forward.h | 10 +- libraries/recording/src/recording/Frame.cpp | 27 ++ libraries/recording/src/recording/Frame.h | 32 +- .../recording/src/recording/Recorder.cpp | 22 +- libraries/recording/src/recording/Recorder.h | 19 +- .../recording/src/recording/impl/ArrayClip.h | 100 ++++++ .../src/recording/impl/BufferClip.cpp | 87 ++--- .../recording/src/recording/impl/BufferClip.h | 23 +- .../recording/src/recording/impl/FileClip.cpp | 128 +++---- .../recording/src/recording/impl/FileClip.h | 46 +-- .../src/recording/impl/OffsetClip.cpp | 25 +- .../recording/src/recording/impl/OffsetClip.h | 14 +- .../src/recording/impl/WrapperClip.cpp | 8 +- .../src/recording/impl/WrapperClip.h | 7 +- 34 files changed, 1270 insertions(+), 680 deletions(-) create mode 100644 interface/resources/qml/RecorderDialog.qml create mode 100644 interface/resources/qml/controls/ButtonAwesome.qml create mode 100644 interface/resources/qml/controls/Player.qml create mode 100644 interface/src/scripting/RecordingScriptingInterface.cpp create mode 100644 interface/src/scripting/RecordingScriptingInterface.h create mode 100644 interface/src/ui/RecorderDialog.cpp create mode 100644 interface/src/ui/RecorderDialog.h create mode 100644 libraries/recording/src/recording/impl/ArrayClip.h diff --git a/examples/utilities/record/recorder.js b/examples/utilities/record/recorder.js index 40476626e8..d08cdd68f3 100644 --- a/examples/utilities/record/recorder.js +++ b/examples/utilities/record/recorder.js @@ -15,11 +15,11 @@ Script.include("../../libraries/toolBars.js"); var recordingFile = "recording.rec"; function setPlayerOptions() { - MyAvatar.setPlayFromCurrentLocation(true); - MyAvatar.setPlayerUseDisplayName(false); - MyAvatar.setPlayerUseAttachments(false); - MyAvatar.setPlayerUseHeadModel(false); - MyAvatar.setPlayerUseSkeletonModel(false); + Recording.setPlayFromCurrentLocation(true); + Recording.setPlayerUseDisplayName(false); + Recording.setPlayerUseAttachments(false); + Recording.setPlayerUseHeadModel(false); + Recording.setPlayerUseSkeletonModel(false); } var windowDimensions = Controller.getViewportDimensions(); @@ -64,16 +64,16 @@ function setupToolBar() { x: 0, y: 0, width: Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT, - alpha: MyAvatar.isPlaying() ? ALPHA_OFF : ALPHA_ON, + alpha: Recording.isPlaying() ? ALPHA_OFF : ALPHA_ON, visible: true - }, true, !MyAvatar.isRecording()); + }, true, !Recording.isRecording()); var playLoopWidthFactor = 1.65; playIcon = toolBar.addTool({ imageURL: TOOL_ICON_URL + "play-pause.svg", width: playLoopWidthFactor * Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT, - alpha: (MyAvatar.isRecording() || MyAvatar.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, + alpha: (Recording.isRecording() || Recording.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, visible: true }, false); @@ -82,7 +82,7 @@ function setupToolBar() { subImage: { x: 0, y: 0, width: playLoopWidthFactor * Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT }, width: playLoopWidthFactor * Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT, - alpha: (MyAvatar.isRecording() || MyAvatar.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, + alpha: (Recording.isRecording() || Recording.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, visible: true }, false); @@ -93,7 +93,7 @@ function setupToolBar() { imageURL: TOOL_ICON_URL + "recording-save.svg", width: Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT, - alpha: (MyAvatar.isRecording() || MyAvatar.isPlaying() || MyAvatar.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, + alpha: (Recording.isRecording() || Recording.isPlaying() || Recording.playerLength() === 0) ? ALPHA_OFF : ALPHA_ON, visible: true }, false); @@ -101,7 +101,7 @@ function setupToolBar() { imageURL: TOOL_ICON_URL + "recording-upload.svg", width: Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT, - alpha: (MyAvatar.isRecording() || MyAvatar.isPlaying()) ? ALPHA_OFF : ALPHA_ON, + alpha: (Recording.isRecording() || Recording.isPlaying()) ? ALPHA_OFF : ALPHA_ON, visible: true }, false); } @@ -147,23 +147,23 @@ function setupTimer() { function updateTimer() { var text = ""; - if (MyAvatar.isRecording()) { - text = formatTime(MyAvatar.recorderElapsed()); + if (Recording.isRecording()) { + text = formatTime(Recording.recorderElapsed()); } else { - text = formatTime(MyAvatar.playerElapsed()) + " / " + - formatTime(MyAvatar.playerLength()); + text = formatTime(Recording.playerElapsed()) + " / " + + formatTime(Recording.playerLength()); } Overlays.editOverlay(timer, { text: text }) - toolBar.changeSpacing(text.length * 8 + ((MyAvatar.isRecording()) ? 15 : 0), spacing); + toolBar.changeSpacing(text.length * 8 + ((Recording.isRecording()) ? 15 : 0), spacing); - if (MyAvatar.isRecording()) { + if (Recording.isRecording()) { slider.pos = 1.0; - } else if (MyAvatar.playerLength() > 0) { - slider.pos = MyAvatar.playerElapsed() / MyAvatar.playerLength(); + } else if (Recording.playerLength() > 0) { + slider.pos = Recording.playerElapsed() / Recording.playerLength(); } Overlays.editOverlay(slider.foreground, { @@ -217,77 +217,77 @@ function moveUI() { function mousePressEvent(event) { clickedOverlay = Overlays.getOverlayAtPoint({ x: event.x, y: event.y }); - if (recordIcon === toolBar.clicked(clickedOverlay, false) && !MyAvatar.isPlaying()) { - if (!MyAvatar.isRecording()) { - MyAvatar.startRecording(); + if (recordIcon === toolBar.clicked(clickedOverlay, false) && !Recording.isPlaying()) { + if (!Recording.isRecording()) { + Recording.startRecording(); toolBar.selectTool(recordIcon, false); toolBar.setAlpha(ALPHA_OFF, playIcon); toolBar.setAlpha(ALPHA_OFF, playLoopIcon); toolBar.setAlpha(ALPHA_OFF, saveIcon); toolBar.setAlpha(ALPHA_OFF, loadIcon); } else { - MyAvatar.stopRecording(); + Recording.stopRecording(); toolBar.selectTool(recordIcon, true ); - MyAvatar.loadLastRecording(); + Recording.loadLastRecording(); toolBar.setAlpha(ALPHA_ON, playIcon); toolBar.setAlpha(ALPHA_ON, playLoopIcon); toolBar.setAlpha(ALPHA_ON, saveIcon); toolBar.setAlpha(ALPHA_ON, loadIcon); } - } else if (playIcon === toolBar.clicked(clickedOverlay) && !MyAvatar.isRecording()) { - if (MyAvatar.isPlaying()) { - MyAvatar.pausePlayer(); + } else if (playIcon === toolBar.clicked(clickedOverlay) && !Recording.isRecording()) { + if (Recording.isPlaying()) { + Recording.pausePlayer(); toolBar.setAlpha(ALPHA_ON, recordIcon); toolBar.setAlpha(ALPHA_ON, saveIcon); toolBar.setAlpha(ALPHA_ON, loadIcon); - } else if (MyAvatar.playerLength() > 0) { + } else if (Recording.playerLength() > 0) { setPlayerOptions(); - MyAvatar.setPlayerLoop(false); - MyAvatar.startPlaying(); + Recording.setPlayerLoop(false); + Recording.startPlaying(); toolBar.setAlpha(ALPHA_OFF, recordIcon); toolBar.setAlpha(ALPHA_OFF, saveIcon); toolBar.setAlpha(ALPHA_OFF, loadIcon); watchStop = true; } - } else if (playLoopIcon === toolBar.clicked(clickedOverlay) && !MyAvatar.isRecording()) { - if (MyAvatar.isPlaying()) { - MyAvatar.pausePlayer(); + } else if (playLoopIcon === toolBar.clicked(clickedOverlay) && !Recording.isRecording()) { + if (Recording.isPlaying()) { + Recording.pausePlayer(); toolBar.setAlpha(ALPHA_ON, recordIcon); toolBar.setAlpha(ALPHA_ON, saveIcon); toolBar.setAlpha(ALPHA_ON, loadIcon); - } else if (MyAvatar.playerLength() > 0) { + } else if (Recording.playerLength() > 0) { setPlayerOptions(); - MyAvatar.setPlayerLoop(true); - MyAvatar.startPlaying(); + Recording.setPlayerLoop(true); + Recording.startPlaying(); toolBar.setAlpha(ALPHA_OFF, recordIcon); toolBar.setAlpha(ALPHA_OFF, saveIcon); toolBar.setAlpha(ALPHA_OFF, loadIcon); } } else if (saveIcon === toolBar.clicked(clickedOverlay)) { - if (!MyAvatar.isRecording() && !MyAvatar.isPlaying() && MyAvatar.playerLength() != 0) { + if (!Recording.isRecording() && !Recording.isPlaying() && Recording.playerLength() != 0) { recordingFile = Window.save("Save recording to file", ".", "Recordings (*.hfr)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { - MyAvatar.saveRecording(recordingFile); + Recording.saveRecording(recordingFile); } } } else if (loadIcon === toolBar.clicked(clickedOverlay)) { - if (!MyAvatar.isRecording() && !MyAvatar.isPlaying()) { + if (!Recording.isRecording() && !Recording.isPlaying()) { recordingFile = Window.browse("Load recorcding from file", ".", "Recordings (*.hfr *.rec *.HFR *.REC)"); if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { - MyAvatar.loadRecording(recordingFile); + Recording.loadRecording(recordingFile); } - if (MyAvatar.playerLength() > 0) { + if (Recording.playerLength() > 0) { toolBar.setAlpha(ALPHA_ON, playIcon); toolBar.setAlpha(ALPHA_ON, playLoopIcon); toolBar.setAlpha(ALPHA_ON, saveIcon); } } - } else if (MyAvatar.playerLength() > 0 && + } else if (Recording.playerLength() > 0 && slider.x < event.x && event.x < slider.x + slider.w && slider.y < event.y && event.y < slider.y + slider.h) { isSliding = true; slider.pos = (event.x - slider.x) / slider.w; - MyAvatar.setPlayerTime(slider.pos * MyAvatar.playerLength()); + Recording.setPlayerTime(slider.pos * Recording.playerLength()); } } var isSliding = false; @@ -296,10 +296,10 @@ function mouseMoveEvent(event) { if (isSliding) { slider.pos = (event.x - slider.x) / slider.w; if (slider.pos < 0.0 || slider.pos > 1.0) { - MyAvatar.stopPlaying(); + Recording.stopPlaying(); slider.pos = 0.0; } - MyAvatar.setPlayerTime(slider.pos * MyAvatar.playerLength()); + Recording.setPlayerTime(slider.pos * Recording.playerLength()); } } @@ -316,7 +316,7 @@ function update() { updateTimer(); - if (watchStop && !MyAvatar.isPlaying()) { + if (watchStop && !Recording.isPlaying()) { watchStop = false; toolBar.setAlpha(ALPHA_ON, recordIcon); toolBar.setAlpha(ALPHA_ON, saveIcon); @@ -325,11 +325,11 @@ function update() { } function scriptEnding() { - if (MyAvatar.isRecording()) { - MyAvatar.stopRecording(); + if (Recording.isRecording()) { + Recording.stopRecording(); } - if (MyAvatar.isPlaying()) { - MyAvatar.stopPlaying(); + if (Recording.isPlaying()) { + Recording.stopPlaying(); } toolBar.cleanup(); Overlays.deleteOverlay(timer); diff --git a/interface/resources/qml/RecorderDialog.qml b/interface/resources/qml/RecorderDialog.qml new file mode 100644 index 0000000000..4f197846aa --- /dev/null +++ b/interface/resources/qml/RecorderDialog.qml @@ -0,0 +1,105 @@ +// +// Created by Bradley Austin Davis on 2015/11/14 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +import Hifi 1.0 +import QtQuick 2.4 +import "controls" +import "styles" + +VrDialog { + id: root + HifiConstants { id: hifi } + + property real spacing: hifi.layout.spacing + property real outerSpacing: hifi.layout.spacing * 2 + + objectName: "RecorderDialog" + + destroyOnInvisible: false + destroyOnCloseButton: false + + contentImplicitWidth: recorderDialog.width + contentImplicitHeight: recorderDialog.height + + RecorderDialog { + id: recorderDialog + x: root.clientX; y: root.clientY + width: 800 + height: 128 + signal play() + signal rewind() + + onPlay: { + console.log("Pressed play") + player.isPlaying = !player.isPlaying + } + + onRewind: { + console.log("Pressed rewind") + player.position = 0 + } + + Row { + height: 32 + ButtonAwesome { + id: cmdRecord + visible: root.showRecordButton + width: 32; height: 32 + text: "\uf111" + iconColor: "red" + onClicked: { + console.log("Pressed record") + status.text = "Recording"; + } + } + } + Text { + id: status + anchors.top: parent.top + anchors.right: parent.right + width: 128 + text: "Idle" + } + + Player { + id: player + y: root.clientY + 64 + height: 64 + anchors.left: parent.left + anchors.right: parent.right + anchors.bottom: parent.bottom + + + +// onClicked: { +// if (recordTimer.running) { +// recordTimer.stop(); +// } +// recordTimer.start(); +// } + Timer { + id: recordTimer; + interval: 1000; running: false; repeat: false + onTriggered: { + console.log("Recording: " + MyAvatar.isRecording()) + MyAvatar.startRecording(); + console.log("Recording: " + MyAvatar.isRecording()) + } + } + + } + + Component.onCompleted: { + player.play.connect(play) + player.rewind.connect(rewind) + + } + } +} + diff --git a/interface/resources/qml/controls/ButtonAwesome.qml b/interface/resources/qml/controls/ButtonAwesome.qml new file mode 100644 index 0000000000..47c9fdc742 --- /dev/null +++ b/interface/resources/qml/controls/ButtonAwesome.qml @@ -0,0 +1,21 @@ +import QtQuick 2.3 +import QtQuick.Controls 1.3 as Original +import QtQuick.Controls.Styles 1.3 as OriginalStyles +import "." +import "../styles" + +Original.Button { + property color iconColor: "black" + FontLoader { id: iconFont; source: "../../fonts/fontawesome-webfont.ttf"; } + style: OriginalStyles.ButtonStyle { + label: Text { + renderType: Text.NativeRendering + verticalAlignment: Text.AlignVCenter + horizontalAlignment: Text.AlignHCenter + font.family: iconFont.name + font.pointSize: 20 + color: control.enabled ? control.iconColor : "gray" + text: control.text + } + } +} diff --git a/interface/resources/qml/controls/Player.qml b/interface/resources/qml/controls/Player.qml new file mode 100644 index 0000000000..8af0b1527d --- /dev/null +++ b/interface/resources/qml/controls/Player.qml @@ -0,0 +1,89 @@ +// +// AddressBarDialog.qml +// +// Created by Austin Davis on 2015/04/14 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +//import Hifi 1.0 +import QtQuick 2.4 +import QtQuick.Controls 1.2 +import QtQuick.Dialogs 1.2 +import QtQuick.Controls.Styles 1.2 +import "../styles" + +Item { + id: root + + signal play() + signal rewind() + + property real duration: 100 + property real position: 50 + property bool isPlaying: false + implicitHeight: 64 + implicitWidth: 640 + + Item { + anchors.top: parent.top + anchors.left: parent.left + anchors.right: parent.right + height: root.height / 2 + Text { + id: labelCurrent + anchors.top: parent.top + anchors.bottom: parent.bottom + anchors.left: parent.left + horizontalAlignment: Text.AlignHCenter + verticalAlignment: Text.AlignVCenter + width: 56 + text: "00:00:00" + } + Slider { + value: root.position / root.duration + anchors.top: parent.top + anchors.topMargin: 2 + anchors.bottomMargin: 2 + anchors.bottom: parent.bottom + anchors.left: labelCurrent.right + anchors.leftMargin: 4 + anchors.right: labelDuration.left + anchors.rightMargin: 4 + } + Text { + id: labelDuration + anchors.top: parent.top + anchors.bottom: parent.bottom + anchors.right: parent.right + horizontalAlignment: Text.AlignHCenter + verticalAlignment: Text.AlignVCenter + width: 56 + text: "00:00:00" + } + } + + Row { + anchors.bottom: parent.bottom + anchors.horizontalCenter: parent.horizontalCenter + height: root.height / 2; + ButtonAwesome { + id: cmdPlay + anchors.top: parent.top + anchors.bottom: parent.bottom + text: isPlaying ? "\uf04c" : "\uf04b" + width: root.height / 2; + onClicked: root.play(); + } + ButtonAwesome { + id: cmdRewind + anchors.top: parent.top + anchors.bottom: parent.bottom + width: root.height / 2 + text: "\uf04a" + onClicked: root.rewind(); + } + } +} diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index d54813135e..96b8ab74a8 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -94,6 +94,8 @@ #include #include #include +#include +#include #include "AnimDebugDraw.h" #include "AudioClient.h" @@ -124,6 +126,7 @@ #include "scripting/LocationScriptingInterface.h" #include "scripting/MenuScriptingInterface.h" #include "scripting/SettingsScriptingInterface.h" +#include "scripting/RecordingScriptingInterface.h" #include "scripting/WebWindowClass.h" #include "scripting/WindowScriptingInterface.h" #include "scripting/ControllerScriptingInterface.h" @@ -132,6 +135,7 @@ #endif #include "Stars.h" #include "ui/AddressBarDialog.h" +#include "ui/RecorderDialog.h" #include "ui/AvatarInputs.h" #include "ui/AssetUploadDialogFactory.h" #include "ui/DataWebDialog.h" @@ -295,6 +299,8 @@ bool setupEssentials(int& argc, char** argv) { Setting::init(); // Set dependencies + DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(NodeType::Agent, listenPort); DependencyManager::set(); @@ -319,6 +325,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -996,6 +1003,7 @@ void Application::initializeGL() { void Application::initializeUi() { AddressBarDialog::registerType(); + RecorderDialog::registerType(); ErrorDialog::registerType(); LoginDialog::registerType(); MessageDialog::registerType(); @@ -1011,6 +1019,7 @@ void Application::initializeUi() { offscreenUi->load("RootMenu.qml"); auto scriptingInterface = DependencyManager::get(); offscreenUi->getRootContext()->setContextProperty("Controller", scriptingInterface.data()); + offscreenUi->getRootContext()->setContextProperty("MyAvatar", getMyAvatar()); _glWidget->installEventFilter(offscreenUi.data()); VrMenu::load(); VrMenu::executeQueuedLambdas(); @@ -1580,8 +1589,9 @@ void Application::keyPressEvent(QKeyEvent* event) { case Qt::Key_X: if (isMeta && isShifted) { - auto offscreenUi = DependencyManager::get(); - offscreenUi->load("TestControllers.qml"); +// auto offscreenUi = DependencyManager::get(); +// offscreenUi->load("TestControllers.qml"); + RecorderDialog::toggle(); } break; @@ -3969,6 +3979,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri RayToOverlayIntersectionResultFromScriptValue); scriptEngine->registerGlobalObject("Desktop", DependencyManager::get().data()); + scriptEngine->registerGlobalObject("Recording", DependencyManager::get().data()); scriptEngine->registerGlobalObject("Window", DependencyManager::get().data()); scriptEngine->registerGetterSetter("location", LocationScriptingInterface::locationGetter, diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index 1a9b8a49e2..b8cf8ab4f1 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "Application.h" #include "Avatar.h" @@ -91,9 +92,9 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { if (isMine) { MyAvatar* myAvatar = static_cast(_owningAvatar); - + auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. - if (!myAvatar->isPlaying()) { + if (!player->isPlaying()) { FaceTracker* faceTracker = qApp->getActiveFaceTracker(); _isFaceTrackerConnected = faceTracker != NULL && !faceTracker->isMuted(); if (_isFaceTrackerConnected) { diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index e040ac2df9..38eb5042f7 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -80,10 +80,6 @@ const QString& DEFAULT_AVATAR_COLLISION_SOUND_URL = "https://hifi-public.s3.amaz const float MyAvatar::ZOOM_MIN = 0.5f; const float MyAvatar::ZOOM_MAX = 25.0f; const float MyAvatar::ZOOM_DEFAULT = 1.5f; -static const QString HEADER_NAME = "com.highfidelity.recording.AvatarData"; -static recording::FrameType AVATAR_FRAME_TYPE = recording::Frame::TYPE_INVALID; -static std::once_flag frameTypeRegistration; - MyAvatar::MyAvatar(RigPointer rig) : Avatar(rig), @@ -121,17 +117,6 @@ MyAvatar::MyAvatar(RigPointer rig) : { using namespace recording; - std::call_once(frameTypeRegistration, [] { - AVATAR_FRAME_TYPE = recording::Frame::registerFrameType(HEADER_NAME); - }); - - // FIXME how to deal with driving multiple avatars locally? - Frame::registerFrameHandler(AVATAR_FRAME_TYPE, [this](Frame::ConstPointer frame) { - qDebug() << "Playback of avatar frame length: " << frame->data.size(); - avatarStateFromFrame(frame->data, this); - }); - - for (int i = 0; i < MAX_DRIVE_KEYS; i++) { _driveKeys[i] = 0.0f; } @@ -326,8 +311,10 @@ void MyAvatar::simulate(float deltaTime) { } // Record avatars movements. - if (_recorder && _recorder->isRecording()) { - _recorder->recordFrame(AVATAR_FRAME_TYPE, avatarStateToFrame(this)); + auto recorder = DependencyManager::get(); + if (recorder->isRecording()) { + static const recording::FrameType FRAME_TYPE = recording::Frame::registerFrameType(AvatarData::FRAME_NAME); + recorder->recordFrame(FRAME_TYPE, toFrame(*this)); } // consider updating our billboard @@ -403,8 +390,8 @@ void MyAvatar::updateFromTrackers(float deltaTime) { glm::vec3 estimatedPosition, estimatedRotation; bool inHmd = qApp->getAvatarUpdater()->isHMDMode(); - - if (isPlaying() && inHmd) { + bool playing = DependencyManager::get()->isPlaying(); + if (inHmd && playing) { return; } @@ -455,7 +442,7 @@ void MyAvatar::updateFromTrackers(float deltaTime) { Head* head = getHead(); - if (inHmd || isPlaying()) { + if (inHmd || playing) { head->setDeltaPitch(estimatedRotation.x); head->setDeltaYaw(estimatedRotation.y); head->setDeltaRoll(estimatedRotation.z); @@ -572,102 +559,6 @@ bool MyAvatar::setJointReferential(const QUuid& id, int jointIndex) { } } -bool MyAvatar::isRecording() { - if (!_recorder) { - return false; - } - if (QThread::currentThread() != thread()) { - bool result; - QMetaObject::invokeMethod(this, "isRecording", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(bool, result)); - return result; - } - return _recorder && _recorder->isRecording(); -} - -float MyAvatar::recorderElapsed() { - if (QThread::currentThread() != thread()) { - float result; - QMetaObject::invokeMethod(this, "recorderElapsed", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(float, result)); - return result; - } - if (!_recorder) { - return 0; - } - return (float)_recorder->position() / (float) MSECS_PER_SECOND; -} - -QMetaObject::Connection _audioClientRecorderConnection; - -void MyAvatar::startRecording() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "startRecording", Qt::BlockingQueuedConnection); - return; - } - - _recorder = std::make_shared(); - // connect to AudioClient's signal so we get input audio - auto audioClient = DependencyManager::get(); - _audioClientRecorderConnection = connect(audioClient.data(), &AudioClient::inputReceived, [] { - // FIXME, missing audio data handling - }); - setRecordingBasis(); - _recorder->start(); -} - -void MyAvatar::stopRecording() { - if (!_recorder) { - return; - } - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "stopRecording", Qt::BlockingQueuedConnection); - return; - } - if (_recorder) { - QObject::disconnect(_audioClientRecorderConnection); - _audioClientRecorderConnection = QMetaObject::Connection(); - _recorder->stop(); - clearRecordingBasis(); - } -} - -void MyAvatar::saveRecording(const QString& filename) { - if (!_recorder) { - qCDebug(interfaceapp) << "There is no recording to save"; - return; - } - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "saveRecording", Qt::BlockingQueuedConnection, - Q_ARG(QString, filename)); - return; - } - - if (_recorder) { - auto clip = _recorder->getClip(); - recording::Clip::toFile(filename, clip); - } -} - -void MyAvatar::loadLastRecording() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "loadLastRecording", Qt::BlockingQueuedConnection); - return; - } - - if (!_recorder || !_recorder->getClip()) { - qCDebug(interfaceapp) << "There is no recording to load"; - return; - } - - if (!_player) { - _player = std::make_shared(); - } - - _player->queueClip(_recorder->getClip()); - _player->play(); -} - void MyAvatar::startAnimation(const QString& url, float fps, float priority, bool loop, bool hold, float firstFrame, float lastFrame, const QStringList& maskedJoints) { if (QThread::currentThread() != thread()) { diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 410b8a978d..16c3abec3e 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -254,13 +254,6 @@ public slots: bool setModelReferential(const QUuid& id); bool setJointReferential(const QUuid& id, int jointIndex); - bool isRecording(); - float recorderElapsed(); - void startRecording(); - void stopRecording(); - void saveRecording(const QString& filename); - void loadLastRecording(); - virtual void rebuildSkeletonBody() override; bool getEnableRigAnimations() const { return _rig->getEnableRig(); } @@ -309,9 +302,6 @@ private: const glm::vec3& translation = glm::vec3(), const glm::quat& rotation = glm::quat(), float scale = 1.0f, bool allowDuplicates = false, bool useSaved = true) override; - const recording::RecorderPointer getRecorder() const { return _recorder; } - const recording::DeckPointer getPlayer() const { return _player; } - //void beginFollowingHMD(); //bool shouldFollowHMD() const; //void followHMD(float deltaTime); @@ -358,8 +348,6 @@ private: eyeContactTarget _eyeContactTarget; - recording::RecorderPointer _recorder; - glm::vec3 _trackedHeadPosition; Setting::Handle _realWorldFieldOfView; diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 1347c69d61..83c8cdfcf5 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "Application.h" #include "Avatar.h" @@ -247,8 +248,8 @@ void SkeletonModel::simulate(float deltaTime, bool fullUpdate) { } MyAvatar* myAvatar = static_cast(_owningAvatar); - if (myAvatar->isPlaying()) { - // Don't take inputs if playing back a recording. + auto player = DependencyManager::get(); + if (player->isPlaying()) { return; } diff --git a/interface/src/scripting/RecordingScriptingInterface.cpp b/interface/src/scripting/RecordingScriptingInterface.cpp new file mode 100644 index 0000000000..7e0c763dfa --- /dev/null +++ b/interface/src/scripting/RecordingScriptingInterface.cpp @@ -0,0 +1,328 @@ +// +// Created by Bradley Austin Davis on 2015/11/13 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "RecordingScriptingInterface.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "avatar/AvatarManager.h" +#include "Application.h" +#include "InterfaceLogging.h" + +typedef int16_t AudioSample; + + +using namespace recording; + +// FIXME move to somewhere audio related? +static const QString AUDIO_FRAME_NAME = "com.highfidelity.recording.Audio"; + +RecordingScriptingInterface::RecordingScriptingInterface() { + static const recording::FrameType AVATAR_FRAME_TYPE = recording::Frame::registerFrameType(AvatarData::FRAME_NAME); + // FIXME how to deal with driving multiple avatars locally? + Frame::registerFrameHandler(AVATAR_FRAME_TYPE, [this](Frame::ConstPointer frame) { + processAvatarFrame(frame); + }); + + static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME); + Frame::registerFrameHandler(AUDIO_FRAME_TYPE, [this](Frame::ConstPointer frame) { + processAudioFrame(frame); + }); + + _player = DependencyManager::get(); + _recorder = DependencyManager::get(); + + auto audioClient = DependencyManager::get(); + connect(audioClient.data(), &AudioClient::inputReceived, this, &RecordingScriptingInterface::processAudioInput); +} + +bool RecordingScriptingInterface::isPlaying() { + return _player->isPlaying(); +} + +bool RecordingScriptingInterface::isPaused() { + return _player->isPaused(); +} + +float RecordingScriptingInterface::playerElapsed() { + return (float)_player->position() / MSECS_PER_SECOND; +} + +float RecordingScriptingInterface::playerLength() { + return _player->length() / MSECS_PER_SECOND; +} + +void RecordingScriptingInterface::loadRecording(const QString& filename) { + using namespace recording; + + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadRecording", Qt::BlockingQueuedConnection, + Q_ARG(QString, filename)); + return; + } + + ClipPointer clip = Clip::fromFile(filename); + if (!clip) { + qWarning() << "Unable to load clip data from " << filename; + } + _player->queueClip(clip); +} + +void RecordingScriptingInterface::startPlaying() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "startPlaying", Qt::BlockingQueuedConnection); + return; + } + auto myAvatar = DependencyManager::get()->getMyAvatar(); + // Playback from the current position + if (_playFromCurrentLocation) { + _dummyAvatar.setRecordingBasis(std::make_shared(myAvatar->getTransform())); + } else { + _dummyAvatar.clearRecordingBasis(); + } + _player->play(); +} + +void RecordingScriptingInterface::setPlayerVolume(float volume) { + // FIXME +} + +void RecordingScriptingInterface::setPlayerAudioOffset(float audioOffset) { + // FIXME +} + +void RecordingScriptingInterface::setPlayerTime(float time) { + _player->seek(time * MSECS_PER_SECOND); +} + +void RecordingScriptingInterface::setPlayFromCurrentLocation(bool playFromCurrentLocation) { + _playFromCurrentLocation = playFromCurrentLocation; +} + +void RecordingScriptingInterface::setPlayerLoop(bool loop) { + _player->loop(loop); +} + +void RecordingScriptingInterface::setPlayerUseDisplayName(bool useDisplayName) { + _useDisplayName = useDisplayName; +} + +void RecordingScriptingInterface::setPlayerUseAttachments(bool useAttachments) { + _useAttachments = useAttachments; +} + +void RecordingScriptingInterface::setPlayerUseHeadModel(bool useHeadModel) { + _useHeadModel = useHeadModel; +} + +void RecordingScriptingInterface::setPlayerUseSkeletonModel(bool useSkeletonModel) { + _useSkeletonModel = useSkeletonModel; +} + +void RecordingScriptingInterface::play() { + _player->play(); +} + +void RecordingScriptingInterface::pausePlayer() { + _player->pause(); +} + +void RecordingScriptingInterface::stopPlaying() { + _player->stop(); +} + +bool RecordingScriptingInterface::isRecording() { + return _recorder->isRecording(); +} + +float RecordingScriptingInterface::recorderElapsed() { + return _recorder->position(); +} + +void RecordingScriptingInterface::startRecording() { + if (_recorder->isRecording()) { + qCWarning(interfaceapp) << "Recorder is already running"; + return; + } + + + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "startRecording", Qt::BlockingQueuedConnection); + return; + } + + _recordingEpoch = Frame::epochForFrameTime(0); + + _audioRecordingBuffer.clear(); + auto myAvatar = DependencyManager::get()->getMyAvatar(); + myAvatar->setRecordingBasis(); + _recorder->start(); +} + +float calculateAudioTime(const QByteArray& audio) { + static const float AUDIO_BYTES_PER_SECOND = AudioConstants::SAMPLE_RATE * sizeof(AudioConstants::AudioSample); + return (float)audio.size() / AUDIO_BYTES_PER_SECOND; +} + +void injectAudioFrame(Clip::Pointer& clip, Frame::Time time, const QByteArray& audio) { + static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME); + clip->addFrame(std::make_shared(AUDIO_FRAME_TYPE, time, audio)); +} + +// Detect too much audio in a single frame, or too much deviation between +// the expected audio length and the computed audio length +bool shouldStartNewAudioFrame(const QByteArray& currentAudioFrame, float expectedAudioLength) { + if (currentAudioFrame.isEmpty()) { + return true; + } + + // 100 milliseconds + float actualAudioLength = calculateAudioTime(currentAudioFrame); + static const float MAX_AUDIO_PACKET_DURATION = 1.0f; + if (actualAudioLength >= MAX_AUDIO_PACKET_DURATION) { + return true; + } + + + float deviation = std::abs(actualAudioLength - expectedAudioLength); + + qDebug() << "Checking buffer deviation current length "; + qDebug() << "Actual: " << actualAudioLength; + qDebug() << "Expected: " << expectedAudioLength; + qDebug() << "Deviation: " << deviation; + + static const float MAX_AUDIO_DEVIATION = 0.1f; + if (deviation >= MAX_AUDIO_PACKET_DURATION) { + return true; + } + + return false; +} + + +void injectAudioFrames(Clip::Pointer& clip, const QList>& audioBuffer) { + Frame::Time lastAudioStartTime = 0; + QByteArray audioFrameBuffer; + for (const auto& audioPacket : audioBuffer) { + float expectedAudioLength = Frame::frameTimeToSeconds(audioPacket.first - lastAudioStartTime); + if (shouldStartNewAudioFrame(audioFrameBuffer, expectedAudioLength)) { + // Time to start a new frame, inject the old one if it exists + if (audioFrameBuffer.size()) { + injectAudioFrame(clip, lastAudioStartTime, audioFrameBuffer); + audioFrameBuffer.clear(); + } + lastAudioStartTime = audioPacket.first; + } + audioFrameBuffer.append(audioPacket.second); + } +} + + +void RecordingScriptingInterface::stopRecording() { + _recorder->stop(); + + _lastClip = _recorder->getClip(); + // post-process the audio into discreet chunks based on times of received samples + injectAudioFrames(_lastClip, _audioRecordingBuffer); + _audioRecordingBuffer.clear(); + _lastClip->seek(0); + Frame::ConstPointer frame; + while (frame = _lastClip->nextFrame()) { + qDebug() << "Frame time " << frame->timeOffset << " size " << frame->data.size(); + } + _lastClip->seek(0); + + auto myAvatar = DependencyManager::get()->getMyAvatar(); + myAvatar->clearRecordingBasis(); +} + +void RecordingScriptingInterface::saveRecording(const QString& filename) { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "saveRecording", Qt::BlockingQueuedConnection, + Q_ARG(QString, filename)); + return; + } + + if (!_lastClip) { + qWarning() << "There is no recording to save"; + return; + } + + recording::Clip::toFile(filename, _lastClip); +} + +void RecordingScriptingInterface::loadLastRecording() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "loadLastRecording", Qt::BlockingQueuedConnection); + return; + } + + if (!_lastClip) { + qCDebug(interfaceapp) << "There is no recording to load"; + return; + } + + _player->queueClip(_lastClip); + _player->play(); +} + +void RecordingScriptingInterface::processAvatarFrame(const Frame::ConstPointer& frame) { + Q_ASSERT(QThread::currentThread() == thread()); + + AvatarData::fromFrame(frame->data, _dummyAvatar); + + auto myAvatar = DependencyManager::get()->getMyAvatar(); + if (_useHeadModel && _dummyAvatar.getFaceModelURL().isValid() && + (_dummyAvatar.getFaceModelURL() != myAvatar->getFaceModelURL())) { + // FIXME + //myAvatar->setFaceModelURL(_dummyAvatar.getFaceModelURL()); + } + + if (_useSkeletonModel && _dummyAvatar.getSkeletonModelURL().isValid() && + (_dummyAvatar.getSkeletonModelURL() != myAvatar->getSkeletonModelURL())) { + // FIXME + //myAvatar->useFullAvatarURL() + } + + if (_useDisplayName && _dummyAvatar.getDisplayName() != myAvatar->getDisplayName()) { + myAvatar->setDisplayName(_dummyAvatar.getDisplayName()); + } + + myAvatar->setPosition(_dummyAvatar.getPosition()); + myAvatar->setOrientation(_dummyAvatar.getOrientation()); + + // FIXME attachments + // FIXME joints + // FIXME head lean + // FIXME head orientation +} + +void RecordingScriptingInterface::processAudioInput(const QByteArray& audio) { + if (_recorder->isRecording()) { + auto audioFrameTime = Frame::frameTimeFromEpoch(_recordingEpoch); + _audioRecordingBuffer.push_back({ audioFrameTime, audio }); + qDebug() << "Got sound packet of size " << audio.size() << " At time " << audioFrameTime; + } +} + +void RecordingScriptingInterface::processAudioFrame(const recording::FrameConstPointer& frame) { + AudioInjectorOptions options; + auto myAvatar = DependencyManager::get()->getMyAvatar(); + options.position = myAvatar->getPosition(); + options.orientation = myAvatar->getOrientation(); + // FIXME store the audio format (sample rate, bits, stereo) in the frame + options.stereo = false; + // FIXME move audio injector to a thread pool model? + AudioInjector::playSoundAndDelete(frame->data, options, nullptr); +} diff --git a/interface/src/scripting/RecordingScriptingInterface.h b/interface/src/scripting/RecordingScriptingInterface.h new file mode 100644 index 0000000000..f7add9480b --- /dev/null +++ b/interface/src/scripting/RecordingScriptingInterface.h @@ -0,0 +1,80 @@ +// +// Created by Bradley Austin Davis on 2015/11/13 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_RecordingScriptingInterface_h +#define hifi_RecordingScriptingInterface_h + +#include + +#include + +#include +#include +#include +#include + +class RecordingScriptingInterface : public QObject, public Dependency { + Q_OBJECT + +public: + RecordingScriptingInterface(); + +public slots: + bool isPlaying(); + bool isPaused(); + float playerElapsed(); + float playerLength(); + void loadRecording(const QString& filename); + void startPlaying(); + void setPlayerVolume(float volume); + void setPlayerAudioOffset(float audioOffset); + void setPlayerTime(float time); + void setPlayFromCurrentLocation(bool playFromCurrentLocation); + void setPlayerLoop(bool loop); + void setPlayerUseDisplayName(bool useDisplayName); + void setPlayerUseAttachments(bool useAttachments); + void setPlayerUseHeadModel(bool useHeadModel); + void setPlayerUseSkeletonModel(bool useSkeletonModel); + void play(); + void pausePlayer(); + void stopPlaying(); + bool isRecording(); + float recorderElapsed(); + void startRecording(); + void stopRecording(); + void saveRecording(const QString& filename); + void loadLastRecording(); + +signals: + void playbackStateChanged(); + // Should this occur for any frame or just for seek calls? + void playbackPositionChanged(); + void looped(); + +private: + using Mutex = std::recursive_mutex; + using Locker = std::unique_lock; + using Flag = std::atomic; + void processAvatarFrame(const recording::FrameConstPointer& frame); + void processAudioFrame(const recording::FrameConstPointer& frame); + void processAudioInput(const QByteArray& audioData); + QSharedPointer _player; + QSharedPointer _recorder; + QList> _audioRecordingBuffer; + quint64 _recordingEpoch { 0 }; + + Flag _playFromCurrentLocation { true }; + Flag _useDisplayName { false }; + Flag _useHeadModel { false }; + Flag _useAttachments { false }; + Flag _useSkeletonModel { false }; + recording::ClipPointer _lastClip; + AvatarData _dummyAvatar; +}; + +#endif // hifi_RecordingScriptingInterface_h diff --git a/interface/src/ui/RecorderDialog.cpp b/interface/src/ui/RecorderDialog.cpp new file mode 100644 index 0000000000..ddefa9fbd9 --- /dev/null +++ b/interface/src/ui/RecorderDialog.cpp @@ -0,0 +1,22 @@ +// +// Created by Bradley Austin Davis on 2015/11/14 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "RecorderDialog.h" + +#include + +#include "DependencyManager.h" + +HIFI_QML_DEF(RecorderDialog) + +RecorderDialog::RecorderDialog(QQuickItem* parent) : OffscreenQmlDialog(parent) { +} + +void RecorderDialog::hide() { + ((QQuickItem*)parent())->setEnabled(false); +} diff --git a/interface/src/ui/RecorderDialog.h b/interface/src/ui/RecorderDialog.h new file mode 100644 index 0000000000..f4f0a7c2d8 --- /dev/null +++ b/interface/src/ui/RecorderDialog.h @@ -0,0 +1,28 @@ +// +// Created by Bradley Austin Davis on 2015/11/14 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once +#ifndef hifi_RecorderDialog_h +#define hifi_RecorderDialog_h + +#include + +class RecorderDialog : public OffscreenQmlDialog { + Q_OBJECT + HIFI_QML_DECL + +public: + RecorderDialog(QQuickItem* parent = nullptr); + +signals: + +protected: + void hide(); +}; + +#endif diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index a47d5f663e..017ef7578a 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -33,8 +33,7 @@ #include #include #include -#include -#include +#include #include "AvatarLogging.h" @@ -45,6 +44,9 @@ using namespace std; const glm::vec3 DEFAULT_LOCAL_AABOX_CORNER(-0.5f); const glm::vec3 DEFAULT_LOCAL_AABOX_SCALE(1.0f); +const QString AvatarData::FRAME_NAME = "com.highfidelity.recording.AvatarData"; +static std::once_flag frameTypeRegistration; + AvatarData::AvatarData() : _sessionUUID(), _position(0.0f), @@ -791,155 +793,10 @@ bool AvatarData::hasReferential() { return _referential != NULL; } -bool AvatarData::isPlaying() { - return _player && _player->isPlaying(); -} - -bool AvatarData::isPaused() { - return _player && _player->isPaused(); -} - -float AvatarData::playerElapsed() { - if (!_player) { - return 0; - } - if (QThread::currentThread() != thread()) { - float result; - QMetaObject::invokeMethod(this, "playerElapsed", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(float, result)); - return result; - } - return (float)_player->position() / (float) MSECS_PER_SECOND; -} - -float AvatarData::playerLength() { - if (!_player) { - return 0; - } - if (QThread::currentThread() != thread()) { - float result; - QMetaObject::invokeMethod(this, "playerLength", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(float, result)); - return result; - } - return (float)_player->length() / (float) MSECS_PER_SECOND; -} - -void AvatarData::loadRecording(const QString& filename) { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "loadRecording", Qt::BlockingQueuedConnection, - Q_ARG(QString, filename)); - return; - } - using namespace recording; - - ClipPointer clip = Clip::fromFile(filename); - if (!clip) { - qWarning() << "Unable to load clip data from " << filename; - } - - _player = std::make_shared(); - _player->queueClip(clip); -} - -void AvatarData::startPlaying() { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "startPlaying", Qt::BlockingQueuedConnection); - return; - } - - if (!_player) { - qWarning() << "No clip loaded for playback"; - return; - } - setRecordingBasis(); - _player->play(); -} - -void AvatarData::setPlayerVolume(float volume) { - // FIXME -} - -void AvatarData::setPlayerAudioOffset(float audioOffset) { - // FIXME -} - -void AvatarData::setPlayerTime(float time) { - if (!_player) { - qWarning() << "No player active"; - return; - } - - _player->seek(time * MSECS_PER_SECOND); -} - -void AvatarData::setPlayFromCurrentLocation(bool playFromCurrentLocation) { - // FIXME -} - -void AvatarData::setPlayerLoop(bool loop) { - if (_player) { - _player->loop(loop); - } -} - -void AvatarData::setPlayerUseDisplayName(bool useDisplayName) { - // FIXME -} - -void AvatarData::setPlayerUseAttachments(bool useAttachments) { - // FIXME -} - -void AvatarData::setPlayerUseHeadModel(bool useHeadModel) { - // FIXME -} - -void AvatarData::setPlayerUseSkeletonModel(bool useSkeletonModel) { - // FIXME -} - -void AvatarData::play() { - if (isPlaying()) { - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "play", Qt::BlockingQueuedConnection); - return; - } - - _player->play(); - } -} - std::shared_ptr AvatarData::getRecordingBasis() const { return _recordingBasis; } -void AvatarData::pausePlayer() { - if (!_player) { - return; - } - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "pausePlayer", Qt::BlockingQueuedConnection); - return; - } - if (_player) { - _player->pause(); - } -} - -void AvatarData::stopPlaying() { - if (!_player) { - return; - } - if (QThread::currentThread() != thread()) { - QMetaObject::invokeMethod(this, "stopPlaying", Qt::BlockingQueuedConnection); - return; - } - if (_player) { - _player->stop(); - } -} - void AvatarData::changeReferential(Referential* ref) { delete _referential; _referential = ref; @@ -1568,26 +1425,26 @@ JointData jointDataFromJsonValue(const QJsonValue& json) { // This allows the application to decide whether playback should be relative to an avatar's // transform at the start of playback, or relative to the transform of the recorded // avatar -QByteArray avatarStateToFrame(const AvatarData* _avatar) { +QByteArray AvatarData::toFrame(const AvatarData& avatar) { QJsonObject root; - if (!_avatar->getFaceModelURL().isEmpty()) { - root[JSON_AVATAR_HEAD_MODEL] = _avatar->getFaceModelURL().toString(); + if (!avatar.getFaceModelURL().isEmpty()) { + root[JSON_AVATAR_HEAD_MODEL] = avatar.getFaceModelURL().toString(); } - if (!_avatar->getSkeletonModelURL().isEmpty()) { - root[JSON_AVATAR_BODY_MODEL] = _avatar->getSkeletonModelURL().toString(); + if (!avatar.getSkeletonModelURL().isEmpty()) { + root[JSON_AVATAR_BODY_MODEL] = avatar.getSkeletonModelURL().toString(); } - if (!_avatar->getDisplayName().isEmpty()) { - root[JSON_AVATAR_DISPLAY_NAME] = _avatar->getDisplayName(); + if (!avatar.getDisplayName().isEmpty()) { + root[JSON_AVATAR_DISPLAY_NAME] = avatar.getDisplayName(); } - if (!_avatar->getAttachmentData().isEmpty()) { + if (!avatar.getAttachmentData().isEmpty()) { // FIXME serialize attachment data } - auto recordingBasis = _avatar->getRecordingBasis(); + auto recordingBasis = avatar.getRecordingBasis(); if (recordingBasis) { // Find the relative transform - auto relativeTransform = recordingBasis->relativeTransform(_avatar->getTransform()); + auto relativeTransform = recordingBasis->relativeTransform(avatar.getTransform()); // if the resulting relative basis is identity, we shouldn't record anything if (!relativeTransform.isIdentity()) { @@ -1595,17 +1452,17 @@ QByteArray avatarStateToFrame(const AvatarData* _avatar) { root[JSON_AVATAR_BASIS] = Transform::toJson(*recordingBasis); } } else { - root[JSON_AVATAR_RELATIVE] = Transform::toJson(_avatar->getTransform()); + root[JSON_AVATAR_RELATIVE] = Transform::toJson(avatar.getTransform()); } // Skeleton pose QJsonArray jointArray; - for (const auto& joint : _avatar->getRawJointData()) { + for (const auto& joint : avatar.getRawJointData()) { jointArray.push_back(toJsonValue(joint)); } root[JSON_AVATAR_JOINT_ARRAY] = jointArray; - const HeadData* head = _avatar->getHeadData(); + const HeadData* head = avatar.getHeadData(); if (head) { QJsonObject headJson; QJsonArray blendshapeCoefficients; @@ -1616,8 +1473,8 @@ QByteArray avatarStateToFrame(const AvatarData* _avatar) { headJson[JSON_AVATAR_HEAD_ROTATION] = toJsonValue(head->getRawOrientation()); headJson[JSON_AVATAR_HEAD_LEAN_FORWARD] = QJsonValue(head->getLeanForward()); headJson[JSON_AVATAR_HEAD_LEAN_SIDEWAYS] = QJsonValue(head->getLeanSideways()); - vec3 relativeLookAt = glm::inverse(_avatar->getOrientation()) * - (head->getLookAtPosition() - _avatar->getPosition()); + vec3 relativeLookAt = glm::inverse(avatar.getOrientation()) * + (head->getLookAtPosition() - avatar.getPosition()); headJson[JSON_AVATAR_HEAD_LOOKAT] = toJsonValue(relativeLookAt); root[JSON_AVATAR_HEAD] = headJson; } @@ -1625,26 +1482,29 @@ QByteArray avatarStateToFrame(const AvatarData* _avatar) { return QJsonDocument(root).toBinaryData(); } -void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar) { +void AvatarData::fromFrame(const QByteArray& frameData, AvatarData& result) { QJsonDocument doc = QJsonDocument::fromBinaryData(frameData); QJsonObject root = doc.object(); if (root.contains(JSON_AVATAR_HEAD_MODEL)) { auto faceModelURL = root[JSON_AVATAR_HEAD_MODEL].toString(); - if (faceModelURL != _avatar->getFaceModelURL().toString()) { - _avatar->setFaceModelURL(faceModelURL); + if (faceModelURL != result.getFaceModelURL().toString()) { + QUrl faceModel(faceModelURL); + if (faceModel.isValid()) { + result.setFaceModelURL(faceModel); + } } } if (root.contains(JSON_AVATAR_BODY_MODEL)) { auto bodyModelURL = root[JSON_AVATAR_BODY_MODEL].toString(); - if (bodyModelURL != _avatar->getSkeletonModelURL().toString()) { - _avatar->setSkeletonModelURL(bodyModelURL); + if (bodyModelURL != result.getSkeletonModelURL().toString()) { + result.setSkeletonModelURL(bodyModelURL); } } if (root.contains(JSON_AVATAR_DISPLAY_NAME)) { auto newDisplayName = root[JSON_AVATAR_DISPLAY_NAME].toString(); - if (newDisplayName != _avatar->getDisplayName()) { - _avatar->setDisplayName(newDisplayName); + if (newDisplayName != result.getDisplayName()) { + result.setDisplayName(newDisplayName); } } @@ -1656,18 +1516,18 @@ void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar) { // The first is more useful for playing back recordings on your own avatar, while // the latter is more useful for playing back other avatars within your scene. - auto currentBasis = _avatar->getRecordingBasis(); + auto currentBasis = result.getRecordingBasis(); if (!currentBasis) { currentBasis = std::make_shared(Transform::fromJson(root[JSON_AVATAR_BASIS])); } auto relativeTransform = Transform::fromJson(root[JSON_AVATAR_RELATIVE]); auto worldTransform = currentBasis->worldTransform(relativeTransform); - _avatar->setPosition(worldTransform.getTranslation()); - _avatar->setOrientation(worldTransform.getRotation()); + result.setPosition(worldTransform.getTranslation()); + result.setOrientation(worldTransform.getRotation()); // TODO: find a way to record/playback the Scale of the avatar - //_avatar->setTargetScale(worldTransform.getScale().x); + //result.setTargetScale(worldTransform.getScale().x); } @@ -1689,13 +1549,13 @@ void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar) { for (const auto& joint : jointArray) { jointRotations.push_back(joint.rotation); } - _avatar->setJointRotations(jointRotations); + result.setJointRotations(jointRotations); } #if 0 // Most head data is relative to the avatar, and needs no basis correction, // but the lookat vector does need correction - HeadData* head = _avatar->_headData; + HeadData* head = result._headData; if (head && root.contains(JSON_AVATAR_HEAD)) { QJsonObject headJson = root[JSON_AVATAR_HEAD].toObject(); if (headJson.contains(JSON_AVATAR_HEAD_BLENDSHAPE_COEFFICIENTS)) { @@ -1718,7 +1578,7 @@ void avatarStateFromFrame(const QByteArray& frameData, AvatarData* _avatar) { if (headJson.contains(JSON_AVATAR_HEAD_LOOKAT)) { auto relativeLookAt = vec3FromJsonValue(headJson[JSON_AVATAR_HEAD_LOOKAT]); if (glm::length2(relativeLookAt) > 0.01) { - head->setLookAtPosition((_avatar->getOrientation() * relativeLookAt) + _avatar->getPosition()); + head->setLookAtPosition((result.getOrientation() * relativeLookAt) + result.getPosition()); } } } diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 26bc9d83ff..e79c0be80a 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -50,13 +50,12 @@ typedef unsigned long long quint64; #include #include #include +#include #include "AABox.h" #include "HandData.h" #include "HeadData.h" #include "PathUtils.h" -#include "Player.h" -#include "Recorder.h" #include "Referential.h" using AvatarSharedPointer = std::shared_ptr; @@ -165,7 +164,13 @@ class AvatarData : public QObject { Q_PROPERTY(QStringList jointNames READ getJointNames) Q_PROPERTY(QUuid sessionUUID READ getSessionUUID) + public: + static const QString FRAME_NAME; + + static void fromFrame(const QByteArray& frameData, AvatarData& avatar); + static QByteArray toFrame(const AvatarData& avatar); + AvatarData(); virtual ~AvatarData(); @@ -348,25 +353,6 @@ public slots: void setJointMappingsFromNetworkReply(); void setSessionUUID(const QUuid& sessionUUID) { _sessionUUID = sessionUUID; } bool hasReferential(); - - bool isPlaying(); - bool isPaused(); - float playerElapsed(); - float playerLength(); - void loadRecording(const QString& filename); - void startPlaying(); - void setPlayerVolume(float volume); - void setPlayerAudioOffset(float audioOffset); - void setPlayerTime(float time); - void setPlayFromCurrentLocation(bool playFromCurrentLocation); - void setPlayerLoop(bool loop); - void setPlayerUseDisplayName(bool useDisplayName); - void setPlayerUseAttachments(bool useAttachments); - void setPlayerUseHeadModel(bool useHeadModel); - void setPlayerUseSkeletonModel(bool useSkeletonModel); - void play(); - void pausePlayer(); - void stopPlaying(); protected: QUuid _sessionUUID; @@ -421,8 +407,6 @@ protected: QWeakPointer _owningAvatarMixer; - recording::DeckPointer _player; - /// Loads the joint indices, names from the FST file (if any) virtual void updateJointMappings(); void changeReferential(Referential* ref); @@ -437,7 +421,7 @@ protected: QMutex avatarLock; // Name is redundant, but it aids searches. // During recording, this holds the starting position, orientation & scale of the recorded avatar - // During playback, it holds the + // During playback, it holds the origin from which to play the relative positions in the clip TransformPointer _recordingBasis; private: diff --git a/libraries/gl/src/gl/OffscreenQmlSurface.cpp b/libraries/gl/src/gl/OffscreenQmlSurface.cpp index b6603deb62..e8a950a16b 100644 --- a/libraries/gl/src/gl/OffscreenQmlSurface.cpp +++ b/libraries/gl/src/gl/OffscreenQmlSurface.cpp @@ -26,7 +26,7 @@ #include "OffscreenGlCanvas.h" // FIXME move to threaded rendering with Qt 5.5 -// #define QML_THREADED +//#define QML_THREADED // Time between receiving a request to render the offscreen UI actually triggering // the render. Could possibly be increased depending on the framerate we expect to @@ -72,7 +72,7 @@ public: OffscreenGlCanvas::create(shareContext); #ifdef QML_THREADED // Qt 5.5 - // _renderControl->prepareThread(_renderThread); + _renderControl->prepareThread(_renderThread); _context->moveToThread(&_thread); moveToThread(&_thread); _thread.setObjectName("QML Thread"); diff --git a/libraries/recording/src/recording/Clip.cpp b/libraries/recording/src/recording/Clip.cpp index 28e4211fe3..abe66ccb2e 100644 --- a/libraries/recording/src/recording/Clip.cpp +++ b/libraries/recording/src/recording/Clip.cpp @@ -23,7 +23,7 @@ Clip::Pointer Clip::fromFile(const QString& filePath) { return result; } -void Clip::toFile(const QString& filePath, Clip::Pointer clip) { +void Clip::toFile(const QString& filePath, const Clip::ConstPointer& clip) { FileClip::write(filePath, clip->duplicate()); } @@ -31,19 +31,10 @@ Clip::Pointer Clip::newClip() { return std::make_shared(); } -Clip::Pointer Clip::duplicate() { - Clip::Pointer result = std::make_shared(); - - Locker lock(_mutex); - Time currentPosition = position(); - seek(0); - - auto frame = nextFrame(); - while (frame) { - result->addFrame(frame); - frame = nextFrame(); - } - - seek(currentPosition); - return result; +void Clip::seek(float offset) { + seekFrameTime(Frame::secondsToFrameTime(offset)); } + +float Clip::position() const { + return Frame::frameTimeToSeconds(positionFrameTime()); +}; diff --git a/libraries/recording/src/recording/Clip.h b/libraries/recording/src/recording/Clip.h index a00ab72c98..722fadf0b2 100644 --- a/libraries/recording/src/recording/Clip.h +++ b/libraries/recording/src/recording/Clip.h @@ -16,6 +16,8 @@ #include +#include "Frame.h" + class QIODevice; namespace recording { @@ -23,16 +25,22 @@ namespace recording { class Clip { public: using Pointer = std::shared_ptr; + using ConstPointer = std::shared_ptr; virtual ~Clip() {} - Pointer duplicate(); + virtual Pointer duplicate() const = 0; - virtual Time duration() const = 0; + virtual QString getName() const = 0; + + virtual float duration() const = 0; virtual size_t frameCount() const = 0; - virtual void seek(Time offset) = 0; - virtual Time position() const = 0; + virtual void seek(float offset) final; + virtual float position() const final; + + virtual void seekFrameTime(Frame::Time offset) = 0; + virtual Frame::Time positionFrameTime() const = 0; virtual FrameConstPointer peekFrame() const = 0; virtual FrameConstPointer nextFrame() = 0; @@ -40,7 +48,7 @@ public: virtual void addFrame(FrameConstPointer) = 0; static Pointer fromFile(const QString& filePath); - static void toFile(const QString& filePath, Pointer clip); + static void toFile(const QString& filePath, const ConstPointer& clip); static Pointer newClip(); protected: diff --git a/libraries/recording/src/recording/Deck.cpp b/libraries/recording/src/recording/Deck.cpp index 10209c26d7..e52fcc16e6 100644 --- a/libraries/recording/src/recording/Deck.cpp +++ b/libraries/recording/src/recording/Deck.cpp @@ -14,31 +14,46 @@ #include "Clip.h" #include "Frame.h" #include "Logging.h" +#include "impl/OffsetClip.h" using namespace recording; -void Deck::queueClip(ClipPointer clip, Time timeOffset) { +Deck::Deck(QObject* parent) + : QObject(parent) {} + +void Deck::queueClip(ClipPointer clip, float timeOffset) { + Locker lock(_mutex); + if (!clip) { qCWarning(recordingLog) << "Clip invalid, ignoring"; return; } - // FIXME if the time offset is not zero, wrap the clip in a OffsetClip wrapper + // FIXME disabling multiple clips for now + _clips.clear(); + + // if the time offset is not zero, wrap in an OffsetClip + if (timeOffset != 0.0f) { + clip = std::make_shared(clip, timeOffset); + } + _clips.push_back(clip); _length = std::max(_length, clip->duration()); } void Deck::play() { + Locker lock(_mutex); if (_pause) { _pause = false; - _startEpoch = usecTimestampNow() - (_position * USECS_PER_MSEC); + _startEpoch = Frame::epochForFrameTime(_position); emit playbackStateChanged(); processFrames(); } } void Deck::pause() { + Locker lock(_mutex); if (!_pause) { _pause = true; emit playbackStateChanged(); @@ -47,9 +62,9 @@ void Deck::pause() { Clip::Pointer Deck::getNextClip() { Clip::Pointer result; - Time soonestFramePosition = INVALID_TIME; + auto soonestFramePosition = Frame::INVALID_TIME; for (const auto& clip : _clips) { - Time nextFramePosition = clip->position(); + auto nextFramePosition = clip->positionFrameTime(); if (nextFramePosition < soonestFramePosition) { result = clip; soonestFramePosition = nextFramePosition; @@ -58,11 +73,16 @@ Clip::Pointer Deck::getNextClip() { return result; } -void Deck::seek(Time position) { - _position = position; - // FIXME reset the frames to the appropriate spot +void Deck::seek(float position) { + Locker lock(_mutex); + _position = Frame::secondsToFrameTime(position); + + // Recompute the start epoch + _startEpoch = Frame::epochForFrameTime(_position); + + // reset the clips to the appropriate spot for (auto& clip : _clips) { - clip->seek(position); + clip->seekFrameTime(_position); } if (!_pause) { @@ -71,35 +91,46 @@ void Deck::seek(Time position) { } } -Time Deck::position() const { - if (_pause) { - return _position; +float Deck::position() const { + Locker lock(_mutex); + auto currentPosition = _position; + if (!_pause) { + currentPosition = Frame::frameTimeFromEpoch(_startEpoch); } - return (usecTimestampNow() - _startEpoch) / USECS_PER_MSEC; + return Frame::frameTimeToSeconds(currentPosition); } -static const Time MIN_FRAME_WAIT_INTERVAL_MS = 1; +static const Frame::Time MIN_FRAME_WAIT_INTERVAL = Frame::secondsToFrameTime(0.001f); +static const Frame::Time MAX_FRAME_PROCESSING_TIME = Frame::secondsToFrameTime(0.002f); void Deck::processFrames() { + Locker lock(_mutex); if (_pause) { return; } - _position = position(); - auto triggerPosition = _position + MIN_FRAME_WAIT_INTERVAL_MS; + auto startingPosition = Frame::frameTimeFromEpoch(_startEpoch); + auto triggerPosition = startingPosition + MIN_FRAME_WAIT_INTERVAL; Clip::Pointer nextClip; + // FIXME add code to start dropping frames if we fall behind. + // Alternatively, add code to cache frames here and then process only the last frame of a given type + // ... the latter will work for Avatar, but not well for audio I suspect. for (nextClip = getNextClip(); nextClip; nextClip = getNextClip()) { - // If the clip is too far in the future, just break out of the handling loop - Time framePosition = nextClip->position(); - if (framePosition > triggerPosition) { + auto currentPosition = Frame::frameTimeFromEpoch(_startEpoch); + if ((currentPosition - startingPosition) >= MAX_FRAME_PROCESSING_TIME) { + qCWarning(recordingLog) << "Exceeded maximum frame processing time, breaking early"; break; } + // If the clip is too far in the future, just break out of the handling loop + Frame::Time framePosition = nextClip->positionFrameTime(); + if (framePosition > triggerPosition) { + break; + } // Handle the frame and advance the clip Frame::handleFrame(nextClip->nextFrame()); } - if (!nextClip) { qCDebug(recordingLog) << "No more frames available"; // No more frames available, so handle the end of playback @@ -107,6 +138,9 @@ void Deck::processFrames() { qCDebug(recordingLog) << "Looping enabled, seeking back to beginning"; // If we have looping enabled, start the playback over seek(0); + // FIXME configure the recording scripting interface to reset the avatar basis on a loop + // if doing relative movement + emit looped(); } else { // otherwise pause playback pause(); @@ -115,9 +149,67 @@ void Deck::processFrames() { } // If we have more clip frames available, set the timer for the next one - Time nextClipPosition = nextClip->position(); - Time interval = nextClipPosition - _position; + _position = Frame::frameTimeFromEpoch(_startEpoch); + auto nextFrameTime = nextClip->positionFrameTime(); + auto interval = Frame::frameTimeToMilliseconds(nextFrameTime - _position); _timer.singleShot(interval, [this] { processFrames(); }); } + +void Deck::removeClip(const ClipConstPointer& clip) { + Locker lock(_mutex); + std::remove_if(_clips.begin(), _clips.end(), [&](const Clip::ConstPointer& testClip)->bool { + return (clip == testClip); + }); +} + +void Deck::removeClip(const QString& clipName) { + Locker lock(_mutex); + std::remove_if(_clips.begin(), _clips.end(), [&](const Clip::ConstPointer& clip)->bool { + return (clip->getName() == clipName); + }); +} + +void Deck::removeAllClips() { + Locker lock(_mutex); + _clips.clear(); +} + +Deck::ClipList Deck::getClips(const QString& clipName) const { + Locker lock(_mutex); + ClipList result = _clips; + return result; +} + + +bool Deck::isPlaying() { + Locker lock(_mutex); + return !_pause; +} + +bool Deck::isPaused() const { + Locker lock(_mutex); + return _pause; +} + +void Deck::stop() { + Locker lock(_mutex); + pause(); + seek(0.0f); +} + +float Deck::length() const { + Locker lock(_mutex); + return _length; +} + +void Deck::loop(bool enable) { + Locker lock(_mutex); + _loop = enable; +} + +bool Deck::isLooping() const { + Locker lock(_mutex); + return _loop; +} diff --git a/libraries/recording/src/recording/Deck.h b/libraries/recording/src/recording/Deck.h index 7086e9759d..1f8d58d5e1 100644 --- a/libraries/recording/src/recording/Deck.h +++ b/libraries/recording/src/recording/Deck.h @@ -12,56 +12,70 @@ #include #include +#include #include #include +#include + +#include #include "Forward.h" +#include "Frame.h" namespace recording { -class Deck : public QObject { +class Deck : public QObject, public ::Dependency { Q_OBJECT public: + using ClipList = std::list; using Pointer = std::shared_ptr; - Deck(QObject* parent = nullptr) : QObject(parent) {} + + Deck(QObject* parent = nullptr); // Place a clip on the deck for recording or playback - void queueClip(ClipPointer clip, Time timeOffset = 0.0f); + void queueClip(ClipPointer clip, float timeOffset = 0.0f); + void removeClip(const ClipConstPointer& clip); + void removeClip(const QString& clipName); + void removeAllClips(); + ClipList getClips(const QString& clipName) const; void play(); - bool isPlaying() { return !_pause; } + bool isPlaying(); void pause(); - bool isPaused() const { return _pause; } + bool isPaused() const; - void stop() { pause(); seek(0.0f); } + void stop(); - Time length() const { return _length; } + float length() const; - void loop(bool enable = true) { _loop = enable; } - bool isLooping() const { return _loop; } + void loop(bool enable = true); + bool isLooping() const; - Time position() const; - void seek(Time position); + float position() const; + void seek(float position); signals: void playbackStateChanged(); + void looped(); private: - using Clips = std::list; + using Mutex = std::recursive_mutex; + using Locker = std::unique_lock; ClipPointer getNextClip(); void processFrames(); + mutable Mutex _mutex; QTimer _timer; - Clips _clips; + ClipList _clips; quint64 _startEpoch { 0 }; - Time _position { 0 }; + Frame::Time _position { 0 }; bool _pause { true }; bool _loop { false }; - Time _length { 0 }; + float _length { 0 }; }; } diff --git a/libraries/recording/src/recording/Forward.h b/libraries/recording/src/recording/Forward.h index 4ba54e23a3..1bc9b31ea9 100644 --- a/libraries/recording/src/recording/Forward.h +++ b/libraries/recording/src/recording/Forward.h @@ -16,10 +16,6 @@ namespace recording { -using Time = uint32_t; - -static const Time INVALID_TIME = std::numeric_limits::max(); - using FrameType = uint16_t; using FrameSize = uint16_t; @@ -36,16 +32,14 @@ class Clip; using ClipPointer = std::shared_ptr; +using ClipConstPointer = std::shared_ptr; + // An interface for playing back clips class Deck; -using DeckPointer = std::shared_ptr; - // An interface for recording a single clip class Recorder; -using RecorderPointer = std::shared_ptr; - } #endif diff --git a/libraries/recording/src/recording/Frame.cpp b/libraries/recording/src/recording/Frame.cpp index 5b0116519f..bff85ea872 100644 --- a/libraries/recording/src/recording/Frame.cpp +++ b/libraries/recording/src/recording/Frame.cpp @@ -12,6 +12,9 @@ #include +#include +#include + using namespace recording; // FIXME move to shared @@ -73,7 +76,31 @@ using Locker = std::unique_lock; static Mutex mutex; static std::once_flag once; +float FrameHeader::frameTimeToSeconds(Frame::Time frameTime) { + float result = frameTime; + result /= MSECS_PER_SECOND; + return result; +} +uint32_t FrameHeader::frameTimeToMilliseconds(Frame::Time frameTime) { + return frameTime; +} + +Frame::Time FrameHeader::frameTimeFromEpoch(quint64 epoch) { + auto intervalMicros = (usecTimestampNow() - epoch); + intervalMicros /= USECS_PER_MSEC; + return (Frame::Time)(intervalMicros); +} + +quint64 FrameHeader::epochForFrameTime(Time frameTime) { + auto epoch = usecTimestampNow(); + epoch -= (frameTime * USECS_PER_MSEC); + return epoch; +} + +Frame::Time FrameHeader::secondsToFrameTime(float seconds) { + return (Time)(seconds * MSECS_PER_SECOND); +} FrameType Frame::registerFrameType(const QString& frameTypeName) { Locker lock(mutex); diff --git a/libraries/recording/src/recording/Frame.h b/libraries/recording/src/recording/Frame.h index f0f53ce144..3cc999f505 100644 --- a/libraries/recording/src/recording/Frame.h +++ b/libraries/recording/src/recording/Frame.h @@ -13,26 +13,46 @@ #include "Forward.h" #include +#include #include namespace recording { -struct Frame { +struct FrameHeader { + using Time = uint32_t; + + static const Time INVALID_TIME = UINT32_MAX; + static const FrameType TYPE_INVALID = 0xFFFF; + static const FrameType TYPE_HEADER = 0x0; + + static Time secondsToFrameTime(float seconds); + static float frameTimeToSeconds(Time frameTime); + + static uint32_t frameTimeToMilliseconds(Time frameTime); + + static Time frameTimeFromEpoch(quint64 epoch); + static quint64 epochForFrameTime(Time frameTime); + + FrameType type { TYPE_INVALID }; + Time timeOffset { 0 }; // milliseconds + + FrameHeader() {} + FrameHeader(FrameType type, Time timeOffset) + : type(type), timeOffset(timeOffset) { } +}; + +struct Frame : public FrameHeader { public: using Pointer = std::shared_ptr; using ConstPointer = std::shared_ptr; using Handler = std::function; - static const FrameType TYPE_INVALID = 0xFFFF; - static const FrameType TYPE_HEADER = 0x0; - FrameType type { TYPE_INVALID }; - Time timeOffset { 0 }; // milliseconds QByteArray data; Frame() {} Frame(FrameType type, float timeOffset, const QByteArray& data) - : type(type), timeOffset(timeOffset), data(data) {} + : FrameHeader(type, timeOffset), data(data) { } static FrameType registerFrameType(const QString& frameTypeName); static QMap getFrameTypes(); diff --git a/libraries/recording/src/recording/Recorder.cpp b/libraries/recording/src/recording/Recorder.cpp index f007367cae..aae31f8ec0 100644 --- a/libraries/recording/src/recording/Recorder.cpp +++ b/libraries/recording/src/recording/Recorder.cpp @@ -16,20 +16,23 @@ using namespace recording; -Recorder::~Recorder() { +Recorder::Recorder(QObject* parent) + : QObject(parent) {} -} - -Time Recorder::position() { +float Recorder::position() { + Locker lock(_mutex); + if (_clip) { + return _clip->duration(); + } return 0.0f; } void Recorder::start() { + Locker lock(_mutex); if (!_recording) { _recording = true; - if (!_clip) { - _clip = std::make_shared(); - } + // FIXME for now just record a new clip every time + _clip = std::make_shared(); _startEpoch = usecTimestampNow(); _timer.start(); emit recordingStateChanged(); @@ -37,6 +40,7 @@ void Recorder::start() { } void Recorder::stop() { + Locker lock(_mutex); if (_recording) { _recording = false; _elapsed = _timer.elapsed(); @@ -45,14 +49,17 @@ void Recorder::stop() { } bool Recorder::isRecording() { + Locker lock(_mutex); return _recording; } void Recorder::clear() { + Locker lock(_mutex); _clip.reset(); } void Recorder::recordFrame(FrameType type, QByteArray frameData) { + Locker lock(_mutex); if (!_recording || !_clip) { return; } @@ -65,6 +72,7 @@ void Recorder::recordFrame(FrameType type, QByteArray frameData) { } ClipPointer Recorder::getClip() { + Locker lock(_mutex); return _clip; } diff --git a/libraries/recording/src/recording/Recorder.h b/libraries/recording/src/recording/Recorder.h index f8346456d4..abbc964389 100644 --- a/libraries/recording/src/recording/Recorder.h +++ b/libraries/recording/src/recording/Recorder.h @@ -10,24 +10,25 @@ #ifndef hifi_Recording_Recorder_h #define hifi_Recording_Recorder_h -#include "Forward.h" +#include #include #include +#include + +#include "Forward.h" + namespace recording { // An interface for interacting with clips, creating them by recording or // playing them back. Also serialization to and from files / network sources -class Recorder : public QObject { +class Recorder : public QObject, public Dependency { Q_OBJECT public: - using Pointer = std::shared_ptr; + Recorder(QObject* parent = nullptr); - Recorder(QObject* parent = nullptr) : QObject(parent) {} - virtual ~Recorder(); - - Time position(); + float position(); // Start recording frames void start(); @@ -49,6 +50,10 @@ signals: void recordingStateChanged(); private: + using Mutex = std::recursive_mutex; + using Locker = std::unique_lock; + + Mutex _mutex; QElapsedTimer _timer; ClipPointer _clip; quint64 _elapsed { 0 }; diff --git a/libraries/recording/src/recording/impl/ArrayClip.h b/libraries/recording/src/recording/impl/ArrayClip.h new file mode 100644 index 0000000000..10b3580228 --- /dev/null +++ b/libraries/recording/src/recording/impl/ArrayClip.h @@ -0,0 +1,100 @@ +// +// Created by Bradley Austin Davis 2015/11/05 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#pragma once +#ifndef hifi_Recording_Impl_ArrayClip_h +#define hifi_Recording_Impl_ArrayClip_h + +#include "../Clip.h" + +#include + +namespace recording { + +template +class ArrayClip : public Clip { +public: + virtual float duration() const override { + Locker lock(_mutex); + if (_frames.empty()) { + return 0; + } + return Frame::frameTimeToSeconds((*_frames.rbegin()).timeOffset); + } + + virtual size_t frameCount() const override { + Locker lock(_mutex); + return _frames.size(); + } + + Clip::Pointer duplicate() const { + auto result = newClip(); + Locker lock(_mutex); + for (size_t i = 0; i < _frames.size(); ++i) { + result->addFrame(readFrame(i)); + } + return result; + } + + virtual void seekFrameTime(Frame::Time offset) { + Locker lock(_mutex); + auto itr = std::lower_bound(_frames.begin(), _frames.end(), offset, + [](const T& a, Frame::Time b)->bool { + return a.timeOffset < b; + } + ); + _frameIndex = itr - _frames.begin(); + } + + virtual Frame::Time positionFrameTime() const override { + Locker lock(_mutex); + Frame::Time result = Frame::INVALID_TIME; + if (_frameIndex < _frames.size()) { + result = _frames[_frameIndex].timeOffset; + } + return result; + } + + virtual FrameConstPointer peekFrame() const override { + Locker lock(_mutex); + FrameConstPointer result; + if (_frameIndex < _frames.size()) { + result = readFrame(_frameIndex); + } + return result; + } + + virtual FrameConstPointer nextFrame() override { + Locker lock(_mutex); + FrameConstPointer result; + if (_frameIndex < _frames.size()) { + result = readFrame(_frameIndex++); + } + return result; + } + + virtual void skipFrame() override { + Locker lock(_mutex); + if (_frameIndex < _frames.size()) { + ++_frameIndex; + } + } + +protected: + virtual void reset() override { + _frameIndex = 0; + } + + virtual FrameConstPointer readFrame(size_t index) const = 0; + std::vector _frames; + mutable size_t _frameIndex { 0 }; +}; + +} + +#endif diff --git a/libraries/recording/src/recording/impl/BufferClip.cpp b/libraries/recording/src/recording/impl/BufferClip.cpp index 87bbfbfef7..c40d9dd42a 100644 --- a/libraries/recording/src/recording/impl/BufferClip.cpp +++ b/libraries/recording/src/recording/impl/BufferClip.cpp @@ -8,85 +8,40 @@ #include "BufferClip.h" -#include +#include +#include #include "../Frame.h" using namespace recording; - -void BufferClip::seek(Time offset) { - Locker lock(_mutex); - auto itr = std::lower_bound(_frames.begin(), _frames.end(), offset, - [](Frame::ConstPointer a, Time b)->bool { - return a->timeOffset < b; - } - ); - _frameIndex = itr - _frames.begin(); +QString BufferClip::getName() const { + return _name; } -Time BufferClip::position() const { - Locker lock(_mutex); - Time result = INVALID_TIME; - if (_frameIndex < _frames.size()) { - result = _frames[_frameIndex]->timeOffset; - } - return result; -} - -FrameConstPointer BufferClip::peekFrame() const { - Locker lock(_mutex); - FrameConstPointer result; - if (_frameIndex < _frames.size()) { - result = _frames[_frameIndex]; - } - return result; -} - -FrameConstPointer BufferClip::nextFrame() { - Locker lock(_mutex); - FrameConstPointer result; - if (_frameIndex < _frames.size()) { - result = _frames[_frameIndex]; - ++_frameIndex; - } - return result; -} void BufferClip::addFrame(FrameConstPointer newFrame) { if (newFrame->timeOffset < 0.0f) { throw std::runtime_error("Frames may not have negative time offsets"); } - auto currentPosition = position(); - seek(newFrame->timeOffset); - { - Locker lock(_mutex); - _frames.insert(_frames.begin() + _frameIndex, newFrame); - } - seek(currentPosition); -} - -void BufferClip::skipFrame() { Locker lock(_mutex); - if (_frameIndex < _frames.size()) { - ++_frameIndex; + auto itr = std::lower_bound(_frames.begin(), _frames.end(), newFrame->timeOffset, + [](const Frame& a, Frame::Time b)->bool { + return a.timeOffset < b; + } + ); + + auto newFrameIndex = itr - _frames.begin(); + //qDebug() << "Adding frame with time offset " << newFrame->timeOffset << " @ index " << newFrameIndex; + _frames.insert(_frames.begin() + newFrameIndex, Frame(*newFrame)); +} + +// Internal only function, needs no locking +FrameConstPointer BufferClip::readFrame(size_t frameIndex) const { + FramePointer result; + if (frameIndex < _frames.size()) { + result = std::make_shared(_frames[frameIndex]); } + return result; } - -void BufferClip::reset() { - Locker lock(_mutex); - _frameIndex = 0; -} - -Time BufferClip::duration() const { - if (_frames.empty()) { - return 0; - } - return (*_frames.rbegin())->timeOffset; -} - -size_t BufferClip::frameCount() const { - return _frames.size(); -} - diff --git a/libraries/recording/src/recording/impl/BufferClip.h b/libraries/recording/src/recording/impl/BufferClip.h index ce81dac730..af8a64716b 100644 --- a/libraries/recording/src/recording/impl/BufferClip.h +++ b/libraries/recording/src/recording/impl/BufferClip.h @@ -10,33 +10,22 @@ #ifndef hifi_Recording_Impl_BufferClip_h #define hifi_Recording_Impl_BufferClip_h -#include "../Clip.h" +#include "ArrayClip.h" -#include +#include namespace recording { -class BufferClip : public Clip { +class BufferClip : public ArrayClip { public: using Pointer = std::shared_ptr; - virtual ~BufferClip() {} - - virtual Time duration() const override; - virtual size_t frameCount() const override; - - virtual void seek(Time offset) override; - virtual Time position() const override; - - virtual FrameConstPointer peekFrame() const override; - virtual FrameConstPointer nextFrame() override; - virtual void skipFrame() override; + virtual QString getName() const override; virtual void addFrame(FrameConstPointer) override; private: - virtual void reset() override; - - std::vector _frames; + virtual FrameConstPointer readFrame(size_t index) const override; + QString _name { QUuid().toString() }; mutable size_t _frameIndex { 0 }; }; diff --git a/libraries/recording/src/recording/impl/FileClip.cpp b/libraries/recording/src/recording/impl/FileClip.cpp index b8e1eb26fa..80aaac4c87 100644 --- a/libraries/recording/src/recording/impl/FileClip.cpp +++ b/libraries/recording/src/recording/impl/FileClip.cpp @@ -18,15 +18,15 @@ #include "../Frame.h" #include "../Logging.h" +#include "BufferClip.h" using namespace recording; -static const qint64 MINIMUM_FRAME_SIZE = sizeof(FrameType) + sizeof(Time) + sizeof(FrameSize); - +static const qint64 MINIMUM_FRAME_SIZE = sizeof(FrameType) + sizeof(Frame::Time) + sizeof(FrameSize); static const QString FRAME_TYPE_MAP = QStringLiteral("frameTypes"); +static const QString FRAME_COMREPSSION_FLAG = QStringLiteral("compressed"); -using FrameHeaderList = std::list; using FrameTranslationMap = QMap; FrameTranslationMap parseTranslationMap(const QJsonDocument& doc) { @@ -49,19 +49,18 @@ FrameTranslationMap parseTranslationMap(const QJsonDocument& doc) { } -FrameHeaderList parseFrameHeaders(uchar* const start, const qint64& size) { - using FrameHeader = FileClip::FrameHeader; - FrameHeaderList results; +FileFrameHeaderList parseFrameHeaders(uchar* const start, const qint64& size) { + FileFrameHeaderList results; auto current = start; auto end = current + size; // Read all the frame headers // FIXME move to Frame::readHeader? while (end - current >= MINIMUM_FRAME_SIZE) { - FrameHeader header; + FileFrameHeader header; memcpy(&(header.type), current, sizeof(FrameType)); current += sizeof(FrameType); - memcpy(&(header.timeOffset), current, sizeof(Time)); - current += sizeof(Time); + memcpy(&(header.timeOffset), current, sizeof(Frame::Time)); + current += sizeof(Frame::Time); memcpy(&(header.size), current, sizeof(FrameSize)); current += sizeof(FrameSize); header.fileOffset = current - start; @@ -72,6 +71,11 @@ FrameHeaderList parseFrameHeaders(uchar* const start, const qint64& size) { current += header.size; results.push_back(header); } + qDebug() << "Parsed source data into " << results.size() << " frames"; + int i = 0; + for (const auto& frameHeader : results) { + qDebug() << "Frame " << i++ << " time " << frameHeader.timeOffset; + } return results; } @@ -89,7 +93,7 @@ FileClip::FileClip(const QString& fileName) : _file(fileName) { return; } - FrameHeaderList parsedFrameHeaders = parseFrameHeaders(_map, size); + auto parsedFrameHeaders = parseFrameHeaders(_map, size); // Verify that at least one frame exists and that the first frame is a header if (0 == parsedFrameHeaders.size()) { @@ -110,6 +114,11 @@ FileClip::FileClip(const QString& fileName) : _file(fileName) { _fileHeader = QJsonDocument::fromBinaryData(fileHeaderData); } + // Check for compression + { + _compressed = _fileHeader.object()[FRAME_COMREPSSION_FLAG].toBool(); + } + // Find the type enum translation map and fix up the frame headers { FrameTranslationMap translationMap = parseTranslationMap(_fileHeader); @@ -120,19 +129,25 @@ FileClip::FileClip(const QString& fileName) : _file(fileName) { qDebug() << translationMap; // Update the loaded headers with the frame data - _frameHeaders.reserve(parsedFrameHeaders.size()); + _frames.reserve(parsedFrameHeaders.size()); for (auto& frameHeader : parsedFrameHeaders) { if (!translationMap.contains(frameHeader.type)) { continue; } frameHeader.type = translationMap[frameHeader.type]; - _frameHeaders.push_back(frameHeader); + _frames.push_back(frameHeader); } } + +} + + +QString FileClip::getName() const { + return _file.fileName(); } // FIXME move to frame? -bool writeFrame(QIODevice& output, const Frame& frame) { +bool writeFrame(QIODevice& output, const Frame& frame, bool compressed = true) { if (frame.type == Frame::TYPE_INVALID) { qWarning() << "Attempting to write invalid frame"; return true; @@ -142,17 +157,24 @@ bool writeFrame(QIODevice& output, const Frame& frame) { if (written != sizeof(FrameType)) { return false; } - written = output.write((char*)&(frame.timeOffset), sizeof(Time)); - if (written != sizeof(Time)) { + //qDebug() << "Writing frame with time offset " << frame.timeOffset; + written = output.write((char*)&(frame.timeOffset), sizeof(Frame::Time)); + if (written != sizeof(Frame::Time)) { return false; } - uint16_t dataSize = frame.data.size(); + QByteArray frameData = frame.data; + if (compressed) { + frameData = qCompress(frameData); + } + + uint16_t dataSize = frameData.size(); written = output.write((char*)&dataSize, sizeof(FrameSize)); if (written != sizeof(uint16_t)) { return false; } + if (dataSize != 0) { - written = output.write(frame.data); + written = output.write(frameData); if (written != dataSize) { return false; } @@ -161,7 +183,8 @@ bool writeFrame(QIODevice& output, const Frame& frame) { } bool FileClip::write(const QString& fileName, Clip::Pointer clip) { - qCDebug(recordingLog) << "Writing clip to file " << fileName; + // FIXME need to move this to a different thread + //qCDebug(recordingLog) << "Writing clip to file " << fileName << " with " << clip->frameCount() << " frames"; if (0 == clip->frameCount()) { return false; @@ -182,10 +205,14 @@ bool FileClip::write(const QString& fileName, Clip::Pointer clip) { QJsonObject rootObject; rootObject.insert(FRAME_TYPE_MAP, frameTypeObj); + // Always mark new files as compressed + rootObject.insert(FRAME_COMREPSSION_FLAG, true); QByteArray headerFrameData = QJsonDocument(rootObject).toBinaryData(); - if (!writeFrame(outputFile, Frame({ Frame::TYPE_HEADER, 0, headerFrameData }))) { + // Never compress the header frame + if (!writeFrame(outputFile, Frame({ Frame::TYPE_HEADER, 0, headerFrameData }), false)) { return false; } + } clip->seek(0); @@ -207,73 +234,24 @@ FileClip::~FileClip() { } } -void FileClip::seek(Time offset) { - Locker lock(_mutex); - auto itr = std::lower_bound(_frameHeaders.begin(), _frameHeaders.end(), offset, - [](const FrameHeader& a, Time b)->bool { - return a.timeOffset < b; - } - ); - _frameIndex = itr - _frameHeaders.begin(); -} - -Time FileClip::position() const { - Locker lock(_mutex); - Time result = INVALID_TIME; - if (_frameIndex < _frameHeaders.size()) { - result = _frameHeaders[_frameIndex].timeOffset; - } - return result; -} - -FramePointer FileClip::readFrame(uint32_t frameIndex) const { +// Internal only function, needs no locking +FrameConstPointer FileClip::readFrame(size_t frameIndex) const { FramePointer result; - if (frameIndex < _frameHeaders.size()) { + if (frameIndex < _frames.size()) { result = std::make_shared(); - const FrameHeader& header = _frameHeaders[frameIndex]; + const auto& header = _frames[frameIndex]; result->type = header.type; result->timeOffset = header.timeOffset; if (header.size) { result->data.insert(0, reinterpret_cast(_map)+header.fileOffset, header.size); + if (_compressed) { + result->data = qUncompress(result->data); + } } } return result; } -FrameConstPointer FileClip::peekFrame() const { - Locker lock(_mutex); - return readFrame(_frameIndex); -} - -FrameConstPointer FileClip::nextFrame() { - Locker lock(_mutex); - auto result = readFrame(_frameIndex); - if (_frameIndex < _frameHeaders.size()) { - ++_frameIndex; - } - return result; -} - -void FileClip::skipFrame() { - ++_frameIndex; -} - -void FileClip::reset() { - _frameIndex = 0; -} - void FileClip::addFrame(FrameConstPointer) { throw std::runtime_error("File clips are read only"); } - -Time FileClip::duration() const { - if (_frameHeaders.empty()) { - return 0; - } - return _frameHeaders.rbegin()->timeOffset; -} - -size_t FileClip::frameCount() const { - return _frameHeaders.size(); -} - diff --git a/libraries/recording/src/recording/impl/FileClip.h b/libraries/recording/src/recording/impl/FileClip.h index 18c62936c1..f103a9aca6 100644 --- a/libraries/recording/src/recording/impl/FileClip.h +++ b/libraries/recording/src/recording/impl/FileClip.h @@ -10,31 +10,34 @@ #ifndef hifi_Recording_Impl_FileClip_h #define hifi_Recording_Impl_FileClip_h -#include "../Clip.h" +#include "ArrayClip.h" + +#include #include #include -#include +#include "../Frame.h" namespace recording { -class FileClip : public Clip { +struct FileFrameHeader : public FrameHeader { + FrameType type; + Frame::Time timeOffset; + uint16_t size; + quint64 fileOffset; +}; + +using FileFrameHeaderList = std::list; + +class FileClip : public ArrayClip { public: using Pointer = std::shared_ptr; FileClip(const QString& file); virtual ~FileClip(); - virtual Time duration() const override; - virtual size_t frameCount() const override; - - virtual void seek(Time offset) override; - virtual Time position() const override; - - virtual FrameConstPointer peekFrame() const override; - virtual FrameConstPointer nextFrame() override; - virtual void skipFrame() override; + virtual QString getName() const override; virtual void addFrame(FrameConstPointer) override; const QJsonDocument& getHeader() { @@ -43,27 +46,12 @@ public: static bool write(const QString& filePath, Clip::Pointer clip); - struct FrameHeader { - FrameType type; - Time timeOffset; - uint16_t size; - quint64 fileOffset; - }; - private: - - virtual void reset() override; - - - using FrameHeaderVector = std::vector; - - FramePointer readFrame(uint32_t frameIndex) const; - + virtual FrameConstPointer readFrame(size_t index) const override; QJsonDocument _fileHeader; QFile _file; - uint32_t _frameIndex { 0 }; uchar* _map { nullptr }; - FrameHeaderVector _frameHeaders; + bool _compressed { true }; }; } diff --git a/libraries/recording/src/recording/impl/OffsetClip.cpp b/libraries/recording/src/recording/impl/OffsetClip.cpp index bccd48d6c8..afca9e0b7a 100644 --- a/libraries/recording/src/recording/impl/OffsetClip.cpp +++ b/libraries/recording/src/recording/impl/OffsetClip.cpp @@ -22,15 +22,15 @@ using namespace recording; -OffsetClip::OffsetClip(const Clip::Pointer& wrappedClip, Time offset) - : WrapperClip(wrappedClip), _offset(offset) { } +OffsetClip::OffsetClip(const Clip::Pointer& wrappedClip, float offset) + : WrapperClip(wrappedClip), _offset(Frame::secondsToFrameTime(offset)) { } -void OffsetClip::seek(Time offset) { - _wrappedClip->seek(offset - _offset); +void OffsetClip::seekFrameTime(Frame::Time offset) { + _wrappedClip->seekFrameTime(offset - _offset); } -Time OffsetClip::position() const { - return _wrappedClip->position() + _offset; +Frame::Time OffsetClip::positionFrameTime() const { + return _wrappedClip->positionFrameTime() + _offset; } FrameConstPointer OffsetClip::peekFrame() const { @@ -45,7 +45,18 @@ FrameConstPointer OffsetClip::nextFrame() { return result; } -Time OffsetClip::duration() const { +float OffsetClip::duration() const { return _wrappedClip->duration() + _offset; } +QString OffsetClip::getName() const { + return _wrappedClip->getName(); +} + +Clip::Pointer OffsetClip::duplicate() const { + return std::make_shared( + _wrappedClip->duplicate(), Frame::frameTimeToSeconds(_offset)); +} + + + diff --git a/libraries/recording/src/recording/impl/OffsetClip.h b/libraries/recording/src/recording/impl/OffsetClip.h index 1c6b005b65..40301adf59 100644 --- a/libraries/recording/src/recording/impl/OffsetClip.h +++ b/libraries/recording/src/recording/impl/OffsetClip.h @@ -18,18 +18,20 @@ class OffsetClip : public WrapperClip { public: using Pointer = std::shared_ptr; - OffsetClip(const Clip::Pointer& wrappedClip, Time offset); - virtual ~OffsetClip(); + OffsetClip(const Clip::Pointer& wrappedClip, float offset); - virtual Time duration() const override; - virtual void seek(Time offset) override; - virtual Time position() const override; + virtual QString getName() const override; + + virtual Clip::Pointer duplicate() const override; + virtual float duration() const override; + virtual void seekFrameTime(Frame::Time offset) override; + virtual Frame::Time positionFrameTime() const override; virtual FrameConstPointer peekFrame() const override; virtual FrameConstPointer nextFrame() override; protected: - const Time _offset; + const Frame::Time _offset; }; } diff --git a/libraries/recording/src/recording/impl/WrapperClip.cpp b/libraries/recording/src/recording/impl/WrapperClip.cpp index f2bbacabf1..955dd47a5e 100644 --- a/libraries/recording/src/recording/impl/WrapperClip.cpp +++ b/libraries/recording/src/recording/impl/WrapperClip.cpp @@ -22,11 +22,11 @@ using namespace recording; WrapperClip::WrapperClip(const Clip::Pointer& wrappedClip) : _wrappedClip(wrappedClip) { } -void WrapperClip::seek(Time offset) { - _wrappedClip->seek(offset); +void WrapperClip::seekFrameTime(Frame::Time offset) { + _wrappedClip->seekFrameTime(offset); } -Time WrapperClip::position() const { +Frame::Time WrapperClip::positionFrameTime() const { return _wrappedClip->position(); } @@ -50,7 +50,7 @@ void WrapperClip::addFrame(FrameConstPointer) { throw std::runtime_error("Wrapper clips are read only"); } -Time WrapperClip::duration() const { +float WrapperClip::duration() const { return _wrappedClip->duration(); } diff --git a/libraries/recording/src/recording/impl/WrapperClip.h b/libraries/recording/src/recording/impl/WrapperClip.h index 3fe013e0ed..77a484b5f7 100644 --- a/libraries/recording/src/recording/impl/WrapperClip.h +++ b/libraries/recording/src/recording/impl/WrapperClip.h @@ -24,13 +24,12 @@ public: using Pointer = std::shared_ptr; WrapperClip(const Clip::Pointer& wrappedClip); - virtual ~WrapperClip(); - virtual Time duration() const override; + virtual float duration() const override; virtual size_t frameCount() const override; - virtual void seek(Time offset) override; - virtual Time position() const override; + virtual void seekFrameTime(Frame::Time offset) override; + virtual Frame::Time positionFrameTime() const override; virtual FrameConstPointer peekFrame() const override; virtual FrameConstPointer nextFrame() override; From d099f61170859097ed649e8ae179fd462d7fc078 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Mon, 16 Nov 2015 14:57:24 -0800 Subject: [PATCH 05/36] Updating audio record/playback mechanism to more closely match actual audio input --- .../scripting/RecordingScriptingInterface.cpp | 78 +------------------ .../scripting/RecordingScriptingInterface.h | 1 - libraries/audio-client/src/AudioClient.cpp | 33 ++++++++ libraries/audio-client/src/AudioClient.h | 1 + 4 files changed, 38 insertions(+), 75 deletions(-) diff --git a/interface/src/scripting/RecordingScriptingInterface.cpp b/interface/src/scripting/RecordingScriptingInterface.cpp index 7e0c763dfa..bf585f5481 100644 --- a/interface/src/scripting/RecordingScriptingInterface.cpp +++ b/interface/src/scripting/RecordingScriptingInterface.cpp @@ -156,7 +156,6 @@ void RecordingScriptingInterface::startRecording() { return; } - if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "startRecording", Qt::BlockingQueuedConnection); return; @@ -164,78 +163,16 @@ void RecordingScriptingInterface::startRecording() { _recordingEpoch = Frame::epochForFrameTime(0); - _audioRecordingBuffer.clear(); auto myAvatar = DependencyManager::get()->getMyAvatar(); myAvatar->setRecordingBasis(); _recorder->start(); } -float calculateAudioTime(const QByteArray& audio) { - static const float AUDIO_BYTES_PER_SECOND = AudioConstants::SAMPLE_RATE * sizeof(AudioConstants::AudioSample); - return (float)audio.size() / AUDIO_BYTES_PER_SECOND; -} - -void injectAudioFrame(Clip::Pointer& clip, Frame::Time time, const QByteArray& audio) { - static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME); - clip->addFrame(std::make_shared(AUDIO_FRAME_TYPE, time, audio)); -} - -// Detect too much audio in a single frame, or too much deviation between -// the expected audio length and the computed audio length -bool shouldStartNewAudioFrame(const QByteArray& currentAudioFrame, float expectedAudioLength) { - if (currentAudioFrame.isEmpty()) { - return true; - } - - // 100 milliseconds - float actualAudioLength = calculateAudioTime(currentAudioFrame); - static const float MAX_AUDIO_PACKET_DURATION = 1.0f; - if (actualAudioLength >= MAX_AUDIO_PACKET_DURATION) { - return true; - } - - - float deviation = std::abs(actualAudioLength - expectedAudioLength); - - qDebug() << "Checking buffer deviation current length "; - qDebug() << "Actual: " << actualAudioLength; - qDebug() << "Expected: " << expectedAudioLength; - qDebug() << "Deviation: " << deviation; - - static const float MAX_AUDIO_DEVIATION = 0.1f; - if (deviation >= MAX_AUDIO_PACKET_DURATION) { - return true; - } - - return false; -} - - -void injectAudioFrames(Clip::Pointer& clip, const QList>& audioBuffer) { - Frame::Time lastAudioStartTime = 0; - QByteArray audioFrameBuffer; - for (const auto& audioPacket : audioBuffer) { - float expectedAudioLength = Frame::frameTimeToSeconds(audioPacket.first - lastAudioStartTime); - if (shouldStartNewAudioFrame(audioFrameBuffer, expectedAudioLength)) { - // Time to start a new frame, inject the old one if it exists - if (audioFrameBuffer.size()) { - injectAudioFrame(clip, lastAudioStartTime, audioFrameBuffer); - audioFrameBuffer.clear(); - } - lastAudioStartTime = audioPacket.first; - } - audioFrameBuffer.append(audioPacket.second); - } -} - - void RecordingScriptingInterface::stopRecording() { _recorder->stop(); _lastClip = _recorder->getClip(); // post-process the audio into discreet chunks based on times of received samples - injectAudioFrames(_lastClip, _audioRecordingBuffer); - _audioRecordingBuffer.clear(); _lastClip->seek(0); Frame::ConstPointer frame; while (frame = _lastClip->nextFrame()) { @@ -310,19 +247,12 @@ void RecordingScriptingInterface::processAvatarFrame(const Frame::ConstPointer& void RecordingScriptingInterface::processAudioInput(const QByteArray& audio) { if (_recorder->isRecording()) { - auto audioFrameTime = Frame::frameTimeFromEpoch(_recordingEpoch); - _audioRecordingBuffer.push_back({ audioFrameTime, audio }); - qDebug() << "Got sound packet of size " << audio.size() << " At time " << audioFrameTime; + static const recording::FrameType AUDIO_FRAME_TYPE = recording::Frame::registerFrameType(AUDIO_FRAME_NAME); + _recorder->recordFrame(AUDIO_FRAME_TYPE, audio); } } void RecordingScriptingInterface::processAudioFrame(const recording::FrameConstPointer& frame) { - AudioInjectorOptions options; - auto myAvatar = DependencyManager::get()->getMyAvatar(); - options.position = myAvatar->getPosition(); - options.orientation = myAvatar->getOrientation(); - // FIXME store the audio format (sample rate, bits, stereo) in the frame - options.stereo = false; - // FIXME move audio injector to a thread pool model? - AudioInjector::playSoundAndDelete(frame->data, options, nullptr); + auto audioClient = DependencyManager::get(); + audioClient->handleRecordedAudioInput(frame->data); } diff --git a/interface/src/scripting/RecordingScriptingInterface.h b/interface/src/scripting/RecordingScriptingInterface.h index f7add9480b..510a4b6898 100644 --- a/interface/src/scripting/RecordingScriptingInterface.h +++ b/interface/src/scripting/RecordingScriptingInterface.h @@ -65,7 +65,6 @@ private: void processAudioInput(const QByteArray& audioData); QSharedPointer _player; QSharedPointer _recorder; - QList> _audioRecordingBuffer; quint64 _recordingEpoch { 0 }; Flag _playFromCurrentLocation { true }; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 72e47073f2..a506fe217c 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -904,6 +904,39 @@ void AudioClient::handleAudioInput() { } } +void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { + if (!_audioPacket) { + // we don't have an audioPacket yet - set that up now + _audioPacket = NLPacket::create(PacketType::MicrophoneAudioWithEcho); + } + // FIXME either discard stereo in the recording or record a stereo flag + const int numNetworkBytes = _isStereoInput + ? AudioConstants::NETWORK_FRAME_BYTES_STEREO + : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkSamples = _isStereoInput + ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO + : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + + auto nodeList = DependencyManager::get(); + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + if (audioMixer && audioMixer->getActiveSocket()) { + glm::vec3 headPosition = _positionGetter(); + glm::quat headOrientation = _orientationGetter(); + quint8 isStereo = _isStereoInput ? 1 : 0; + _audioPacket->reset(); + _audioPacket->setType(PacketType::MicrophoneAudioWithEcho); + _audioPacket->writePrimitive(_outgoingAvatarAudioSequenceNumber); + _audioPacket->writePrimitive(isStereo); + _audioPacket->writePrimitive(headPosition); + _audioPacket->writePrimitive(headOrientation); + _audioPacket->write(audio); + _stats.sentPacket(); + nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::SendAudioPacket); + nodeList->sendUnreliablePacket(*_audioPacket, *audioMixer); + _outgoingAvatarAudioSequenceNumber++; + } +} + void AudioClient::processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer) { const int numNetworkOutputSamples = inputBuffer.size() / sizeof(int16_t); const int numDeviceOutputSamples = numNetworkOutputSamples * (_outputFormat.sampleRate() * _outputFormat.channelCount()) diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index e699ee9266..7d2b5a783f 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -147,6 +147,7 @@ public slots: void sendDownstreamAudioStatsPacket() { _stats.sendDownstreamAudioStatsPacket(); } void handleAudioInput(); + void handleRecordedAudioInput(const QByteArray& audio); void reset(); void audioMixerKilled(); void toggleMute(); From b3b73e8cd110293af3778f4179568035726a6552 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 17 Nov 2015 12:02:35 +1300 Subject: [PATCH 06/36] Fix particle aging --- libraries/entities/src/ParticleEffectEntityItem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/entities/src/ParticleEffectEntityItem.cpp b/libraries/entities/src/ParticleEffectEntityItem.cpp index 263d7dce0c..06fcdb495c 100644 --- a/libraries/entities/src/ParticleEffectEntityItem.cpp +++ b/libraries/entities/src/ParticleEffectEntityItem.cpp @@ -662,7 +662,7 @@ void ParticleEffectEntityItem::stepSimulation(float deltaTime) { // move head forward _particleHeadIndex = (_particleHeadIndex + 1) % _maxParticles; } else { - float age = 1.0f - _particleLifetimes[i] / _lifespan; // 0.0 .. 1.0 + float age = _particleLifetimes[i] / _lifespan; // 0.0 .. 1.0 updateRadius(i, age); updateColor(i, age); updateAlpha(i, age); From d42a1a721f50e5de110ba114b192cd7125dc429d Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 16 Nov 2015 15:26:17 -0800 Subject: [PATCH 07/36] first cut at messages-mixer --- assignment-client/src/Agent.cpp | 11 +- assignment-client/src/Agent.h | 1 + assignment-client/src/AssignmentFactory.cpp | 3 + .../src/messages/MessagesMixer.cpp | 535 ++++++++++++++++++ .../src/messages/MessagesMixer.h | 58 ++ .../src/messages/MessagesMixerClientData.cpp | 55 ++ .../src/messages/MessagesMixerClientData.h | 105 ++++ .../resources/describe-settings.json | 16 + domain-server/src/DomainGatekeeper.cpp | 5 +- domain-server/src/DomainServer.cpp | 1 - .../src/DomainServerSettingsManager.cpp | 5 +- libraries/networking/src/Assignment.cpp | 4 + libraries/networking/src/Assignment.h | 2 +- libraries/networking/src/DomainHandler.cpp | 2 + libraries/networking/src/MessagesClient.cpp | 113 ++++ libraries/networking/src/MessagesClient.h | 40 ++ libraries/networking/src/NetworkLogging.cpp | 1 + libraries/networking/src/NetworkLogging.h | 1 + libraries/networking/src/Node.cpp | 1 + libraries/networking/src/NodeType.h | 1 + .../networking/src/ThreadedAssignment.cpp | 3 + libraries/networking/src/udt/PacketHeaders.h | 3 +- 22 files changed, 959 insertions(+), 7 deletions(-) create mode 100644 assignment-client/src/messages/MessagesMixer.cpp create mode 100644 assignment-client/src/messages/MessagesMixer.h create mode 100644 assignment-client/src/messages/MessagesMixerClientData.cpp create mode 100644 assignment-client/src/messages/MessagesMixerClientData.h create mode 100644 libraries/networking/src/MessagesClient.cpp create mode 100644 libraries/networking/src/MessagesClient.h diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 063bf24de8..0d719d6806 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -55,6 +55,7 @@ Agent::Agent(NLPacket& packet) : { PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase }, this, "handleOctreePacket"); packetReceiver.registerListener(PacketType::Jurisdiction, this, "handleJurisdictionPacket"); + packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagePacket"); } void Agent::handleOctreePacket(QSharedPointer packet, SharedNodePointer senderNode) { @@ -93,7 +94,15 @@ void Agent::handleJurisdictionPacket(QSharedPointer packet, SharedNode DependencyManager::get()->getJurisdictionListener()-> queueReceivedPacket(packet, senderNode); } -} +} + +void Agent::handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode) { + auto packetType = packet->getType(); + + if (packetType == PacketType::MessagesData) { + qDebug() << "got a messages packet"; + } +} void Agent::handleAudioPacket(QSharedPointer packet) { _receivedAudioStream.parseData(*packet); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index ab000015d5..be3a0db293 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -58,6 +58,7 @@ private slots: void handleAudioPacket(QSharedPointer packet); void handleOctreePacket(QSharedPointer packet, SharedNodePointer senderNode); void handleJurisdictionPacket(QSharedPointer packet, SharedNodePointer senderNode); + void handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode); void sendPingRequests(); void processAgentAvatarAndAudio(float deltaTime); diff --git a/assignment-client/src/AssignmentFactory.cpp b/assignment-client/src/AssignmentFactory.cpp index cacc523ebd..c4cd6821ef 100644 --- a/assignment-client/src/AssignmentFactory.cpp +++ b/assignment-client/src/AssignmentFactory.cpp @@ -17,6 +17,7 @@ #include "avatars/AvatarMixer.h" #include "entities/EntityServer.h" #include "assets/AssetServer.h" +#include "messages/MessagesMixer.h" ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { @@ -36,6 +37,8 @@ ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) { return new EntityServer(packet); case Assignment::AssetServerType: return new AssetServer(packet); + case Assignment::MessagesMixerType: + return new MessagesMixer(packet); default: return NULL; } diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp new file mode 100644 index 0000000000..70b0c1b2cf --- /dev/null +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -0,0 +1,535 @@ +// +// MessagesMixer.cpp +// assignment-client/src/messages +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "MessagesMixerClientData.h" +#include "MessagesMixer.h" + +const QString MESSAGES_MIXER_LOGGING_NAME = "messages-mixer"; + +const int MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND = 60; +const unsigned int MESSAGES_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; + +MessagesMixer::MessagesMixer(NLPacket& packet) : + ThreadedAssignment(packet), + _broadcastThread(), + _lastFrameTimestamp(QDateTime::currentMSecsSinceEpoch()), + _trailingSleepRatio(1.0f), + _performanceThrottlingRatio(0.0f), + _sumListeners(0), + _numStatFrames(0), + _sumBillboardPackets(0), + _sumIdentityPackets(0) +{ + // make sure we hear about node kills so we can tell the other nodes + connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &MessagesMixer::nodeKilled); + + auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); + packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesDataPacket"); +} + +MessagesMixer::~MessagesMixer() { + if (_broadcastTimer) { + _broadcastTimer->deleteLater(); + } + + _broadcastThread.quit(); + _broadcastThread.wait(); +} + +// An 80% chance of sending a identity packet within a 5 second interval. +// assuming 60 htz update rate. +const float BILLBOARD_AND_IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; + +void MessagesMixer::broadcastMessagesData() { + qDebug() << "MessagesMixer::broadcastMessagesData()..."; + + int idleTime = QDateTime::currentMSecsSinceEpoch() - _lastFrameTimestamp; + + ++_numStatFrames; + + const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; + const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; + + const float RATIO_BACK_OFF = 0.02f; + + const int TRAILING_AVERAGE_FRAMES = 100; + int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; + + const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; + const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; + + // NOTE: The following code calculates the _performanceThrottlingRatio based on how much the messages-mixer was + // able to sleep. This will eventually be used to ask for an additional messages-mixer to help out. Currently the value + // is unused as it is assumed this should not be hit before the messages-mixer hits the desired bandwidth limit per client. + // It is reported in the domain-server stats for the messages-mixer. + + _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) + + (idleTime * CURRENT_FRAME_RATIO / (float) MESSAGES_DATA_SEND_INTERVAL_MSECS); + + float lastCutoffRatio = _performanceThrottlingRatio; + bool hasRatioChanged = false; + + if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { + if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { + // we're struggling - change our performance throttling ratio + _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); + + qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; + hasRatioChanged = true; + } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { + // we've recovered and can back off the performance throttling + _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; + + if (_performanceThrottlingRatio < 0) { + _performanceThrottlingRatio = 0; + } + + qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" + << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; + hasRatioChanged = true; + } + + if (hasRatioChanged) { + framesSinceCutoffEvent = 0; + } + } + + if (!hasRatioChanged) { + ++framesSinceCutoffEvent; + } + + auto nodeList = DependencyManager::get(); + + // setup for distributed random floating point values + std::random_device randomDevice; + std::mt19937 generator(randomDevice()); + std::uniform_real_distribution distribution; + + qDebug() << "MessagesMixer::broadcastMessagesData()... calling nodeList->eachMatchingNode()"; + + nodeList->eachMatchingNode( + [&](const SharedNodePointer& node)->bool { + if (!node->getLinkedData()) { + return false; + } + if (node->getType() != NodeType::Agent) { + return false; + } + if (!node->getActiveSocket()) { + return false; + } + return true; + }, + [&](const SharedNodePointer& node) { + MessagesMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + MutexTryLocker lock(nodeData->getMutex()); + if (!lock.isLocked()) { + return; + } + ++_sumListeners; + + AvatarData& avatar = nodeData->getAvatar(); + glm::vec3 myPosition = avatar.getPosition(); + + // reset the internal state for correct random number distribution + distribution.reset(); + + // reset the max distance for this frame + float maxAvatarDistanceThisFrame = 0.0f; + + // reset the number of sent avatars + nodeData->resetNumAvatarsSentLastFrame(); + + // keep a counter of the number of considered avatars + int numOtherAvatars = 0; + + // keep track of outbound data rate specifically for avatar data + int numAvatarDataBytes = 0; + + // keep track of the number of other avatars held back in this frame + int numAvatarsHeldBack = 0; + + // keep track of the number of other avatar frames skipped + int numAvatarsWithSkippedFrames = 0; + + // use the data rate specifically for avatar data for FRD adjustment checks + float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps(); + + // Check if it is time to adjust what we send this client based on the observed + // bandwidth to this node. We do this once a second, which is also the window for + // the bandwidth reported by node->getOutboundBandwidth(); + if (nodeData->getNumFramesSinceFRDAdjustment() > MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND) { + + const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f; + const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO); + const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f)); + + // get the current full rate distance so we can work with it + float currentFullRateDistance = nodeData->getFullRateDistance(); + + if (avatarDataRateLastSecond > _maxKbpsPerNode) { + + // is the FRD greater than the farthest avatar? + // if so, before we calculate anything, set it to that distance + currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance()); + + // we're adjusting the full rate distance to target a bandwidth in the middle + // of the hysterisis gap + currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; + + nodeData->setFullRateDistance(currentFullRateDistance); + nodeData->resetNumFramesSinceFRDAdjustment(); + } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance() + && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) { + // we are constrained AND we've recovered to below the acceptable ratio + // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap + currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; + + nodeData->setFullRateDistance(currentFullRateDistance); + nodeData->resetNumFramesSinceFRDAdjustment(); + } + } else { + nodeData->incrementNumFramesSinceFRDAdjustment(); + } + + // setup a PacketList for the avatarPackets + auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + + // this is an AGENT we have received head data from + // send back a packet with other active node data to this node + nodeList->eachMatchingNode( + [&](const SharedNodePointer& otherNode)->bool { + if (!otherNode->getLinkedData()) { + return false; + } + if (otherNode->getUUID() == node->getUUID()) { + return false; + } + + return true; + }, + [&](const SharedNodePointer& otherNode) { + ++numOtherAvatars; + + MessagesMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + MutexTryLocker lock(otherNodeData->getMutex()); + if (!lock.isLocked()) { + return; + } + + // make sure we send out identity and billboard packets to and from new arrivals. + bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); + + // we will also force a send of billboard or identity packet + // if either has changed in the last frame + if (otherNodeData->getBillboardChangeTimestamp() > 0 + && (forceSend + || otherNodeData->getBillboardChangeTimestamp() > _lastFrameTimestamp + || distribution(generator) < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) { + + QByteArray rfcUUID = otherNode->getUUID().toRfc4122(); + QByteArray billboard = otherNodeData->getAvatar().getBillboard(); + + auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, rfcUUID.size() + billboard.size()); + billboardPacket->write(rfcUUID); + billboardPacket->write(billboard); + + nodeList->sendPacket(std::move(billboardPacket), *node); + + ++_sumBillboardPackets; + } + + if (otherNodeData->getIdentityChangeTimestamp() > 0 + && (forceSend + || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp + || distribution(generator) < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) { + + QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); + + auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); + + individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNode->getUUID().toRfc4122()); + + identityPacket->write(individualData); + + nodeList->sendPacket(std::move(identityPacket), *node); + + ++_sumIdentityPackets; + } + + AvatarData& otherAvatar = otherNodeData->getAvatar(); + // Decide whether to send this avatar's data based on it's distance from us + + // The full rate distance is the distance at which EVERY update will be sent for this avatar + // at twice the full rate distance, there will be a 50% chance of sending this avatar's update + glm::vec3 otherPosition = otherAvatar.getPosition(); + float distanceToAvatar = glm::length(myPosition - otherPosition); + + // potentially update the max full rate distance for this frame + maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); + + if (distanceToAvatar != 0.0f + && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { + return; + } + + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); + + if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { + // we got out out of order packets from the sender, track it + otherNodeData->incrementNumOutOfOrderSends(); + } + + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + return; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + + // we're going to send this avatar + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += + avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO)); + + avatarPacketList->endSegment(); + }); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + // send the avatar data PacketList + nodeList->sendPacketList(std::move(avatarPacketList), *node); + + // record the bytes sent for other avatar data in the MessagesMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes); + + // record the number of avatars held back this frame + nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); + nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); + + if (numOtherAvatars == 0) { + // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send + nodeData->setMaxAvatarDistance(FLT_MAX); + } else { + nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); + } + } + ); + + qDebug() << "MessagesMixer::broadcastMessagesData()... calling nodeList->eachMatchingNode() for encode..."; + + // We're done encoding this version of the otherAvatars. Update their "lastSent" joint-states so + // that we can notice differences, next time around. + nodeList->eachMatchingNode( + [&](const SharedNodePointer& otherNode)->bool { + if (!otherNode->getLinkedData()) { + return false; + } + if (otherNode->getType() != NodeType::Agent) { + return false; + } + if (!otherNode->getActiveSocket()) { + return false; + } + return true; + }, + [&](const SharedNodePointer& otherNode) { + MessagesMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + MutexTryLocker lock(otherNodeData->getMutex()); + if (!lock.isLocked()) { + return; + } + AvatarData& otherAvatar = otherNodeData->getAvatar(); + otherAvatar.doneEncoding(false); + }); + + _lastFrameTimestamp = QDateTime::currentMSecsSinceEpoch(); +} + +void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { + if (killedNode->getType() == NodeType::Agent + && killedNode->getLinkedData()) { + auto nodeList = DependencyManager::get(); + + // this was an avatar we were sending to other people + // send a kill packet for it to our other nodes + auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); + killPacket->write(killedNode->getUUID().toRfc4122()); + + nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); + + // we also want to remove sequence number data for this avatar on our other avatars + // so invoke the appropriate method on the MessagesMixerClientData for other avatars + nodeList->eachMatchingNode( + [&](const SharedNodePointer& node)->bool { + if (!node->getLinkedData()) { + return false; + } + + if (node->getUUID() == killedNode->getUUID()) { + return false; + } + + return true; + }, + [&](const SharedNodePointer& node) { + QMetaObject::invokeMethod(node->getLinkedData(), + "removeLastBroadcastSequenceNumber", + Qt::AutoConnection, + Q_ARG(const QUuid&, QUuid(killedNode->getUUID()))); + } + ); + } +} + +void MessagesMixer::handleMessagesDataPacket(QSharedPointer packet, SharedNodePointer senderNode) { + auto nodeList = DependencyManager::get(); + nodeList->updateNodeWithDataFromPacket(packet, senderNode); +} + +void MessagesMixer::sendStatsPacket() { + QJsonObject statsObject; + statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; + statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; + statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; + + QJsonObject messagesObject; + + auto nodeList = DependencyManager::get(); + // add stats for each listerner + nodeList->eachNode([&](const SharedNodePointer& node) { + QJsonObject messagesStats; + + const QString NODE_OUTBOUND_KBPS_STAT_KEY = "outbound_kbps"; + const QString NODE_INBOUND_KBPS_STAT_KEY = "inbound_kbps"; + + // add the key to ask the domain-server for a username replacement, if it has it + messagesStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidStringWithoutCurlyBraces(node->getUUID()); + messagesStats[NODE_OUTBOUND_KBPS_STAT_KEY] = node->getOutboundBandwidth(); + messagesStats[NODE_INBOUND_KBPS_STAT_KEY] = node->getInboundBandwidth(); + + MessagesMixerClientData* clientData = static_cast(node->getLinkedData()); + if (clientData) { + MutexTryLocker lock(clientData->getMutex()); + if (lock.isLocked()) { + clientData->loadJSONStats(messagesStats); + + // add the diff between the full outbound bandwidth and the measured bandwidth for AvatarData send only + messagesStats["delta_full_vs_avatar_data_kbps"] = + messagesStats[NODE_OUTBOUND_KBPS_STAT_KEY].toDouble() - messagesStats[OUTBOUND_MESSAGES_DATA_STATS_KEY].toDouble(); + } + } + + messagesObject[uuidStringWithoutCurlyBraces(node->getUUID())] = messagesStats; + }); + + statsObject["messages"] = messagesObject; + ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); + + _sumListeners = 0; + _numStatFrames = 0; +} + +void MessagesMixer::run() { + ThreadedAssignment::commonInit(MESSAGES_MIXER_LOGGING_NAME, NodeType::MessagesMixer); + + NodeType_t owningNodeType = DependencyManager::get()->getOwnerType(); + qDebug() << "owningNodeType:" << owningNodeType; + + auto nodeList = DependencyManager::get(); + nodeList->addNodeTypeToInterestSet(NodeType::Agent); + + nodeList->linkedDataCreateCallback = [] (Node* node) { + node->setLinkedData(new MessagesMixerClientData()); + }; + + // setup the timer that will be fired on the broadcast thread + _broadcastTimer = new QTimer; + _broadcastTimer->setInterval(MESSAGES_DATA_SEND_INTERVAL_MSECS); + _broadcastTimer->moveToThread(&_broadcastThread); + + // connect appropriate signals and slots + connect(_broadcastTimer, &QTimer::timeout, this, &MessagesMixer::broadcastMessagesData, Qt::DirectConnection); + connect(&_broadcastThread, SIGNAL(started()), _broadcastTimer, SLOT(start())); + + // wait until we have the domain-server settings, otherwise we bail + DomainHandler& domainHandler = nodeList->getDomainHandler(); + + qDebug() << "Waiting for domain settings from domain-server."; + + // block until we get the settingsRequestComplete signal + + QEventLoop loop; + connect(&domainHandler, &DomainHandler::settingsReceived, &loop, &QEventLoop::quit); + connect(&domainHandler, &DomainHandler::settingsReceiveFail, &loop, &QEventLoop::quit); + domainHandler.requestDomainSettings(); + loop.exec(); + + if (domainHandler.getSettingsObject().isEmpty()) { + qDebug() << "Failed to retreive settings object from domain-server. Bailing on assignment."; + setFinished(true); + return; + } + + // parse the settings to pull out the values we need + parseDomainServerSettings(domainHandler.getSettingsObject()); + + // start the broadcastThread + _broadcastThread.start(); +} + +void MessagesMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { + qDebug() << "MessagesMixer::parseDomainServerSettings() domainSettings:" << domainSettings; + const QString MESSAGES_MIXER_SETTINGS_KEY = "messages_mixer"; + const QString NODE_SEND_BANDWIDTH_KEY = "max_node_send_bandwidth"; + + const float DEFAULT_NODE_SEND_BANDWIDTH = 1.0f; + QJsonValue nodeBandwidthValue = domainSettings[MESSAGES_MIXER_SETTINGS_KEY].toObject()[NODE_SEND_BANDWIDTH_KEY]; + if (!nodeBandwidthValue.isDouble()) { + qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; + } + + _maxKbpsPerNode = nodeBandwidthValue.toDouble(DEFAULT_NODE_SEND_BANDWIDTH) * KILO_PER_MEGA; + qDebug() << "The maximum send bandwidth per node is" << _maxKbpsPerNode << "kbps."; +} diff --git a/assignment-client/src/messages/MessagesMixer.h b/assignment-client/src/messages/MessagesMixer.h new file mode 100644 index 0000000000..d96a20dd18 --- /dev/null +++ b/assignment-client/src/messages/MessagesMixer.h @@ -0,0 +1,58 @@ +// +// MessagesMixer.h +// assignment-client/src/messages +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// The avatar mixer receives head, hand and positional data from all connected +// nodes, and broadcasts that data back to them, every BROADCAST_INTERVAL ms. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MessagesMixer_h +#define hifi_MessagesMixer_h + +#include + +/// Handles assignments of type MessagesMixer - distribution of avatar data to various clients +class MessagesMixer : public ThreadedAssignment { + Q_OBJECT +public: + MessagesMixer(NLPacket& packet); + ~MessagesMixer(); +public slots: + /// runs the avatar mixer + void run(); + + void nodeKilled(SharedNodePointer killedNode); + + void sendStatsPacket(); + +private slots: + void handleMessagesDataPacket(QSharedPointer packet, SharedNodePointer senderNode); + +private: + void broadcastMessagesData(); + void parseDomainServerSettings(const QJsonObject& domainSettings); + + QThread _broadcastThread; + + quint64 _lastFrameTimestamp; + + float _trailingSleepRatio; + float _performanceThrottlingRatio; + + int _sumListeners; + int _numStatFrames; + int _sumBillboardPackets; + int _sumIdentityPackets; + + float _maxKbpsPerNode = 0.0f; + + QTimer* _broadcastTimer = nullptr; +}; + +#endif // hifi_MessagesMixer_h diff --git a/assignment-client/src/messages/MessagesMixerClientData.cpp b/assignment-client/src/messages/MessagesMixerClientData.cpp new file mode 100644 index 0000000000..6aa8f39c22 --- /dev/null +++ b/assignment-client/src/messages/MessagesMixerClientData.cpp @@ -0,0 +1,55 @@ +// +// MessagesMixerClientData.cpp +// assignment-client/src/messages +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include "MessagesMixerClientData.h" + +int MessagesMixerClientData::parseData(NLPacket& packet) { + // pull the sequence number from the data first + packet.readPrimitive(&_lastReceivedSequenceNumber); + + // compute the offset to the data payload + return _avatar.parseDataFromBuffer(packet.readWithoutCopy(packet.bytesLeftToRead())); +} + +bool MessagesMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid) { + if (_hasReceivedFirstPacketsFrom.find(uuid) == _hasReceivedFirstPacketsFrom.end()) { + _hasReceivedFirstPacketsFrom.insert(uuid); + return false; + } + return true; +} + +uint16_t MessagesMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const { + // return the matching PacketSequenceNumber, or the default if we don't have it + auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID); + if (nodeMatch != _lastBroadcastSequenceNumbers.end()) { + return nodeMatch->second; + } else { + return 0; + } +} + +void MessagesMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { + jsonObject["display_name"] = _avatar.getDisplayName(); + jsonObject["full_rate_distance"] = _fullRateDistance; + jsonObject["max_av_distance"] = _maxAvatarDistance; + jsonObject["num_avs_sent_last_frame"] = _numAvatarsSentLastFrame; + jsonObject["avg_other_av_starves_per_second"] = getAvgNumOtherAvatarStarvesPerSecond(); + jsonObject["avg_other_av_skips_per_second"] = getAvgNumOtherAvatarSkipsPerSecond(); + jsonObject["total_num_out_of_order_sends"] = _numOutOfOrderSends; + + jsonObject[OUTBOUND_MESSAGES_DATA_STATS_KEY] = getOutboundAvatarDataKbps(); + jsonObject[INBOUND_MESSAGES_DATA_STATS_KEY] = _avatar.getAverageBytesReceivedPerSecond() / (float) BYTES_PER_KILOBIT; + + jsonObject["av_data_receive_rate"] = _avatar.getReceiveRate(); +} diff --git a/assignment-client/src/messages/MessagesMixerClientData.h b/assignment-client/src/messages/MessagesMixerClientData.h new file mode 100644 index 0000000000..1667df431f --- /dev/null +++ b/assignment-client/src/messages/MessagesMixerClientData.h @@ -0,0 +1,105 @@ +// +// MessagesMixerClientData.h +// assignment-client/src/messages +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MessagesMixerClientData_h +#define hifi_MessagesMixerClientData_h + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +const QString OUTBOUND_MESSAGES_DATA_STATS_KEY = "outbound_av_data_kbps"; +const QString INBOUND_MESSAGES_DATA_STATS_KEY = "inbound_av_data_kbps"; + +class MessagesMixerClientData : public NodeData { + Q_OBJECT +public: + int parseData(NLPacket& packet); + AvatarData& getAvatar() { return _avatar; } + + bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid); + + uint16_t getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const; + void setLastBroadcastSequenceNumber(const QUuid& nodeUUID, uint16_t sequenceNumber) + { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; } + Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); } + + uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; } + + quint64 getBillboardChangeTimestamp() const { return _billboardChangeTimestamp; } + void setBillboardChangeTimestamp(quint64 billboardChangeTimestamp) { _billboardChangeTimestamp = billboardChangeTimestamp; } + + quint64 getIdentityChangeTimestamp() const { return _identityChangeTimestamp; } + void setIdentityChangeTimestamp(quint64 identityChangeTimestamp) { _identityChangeTimestamp = identityChangeTimestamp; } + + void setFullRateDistance(float fullRateDistance) { _fullRateDistance = fullRateDistance; } + float getFullRateDistance() const { return _fullRateDistance; } + + void setMaxAvatarDistance(float maxAvatarDistance) { _maxAvatarDistance = maxAvatarDistance; } + float getMaxAvatarDistance() const { return _maxAvatarDistance; } + + void resetNumAvatarsSentLastFrame() { _numAvatarsSentLastFrame = 0; } + void incrementNumAvatarsSentLastFrame() { ++_numAvatarsSentLastFrame; } + int getNumAvatarsSentLastFrame() const { return _numAvatarsSentLastFrame; } + + void recordNumOtherAvatarStarves(int numAvatarsHeldBack) { _otherAvatarStarves.updateAverage((float) numAvatarsHeldBack); } + float getAvgNumOtherAvatarStarvesPerSecond() const { return _otherAvatarStarves.getAverageSampleValuePerSecond(); } + + void recordNumOtherAvatarSkips(int numOtherAvatarSkips) { _otherAvatarSkips.updateAverage((float) numOtherAvatarSkips); } + float getAvgNumOtherAvatarSkipsPerSecond() const { return _otherAvatarSkips.getAverageSampleValuePerSecond(); } + + void incrementNumOutOfOrderSends() { ++_numOutOfOrderSends; } + + int getNumFramesSinceFRDAdjustment() const { return _numFramesSinceAdjustment; } + void incrementNumFramesSinceFRDAdjustment() { ++_numFramesSinceAdjustment; } + void resetNumFramesSinceFRDAdjustment() { _numFramesSinceAdjustment = 0; } + + void recordSentAvatarData(int numBytes) { _avgOtherAvatarDataRate.updateAverage((float) numBytes); } + + float getOutboundAvatarDataKbps() const + { return _avgOtherAvatarDataRate.getAverageSampleValuePerSecond() / (float) BYTES_PER_KILOBIT; } + + void loadJSONStats(QJsonObject& jsonObject) const; +private: + AvatarData _avatar; + + uint16_t _lastReceivedSequenceNumber { 0 }; + std::unordered_map _lastBroadcastSequenceNumbers; + std::unordered_set _hasReceivedFirstPacketsFrom; + + quint64 _billboardChangeTimestamp = 0; + quint64 _identityChangeTimestamp = 0; + + float _fullRateDistance = FLT_MAX; + float _maxAvatarDistance = FLT_MAX; + + int _numAvatarsSentLastFrame = 0; + int _numFramesSinceAdjustment = 0; + + SimpleMovingAverage _otherAvatarStarves; + SimpleMovingAverage _otherAvatarSkips; + int _numOutOfOrderSends = 0; + + SimpleMovingAverage _avgOtherAvatarDataRate; +}; + +#endif // hifi_MessagesMixerClientData_h diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index e0038117f0..b2443b8bd4 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -549,6 +549,22 @@ "advanced": true } ] + }, + { + "name": "messages_mixer", + "label": "Messages Mixer", + "assignment-types": [4], + "settings": [ + { + "name": "max_node_send_bandwidth", + "type": "double", + "label": "Per-Node Bandwidth", + "help": "Desired maximum send bandwidth (in Megabits per second) to each node", + "placeholder": 1.0, + "default": 1.0, + "advanced": true + } + ] } ] } diff --git a/domain-server/src/DomainGatekeeper.cpp b/domain-server/src/DomainGatekeeper.cpp index d360ab4802..55f0fb2d2b 100644 --- a/domain-server/src/DomainGatekeeper.cpp +++ b/domain-server/src/DomainGatekeeper.cpp @@ -48,7 +48,8 @@ QUuid DomainGatekeeper::assignmentUUIDForPendingAssignment(const QUuid& tempUUID const NodeSet STATICALLY_ASSIGNED_NODES = NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer << NodeType::EntityServer - << NodeType::AssetServer; + << NodeType::AssetServer + << NodeType::MessagesMixer; void DomainGatekeeper::processConnectRequestPacket(QSharedPointer packet) { if (packet->getPayloadSize() == 0) { @@ -66,7 +67,7 @@ void DomainGatekeeper::processConnectRequestPacket(QSharedPointer pack } static const NodeSet VALID_NODE_TYPES { - NodeType::AudioMixer, NodeType::AvatarMixer, NodeType::AssetServer, NodeType::EntityServer, NodeType::Agent + NodeType::AudioMixer, NodeType::AvatarMixer, NodeType::AssetServer, NodeType::EntityServer, NodeType::Agent, NodeType::MessagesMixer }; if (!VALID_NODE_TYPES.contains(nodeConnection.nodeType)) { diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index b5fd9f2b20..1db277f47f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -553,7 +553,6 @@ void DomainServer::populateDefaultStaticAssignmentsExcludingTypes(const QSet(static_cast(defaultedType) + 1)) { if (!excludedTypes.contains(defaultedType) - && defaultedType != Assignment::UNUSED_0 && defaultedType != Assignment::UNUSED_1 && defaultedType != Assignment::AgentType) { diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index f650089486..924e19e1fc 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -131,7 +131,8 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList appSettings.setValue(JSON_SETTINGS_VERSION_KEY, _descriptionVersion); } -QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString &keyPath) { +QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString& keyPath) { + qDebug() << "DomainServerSettingsManager::valueOrDefaultValueForKeyPath() keyPath:" << keyPath; const QVariant* foundValue = valueForKeyPath(_configMap.getMergedConfig(), keyPath); if (foundValue) { @@ -226,6 +227,8 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection } QJsonObject DomainServerSettingsManager::responseObjectForType(const QString& typeValue, bool isAuthenticated) { + qDebug() << "DomainServerSettingsManager::responseObjectForType() typeValue:" << typeValue; + QJsonObject responseObject; if (!typeValue.isEmpty() || isAuthenticated) { diff --git a/libraries/networking/src/Assignment.cpp b/libraries/networking/src/Assignment.cpp index 66c6bf2a2c..5fdedeafb9 100644 --- a/libraries/networking/src/Assignment.cpp +++ b/libraries/networking/src/Assignment.cpp @@ -30,6 +30,8 @@ Assignment::Type Assignment::typeForNodeType(NodeType_t nodeType) { return Assignment::EntityServerType; case NodeType::AssetServer: return Assignment::AssetServerType; + case NodeType::MessagesMixer: + return Assignment::MessagesMixerType; default: return Assignment::AllTypes; } @@ -131,6 +133,8 @@ const char* Assignment::getTypeName() const { return "asset-server"; case Assignment::EntityServerType: return "entity-server"; + case Assignment::MessagesMixerType: + return "messages-mixer"; default: return "unknown"; } diff --git a/libraries/networking/src/Assignment.h b/libraries/networking/src/Assignment.h index ee3d9cb5fd..9639411eec 100644 --- a/libraries/networking/src/Assignment.h +++ b/libraries/networking/src/Assignment.h @@ -30,7 +30,7 @@ public: AvatarMixerType = 1, AgentType = 2, AssetServerType = 3, - UNUSED_0 = 4, + MessagesMixerType = 4, UNUSED_1 = 5, EntityServerType = 6, AllTypes = 7 diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index df024b361d..9f411c59f1 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -242,6 +242,8 @@ void DomainHandler::requestDomainSettings() { Assignment::Type assignmentType = Assignment::typeForNodeType(DependencyManager::get()->getOwnerType()); + qCDebug(networking) << "Requesting settings from domain server for assignmentType:" << assignmentType; + auto packet = NLPacket::create(PacketType::DomainSettingsRequest, sizeof(assignmentType), true, false); packet->writePrimitive(assignmentType); diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp new file mode 100644 index 0000000000..d3d44b6fdc --- /dev/null +++ b/libraries/networking/src/MessagesClient.cpp @@ -0,0 +1,113 @@ +// +// MessagesClient.cpp +// libraries/networking/src +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MessagesClient.h" + +#include + +#include +#include +#include +#include + +#include "AssetRequest.h" +#include "AssetUpload.h" +#include "AssetUtils.h" +#include "NetworkAccessManager.h" +#include "NetworkLogging.h" +#include "NodeList.h" +#include "PacketReceiver.h" +#include "ResourceCache.h" + +MessagesClient::MessagesClient() { + + setCustomDeleter([](Dependency* dependency){ + static_cast(dependency)->deleteLater(); + }); + + auto nodeList = DependencyManager::get(); + auto& packetReceiver = nodeList->getPacketReceiver(); + + packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagePacket"); + + connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &MessagesClient::handleNodeKilled); +} + +void MessagesClient::init() { + if (QThread::currentThread() != thread()) { + QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection); + } + + // Setup disk cache if not already + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + if (!networkAccessManager.cache()) { + QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation); + cachePath = !cachePath.isEmpty() ? cachePath : "interfaceCache"; + + QNetworkDiskCache* cache = new QNetworkDiskCache(); + cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE); + cache->setCacheDirectory(cachePath); + networkAccessManager.setCache(cache); + qCDebug(asset_client) << "MessagesClient disk cache setup at" << cachePath + << "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)"; + } +} + +bool haveMessagesMixer() { + auto nodeList = DependencyManager::get(); + SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); + + if (!messagesMixer) { + qCWarning(messages_client) << "Could not complete MessagesClient operation " + << "since you are not currently connected to a messages-mixer."; + return false; + } + + return true; +} + +void MessagesClient::handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode) { + auto packetType = packet->getType(); + + if (packetType == PacketType::MessagesData) { + qDebug() << "got a messages packet"; + } +} + +void MessagesClient::sendMessage(const QString& channel, const QString& message) { + auto nodeList = DependencyManager::get(); + SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); + + if (messagesMixer) { + auto packetList = NLPacketList::create(PacketType::MessagesData, QByteArray(), true, true); + + #if 0 + auto messageID = ++_currentID; + packetList->writePrimitive(messageID); + + packetList->writePrimitive(static_cast(extension.length())); + packetList->write(extension.toLatin1().constData(), extension.length()); + + uint64_t size = data.length(); + packetList->writePrimitive(size); + packetList->write(data.constData(), size); + + nodeList->sendPacketList(std::move(packetList), *assetServer); + #endif + } +} + +void MessagesClient::handleNodeKilled(SharedNodePointer node) { + if (node->getType() != NodeType::MessagesMixer) { + return; + } + +} diff --git a/libraries/networking/src/MessagesClient.h b/libraries/networking/src/MessagesClient.h new file mode 100644 index 0000000000..f1f13bfe20 --- /dev/null +++ b/libraries/networking/src/MessagesClient.h @@ -0,0 +1,40 @@ +// +// MessagesClient.h +// libraries/networking/src +// +// Created by Brad hefta-Gaub on 11/16/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#ifndef hifi_MessagesClient_h +#define hifi_MessagesClient_h + +#include + +#include + +#include "LimitedNodeList.h" +#include "NLPacket.h" +#include "Node.h" + +class MessagesClient : public QObject, public Dependency { + Q_OBJECT +public: + MessagesClient(); + + Q_INVOKABLE void init(); + + Q_INVOKABLE void sendMessage(const QString& channel, const QString& message); + +private slots: + void handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode); + void handleNodeKilled(SharedNodePointer node); + +private: +}; + +#endif diff --git a/libraries/networking/src/NetworkLogging.cpp b/libraries/networking/src/NetworkLogging.cpp index 28b209960e..01abd6ae19 100644 --- a/libraries/networking/src/NetworkLogging.cpp +++ b/libraries/networking/src/NetworkLogging.cpp @@ -13,3 +13,4 @@ Q_LOGGING_CATEGORY(networking, "hifi.networking") Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client") +Q_LOGGING_CATEGORY(messages_client, "hifi.networking.messages_client") diff --git a/libraries/networking/src/NetworkLogging.h b/libraries/networking/src/NetworkLogging.h index 838bbb57d2..37ebc1933d 100644 --- a/libraries/networking/src/NetworkLogging.h +++ b/libraries/networking/src/NetworkLogging.h @@ -16,5 +16,6 @@ Q_DECLARE_LOGGING_CATEGORY(networking) Q_DECLARE_LOGGING_CATEGORY(asset_client) +Q_DECLARE_LOGGING_CATEGORY(messages_client) #endif // hifi_NetworkLogging_h diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 5fea670dd0..243dca78e2 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -32,6 +32,7 @@ void NodeType::init() { TypeNameHash.insert(NodeType::Agent, "Agent"); TypeNameHash.insert(NodeType::AudioMixer, "Audio Mixer"); TypeNameHash.insert(NodeType::AvatarMixer, "Avatar Mixer"); + TypeNameHash.insert(NodeType::MessagesMixer, "Messages Mixer"); TypeNameHash.insert(NodeType::AssetServer, "Asset Server"); TypeNameHash.insert(NodeType::Unassigned, "Unassigned"); } diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index e680f218db..d4377f4610 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -23,6 +23,7 @@ namespace NodeType { const NodeType_t AudioMixer = 'M'; const NodeType_t AvatarMixer = 'W'; const NodeType_t AssetServer = 'A'; + const NodeType_t MessagesMixer = 'm'; const NodeType_t Unassigned = 1; void init(); diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 0422c03297..992b3be2b4 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -119,9 +119,12 @@ void ThreadedAssignment::stopSendingStats() { } void ThreadedAssignment::checkInWithDomainServerOrExit() { + qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit()...."; if (DependencyManager::get()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) { + qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit().... getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS"; setFinished(true); } else { + qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit().... calling DependencyManager::get()->sendDomainServerCheckIn()"; DependencyManager::get()->sendDomainServerCheckIn(); } } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 82d905bf28..099e842c27 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -79,7 +79,8 @@ enum class PacketType : uint8_t { AssetUpload, AssetUploadReply, AssetGetInfo, - AssetGetInfoReply + AssetGetInfoReply, + MessagesData }; const int NUM_BYTES_MD5_HASH = 16; From f17af601ab5dbe550238ad6231750bdb10b45d89 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 15:52:48 -0800 Subject: [PATCH 08/36] update BUILD for cmake version change --- BUILD.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.md b/BUILD.md index 5abb3ae4e7..a24524af6f 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,6 +1,6 @@ ###Dependencies -* [cmake](http://www.cmake.org/cmake/resources/software.html) ~> 2.8.12.2 +* [cmake](http://www.cmake.org/cmake/resources/software.html) ~> 3.3.2 * [Qt](http://www.qt.io/download-open-source) ~> 5.4.1 * [OpenSSL](https://www.openssl.org/community/binaries.html) ~> 1.0.1m * IMPORTANT: Using the recommended version of OpenSSL is critical to avoid security vulnerabilities. From d2b8ba740abb14b80d15f04868db31778182ed97 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 16 Nov 2015 15:59:52 -0800 Subject: [PATCH 09/36] change BUILD guides for 5.5.1 and homebrew recs --- BUILD.md | 15 +++++++-------- BUILD_ANDROID.md | 14 +++++++------- BUILD_OSX.md | 13 ++++--------- BUILD_WIN.md | 10 +++++----- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/BUILD.md b/BUILD.md index a24524af6f..c1ccd3193e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,7 +1,7 @@ ###Dependencies * [cmake](http://www.cmake.org/cmake/resources/software.html) ~> 3.3.2 -* [Qt](http://www.qt.io/download-open-source) ~> 5.4.1 +* [Qt](http://www.qt.io/download-open-source) ~> 5.5.1 * [OpenSSL](https://www.openssl.org/community/binaries.html) ~> 1.0.1m * IMPORTANT: Using the recommended version of OpenSSL is critical to avoid security vulnerabilities. * [VHACD](https://github.com/virneo/v-hacd)(clone this repository)(Optional) @@ -21,10 +21,10 @@ * [SDL2](https://www.libsdl.org/download-2.0.php) ~> 2.0.3 * [soxr](http://soxr.sourceforge.net) ~> 0.1.1 * [Intel Threading Building Blocks](https://www.threadingbuildingblocks.org/) ~> 4.3 -* [Sixense](http://sixense.com/) ~> 071615 +* [Sixense](http://sixense.com/) ~> 071615 * [zlib](http://www.zlib.net/) ~> 1.28 (Win32 only) -The above dependencies will be downloaded, built, linked and included automatically by CMake where we require them. The CMakeLists files that handle grabbing each of the following external dependencies can be found in the [cmake/externals folder](cmake/externals). The resulting downloads, source files and binaries will be placed in the `build/ext` folder in each of the subfolders for each external project. +The above dependencies will be downloaded, built, linked and included automatically by CMake where we require them. The CMakeLists files that handle grabbing each of the following external dependencies can be found in the [cmake/externals folder](cmake/externals). The resulting downloads, source files and binaries will be placed in the `build/ext` folder in each of the subfolders for each external project. These are not placed in your normal build tree when doing an out of source build so that they do not need to be re-downloaded and re-compiled every time the CMake build folder is cleared. Should you want to force a re-download and re-compile of a specific external, you can simply remove that directory from the appropriate subfolder in `build/ext`. Should you want to force a re-download and re-compile of all externals, just remove the `build/ext` folder. @@ -42,12 +42,12 @@ Hifi uses CMake to generate build files and project files for your platform. ####Qt In order for CMake to find the Qt5 find modules, you will need to set an ENV variable pointing to your Qt installation. -For example, a Qt5 5.4.1 installation to /usr/local/qt5 would require that QT_CMAKE_PREFIX_PATH be set with the following command. This can either be entered directly into your shell session before you build or in your shell profile (e.g.: ~/.bash_profile, ~/.bashrc, ~/.zshrc - this depends on your shell and environment). +For example, a Qt5 5.5.1 installation to /usr/local/qt5 would require that QT_CMAKE_PREFIX_PATH be set with the following command. This can either be entered directly into your shell session before you build or in your shell profile (e.g.: ~/.bash_profile, ~/.bashrc, ~/.zshrc - this depends on your shell and environment). The path it needs to be set to will depend on where and how Qt5 was installed. e.g. - export QT_CMAKE_PREFIX_PATH=/usr/local/qt/5.4.1/clang_64/lib/cmake/ - export QT_CMAKE_PREFIX_PATH=/usr/local/Cellar/qt5/5.4.1/lib/cmake + export QT_CMAKE_PREFIX_PATH=/usr/local/qt/5.5.1/clang_64/lib/cmake/ + export QT_CMAKE_PREFIX_PATH=/usr/local/Cellar/qt5/5.5.1/lib/cmake export QT_CMAKE_PREFIX_PATH=/usr/local/opt/qt5/lib/cmake ####Generating build files @@ -64,7 +64,7 @@ Any variables that need to be set for CMake to find dependencies can be set as E For example, to pass the QT_CMAKE_PREFIX_PATH variable during build file generation: - cmake .. -DQT_CMAKE_PREFIX_PATH=/usr/local/qt/5.4.1/lib/cmake + cmake .. -DQT_CMAKE_PREFIX_PATH=/usr/local/qt/5.5.1/lib/cmake ####Finding Dependencies @@ -83,4 +83,3 @@ In the examples below the variable $NAME would be replaced by the name of the de ####Devices You can support external input/output devices such as Leap Motion, MIDI, and more by adding each individual SDK in the visible building path. Refer to the readme file available in each device folder in [interface/external/](interface/external) for the detailed explanation of the requirements to use the device. - diff --git a/BUILD_ANDROID.md b/BUILD_ANDROID.md index 9f86e7e925..88294f3040 100644 --- a/BUILD_ANDROID.md +++ b/BUILD_ANDROID.md @@ -14,7 +14,7 @@ You will need the following tools to build our Android targets. * Install the latest Platform-tools * Install the latest Build-tools * Install the SDK Platform for API Level 19 - * Install Sources for Android SDK for API Level 19 + * Install Sources for Android SDK for API Level 19 * Install the ARM EABI v7a System Image if you want to run an emulator. You will also need to cross-compile the dependencies required for all platforms for Android, and help CMake find these compiled libraries on your machine. @@ -25,7 +25,7 @@ You will also need to cross-compile the dependencies required for all platforms ####ANDROID_LIB_DIR -Since you won't be installing Android dependencies to system paths on your development machine, CMake will need a little help tracking down your Android dependencies. +Since you won't be installing Android dependencies to system paths on your development machine, CMake will need a little help tracking down your Android dependencies. This is most easily accomplished by installing all Android dependencies in the same folder. You can place this folder wherever you like on your machine. In this build guide and across our CMakeLists files this folder is referred to as `ANDROID_LIB_DIR`. You can set `ANDROID_LIB_DIR` in your environment or by passing when you run CMake. @@ -45,7 +45,7 @@ The original instructions to compile OpenSSL for Android from your host environm Download the [OpenSSL source](https://www.openssl.org/source/) and extract the tarball inside your `ANDROID_LIB_DIR`. Rename the extracted folder to `openssl`. -You will need the [setenv-android.sh script](http://wiki.openssl.org/index.php/File:Setenv-android.sh) from the OpenSSL wiki. +You will need the [setenv-android.sh script](http://wiki.openssl.org/index.php/File:Setenv-android.sh) from the OpenSSL wiki. You must change three values at the top of the `setenv-android.sh` script - `_ANDROID_NDK`, `_ANDROID_EABI` and `_ANDROID_API`. `_ANDROID_NDK` should be `android-ndk-r10`, `_ANDROID_EABI` should be `arm-linux-androidebi-4.9` and `_ANDROID_API` should be `19`. @@ -62,8 +62,8 @@ source setenv-android.sh Then, from the OpenSSL directory, run the following commands. ``` -perl -pi -e 's/install: all install_docs install_sw/install: install_docs install_sw/g' Makefile.org -./config shared -no-ssl2 -no-ssl3 -no-comp -no-hw -no-engine --openssldir=/usr/local/ssl/$ANDROID_API +perl -pi -e 's/install: all install_docs install_sw/install: install_docs install_sw/g' Makefile.org +./config shared -no-ssl2 -no-ssl3 -no-comp -no-hw -no-engine --openssldir=/usr/local/ssl/$ANDROID_API make depend make all ``` @@ -78,7 +78,7 @@ The Oculus Mobile SDK is optional, for Gear VR support. It is not required to co Download the [Oculus Mobile SDK](https://developer.oculus.com/downloads/#sdk=mobile) and extract the archive inside your `ANDROID_LIB_DIR` folder. Rename the extracted folder to `libovr`. -From the VRLib directory, use ndk-build to build VrLib. +From the VRLib directory, use ndk-build to build VrLib. ``` cd VRLib @@ -107,4 +107,4 @@ The following must be set in your environment: The following must be passed to CMake when it is run: -* USE_ANDROID_TOOLCHAIN - set to true to build for Android \ No newline at end of file +* USE_ANDROID_TOOLCHAIN - set to true to build for Android diff --git a/BUILD_OSX.md b/BUILD_OSX.md index 9d1357d672..54360ad4b8 100644 --- a/BUILD_OSX.md +++ b/BUILD_OSX.md @@ -3,20 +3,15 @@ Please read the [general build guide](BUILD.md) for information on dependencies ###Homebrew [Homebrew](http://brew.sh/) is an excellent package manager for OS X. It makes install of all High Fidelity dependencies very simple. - brew tap highfidelity/homebrew-formulas - brew install cmake openssl - brew install highfidelity/formulas/qt5 - brew link qt5 --force + brew install cmake openssl qt5 -We have a [homebrew formulas repository](https://github.com/highfidelity/homebrew-formulas) that you can use/tap to install some of the dependencies. In the code block above qt5 is installed from a formula in this repository. - -*Our [qt5 homebrew formula](https://raw.github.com/highfidelity/homebrew-formulas/master/qt5.rb) is for a patched version of Qt 5.4.x stable that removes wireless network scanning that can reduce real-time audio performance. We recommended you use this formula to install Qt.* +We no longer require install of qt5 via our [homebrew formulas repository](https://github.com/highfidelity/homebrew-formulas). Versions of Qt that are 5.5.x and above provide a mechanism to disable the wireless scanning we previously had a custom patch for. ###Qt -Assuming you've installed Qt 5 using the homebrew instructions above, you'll need to set QT_CMAKE_PREFIX_PATH so CMake can find your installation of Qt. For Qt 5.4.1 installed via homebrew, set QT_CMAKE_PREFIX_PATH as follows. +Assuming you've installed Qt 5 using the homebrew instructions above, you'll need to set QT_CMAKE_PREFIX_PATH so CMake can find your installation of Qt. For Qt 5.5.1 installed via homebrew, set QT_CMAKE_PREFIX_PATH as follows. - export QT_CMAKE_PREFIX_PATH=/usr/local/Cellar/qt5/5.4.1/lib/cmake + export QT_CMAKE_PREFIX_PATH=/usr/local/Cellar/qt5/5.5.1/lib/cmake ###Xcode If Xcode is your editor of choice, you can ask CMake to generate Xcode project files instead of Unix Makefiles. diff --git a/BUILD_WIN.md b/BUILD_WIN.md index 89646e99ff..48781ca34a 100644 --- a/BUILD_WIN.md +++ b/BUILD_WIN.md @@ -16,7 +16,7 @@ If using Visual Studio 2013 and building as a Visual Studio 2013 project you nee ####nmake -Some of the external projects may require nmake to compile and install. If it is not installed at the location listed below, please ensure that it is in your PATH so CMake can find it when required. +Some of the external projects may require nmake to compile and install. If it is not installed at the location listed below, please ensure that it is in your PATH so CMake can find it when required. We expect nmake.exe to be located at the following path. @@ -29,19 +29,19 @@ NOTE: Qt does not support 64-bit builds on Windows 7, so you must use the 32-bit * [Download the online installer](http://qt-project.org/downloads) * When it asks you to select components, ONLY select the following: - * Qt > Qt 5.4.1 > **msvc2013 32-bit OpenGL** + * Qt > Qt 5.5.1 > **msvc2013 32-bit** -* [Download the offline installer](http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-windows-x86-msvc2013_opengl-5.4.1.exe) +* [Download the offline installer](http://download.qt.io/official_releases/qt/5.5/5.5.1/qt-opensource-windows-x86-msvc2013-5.5.1.exe) Once Qt is installed, you need to manually configure the following: -* Set the QT_CMAKE_PREFIX_PATH environment variable to your `Qt\5.4.1\msvc2013_opengl\lib\cmake` directory. +* Set the QT_CMAKE_PREFIX_PATH environment variable to your `Qt\5.5.1\msvc2013\lib\cmake` directory. * You can set an environment variable from Control Panel > System > Advanced System Settings > Environment Variables > New ###External Libraries As it stands, Hifi/Interface is a 32-bit application, so all libraries should also be 32-bit. -CMake will need to know where the headers and libraries for required external dependencies are. +CMake will need to know where the headers and libraries for required external dependencies are. We use CMake's `fixup_bundle` to find the DLLs all of our exectuable targets require, and then copy them beside the executable in a post-build step. If `fixup_bundle` is having problems finding a DLL, you can fix it manually on your end by adding the folder containing that DLL to your path. Let us know which DLL CMake had trouble finding, as it is possible a tweak to our CMake files is required. From 8a27a2fba51c8a778cc234ac6bf1df2b3a1e3441 Mon Sep 17 00:00:00 2001 From: Brad Davis Date: Mon, 16 Nov 2015 16:29:10 -0800 Subject: [PATCH 10/36] Fixing recording interface times --- interface/src/scripting/RecordingScriptingInterface.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interface/src/scripting/RecordingScriptingInterface.cpp b/interface/src/scripting/RecordingScriptingInterface.cpp index bf585f5481..32bd6fde97 100644 --- a/interface/src/scripting/RecordingScriptingInterface.cpp +++ b/interface/src/scripting/RecordingScriptingInterface.cpp @@ -56,11 +56,11 @@ bool RecordingScriptingInterface::isPaused() { } float RecordingScriptingInterface::playerElapsed() { - return (float)_player->position() / MSECS_PER_SECOND; + return _player->position(); } float RecordingScriptingInterface::playerLength() { - return _player->length() / MSECS_PER_SECOND; + return _player->length(); } void RecordingScriptingInterface::loadRecording(const QString& filename) { @@ -103,7 +103,7 @@ void RecordingScriptingInterface::setPlayerAudioOffset(float audioOffset) { } void RecordingScriptingInterface::setPlayerTime(float time) { - _player->seek(time * MSECS_PER_SECOND); + _player->seek(time); } void RecordingScriptingInterface::setPlayFromCurrentLocation(bool playFromCurrentLocation) { From b6c27588b6e741e8b820b7791cd24cb83791954a Mon Sep 17 00:00:00 2001 From: EdgarPironti Date: Mon, 16 Nov 2015 16:50:47 -0800 Subject: [PATCH 11/36] ControlledAC.js refactoring --- examples/acScripts/ControlledAC.js | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/acScripts/ControlledAC.js b/examples/acScripts/ControlledAC.js index 41a8a2b257..cbc5f64b8f 100644 --- a/examples/acScripts/ControlledAC.js +++ b/examples/acScripts/ControlledAC.js @@ -38,11 +38,11 @@ var SHOW = 4; var HIDE = 5; var LOAD = 6; -Avatar.setPlayFromCurrentLocation(playFromCurrentLocation); -Avatar.setPlayerUseDisplayName(useDisplayName); -Avatar.setPlayerUseAttachments(useAttachments); -Avatar.setPlayerUseHeadModel(false); -Avatar.setPlayerUseSkeletonModel(useAvatarModel); +Recording.setPlayFromCurrentLocation(playFromCurrentLocation); +Recording.setPlayerUseDisplayName(useDisplayName); +Recording.setPlayerUseAttachments(useAttachments); +Recording.setPlayerUseHeadModel(false); +Recording.setPlayerUseSkeletonModel(useAvatarModel); function setupEntityViewer() { var entityViewerOffset = 10; @@ -96,25 +96,25 @@ function update(event) { if (!Agent.isAvatar) { Agent.isAvatar = true; } - if (!Avatar.isPlaying()) { - Avatar.startPlaying(); + if (!Recording.isPlaying()) { + Recording.startPlaying(); } - Avatar.setPlayerLoop(false); + Recording.setPlayerLoop(false); break; case PLAY_LOOP: print("Play loop"); if (!Agent.isAvatar) { Agent.isAvatar = true; } - if (!Avatar.isPlaying()) { - Avatar.startPlaying(); + if (!Recording.isPlaying()) { + Recording.startPlaying(); } - Avatar.setPlayerLoop(true); + Recording.setPlayerLoop(true); break; case STOP: print("Stop"); - if (Avatar.isPlaying()) { - Avatar.stopPlaying(); + if (Recording.isPlaying()) { + Recording.stopPlaying(); } break; case SHOW: @@ -125,15 +125,15 @@ function update(event) { break; case HIDE: print("Hide"); - if (Avatar.isPlaying()) { - Avatar.stopPlaying(); + if (Recording.isPlaying()) { + Recording.stopPlaying(); } Agent.isAvatar = false; break; case LOAD: print("Load"); if(clip_url !== null) { - Avatar.loadRecording(clip_url); + Recording.loadRecording(clip_url); } break; case DO_NOTHING: @@ -143,8 +143,8 @@ function update(event) { break; } - if (Avatar.isPlaying()) { - Avatar.play(); + if (Recording.isPlaying()) { + Recording.play(); } } From 12f206e2f04bdac6f3c980069f4e6037e51b611b Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 16 Nov 2015 17:00:03 -0800 Subject: [PATCH 12/36] more work on messages --- .../src/messages/MessagesMixer.cpp | 364 ++---------------- .../src/messages/MessagesMixer.h | 2 +- interface/src/Application.cpp | 11 + libraries/networking/src/MessagesClient.cpp | 25 +- libraries/networking/src/MessagesClient.h | 2 +- 5 files changed, 47 insertions(+), 357 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 70b0c1b2cf..6177850532 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -48,7 +48,7 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &MessagesMixer::nodeKilled); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesDataPacket"); + packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); } MessagesMixer::~MessagesMixer() { @@ -64,341 +64,13 @@ MessagesMixer::~MessagesMixer() { // assuming 60 htz update rate. const float BILLBOARD_AND_IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; -void MessagesMixer::broadcastMessagesData() { - qDebug() << "MessagesMixer::broadcastMessagesData()..."; - - int idleTime = QDateTime::currentMSecsSinceEpoch() - _lastFrameTimestamp; - - ++_numStatFrames; - - const float STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.10f; - const float BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD = 0.20f; - - const float RATIO_BACK_OFF = 0.02f; - - const int TRAILING_AVERAGE_FRAMES = 100; - int framesSinceCutoffEvent = TRAILING_AVERAGE_FRAMES; - - const float CURRENT_FRAME_RATIO = 1.0f / TRAILING_AVERAGE_FRAMES; - const float PREVIOUS_FRAMES_RATIO = 1.0f - CURRENT_FRAME_RATIO; - - // NOTE: The following code calculates the _performanceThrottlingRatio based on how much the messages-mixer was - // able to sleep. This will eventually be used to ask for an additional messages-mixer to help out. Currently the value - // is unused as it is assumed this should not be hit before the messages-mixer hits the desired bandwidth limit per client. - // It is reported in the domain-server stats for the messages-mixer. - - _trailingSleepRatio = (PREVIOUS_FRAMES_RATIO * _trailingSleepRatio) - + (idleTime * CURRENT_FRAME_RATIO / (float) MESSAGES_DATA_SEND_INTERVAL_MSECS); - - float lastCutoffRatio = _performanceThrottlingRatio; - bool hasRatioChanged = false; - - if (framesSinceCutoffEvent >= TRAILING_AVERAGE_FRAMES) { - if (_trailingSleepRatio <= STRUGGLE_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD) { - // we're struggling - change our performance throttling ratio - _performanceThrottlingRatio = _performanceThrottlingRatio + (0.5f * (1.0f - _performanceThrottlingRatio)); - - qDebug() << "Mixer is struggling, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; - } else if (_trailingSleepRatio >= BACK_OFF_TRIGGER_SLEEP_PERCENTAGE_THRESHOLD && _performanceThrottlingRatio != 0) { - // we've recovered and can back off the performance throttling - _performanceThrottlingRatio = _performanceThrottlingRatio - RATIO_BACK_OFF; - - if (_performanceThrottlingRatio < 0) { - _performanceThrottlingRatio = 0; - } - - qDebug() << "Mixer is recovering, sleeping" << _trailingSleepRatio * 100 << "% of frame time. Old cutoff was" - << lastCutoffRatio << "and is now" << _performanceThrottlingRatio; - hasRatioChanged = true; - } - - if (hasRatioChanged) { - framesSinceCutoffEvent = 0; - } - } - - if (!hasRatioChanged) { - ++framesSinceCutoffEvent; - } - - auto nodeList = DependencyManager::get(); - - // setup for distributed random floating point values - std::random_device randomDevice; - std::mt19937 generator(randomDevice()); - std::uniform_real_distribution distribution; - - qDebug() << "MessagesMixer::broadcastMessagesData()... calling nodeList->eachMatchingNode()"; - - nodeList->eachMatchingNode( - [&](const SharedNodePointer& node)->bool { - if (!node->getLinkedData()) { - return false; - } - if (node->getType() != NodeType::Agent) { - return false; - } - if (!node->getActiveSocket()) { - return false; - } - return true; - }, - [&](const SharedNodePointer& node) { - MessagesMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - MutexTryLocker lock(nodeData->getMutex()); - if (!lock.isLocked()) { - return; - } - ++_sumListeners; - - AvatarData& avatar = nodeData->getAvatar(); - glm::vec3 myPosition = avatar.getPosition(); - - // reset the internal state for correct random number distribution - distribution.reset(); - - // reset the max distance for this frame - float maxAvatarDistanceThisFrame = 0.0f; - - // reset the number of sent avatars - nodeData->resetNumAvatarsSentLastFrame(); - - // keep a counter of the number of considered avatars - int numOtherAvatars = 0; - - // keep track of outbound data rate specifically for avatar data - int numAvatarDataBytes = 0; - - // keep track of the number of other avatars held back in this frame - int numAvatarsHeldBack = 0; - - // keep track of the number of other avatar frames skipped - int numAvatarsWithSkippedFrames = 0; - - // use the data rate specifically for avatar data for FRD adjustment checks - float avatarDataRateLastSecond = nodeData->getOutboundAvatarDataKbps(); - - // Check if it is time to adjust what we send this client based on the observed - // bandwidth to this node. We do this once a second, which is also the window for - // the bandwidth reported by node->getOutboundBandwidth(); - if (nodeData->getNumFramesSinceFRDAdjustment() > MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND) { - - const float FRD_ADJUSTMENT_ACCEPTABLE_RATIO = 0.8f; - const float HYSTERISIS_GAP = (1 - FRD_ADJUSTMENT_ACCEPTABLE_RATIO); - const float HYSTERISIS_MIDDLE_PERCENTAGE = (1 - (HYSTERISIS_GAP * 0.5f)); - - // get the current full rate distance so we can work with it - float currentFullRateDistance = nodeData->getFullRateDistance(); - - if (avatarDataRateLastSecond > _maxKbpsPerNode) { - - // is the FRD greater than the farthest avatar? - // if so, before we calculate anything, set it to that distance - currentFullRateDistance = std::min(currentFullRateDistance, nodeData->getMaxAvatarDistance()); - - // we're adjusting the full rate distance to target a bandwidth in the middle - // of the hysterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } else if (currentFullRateDistance < nodeData->getMaxAvatarDistance() - && avatarDataRateLastSecond < _maxKbpsPerNode * FRD_ADJUSTMENT_ACCEPTABLE_RATIO) { - // we are constrained AND we've recovered to below the acceptable ratio - // lets adjust the full rate distance to target a bandwidth in the middle of the hyterisis gap - currentFullRateDistance *= (_maxKbpsPerNode * HYSTERISIS_MIDDLE_PERCENTAGE) / avatarDataRateLastSecond; - - nodeData->setFullRateDistance(currentFullRateDistance); - nodeData->resetNumFramesSinceFRDAdjustment(); - } - } else { - nodeData->incrementNumFramesSinceFRDAdjustment(); - } - - // setup a PacketList for the avatarPackets - auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - - // this is an AGENT we have received head data from - // send back a packet with other active node data to this node - nodeList->eachMatchingNode( - [&](const SharedNodePointer& otherNode)->bool { - if (!otherNode->getLinkedData()) { - return false; - } - if (otherNode->getUUID() == node->getUUID()) { - return false; - } - - return true; - }, - [&](const SharedNodePointer& otherNode) { - ++numOtherAvatars; - - MessagesMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - MutexTryLocker lock(otherNodeData->getMutex()); - if (!lock.isLocked()) { - return; - } - - // make sure we send out identity and billboard packets to and from new arrivals. - bool forceSend = !otherNodeData->checkAndSetHasReceivedFirstPacketsFrom(node->getUUID()); - - // we will also force a send of billboard or identity packet - // if either has changed in the last frame - if (otherNodeData->getBillboardChangeTimestamp() > 0 - && (forceSend - || otherNodeData->getBillboardChangeTimestamp() > _lastFrameTimestamp - || distribution(generator) < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) { - - QByteArray rfcUUID = otherNode->getUUID().toRfc4122(); - QByteArray billboard = otherNodeData->getAvatar().getBillboard(); - - auto billboardPacket = NLPacket::create(PacketType::AvatarBillboard, rfcUUID.size() + billboard.size()); - billboardPacket->write(rfcUUID); - billboardPacket->write(billboard); - - nodeList->sendPacket(std::move(billboardPacket), *node); - - ++_sumBillboardPackets; - } - - if (otherNodeData->getIdentityChangeTimestamp() > 0 - && (forceSend - || otherNodeData->getIdentityChangeTimestamp() > _lastFrameTimestamp - || distribution(generator) < BILLBOARD_AND_IDENTITY_SEND_PROBABILITY)) { - - QByteArray individualData = otherNodeData->getAvatar().identityByteArray(); - - auto identityPacket = NLPacket::create(PacketType::AvatarIdentity, individualData.size()); - - individualData.replace(0, NUM_BYTES_RFC4122_UUID, otherNode->getUUID().toRfc4122()); - - identityPacket->write(individualData); - - nodeList->sendPacket(std::move(identityPacket), *node); - - ++_sumIdentityPackets; - } - - AvatarData& otherAvatar = otherNodeData->getAvatar(); - // Decide whether to send this avatar's data based on it's distance from us - - // The full rate distance is the distance at which EVERY update will be sent for this avatar - // at twice the full rate distance, there will be a 50% chance of sending this avatar's update - glm::vec3 otherPosition = otherAvatar.getPosition(); - float distanceToAvatar = glm::length(myPosition - otherPosition); - - // potentially update the max full rate distance for this frame - maxAvatarDistanceThisFrame = std::max(maxAvatarDistanceThisFrame, distanceToAvatar); - - if (distanceToAvatar != 0.0f - && distribution(generator) > (nodeData->getFullRateDistance() / distanceToAvatar)) { - return; - } - - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(otherNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = otherNodeData->getLastReceivedSequenceNumber(); - - if (lastSeqToReceiver > lastSeqFromSender && lastSeqToReceiver != UINT16_MAX) { - // we got out out of order packets from the sender, track it - otherNodeData->incrementNumOutOfOrderSends(); - } - - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - return; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - - // we're going to send this avatar - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += - avatarPacketList->write(otherAvatar.toByteArray(false, distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO)); - - avatarPacketList->endSegment(); - }); - - // close the current packet so that we're always sending something - avatarPacketList->closeCurrentPacket(true); - - // send the avatar data PacketList - nodeList->sendPacketList(std::move(avatarPacketList), *node); - - // record the bytes sent for other avatar data in the MessagesMixerClientData - nodeData->recordSentAvatarData(numAvatarDataBytes); - - // record the number of avatars held back this frame - nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); - nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); - - if (numOtherAvatars == 0) { - // update the full rate distance to FLOAT_MAX since we didn't have any other avatars to send - nodeData->setMaxAvatarDistance(FLT_MAX); - } else { - nodeData->setMaxAvatarDistance(maxAvatarDistanceThisFrame); - } - } - ); - - qDebug() << "MessagesMixer::broadcastMessagesData()... calling nodeList->eachMatchingNode() for encode..."; - - // We're done encoding this version of the otherAvatars. Update their "lastSent" joint-states so - // that we can notice differences, next time around. - nodeList->eachMatchingNode( - [&](const SharedNodePointer& otherNode)->bool { - if (!otherNode->getLinkedData()) { - return false; - } - if (otherNode->getType() != NodeType::Agent) { - return false; - } - if (!otherNode->getActiveSocket()) { - return false; - } - return true; - }, - [&](const SharedNodePointer& otherNode) { - MessagesMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - MutexTryLocker lock(otherNodeData->getMutex()); - if (!lock.isLocked()) { - return; - } - AvatarData& otherAvatar = otherNodeData->getAvatar(); - otherAvatar.doneEncoding(false); - }); - - _lastFrameTimestamp = QDateTime::currentMSecsSinceEpoch(); -} - void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { + qDebug() << "MessagesMixer::nodeKilled()... node:" << killedNode->getUUID(); + if (killedNode->getType() == NodeType::Agent && killedNode->getLinkedData()) { auto nodeList = DependencyManager::get(); - // this was an avatar we were sending to other people - // send a kill packet for it to our other nodes - auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); - killPacket->write(killedNode->getUUID().toRfc4122()); - - nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); - // we also want to remove sequence number data for this avatar on our other avatars // so invoke the appropriate method on the MessagesMixerClientData for other avatars nodeList->eachMatchingNode( @@ -414,18 +86,32 @@ void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { return true; }, [&](const SharedNodePointer& node) { - QMetaObject::invokeMethod(node->getLinkedData(), - "removeLastBroadcastSequenceNumber", - Qt::AutoConnection, - Q_ARG(const QUuid&, QUuid(killedNode->getUUID()))); + qDebug() << "eachMatchingNode()... node:" << node->getUUID(); } ); } } -void MessagesMixer::handleMessagesDataPacket(QSharedPointer packet, SharedNodePointer senderNode) { +void MessagesMixer::handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode) { + qDebug() << "MessagesMixer::handleMessagesPacket()... senderNode:" << senderNode->getUUID(); + auto nodeList = DependencyManager::get(); - nodeList->updateNodeWithDataFromPacket(packet, senderNode); + //nodeList->updateNodeWithDataFromPacket(packet, senderNode); + + QByteArray data = packetList->getMessage(); + auto packetType = packetList->getType(); + + if (packetType == PacketType::MessagesData) { + QString message = QString::fromUtf8(data); + qDebug() << "got a messages packet:" << message; + + // this was an avatar we were sending to other people + // send a kill packet for it to our other nodes + //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); + //killPacket->write(killedNode->getUUID().toRfc4122()); + //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); + + } } void MessagesMixer::sendStatsPacket() { @@ -484,6 +170,7 @@ void MessagesMixer::run() { node->setLinkedData(new MessagesMixerClientData()); }; + /* // setup the timer that will be fired on the broadcast thread _broadcastTimer = new QTimer; _broadcastTimer->setInterval(MESSAGES_DATA_SEND_INTERVAL_MSECS); @@ -492,6 +179,7 @@ void MessagesMixer::run() { // connect appropriate signals and slots connect(_broadcastTimer, &QTimer::timeout, this, &MessagesMixer::broadcastMessagesData, Qt::DirectConnection); connect(&_broadcastThread, SIGNAL(started()), _broadcastTimer, SLOT(start())); + */ // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); @@ -516,7 +204,7 @@ void MessagesMixer::run() { parseDomainServerSettings(domainHandler.getSettingsObject()); // start the broadcastThread - _broadcastThread.start(); + //_broadcastThread.start(); } void MessagesMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { diff --git a/assignment-client/src/messages/MessagesMixer.h b/assignment-client/src/messages/MessagesMixer.h index d96a20dd18..0d390b83a4 100644 --- a/assignment-client/src/messages/MessagesMixer.h +++ b/assignment-client/src/messages/MessagesMixer.h @@ -32,7 +32,7 @@ public slots: void sendStatsPacket(); private slots: - void handleMessagesDataPacket(QSharedPointer packet, SharedNodePointer senderNode); + void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); private: void broadcastMessagesData(); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 96b8ab74a8..692b372ae4 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -70,6 +70,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,7 @@ bool setupEssentials(int& argc, char** argv) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); return true; @@ -484,6 +486,14 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : connect(assetThread, &QThread::started, assetClient.data(), &AssetClient::init); assetThread->start(); + // Setup MessagesClient + auto messagesClient = DependencyManager::get(); + QThread* messagesThread = new QThread; + messagesThread->setObjectName("Messages Client Thread"); + messagesClient->moveToThread(messagesThread); + connect(messagesThread, &QThread::started, messagesClient.data(), &MessagesClient::init); + messagesThread->start(); + const DomainHandler& domainHandler = nodeList->getDomainHandler(); connect(&domainHandler, SIGNAL(hostnameChanged(const QString&)), SLOT(domainChanged(const QString&))); @@ -4019,6 +4029,7 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerFunction("HMD", "getHUDLookAtPosition3D", HMDScriptingInterface::getHUDLookAtPosition3D, 0); scriptEngine->registerGlobalObject("Scene", DependencyManager::get().data()); + scriptEngine->registerGlobalObject("Messages", DependencyManager::get().data()); scriptEngine->registerGlobalObject("ScriptDiscoveryService", this->getRunningScriptsWidget()); diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index d3d44b6fdc..7517bb6535 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -74,34 +74,25 @@ bool haveMessagesMixer() { return true; } -void MessagesClient::handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode) { - auto packetType = packet->getType(); +void MessagesClient::handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode) { + QByteArray data = packetList->getMessage(); + auto packetType = packetList->getType(); if (packetType == PacketType::MessagesData) { - qDebug() << "got a messages packet"; + QString message = QString::fromUtf8(data); + qDebug() << "got a messages packet:" << message; } } void MessagesClient::sendMessage(const QString& channel, const QString& message) { + qDebug() << "MessagesClient::sendMessage() channel:" << channel << "message:" << message; auto nodeList = DependencyManager::get(); SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); if (messagesMixer) { auto packetList = NLPacketList::create(PacketType::MessagesData, QByteArray(), true, true); - - #if 0 - auto messageID = ++_currentID; - packetList->writePrimitive(messageID); - - packetList->writePrimitive(static_cast(extension.length())); - packetList->write(extension.toLatin1().constData(), extension.length()); - - uint64_t size = data.length(); - packetList->writePrimitive(size); - packetList->write(data.constData(), size); - - nodeList->sendPacketList(std::move(packetList), *assetServer); - #endif + packetList->write(message.toUtf8()); + nodeList->sendPacketList(std::move(packetList), *messagesMixer); } } diff --git a/libraries/networking/src/MessagesClient.h b/libraries/networking/src/MessagesClient.h index f1f13bfe20..121e6041b1 100644 --- a/libraries/networking/src/MessagesClient.h +++ b/libraries/networking/src/MessagesClient.h @@ -31,7 +31,7 @@ public: Q_INVOKABLE void sendMessage(const QString& channel, const QString& message); private slots: - void handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode); + void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); void handleNodeKilled(SharedNodePointer node); private: From f40ff69c750b499c3d1f74fd02b6393af122a2ae Mon Sep 17 00:00:00 2001 From: AlessandroSigna Date: Mon, 16 Nov 2015 17:27:00 -0800 Subject: [PATCH 13/36] added scripts for group recording --- .../entityScripts/recordingEntityScript.js | 91 ++++++++++++++ examples/entityScripts/recordingMaster.js | 116 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 examples/entityScripts/recordingEntityScript.js create mode 100644 examples/entityScripts/recordingMaster.js diff --git a/examples/entityScripts/recordingEntityScript.js b/examples/entityScripts/recordingEntityScript.js new file mode 100644 index 0000000000..ede6f4fbe2 --- /dev/null +++ b/examples/entityScripts/recordingEntityScript.js @@ -0,0 +1,91 @@ +// +// recordingEntityScript.js +// examples/entityScripts +// +// Created by Alessandro Signa on 11/12/15. +// Copyright 2015 High Fidelity, Inc. +// + +// All the avatars in the area when the master presses the button will start/stop recording. +// + +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + + + + +(function() { + var insideRecorderArea = false; + var enteredInTime = false; + var isAvatarRecording = false; + var _this; + + function recordingEntity() { + _this = this; + return; + } + + recordingEntity.prototype = { + update: function(){ + var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); + var isRecordingStarted = userData.recordingKey.isRecordingStarted; + if(isRecordingStarted && !isAvatarRecording){ + _this.startRecording(); + }else if((!isRecordingStarted && isAvatarRecording) || (isAvatarRecording && !insideRecorderArea)){ + _this.stopRecording(); + }else if(!isRecordingStarted && insideRecorderArea && !enteredInTime){ + //if an avatar enters the zone while a recording is started he will be able to participate to the next group recording + enteredInTime = true; + } + + }, + preload: function(entityID) { + this.entityID = entityID; + Script.update.connect(_this.update); + }, + enterEntity: function(entityID) { + print("entering in the recording area"); + insideRecorderArea = true; + var userData = JSON.parse(Entities.getEntityProperties(_this.entityID, ["userData"]).userData); + var isRecordingStarted = userData.recordingKey.isRecordingStarted; + if(!isRecordingStarted){ + //i'm in the recording area in time (before the event starts) + enteredInTime = true; + } + }, + leaveEntity: function(entityID) { + print("leaving the recording area"); + insideRecorderArea = false; + enteredInTime = false; + }, + + startRecording: function(entityID){ + if(enteredInTime && !isAvatarRecording){ + print("RECORDING STARTED"); + Recording.startRecording(); + isAvatarRecording = true; + } + }, + + stopRecording: function(entityID){ + if(isAvatarRecording){ + print("RECORDING ENDED"); + Recording.stopRecording(); + Recording.loadLastRecording(); + isAvatarRecording = false; + recordingFile = Window.save("Save recording to file", "./groupRecording", "Recordings (*.hfr)"); + if (!(recordingFile === "null" || recordingFile === null || recordingFile === "")) { + Recording.saveRecording(recordingFile); + } + } + }, + clean: function(entityID) { + Script.update.disconnect(_this.update); + } + } + + + + return new recordingEntity(); +}); diff --git a/examples/entityScripts/recordingMaster.js b/examples/entityScripts/recordingMaster.js new file mode 100644 index 0000000000..3cec521ce0 --- /dev/null +++ b/examples/entityScripts/recordingMaster.js @@ -0,0 +1,116 @@ +// +// recordingMaster.js +// examples/entityScripts +// +// Created by Alessandro Signa on 11/12/15. +// Copyright 2015 High Fidelity, Inc. +// +// Run this script to spawn a box (recorder) and drive the start/end of the recording for anyone who is inside the box +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +var PARAMS_SCRIPT_URL = Script.resolvePath('recordingEntityScript.js'); + +HIFI_PUBLIC_BUCKET = "http://s3.amazonaws.com/hifi-public/"; +Script.include("../libraries/toolBars.js"); +Script.include("../libraries/utils.js"); + + + +var rotation = Quat.safeEulerAngles(Camera.getOrientation()); +rotation = Quat.fromPitchYawRollDegrees(0, rotation.y, 0); +var center = Vec3.sum(MyAvatar.position, Vec3.multiply(1, Quat.getFront(rotation))); + +var TOOL_ICON_URL = HIFI_PUBLIC_BUCKET + "images/tools/"; +var ALPHA_ON = 1.0; +var ALPHA_OFF = 0.7; +var COLOR_TOOL_BAR = { red: 0, green: 0, blue: 0 }; + +var toolBar = null; +var recordIcon; + + + +var isRecording = false; + +var recordAreaEntity = Entities.addEntity({ + name: 'recorderEntity', + dimensions: { + x: 2, + y: 1, + z: 2 + }, + type: 'Box', + position: center, + color: { + red: 255, + green: 255, + blue: 255 + }, + visible: true, + ignoreForCollisions: true, + script: PARAMS_SCRIPT_URL, + + userData: JSON.stringify({ + recordingKey: { + isRecordingStarted: false + } + }) +}); + + +setupToolBar(); + +function setupToolBar() { + if (toolBar != null) { + print("Multiple calls to setupToolBar()"); + return; + } + Tool.IMAGE_HEIGHT /= 2; + Tool.IMAGE_WIDTH /= 2; + + toolBar = new ToolBar(0, 0, ToolBar.HORIZONTAL); //put the button in the up-left corner + + toolBar.setBack(COLOR_TOOL_BAR, ALPHA_OFF); + + recordIcon = toolBar.addTool({ + imageURL: TOOL_ICON_URL + "recording-record.svg", + subImage: { x: 0, y: 0, width: Tool.IMAGE_WIDTH, height: Tool.IMAGE_HEIGHT }, + x: 0, y: 0, + width: Tool.IMAGE_WIDTH, + height: Tool.IMAGE_HEIGHT, + alpha: Recording.isPlaying() ? ALPHA_OFF : ALPHA_ON, + visible: true + }, true, isRecording); + +} + +function mousePressEvent(event) { + clickedOverlay = Overlays.getOverlayAtPoint({ x: event.x, y: event.y }); + if (recordIcon === toolBar.clicked(clickedOverlay, false)) { + if (!isRecording) { + print("I'm the master. I want to start recording"); + isRecording = true; + setEntityCustomData("recordingKey", recordAreaEntity, {isRecordingStarted: true}); + + } else { + print("I want to stop recording"); + isRecording = false; + setEntityCustomData("recordingKey", recordAreaEntity, {isRecordingStarted: false}); + + } + } +} + + +function cleanup() { + toolBar.cleanup(); + Entities.callEntityMethod(recordAreaEntity, 'clean'); //have to call this before deleting to avoid the JSON warnings + Entities.deleteEntity(recordAreaEntity); +} + + + + Script.scriptEnding.connect(cleanup); + Controller.mousePressEvent.connect(mousePressEvent); \ No newline at end of file From 1e0b66a68ffab52f6240273541a4ea4c64da9fd5 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 16 Nov 2015 19:23:39 -0800 Subject: [PATCH 14/36] more work --- .../src/messages/MessagesMixer.cpp | 28 +++++++++++++++++-- .../src/messages/MessagesMixer.h | 5 ++-- interface/src/Application.cpp | 2 +- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 6177850532..11e362ad94 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -92,8 +92,8 @@ void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { } } -void MessagesMixer::handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode) { - qDebug() << "MessagesMixer::handleMessagesPacket()... senderNode:" << senderNode->getUUID(); +void MessagesMixer::handleMessagesPacketList(QSharedPointer packetList, SharedNodePointer senderNode) { + qDebug() << "MessagesMixer::handleMessagesPacketList()... senderNode:" << senderNode->getUUID(); auto nodeList = DependencyManager::get(); //nodeList->updateNodeWithDataFromPacket(packet, senderNode); @@ -114,6 +114,30 @@ void MessagesMixer::handleMessagesPacket(QSharedPointer packetList } } +void MessagesMixer::handleMessagesPacket(QSharedPointer packet, SharedNodePointer sendingNode) { + qDebug() << "MessagesMixer::handleMessagesPacket()... senderNode:" << sendingNode->getUUID(); + + /* + auto nodeList = DependencyManager::get(); + //nodeList->updateNodeWithDataFromPacket(packet, senderNode); + + QByteArray data = packetList->getMessage(); + auto packetType = packetList->getType(); + + if (packetType == PacketType::MessagesData) { + QString message = QString::fromUtf8(data); + qDebug() << "got a messages packet:" << message; + + // this was an avatar we were sending to other people + // send a kill packet for it to our other nodes + //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); + //killPacket->write(killedNode->getUUID().toRfc4122()); + //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); + + } + */ +} + void MessagesMixer::sendStatsPacket() { QJsonObject statsObject; statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; diff --git a/assignment-client/src/messages/MessagesMixer.h b/assignment-client/src/messages/MessagesMixer.h index 0d390b83a4..057796bc54 100644 --- a/assignment-client/src/messages/MessagesMixer.h +++ b/assignment-client/src/messages/MessagesMixer.h @@ -32,8 +32,9 @@ public slots: void sendStatsPacket(); private slots: - void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); - + void handleMessagesPacketList(QSharedPointer packetList, SharedNodePointer senderNode); + void handleMessagesPacket(QSharedPointer packet, SharedNodePointer sendingNode); + private: void broadcastMessagesData(); void parseDomainServerSettings(const QJsonObject& domainSettings); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 692b372ae4..712fb7dc02 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -560,7 +560,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer) : // tell the NodeList instance who to tell the domain server we care about nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer - << NodeType::EntityServer << NodeType::AssetServer); + << NodeType::EntityServer << NodeType::AssetServer << NodeType::MessagesMixer); // connect to the packet sent signal of the _entityEditSender connect(&_entityEditSender, &EntityEditPacketSender::packetSent, this, &Application::packetSent); From 5f04e4a16715e49275cc155ad7b1e45fd925f965 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 16 Nov 2015 19:29:54 -0800 Subject: [PATCH 15/36] more work --- assignment-client/src/messages/MessagesMixer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 11e362ad94..988e2bfeac 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -49,6 +49,7 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); + packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacketList"); } MessagesMixer::~MessagesMixer() { From eb4bb1cc03ceb3a5dbdede1e83eb54dc8d3cc297 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Mon, 16 Nov 2015 19:34:53 -0800 Subject: [PATCH 16/36] more work --- assignment-client/src/messages/MessagesMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 988e2bfeac..5b1637e337 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -49,7 +49,7 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacketList"); + packetReceiver.registerMessageListener(PacketType::MessagesData, this, "handleMessagesPacketList"); } MessagesMixer::~MessagesMixer() { From f712fae4d26df69c9d7e82c3e1824f20d49d01c4 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 17 Nov 2015 08:56:21 -0800 Subject: [PATCH 17/36] more hacking --- assignment-client/src/messages/MessagesMixer.cpp | 4 ++-- libraries/networking/src/PacketReceiver.cpp | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 988e2bfeac..d9cc772eb2 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -48,8 +48,8 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &MessagesMixer::nodeKilled); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacketList"); + //packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); + packetReceiver.registerMessageListener(PacketType::MessagesData, this, "handleMessagesPacketList"); } MessagesMixer::~MessagesMixer() { diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index 9d25724f6c..81d8c5ee73 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -95,6 +95,8 @@ void PacketReceiver::registerDirectListenerForTypes(PacketTypeList types, } bool PacketReceiver::registerMessageListener(PacketType type, QObject* listener, const char* slot) { + qCDebug(networking) << "PacketReceiver::registerMessageListener() packet list type" << type; + Q_ASSERT_X(listener, "PacketReceiver::registerMessageListener", "No object to register"); Q_ASSERT_X(slot, "PacketReceiver::registerMessageListener", "No slot to register"); @@ -110,8 +112,12 @@ bool PacketReceiver::registerMessageListener(PacketType type, QObject* listener, // add the mapping _packetListListenerMap[type] = ObjectMethodPair(QPointer(listener), matchingMethod); + + qCDebug(networking) << "Registering a packet listener for packet list type" << type; + return true; } else { + qCDebug(networking) << "NOT Registering a packet listener for packet list type" << type; return false; } } @@ -352,7 +358,7 @@ void PacketReceiver::handleVerifiedPacketList(std::unique_ptr p } } else if (it == _packetListListenerMap.end()) { - qCWarning(networking) << "No listener found for packet type" << nlPacketList->getType(); + qCWarning(networking) << "No listener found for packet list type" << nlPacketList->getType(); // insert a dummy listener so we don't print this again _packetListListenerMap.insert(nlPacketList->getType(), { nullptr, QMetaMethod() }); From 6b61ec569ca5203e5a3ede40ca12c56dd98f2fb4 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 09:47:50 -0800 Subject: [PATCH 18/36] more work on channels --- .../src/messages/MessagesMixer.cpp | 135 +++++++----------- .../src/messages/MessagesMixer.h | 21 ++- .../src/messages/MessagesMixerClientData.cpp | 55 ------- .../src/messages/MessagesMixerClientData.h | 105 -------------- libraries/networking/src/MessagesClient.cpp | 12 +- libraries/networking/src/udt/PacketHeaders.h | 4 +- 6 files changed, 73 insertions(+), 259 deletions(-) delete mode 100644 assignment-client/src/messages/MessagesMixerClientData.cpp delete mode 100644 assignment-client/src/messages/MessagesMixerClientData.h diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index d9cc772eb2..16b24088e2 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -25,7 +26,6 @@ #include #include -#include "MessagesMixerClientData.h" #include "MessagesMixer.h" const QString MESSAGES_MIXER_LOGGING_NAME = "messages-mixer"; @@ -35,7 +35,6 @@ const unsigned int MESSAGES_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) MESSAGES_ MessagesMixer::MessagesMixer(NLPacket& packet) : ThreadedAssignment(packet), - _broadcastThread(), _lastFrameTimestamp(QDateTime::currentMSecsSinceEpoch()), _trailingSleepRatio(1.0f), _performanceThrottlingRatio(0.0f), @@ -48,17 +47,12 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &MessagesMixer::nodeKilled); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); - //packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagesPacket"); - packetReceiver.registerMessageListener(PacketType::MessagesData, this, "handleMessagesPacketList"); + packetReceiver.registerMessageListener(PacketType::MessagesData, this, "handleMessages"); + packetReceiver.registerMessageListener(PacketType::MessagesSubscribe, this, "handleMessagesSubscribe"); + packetReceiver.registerMessageListener(PacketType::MessagesUnsubscribe, this, "handleMessagesUnsubscribe"); } MessagesMixer::~MessagesMixer() { - if (_broadcastTimer) { - _broadcastTimer->deleteLater(); - } - - _broadcastThread.quit(); - _broadcastThread.wait(); } // An 80% chance of sending a identity packet within a 5 second interval. @@ -93,52 +87,55 @@ void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { } } -void MessagesMixer::handleMessagesPacketList(QSharedPointer packetList, SharedNodePointer senderNode) { - qDebug() << "MessagesMixer::handleMessagesPacketList()... senderNode:" << senderNode->getUUID(); +void MessagesMixer::handleMessages(QSharedPointer packetList, SharedNodePointer senderNode) { + Q_ASSERT(packetList->getType() == PacketType::MessagesData); + qDebug() << "MessagesMixer::handleMessages()... senderNode:" << senderNode->getUUID(); + + QByteArray packetData = packetList->getMessage(); + QBuffer packet{ &packetData }; + packet.open(QIODevice::ReadOnly); + + quint16 channelLength; + packet.read(reinterpret_cast(&channelLength), sizeof(channelLength)); + auto channelData = packet.read(channelLength); + QString channel = QString::fromUtf8(channelData); + + quint16 messageLength; + packet.read(reinterpret_cast(&messageLength), sizeof(messageLength)); + auto messageData = packet.read(messageLength); + QString message = QString::fromUtf8(messageData); + + auto nodeList = DependencyManager::get(); - //nodeList->updateNodeWithDataFromPacket(packet, senderNode); - QByteArray data = packetList->getMessage(); - auto packetType = packetList->getType(); + qDebug() << "got a messages:" << message << "on channel:" << channel << "from node:" << senderNode->getUUID(); - if (packetType == PacketType::MessagesData) { - QString message = QString::fromUtf8(data); - qDebug() << "got a messages packet:" << message; + // this was an avatar we were sending to other people + // send a kill packet for it to our other nodes + //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); + //killPacket->write(killedNode->getUUID().toRfc4122()); + //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); +} - // this was an avatar we were sending to other people - // send a kill packet for it to our other nodes - //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); - //killPacket->write(killedNode->getUUID().toRfc4122()); - //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); +void MessagesMixer::handleMessagesSubscribe(QSharedPointer packetList, SharedNodePointer senderNode) { + Q_ASSERT(packetList->getType() == PacketType::MessagesSubscribe); + QString channel = QString::fromUtf8(packetList->getMessage()); + qDebug() << "Node [" << senderNode->getUUID() << "] subscribed to channel:" << channel; + _channelSubscribers[channel] << senderNode->getUUID(); +} +void MessagesMixer::handleMessagesUnsubscribe(QSharedPointer packetList, SharedNodePointer senderNode) { + Q_ASSERT(packetList->getType() == PacketType::MessagesUnsubscribe); + QString channel = QString::fromUtf8(packetList->getMessage()); + qDebug() << "Node [" << senderNode->getUUID() << "] unsubscribed from channel:" << channel; + + if (_channelSubscribers.contains(channel)) { + _channelSubscribers[channel].remove(senderNode->getUUID()); } } -void MessagesMixer::handleMessagesPacket(QSharedPointer packet, SharedNodePointer sendingNode) { - qDebug() << "MessagesMixer::handleMessagesPacket()... senderNode:" << sendingNode->getUUID(); - - /* - auto nodeList = DependencyManager::get(); - //nodeList->updateNodeWithDataFromPacket(packet, senderNode); - - QByteArray data = packetList->getMessage(); - auto packetType = packetList->getType(); - - if (packetType == PacketType::MessagesData) { - QString message = QString::fromUtf8(data); - qDebug() << "got a messages packet:" << message; - - // this was an avatar we were sending to other people - // send a kill packet for it to our other nodes - //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); - //killPacket->write(killedNode->getUUID().toRfc4122()); - //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); - - } - */ -} - +// FIXME - make these stats relevant void MessagesMixer::sendStatsPacket() { QJsonObject statsObject; statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; @@ -160,18 +157,6 @@ void MessagesMixer::sendStatsPacket() { messagesStats[NODE_OUTBOUND_KBPS_STAT_KEY] = node->getOutboundBandwidth(); messagesStats[NODE_INBOUND_KBPS_STAT_KEY] = node->getInboundBandwidth(); - MessagesMixerClientData* clientData = static_cast(node->getLinkedData()); - if (clientData) { - MutexTryLocker lock(clientData->getMutex()); - if (lock.isLocked()) { - clientData->loadJSONStats(messagesStats); - - // add the diff between the full outbound bandwidth and the measured bandwidth for AvatarData send only - messagesStats["delta_full_vs_avatar_data_kbps"] = - messagesStats[NODE_OUTBOUND_KBPS_STAT_KEY].toDouble() - messagesStats[OUTBOUND_MESSAGES_DATA_STATS_KEY].toDouble(); - } - } - messagesObject[uuidStringWithoutCurlyBraces(node->getUUID())] = messagesStats; }); @@ -192,27 +177,15 @@ void MessagesMixer::run() { nodeList->addNodeTypeToInterestSet(NodeType::Agent); nodeList->linkedDataCreateCallback = [] (Node* node) { - node->setLinkedData(new MessagesMixerClientData()); + // no need to link data }; - /* - // setup the timer that will be fired on the broadcast thread - _broadcastTimer = new QTimer; - _broadcastTimer->setInterval(MESSAGES_DATA_SEND_INTERVAL_MSECS); - _broadcastTimer->moveToThread(&_broadcastThread); - - // connect appropriate signals and slots - connect(_broadcastTimer, &QTimer::timeout, this, &MessagesMixer::broadcastMessagesData, Qt::DirectConnection); - connect(&_broadcastThread, SIGNAL(started()), _broadcastTimer, SLOT(start())); - */ - // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); qDebug() << "Waiting for domain settings from domain-server."; // block until we get the settingsRequestComplete signal - QEventLoop loop; connect(&domainHandler, &DomainHandler::settingsReceived, &loop, &QEventLoop::quit); connect(&domainHandler, &DomainHandler::settingsReceiveFail, &loop, &QEventLoop::quit); @@ -227,22 +200,16 @@ void MessagesMixer::run() { // parse the settings to pull out the values we need parseDomainServerSettings(domainHandler.getSettingsObject()); - - // start the broadcastThread - //_broadcastThread.start(); } void MessagesMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { qDebug() << "MessagesMixer::parseDomainServerSettings() domainSettings:" << domainSettings; const QString MESSAGES_MIXER_SETTINGS_KEY = "messages_mixer"; - const QString NODE_SEND_BANDWIDTH_KEY = "max_node_send_bandwidth"; - const float DEFAULT_NODE_SEND_BANDWIDTH = 1.0f; - QJsonValue nodeBandwidthValue = domainSettings[MESSAGES_MIXER_SETTINGS_KEY].toObject()[NODE_SEND_BANDWIDTH_KEY]; - if (!nodeBandwidthValue.isDouble()) { - qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; - } - - _maxKbpsPerNode = nodeBandwidthValue.toDouble(DEFAULT_NODE_SEND_BANDWIDTH) * KILO_PER_MEGA; - qDebug() << "The maximum send bandwidth per node is" << _maxKbpsPerNode << "kbps."; + // TODO - if we want options, parse them here... + // + // QJsonValue nodeBandwidthValue = domainSettings[MESSAGES_MIXER_SETTINGS_KEY].toObject()[NODE_SEND_BANDWIDTH_KEY]; + // if (!nodeBandwidthValue.isDouble()) { + // qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; + // } } diff --git a/assignment-client/src/messages/MessagesMixer.h b/assignment-client/src/messages/MessagesMixer.h index 057796bc54..cd15ea7d2a 100644 --- a/assignment-client/src/messages/MessagesMixer.h +++ b/assignment-client/src/messages/MessagesMixer.h @@ -23,23 +23,22 @@ class MessagesMixer : public ThreadedAssignment { public: MessagesMixer(NLPacket& packet); ~MessagesMixer(); -public slots: - /// runs the avatar mixer - void run(); +public slots: + void run(); void nodeKilled(SharedNodePointer killedNode); - void sendStatsPacket(); private slots: - void handleMessagesPacketList(QSharedPointer packetList, SharedNodePointer senderNode); - void handleMessagesPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void handleMessages(QSharedPointer packetList, SharedNodePointer senderNode); + void handleMessagesSubscribe(QSharedPointer packetList, SharedNodePointer senderNode); + void handleMessagesUnsubscribe(QSharedPointer packetList, SharedNodePointer senderNode); private: - void broadcastMessagesData(); void parseDomainServerSettings(const QJsonObject& domainSettings); - - QThread _broadcastThread; + + + QHash> _channelSubscribers; quint64 _lastFrameTimestamp; @@ -50,10 +49,6 @@ private: int _numStatFrames; int _sumBillboardPackets; int _sumIdentityPackets; - - float _maxKbpsPerNode = 0.0f; - - QTimer* _broadcastTimer = nullptr; }; #endif // hifi_MessagesMixer_h diff --git a/assignment-client/src/messages/MessagesMixerClientData.cpp b/assignment-client/src/messages/MessagesMixerClientData.cpp deleted file mode 100644 index 6aa8f39c22..0000000000 --- a/assignment-client/src/messages/MessagesMixerClientData.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// -// MessagesMixerClientData.cpp -// assignment-client/src/messages -// -// Created by Brad hefta-Gaub on 11/16/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include - -#include "MessagesMixerClientData.h" - -int MessagesMixerClientData::parseData(NLPacket& packet) { - // pull the sequence number from the data first - packet.readPrimitive(&_lastReceivedSequenceNumber); - - // compute the offset to the data payload - return _avatar.parseDataFromBuffer(packet.readWithoutCopy(packet.bytesLeftToRead())); -} - -bool MessagesMixerClientData::checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid) { - if (_hasReceivedFirstPacketsFrom.find(uuid) == _hasReceivedFirstPacketsFrom.end()) { - _hasReceivedFirstPacketsFrom.insert(uuid); - return false; - } - return true; -} - -uint16_t MessagesMixerClientData::getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const { - // return the matching PacketSequenceNumber, or the default if we don't have it - auto nodeMatch = _lastBroadcastSequenceNumbers.find(nodeUUID); - if (nodeMatch != _lastBroadcastSequenceNumbers.end()) { - return nodeMatch->second; - } else { - return 0; - } -} - -void MessagesMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { - jsonObject["display_name"] = _avatar.getDisplayName(); - jsonObject["full_rate_distance"] = _fullRateDistance; - jsonObject["max_av_distance"] = _maxAvatarDistance; - jsonObject["num_avs_sent_last_frame"] = _numAvatarsSentLastFrame; - jsonObject["avg_other_av_starves_per_second"] = getAvgNumOtherAvatarStarvesPerSecond(); - jsonObject["avg_other_av_skips_per_second"] = getAvgNumOtherAvatarSkipsPerSecond(); - jsonObject["total_num_out_of_order_sends"] = _numOutOfOrderSends; - - jsonObject[OUTBOUND_MESSAGES_DATA_STATS_KEY] = getOutboundAvatarDataKbps(); - jsonObject[INBOUND_MESSAGES_DATA_STATS_KEY] = _avatar.getAverageBytesReceivedPerSecond() / (float) BYTES_PER_KILOBIT; - - jsonObject["av_data_receive_rate"] = _avatar.getReceiveRate(); -} diff --git a/assignment-client/src/messages/MessagesMixerClientData.h b/assignment-client/src/messages/MessagesMixerClientData.h deleted file mode 100644 index 1667df431f..0000000000 --- a/assignment-client/src/messages/MessagesMixerClientData.h +++ /dev/null @@ -1,105 +0,0 @@ -// -// MessagesMixerClientData.h -// assignment-client/src/messages -// -// Created by Brad hefta-Gaub on 11/16/2015. -// Copyright 2015 High Fidelity, Inc. -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_MessagesMixerClientData_h -#define hifi_MessagesMixerClientData_h - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -const QString OUTBOUND_MESSAGES_DATA_STATS_KEY = "outbound_av_data_kbps"; -const QString INBOUND_MESSAGES_DATA_STATS_KEY = "inbound_av_data_kbps"; - -class MessagesMixerClientData : public NodeData { - Q_OBJECT -public: - int parseData(NLPacket& packet); - AvatarData& getAvatar() { return _avatar; } - - bool checkAndSetHasReceivedFirstPacketsFrom(const QUuid& uuid); - - uint16_t getLastBroadcastSequenceNumber(const QUuid& nodeUUID) const; - void setLastBroadcastSequenceNumber(const QUuid& nodeUUID, uint16_t sequenceNumber) - { _lastBroadcastSequenceNumbers[nodeUUID] = sequenceNumber; } - Q_INVOKABLE void removeLastBroadcastSequenceNumber(const QUuid& nodeUUID) { _lastBroadcastSequenceNumbers.erase(nodeUUID); } - - uint16_t getLastReceivedSequenceNumber() const { return _lastReceivedSequenceNumber; } - - quint64 getBillboardChangeTimestamp() const { return _billboardChangeTimestamp; } - void setBillboardChangeTimestamp(quint64 billboardChangeTimestamp) { _billboardChangeTimestamp = billboardChangeTimestamp; } - - quint64 getIdentityChangeTimestamp() const { return _identityChangeTimestamp; } - void setIdentityChangeTimestamp(quint64 identityChangeTimestamp) { _identityChangeTimestamp = identityChangeTimestamp; } - - void setFullRateDistance(float fullRateDistance) { _fullRateDistance = fullRateDistance; } - float getFullRateDistance() const { return _fullRateDistance; } - - void setMaxAvatarDistance(float maxAvatarDistance) { _maxAvatarDistance = maxAvatarDistance; } - float getMaxAvatarDistance() const { return _maxAvatarDistance; } - - void resetNumAvatarsSentLastFrame() { _numAvatarsSentLastFrame = 0; } - void incrementNumAvatarsSentLastFrame() { ++_numAvatarsSentLastFrame; } - int getNumAvatarsSentLastFrame() const { return _numAvatarsSentLastFrame; } - - void recordNumOtherAvatarStarves(int numAvatarsHeldBack) { _otherAvatarStarves.updateAverage((float) numAvatarsHeldBack); } - float getAvgNumOtherAvatarStarvesPerSecond() const { return _otherAvatarStarves.getAverageSampleValuePerSecond(); } - - void recordNumOtherAvatarSkips(int numOtherAvatarSkips) { _otherAvatarSkips.updateAverage((float) numOtherAvatarSkips); } - float getAvgNumOtherAvatarSkipsPerSecond() const { return _otherAvatarSkips.getAverageSampleValuePerSecond(); } - - void incrementNumOutOfOrderSends() { ++_numOutOfOrderSends; } - - int getNumFramesSinceFRDAdjustment() const { return _numFramesSinceAdjustment; } - void incrementNumFramesSinceFRDAdjustment() { ++_numFramesSinceAdjustment; } - void resetNumFramesSinceFRDAdjustment() { _numFramesSinceAdjustment = 0; } - - void recordSentAvatarData(int numBytes) { _avgOtherAvatarDataRate.updateAverage((float) numBytes); } - - float getOutboundAvatarDataKbps() const - { return _avgOtherAvatarDataRate.getAverageSampleValuePerSecond() / (float) BYTES_PER_KILOBIT; } - - void loadJSONStats(QJsonObject& jsonObject) const; -private: - AvatarData _avatar; - - uint16_t _lastReceivedSequenceNumber { 0 }; - std::unordered_map _lastBroadcastSequenceNumbers; - std::unordered_set _hasReceivedFirstPacketsFrom; - - quint64 _billboardChangeTimestamp = 0; - quint64 _identityChangeTimestamp = 0; - - float _fullRateDistance = FLT_MAX; - float _maxAvatarDistance = FLT_MAX; - - int _numAvatarsSentLastFrame = 0; - int _numFramesSinceAdjustment = 0; - - SimpleMovingAverage _otherAvatarStarves; - SimpleMovingAverage _otherAvatarSkips; - int _numOutOfOrderSends = 0; - - SimpleMovingAverage _avgOtherAvatarDataRate; -}; - -#endif // hifi_MessagesMixerClientData_h diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index 7517bb6535..cee7206c31 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -91,7 +91,17 @@ void MessagesClient::sendMessage(const QString& channel, const QString& message) if (messagesMixer) { auto packetList = NLPacketList::create(PacketType::MessagesData, QByteArray(), true, true); - packetList->write(message.toUtf8()); + + auto channelUtf8 = channel.toUtf8(); + quint16 channelLength = channelUtf8.length(); + packetList->writePrimitive(channelLength); + packetList->write(channelUtf8); + + auto messageUtf8 = message.toUtf8(); + quint16 messageLength = messageUtf8.length(); + packetList->writePrimitive(messageLength); + packetList->write(messageUtf8); + nodeList->sendPacketList(std::move(packetList), *messagesMixer); } } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index d1287d4a08..e0a847dcc6 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -88,7 +88,9 @@ public: AssetGetInfoReply, DomainDisconnectRequest, DomainServerRemovedNode, - MessagesData + MessagesData, + MessagesSubscribe, + MessagesUnsubscribe }; }; From f9a674bca5ab5cde3eafe40a7d363ab7ad73756d Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 10:01:31 -0800 Subject: [PATCH 19/36] implement subscribe/unsubscribe in MessagesClient --- libraries/networking/src/MessagesClient.cpp | 27 +++++++++++++++++++++ libraries/networking/src/MessagesClient.h | 2 ++ 2 files changed, 29 insertions(+) diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index cee7206c31..175c31beb8 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -106,6 +106,33 @@ void MessagesClient::sendMessage(const QString& channel, const QString& message) } } +// FIXME - we should keep track of the channels we are subscribed to locally, and +// in the event that they mixer goes away and/or comes back we should automatically +// resubscribe to those channels +void MessagesClient::subscribe(const QString& channel) { + qDebug() << "MessagesClient::subscribe() channel:" << channel; + auto nodeList = DependencyManager::get(); + SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); + + if (messagesMixer) { + auto packetList = NLPacketList::create(PacketType::MessagesSubscribe, QByteArray(), true, true); + packetList->write(channel.toUtf8()); + nodeList->sendPacketList(std::move(packetList), *messagesMixer); + } +} + +void MessagesClient::unsubscribe(const QString& channel) { + qDebug() << "MessagesClient::unsubscribe() channel:" << channel; + auto nodeList = DependencyManager::get(); + SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); + + if (messagesMixer) { + auto packetList = NLPacketList::create(PacketType::MessagesUnsubscribe, QByteArray(), true, true); + packetList->write(channel.toUtf8()); + nodeList->sendPacketList(std::move(packetList), *messagesMixer); + } +} + void MessagesClient::handleNodeKilled(SharedNodePointer node) { if (node->getType() != NodeType::MessagesMixer) { return; diff --git a/libraries/networking/src/MessagesClient.h b/libraries/networking/src/MessagesClient.h index 121e6041b1..2c699850f6 100644 --- a/libraries/networking/src/MessagesClient.h +++ b/libraries/networking/src/MessagesClient.h @@ -29,6 +29,8 @@ public: Q_INVOKABLE void init(); Q_INVOKABLE void sendMessage(const QString& channel, const QString& message); + Q_INVOKABLE void subscribe(const QString& channel); + Q_INVOKABLE void unsubscribe(const QString& channel); private slots: void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); From 1f7c70a0d1ef141512a9db231585bc650b468bc1 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 10:28:50 -0800 Subject: [PATCH 20/36] implement proper sending --- .../src/messages/MessagesMixer.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 16b24088e2..8f69b86367 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -116,6 +116,31 @@ void MessagesMixer::handleMessages(QSharedPointer packetList, Shar //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); //killPacket->write(killedNode->getUUID().toRfc4122()); //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); + + nodeList->eachMatchingNode( + [&](const SharedNodePointer& node)->bool { + + return node->getType() == NodeType::Agent && node->getActiveSocket() && + _channelSubscribers[channel].contains(node->getUUID()); + }, + [&](const SharedNodePointer& node) { + + qDebug() << "sending a messages:" << message << "on channel:" << channel << "to node:" << node->getUUID(); + + auto packetList = NLPacketList::create(PacketType::MessagesData, QByteArray(), true, true); + + auto channelUtf8 = channel.toUtf8(); + quint16 channelLength = channelUtf8.length(); + packetList->writePrimitive(channelLength); + packetList->write(channelUtf8); + + auto messageUtf8 = message.toUtf8(); + quint16 messageLength = messageUtf8.length(); + packetList->writePrimitive(messageLength); + packetList->write(messageUtf8); + + nodeList->sendPacketList(std::move(packetList), *node); + }); } void MessagesMixer::handleMessagesSubscribe(QSharedPointer packetList, SharedNodePointer senderNode) { From c27a127739a34e9203a56ef6d25b86aeb7484dad Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 10:41:26 -0800 Subject: [PATCH 21/36] add example script --- examples/example/messagesExample.js | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 examples/example/messagesExample.js diff --git a/examples/example/messagesExample.js b/examples/example/messagesExample.js new file mode 100644 index 0000000000..39ee4a3dbe --- /dev/null +++ b/examples/example/messagesExample.js @@ -0,0 +1,38 @@ +var totalTime = 0; +var unsubscribedForTime = 0; +var subscribedForTime = 0; +var subscribed = false; +var SWITCH_SUBSCRIPTION_TIME = 10; +Script.update.connect(function (deltaTime) { + var channel = "example"; + totalTime += deltaTime; + if (!subscribed) { + unsubscribedForTime += deltaTime; + } else { + subscribedForTime += deltaTime; + } + + if (totalTime > 5) { + + // if we've been unsubscribed for SWITCH_SUBSCRIPTION_TIME seconds, subscribe + if (!subscribed && unsubscribedForTime > SWITCH_SUBSCRIPTION_TIME) { + print("---- subscribing ----"); + Messages.subscribe(channel); + subscribed = true; + subscribedForTime = 0; + } + + // if we've been subscribed for SWITCH_SUBSCRIPTION_TIME seconds, unsubscribe + if (subscribed && subscribedForTime > SWITCH_SUBSCRIPTION_TIME) { + print("---- unsubscribing ----"); + Messages.unsubscribe(channel); + subscribed = false; + unsubscribedForTime = 0; + } + + // Even if not subscribed, still publish + var message = "update() deltaTime:" + deltaTime; + //print(message); + Messages.sendMessage(channel, message); + } +}); \ No newline at end of file From 034debc4833d42104379c3d233ceb2fc3ffefd0f Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 10:59:53 -0800 Subject: [PATCH 22/36] cleanup --- .../src/messages/MessagesMixer.cpp | 77 ++----------------- .../src/messages/MessagesMixer.h | 11 --- .../resources/describe-settings.json | 6 +- .../src/DomainServerSettingsManager.cpp | 3 - libraries/networking/src/MessagesClient.cpp | 2 +- 5 files changed, 11 insertions(+), 88 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 8f69b86367..1c4ddb564b 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -34,14 +34,7 @@ const int MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND = 60; const unsigned int MESSAGES_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; MessagesMixer::MessagesMixer(NLPacket& packet) : - ThreadedAssignment(packet), - _lastFrameTimestamp(QDateTime::currentMSecsSinceEpoch()), - _trailingSleepRatio(1.0f), - _performanceThrottlingRatio(0.0f), - _sumListeners(0), - _numStatFrames(0), - _sumBillboardPackets(0), - _sumIdentityPackets(0) + ThreadedAssignment(packet) { // make sure we hear about node kills so we can tell the other nodes connect(DependencyManager::get().data(), &NodeList::nodeKilled, this, &MessagesMixer::nodeKilled); @@ -55,43 +48,14 @@ MessagesMixer::MessagesMixer(NLPacket& packet) : MessagesMixer::~MessagesMixer() { } -// An 80% chance of sending a identity packet within a 5 second interval. -// assuming 60 htz update rate. -const float BILLBOARD_AND_IDENTITY_SEND_PROBABILITY = 1.0f / 187.0f; - void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { qDebug() << "MessagesMixer::nodeKilled()... node:" << killedNode->getUUID(); - - if (killedNode->getType() == NodeType::Agent - && killedNode->getLinkedData()) { - auto nodeList = DependencyManager::get(); - - // we also want to remove sequence number data for this avatar on our other avatars - // so invoke the appropriate method on the MessagesMixerClientData for other avatars - nodeList->eachMatchingNode( - [&](const SharedNodePointer& node)->bool { - if (!node->getLinkedData()) { - return false; - } - - if (node->getUUID() == killedNode->getUUID()) { - return false; - } - - return true; - }, - [&](const SharedNodePointer& node) { - qDebug() << "eachMatchingNode()... node:" << node->getUUID(); - } - ); - } + // FIXME - remove the node from the subscription maps } void MessagesMixer::handleMessages(QSharedPointer packetList, SharedNodePointer senderNode) { Q_ASSERT(packetList->getType() == PacketType::MessagesData); - qDebug() << "MessagesMixer::handleMessages()... senderNode:" << senderNode->getUUID(); - QByteArray packetData = packetList->getMessage(); QBuffer packet{ &packetData }; packet.open(QIODevice::ReadOnly); @@ -109,14 +73,6 @@ void MessagesMixer::handleMessages(QSharedPointer packetList, Shar auto nodeList = DependencyManager::get(); - qDebug() << "got a messages:" << message << "on channel:" << channel << "from node:" << senderNode->getUUID(); - - // this was an avatar we were sending to other people - // send a kill packet for it to our other nodes - //auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID); - //killPacket->write(killedNode->getUUID().toRfc4122()); - //nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); - nodeList->eachMatchingNode( [&](const SharedNodePointer& node)->bool { @@ -125,8 +81,6 @@ void MessagesMixer::handleMessages(QSharedPointer packetList, Shar }, [&](const SharedNodePointer& node) { - qDebug() << "sending a messages:" << message << "on channel:" << channel << "to node:" << node->getUUID(); - auto packetList = NLPacketList::create(PacketType::MessagesData, QByteArray(), true, true); auto channelUtf8 = channel.toUtf8(); @@ -163,47 +117,37 @@ void MessagesMixer::handleMessagesUnsubscribe(QSharedPointer packe // FIXME - make these stats relevant void MessagesMixer::sendStatsPacket() { QJsonObject statsObject; - statsObject["average_listeners_last_second"] = (float) _sumListeners / (float) _numStatFrames; - statsObject["trailing_sleep_percentage"] = _trailingSleepRatio * 100; - statsObject["performance_throttling_ratio"] = _performanceThrottlingRatio; - QJsonObject messagesObject; - auto nodeList = DependencyManager::get(); // add stats for each listerner nodeList->eachNode([&](const SharedNodePointer& node) { QJsonObject messagesStats; - const QString NODE_OUTBOUND_KBPS_STAT_KEY = "outbound_kbps"; - const QString NODE_INBOUND_KBPS_STAT_KEY = "inbound_kbps"; - // add the key to ask the domain-server for a username replacement, if it has it messagesStats[USERNAME_UUID_REPLACEMENT_STATS_KEY] = uuidStringWithoutCurlyBraces(node->getUUID()); - messagesStats[NODE_OUTBOUND_KBPS_STAT_KEY] = node->getOutboundBandwidth(); - messagesStats[NODE_INBOUND_KBPS_STAT_KEY] = node->getInboundBandwidth(); + messagesStats["outbound_kbps"] = node->getOutboundBandwidth(); + messagesStats["inbound_kbps"] = node->getInboundBandwidth(); messagesObject[uuidStringWithoutCurlyBraces(node->getUUID())] = messagesStats; }); statsObject["messages"] = messagesObject; ThreadedAssignment::addPacketStatsAndSendStatsPacket(statsObject); - - _sumListeners = 0; - _numStatFrames = 0; } void MessagesMixer::run() { ThreadedAssignment::commonInit(MESSAGES_MIXER_LOGGING_NAME, NodeType::MessagesMixer); NodeType_t owningNodeType = DependencyManager::get()->getOwnerType(); - qDebug() << "owningNodeType:" << owningNodeType; auto nodeList = DependencyManager::get(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); + /* nodeList->linkedDataCreateCallback = [] (Node* node) { // no need to link data }; + */ // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); @@ -228,13 +172,6 @@ void MessagesMixer::run() { } void MessagesMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { - qDebug() << "MessagesMixer::parseDomainServerSettings() domainSettings:" << domainSettings; - const QString MESSAGES_MIXER_SETTINGS_KEY = "messages_mixer"; - // TODO - if we want options, parse them here... - // - // QJsonValue nodeBandwidthValue = domainSettings[MESSAGES_MIXER_SETTINGS_KEY].toObject()[NODE_SEND_BANDWIDTH_KEY]; - // if (!nodeBandwidthValue.isDouble()) { - // qDebug() << NODE_SEND_BANDWIDTH_KEY << "is not a double - will continue with default value"; - // } + const QString MESSAGES_MIXER_SETTINGS_KEY = "messages_mixer"; } diff --git a/assignment-client/src/messages/MessagesMixer.h b/assignment-client/src/messages/MessagesMixer.h index cd15ea7d2a..12667bcc1b 100644 --- a/assignment-client/src/messages/MessagesMixer.h +++ b/assignment-client/src/messages/MessagesMixer.h @@ -37,18 +37,7 @@ private slots: private: void parseDomainServerSettings(const QJsonObject& domainSettings); - QHash> _channelSubscribers; - - quint64 _lastFrameTimestamp; - - float _trailingSleepRatio; - float _performanceThrottlingRatio; - - int _sumListeners; - int _numStatFrames; - int _sumBillboardPackets; - int _sumIdentityPackets; }; #endif // hifi_MessagesMixer_h diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index b2443b8bd4..c2410a1d0c 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -556,10 +556,10 @@ "assignment-types": [4], "settings": [ { - "name": "max_node_send_bandwidth", + "name": "unused", "type": "double", - "label": "Per-Node Bandwidth", - "help": "Desired maximum send bandwidth (in Megabits per second) to each node", + "label": "Unused setting", + "help": "an asofyet unused setting", "placeholder": 1.0, "default": 1.0, "advanced": true diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index 924e19e1fc..88fc6a6cad 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -132,7 +132,6 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList } QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString& keyPath) { - qDebug() << "DomainServerSettingsManager::valueOrDefaultValueForKeyPath() keyPath:" << keyPath; const QVariant* foundValue = valueForKeyPath(_configMap.getMergedConfig(), keyPath); if (foundValue) { @@ -227,8 +226,6 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection } QJsonObject DomainServerSettingsManager::responseObjectForType(const QString& typeValue, bool isAuthenticated) { - qDebug() << "DomainServerSettingsManager::responseObjectForType() typeValue:" << typeValue; - QJsonObject responseObject; if (!typeValue.isEmpty() || isAuthenticated) { diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index 175c31beb8..9823a7130e 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -137,5 +137,5 @@ void MessagesClient::handleNodeKilled(SharedNodePointer node) { if (node->getType() != NodeType::MessagesMixer) { return; } - + // FIXME - do we need to do any special bookkeeping for when the messages mixer is no longer available } From 60ae1259fa7de0239eae1f5b8b8434e04c8158b4 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 11:06:27 -0800 Subject: [PATCH 23/36] more cleanup work --- libraries/networking/src/MessagesClient.cpp | 21 ------------------- libraries/networking/src/MessagesClient.h | 2 -- libraries/networking/src/PacketReceiver.cpp | 4 +--- .../networking/src/ThreadedAssignment.cpp | 3 --- 4 files changed, 1 insertion(+), 29 deletions(-) diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index 9823a7130e..60f1146c92 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -14,18 +14,11 @@ #include #include -#include #include -#include -#include "AssetRequest.h" -#include "AssetUpload.h" -#include "AssetUtils.h" -#include "NetworkAccessManager.h" #include "NetworkLogging.h" #include "NodeList.h" #include "PacketReceiver.h" -#include "ResourceCache.h" MessagesClient::MessagesClient() { @@ -45,20 +38,6 @@ void MessagesClient::init() { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "init", Qt::BlockingQueuedConnection); } - - // Setup disk cache if not already - QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - if (!networkAccessManager.cache()) { - QString cachePath = QStandardPaths::writableLocation(QStandardPaths::DataLocation); - cachePath = !cachePath.isEmpty() ? cachePath : "interfaceCache"; - - QNetworkDiskCache* cache = new QNetworkDiskCache(); - cache->setMaximumCacheSize(MAXIMUM_CACHE_SIZE); - cache->setCacheDirectory(cachePath); - networkAccessManager.setCache(cache); - qCDebug(asset_client) << "MessagesClient disk cache setup at" << cachePath - << "(size:" << MAXIMUM_CACHE_SIZE / BYTES_PER_GIGABYTES << "GB)"; - } } bool haveMessagesMixer() { diff --git a/libraries/networking/src/MessagesClient.h b/libraries/networking/src/MessagesClient.h index 2c699850f6..a79b855be9 100644 --- a/libraries/networking/src/MessagesClient.h +++ b/libraries/networking/src/MessagesClient.h @@ -35,8 +35,6 @@ public: private slots: void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); void handleNodeKilled(SharedNodePointer node); - -private: }; #endif diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index 81d8c5ee73..0a3ea86399 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -95,8 +95,6 @@ void PacketReceiver::registerDirectListenerForTypes(PacketTypeList types, } bool PacketReceiver::registerMessageListener(PacketType type, QObject* listener, const char* slot) { - qCDebug(networking) << "PacketReceiver::registerMessageListener() packet list type" << type; - Q_ASSERT_X(listener, "PacketReceiver::registerMessageListener", "No object to register"); Q_ASSERT_X(slot, "PacketReceiver::registerMessageListener", "No slot to register"); @@ -117,7 +115,7 @@ bool PacketReceiver::registerMessageListener(PacketType type, QObject* listener, return true; } else { - qCDebug(networking) << "NOT Registering a packet listener for packet list type" << type; + qCWarning(networking) << "FAILED to Register a packet listener for packet list type" << type; return false; } } diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index b204982896..6855c2eec3 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -124,12 +124,9 @@ void ThreadedAssignment::stopSendingStats() { } void ThreadedAssignment::checkInWithDomainServerOrExit() { - qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit()...."; if (DependencyManager::get()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) { - qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit().... getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS"; setFinished(true); } else { - qDebug() << "ThreadedAssignment::checkInWithDomainServerOrExit().... calling DependencyManager::get()->sendDomainServerCheckIn()"; DependencyManager::get()->sendDomainServerCheckIn(); } } From 1c2c37ff44a95ae675fbca4eb2d4ec58c51664e3 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 11:10:12 -0800 Subject: [PATCH 24/36] more cleanup work --- assignment-client/src/messages/MessagesMixer.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index 1c4ddb564b..b97afad85f 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -30,9 +30,6 @@ const QString MESSAGES_MIXER_LOGGING_NAME = "messages-mixer"; -const int MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND = 60; -const unsigned int MESSAGES_DATA_SEND_INTERVAL_MSECS = (1.0f / (float) MESSAGES_MIXER_BROADCAST_FRAMES_PER_SECOND) * 1000; - MessagesMixer::MessagesMixer(NLPacket& packet) : ThreadedAssignment(packet) { @@ -143,12 +140,6 @@ void MessagesMixer::run() { auto nodeList = DependencyManager::get(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); - /* - nodeList->linkedDataCreateCallback = [] (Node* node) { - // no need to link data - }; - */ - // wait until we have the domain-server settings, otherwise we bail DomainHandler& domainHandler = nodeList->getDomainHandler(); From e7a8df306c881e44ef0cfb5bf6734d7f5924f139 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 11:18:36 -0800 Subject: [PATCH 25/36] fix warnings --- assignment-client/src/entities/EntityServer.cpp | 1 - assignment-client/src/messages/MessagesMixer.cpp | 2 -- interface/src/avatar/Head.cpp | 1 - interface/src/avatar/SkeletonModel.cpp | 1 - libraries/audio-client/src/AudioClient.cpp | 7 +------ libraries/recording/src/recording/impl/BufferClip.h | 3 ++- libraries/shared/src/shared/JSONHelpers.cpp | 1 - 7 files changed, 3 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/entities/EntityServer.cpp b/assignment-client/src/entities/EntityServer.cpp index 5754a9e057..2fafaa6731 100644 --- a/assignment-client/src/entities/EntityServer.cpp +++ b/assignment-client/src/entities/EntityServer.cpp @@ -112,7 +112,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN quint64 deletePacketSentAt = usecTimestampNow(); EntityTreePointer tree = std::static_pointer_cast(_tree); auto recentlyDeleted = tree->getRecentlyDeletedEntityIDs(); - bool hasMoreToSend = true; packetsSent = 0; diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index b97afad85f..ae1ac74306 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -135,8 +135,6 @@ void MessagesMixer::sendStatsPacket() { void MessagesMixer::run() { ThreadedAssignment::commonInit(MESSAGES_MIXER_LOGGING_NAME, NodeType::MessagesMixer); - NodeType_t owningNodeType = DependencyManager::get()->getOwnerType(); - auto nodeList = DependencyManager::get(); nodeList->addNodeTypeToInterestSet(NodeType::Agent); diff --git a/interface/src/avatar/Head.cpp b/interface/src/avatar/Head.cpp index b8cf8ab4f1..e8452583fc 100644 --- a/interface/src/avatar/Head.cpp +++ b/interface/src/avatar/Head.cpp @@ -91,7 +91,6 @@ void Head::simulate(float deltaTime, bool isMine, bool billboard) { } if (isMine) { - MyAvatar* myAvatar = static_cast(_owningAvatar); auto player = DependencyManager::get(); // Only use face trackers when not playing back a recording. if (!player->isPlaying()) { diff --git a/interface/src/avatar/SkeletonModel.cpp b/interface/src/avatar/SkeletonModel.cpp index 83c8cdfcf5..87f0e631f2 100644 --- a/interface/src/avatar/SkeletonModel.cpp +++ b/interface/src/avatar/SkeletonModel.cpp @@ -247,7 +247,6 @@ void SkeletonModel::simulate(float deltaTime, bool fullUpdate) { return; // only simulate for own avatar } - MyAvatar* myAvatar = static_cast(_owningAvatar); auto player = DependencyManager::get(); if (player->isPlaying()) { return; diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a506fe217c..50bfd995f2 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -909,13 +909,8 @@ void AudioClient::handleRecordedAudioInput(const QByteArray& audio) { // we don't have an audioPacket yet - set that up now _audioPacket = NLPacket::create(PacketType::MicrophoneAudioWithEcho); } + // FIXME either discard stereo in the recording or record a stereo flag - const int numNetworkBytes = _isStereoInput - ? AudioConstants::NETWORK_FRAME_BYTES_STEREO - : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; - const int numNetworkSamples = _isStereoInput - ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO - : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; auto nodeList = DependencyManager::get(); SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); diff --git a/libraries/recording/src/recording/impl/BufferClip.h b/libraries/recording/src/recording/impl/BufferClip.h index af8a64716b..1ea79f3df2 100644 --- a/libraries/recording/src/recording/impl/BufferClip.h +++ b/libraries/recording/src/recording/impl/BufferClip.h @@ -26,7 +26,8 @@ public: private: virtual FrameConstPointer readFrame(size_t index) const override; QString _name { QUuid().toString() }; - mutable size_t _frameIndex { 0 }; + + //mutable size_t _frameIndex { 0 }; // FIXME - not in use }; } diff --git a/libraries/shared/src/shared/JSONHelpers.cpp b/libraries/shared/src/shared/JSONHelpers.cpp index 52ece73490..c0a8820d95 100644 --- a/libraries/shared/src/shared/JSONHelpers.cpp +++ b/libraries/shared/src/shared/JSONHelpers.cpp @@ -27,7 +27,6 @@ QJsonValue glmToJson(const T& t) { template T glmFromJson(const QJsonValue& json) { - static const T DEFAULT_VALUE = T(); T result; if (json.isArray()) { QJsonArray array = json.toArray(); From 5b9791d8002e79e78707f6d7f10df7721fe8908e Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 11:50:10 -0800 Subject: [PATCH 26/36] add message received signal --- .../src/messages/MessagesMixer.cpp | 11 ------ examples/example/messagesExample.js | 5 +++ libraries/networking/src/DomainHandler.cpp | 2 -- libraries/networking/src/MessagesClient.cpp | 36 ++++++++----------- libraries/networking/src/MessagesClient.h | 3 ++ 5 files changed, 22 insertions(+), 35 deletions(-) diff --git a/assignment-client/src/messages/MessagesMixer.cpp b/assignment-client/src/messages/MessagesMixer.cpp index ae1ac74306..21e3fdc4c5 100644 --- a/assignment-client/src/messages/MessagesMixer.cpp +++ b/assignment-client/src/messages/MessagesMixer.cpp @@ -9,22 +9,13 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#include -#include - #include -#include #include -#include -#include #include #include #include #include -#include -#include -#include #include "MessagesMixer.h" @@ -46,7 +37,6 @@ MessagesMixer::~MessagesMixer() { } void MessagesMixer::nodeKilled(SharedNodePointer killedNode) { - qDebug() << "MessagesMixer::nodeKilled()... node:" << killedNode->getUUID(); // FIXME - remove the node from the subscription maps } @@ -67,7 +57,6 @@ void MessagesMixer::handleMessages(QSharedPointer packetList, Shar auto messageData = packet.read(messageLength); QString message = QString::fromUtf8(messageData); - auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode( diff --git a/examples/example/messagesExample.js b/examples/example/messagesExample.js index 39ee4a3dbe..11827f019f 100644 --- a/examples/example/messagesExample.js +++ b/examples/example/messagesExample.js @@ -35,4 +35,9 @@ Script.update.connect(function (deltaTime) { //print(message); Messages.sendMessage(channel, message); } +}); + + +Messages.messageReceived.connect(function (channel, message) { + print("message received on channel:" + channel + ", message:" + message); }); \ No newline at end of file diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index afb2dde266..f7d26f25c5 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -260,8 +260,6 @@ void DomainHandler::requestDomainSettings() { Assignment::Type assignmentType = Assignment::typeForNodeType(DependencyManager::get()->getOwnerType()); - qCDebug(networking) << "Requesting settings from domain server for assignmentType:" << assignmentType; - auto packet = NLPacket::create(PacketType::DomainSettingsRequest, sizeof(assignmentType), true, false); packet->writePrimitive(assignmentType); diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index 60f1146c92..1f9a8c123c 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -40,31 +40,25 @@ void MessagesClient::init() { } } -bool haveMessagesMixer() { - auto nodeList = DependencyManager::get(); - SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); - - if (!messagesMixer) { - qCWarning(messages_client) << "Could not complete MessagesClient operation " - << "since you are not currently connected to a messages-mixer."; - return false; - } - - return true; -} - void MessagesClient::handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode) { - QByteArray data = packetList->getMessage(); - auto packetType = packetList->getType(); + QByteArray packetData = packetList->getMessage(); + QBuffer packet{ &packetData }; + packet.open(QIODevice::ReadOnly); - if (packetType == PacketType::MessagesData) { - QString message = QString::fromUtf8(data); - qDebug() << "got a messages packet:" << message; - } + quint16 channelLength; + packet.read(reinterpret_cast(&channelLength), sizeof(channelLength)); + auto channelData = packet.read(channelLength); + QString channel = QString::fromUtf8(channelData); + + quint16 messageLength; + packet.read(reinterpret_cast(&messageLength), sizeof(messageLength)); + auto messageData = packet.read(messageLength); + QString message = QString::fromUtf8(messageData); + + emit messageReceived(channel, message); } void MessagesClient::sendMessage(const QString& channel, const QString& message) { - qDebug() << "MessagesClient::sendMessage() channel:" << channel << "message:" << message; auto nodeList = DependencyManager::get(); SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); @@ -89,7 +83,6 @@ void MessagesClient::sendMessage(const QString& channel, const QString& message) // in the event that they mixer goes away and/or comes back we should automatically // resubscribe to those channels void MessagesClient::subscribe(const QString& channel) { - qDebug() << "MessagesClient::subscribe() channel:" << channel; auto nodeList = DependencyManager::get(); SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); @@ -101,7 +94,6 @@ void MessagesClient::subscribe(const QString& channel) { } void MessagesClient::unsubscribe(const QString& channel) { - qDebug() << "MessagesClient::unsubscribe() channel:" << channel; auto nodeList = DependencyManager::get(); SharedNodePointer messagesMixer = nodeList->soloNodeOfType(NodeType::MessagesMixer); diff --git a/libraries/networking/src/MessagesClient.h b/libraries/networking/src/MessagesClient.h index a79b855be9..13e908e129 100644 --- a/libraries/networking/src/MessagesClient.h +++ b/libraries/networking/src/MessagesClient.h @@ -32,6 +32,9 @@ public: Q_INVOKABLE void subscribe(const QString& channel); Q_INVOKABLE void unsubscribe(const QString& channel); +signals: + void messageReceived(const QString& channel, const QString& message); + private slots: void handleMessagesPacket(QSharedPointer packetList, SharedNodePointer senderNode); void handleNodeKilled(SharedNodePointer node); From 46c8d7b3f84d51f114cc87bdd7141ffca03929a3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 17 Nov 2015 12:32:45 -0800 Subject: [PATCH 27/36] fix for release build undeclared identifier --- plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp b/plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp index 3898d586ad..ddf251778f 100644 --- a/plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp +++ b/plugins/oculusLegacy/src/OculusLegacyDisplayPlugin.cpp @@ -167,10 +167,7 @@ void OculusLegacyDisplayPlugin::activate() { } }); - #ifndef QT_NO_DEBUG - ovrBool result = - #endif - ovrHmd_ConfigureRendering(_hmd, &config.Config, distortionCaps, _eyeFovs, _eyeRenderDescs); + ovrBool result = ovrHmd_ConfigureRendering(_hmd, &config.Config, distortionCaps, _eyeFovs, _eyeRenderDescs); Q_ASSERT(result); } From 32bf81ef0db6d46dfcb79d1d782fc23997f70dbe Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 12:59:12 -0800 Subject: [PATCH 28/36] move MessagesClient to agent --- assignment-client/src/Agent.cpp | 20 +++++++++++--------- assignment-client/src/Agent.h | 1 - interface/src/Application.cpp | 1 - libraries/script-engine/src/ScriptEngine.cpp | 5 +++++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 0d719d6806..1f56118177 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -55,7 +56,6 @@ Agent::Agent(NLPacket& packet) : { PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase }, this, "handleOctreePacket"); packetReceiver.registerListener(PacketType::Jurisdiction, this, "handleJurisdictionPacket"); - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagePacket"); } void Agent::handleOctreePacket(QSharedPointer packet, SharedNodePointer senderNode) { @@ -96,14 +96,6 @@ void Agent::handleJurisdictionPacket(QSharedPointer packet, SharedNode } } -void Agent::handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode) { - auto packetType = packet->getType(); - - if (packetType == PacketType::MessagesData) { - qDebug() << "got a messages packet"; - } -} - void Agent::handleAudioPacket(QSharedPointer packet) { _receivedAudioStream.parseData(*packet); @@ -118,11 +110,21 @@ const int PING_INTERVAL = 1000; void Agent::run() { ThreadedAssignment::commonInit(AGENT_LOGGING_NAME, NodeType::Agent); + // Setup MessagesClient + auto messagesClient = DependencyManager::set(); + QThread* messagesThread = new QThread; + messagesThread->setObjectName("Messages Client Thread"); + messagesClient->moveToThread(messagesThread); + connect(messagesThread, &QThread::started, messagesClient.data(), &MessagesClient::init); + messagesThread->start(); + + auto nodeList = DependencyManager::get(); nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer << NodeType::EntityServer + << NodeType::MessagesMixer ); _pingTimer = new QTimer(this); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index be3a0db293..ab000015d5 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -58,7 +58,6 @@ private slots: void handleAudioPacket(QSharedPointer packet); void handleOctreePacket(QSharedPointer packet, SharedNodePointer senderNode); void handleJurisdictionPacket(QSharedPointer packet, SharedNodePointer senderNode); - void handleMessagesPacket(QSharedPointer packet, SharedNodePointer senderNode); void sendPingRequests(); void processAgentAvatarAndAudio(float deltaTime); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 712fb7dc02..c77bb9a114 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -4029,7 +4029,6 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEngine* scri scriptEngine->registerFunction("HMD", "getHUDLookAtPosition3D", HMDScriptingInterface::getHUDLookAtPosition3D, 0); scriptEngine->registerGlobalObject("Scene", DependencyManager::get().data()); - scriptEngine->registerGlobalObject("Messages", DependencyManager::get().data()); scriptEngine->registerGlobalObject("ScriptDiscoveryService", this->getRunningScriptsWidget()); diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 0f62bf8cd5..611be863c2 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -375,6 +376,10 @@ void ScriptEngine::init() { auto scriptingInterface = DependencyManager::get(); registerGlobalObject("Controller", scriptingInterface.data()); UserInputMapper::registerControllerTypes(this); + + + registerGlobalObject("Messages", DependencyManager::get().data()); + } void ScriptEngine::registerValue(const QString& valueName, QScriptValue value) { From 2d3fe497e46c6dc7502543040ed18bfd93f1c112 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 13:29:28 -0800 Subject: [PATCH 29/36] fix typo --- libraries/networking/src/MessagesClient.cpp | 6 +----- libraries/networking/src/PacketReceiver.cpp | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/MessagesClient.cpp b/libraries/networking/src/MessagesClient.cpp index 1f9a8c123c..ac2bf55033 100644 --- a/libraries/networking/src/MessagesClient.cpp +++ b/libraries/networking/src/MessagesClient.cpp @@ -21,16 +21,12 @@ #include "PacketReceiver.h" MessagesClient::MessagesClient() { - setCustomDeleter([](Dependency* dependency){ static_cast(dependency)->deleteLater(); }); - auto nodeList = DependencyManager::get(); auto& packetReceiver = nodeList->getPacketReceiver(); - - packetReceiver.registerListener(PacketType::MessagesData, this, "handleMessagePacket"); - + packetReceiver.registerMessageListener(PacketType::MessagesData, this, "handleMessagesPacket"); connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, &MessagesClient::handleNodeKilled); } diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index 0a3ea86399..07f25fee5f 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -97,7 +97,7 @@ void PacketReceiver::registerDirectListenerForTypes(PacketTypeList types, bool PacketReceiver::registerMessageListener(PacketType type, QObject* listener, const char* slot) { Q_ASSERT_X(listener, "PacketReceiver::registerMessageListener", "No object to register"); Q_ASSERT_X(slot, "PacketReceiver::registerMessageListener", "No slot to register"); - + QMetaMethod matchingMethod = matchingMethodForListener(type, listener, slot); if (matchingMethod.isValid()) { From 2f142eb0887e2ff6315171b6a66fa3b33c711ce1 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 13:33:30 -0800 Subject: [PATCH 30/36] add a receiver example --- examples/example/messagesReceiverExample.js | 22 ++++++++++++++++++++ libraries/script-engine/src/ScriptEngine.cpp | 5 +---- 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 examples/example/messagesReceiverExample.js diff --git a/examples/example/messagesReceiverExample.js b/examples/example/messagesReceiverExample.js new file mode 100644 index 0000000000..31020a4c8a --- /dev/null +++ b/examples/example/messagesReceiverExample.js @@ -0,0 +1,22 @@ +var totalTime = 0; +var subscribed = false; +var WAIT_FOR_SUBSCRIPTION_TIME = 10; +function myUpdate(deltaTime) { + var channel = "example"; + totalTime += deltaTime; + + if (totalTime > WAIT_FOR_SUBSCRIPTION_TIME && !subscribed) { + + print("---- subscribing ----"); + Messages.subscribe(channel); + subscribed = true; + Script.update.disconnect(myUpdate); + } +} + +Script.update.connect(myUpdate); + +Messages.messageReceived.connect(function (channel, message) { + print("message received on channel:" + channel + ", message:" + message); +}); + diff --git a/libraries/script-engine/src/ScriptEngine.cpp b/libraries/script-engine/src/ScriptEngine.cpp index 611be863c2..c17b091643 100644 --- a/libraries/script-engine/src/ScriptEngine.cpp +++ b/libraries/script-engine/src/ScriptEngine.cpp @@ -367,6 +367,7 @@ void ScriptEngine::init() { registerGlobalObject("Vec3", &_vec3Library); registerGlobalObject("Uuid", &_uuidLibrary); registerGlobalObject("AnimationCache", DependencyManager::get().data()); + registerGlobalObject("Messages", DependencyManager::get().data()); qScriptRegisterMetaType(this, animVarMapToScriptValue, animVarMapFromScriptValue); qScriptRegisterMetaType(this, resultHandlerToScriptValue, resultHandlerFromScriptValue); @@ -376,10 +377,6 @@ void ScriptEngine::init() { auto scriptingInterface = DependencyManager::get(); registerGlobalObject("Controller", scriptingInterface.data()); UserInputMapper::registerControllerTypes(this); - - - registerGlobalObject("Messages", DependencyManager::get().data()); - } void ScriptEngine::registerValue(const QString& valueName, QScriptValue value) { From f54374b66abb8d07890ccf0b3eb62d3b99e99234 Mon Sep 17 00:00:00 2001 From: Brad Hefta-Gaub Date: Tue, 17 Nov 2015 13:40:04 -0800 Subject: [PATCH 31/36] remove unused settings --- domain-server/resources/describe-settings.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index c2410a1d0c..e0038117f0 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -549,22 +549,6 @@ "advanced": true } ] - }, - { - "name": "messages_mixer", - "label": "Messages Mixer", - "assignment-types": [4], - "settings": [ - { - "name": "unused", - "type": "double", - "label": "Unused setting", - "help": "an asofyet unused setting", - "placeholder": 1.0, - "default": 1.0, - "advanced": true - } - ] } ] } From e93b5c5838614e37a67fd9dceb9739954217e7c0 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Tue, 17 Nov 2015 14:02:27 -0800 Subject: [PATCH 32/36] Bug fixes for avatars with no eyes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed default eye position to 1.9 meters because the hifi_team avatars are 2.0 meters tall. Also, prevent array access with negative indices when eye bones are missing. ಠ_ಠ --- interface/src/avatar/MyAvatar.cpp | 68 ++++++++++++++++++++++++ libraries/animation/src/AnimSkeleton.cpp | 2 +- libraries/animation/src/Rig.cpp | 34 +++++++----- 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/interface/src/avatar/MyAvatar.cpp b/interface/src/avatar/MyAvatar.cpp index 38eb5042f7..2eb005fc1c 100644 --- a/interface/src/avatar/MyAvatar.cpp +++ b/interface/src/avatar/MyAvatar.cpp @@ -379,6 +379,13 @@ void MyAvatar::updateHMDFollowVelocity() { // update sensor to world matrix from current body position and hmd sensor. // This is so the correct camera can be used for rendering. void MyAvatar::updateSensorToWorldMatrix() { + +#ifdef DEBUG_RENDERING + // draw marker about avatar's position + const glm::vec4 red(1.0f, 0.0f, 0.0f, 1.0f); + DebugDraw::getInstance().addMyAvatarMarker("pos", glm::quat(), glm::vec3(), red); +#endif + // update the sensor mat so that the body position will end up in the desired // position when driven from the head. glm::mat4 desiredMat = createMatFromQuatAndPos(getOrientation(), getPosition()); @@ -1859,6 +1866,7 @@ glm::quat MyAvatar::getWorldBodyOrientation() const { return glm::quat_cast(_sensorToWorldMatrix * _bodySensorMatrix); } +#if 0 // derive avatar body position and orientation from the current HMD Sensor location. // results are in sensor space glm::mat4 MyAvatar::deriveBodyFromHMDSensor() const { @@ -1876,6 +1884,66 @@ glm::mat4 MyAvatar::deriveBodyFromHMDSensor() const { } return glm::mat4(); } +#else +// old school meat hook style +glm::mat4 MyAvatar::deriveBodyFromHMDSensor() const { + + // HMD is in sensor space. + const glm::vec3 hmdPosition = getHMDSensorPosition(); + const glm::quat hmdOrientation = getHMDSensorOrientation(); + const glm::quat hmdOrientationYawOnly = cancelOutRollAndPitch(hmdOrientation); + + /* + const glm::vec3 DEFAULT_RIGHT_EYE_POS(-0.3f, 1.6f, 0.0f); + const glm::vec3 DEFAULT_LEFT_EYE_POS(0.3f, 1.6f, 0.0f); + const glm::vec3 DEFAULT_NECK_POS(0.0f, 1.5f, 0.0f); + const glm::vec3 DEFAULT_HIPS_POS(0.0f, 1.0f, 0.0f); + */ + + // 2 meter tall dude + const glm::vec3 DEFAULT_RIGHT_EYE_POS(-0.3f, 1.9f, 0.0f); + const glm::vec3 DEFAULT_LEFT_EYE_POS(0.3f, 1.9f, 0.0f); + const glm::vec3 DEFAULT_NECK_POS(0.0f, 1.70f, 0.0f); + const glm::vec3 DEFAULT_HIPS_POS(0.0f, 1.05f, 0.0f); + + vec3 localEyes, localNeck; + if (!_debugDrawSkeleton) { + const glm::quat rotY180 = glm::angleAxis((float)PI, glm::vec3(0.0f, 1.0f, 0.0f)); + localEyes = rotY180 * (((DEFAULT_RIGHT_EYE_POS + DEFAULT_LEFT_EYE_POS) / 2.0f) - DEFAULT_HIPS_POS); + localNeck = rotY180 * (DEFAULT_NECK_POS - DEFAULT_HIPS_POS); + } else { + // TODO: At the moment MyAvatar does not have access to the rig, which has the skeleton, which has the bind poses. + // for now use the _debugDrawSkeleton, which is initialized with the same FBX model as the rig. + + // TODO: cache these indices. + int rightEyeIndex = _debugDrawSkeleton->nameToJointIndex("RightEye"); + int leftEyeIndex = _debugDrawSkeleton->nameToJointIndex("LeftEye"); + int neckIndex = _debugDrawSkeleton->nameToJointIndex("Neck"); + int hipsIndex = _debugDrawSkeleton->nameToJointIndex("Hips"); + + glm::vec3 absRightEyePos = rightEyeIndex != -1 ? _debugDrawSkeleton->getAbsoluteBindPose(rightEyeIndex).trans : DEFAULT_RIGHT_EYE_POS; + glm::vec3 absLeftEyePos = leftEyeIndex != -1 ? _debugDrawSkeleton->getAbsoluteBindPose(leftEyeIndex).trans : DEFAULT_LEFT_EYE_POS; + glm::vec3 absNeckPos = neckIndex != -1 ? _debugDrawSkeleton->getAbsoluteBindPose(neckIndex).trans : DEFAULT_NECK_POS; + glm::vec3 absHipsPos = neckIndex != -1 ? _debugDrawSkeleton->getAbsoluteBindPose(hipsIndex).trans : DEFAULT_HIPS_POS; + + const glm::quat rotY180 = glm::angleAxis((float)PI, glm::vec3(0.0f, 1.0f, 0.0f)); + localEyes = rotY180 * (((absRightEyePos + absLeftEyePos) / 2.0f) - absHipsPos); + localNeck = rotY180 * (absNeckPos - absHipsPos); + } + + // apply simplistic head/neck model + // figure out where the avatar body should be by applying offsets from the avatar's neck & head joints. + + // eyeToNeck offset is relative full HMD orientation. + // while neckToRoot offset is only relative to HMDs yaw. + glm::vec3 eyeToNeck = hmdOrientation * (localNeck - localEyes); + glm::vec3 neckToRoot = hmdOrientationYawOnly * -localNeck; + glm::vec3 bodyPos = hmdPosition + eyeToNeck + neckToRoot; + + // avatar facing is determined solely by hmd orientation. + return createMatFromQuatAndPos(hmdOrientationYawOnly, bodyPos); +} +#endif glm::vec3 MyAvatar::getPositionForAudio() { switch (_audioListenerMode) { diff --git a/libraries/animation/src/AnimSkeleton.cpp b/libraries/animation/src/AnimSkeleton.cpp index 0db7473c9c..7ec8db1490 100644 --- a/libraries/animation/src/AnimSkeleton.cpp +++ b/libraries/animation/src/AnimSkeleton.cpp @@ -79,7 +79,7 @@ const QString& AnimSkeleton::getJointName(int jointIndex) const { } AnimPose AnimSkeleton::getAbsolutePose(int jointIndex, const AnimPoseVec& poses) const { - if (jointIndex < 0) { + if (jointIndex < 0 || jointIndex >= (int)poses.size() || jointIndex >= (int)_joints.size()) { return AnimPose::identity; } else { return getAbsolutePose(_joints[jointIndex].parentIndex, poses) * poses[jointIndex]; diff --git a/libraries/animation/src/Rig.cpp b/libraries/animation/src/Rig.cpp index 9b6221a370..90f068d1ef 100644 --- a/libraries/animation/src/Rig.cpp +++ b/libraries/animation/src/Rig.cpp @@ -23,6 +23,19 @@ #include "AnimSkeleton.h" #include "IKTarget.h" +/* +const glm::vec3 DEFAULT_RIGHT_EYE_POS(-0.3f, 1.6f, 0.0f); +const glm::vec3 DEFAULT_LEFT_EYE_POS(0.3f, 1.6f, 0.0f); +const glm::vec3 DEFAULT_HEAD_POS(0.0f, 1.55f, 0.0f); +const glm::vec3 DEFAULT_NECK_POS(0.0f, 1.5f, 0.0f); +*/ + +// 2 meter tall dude +const glm::vec3 DEFAULT_RIGHT_EYE_POS(-0.3f, 1.9f, 0.0f); +const glm::vec3 DEFAULT_LEFT_EYE_POS(0.3f, 1.9f, 0.0f); +const glm::vec3 DEFAULT_HEAD_POS(0.0f, 1.75f, 0.0f); +const glm::vec3 DEFAULT_NECK_POS(0.0f, 1.70f, 0.0f); + void insertSorted(QList& handles, const AnimationHandlePointer& handle) { for (QList::iterator it = handles.begin(); it != handles.end(); it++) { if (handle->getPriority() > (*it)->getPriority()) { @@ -410,17 +423,19 @@ void Rig::calcAnimAlpha(float speed, const std::vector& referenceSpeeds, void Rig::computeEyesInRootFrame(const AnimPoseVec& poses) { // TODO: use cached eye/hips indices for these calculations int numPoses = poses.size(); - int rightEyeIndex = _animSkeleton->nameToJointIndex(QString("RightEye")); - int leftEyeIndex = _animSkeleton->nameToJointIndex(QString("LeftEye")); - if (numPoses > rightEyeIndex && numPoses > leftEyeIndex - && rightEyeIndex > 0 && leftEyeIndex > 0) { - int hipsIndex = _animSkeleton->nameToJointIndex(QString("Hips")); - int headIndex = _animSkeleton->nameToJointIndex(QString("Head")); - if (hipsIndex >= 0 && headIndex > 0) { + int hipsIndex = _animSkeleton->nameToJointIndex(QString("Hips")); + int headIndex = _animSkeleton->nameToJointIndex(QString("Head")); + if (hipsIndex > 0 && headIndex > 0) { + int rightEyeIndex = _animSkeleton->nameToJointIndex(QString("RightEye")); + int leftEyeIndex = _animSkeleton->nameToJointIndex(QString("LeftEye")); + if (numPoses > rightEyeIndex && numPoses > leftEyeIndex && rightEyeIndex > 0 && leftEyeIndex > 0) { glm::vec3 rightEye = _animSkeleton->getAbsolutePose(rightEyeIndex, poses).trans; glm::vec3 leftEye = _animSkeleton->getAbsolutePose(leftEyeIndex, poses).trans; glm::vec3 hips = _animSkeleton->getAbsolutePose(hipsIndex, poses).trans; _eyesInRootFrame = 0.5f * (rightEye + leftEye) - hips; + } else { + glm::vec3 hips = _animSkeleton->getAbsolutePose(hipsIndex, poses).trans; + _eyesInRootFrame = 0.5f * (DEFAULT_RIGHT_EYE_POS + DEFAULT_LEFT_EYE_POS) - hips; } } } @@ -1172,11 +1187,6 @@ static void computeHeadNeckAnimVars(AnimSkeleton::ConstPointer skeleton, const A int headIndex = skeleton->nameToJointIndex("Head"); int neckIndex = skeleton->nameToJointIndex("Neck"); - const glm::vec3 DEFAULT_RIGHT_EYE_POS(-0.3f, 1.6f, 0.0f); - const glm::vec3 DEFAULT_LEFT_EYE_POS(0.3f, 1.6f, 0.0f); - const glm::vec3 DEFAULT_HEAD_POS(0.0f, 1.55f, 0.0f); - const glm::vec3 DEFAULT_NECK_POS(0.0f, 1.5f, 0.0f); - // Use absolute bindPose positions just in case the relBindPose have rotations we don't expect. glm::vec3 absRightEyePos = rightEyeIndex != -1 ? skeleton->getAbsoluteBindPose(rightEyeIndex).trans : DEFAULT_RIGHT_EYE_POS; glm::vec3 absLeftEyePos = leftEyeIndex != -1 ? skeleton->getAbsoluteBindPose(leftEyeIndex).trans : DEFAULT_LEFT_EYE_POS; From 2fe9af54d4c579d64c850a7d991e746dfb8a279f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 18 Nov 2015 13:59:32 -0800 Subject: [PATCH 33/36] Add textures property to polyline --- .../src/RenderablePolyLineEntityItem.cpp | 17 ++++++++++++----- .../src/RenderablePolyLineEntityItem.h | 8 ++++++-- libraries/entities/src/PolyLineEntityItem.cpp | 8 +++++++- libraries/entities/src/PolyLineEntityItem.h | 11 ++++++++++- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/libraries/entities-renderer/src/RenderablePolyLineEntityItem.cpp b/libraries/entities-renderer/src/RenderablePolyLineEntityItem.cpp index 32418199b8..76cf4fac3d 100644 --- a/libraries/entities-renderer/src/RenderablePolyLineEntityItem.cpp +++ b/libraries/entities-renderer/src/RenderablePolyLineEntityItem.cpp @@ -36,7 +36,6 @@ PolyLineEntityItem(entityItemID, properties) { gpu::PipelinePointer RenderablePolyLineEntityItem::_pipeline; gpu::Stream::FormatPointer RenderablePolyLineEntityItem::_format; -gpu::TexturePointer RenderablePolyLineEntityItem::_texture; int32_t RenderablePolyLineEntityItem::PAINTSTROKE_GPU_SLOT; void RenderablePolyLineEntityItem::createPipeline() { @@ -44,9 +43,6 @@ void RenderablePolyLineEntityItem::createPipeline() { static const int COLOR_OFFSET = 24; static const int TEXTURE_OFFSET = 28; - auto textureCache = DependencyManager::get(); - QString path = PathUtils::resourcesPath() + "images/paintStroke.png"; - _texture = textureCache->getImageTexture(path); _format.reset(new gpu::Stream::Format()); _format->setAttribute(gpu::Stream::POSITION, 0, gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ), 0); _format->setAttribute(gpu::Stream::NORMAL, 0, gpu::Element(gpu::VEC3, gpu::FLOAT, gpu::XYZ), NORMAL_OFFSET); @@ -132,6 +128,13 @@ void RenderablePolyLineEntityItem::render(RenderArgs* args) { createPipeline(); } + if (!_texture || _texturesChangedFlag) { + auto textureCache = DependencyManager::get(); + QString path = _textures.isEmpty() ? PathUtils::resourcesPath() + "images/paintStroke.png" : _textures; + _texture = textureCache->getTexture(QUrl(path)); + _texturesChangedFlag = false; + } + PerformanceTimer perfTimer("RenderablePolyLineEntityItem::render"); Q_ASSERT(getType() == EntityTypes::PolyLine); @@ -147,7 +150,11 @@ void RenderablePolyLineEntityItem::render(RenderArgs* args) { batch.setModelTransform(transform); batch.setPipeline(_pipeline); - batch.setResourceTexture(PAINTSTROKE_GPU_SLOT, _texture); + if (_texture->isLoaded()) { + batch.setResourceTexture(PAINTSTROKE_GPU_SLOT, _texture->getGPUTexture()); + } else { + batch.setResourceTexture(PAINTSTROKE_GPU_SLOT, args->_whiteTexture); + } batch.setInputFormat(_format); batch.setInputBuffer(0, _verticesBuffer, 0, _format->getChannels().at(0)._stride); diff --git a/libraries/entities-renderer/src/RenderablePolyLineEntityItem.h b/libraries/entities-renderer/src/RenderablePolyLineEntityItem.h index c8a47cce0c..618f8c66a6 100644 --- a/libraries/entities-renderer/src/RenderablePolyLineEntityItem.h +++ b/libraries/entities-renderer/src/RenderablePolyLineEntityItem.h @@ -12,10 +12,13 @@ #ifndef hifi_RenderablePolyLineEntityItem_h #define hifi_RenderablePolyLineEntityItem_h + #include +#include #include #include "RenderableEntityItem.h" -#include +#include + #include @@ -29,9 +32,10 @@ public: SIMPLE_RENDERABLE(); + NetworkTexturePointer _texture; + static gpu::PipelinePointer _pipeline; static gpu::Stream::FormatPointer _format; - static gpu::TexturePointer _texture; static int32_t PAINTSTROKE_GPU_SLOT; protected: diff --git a/libraries/entities/src/PolyLineEntityItem.cpp b/libraries/entities/src/PolyLineEntityItem.cpp index f1be431ce8..45c967f78d 100644 --- a/libraries/entities/src/PolyLineEntityItem.cpp +++ b/libraries/entities/src/PolyLineEntityItem.cpp @@ -37,7 +37,8 @@ _pointsChanged(true), _points(QVector(0.0f)), _vertices(QVector(0.0f)), _normals(QVector(0.0f)), -_strokeWidths(QVector(0.0f)) +_strokeWidths(QVector(0.0f)), +_textures("") { _type = EntityTypes::PolyLine; _created = properties.getCreated(); @@ -56,6 +57,7 @@ EntityItemProperties PolyLineEntityItem::getProperties(EntityPropertyFlags desir COPY_ENTITY_PROPERTY_TO_PROPERTIES(linePoints, getLinePoints); COPY_ENTITY_PROPERTY_TO_PROPERTIES(normals, getNormals); COPY_ENTITY_PROPERTY_TO_PROPERTIES(strokeWidths, getStrokeWidths); + COPY_ENTITY_PROPERTY_TO_PROPERTIES(textures, getTextures); properties._glowLevel = getGlowLevel(); properties._glowLevelChanged = false; @@ -72,6 +74,7 @@ bool PolyLineEntityItem::setProperties(const EntityItemProperties& properties) { SET_ENTITY_PROPERTY_FROM_PROPERTIES(linePoints, setLinePoints); SET_ENTITY_PROPERTY_FROM_PROPERTIES(normals, setNormals); SET_ENTITY_PROPERTY_FROM_PROPERTIES(strokeWidths, setStrokeWidths); + SET_ENTITY_PROPERTY_FROM_PROPERTIES(textures, setTextures); if (somethingChanged) { bool wantDebug = false; @@ -196,6 +199,7 @@ int PolyLineEntityItem::readEntitySubclassDataFromBuffer(const unsigned char* da READ_ENTITY_PROPERTY(PROP_LINE_POINTS, QVector, setLinePoints); READ_ENTITY_PROPERTY(PROP_NORMALS, QVector, setNormals); READ_ENTITY_PROPERTY(PROP_STROKE_WIDTHS, QVector, setStrokeWidths); + READ_ENTITY_PROPERTY(PROP_TEXTURES, QString, setTextures); return bytesRead; } @@ -209,6 +213,7 @@ EntityPropertyFlags PolyLineEntityItem::getEntityProperties(EncodeBitstreamParam requestedProperties += PROP_LINE_POINTS; requestedProperties += PROP_NORMALS; requestedProperties += PROP_STROKE_WIDTHS; + requestedProperties += PROP_TEXTURES; return requestedProperties; } @@ -228,6 +233,7 @@ void PolyLineEntityItem::appendSubclassData(OctreePacketData* packetData, Encode APPEND_ENTITY_PROPERTY(PROP_LINE_POINTS, getLinePoints()); APPEND_ENTITY_PROPERTY(PROP_NORMALS, getNormals()); APPEND_ENTITY_PROPERTY(PROP_STROKE_WIDTHS, getStrokeWidths()); + APPEND_ENTITY_PROPERTY(PROP_TEXTURES, getTextures()); } void PolyLineEntityItem::debugDump() const { diff --git a/libraries/entities/src/PolyLineEntityItem.h b/libraries/entities/src/PolyLineEntityItem.h index 86a1dfb6e0..9e9d3f9013 100644 --- a/libraries/entities/src/PolyLineEntityItem.h +++ b/libraries/entities/src/PolyLineEntityItem.h @@ -67,7 +67,14 @@ class PolyLineEntityItem : public EntityItem { bool setStrokeWidths(const QVector& strokeWidths); const QVector& getStrokeWidths() const{ return _strokeWidths; } - + + const QString& getTextures() const { return _textures; } + void setTextures(const QString& textures) { + if (_textures != textures) { + _textures = textures; + _texturesChangedFlag = true; + } + } virtual ShapeType getShapeType() const { return SHAPE_TYPE_LINE; } @@ -90,6 +97,8 @@ class PolyLineEntityItem : public EntityItem { QVector _vertices; QVector _normals; QVector _strokeWidths; + QString _textures; + bool _texturesChangedFlag { false }; mutable QReadWriteLock _quadReadWriteLock; }; From 3d0afc3c5ff0910d2b92b5ee9843d844d4315696 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Wed, 18 Nov 2015 14:00:00 -0800 Subject: [PATCH 34/36] Update entities version number for polyline change --- libraries/networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f5c66617a8..2411ee23ac 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -41,7 +41,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::EntityAdd: case PacketType::EntityEdit: case PacketType::EntityData: - return VERSION_ENTITIES_PARTICLES_ADDITIVE_BLENDING; + return VERSION_ENTITIES_POLYLINE_TEXTURE; case PacketType::AvatarData: case PacketType::BulkAvatarData: default: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index e0a847dcc6..23df89b3d7 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -159,5 +159,6 @@ const PacketVersion VERSION_ENTITIES_ANIMATION_PROPERTIES_GROUP = 46; const PacketVersion VERSION_ENTITIES_KEYLIGHT_PROPERTIES_GROUP = 47; const PacketVersion VERSION_ENTITIES_KEYLIGHT_PROPERTIES_GROUP_BIS = 48; const PacketVersion VERSION_ENTITIES_PARTICLES_ADDITIVE_BLENDING = 49; +const PacketVersion VERSION_ENTITIES_POLYLINE_TEXTURE = 50; #endif // hifi_PacketHeaders_h From b122dad7eab3b28936409c6ef2315eefbfdb6609 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 19 Nov 2015 15:16:23 -0800 Subject: [PATCH 35/36] Add begin and middle textures to InfiniteLine --- examples/libraries/line.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/libraries/line.js b/examples/libraries/line.js index c21bf2f3ad..d5e5c83028 100644 --- a/examples/libraries/line.js +++ b/examples/libraries/line.js @@ -19,10 +19,11 @@ var MAX_LINE_LENGTH = 40; // This must be 2 or greater; var DEFAULT_STROKE_WIDTH = 0.1; var DEFAULT_LIFETIME = 20; var DEFAULT_COLOR = { red: 255, green: 255, blue: 255 }; -var PolyLine = function(position, color, lifetime) { +var PolyLine = function(position, color, lifetime, texture) { this.position = position; this.color = color; this.lifetime = lifetime === undefined ? DEFAULT_LIFETIME : lifetime; + this.texture = texture ? texture : ""; this.points = [ ]; this.strokeWidths = [ @@ -37,7 +38,8 @@ var PolyLine = function(position, color, lifetime) { strokeWidths: this.strokeWidths, dimensions: LINE_DIMENSIONS, color: color, - lifetime: lifetime + lifetime: lifetime, + textures: this.texture }); }; @@ -98,26 +100,29 @@ PolyLine.prototype.destroy = function() { // InfiniteLine -InfiniteLine = function(position, color, lifetime) { +InfiniteLine = function(position, color, lifetime, textureBegin, textureMiddle) { this.position = position; this.color = color; this.lifetime = lifetime === undefined ? DEFAULT_LIFETIME : lifetime; this.lines = []; this.size = 0; + + this.textureBegin = textureBegin ? textureBegin : ""; + this.textureMiddle = textureMiddle ? textureMiddle : ""; }; InfiniteLine.prototype.enqueuePoint = function(position, strokeWidth) { var currentLine; if (this.lines.length == 0) { - currentLine = new PolyLine(position, this.color, this.lifetime); + currentLine = new PolyLine(position, this.color, this.lifetime, this.textureBegin); this.lines.push(currentLine); } else { currentLine = this.lines[this.lines.length - 1]; } if (currentLine.isFull()) { - var newLine = new PolyLine(currentLine.getLastPoint(), this.color, this.lifetime); + var newLine = new PolyLine(currentLine.getLastPoint(), this.color, this.lifetime, this.textureMiddle); newLine.enqueuePoint(currentLine.getLastPoint(), strokeWidth); this.lines.push(newLine); currentLine = newLine; From 9e31614fce4379adf4e5675d54a7e5c40af174fc Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 19 Nov 2015 15:16:46 -0800 Subject: [PATCH 36/36] Fix textures property not being sent for polyline --- libraries/entities/src/EntityItemProperties.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/entities/src/EntityItemProperties.cpp b/libraries/entities/src/EntityItemProperties.cpp index 4f5d256969..78a4f3e8b6 100644 --- a/libraries/entities/src/EntityItemProperties.cpp +++ b/libraries/entities/src/EntityItemProperties.cpp @@ -432,6 +432,7 @@ QScriptValue EntityItemProperties::copyToScriptValue(QScriptEngine* engine, bool COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_LINE_POINTS, linePoints); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_NORMALS, normals); COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_STROKE_WIDTHS, strokeWidths); + COPY_PROPERTY_TO_QSCRIPTVALUE(PROP_TEXTURES, textures); } // Sitting properties support @@ -1011,6 +1012,7 @@ bool EntityItemProperties::encodeEntityEditPacket(PacketType command, EntityItem APPEND_ENTITY_PROPERTY(PROP_LINE_POINTS, properties.getLinePoints()); APPEND_ENTITY_PROPERTY(PROP_NORMALS, properties.getNormals()); APPEND_ENTITY_PROPERTY(PROP_STROKE_WIDTHS, properties.getStrokeWidths()); + APPEND_ENTITY_PROPERTY(PROP_TEXTURES, properties.getTextures()); } APPEND_ENTITY_PROPERTY(PROP_MARKETPLACE_ID, properties.getMarketplaceID()); @@ -1287,6 +1289,7 @@ bool EntityItemProperties::decodeEntityEditPacket(const unsigned char* data, int READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_LINE_POINTS, QVector, setLinePoints); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_NORMALS, QVector, setNormals); READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_STROKE_WIDTHS, QVector, setStrokeWidths); + READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_TEXTURES, QString, setTextures); } READ_ENTITY_PROPERTY_TO_PROPERTIES(PROP_MARKETPLACE_ID, QString, setMarketplaceID);