From 7dbdc9ca0a1d413aff93fcdce106a7a1d401e2d5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Apr 2016 17:04:20 -0700 Subject: [PATCH 1/6] use tar-fs and zlib for content extraction --- server-console/package.json | 2 +- server-console/src/main.js | 57 +++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/server-console/package.json b/server-console/package.json index 44f7b3d05e..ece562c1c4 100644 --- a/server-console/package.json +++ b/server-console/package.json @@ -32,7 +32,7 @@ "os-homedir": "^1.0.1", "request": "2.67.0", "request-progress": "1.0.2", - "unzip": "0.1.11", + "tar-fs": "^1.12.0", "yargs": "^3.30.0" } } diff --git a/server-console/src/main.js b/server-console/src/main.js index 26ec2dd536..29e874f453 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -17,7 +17,8 @@ const path = require('path'); const fs = require('fs-extra'); const Tail = require('always-tail'); const http = require('http'); -const unzip = require('unzip'); +const zlib = require('zlib'); +const tar = require('tar-fs'); const request = require('request'); const progress = require('request-progress'); @@ -90,6 +91,10 @@ function getRootHifiDataDirectory() { } } +function getDomainServerClientResourcesDirectory() { + return path.join(getRootHifiDataDirectory(), '/domain-server'); +} + function getAssignmentClientResourcesDirectory() { return path.join(getRootHifiDataDirectory(), '/assignment-client'); } @@ -477,18 +482,34 @@ function updateTrayMenu(serverState) { const httpStatusPort = 60332; function maybeInstallDefaultContentSet(onComplete) { - // Check for existing AC data + // Check for existing data const acResourceDirectory = getAssignmentClientResourcesDirectory(); + console.log("Checking for existence of " + acResourceDirectory); - var userHasExistingServerData = true; + + var userHasExistingACData = true; try { fs.accessSync(acResourceDirectory); + console.log("Found directory " + acResourceDirectory); } catch (e) { console.log(e); - userHasExistingServerData = false; + userHasExistingACData = false; } - if (userHasExistingServerData) { + const dsResourceDirectory = getDomainServerClientResourcesDirectory(); + + console.log("checking for existence of " + dsResourceDirectory); + + var userHasExistingDSData = true; + try { + fs.accessSync(dsResourceDirectory); + console.log("Found directory " + dsResourceDirectory); + } catch (e) { + console.log(e); + userHasExistingDSData = false; + } + + if (userHasExistingACData || userHasExistingDSData) { console.log("User has existing data, suppressing downloader"); onComplete(); return; @@ -514,16 +535,19 @@ function maybeInstallDefaultContentSet(onComplete) { electron.ipcMain.on('ready', function() { console.log("got ready"); + var currentState = ''; + function sendStateUpdate(state, args) { // console.log(state, window, args); window.webContents.send('update', { state: state, args: args }); + currentState = state; } var aborted = false; // Start downloading content set var req = progress(request.get({ - url: "https://s3.amazonaws.com/hifi-public/homeset/ContentSet-Lounge.zip" + url: "http://cachefly.highfidelity.com/home.tgz" }, function(error, responseMessage, responseData) { if (aborted) { return; @@ -546,24 +570,21 @@ function maybeInstallDefaultContentSet(onComplete) { sendStateUpdate('downloading', state); } }); - var unzipper = unzip.Extract({ - path: acResourceDirectory, - verbose: true - }); - unzipper.on('close', function() { - console.log("Done", arguments); - sendStateUpdate('complete'); - }); - unzipper.on('error', function (err) { - console.log("aborting"); + + req.pipe(zlib.createGunzip()).pipe(tar.extract(getRootHifiDataDirectory())).on('error', function(){ + console.log("Aborting request because gunzip/untar failed"); aborted = true; req.abort(); - console.log("ERROR"); + console.log("ERROR" + err); + sendStateUpdate('error', { message: "Error installing resources." }); + }).on('finish', function(){ + // response and decompression complete, return + console.log("Done", arguments); + sendStateUpdate('complete'); }); - req.pipe(unzipper); window.on('closed', function() { if (currentState == 'downloading') { From db4f1a8acbf4014fe2f0f28f3f4a8d5a70fc69e6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Apr 2016 17:36:14 -0700 Subject: [PATCH 2/6] split gunzip and untar errors --- server-console/src/main.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server-console/src/main.js b/server-console/src/main.js index 29e874f453..20802fac81 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -571,7 +571,7 @@ function maybeInstallDefaultContentSet(onComplete) { } }); - req.pipe(zlib.createGunzip()).pipe(tar.extract(getRootHifiDataDirectory())).on('error', function(){ + function extractError(err) { console.log("Aborting request because gunzip/untar failed"); aborted = true; req.abort(); @@ -580,7 +580,12 @@ function maybeInstallDefaultContentSet(onComplete) { sendStateUpdate('error', { message: "Error installing resources." }); - }).on('finish', function(){ + } + + var gunzip = zlib.createGunzip(); + gunzip.on('error', extractError); + + req.pipe(gunzip).pipe(tar.extract(getRootHifiDataDirectory())).on('error', extractError).on('finish', function(){ // response and decompression complete, return console.log("Done", arguments); sendStateUpdate('complete'); From 149bf0be923ade89d3255d3a725c20cc8362e281 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 11 Apr 2016 17:41:05 -0700 Subject: [PATCH 3/6] allow zlib as much memory as possible for faster speed --- server-console/src/main.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server-console/src/main.js b/server-console/src/main.js index 20802fac81..c7917eaf59 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -582,8 +582,13 @@ function maybeInstallDefaultContentSet(onComplete) { }); } - var gunzip = zlib.createGunzip(); + var gunzip = zlib.createGunzip({ + level: 9 + }); gunzip.on('error', extractError); + gunzip.on('finish', function(){ + console.log("GUNZIP DONE"); + }); req.pipe(gunzip).pipe(tar.extract(getRootHifiDataDirectory())).on('error', extractError).on('finish', function(){ // response and decompression complete, return From 8850ebd1583a076abb52a3b915927dd64602f0a2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 11:40:11 -0700 Subject: [PATCH 4/6] update electron/electron-prebuilt, remove gunzip option --- server-console/package.json | 6 +++--- server-console/packager.js | 2 +- server-console/src/downloader.html | 2 +- server-console/src/main.js | 11 +++-------- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/server-console/package.json b/server-console/package.json index ece562c1c4..d97f72609b 100644 --- a/server-console/package.json +++ b/server-console/package.json @@ -9,8 +9,8 @@ ], "devDependencies": { "electron-compilers": "^1.0.1", - "electron-packager": "^5.2.1", - "electron-prebuilt": "0.35.4" + "electron-packager": "^6.0.2", + "electron-prebuilt": "0.37.5" }, "repository": { "type": "git", @@ -30,7 +30,7 @@ "fs-extra": "^0.26.4", "node-notifier": "^4.4.0", "os-homedir": "^1.0.1", - "request": "2.67.0", + "request": "^2.67.0", "request-progress": "1.0.2", "tar-fs": "^1.12.0", "yargs": "^3.30.0" diff --git a/server-console/packager.js b/server-console/packager.js index 657098d2f4..bc3b8989d2 100644 --- a/server-console/packager.js +++ b/server-console/packager.js @@ -17,7 +17,7 @@ var iconName = argv.production ? "console" : "console-beta"; var options = { dir: __dirname, name: "server-console", - version: "0.35.4", + version: "0.37.5", overwrite: true, prune: true, arch: "x64", diff --git a/server-console/src/downloader.html b/server-console/src/downloader.html index a7ba7bd7b7..07d3392246 100644 --- a/server-console/src/downloader.html +++ b/server-console/src/downloader.html @@ -16,7 +16,7 @@
-

Installing...

+

Extracting...

Just a moment
diff --git a/server-console/src/main.js b/server-console/src/main.js index c7917eaf59..ee91ca01f3 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -561,14 +561,14 @@ function maybeInstallDefaultContentSet(onComplete) { sendStateUpdate('error', { message: message }); - } else { - sendStateUpdate('installing'); } }), { throttle: 250 }).on('progress', function(state) { if (!aborted) { // Update progress popup sendStateUpdate('downloading', state); } + }).on('end', function(){ + sendStateUpdate('installing'); }); function extractError(err) { @@ -582,13 +582,8 @@ function maybeInstallDefaultContentSet(onComplete) { }); } - var gunzip = zlib.createGunzip({ - level: 9 - }); + var gunzip = zlib.createGunzip(); gunzip.on('error', extractError); - gunzip.on('finish', function(){ - console.log("GUNZIP DONE"); - }); req.pipe(gunzip).pipe(tar.extract(getRootHifiDataDirectory())).on('error', extractError).on('finish', function(){ // response and decompression complete, return From c7d184ad6b2c3d98d3dd2b0bbd9cfb3873a1af06 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Apr 2016 12:04:27 -0700 Subject: [PATCH 5/6] update comment for completed untar --- server-console/src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-console/src/main.js b/server-console/src/main.js index ee91ca01f3..6465f6b0a3 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -587,7 +587,7 @@ function maybeInstallDefaultContentSet(onComplete) { req.pipe(gunzip).pipe(tar.extract(getRootHifiDataDirectory())).on('error', extractError).on('finish', function(){ // response and decompression complete, return - console.log("Done", arguments); + console.log("Finished unarchiving home content set"); sendStateUpdate('complete'); }); From d7450b8178394bd2a0244245ce0fe75edb608796 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 13 Apr 2016 14:25:23 -0700 Subject: [PATCH 6/6] release pending ATP script request for active AssetServer --- assignment-client/src/Agent.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index d0433b5d60..ef20dc82e2 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -195,6 +195,7 @@ void Agent::requestScript() { // then wait for the nodeConnected signal to fire off the request auto assetServer = nodeList->soloNodeOfType(NodeType::AssetServer); + if (!assetServer || !assetServer->getActiveSocket()) { qDebug() << "Waiting to connect to Asset Server for ATP script download."; _pendingScriptRequest = request; @@ -209,7 +210,7 @@ void Agent::requestScript() { } void Agent::nodeActivated(SharedNodePointer activatedNode) { - if (_pendingScriptRequest) { + if (_pendingScriptRequest && activatedNode->getType() == NodeType::AssetServer) { qInfo() << "Requesting script at URL" << qPrintable(_pendingScriptRequest->getUrl().toString()); _pendingScriptRequest->send();