From 85046190670fc50dae7ba76659e2a2cab3908a45 Mon Sep 17 00:00:00 2001
From: Brad Davis <bdavis@saintandreas.org>
Date: Mon, 3 Dec 2018 12:54:04 -0800
Subject: [PATCH] Dockerized build, updated build script, ndk18
---
.gitignore | 15 +-
android/Dockerfile | 92 ++++++++++++
android/build.gradle | 4 +-
android/build_android.sh | 4 +
hifi_android.py | 286 ++++++++++++++++++++++++++++++++++++
hifi_singleton.py | 46 ++++++
hifi_utils.py | 124 ++++++++++++++++
hifi_vcpkg.py | 216 ++++++++++++++++++++++++++++
interface/src/Menu.cpp | 2 +-
prebuild.py | 304 +++++++++------------------------------
10 files changed, 848 insertions(+), 245 deletions(-)
create mode 100644 android/Dockerfile
create mode 100755 android/build_android.sh
create mode 100644 hifi_android.py
create mode 100644 hifi_singleton.py
create mode 100644 hifi_utils.py
create mode 100644 hifi_vcpkg.py
diff --git a/.gitignore b/.gitignore
index ef1a7b215e..09b58d71ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,10 +17,11 @@ Makefile
local.properties
android/gradle*
android/.gradle
-android/app/src/main/jniLibs
-android/app/libs
-android/app/src/main/res/values/libs.xml
-android/app/src/main/assets/bundled
+android/**/src/main/jniLibs
+android/**/libs
+android/**/src/main/res/values/libs.xml
+android/**/src/main/assets
+android/**/gradle*
# VSCode
# List taken from Github Global Ignores master@435c4d92
@@ -83,9 +84,6 @@ npm-debug.log
# Android studio files
*___jb_old___
-# Generated assets for Android
-android/app/src/main/assets
-
# Resource binary file
interface/compiledResources
@@ -95,6 +93,9 @@ interface/resources/GPUCache/*
# package lock file for JSDoc tool
tools/jsdoc/package-lock.json
+# Python compile artifacts
+**/__pycache__
+
# ignore unneeded unity project files for avatar exporter
tools/unity-avatar-exporter/Library
tools/unity-avatar-exporter/Packages
diff --git a/android/Dockerfile b/android/Dockerfile
new file mode 100644
index 0000000000..2a6943cbc2
--- /dev/null
+++ b/android/Dockerfile
@@ -0,0 +1,92 @@
+FROM openjdk:8
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN apt-get update && apt-get -y install \
+ curl \
+ gnupg \
+ software-properties-common \
+ unzip \
+ -
+
+# --- Versions and Download paths
+ENV ANDROID_HOME="/usr/local/android-sdk" \
+ ANDROID_NDK_HOME="/usr/local/android-ndk" \
+ ANDROID_SDK_HOME="/usr/local/android-sdk-home" \
+ ANDROID_VERSION=26 \
+ ANDROID_BUILD_TOOLS_VERSION=28.0.3 \
+ ANDROID_NDK_VERSION=r18
+
+ENV SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip" \
+ NDK_URL="https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
+
+# --- Android SDK
+RUN mkdir -p "$ANDROID_HOME" "$ANDROID_SDK_HOME" && \
+ cd "$ANDROID_HOME" && \
+ curl -s -S -o sdk.zip -L "${SDK_URL}" && \
+ unzip sdk.zip && \
+ rm sdk.zip && \
+ yes | $ANDROID_HOME/tools/bin/sdkmanager --licenses
+
+# Install Android Build Tool and Libraries
+RUN $ANDROID_HOME/tools/bin/sdkmanager --update
+RUN $ANDROID_HOME/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" \
+ "platforms;android-${ANDROID_VERSION}" \
+ "platform-tools"
+
+RUN chmod -R a+w "${ANDROID_HOME}"
+RUN chmod -R a+w "${ANDROID_SDK_HOME}"
+
+# --- Android NDK
+# download
+RUN mkdir /usr/local/android-ndk-tmp && \
+ cd /usr/local/android-ndk-tmp && \
+ curl -s -S -o ndk.zip -L "${NDK_URL}" && \
+ unzip -q ndk.zip && \
+ mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \
+ cd ${ANDROID_NDK_HOME} && \
+ rm -rf /usr/local/android-ndk-tmp
+
+ENV PATH ${PATH}:${ANDROID_NDK_HOME}
+
+RUN apt-get -y install \
+ g++ \
+ gcc \
+ -
+
+# --- Gradle
+ARG BUILD_UID=1001
+RUN useradd -ms /bin/bash -u $BUILD_UID jenkins
+USER jenkins
+WORKDIR /home/jenkins
+
+# Hifi dependencies
+ENV HIFI_BASE="/home/jenkins/hifi_android"
+ENV HIFI_ANDROID_PRECOMPILED="$HIFI_BASE/dependencies"
+ENV HIFI_VCPKG_BASE="$HIFI_BASE/vcpkg"
+
+RUN mkdir "$HIFI_BASE" && \
+ mkdir "$HIFI_VCPKG_BASE" && \
+ mkdir "$HIFI_ANDROID_PRECOMPILED"
+
+RUN git clone https://github.com/jherico/hifi.git && \
+ cd ~/hifi && \
+ git checkout feature/build/gradle-wrapper
+
+
+WORKDIR /home/jenkins/hifi
+
+RUN touch .test4 && \
+ git fetch && git reset origin/feature/build/gradle-wrapper --hard
+
+RUN mkdir build
+
+# Pre-cache the vcpkg managed dependencies
+WORKDIR /home/jenkins/hifi/build
+RUN python3 ../prebuild.py --build-root `pwd` --android
+
+# Pre-cache the gradle dependencies
+WORKDIR /home/jenkins/hifi/android
+RUN ./gradlew -m tasks -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED
+RUN ./gradlew extractDependencies -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED
+
diff --git a/android/build.gradle b/android/build.gradle
index 771db089d0..8d03b9f6b3 100644
--- a/android/build.gradle
+++ b/android/build.gradle
@@ -392,7 +392,7 @@ task extractDependencies(dependsOn: verifyDependencyDownloads) {
}
// Copies the non Qt dependencies. Qt dependencies (primary libraries and plugins) are handled by the qtBundle task
-task copyDependencies(dependsOn: [ extractDependencies ]) {
+task copyDependencies() {
doLast {
packages.each { entry ->
def packageName = entry.key
@@ -414,7 +414,7 @@ task copyDependencies(dependsOn: [ extractDependencies ]) {
}
}
-task extractGvrBinaries(dependsOn: extractDependencies) {
+task extractGvrBinaries() {
doLast {
def gvrLibFolder = new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries');
zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-audio-1.101.0.aar')).visit { element ->
diff --git a/android/build_android.sh b/android/build_android.sh
new file mode 100755
index 0000000000..f98bd1a4b2
--- /dev/null
+++ b/android/build_android.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -xeuo pipefail
+./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} setupDependencies
+./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} app:${ANDROID_BUILD_TARGET}
\ No newline at end of file
diff --git a/hifi_android.py b/hifi_android.py
new file mode 100644
index 0000000000..e3944cda9a
--- /dev/null
+++ b/hifi_android.py
@@ -0,0 +1,286 @@
+import hifi_utils
+import json
+import os
+import platform
+import re
+import shutil
+import xml.etree.ElementTree as ET
+import functools
+
+print = functools.partial(print, flush=True)
+
+ANDROID_PACKAGE_URL = 'https://hifi-public.s3.amazonaws.com/dependencies/android/'
+
+ANDROID_PACKAGES = {
+ 'qt' : {
+ 'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz',
+ 'versionId': '3S97HBM5G5Xw9EfE52sikmgdN3t6C2MN',
+ 'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df',
+ },
+ 'bullet': {
+ 'file': 'bullet-2.88_armv8-libcpp.tgz',
+ 'versionId': 'S8YaoED0Cl8sSb8fSV7Q2G1lQJSNDxqg',
+ 'checksum': '81642779ccb110f8c7338e8739ac38a0',
+ },
+ 'draco': {
+ 'file': 'draco_armv8-libcpp.tgz',
+ 'versionId': '3.B.uBj31kWlgND3_R2xwQzT_TP6Dz_8',
+ 'checksum': '617a80d213a5ec69fbfa21a1f2f738cd',
+ },
+ 'glad': {
+ 'file': 'glad_armv8-libcpp.zip',
+ 'versionId': 'r5Zran.JSCtvrrB6Q4KaqfIoALPw3lYY',
+ 'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449',
+ },
+ 'gvr': {
+ 'file': 'gvrsdk_v1.101.0.tgz',
+ 'versionId': 'nqBV_j81Uc31rC7bKIrlya_Hah4v3y5r',
+ 'checksum': '57fd02baa069176ba18597a29b6b4fc7',
+ },
+ 'nvtt': {
+ 'file': 'nvtt_armv8-libcpp.zip',
+ 'versionId': 'lmkBVR5t4UF1UUwMwEirnk9H_8Nt90IO',
+ 'checksum': 'eb46d0b683e66987190ed124aabf8910',
+ 'sharedLibFolder': 'lib',
+ 'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so']
+ },
+ 'oculus': {
+ 'file': 'ovr_sdk_mobile_1.19.0.zip',
+ 'versionId': 's_RN1vlEvUi3pnT7WPxUC4pQ0RJBs27y',
+ 'checksum': '98f0afb62861f1f02dd8110b31ed30eb',
+ 'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release',
+ 'includeLibs': ['libvrapi.so']
+ },
+ 'openssl': {
+ 'file': 'openssl-1.1.0g_armv8.tgz',
+ 'versionId': 'AiiPjmgUZTgNj7YV1EEx2lL47aDvvvAW',
+ 'checksum': 'cabb681fbccd79594f65fcc266e02f32'
+ },
+ 'polyvox': {
+ 'file': 'polyvox_armv8-libcpp.tgz',
+ 'versionId': 'A2kbKiNhpIenGq23bKRRzg7IMAI5BI92',
+ 'checksum': 'dba88b3a098747af4bb169e9eb9af57e',
+ 'sharedLibFolder': 'lib',
+ 'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'],
+ },
+ 'tbb': {
+ 'file': 'tbb-2018_U1_armv8_libcpp.tgz',
+ 'versionId': 'mrRbWnv4O4evcM1quRH43RJqimlRtaKB',
+ 'checksum': '20768f298f53b195e71b414b0ae240c4',
+ 'sharedLibFolder': 'lib/release',
+ 'includeLibs': ['libtbb.so', 'libtbbmalloc.so'],
+ },
+ 'hifiAC': {
+ 'baseUrl': 'http://s3.amazonaws.com/hifi-public/dependencies/',
+ 'file': 'codecSDK-android_armv8-2.0.zip',
+ 'checksum': '1cbef929675818fc64c4101b72f84a6a'
+ },
+ 'etc2comp': {
+ 'file': 'etc2comp-patched-armv8-libcpp.tgz',
+ 'versionId': 'bHhGECRAQR1vkpshBcK6ByNc1BQIM8gU',
+ 'checksum': '14b02795d774457a33bbc60e00a786bc'
+ },
+ 'breakpad': {
+ 'file': 'breakpad.tgz',
+ 'versionId': '8VrYXz7oyc.QBxNia0BVJOUBvrFO61jI',
+ 'checksum': 'ddcb23df336b08017042ba4786db1d9e',
+ 'sharedLibFolder': 'lib',
+ 'includeLibs': {'libbreakpad_client.a'}
+ }
+}
+
+ANDROID_PLATFORM_PACKAGES = {
+ 'Darwin' : {
+ 'qt': {
+ 'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz',
+ 'versionId': 'OxBD7iKINv1HbyOXmAmDrBb8AF3N.Kup',
+ 'checksum': 'c83cc477c08a892e00c71764dca051a0'
+ },
+ },
+ 'Windows' : {
+ 'qt': {
+ 'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz',
+ 'versionId': 'JfWM0P_Mz5Qp0LwpzhrsRwN3fqlLSFeT',
+ 'checksum': '0582191cc55431aa4f660848a542883e'
+ },
+ }
+}
+
+QT5_DEPS = [
+ 'Qt5Concurrent',
+ 'Qt5Core',
+ 'Qt5Gui',
+ 'Qt5Multimedia',
+ 'Qt5Network',
+ 'Qt5OpenGL',
+ 'Qt5Qml',
+ 'Qt5Quick',
+ 'Qt5QuickControls2',
+ 'Qt5QuickTemplates2',
+ 'Qt5Script',
+ 'Qt5ScriptTools',
+ 'Qt5Svg',
+ 'Qt5WebChannel',
+ 'Qt5WebSockets',
+ 'Qt5Widgets',
+ 'Qt5XmlPatterns',
+ # Android specific
+ 'Qt5AndroidExtras',
+ 'Qt5WebView',
+]
+
+def getPlatformPackages():
+ result = ANDROID_PACKAGES.copy()
+ system = platform.system()
+ if system in ANDROID_PLATFORM_PACKAGES:
+ platformPackages = ANDROID_PLATFORM_PACKAGES[system]
+ result = { **result, **platformPackages }
+ return result
+
+def getPackageUrl(package):
+ url = ANDROID_PACKAGE_URL
+ if 'baseUrl' in package:
+ url = package['baseUrl']
+ url += package['file']
+ if 'versionId' in package:
+ url += '?versionId=' + package['versionId']
+ return url
+
+def copyAndroidLibs(packagePath, appPath):
+ androidPackages = getPlatformPackages()
+ jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a')
+ if not os.path.isdir(jniPath):
+ os.makedirs(jniPath)
+ for packageName in androidPackages:
+ package = androidPackages[packageName]
+ if 'sharedLibFolder' in package:
+ sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder'])
+ if 'includeLibs' in package:
+ for lib in package['includeLibs']:
+ sourceFile = os.path.join(sharedLibFolder, lib)
+ destFile = os.path.join(jniPath, os.path.split(lib)[1])
+ if not os.path.isfile(destFile):
+ print("Copying {}".format(lib))
+ shutil.copy(sourceFile, destFile)
+
+class QtPackager:
+ def __init__(self, appPath, qtRootPath):
+ self.appPath = appPath
+ self.qtRootPath = qtRootPath
+ self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a')
+ self.assetPath = os.path.join(self.appPath, 'src/main/assets')
+ self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--')
+ # Jars go into the qt library
+ self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs'))
+ self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml')
+ self.files = []
+ self.features = []
+ self.permissions = []
+
+ def copyQtDeps(self):
+ for lib in QT5_DEPS:
+ libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib))
+ if not os.path.exists(libfile):
+ continue
+ self.files.append(libfile)
+ androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib))
+ if not os.path.exists(androidDeps):
+ continue
+
+ tree = ET.parse(androidDeps)
+ root = tree.getroot()
+ for item in root.findall('./dependencies/lib/depends/*'):
+ if (item.tag == 'lib') or (item.tag == 'bundled'):
+ relativeFilename = item.attrib['file']
+ if (relativeFilename.startswith('qml')):
+ continue
+ filename = os.path.join(self.qtRootPath, relativeFilename)
+ self.files.extend(hifi_utils.recursiveFileList(filename))
+ elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1":
+ self.files.append(os.path.join(self.qtRootPath, item.attrib['file']))
+ elif item.tag == 'permission':
+ self.permissions.append(item.attrib['name'])
+ elif item.tag == 'feature':
+ self.features.append(item.attrib['name'])
+
+ def scanQmlImports(self):
+ qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner')
+ system = platform.system()
+ if 'Windows' == system:
+ qmlImportCommandFile += ".exe"
+ if not os.path.isfile(qmlImportCommandFile):
+ raise RuntimeError("Couldn't find qml import scanner")
+ qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml')
+ qmlImportPath = os.path.join(self.qtRootPath, 'qml')
+ commandResult = hifi_utils.executeSubprocessCapture([
+ qmlImportCommandFile,
+ '-rootPath', qmlRootPath,
+ '-importPath', qmlImportPath
+ ])
+ qmlImportResults = json.loads(commandResult)
+ for item in qmlImportResults:
+ if 'path' not in item:
+ print("Warning: QML import could not be resolved in any of the import paths: {}".format(item['name']))
+ continue
+ path = os.path.realpath(item['path'])
+ if not os.path.exists(path):
+ continue
+ basePath = path
+ if os.path.isfile(basePath):
+ basePath = os.path.dirname(basePath)
+ basePath = os.path.normcase(basePath)
+ if basePath.startswith(qmlRootPath):
+ continue
+ self.files.extend(hifi_utils.recursiveFileList(path))
+
+ def processFiles(self):
+ self.files = list(set(self.files))
+ self.files.sort()
+ libsXmlRoot = ET.Element('resources')
+ qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'})
+ bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'})
+ bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'})
+ libPrefix = 'lib'
+ for sourceFile in self.files:
+ if not os.path.isfile(sourceFile):
+ raise RuntimeError("Unable to find dependency file " + sourceFile)
+ relativePath = os.path.relpath(sourceFile, self.qtRootPath)
+ destinationFile = None
+ if relativePath.endswith('.so'):
+ garbledFileName = None
+ if relativePath.startswith(libPrefix):
+ garbledFileName = relativePath[4:]
+ p = re.compile(r'lib(Qt5.*).so')
+ m = p.search(garbledFileName)
+ if not m:
+ raise RuntimeError("Huh?")
+ libName = m.group(1)
+ ET.SubElement(qtLibsNode, 'item').text = libName
+ else:
+ garbledFileName = 'lib' + relativePath.replace('\\', '_'[0])
+ value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/')
+ ET.SubElement(bundledLibsNode, 'item').text = value
+ destinationFile = os.path.join(self.jniPath, garbledFileName)
+ elif relativePath.startswith('jar'):
+ destinationFile = os.path.join(self.jarPath, relativePath[4:])
+ else:
+ value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/')
+ ET.SubElement(bundledAssetsNode, 'item').text = value
+ destinationFile = os.path.join(self.qtAssetPath, relativePath)
+
+ destinationParent = os.path.realpath(os.path.dirname(destinationFile))
+ if not os.path.isdir(destinationParent):
+ os.makedirs(destinationParent)
+ if not os.path.isfile(destinationFile):
+ shutil.copy(sourceFile, destinationFile)
+
+ tree = ET.ElementTree(libsXmlRoot)
+ tree.write(self.xmlFile, 'UTF-8', True)
+
+ def bundle(self):
+ if not os.path.isfile(self.xmlFile) or True:
+ self.copyQtDeps()
+ self.scanQmlImports()
+ self.processFiles()
+
+
diff --git a/hifi_singleton.py b/hifi_singleton.py
new file mode 100644
index 0000000000..692948c80b
--- /dev/null
+++ b/hifi_singleton.py
@@ -0,0 +1,46 @@
+import os
+import platform
+import time
+
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
+
+# Used to ensure only one instance of the script runs at a time
+class Singleton:
+ def __init__(self, path):
+ self.fh = None
+ self.windows = 'Windows' == platform.system()
+ self.path = path
+
+ def __enter__(self):
+ success = False
+ while not success:
+ try:
+ if self.windows:
+ if os.path.exists(self.path):
+ os.unlink(self.path)
+ self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
+ else:
+ self.fh = open(self.path, 'x')
+ fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ success = True
+ except EnvironmentError as err:
+ if self.fh is not None:
+ if self.windows:
+ os.close(self.fh)
+ else:
+ self.fh.close()
+ self.fh = None
+ print("Couldn't aquire lock, retrying in 10 seconds")
+ time.sleep(10)
+ return self
+
+ def __exit__(self, type, value, traceback):
+ if self.windows:
+ os.close(self.fh)
+ else:
+ fcntl.lockf(self.fh, fcntl.LOCK_UN)
+ self.fh.close()
+ os.unlink(self.path)
\ No newline at end of file
diff --git a/hifi_utils.py b/hifi_utils.py
new file mode 100644
index 0000000000..f53258d4f6
--- /dev/null
+++ b/hifi_utils.py
@@ -0,0 +1,124 @@
+import os
+import hashlib
+import platform
+import shutil
+import ssl
+import subprocess
+import sys
+import tarfile
+import urllib
+import urllib.request
+import zipfile
+import tempfile
+import time
+import functools
+
+print = functools.partial(print, flush=True)
+
+def scriptRelative(*paths):
+ scriptdir = os.path.dirname(os.path.realpath(sys.argv[0]))
+ result = os.path.join(scriptdir, *paths)
+ result = os.path.realpath(result)
+ result = os.path.normcase(result)
+ return result
+
+
+def recursiveFileList(startPath):
+ result = []
+ if os.path.isfile(startPath):
+ result.append(startPath)
+ elif os.path.isdir(startPath):
+ for dirName, subdirList, fileList in os.walk(startPath):
+ for fname in fileList:
+ result.append(os.path.realpath(os.path.join(startPath, dirName, fname)))
+ result.sort()
+ return result
+
+
+def executeSubprocessCapture(processArgs):
+ processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if (0 != processResult.returncode):
+ raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format(
+ processArgs[0],
+ ' '.join(processArgs[1:]),
+ processResult.stdout.decode('utf-8'),
+ processResult.stderr.decode('utf-8')))
+ return processResult.stdout.decode('utf-8')
+
+def executeSubprocess(processArgs, folder=None, env=None):
+ restoreDir = None
+ if folder != None:
+ restoreDir = os.getcwd()
+ os.chdir(folder)
+
+ process = subprocess.Popen(
+ processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env)
+ process.wait()
+
+ if (0 != process.returncode):
+ raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format(
+ processArgs[0],
+ ' '.join(processArgs[1:]),
+ ))
+
+ if restoreDir != None:
+ os.chdir(restoreDir)
+
+
+def hashFile(file, hasher = hashlib.sha512()):
+ with open(file, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hasher.update(chunk)
+ return hasher.hexdigest()
+
+# Assumes input files are in deterministic order
+def hashFiles(filenames):
+ hasher = hashlib.sha256()
+ for filename in filenames:
+ with open(filename, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hasher.update(chunk)
+ return hasher.hexdigest()
+
+def hashFolder(folder):
+ filenames = recursiveFileList(folder)
+ return hashFiles(filenames)
+
+def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3):
+ for i in range(retries):
+ tempFileName = None
+ # OSX Python doesn't support SSL, so we need to bypass it.
+ # However, we still validate the downloaded file's sha512 hash
+ if 'Darwin' == platform.system():
+ tempFileDescriptor, tempFileName = tempfile.mkstemp()
+ context = ssl._create_unverified_context()
+ with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile:
+ shutil.copyfileobj(response, tempFile)
+ else:
+ tempFileName, headers = urllib.request.urlretrieve(url)
+
+ # for some reason the hash we get back from the downloaded file is sometimes wrong if we check it right away
+ # but if we examine the file later, it is correct.
+ time.sleep(3)
+ downloadHash = hashFile(tempFileName, hasher)
+ # Verify the hash
+ if hash is not None and hash != downloadHash:
+ print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url))
+ os.remove(tempFileName)
+ continue
+
+ return tempFileName
+
+ raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url))
+
+
+def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False):
+ tempFileName = downloadFile(url, hash, hasher)
+ if isZip:
+ with zipfile.ZipFile(tempFileName) as zip:
+ zip.extractall(destPath)
+ else:
+ # Extract the archive
+ with tarfile.open(tempFileName, 'r:gz') as tgz:
+ tgz.extractall(destPath)
+ os.remove(tempFileName)
diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py
new file mode 100644
index 0000000000..5492109864
--- /dev/null
+++ b/hifi_vcpkg.py
@@ -0,0 +1,216 @@
+import hifi_utils
+import hifi_android
+import hashlib
+import os
+import platform
+import re
+import shutil
+import tempfile
+import json
+import xml.etree.ElementTree as ET
+import functools
+
+print = functools.partial(print, flush=True)
+
+# Encapsulates the vcpkg system
+class VcpkgRepo:
+ CMAKE_TEMPLATE = """
+set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")
+set(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}")
+set(VCPKG_INSTALL_ROOT "{}")
+set(VCPKG_TOOLS_DIR "{}")
+"""
+
+ CMAKE_TEMPLATE_NON_ANDROID = """
+# If the cached cmake toolchain path is different from the computed one, exit
+if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE))
+ message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake")
+endif()
+"""
+
+ def __init__(self, args):
+ self.args = args
+ # our custom ports, relative to the script location
+ self.sourcePortsPath = args.ports_path
+ self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8]
+ self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake')
+
+ # OS dependent information
+ system = platform.system()
+
+ if self.args.vcpkg_root is not None:
+ self.path = args.vcpkg_root
+ else:
+ if 'Darwin' == system:
+ defaultBasePath = os.path.expanduser('~/hifi/vcpkg')
+ else:
+ defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg')
+ self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath)
+ if self.basePath == defaultBasePath:
+ print("Warning: Environment variable HIFI_VCPKG_BASE not set, using {}".format(defaultBasePath))
+ if self.args.android:
+ self.basePath = os.path.join(self.basePath, 'android')
+ if (not os.path.isdir(self.basePath)):
+ os.makedirs(self.basePath)
+ self.path = os.path.join(self.basePath, self.id)
+
+ print("Using vcpkg path {}".format(self.path))
+ lockDir, lockName = os.path.split(self.path)
+ lockName += '.lock'
+ if not os.path.isdir(lockDir):
+ os.makedirs(lockDir)
+
+ self.lockFile = os.path.join(lockDir, lockName)
+ self.tagFile = os.path.join(self.path, '.id')
+ # A format version attached to the tag file... increment when you want to force the build systems to rebuild
+ # without the contents of the ports changing
+ self.version = 1
+ self.tagContents = "{}_{}".format(self.id, self.version)
+
+ if 'Windows' == system:
+ self.exe = os.path.join(self.path, 'vcpkg.exe')
+ self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ'
+ self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e'
+ self.hostTriplet = 'x64-windows'
+ elif 'Darwin' == system:
+ self.exe = os.path.join(self.path, 'vcpkg')
+ self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc'
+ self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d'
+ self.hostTriplet = 'x64-osx'
+ else:
+ self.exe = os.path.join(self.path, 'vcpkg')
+ self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU'
+ self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d'
+ self.hostTriplet = 'x64-linux'
+
+ if self.args.android:
+ self.triplet = 'arm64-android'
+ self.androidPackagePath = os.path.join(self.path, 'android')
+ else:
+ self.triplet = self.hostTriplet
+
+ def upToDate(self):
+ # Prevent doing a clean if we've explcitly set a directory for vcpkg
+ if self.args.vcpkg_root is not None:
+ return True
+
+ if self.args.force_build:
+ print("Force build, out of date")
+ return False
+ if not os.path.isfile(self.exe):
+ print("Exe file {} not found, out of date".format(self.exe))
+ return False
+ if not os.path.isfile(self.tagFile):
+ print("Tag file {} not found, out of date".format(self.tagFile))
+ return False
+ with open(self.tagFile, 'r') as f:
+ storedTag = f.read()
+ if storedTag != self.tagContents:
+ print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents))
+ return False
+ return True
+
+ def clean(self):
+ print("Cleaning vcpkg installation at {}".format(self.path))
+ if os.path.isdir(self.path):
+ print("Removing {}".format(self.path))
+ shutil.rmtree(self.path, ignore_errors=True)
+
+ # Make sure the VCPKG prerequisites are all there.
+ def bootstrap(self):
+ if self.upToDate():
+ return
+
+ self.clean()
+
+ downloadVcpkg = False
+ if self.args.force_bootstrap:
+ print("Forcing bootstrap")
+ downloadVcpkg = True
+
+ if not downloadVcpkg and not os.path.isfile(self.exe):
+ print("Missing executable, boostrapping")
+ downloadVcpkg = True
+
+ # Make sure we have a vcpkg executable
+ testFile = os.path.join(self.path, '.vcpkg-root')
+ if not downloadVcpkg and not os.path.isfile(testFile):
+ print("Missing {}, bootstrapping".format(testFile))
+ downloadVcpkg = True
+
+ if downloadVcpkg:
+ print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
+ hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash)
+
+ print("Replacing port files")
+ portsPath = os.path.join(self.path, 'ports')
+ if (os.path.islink(portsPath)):
+ os.unlink(portsPath)
+ if (os.path.isdir(portsPath)):
+ shutil.rmtree(portsPath, ignore_errors=True)
+ shutil.copytree(self.sourcePortsPath, portsPath)
+
+ def run(self, commands):
+ actualCommands = [self.exe, '--vcpkg-root', self.path]
+ actualCommands.extend(commands)
+ print("Running command")
+ print(actualCommands)
+ hifi_utils.executeSubprocess(actualCommands, folder=self.path)
+
+ def setupDependencies(self):
+ # Special case for android, grab a bunch of binaries
+ # FIXME remove special casing for android builds eventually
+ if self.args.android:
+ print("Installing Android binaries")
+ self.setupAndroidDependencies()
+
+ print("Installing host tools")
+ self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools'])
+
+ # If not android, install the hifi-client-deps libraries
+ if not self.args.android:
+ print("Installing build dependencies")
+ self.run(['install', '--triplet', self.triplet, 'hifi-client-deps'])
+
+ def cleanBuilds(self):
+ # Remove temporary build artifacts
+ builddir = os.path.join(self.path, 'buildtrees')
+ if os.path.isdir(builddir):
+ print("Wiping build trees")
+ shutil.rmtree(builddir, ignore_errors=True)
+
+ def setupAndroidDependencies(self):
+ # vcpkg prebuilt
+ if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')):
+ dest = os.path.join(self.path, 'installed')
+ url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
+ # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
+ #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
+ #hifi_utils.downloadAndExtract(url, dest, hash)
+ hifi_utils.downloadAndExtract(url, dest)
+
+ def writeTag(self):
+ print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
+ with open(self.tagFile, 'w') as f:
+ f.write(self.tagContents)
+
+ def writeConfig(self):
+ print("Writing cmake config to {}".format(self.configFilePath))
+ # Write out the configuration for use by CMake
+ cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
+ installPath = os.path.join(self.path, 'installed', self.triplet)
+ toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools')
+ cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE
+ if not self.args.android:
+ cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID
+ cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath).replace('\\', '/')
+ with open(self.configFilePath, 'w') as f:
+ f.write(cmakeConfig)
+
+ def cleanOldBuilds(self):
+ # FIXME because we have the base directory, and because a build will
+ # update the tag file on every run, we can scan the base dir for sub directories containing
+ # a tag file that is older than N days, and if found, delete the directory, recovering space
+ print("Not implemented")
+
+
diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp
index e9a44b1e87..bbbd8db89f 100644
--- a/interface/src/Menu.cpp
+++ b/interface/src/Menu.cpp
@@ -802,7 +802,7 @@ Menu::Menu() {
connect(action, &QAction::triggered, qApp, []() { std::thread(crash::newFault).join(); });
// Developer > Show Statistics
- addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats);
+ addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats, 0, true);
// Developer > Show Animation Statistics
addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::AnimStats);
diff --git a/prebuild.py b/prebuild.py
index b8cb2c96ae..a758dcbea2 100644
--- a/prebuild.py
+++ b/prebuild.py
@@ -1,5 +1,26 @@
#!python
+# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring
+# tools required by the build as well as dependencies on which we rely.
+#
+# By using this script, we can reduce the requirements for a developer getting started to:
+#
+# * A working C++ dev environment like visual studio, xcode, gcc, or clang
+# * Qt
+# * CMake
+# * Python 3.x
+#
+# The function of the build script is to acquire, if not already present, all the other build requirements
+# The build script should be idempotent. If you run it with the same arguments multiple times, that should
+# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not
+# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should
+# execute quickly, determining that no work is to be done
+
+import hifi_singleton
+import hifi_utils
+import hifi_android
+import hifi_vcpkg
+
import argparse
import concurrent
import hashlib
@@ -9,252 +30,65 @@ import os
import platform
import shutil
import ssl
-import subprocess
import sys
-import tarfile
+import re
import tempfile
import time
-import urllib.request
import functools
print = functools.partial(print, flush=True)
-def executeSubprocess(processArgs, folder=None, env=None):
- restoreDir = None
- if folder != None:
- restoreDir = os.getcwd()
- os.chdir(folder)
-
- process = subprocess.Popen(
- processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env)
- process.wait()
-
- if (0 != process.returncode):
- raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format(
- processArgs[0],
- ' '.join(processArgs[1:]),
- ))
-
- if restoreDir != None:
- os.chdir(restoreDir)
-
-
-def hashFile(file):
- hasher = hashlib.sha512()
- with open(file, "rb") as f:
- for chunk in iter(lambda: f.read(4096), b""):
- hasher.update(chunk)
- return hasher.hexdigest()
-
-
-def hashFolder(folder):
- hasher = hashlib.sha256()
- for dirName, subdirList, fileList in os.walk(folder):
- for fname in fileList:
- with open(os.path.join(folder, dirName, fname), "rb") as f:
- for chunk in iter(lambda: f.read(4096), b""):
- hasher.update(chunk)
- return hasher.hexdigest()
-
-
-def downloadAndExtract(url, destPath, hash=None):
- tempFileDescriptor, tempFileName = tempfile.mkstemp()
- # OSX Python doesn't support SSL, so we need to bypass it.
- # However, we still validate the downloaded file's sha512 hash
- context = ssl._create_unverified_context()
- with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile:
- shutil.copyfileobj(response, tempFile)
-
- # Verify the hash
- if hash and hash != hashFile(tempFileName):
- raise RuntimeError("Downloaded file does not match hash")
-
- # Extract the archive
- with tarfile.open(tempFileName, 'r:gz') as tgz:
- tgz.extractall(destPath)
- os.remove(tempFileName)
-
-
-class VcpkgRepo:
- def __init__(self):
- global args
- scriptPath = os.path.dirname(os.path.realpath(sys.argv[0]))
- # our custom ports, relative to the script location
- self.sourcePortsPath = os.path.join(scriptPath, 'cmake', 'ports')
- # FIXME Revert to ports hash before release
- self.id = hashFolder(self.sourcePortsPath)[:8]
- # OS dependent information
- system = platform.system()
-
- if args.vcpkg_root is not None:
- print("override vcpkg path with " + args.vcpkg_root)
- self.path = args.vcpkg_root
- else:
- if 'Darwin' == system:
- defaultBasePath = os.path.expanduser('~/hifi/vcpkg')
- else:
- defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg')
- basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath)
- if (not os.path.isdir(basePath)):
- os.makedirs(basePath)
- self.path = os.path.join(basePath, self.id)
-
- self.tagFile = os.path.join(self.path, '.id')
- # A format version attached to the tag file... increment when you want to force the build systems to rebuild
- # without the contents of the ports changing
- self.version = 1
- self.tagContents = "{}_{}".format(self.id, self.version)
-
- print("prebuild path: " + self.path)
- if 'Windows' == system:
- self.exe = os.path.join(self.path, 'vcpkg.exe')
- self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ'
- self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e'
- self.hostTriplet = 'x64-windows'
- elif 'Darwin' == system:
- self.exe = os.path.join(self.path, 'vcpkg')
- self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc'
- self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d'
- self.hostTriplet = 'x64-osx'
- else:
- self.exe = os.path.join(self.path, 'vcpkg')
- self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU'
- self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d'
- self.hostTriplet = 'x64-linux'
-
- if args.android:
- self.triplet = 'arm64-android'
- else:
- self.triplet = self.hostTriplet
-
- def outOfDate(self):
- global args
- # Prevent doing a clean if we've explcitly set a directory for vcpkg
- if args.vcpkg_root is not None:
- return False
- if args.force_build:
- return True
- print("Looking for tag file {}".format(self.tagFile))
- if not os.path.isfile(self.tagFile):
- return True
- with open(self.tagFile, 'r') as f:
- storedTag = f.read()
- print("Found stored tag {}".format(storedTag))
- if storedTag != self.tagContents:
- print("Doesn't match computed tag {}".format(self.tagContents))
- return True
- return False
-
- def clean(self):
- cleanPath = self.path
- print("Cleaning vcpkg installation at {}".format(cleanPath))
- if os.path.isdir(self.path):
- print("Removing {}".format(cleanPath))
- shutil.rmtree(cleanPath, ignore_errors=True)
-
- def bootstrap(self):
- global args
- if self.outOfDate():
- self.clean()
-
- # don't download the vcpkg binaries if we're working with an explicit
- # vcpkg directory (possibly a git checkout)
- if args.vcpkg_root is None:
- downloadVcpkg = False
- if args.force_bootstrap:
- print("Forcing bootstrap")
- downloadVcpkg = True
-
- if not downloadVcpkg and not os.path.isfile(self.exe):
- print("Missing executable, boostrapping")
- downloadVcpkg = True
-
- # Make sure we have a vcpkg executable
- testFile = os.path.join(self.path, '.vcpkg-root')
- if not downloadVcpkg and not os.path.isfile(testFile):
- print("Missing {}, bootstrapping".format(testFile))
- downloadVcpkg = True
-
- if downloadVcpkg:
- print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
- downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash)
-
- print("Replacing port files")
- portsPath = os.path.join(self.path, 'ports')
- if (os.path.islink(portsPath)):
- os.unlink(portsPath)
- if (os.path.isdir(portsPath)):
- shutil.rmtree(portsPath, ignore_errors=True)
- shutil.copytree(self.sourcePortsPath, portsPath)
-
- def downloadAndroidDependencies(self):
- url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
- hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
- dest = os.path.join(self.path, 'installed')
- downloadAndExtract(url, dest, hash)
-
- def run(self, commands):
- actualCommands = [self.exe, '--vcpkg-root', self.path]
- actualCommands.extend(commands)
- print("Running command")
- print(actualCommands)
- executeSubprocess(actualCommands, folder=self.path)
-
- def buildDependencies(self):
- global args
- print("Installing host tools")
- self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools'])
- # Special case for android, grab a bunch of binaries
- if args.android:
- self.downloadAndroidDependencies()
- return
-
- print("Installing build dependencies")
- self.run(['install', '--triplet', self.triplet, 'hifi-client-deps'])
- # Remove temporary build artifacts
- builddir = os.path.join(self.path, 'buildtrees')
- if os.path.isdir(builddir):
- print("Wiping build trees")
- shutil.rmtree(builddir, ignore_errors=True)
-
- def writeConfig(self):
- global args
- configFilePath = os.path.join(args.build_root, 'vcpkg.cmake')
- print("Writing cmake config to {}".format(configFilePath))
- # Write out the configuration for use by CMake
- cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
- installPath = os.path.join(self.path, 'installed', self.triplet)
- toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools')
- cmakeTemplate = 'set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")\n'
- cmakeTemplate += 'set(VCPKG_INSTALL_ROOT "{}" CACHE FILEPATH "vcpkg installed packages path")\n'
- cmakeTemplate += 'set(VCPKG_TOOLS_DIR "{}" CACHE FILEPATH "vcpkg installed packages path")\n'
- cmakeConfig = cmakeTemplate.format(cmakeScript, installPath, toolsPath).replace('\\', '/')
- with open(configFilePath, 'w') as f:
- f.write(cmakeConfig)
-
- def writeTag(self):
- print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
- with open(self.tagFile, 'w') as f:
- f.write(self.tagContents)
+def parse_args():
+ # our custom ports, relative to the script location
+ defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports')
+ from argparse import ArgumentParser
+ parser = ArgumentParser(description='Prepare build dependencies.')
+ parser.add_argument('--android', action='store_true')
+ #parser.add_argument('--android', type=str)
+ parser.add_argument('--debug', action='store_true')
+ parser.add_argument('--force-bootstrap', action='store_true')
+ parser.add_argument('--force-build', action='store_true')
+ parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
+ parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
+ parser.add_argument('--ports-path', type=str, default=defaultPortsPath)
+ if True:
+ args = parser.parse_args()
+ else:
+ args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/hifi/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a'])
+ return args
def main():
- vcpkg = VcpkgRepo()
- vcpkg.bootstrap()
- vcpkg.buildDependencies()
- vcpkg.writeConfig()
- vcpkg.writeTag()
+ # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build
+ # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for
+ # building OpenSSL
+ removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE']
+ for var in removeEnvVars:
+ if var in os.environ:
+ del os.environ[var]
+ args = parse_args()
+ # Only allow one instance of the program to run at a time
+ pm = hifi_vcpkg.VcpkgRepo(args)
+ with hifi_singleton.Singleton(pm.lockFile) as lock:
+ if not pm.upToDate():
+ pm.bootstrap()
+ # Always write the tag, even if we changed nothing. This
+ # allows vcpkg to reclaim disk space by identifying directories with
+ # tags that haven't been touched in a long time
+ pm.writeTag()
-from argparse import ArgumentParser
-parser = ArgumentParser(description='Prepare build dependencies.')
-parser.add_argument('--android', action='store_true')
-parser.add_argument('--debug', action='store_true')
-parser.add_argument('--force-bootstrap', action='store_true')
-parser.add_argument('--force-build', action='store_true')
-parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
-parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
+ # Grab our required dependencies:
+ # * build host tools, like spirv-cross and scribe
+ # * build client dependencies like openssl and nvtt
+ pm.setupDependencies()
-args = parser.parse_args()
+ # wipe out the build directories (after writing the tag, since failure
+ # here shouldn't invalidte the vcpkg install)
+ pm.cleanBuilds()
+
+ # Write the vcpkg config to the build directory last
+ pm.writeConfig()
+
+print(sys.argv)
main()
-