diff --git a/.gitignore b/.gitignore index ef1a7b215e..09b58d71ef 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,11 @@ Makefile local.properties android/gradle* android/.gradle -android/app/src/main/jniLibs -android/app/libs -android/app/src/main/res/values/libs.xml -android/app/src/main/assets/bundled +android/**/src/main/jniLibs +android/**/libs +android/**/src/main/res/values/libs.xml +android/**/src/main/assets +android/**/gradle* # VSCode # List taken from Github Global Ignores master@435c4d92 @@ -83,9 +84,6 @@ npm-debug.log # Android studio files *___jb_old___ -# Generated assets for Android -android/app/src/main/assets - # Resource binary file interface/compiledResources @@ -95,6 +93,9 @@ interface/resources/GPUCache/* # package lock file for JSDoc tool tools/jsdoc/package-lock.json +# Python compile artifacts +**/__pycache__ + # ignore unneeded unity project files for avatar exporter tools/unity-avatar-exporter/Library tools/unity-avatar-exporter/Packages diff --git a/android/Dockerfile b/android/Dockerfile new file mode 100644 index 0000000000..2a6943cbc2 --- /dev/null +++ b/android/Dockerfile @@ -0,0 +1,92 @@ +FROM openjdk:8 + +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +RUN apt-get update && apt-get -y install \ + curl \ + gnupg \ + software-properties-common \ + unzip \ + - + +# --- Versions and Download paths +ENV ANDROID_HOME="/usr/local/android-sdk" \ + ANDROID_NDK_HOME="/usr/local/android-ndk" \ + ANDROID_SDK_HOME="/usr/local/android-sdk-home" \ + ANDROID_VERSION=26 \ + ANDROID_BUILD_TOOLS_VERSION=28.0.3 \ + ANDROID_NDK_VERSION=r18 + +ENV SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip" \ + NDK_URL="https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip" + +# --- Android SDK +RUN mkdir -p "$ANDROID_HOME" "$ANDROID_SDK_HOME" && \ + cd "$ANDROID_HOME" && \ + curl -s -S -o sdk.zip -L "${SDK_URL}" && \ + unzip sdk.zip && \ + rm sdk.zip && \ + yes | $ANDROID_HOME/tools/bin/sdkmanager --licenses + +# Install Android Build Tool and Libraries +RUN $ANDROID_HOME/tools/bin/sdkmanager --update +RUN $ANDROID_HOME/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" \ + "platforms;android-${ANDROID_VERSION}" \ + "platform-tools" + +RUN chmod -R a+w "${ANDROID_HOME}" +RUN chmod -R a+w "${ANDROID_SDK_HOME}" + +# --- Android NDK +# download +RUN mkdir /usr/local/android-ndk-tmp && \ + cd /usr/local/android-ndk-tmp && \ + curl -s -S -o ndk.zip -L "${NDK_URL}" && \ + unzip -q ndk.zip && \ + mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \ + cd ${ANDROID_NDK_HOME} && \ + rm -rf /usr/local/android-ndk-tmp + +ENV PATH ${PATH}:${ANDROID_NDK_HOME} + +RUN apt-get -y install \ + g++ \ + gcc \ + - + +# --- Gradle +ARG BUILD_UID=1001 +RUN useradd -ms /bin/bash -u $BUILD_UID jenkins +USER jenkins +WORKDIR /home/jenkins + +# Hifi dependencies +ENV HIFI_BASE="/home/jenkins/hifi_android" +ENV HIFI_ANDROID_PRECOMPILED="$HIFI_BASE/dependencies" +ENV HIFI_VCPKG_BASE="$HIFI_BASE/vcpkg" + +RUN mkdir "$HIFI_BASE" && \ + mkdir "$HIFI_VCPKG_BASE" && \ + mkdir "$HIFI_ANDROID_PRECOMPILED" + +RUN git clone https://github.com/jherico/hifi.git && \ + cd ~/hifi && \ + git checkout feature/build/gradle-wrapper + + +WORKDIR /home/jenkins/hifi + +RUN touch .test4 && \ + git fetch && git reset origin/feature/build/gradle-wrapper --hard + +RUN mkdir build + +# Pre-cache the vcpkg managed dependencies +WORKDIR /home/jenkins/hifi/build +RUN python3 ../prebuild.py --build-root `pwd` --android + +# Pre-cache the gradle dependencies +WORKDIR /home/jenkins/hifi/android +RUN ./gradlew -m tasks -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED +RUN ./gradlew extractDependencies -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED + diff --git a/android/build.gradle b/android/build.gradle index 771db089d0..8d03b9f6b3 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -392,7 +392,7 @@ task extractDependencies(dependsOn: verifyDependencyDownloads) { } // Copies the non Qt dependencies. Qt dependencies (primary libraries and plugins) are handled by the qtBundle task -task copyDependencies(dependsOn: [ extractDependencies ]) { +task copyDependencies() { doLast { packages.each { entry -> def packageName = entry.key @@ -414,7 +414,7 @@ task copyDependencies(dependsOn: [ extractDependencies ]) { } } -task extractGvrBinaries(dependsOn: extractDependencies) { +task extractGvrBinaries() { doLast { def gvrLibFolder = new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries'); zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-audio-1.101.0.aar')).visit { element -> diff --git a/android/build_android.sh b/android/build_android.sh new file mode 100755 index 0000000000..f98bd1a4b2 --- /dev/null +++ b/android/build_android.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -xeuo pipefail +./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} setupDependencies +./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} app:${ANDROID_BUILD_TARGET} \ No newline at end of file diff --git a/hifi_android.py b/hifi_android.py new file mode 100644 index 0000000000..e3944cda9a --- /dev/null +++ b/hifi_android.py @@ -0,0 +1,286 @@ +import hifi_utils +import json +import os +import platform +import re +import shutil +import xml.etree.ElementTree as ET +import functools + +print = functools.partial(print, flush=True) + +ANDROID_PACKAGE_URL = 'https://hifi-public.s3.amazonaws.com/dependencies/android/' + +ANDROID_PACKAGES = { + 'qt' : { + 'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz', + 'versionId': '3S97HBM5G5Xw9EfE52sikmgdN3t6C2MN', + 'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df', + }, + 'bullet': { + 'file': 'bullet-2.88_armv8-libcpp.tgz', + 'versionId': 'S8YaoED0Cl8sSb8fSV7Q2G1lQJSNDxqg', + 'checksum': '81642779ccb110f8c7338e8739ac38a0', + }, + 'draco': { + 'file': 'draco_armv8-libcpp.tgz', + 'versionId': '3.B.uBj31kWlgND3_R2xwQzT_TP6Dz_8', + 'checksum': '617a80d213a5ec69fbfa21a1f2f738cd', + }, + 'glad': { + 'file': 'glad_armv8-libcpp.zip', + 'versionId': 'r5Zran.JSCtvrrB6Q4KaqfIoALPw3lYY', + 'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449', + }, + 'gvr': { + 'file': 'gvrsdk_v1.101.0.tgz', + 'versionId': 'nqBV_j81Uc31rC7bKIrlya_Hah4v3y5r', + 'checksum': '57fd02baa069176ba18597a29b6b4fc7', + }, + 'nvtt': { + 'file': 'nvtt_armv8-libcpp.zip', + 'versionId': 'lmkBVR5t4UF1UUwMwEirnk9H_8Nt90IO', + 'checksum': 'eb46d0b683e66987190ed124aabf8910', + 'sharedLibFolder': 'lib', + 'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so'] + }, + 'oculus': { + 'file': 'ovr_sdk_mobile_1.19.0.zip', + 'versionId': 's_RN1vlEvUi3pnT7WPxUC4pQ0RJBs27y', + 'checksum': '98f0afb62861f1f02dd8110b31ed30eb', + 'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release', + 'includeLibs': ['libvrapi.so'] + }, + 'openssl': { + 'file': 'openssl-1.1.0g_armv8.tgz', + 'versionId': 'AiiPjmgUZTgNj7YV1EEx2lL47aDvvvAW', + 'checksum': 'cabb681fbccd79594f65fcc266e02f32' + }, + 'polyvox': { + 'file': 'polyvox_armv8-libcpp.tgz', + 'versionId': 'A2kbKiNhpIenGq23bKRRzg7IMAI5BI92', + 'checksum': 'dba88b3a098747af4bb169e9eb9af57e', + 'sharedLibFolder': 'lib', + 'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'], + }, + 'tbb': { + 'file': 'tbb-2018_U1_armv8_libcpp.tgz', + 'versionId': 'mrRbWnv4O4evcM1quRH43RJqimlRtaKB', + 'checksum': '20768f298f53b195e71b414b0ae240c4', + 'sharedLibFolder': 'lib/release', + 'includeLibs': ['libtbb.so', 'libtbbmalloc.so'], + }, + 'hifiAC': { + 'baseUrl': 'http://s3.amazonaws.com/hifi-public/dependencies/', + 'file': 'codecSDK-android_armv8-2.0.zip', + 'checksum': '1cbef929675818fc64c4101b72f84a6a' + }, + 'etc2comp': { + 'file': 'etc2comp-patched-armv8-libcpp.tgz', + 'versionId': 'bHhGECRAQR1vkpshBcK6ByNc1BQIM8gU', + 'checksum': '14b02795d774457a33bbc60e00a786bc' + }, + 'breakpad': { + 'file': 'breakpad.tgz', + 'versionId': '8VrYXz7oyc.QBxNia0BVJOUBvrFO61jI', + 'checksum': 'ddcb23df336b08017042ba4786db1d9e', + 'sharedLibFolder': 'lib', + 'includeLibs': {'libbreakpad_client.a'} + } +} + +ANDROID_PLATFORM_PACKAGES = { + 'Darwin' : { + 'qt': { + 'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz', + 'versionId': 'OxBD7iKINv1HbyOXmAmDrBb8AF3N.Kup', + 'checksum': 'c83cc477c08a892e00c71764dca051a0' + }, + }, + 'Windows' : { + 'qt': { + 'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz', + 'versionId': 'JfWM0P_Mz5Qp0LwpzhrsRwN3fqlLSFeT', + 'checksum': '0582191cc55431aa4f660848a542883e' + }, + } +} + +QT5_DEPS = [ + 'Qt5Concurrent', + 'Qt5Core', + 'Qt5Gui', + 'Qt5Multimedia', + 'Qt5Network', + 'Qt5OpenGL', + 'Qt5Qml', + 'Qt5Quick', + 'Qt5QuickControls2', + 'Qt5QuickTemplates2', + 'Qt5Script', + 'Qt5ScriptTools', + 'Qt5Svg', + 'Qt5WebChannel', + 'Qt5WebSockets', + 'Qt5Widgets', + 'Qt5XmlPatterns', + # Android specific + 'Qt5AndroidExtras', + 'Qt5WebView', +] + +def getPlatformPackages(): + result = ANDROID_PACKAGES.copy() + system = platform.system() + if system in ANDROID_PLATFORM_PACKAGES: + platformPackages = ANDROID_PLATFORM_PACKAGES[system] + result = { **result, **platformPackages } + return result + +def getPackageUrl(package): + url = ANDROID_PACKAGE_URL + if 'baseUrl' in package: + url = package['baseUrl'] + url += package['file'] + if 'versionId' in package: + url += '?versionId=' + package['versionId'] + return url + +def copyAndroidLibs(packagePath, appPath): + androidPackages = getPlatformPackages() + jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a') + if not os.path.isdir(jniPath): + os.makedirs(jniPath) + for packageName in androidPackages: + package = androidPackages[packageName] + if 'sharedLibFolder' in package: + sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder']) + if 'includeLibs' in package: + for lib in package['includeLibs']: + sourceFile = os.path.join(sharedLibFolder, lib) + destFile = os.path.join(jniPath, os.path.split(lib)[1]) + if not os.path.isfile(destFile): + print("Copying {}".format(lib)) + shutil.copy(sourceFile, destFile) + +class QtPackager: + def __init__(self, appPath, qtRootPath): + self.appPath = appPath + self.qtRootPath = qtRootPath + self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a') + self.assetPath = os.path.join(self.appPath, 'src/main/assets') + self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--') + # Jars go into the qt library + self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs')) + self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml') + self.files = [] + self.features = [] + self.permissions = [] + + def copyQtDeps(self): + for lib in QT5_DEPS: + libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib)) + if not os.path.exists(libfile): + continue + self.files.append(libfile) + androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib)) + if not os.path.exists(androidDeps): + continue + + tree = ET.parse(androidDeps) + root = tree.getroot() + for item in root.findall('./dependencies/lib/depends/*'): + if (item.tag == 'lib') or (item.tag == 'bundled'): + relativeFilename = item.attrib['file'] + if (relativeFilename.startswith('qml')): + continue + filename = os.path.join(self.qtRootPath, relativeFilename) + self.files.extend(hifi_utils.recursiveFileList(filename)) + elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1": + self.files.append(os.path.join(self.qtRootPath, item.attrib['file'])) + elif item.tag == 'permission': + self.permissions.append(item.attrib['name']) + elif item.tag == 'feature': + self.features.append(item.attrib['name']) + + def scanQmlImports(self): + qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner') + system = platform.system() + if 'Windows' == system: + qmlImportCommandFile += ".exe" + if not os.path.isfile(qmlImportCommandFile): + raise RuntimeError("Couldn't find qml import scanner") + qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml') + qmlImportPath = os.path.join(self.qtRootPath, 'qml') + commandResult = hifi_utils.executeSubprocessCapture([ + qmlImportCommandFile, + '-rootPath', qmlRootPath, + '-importPath', qmlImportPath + ]) + qmlImportResults = json.loads(commandResult) + for item in qmlImportResults: + if 'path' not in item: + print("Warning: QML import could not be resolved in any of the import paths: {}".format(item['name'])) + continue + path = os.path.realpath(item['path']) + if not os.path.exists(path): + continue + basePath = path + if os.path.isfile(basePath): + basePath = os.path.dirname(basePath) + basePath = os.path.normcase(basePath) + if basePath.startswith(qmlRootPath): + continue + self.files.extend(hifi_utils.recursiveFileList(path)) + + def processFiles(self): + self.files = list(set(self.files)) + self.files.sort() + libsXmlRoot = ET.Element('resources') + qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'}) + bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'}) + bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'}) + libPrefix = 'lib' + for sourceFile in self.files: + if not os.path.isfile(sourceFile): + raise RuntimeError("Unable to find dependency file " + sourceFile) + relativePath = os.path.relpath(sourceFile, self.qtRootPath) + destinationFile = None + if relativePath.endswith('.so'): + garbledFileName = None + if relativePath.startswith(libPrefix): + garbledFileName = relativePath[4:] + p = re.compile(r'lib(Qt5.*).so') + m = p.search(garbledFileName) + if not m: + raise RuntimeError("Huh?") + libName = m.group(1) + ET.SubElement(qtLibsNode, 'item').text = libName + else: + garbledFileName = 'lib' + relativePath.replace('\\', '_'[0]) + value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/') + ET.SubElement(bundledLibsNode, 'item').text = value + destinationFile = os.path.join(self.jniPath, garbledFileName) + elif relativePath.startswith('jar'): + destinationFile = os.path.join(self.jarPath, relativePath[4:]) + else: + value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/') + ET.SubElement(bundledAssetsNode, 'item').text = value + destinationFile = os.path.join(self.qtAssetPath, relativePath) + + destinationParent = os.path.realpath(os.path.dirname(destinationFile)) + if not os.path.isdir(destinationParent): + os.makedirs(destinationParent) + if not os.path.isfile(destinationFile): + shutil.copy(sourceFile, destinationFile) + + tree = ET.ElementTree(libsXmlRoot) + tree.write(self.xmlFile, 'UTF-8', True) + + def bundle(self): + if not os.path.isfile(self.xmlFile) or True: + self.copyQtDeps() + self.scanQmlImports() + self.processFiles() + + diff --git a/hifi_singleton.py b/hifi_singleton.py new file mode 100644 index 0000000000..692948c80b --- /dev/null +++ b/hifi_singleton.py @@ -0,0 +1,46 @@ +import os +import platform +import time + +try: + import fcntl +except ImportError: + fcntl = None + +# Used to ensure only one instance of the script runs at a time +class Singleton: + def __init__(self, path): + self.fh = None + self.windows = 'Windows' == platform.system() + self.path = path + + def __enter__(self): + success = False + while not success: + try: + if self.windows: + if os.path.exists(self.path): + os.unlink(self.path) + self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR) + else: + self.fh = open(self.path, 'x') + fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + success = True + except EnvironmentError as err: + if self.fh is not None: + if self.windows: + os.close(self.fh) + else: + self.fh.close() + self.fh = None + print("Couldn't aquire lock, retrying in 10 seconds") + time.sleep(10) + return self + + def __exit__(self, type, value, traceback): + if self.windows: + os.close(self.fh) + else: + fcntl.lockf(self.fh, fcntl.LOCK_UN) + self.fh.close() + os.unlink(self.path) \ No newline at end of file diff --git a/hifi_utils.py b/hifi_utils.py new file mode 100644 index 0000000000..f53258d4f6 --- /dev/null +++ b/hifi_utils.py @@ -0,0 +1,124 @@ +import os +import hashlib +import platform +import shutil +import ssl +import subprocess +import sys +import tarfile +import urllib +import urllib.request +import zipfile +import tempfile +import time +import functools + +print = functools.partial(print, flush=True) + +def scriptRelative(*paths): + scriptdir = os.path.dirname(os.path.realpath(sys.argv[0])) + result = os.path.join(scriptdir, *paths) + result = os.path.realpath(result) + result = os.path.normcase(result) + return result + + +def recursiveFileList(startPath): + result = [] + if os.path.isfile(startPath): + result.append(startPath) + elif os.path.isdir(startPath): + for dirName, subdirList, fileList in os.walk(startPath): + for fname in fileList: + result.append(os.path.realpath(os.path.join(startPath, dirName, fname))) + result.sort() + return result + + +def executeSubprocessCapture(processArgs): + processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if (0 != processResult.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format( + processArgs[0], + ' '.join(processArgs[1:]), + processResult.stdout.decode('utf-8'), + processResult.stderr.decode('utf-8'))) + return processResult.stdout.decode('utf-8') + +def executeSubprocess(processArgs, folder=None, env=None): + restoreDir = None + if folder != None: + restoreDir = os.getcwd() + os.chdir(folder) + + process = subprocess.Popen( + processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env) + process.wait() + + if (0 != process.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format( + processArgs[0], + ' '.join(processArgs[1:]), + )) + + if restoreDir != None: + os.chdir(restoreDir) + + +def hashFile(file, hasher = hashlib.sha512()): + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +# Assumes input files are in deterministic order +def hashFiles(filenames): + hasher = hashlib.sha256() + for filename in filenames: + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def hashFolder(folder): + filenames = recursiveFileList(folder) + return hashFiles(filenames) + +def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3): + for i in range(retries): + tempFileName = None + # OSX Python doesn't support SSL, so we need to bypass it. + # However, we still validate the downloaded file's sha512 hash + if 'Darwin' == platform.system(): + tempFileDescriptor, tempFileName = tempfile.mkstemp() + context = ssl._create_unverified_context() + with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile: + shutil.copyfileobj(response, tempFile) + else: + tempFileName, headers = urllib.request.urlretrieve(url) + + # for some reason the hash we get back from the downloaded file is sometimes wrong if we check it right away + # but if we examine the file later, it is correct. + time.sleep(3) + downloadHash = hashFile(tempFileName, hasher) + # Verify the hash + if hash is not None and hash != downloadHash: + print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url)) + os.remove(tempFileName) + continue + + return tempFileName + + raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url)) + + +def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False): + tempFileName = downloadFile(url, hash, hasher) + if isZip: + with zipfile.ZipFile(tempFileName) as zip: + zip.extractall(destPath) + else: + # Extract the archive + with tarfile.open(tempFileName, 'r:gz') as tgz: + tgz.extractall(destPath) + os.remove(tempFileName) diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py new file mode 100644 index 0000000000..5492109864 --- /dev/null +++ b/hifi_vcpkg.py @@ -0,0 +1,216 @@ +import hifi_utils +import hifi_android +import hashlib +import os +import platform +import re +import shutil +import tempfile +import json +import xml.etree.ElementTree as ET +import functools + +print = functools.partial(print, flush=True) + +# Encapsulates the vcpkg system +class VcpkgRepo: + CMAKE_TEMPLATE = """ +set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file") +set(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}") +set(VCPKG_INSTALL_ROOT "{}") +set(VCPKG_TOOLS_DIR "{}") +""" + + CMAKE_TEMPLATE_NON_ANDROID = """ +# If the cached cmake toolchain path is different from the computed one, exit +if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE)) + message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake") +endif() +""" + + def __init__(self, args): + self.args = args + # our custom ports, relative to the script location + self.sourcePortsPath = args.ports_path + self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8] + self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake') + + # OS dependent information + system = platform.system() + + if self.args.vcpkg_root is not None: + self.path = args.vcpkg_root + else: + if 'Darwin' == system: + defaultBasePath = os.path.expanduser('~/hifi/vcpkg') + else: + defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg') + self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath) + if self.basePath == defaultBasePath: + print("Warning: Environment variable HIFI_VCPKG_BASE not set, using {}".format(defaultBasePath)) + if self.args.android: + self.basePath = os.path.join(self.basePath, 'android') + if (not os.path.isdir(self.basePath)): + os.makedirs(self.basePath) + self.path = os.path.join(self.basePath, self.id) + + print("Using vcpkg path {}".format(self.path)) + lockDir, lockName = os.path.split(self.path) + lockName += '.lock' + if not os.path.isdir(lockDir): + os.makedirs(lockDir) + + self.lockFile = os.path.join(lockDir, lockName) + self.tagFile = os.path.join(self.path, '.id') + # A format version attached to the tag file... increment when you want to force the build systems to rebuild + # without the contents of the ports changing + self.version = 1 + self.tagContents = "{}_{}".format(self.id, self.version) + + if 'Windows' == system: + self.exe = os.path.join(self.path, 'vcpkg.exe') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ' + self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e' + self.hostTriplet = 'x64-windows' + elif 'Darwin' == system: + self.exe = os.path.join(self.path, 'vcpkg') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc' + self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d' + self.hostTriplet = 'x64-osx' + else: + self.exe = os.path.join(self.path, 'vcpkg') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU' + self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d' + self.hostTriplet = 'x64-linux' + + if self.args.android: + self.triplet = 'arm64-android' + self.androidPackagePath = os.path.join(self.path, 'android') + else: + self.triplet = self.hostTriplet + + def upToDate(self): + # Prevent doing a clean if we've explcitly set a directory for vcpkg + if self.args.vcpkg_root is not None: + return True + + if self.args.force_build: + print("Force build, out of date") + return False + if not os.path.isfile(self.exe): + print("Exe file {} not found, out of date".format(self.exe)) + return False + if not os.path.isfile(self.tagFile): + print("Tag file {} not found, out of date".format(self.tagFile)) + return False + with open(self.tagFile, 'r') as f: + storedTag = f.read() + if storedTag != self.tagContents: + print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents)) + return False + return True + + def clean(self): + print("Cleaning vcpkg installation at {}".format(self.path)) + if os.path.isdir(self.path): + print("Removing {}".format(self.path)) + shutil.rmtree(self.path, ignore_errors=True) + + # Make sure the VCPKG prerequisites are all there. + def bootstrap(self): + if self.upToDate(): + return + + self.clean() + + downloadVcpkg = False + if self.args.force_bootstrap: + print("Forcing bootstrap") + downloadVcpkg = True + + if not downloadVcpkg and not os.path.isfile(self.exe): + print("Missing executable, boostrapping") + downloadVcpkg = True + + # Make sure we have a vcpkg executable + testFile = os.path.join(self.path, '.vcpkg-root') + if not downloadVcpkg and not os.path.isfile(testFile): + print("Missing {}, bootstrapping".format(testFile)) + downloadVcpkg = True + + if downloadVcpkg: + print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) + hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) + + print("Replacing port files") + portsPath = os.path.join(self.path, 'ports') + if (os.path.islink(portsPath)): + os.unlink(portsPath) + if (os.path.isdir(portsPath)): + shutil.rmtree(portsPath, ignore_errors=True) + shutil.copytree(self.sourcePortsPath, portsPath) + + def run(self, commands): + actualCommands = [self.exe, '--vcpkg-root', self.path] + actualCommands.extend(commands) + print("Running command") + print(actualCommands) + hifi_utils.executeSubprocess(actualCommands, folder=self.path) + + def setupDependencies(self): + # Special case for android, grab a bunch of binaries + # FIXME remove special casing for android builds eventually + if self.args.android: + print("Installing Android binaries") + self.setupAndroidDependencies() + + print("Installing host tools") + self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools']) + + # If not android, install the hifi-client-deps libraries + if not self.args.android: + print("Installing build dependencies") + self.run(['install', '--triplet', self.triplet, 'hifi-client-deps']) + + def cleanBuilds(self): + # Remove temporary build artifacts + builddir = os.path.join(self.path, 'buildtrees') + if os.path.isdir(builddir): + print("Wiping build trees") + shutil.rmtree(builddir, ignore_errors=True) + + def setupAndroidDependencies(self): + # vcpkg prebuilt + if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')): + dest = os.path.join(self.path, 'installed') + url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" + # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash + #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" + #hifi_utils.downloadAndExtract(url, dest, hash) + hifi_utils.downloadAndExtract(url, dest) + + def writeTag(self): + print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) + with open(self.tagFile, 'w') as f: + f.write(self.tagContents) + + def writeConfig(self): + print("Writing cmake config to {}".format(self.configFilePath)) + # Write out the configuration for use by CMake + cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') + installPath = os.path.join(self.path, 'installed', self.triplet) + toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools') + cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE + if not self.args.android: + cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID + cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath).replace('\\', '/') + with open(self.configFilePath, 'w') as f: + f.write(cmakeConfig) + + def cleanOldBuilds(self): + # FIXME because we have the base directory, and because a build will + # update the tag file on every run, we can scan the base dir for sub directories containing + # a tag file that is older than N days, and if found, delete the directory, recovering space + print("Not implemented") + + diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index e9a44b1e87..bbbd8db89f 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -802,7 +802,7 @@ Menu::Menu() { connect(action, &QAction::triggered, qApp, []() { std::thread(crash::newFault).join(); }); // Developer > Show Statistics - addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats); + addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats, 0, true); // Developer > Show Animation Statistics addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::AnimStats); diff --git a/prebuild.py b/prebuild.py index b8cb2c96ae..a758dcbea2 100644 --- a/prebuild.py +++ b/prebuild.py @@ -1,5 +1,26 @@ #!python +# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring +# tools required by the build as well as dependencies on which we rely. +# +# By using this script, we can reduce the requirements for a developer getting started to: +# +# * A working C++ dev environment like visual studio, xcode, gcc, or clang +# * Qt +# * CMake +# * Python 3.x +# +# The function of the build script is to acquire, if not already present, all the other build requirements +# The build script should be idempotent. If you run it with the same arguments multiple times, that should +# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not +# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should +# execute quickly, determining that no work is to be done + +import hifi_singleton +import hifi_utils +import hifi_android +import hifi_vcpkg + import argparse import concurrent import hashlib @@ -9,252 +30,65 @@ import os import platform import shutil import ssl -import subprocess import sys -import tarfile +import re import tempfile import time -import urllib.request import functools print = functools.partial(print, flush=True) -def executeSubprocess(processArgs, folder=None, env=None): - restoreDir = None - if folder != None: - restoreDir = os.getcwd() - os.chdir(folder) - - process = subprocess.Popen( - processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env) - process.wait() - - if (0 != process.returncode): - raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format( - processArgs[0], - ' '.join(processArgs[1:]), - )) - - if restoreDir != None: - os.chdir(restoreDir) - - -def hashFile(file): - hasher = hashlib.sha512() - with open(file, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def hashFolder(folder): - hasher = hashlib.sha256() - for dirName, subdirList, fileList in os.walk(folder): - for fname in fileList: - with open(os.path.join(folder, dirName, fname), "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def downloadAndExtract(url, destPath, hash=None): - tempFileDescriptor, tempFileName = tempfile.mkstemp() - # OSX Python doesn't support SSL, so we need to bypass it. - # However, we still validate the downloaded file's sha512 hash - context = ssl._create_unverified_context() - with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile: - shutil.copyfileobj(response, tempFile) - - # Verify the hash - if hash and hash != hashFile(tempFileName): - raise RuntimeError("Downloaded file does not match hash") - - # Extract the archive - with tarfile.open(tempFileName, 'r:gz') as tgz: - tgz.extractall(destPath) - os.remove(tempFileName) - - -class VcpkgRepo: - def __init__(self): - global args - scriptPath = os.path.dirname(os.path.realpath(sys.argv[0])) - # our custom ports, relative to the script location - self.sourcePortsPath = os.path.join(scriptPath, 'cmake', 'ports') - # FIXME Revert to ports hash before release - self.id = hashFolder(self.sourcePortsPath)[:8] - # OS dependent information - system = platform.system() - - if args.vcpkg_root is not None: - print("override vcpkg path with " + args.vcpkg_root) - self.path = args.vcpkg_root - else: - if 'Darwin' == system: - defaultBasePath = os.path.expanduser('~/hifi/vcpkg') - else: - defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg') - basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath) - if (not os.path.isdir(basePath)): - os.makedirs(basePath) - self.path = os.path.join(basePath, self.id) - - self.tagFile = os.path.join(self.path, '.id') - # A format version attached to the tag file... increment when you want to force the build systems to rebuild - # without the contents of the ports changing - self.version = 1 - self.tagContents = "{}_{}".format(self.id, self.version) - - print("prebuild path: " + self.path) - if 'Windows' == system: - self.exe = os.path.join(self.path, 'vcpkg.exe') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ' - self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e' - self.hostTriplet = 'x64-windows' - elif 'Darwin' == system: - self.exe = os.path.join(self.path, 'vcpkg') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc' - self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d' - self.hostTriplet = 'x64-osx' - else: - self.exe = os.path.join(self.path, 'vcpkg') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU' - self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d' - self.hostTriplet = 'x64-linux' - - if args.android: - self.triplet = 'arm64-android' - else: - self.triplet = self.hostTriplet - - def outOfDate(self): - global args - # Prevent doing a clean if we've explcitly set a directory for vcpkg - if args.vcpkg_root is not None: - return False - if args.force_build: - return True - print("Looking for tag file {}".format(self.tagFile)) - if not os.path.isfile(self.tagFile): - return True - with open(self.tagFile, 'r') as f: - storedTag = f.read() - print("Found stored tag {}".format(storedTag)) - if storedTag != self.tagContents: - print("Doesn't match computed tag {}".format(self.tagContents)) - return True - return False - - def clean(self): - cleanPath = self.path - print("Cleaning vcpkg installation at {}".format(cleanPath)) - if os.path.isdir(self.path): - print("Removing {}".format(cleanPath)) - shutil.rmtree(cleanPath, ignore_errors=True) - - def bootstrap(self): - global args - if self.outOfDate(): - self.clean() - - # don't download the vcpkg binaries if we're working with an explicit - # vcpkg directory (possibly a git checkout) - if args.vcpkg_root is None: - downloadVcpkg = False - if args.force_bootstrap: - print("Forcing bootstrap") - downloadVcpkg = True - - if not downloadVcpkg and not os.path.isfile(self.exe): - print("Missing executable, boostrapping") - downloadVcpkg = True - - # Make sure we have a vcpkg executable - testFile = os.path.join(self.path, '.vcpkg-root') - if not downloadVcpkg and not os.path.isfile(testFile): - print("Missing {}, bootstrapping".format(testFile)) - downloadVcpkg = True - - if downloadVcpkg: - print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) - downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) - - print("Replacing port files") - portsPath = os.path.join(self.path, 'ports') - if (os.path.islink(portsPath)): - os.unlink(portsPath) - if (os.path.isdir(portsPath)): - shutil.rmtree(portsPath, ignore_errors=True) - shutil.copytree(self.sourcePortsPath, portsPath) - - def downloadAndroidDependencies(self): - url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" - hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" - dest = os.path.join(self.path, 'installed') - downloadAndExtract(url, dest, hash) - - def run(self, commands): - actualCommands = [self.exe, '--vcpkg-root', self.path] - actualCommands.extend(commands) - print("Running command") - print(actualCommands) - executeSubprocess(actualCommands, folder=self.path) - - def buildDependencies(self): - global args - print("Installing host tools") - self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools']) - # Special case for android, grab a bunch of binaries - if args.android: - self.downloadAndroidDependencies() - return - - print("Installing build dependencies") - self.run(['install', '--triplet', self.triplet, 'hifi-client-deps']) - # Remove temporary build artifacts - builddir = os.path.join(self.path, 'buildtrees') - if os.path.isdir(builddir): - print("Wiping build trees") - shutil.rmtree(builddir, ignore_errors=True) - - def writeConfig(self): - global args - configFilePath = os.path.join(args.build_root, 'vcpkg.cmake') - print("Writing cmake config to {}".format(configFilePath)) - # Write out the configuration for use by CMake - cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') - installPath = os.path.join(self.path, 'installed', self.triplet) - toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools') - cmakeTemplate = 'set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")\n' - cmakeTemplate += 'set(VCPKG_INSTALL_ROOT "{}" CACHE FILEPATH "vcpkg installed packages path")\n' - cmakeTemplate += 'set(VCPKG_TOOLS_DIR "{}" CACHE FILEPATH "vcpkg installed packages path")\n' - cmakeConfig = cmakeTemplate.format(cmakeScript, installPath, toolsPath).replace('\\', '/') - with open(configFilePath, 'w') as f: - f.write(cmakeConfig) - - def writeTag(self): - print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) - with open(self.tagFile, 'w') as f: - f.write(self.tagContents) +def parse_args(): + # our custom ports, relative to the script location + defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports') + from argparse import ArgumentParser + parser = ArgumentParser(description='Prepare build dependencies.') + parser.add_argument('--android', action='store_true') + #parser.add_argument('--android', type=str) + parser.add_argument('--debug', action='store_true') + parser.add_argument('--force-bootstrap', action='store_true') + parser.add_argument('--force-build', action='store_true') + parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') + parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') + parser.add_argument('--ports-path', type=str, default=defaultPortsPath) + if True: + args = parser.parse_args() + else: + args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/hifi/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a']) + return args def main(): - vcpkg = VcpkgRepo() - vcpkg.bootstrap() - vcpkg.buildDependencies() - vcpkg.writeConfig() - vcpkg.writeTag() + # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build + # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for + # building OpenSSL + removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE'] + for var in removeEnvVars: + if var in os.environ: + del os.environ[var] + args = parse_args() + # Only allow one instance of the program to run at a time + pm = hifi_vcpkg.VcpkgRepo(args) + with hifi_singleton.Singleton(pm.lockFile) as lock: + if not pm.upToDate(): + pm.bootstrap() + # Always write the tag, even if we changed nothing. This + # allows vcpkg to reclaim disk space by identifying directories with + # tags that haven't been touched in a long time + pm.writeTag() -from argparse import ArgumentParser -parser = ArgumentParser(description='Prepare build dependencies.') -parser.add_argument('--android', action='store_true') -parser.add_argument('--debug', action='store_true') -parser.add_argument('--force-bootstrap', action='store_true') -parser.add_argument('--force-build', action='store_true') -parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') -parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') + # Grab our required dependencies: + # * build host tools, like spirv-cross and scribe + # * build client dependencies like openssl and nvtt + pm.setupDependencies() -args = parser.parse_args() + # wipe out the build directories (after writing the tag, since failure + # here shouldn't invalidte the vcpkg install) + pm.cleanBuilds() + + # Write the vcpkg config to the build directory last + pm.writeConfig() + +print(sys.argv) main() -