mirror of
https://github.com/HifiExperiments/overte.git
synced 2025-04-05 21:43:07 +02:00
Dockerized build, updated build script, ndk18
This commit is contained in:
Brad Davis
parent
79d89d823e
commit
8504619067
10 changed files with 848 additions and 245 deletions
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -17,10 +17,11 @@ Makefile
|
|||
local.properties
|
||||
android/gradle*
|
||||
android/.gradle
|
||||
android/app/src/main/jniLibs
|
||||
android/app/libs
|
||||
android/app/src/main/res/values/libs.xml
|
||||
android/app/src/main/assets/bundled
|
||||
android/**/src/main/jniLibs
|
||||
android/**/libs
|
||||
android/**/src/main/res/values/libs.xml
|
||||
android/**/src/main/assets
|
||||
android/**/gradle*
|
||||
|
||||
# VSCode
|
||||
# List taken from Github Global Ignores master@435c4d92
|
||||
|
|
@ -83,9 +84,6 @@ npm-debug.log
|
|||
# Android studio files
|
||||
*___jb_old___
|
||||
|
||||
# Generated assets for Android
|
||||
android/app/src/main/assets
|
||||
|
||||
# Resource binary file
|
||||
interface/compiledResources
|
||||
|
||||
|
|
@ -95,6 +93,9 @@ interface/resources/GPUCache/*
|
|||
# package lock file for JSDoc tool
|
||||
tools/jsdoc/package-lock.json
|
||||
|
||||
# Python compile artifacts
|
||||
**/__pycache__
|
||||
|
||||
# ignore unneeded unity project files for avatar exporter
|
||||
tools/unity-avatar-exporter/Library
|
||||
tools/unity-avatar-exporter/Packages
|
||||
|
|
|
|||
92
android/Dockerfile
Normal file
92
android/Dockerfile
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
FROM openjdk:8
|
||||
|
||||
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
|
||||
|
||||
RUN apt-get update && apt-get -y install \
|
||||
curl \
|
||||
gnupg \
|
||||
software-properties-common \
|
||||
unzip \
|
||||
-
|
||||
|
||||
# --- Versions and Download paths
|
||||
ENV ANDROID_HOME="/usr/local/android-sdk" \
|
||||
ANDROID_NDK_HOME="/usr/local/android-ndk" \
|
||||
ANDROID_SDK_HOME="/usr/local/android-sdk-home" \
|
||||
ANDROID_VERSION=26 \
|
||||
ANDROID_BUILD_TOOLS_VERSION=28.0.3 \
|
||||
ANDROID_NDK_VERSION=r18
|
||||
|
||||
ENV SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip" \
|
||||
NDK_URL="https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
|
||||
|
||||
# --- Android SDK
|
||||
RUN mkdir -p "$ANDROID_HOME" "$ANDROID_SDK_HOME" && \
|
||||
cd "$ANDROID_HOME" && \
|
||||
curl -s -S -o sdk.zip -L "${SDK_URL}" && \
|
||||
unzip sdk.zip && \
|
||||
rm sdk.zip && \
|
||||
yes | $ANDROID_HOME/tools/bin/sdkmanager --licenses
|
||||
|
||||
# Install Android Build Tool and Libraries
|
||||
RUN $ANDROID_HOME/tools/bin/sdkmanager --update
|
||||
RUN $ANDROID_HOME/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" \
|
||||
"platforms;android-${ANDROID_VERSION}" \
|
||||
"platform-tools"
|
||||
|
||||
RUN chmod -R a+w "${ANDROID_HOME}"
|
||||
RUN chmod -R a+w "${ANDROID_SDK_HOME}"
|
||||
|
||||
# --- Android NDK
|
||||
# download
|
||||
RUN mkdir /usr/local/android-ndk-tmp && \
|
||||
cd /usr/local/android-ndk-tmp && \
|
||||
curl -s -S -o ndk.zip -L "${NDK_URL}" && \
|
||||
unzip -q ndk.zip && \
|
||||
mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \
|
||||
cd ${ANDROID_NDK_HOME} && \
|
||||
rm -rf /usr/local/android-ndk-tmp
|
||||
|
||||
ENV PATH ${PATH}:${ANDROID_NDK_HOME}
|
||||
|
||||
RUN apt-get -y install \
|
||||
g++ \
|
||||
gcc \
|
||||
-
|
||||
|
||||
# --- Gradle
|
||||
ARG BUILD_UID=1001
|
||||
RUN useradd -ms /bin/bash -u $BUILD_UID jenkins
|
||||
USER jenkins
|
||||
WORKDIR /home/jenkins
|
||||
|
||||
# Hifi dependencies
|
||||
ENV HIFI_BASE="/home/jenkins/hifi_android"
|
||||
ENV HIFI_ANDROID_PRECOMPILED="$HIFI_BASE/dependencies"
|
||||
ENV HIFI_VCPKG_BASE="$HIFI_BASE/vcpkg"
|
||||
|
||||
RUN mkdir "$HIFI_BASE" && \
|
||||
mkdir "$HIFI_VCPKG_BASE" && \
|
||||
mkdir "$HIFI_ANDROID_PRECOMPILED"
|
||||
|
||||
RUN git clone https://github.com/jherico/hifi.git && \
|
||||
cd ~/hifi && \
|
||||
git checkout feature/build/gradle-wrapper
|
||||
|
||||
|
||||
WORKDIR /home/jenkins/hifi
|
||||
|
||||
RUN touch .test4 && \
|
||||
git fetch && git reset origin/feature/build/gradle-wrapper --hard
|
||||
|
||||
RUN mkdir build
|
||||
|
||||
# Pre-cache the vcpkg managed dependencies
|
||||
WORKDIR /home/jenkins/hifi/build
|
||||
RUN python3 ../prebuild.py --build-root `pwd` --android
|
||||
|
||||
# Pre-cache the gradle dependencies
|
||||
WORKDIR /home/jenkins/hifi/android
|
||||
RUN ./gradlew -m tasks -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED
|
||||
RUN ./gradlew extractDependencies -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED
|
||||
|
||||
|
|
@ -392,7 +392,7 @@ task extractDependencies(dependsOn: verifyDependencyDownloads) {
|
|||
}
|
||||
|
||||
// Copies the non Qt dependencies. Qt dependencies (primary libraries and plugins) are handled by the qtBundle task
|
||||
task copyDependencies(dependsOn: [ extractDependencies ]) {
|
||||
task copyDependencies() {
|
||||
doLast {
|
||||
packages.each { entry ->
|
||||
def packageName = entry.key
|
||||
|
|
@ -414,7 +414,7 @@ task copyDependencies(dependsOn: [ extractDependencies ]) {
|
|||
}
|
||||
}
|
||||
|
||||
task extractGvrBinaries(dependsOn: extractDependencies) {
|
||||
task extractGvrBinaries() {
|
||||
doLast {
|
||||
def gvrLibFolder = new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries');
|
||||
zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-audio-1.101.0.aar')).visit { element ->
|
||||
|
|
|
|||
4
android/build_android.sh
Executable file
4
android/build_android.sh
Executable file
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
set -xeuo pipefail
|
||||
./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} setupDependencies
|
||||
./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} app:${ANDROID_BUILD_TARGET}
|
||||
286
hifi_android.py
Normal file
286
hifi_android.py
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
import hifi_utils
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import xml.etree.ElementTree as ET
|
||||
import functools
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
|
||||
ANDROID_PACKAGE_URL = 'https://hifi-public.s3.amazonaws.com/dependencies/android/'
|
||||
|
||||
ANDROID_PACKAGES = {
|
||||
'qt' : {
|
||||
'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz',
|
||||
'versionId': '3S97HBM5G5Xw9EfE52sikmgdN3t6C2MN',
|
||||
'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df',
|
||||
},
|
||||
'bullet': {
|
||||
'file': 'bullet-2.88_armv8-libcpp.tgz',
|
||||
'versionId': 'S8YaoED0Cl8sSb8fSV7Q2G1lQJSNDxqg',
|
||||
'checksum': '81642779ccb110f8c7338e8739ac38a0',
|
||||
},
|
||||
'draco': {
|
||||
'file': 'draco_armv8-libcpp.tgz',
|
||||
'versionId': '3.B.uBj31kWlgND3_R2xwQzT_TP6Dz_8',
|
||||
'checksum': '617a80d213a5ec69fbfa21a1f2f738cd',
|
||||
},
|
||||
'glad': {
|
||||
'file': 'glad_armv8-libcpp.zip',
|
||||
'versionId': 'r5Zran.JSCtvrrB6Q4KaqfIoALPw3lYY',
|
||||
'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449',
|
||||
},
|
||||
'gvr': {
|
||||
'file': 'gvrsdk_v1.101.0.tgz',
|
||||
'versionId': 'nqBV_j81Uc31rC7bKIrlya_Hah4v3y5r',
|
||||
'checksum': '57fd02baa069176ba18597a29b6b4fc7',
|
||||
},
|
||||
'nvtt': {
|
||||
'file': 'nvtt_armv8-libcpp.zip',
|
||||
'versionId': 'lmkBVR5t4UF1UUwMwEirnk9H_8Nt90IO',
|
||||
'checksum': 'eb46d0b683e66987190ed124aabf8910',
|
||||
'sharedLibFolder': 'lib',
|
||||
'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so']
|
||||
},
|
||||
'oculus': {
|
||||
'file': 'ovr_sdk_mobile_1.19.0.zip',
|
||||
'versionId': 's_RN1vlEvUi3pnT7WPxUC4pQ0RJBs27y',
|
||||
'checksum': '98f0afb62861f1f02dd8110b31ed30eb',
|
||||
'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release',
|
||||
'includeLibs': ['libvrapi.so']
|
||||
},
|
||||
'openssl': {
|
||||
'file': 'openssl-1.1.0g_armv8.tgz',
|
||||
'versionId': 'AiiPjmgUZTgNj7YV1EEx2lL47aDvvvAW',
|
||||
'checksum': 'cabb681fbccd79594f65fcc266e02f32'
|
||||
},
|
||||
'polyvox': {
|
||||
'file': 'polyvox_armv8-libcpp.tgz',
|
||||
'versionId': 'A2kbKiNhpIenGq23bKRRzg7IMAI5BI92',
|
||||
'checksum': 'dba88b3a098747af4bb169e9eb9af57e',
|
||||
'sharedLibFolder': 'lib',
|
||||
'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'],
|
||||
},
|
||||
'tbb': {
|
||||
'file': 'tbb-2018_U1_armv8_libcpp.tgz',
|
||||
'versionId': 'mrRbWnv4O4evcM1quRH43RJqimlRtaKB',
|
||||
'checksum': '20768f298f53b195e71b414b0ae240c4',
|
||||
'sharedLibFolder': 'lib/release',
|
||||
'includeLibs': ['libtbb.so', 'libtbbmalloc.so'],
|
||||
},
|
||||
'hifiAC': {
|
||||
'baseUrl': 'http://s3.amazonaws.com/hifi-public/dependencies/',
|
||||
'file': 'codecSDK-android_armv8-2.0.zip',
|
||||
'checksum': '1cbef929675818fc64c4101b72f84a6a'
|
||||
},
|
||||
'etc2comp': {
|
||||
'file': 'etc2comp-patched-armv8-libcpp.tgz',
|
||||
'versionId': 'bHhGECRAQR1vkpshBcK6ByNc1BQIM8gU',
|
||||
'checksum': '14b02795d774457a33bbc60e00a786bc'
|
||||
},
|
||||
'breakpad': {
|
||||
'file': 'breakpad.tgz',
|
||||
'versionId': '8VrYXz7oyc.QBxNia0BVJOUBvrFO61jI',
|
||||
'checksum': 'ddcb23df336b08017042ba4786db1d9e',
|
||||
'sharedLibFolder': 'lib',
|
||||
'includeLibs': {'libbreakpad_client.a'}
|
||||
}
|
||||
}
|
||||
|
||||
ANDROID_PLATFORM_PACKAGES = {
|
||||
'Darwin' : {
|
||||
'qt': {
|
||||
'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz',
|
||||
'versionId': 'OxBD7iKINv1HbyOXmAmDrBb8AF3N.Kup',
|
||||
'checksum': 'c83cc477c08a892e00c71764dca051a0'
|
||||
},
|
||||
},
|
||||
'Windows' : {
|
||||
'qt': {
|
||||
'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz',
|
||||
'versionId': 'JfWM0P_Mz5Qp0LwpzhrsRwN3fqlLSFeT',
|
||||
'checksum': '0582191cc55431aa4f660848a542883e'
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
QT5_DEPS = [
|
||||
'Qt5Concurrent',
|
||||
'Qt5Core',
|
||||
'Qt5Gui',
|
||||
'Qt5Multimedia',
|
||||
'Qt5Network',
|
||||
'Qt5OpenGL',
|
||||
'Qt5Qml',
|
||||
'Qt5Quick',
|
||||
'Qt5QuickControls2',
|
||||
'Qt5QuickTemplates2',
|
||||
'Qt5Script',
|
||||
'Qt5ScriptTools',
|
||||
'Qt5Svg',
|
||||
'Qt5WebChannel',
|
||||
'Qt5WebSockets',
|
||||
'Qt5Widgets',
|
||||
'Qt5XmlPatterns',
|
||||
# Android specific
|
||||
'Qt5AndroidExtras',
|
||||
'Qt5WebView',
|
||||
]
|
||||
|
||||
def getPlatformPackages():
|
||||
result = ANDROID_PACKAGES.copy()
|
||||
system = platform.system()
|
||||
if system in ANDROID_PLATFORM_PACKAGES:
|
||||
platformPackages = ANDROID_PLATFORM_PACKAGES[system]
|
||||
result = { **result, **platformPackages }
|
||||
return result
|
||||
|
||||
def getPackageUrl(package):
|
||||
url = ANDROID_PACKAGE_URL
|
||||
if 'baseUrl' in package:
|
||||
url = package['baseUrl']
|
||||
url += package['file']
|
||||
if 'versionId' in package:
|
||||
url += '?versionId=' + package['versionId']
|
||||
return url
|
||||
|
||||
def copyAndroidLibs(packagePath, appPath):
|
||||
androidPackages = getPlatformPackages()
|
||||
jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a')
|
||||
if not os.path.isdir(jniPath):
|
||||
os.makedirs(jniPath)
|
||||
for packageName in androidPackages:
|
||||
package = androidPackages[packageName]
|
||||
if 'sharedLibFolder' in package:
|
||||
sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder'])
|
||||
if 'includeLibs' in package:
|
||||
for lib in package['includeLibs']:
|
||||
sourceFile = os.path.join(sharedLibFolder, lib)
|
||||
destFile = os.path.join(jniPath, os.path.split(lib)[1])
|
||||
if not os.path.isfile(destFile):
|
||||
print("Copying {}".format(lib))
|
||||
shutil.copy(sourceFile, destFile)
|
||||
|
||||
class QtPackager:
|
||||
def __init__(self, appPath, qtRootPath):
|
||||
self.appPath = appPath
|
||||
self.qtRootPath = qtRootPath
|
||||
self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a')
|
||||
self.assetPath = os.path.join(self.appPath, 'src/main/assets')
|
||||
self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--')
|
||||
# Jars go into the qt library
|
||||
self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs'))
|
||||
self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml')
|
||||
self.files = []
|
||||
self.features = []
|
||||
self.permissions = []
|
||||
|
||||
def copyQtDeps(self):
|
||||
for lib in QT5_DEPS:
|
||||
libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib))
|
||||
if not os.path.exists(libfile):
|
||||
continue
|
||||
self.files.append(libfile)
|
||||
androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib))
|
||||
if not os.path.exists(androidDeps):
|
||||
continue
|
||||
|
||||
tree = ET.parse(androidDeps)
|
||||
root = tree.getroot()
|
||||
for item in root.findall('./dependencies/lib/depends/*'):
|
||||
if (item.tag == 'lib') or (item.tag == 'bundled'):
|
||||
relativeFilename = item.attrib['file']
|
||||
if (relativeFilename.startswith('qml')):
|
||||
continue
|
||||
filename = os.path.join(self.qtRootPath, relativeFilename)
|
||||
self.files.extend(hifi_utils.recursiveFileList(filename))
|
||||
elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1":
|
||||
self.files.append(os.path.join(self.qtRootPath, item.attrib['file']))
|
||||
elif item.tag == 'permission':
|
||||
self.permissions.append(item.attrib['name'])
|
||||
elif item.tag == 'feature':
|
||||
self.features.append(item.attrib['name'])
|
||||
|
||||
def scanQmlImports(self):
|
||||
qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner')
|
||||
system = platform.system()
|
||||
if 'Windows' == system:
|
||||
qmlImportCommandFile += ".exe"
|
||||
if not os.path.isfile(qmlImportCommandFile):
|
||||
raise RuntimeError("Couldn't find qml import scanner")
|
||||
qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml')
|
||||
qmlImportPath = os.path.join(self.qtRootPath, 'qml')
|
||||
commandResult = hifi_utils.executeSubprocessCapture([
|
||||
qmlImportCommandFile,
|
||||
'-rootPath', qmlRootPath,
|
||||
'-importPath', qmlImportPath
|
||||
])
|
||||
qmlImportResults = json.loads(commandResult)
|
||||
for item in qmlImportResults:
|
||||
if 'path' not in item:
|
||||
print("Warning: QML import could not be resolved in any of the import paths: {}".format(item['name']))
|
||||
continue
|
||||
path = os.path.realpath(item['path'])
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
basePath = path
|
||||
if os.path.isfile(basePath):
|
||||
basePath = os.path.dirname(basePath)
|
||||
basePath = os.path.normcase(basePath)
|
||||
if basePath.startswith(qmlRootPath):
|
||||
continue
|
||||
self.files.extend(hifi_utils.recursiveFileList(path))
|
||||
|
||||
def processFiles(self):
|
||||
self.files = list(set(self.files))
|
||||
self.files.sort()
|
||||
libsXmlRoot = ET.Element('resources')
|
||||
qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'})
|
||||
bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'})
|
||||
bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'})
|
||||
libPrefix = 'lib'
|
||||
for sourceFile in self.files:
|
||||
if not os.path.isfile(sourceFile):
|
||||
raise RuntimeError("Unable to find dependency file " + sourceFile)
|
||||
relativePath = os.path.relpath(sourceFile, self.qtRootPath)
|
||||
destinationFile = None
|
||||
if relativePath.endswith('.so'):
|
||||
garbledFileName = None
|
||||
if relativePath.startswith(libPrefix):
|
||||
garbledFileName = relativePath[4:]
|
||||
p = re.compile(r'lib(Qt5.*).so')
|
||||
m = p.search(garbledFileName)
|
||||
if not m:
|
||||
raise RuntimeError("Huh?")
|
||||
libName = m.group(1)
|
||||
ET.SubElement(qtLibsNode, 'item').text = libName
|
||||
else:
|
||||
garbledFileName = 'lib' + relativePath.replace('\\', '_'[0])
|
||||
value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/')
|
||||
ET.SubElement(bundledLibsNode, 'item').text = value
|
||||
destinationFile = os.path.join(self.jniPath, garbledFileName)
|
||||
elif relativePath.startswith('jar'):
|
||||
destinationFile = os.path.join(self.jarPath, relativePath[4:])
|
||||
else:
|
||||
value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/')
|
||||
ET.SubElement(bundledAssetsNode, 'item').text = value
|
||||
destinationFile = os.path.join(self.qtAssetPath, relativePath)
|
||||
|
||||
destinationParent = os.path.realpath(os.path.dirname(destinationFile))
|
||||
if not os.path.isdir(destinationParent):
|
||||
os.makedirs(destinationParent)
|
||||
if not os.path.isfile(destinationFile):
|
||||
shutil.copy(sourceFile, destinationFile)
|
||||
|
||||
tree = ET.ElementTree(libsXmlRoot)
|
||||
tree.write(self.xmlFile, 'UTF-8', True)
|
||||
|
||||
def bundle(self):
|
||||
if not os.path.isfile(self.xmlFile) or True:
|
||||
self.copyQtDeps()
|
||||
self.scanQmlImports()
|
||||
self.processFiles()
|
||||
|
||||
|
||||
46
hifi_singleton.py
Normal file
46
hifi_singleton.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import os
|
||||
import platform
|
||||
import time
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError:
|
||||
fcntl = None
|
||||
|
||||
# Used to ensure only one instance of the script runs at a time
|
||||
class Singleton:
|
||||
def __init__(self, path):
|
||||
self.fh = None
|
||||
self.windows = 'Windows' == platform.system()
|
||||
self.path = path
|
||||
|
||||
def __enter__(self):
|
||||
success = False
|
||||
while not success:
|
||||
try:
|
||||
if self.windows:
|
||||
if os.path.exists(self.path):
|
||||
os.unlink(self.path)
|
||||
self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
|
||||
else:
|
||||
self.fh = open(self.path, 'x')
|
||||
fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
success = True
|
||||
except EnvironmentError as err:
|
||||
if self.fh is not None:
|
||||
if self.windows:
|
||||
os.close(self.fh)
|
||||
else:
|
||||
self.fh.close()
|
||||
self.fh = None
|
||||
print("Couldn't aquire lock, retrying in 10 seconds")
|
||||
time.sleep(10)
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
if self.windows:
|
||||
os.close(self.fh)
|
||||
else:
|
||||
fcntl.lockf(self.fh, fcntl.LOCK_UN)
|
||||
self.fh.close()
|
||||
os.unlink(self.path)
|
||||
124
hifi_utils.py
Normal file
124
hifi_utils.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import os
|
||||
import hashlib
|
||||
import platform
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib
|
||||
import urllib.request
|
||||
import zipfile
|
||||
import tempfile
|
||||
import time
|
||||
import functools
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
|
||||
def scriptRelative(*paths):
|
||||
scriptdir = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
result = os.path.join(scriptdir, *paths)
|
||||
result = os.path.realpath(result)
|
||||
result = os.path.normcase(result)
|
||||
return result
|
||||
|
||||
|
||||
def recursiveFileList(startPath):
|
||||
result = []
|
||||
if os.path.isfile(startPath):
|
||||
result.append(startPath)
|
||||
elif os.path.isdir(startPath):
|
||||
for dirName, subdirList, fileList in os.walk(startPath):
|
||||
for fname in fileList:
|
||||
result.append(os.path.realpath(os.path.join(startPath, dirName, fname)))
|
||||
result.sort()
|
||||
return result
|
||||
|
||||
|
||||
def executeSubprocessCapture(processArgs):
|
||||
processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if (0 != processResult.returncode):
|
||||
raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format(
|
||||
processArgs[0],
|
||||
' '.join(processArgs[1:]),
|
||||
processResult.stdout.decode('utf-8'),
|
||||
processResult.stderr.decode('utf-8')))
|
||||
return processResult.stdout.decode('utf-8')
|
||||
|
||||
def executeSubprocess(processArgs, folder=None, env=None):
|
||||
restoreDir = None
|
||||
if folder != None:
|
||||
restoreDir = os.getcwd()
|
||||
os.chdir(folder)
|
||||
|
||||
process = subprocess.Popen(
|
||||
processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env)
|
||||
process.wait()
|
||||
|
||||
if (0 != process.returncode):
|
||||
raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format(
|
||||
processArgs[0],
|
||||
' '.join(processArgs[1:]),
|
||||
))
|
||||
|
||||
if restoreDir != None:
|
||||
os.chdir(restoreDir)
|
||||
|
||||
|
||||
def hashFile(file, hasher = hashlib.sha512()):
|
||||
with open(file, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
# Assumes input files are in deterministic order
|
||||
def hashFiles(filenames):
|
||||
hasher = hashlib.sha256()
|
||||
for filename in filenames:
|
||||
with open(filename, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
def hashFolder(folder):
|
||||
filenames = recursiveFileList(folder)
|
||||
return hashFiles(filenames)
|
||||
|
||||
def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3):
|
||||
for i in range(retries):
|
||||
tempFileName = None
|
||||
# OSX Python doesn't support SSL, so we need to bypass it.
|
||||
# However, we still validate the downloaded file's sha512 hash
|
||||
if 'Darwin' == platform.system():
|
||||
tempFileDescriptor, tempFileName = tempfile.mkstemp()
|
||||
context = ssl._create_unverified_context()
|
||||
with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile:
|
||||
shutil.copyfileobj(response, tempFile)
|
||||
else:
|
||||
tempFileName, headers = urllib.request.urlretrieve(url)
|
||||
|
||||
# for some reason the hash we get back from the downloaded file is sometimes wrong if we check it right away
|
||||
# but if we examine the file later, it is correct.
|
||||
time.sleep(3)
|
||||
downloadHash = hashFile(tempFileName, hasher)
|
||||
# Verify the hash
|
||||
if hash is not None and hash != downloadHash:
|
||||
print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url))
|
||||
os.remove(tempFileName)
|
||||
continue
|
||||
|
||||
return tempFileName
|
||||
|
||||
raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url))
|
||||
|
||||
|
||||
def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False):
|
||||
tempFileName = downloadFile(url, hash, hasher)
|
||||
if isZip:
|
||||
with zipfile.ZipFile(tempFileName) as zip:
|
||||
zip.extractall(destPath)
|
||||
else:
|
||||
# Extract the archive
|
||||
with tarfile.open(tempFileName, 'r:gz') as tgz:
|
||||
tgz.extractall(destPath)
|
||||
os.remove(tempFileName)
|
||||
216
hifi_vcpkg.py
Normal file
216
hifi_vcpkg.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
import hifi_utils
|
||||
import hifi_android
|
||||
import hashlib
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import json
|
||||
import xml.etree.ElementTree as ET
|
||||
import functools
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
|
||||
# Encapsulates the vcpkg system
|
||||
class VcpkgRepo:
|
||||
CMAKE_TEMPLATE = """
|
||||
set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")
|
||||
set(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}")
|
||||
set(VCPKG_INSTALL_ROOT "{}")
|
||||
set(VCPKG_TOOLS_DIR "{}")
|
||||
"""
|
||||
|
||||
CMAKE_TEMPLATE_NON_ANDROID = """
|
||||
# If the cached cmake toolchain path is different from the computed one, exit
|
||||
if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE))
|
||||
message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake")
|
||||
endif()
|
||||
"""
|
||||
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
# our custom ports, relative to the script location
|
||||
self.sourcePortsPath = args.ports_path
|
||||
self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8]
|
||||
self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake')
|
||||
|
||||
# OS dependent information
|
||||
system = platform.system()
|
||||
|
||||
if self.args.vcpkg_root is not None:
|
||||
self.path = args.vcpkg_root
|
||||
else:
|
||||
if 'Darwin' == system:
|
||||
defaultBasePath = os.path.expanduser('~/hifi/vcpkg')
|
||||
else:
|
||||
defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg')
|
||||
self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath)
|
||||
if self.basePath == defaultBasePath:
|
||||
print("Warning: Environment variable HIFI_VCPKG_BASE not set, using {}".format(defaultBasePath))
|
||||
if self.args.android:
|
||||
self.basePath = os.path.join(self.basePath, 'android')
|
||||
if (not os.path.isdir(self.basePath)):
|
||||
os.makedirs(self.basePath)
|
||||
self.path = os.path.join(self.basePath, self.id)
|
||||
|
||||
print("Using vcpkg path {}".format(self.path))
|
||||
lockDir, lockName = os.path.split(self.path)
|
||||
lockName += '.lock'
|
||||
if not os.path.isdir(lockDir):
|
||||
os.makedirs(lockDir)
|
||||
|
||||
self.lockFile = os.path.join(lockDir, lockName)
|
||||
self.tagFile = os.path.join(self.path, '.id')
|
||||
# A format version attached to the tag file... increment when you want to force the build systems to rebuild
|
||||
# without the contents of the ports changing
|
||||
self.version = 1
|
||||
self.tagContents = "{}_{}".format(self.id, self.version)
|
||||
|
||||
if 'Windows' == system:
|
||||
self.exe = os.path.join(self.path, 'vcpkg.exe')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ'
|
||||
self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e'
|
||||
self.hostTriplet = 'x64-windows'
|
||||
elif 'Darwin' == system:
|
||||
self.exe = os.path.join(self.path, 'vcpkg')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc'
|
||||
self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d'
|
||||
self.hostTriplet = 'x64-osx'
|
||||
else:
|
||||
self.exe = os.path.join(self.path, 'vcpkg')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU'
|
||||
self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d'
|
||||
self.hostTriplet = 'x64-linux'
|
||||
|
||||
if self.args.android:
|
||||
self.triplet = 'arm64-android'
|
||||
self.androidPackagePath = os.path.join(self.path, 'android')
|
||||
else:
|
||||
self.triplet = self.hostTriplet
|
||||
|
||||
def upToDate(self):
|
||||
# Prevent doing a clean if we've explcitly set a directory for vcpkg
|
||||
if self.args.vcpkg_root is not None:
|
||||
return True
|
||||
|
||||
if self.args.force_build:
|
||||
print("Force build, out of date")
|
||||
return False
|
||||
if not os.path.isfile(self.exe):
|
||||
print("Exe file {} not found, out of date".format(self.exe))
|
||||
return False
|
||||
if not os.path.isfile(self.tagFile):
|
||||
print("Tag file {} not found, out of date".format(self.tagFile))
|
||||
return False
|
||||
with open(self.tagFile, 'r') as f:
|
||||
storedTag = f.read()
|
||||
if storedTag != self.tagContents:
|
||||
print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents))
|
||||
return False
|
||||
return True
|
||||
|
||||
def clean(self):
|
||||
print("Cleaning vcpkg installation at {}".format(self.path))
|
||||
if os.path.isdir(self.path):
|
||||
print("Removing {}".format(self.path))
|
||||
shutil.rmtree(self.path, ignore_errors=True)
|
||||
|
||||
# Make sure the VCPKG prerequisites are all there.
|
||||
def bootstrap(self):
|
||||
if self.upToDate():
|
||||
return
|
||||
|
||||
self.clean()
|
||||
|
||||
downloadVcpkg = False
|
||||
if self.args.force_bootstrap:
|
||||
print("Forcing bootstrap")
|
||||
downloadVcpkg = True
|
||||
|
||||
if not downloadVcpkg and not os.path.isfile(self.exe):
|
||||
print("Missing executable, boostrapping")
|
||||
downloadVcpkg = True
|
||||
|
||||
# Make sure we have a vcpkg executable
|
||||
testFile = os.path.join(self.path, '.vcpkg-root')
|
||||
if not downloadVcpkg and not os.path.isfile(testFile):
|
||||
print("Missing {}, bootstrapping".format(testFile))
|
||||
downloadVcpkg = True
|
||||
|
||||
if downloadVcpkg:
|
||||
print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
|
||||
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash)
|
||||
|
||||
print("Replacing port files")
|
||||
portsPath = os.path.join(self.path, 'ports')
|
||||
if (os.path.islink(portsPath)):
|
||||
os.unlink(portsPath)
|
||||
if (os.path.isdir(portsPath)):
|
||||
shutil.rmtree(portsPath, ignore_errors=True)
|
||||
shutil.copytree(self.sourcePortsPath, portsPath)
|
||||
|
||||
def run(self, commands):
|
||||
actualCommands = [self.exe, '--vcpkg-root', self.path]
|
||||
actualCommands.extend(commands)
|
||||
print("Running command")
|
||||
print(actualCommands)
|
||||
hifi_utils.executeSubprocess(actualCommands, folder=self.path)
|
||||
|
||||
def setupDependencies(self):
|
||||
# Special case for android, grab a bunch of binaries
|
||||
# FIXME remove special casing for android builds eventually
|
||||
if self.args.android:
|
||||
print("Installing Android binaries")
|
||||
self.setupAndroidDependencies()
|
||||
|
||||
print("Installing host tools")
|
||||
self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools'])
|
||||
|
||||
# If not android, install the hifi-client-deps libraries
|
||||
if not self.args.android:
|
||||
print("Installing build dependencies")
|
||||
self.run(['install', '--triplet', self.triplet, 'hifi-client-deps'])
|
||||
|
||||
def cleanBuilds(self):
|
||||
# Remove temporary build artifacts
|
||||
builddir = os.path.join(self.path, 'buildtrees')
|
||||
if os.path.isdir(builddir):
|
||||
print("Wiping build trees")
|
||||
shutil.rmtree(builddir, ignore_errors=True)
|
||||
|
||||
def setupAndroidDependencies(self):
|
||||
# vcpkg prebuilt
|
||||
if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')):
|
||||
dest = os.path.join(self.path, 'installed')
|
||||
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
|
||||
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
|
||||
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
|
||||
#hifi_utils.downloadAndExtract(url, dest, hash)
|
||||
hifi_utils.downloadAndExtract(url, dest)
|
||||
|
||||
def writeTag(self):
|
||||
print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
|
||||
with open(self.tagFile, 'w') as f:
|
||||
f.write(self.tagContents)
|
||||
|
||||
def writeConfig(self):
|
||||
print("Writing cmake config to {}".format(self.configFilePath))
|
||||
# Write out the configuration for use by CMake
|
||||
cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
|
||||
installPath = os.path.join(self.path, 'installed', self.triplet)
|
||||
toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools')
|
||||
cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE
|
||||
if not self.args.android:
|
||||
cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID
|
||||
cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath).replace('\\', '/')
|
||||
with open(self.configFilePath, 'w') as f:
|
||||
f.write(cmakeConfig)
|
||||
|
||||
def cleanOldBuilds(self):
|
||||
# FIXME because we have the base directory, and because a build will
|
||||
# update the tag file on every run, we can scan the base dir for sub directories containing
|
||||
# a tag file that is older than N days, and if found, delete the directory, recovering space
|
||||
print("Not implemented")
|
||||
|
||||
|
||||
|
|
@ -802,7 +802,7 @@ Menu::Menu() {
|
|||
connect(action, &QAction::triggered, qApp, []() { std::thread(crash::newFault).join(); });
|
||||
|
||||
// Developer > Show Statistics
|
||||
addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats);
|
||||
addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats, 0, true);
|
||||
|
||||
// Developer > Show Animation Statistics
|
||||
addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::AnimStats);
|
||||
|
|
|
|||
304
prebuild.py
304
prebuild.py
|
|
@ -1,5 +1,26 @@
|
|||
#!python
|
||||
|
||||
# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring
|
||||
# tools required by the build as well as dependencies on which we rely.
|
||||
#
|
||||
# By using this script, we can reduce the requirements for a developer getting started to:
|
||||
#
|
||||
# * A working C++ dev environment like visual studio, xcode, gcc, or clang
|
||||
# * Qt
|
||||
# * CMake
|
||||
# * Python 3.x
|
||||
#
|
||||
# The function of the build script is to acquire, if not already present, all the other build requirements
|
||||
# The build script should be idempotent. If you run it with the same arguments multiple times, that should
|
||||
# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not
|
||||
# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should
|
||||
# execute quickly, determining that no work is to be done
|
||||
|
||||
import hifi_singleton
|
||||
import hifi_utils
|
||||
import hifi_android
|
||||
import hifi_vcpkg
|
||||
|
||||
import argparse
|
||||
import concurrent
|
||||
import hashlib
|
||||
|
|
@ -9,252 +30,65 @@ import os
|
|||
import platform
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
import functools
|
||||
|
||||
print = functools.partial(print, flush=True)
|
||||
|
||||
def executeSubprocess(processArgs, folder=None, env=None):
|
||||
restoreDir = None
|
||||
if folder != None:
|
||||
restoreDir = os.getcwd()
|
||||
os.chdir(folder)
|
||||
|
||||
process = subprocess.Popen(
|
||||
processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env)
|
||||
process.wait()
|
||||
|
||||
if (0 != process.returncode):
|
||||
raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format(
|
||||
processArgs[0],
|
||||
' '.join(processArgs[1:]),
|
||||
))
|
||||
|
||||
if restoreDir != None:
|
||||
os.chdir(restoreDir)
|
||||
|
||||
|
||||
def hashFile(file):
|
||||
hasher = hashlib.sha512()
|
||||
with open(file, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def hashFolder(folder):
|
||||
hasher = hashlib.sha256()
|
||||
for dirName, subdirList, fileList in os.walk(folder):
|
||||
for fname in fileList:
|
||||
with open(os.path.join(folder, dirName, fname), "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def downloadAndExtract(url, destPath, hash=None):
|
||||
tempFileDescriptor, tempFileName = tempfile.mkstemp()
|
||||
# OSX Python doesn't support SSL, so we need to bypass it.
|
||||
# However, we still validate the downloaded file's sha512 hash
|
||||
context = ssl._create_unverified_context()
|
||||
with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile:
|
||||
shutil.copyfileobj(response, tempFile)
|
||||
|
||||
# Verify the hash
|
||||
if hash and hash != hashFile(tempFileName):
|
||||
raise RuntimeError("Downloaded file does not match hash")
|
||||
|
||||
# Extract the archive
|
||||
with tarfile.open(tempFileName, 'r:gz') as tgz:
|
||||
tgz.extractall(destPath)
|
||||
os.remove(tempFileName)
|
||||
|
||||
|
||||
class VcpkgRepo:
|
||||
def __init__(self):
|
||||
global args
|
||||
scriptPath = os.path.dirname(os.path.realpath(sys.argv[0]))
|
||||
# our custom ports, relative to the script location
|
||||
self.sourcePortsPath = os.path.join(scriptPath, 'cmake', 'ports')
|
||||
# FIXME Revert to ports hash before release
|
||||
self.id = hashFolder(self.sourcePortsPath)[:8]
|
||||
# OS dependent information
|
||||
system = platform.system()
|
||||
|
||||
if args.vcpkg_root is not None:
|
||||
print("override vcpkg path with " + args.vcpkg_root)
|
||||
self.path = args.vcpkg_root
|
||||
else:
|
||||
if 'Darwin' == system:
|
||||
defaultBasePath = os.path.expanduser('~/hifi/vcpkg')
|
||||
else:
|
||||
defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg')
|
||||
basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath)
|
||||
if (not os.path.isdir(basePath)):
|
||||
os.makedirs(basePath)
|
||||
self.path = os.path.join(basePath, self.id)
|
||||
|
||||
self.tagFile = os.path.join(self.path, '.id')
|
||||
# A format version attached to the tag file... increment when you want to force the build systems to rebuild
|
||||
# without the contents of the ports changing
|
||||
self.version = 1
|
||||
self.tagContents = "{}_{}".format(self.id, self.version)
|
||||
|
||||
print("prebuild path: " + self.path)
|
||||
if 'Windows' == system:
|
||||
self.exe = os.path.join(self.path, 'vcpkg.exe')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ'
|
||||
self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e'
|
||||
self.hostTriplet = 'x64-windows'
|
||||
elif 'Darwin' == system:
|
||||
self.exe = os.path.join(self.path, 'vcpkg')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc'
|
||||
self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d'
|
||||
self.hostTriplet = 'x64-osx'
|
||||
else:
|
||||
self.exe = os.path.join(self.path, 'vcpkg')
|
||||
self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU'
|
||||
self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d'
|
||||
self.hostTriplet = 'x64-linux'
|
||||
|
||||
if args.android:
|
||||
self.triplet = 'arm64-android'
|
||||
else:
|
||||
self.triplet = self.hostTriplet
|
||||
|
||||
def outOfDate(self):
|
||||
global args
|
||||
# Prevent doing a clean if we've explcitly set a directory for vcpkg
|
||||
if args.vcpkg_root is not None:
|
||||
return False
|
||||
if args.force_build:
|
||||
return True
|
||||
print("Looking for tag file {}".format(self.tagFile))
|
||||
if not os.path.isfile(self.tagFile):
|
||||
return True
|
||||
with open(self.tagFile, 'r') as f:
|
||||
storedTag = f.read()
|
||||
print("Found stored tag {}".format(storedTag))
|
||||
if storedTag != self.tagContents:
|
||||
print("Doesn't match computed tag {}".format(self.tagContents))
|
||||
return True
|
||||
return False
|
||||
|
||||
def clean(self):
|
||||
cleanPath = self.path
|
||||
print("Cleaning vcpkg installation at {}".format(cleanPath))
|
||||
if os.path.isdir(self.path):
|
||||
print("Removing {}".format(cleanPath))
|
||||
shutil.rmtree(cleanPath, ignore_errors=True)
|
||||
|
||||
def bootstrap(self):
|
||||
global args
|
||||
if self.outOfDate():
|
||||
self.clean()
|
||||
|
||||
# don't download the vcpkg binaries if we're working with an explicit
|
||||
# vcpkg directory (possibly a git checkout)
|
||||
if args.vcpkg_root is None:
|
||||
downloadVcpkg = False
|
||||
if args.force_bootstrap:
|
||||
print("Forcing bootstrap")
|
||||
downloadVcpkg = True
|
||||
|
||||
if not downloadVcpkg and not os.path.isfile(self.exe):
|
||||
print("Missing executable, boostrapping")
|
||||
downloadVcpkg = True
|
||||
|
||||
# Make sure we have a vcpkg executable
|
||||
testFile = os.path.join(self.path, '.vcpkg-root')
|
||||
if not downloadVcpkg and not os.path.isfile(testFile):
|
||||
print("Missing {}, bootstrapping".format(testFile))
|
||||
downloadVcpkg = True
|
||||
|
||||
if downloadVcpkg:
|
||||
print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
|
||||
downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash)
|
||||
|
||||
print("Replacing port files")
|
||||
portsPath = os.path.join(self.path, 'ports')
|
||||
if (os.path.islink(portsPath)):
|
||||
os.unlink(portsPath)
|
||||
if (os.path.isdir(portsPath)):
|
||||
shutil.rmtree(portsPath, ignore_errors=True)
|
||||
shutil.copytree(self.sourcePortsPath, portsPath)
|
||||
|
||||
def downloadAndroidDependencies(self):
|
||||
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
|
||||
hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
|
||||
dest = os.path.join(self.path, 'installed')
|
||||
downloadAndExtract(url, dest, hash)
|
||||
|
||||
def run(self, commands):
|
||||
actualCommands = [self.exe, '--vcpkg-root', self.path]
|
||||
actualCommands.extend(commands)
|
||||
print("Running command")
|
||||
print(actualCommands)
|
||||
executeSubprocess(actualCommands, folder=self.path)
|
||||
|
||||
def buildDependencies(self):
|
||||
global args
|
||||
print("Installing host tools")
|
||||
self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools'])
|
||||
# Special case for android, grab a bunch of binaries
|
||||
if args.android:
|
||||
self.downloadAndroidDependencies()
|
||||
return
|
||||
|
||||
print("Installing build dependencies")
|
||||
self.run(['install', '--triplet', self.triplet, 'hifi-client-deps'])
|
||||
# Remove temporary build artifacts
|
||||
builddir = os.path.join(self.path, 'buildtrees')
|
||||
if os.path.isdir(builddir):
|
||||
print("Wiping build trees")
|
||||
shutil.rmtree(builddir, ignore_errors=True)
|
||||
|
||||
def writeConfig(self):
|
||||
global args
|
||||
configFilePath = os.path.join(args.build_root, 'vcpkg.cmake')
|
||||
print("Writing cmake config to {}".format(configFilePath))
|
||||
# Write out the configuration for use by CMake
|
||||
cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
|
||||
installPath = os.path.join(self.path, 'installed', self.triplet)
|
||||
toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools')
|
||||
cmakeTemplate = 'set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")\n'
|
||||
cmakeTemplate += 'set(VCPKG_INSTALL_ROOT "{}" CACHE FILEPATH "vcpkg installed packages path")\n'
|
||||
cmakeTemplate += 'set(VCPKG_TOOLS_DIR "{}" CACHE FILEPATH "vcpkg installed packages path")\n'
|
||||
cmakeConfig = cmakeTemplate.format(cmakeScript, installPath, toolsPath).replace('\\', '/')
|
||||
with open(configFilePath, 'w') as f:
|
||||
f.write(cmakeConfig)
|
||||
|
||||
def writeTag(self):
|
||||
print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
|
||||
with open(self.tagFile, 'w') as f:
|
||||
f.write(self.tagContents)
|
||||
def parse_args():
|
||||
# our custom ports, relative to the script location
|
||||
defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports')
|
||||
from argparse import ArgumentParser
|
||||
parser = ArgumentParser(description='Prepare build dependencies.')
|
||||
parser.add_argument('--android', action='store_true')
|
||||
#parser.add_argument('--android', type=str)
|
||||
parser.add_argument('--debug', action='store_true')
|
||||
parser.add_argument('--force-bootstrap', action='store_true')
|
||||
parser.add_argument('--force-build', action='store_true')
|
||||
parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
|
||||
parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
|
||||
parser.add_argument('--ports-path', type=str, default=defaultPortsPath)
|
||||
if True:
|
||||
args = parser.parse_args()
|
||||
else:
|
||||
args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/hifi/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a'])
|
||||
return args
|
||||
|
||||
def main():
|
||||
vcpkg = VcpkgRepo()
|
||||
vcpkg.bootstrap()
|
||||
vcpkg.buildDependencies()
|
||||
vcpkg.writeConfig()
|
||||
vcpkg.writeTag()
|
||||
# Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build
|
||||
# VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for
|
||||
# building OpenSSL
|
||||
removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE']
|
||||
for var in removeEnvVars:
|
||||
if var in os.environ:
|
||||
del os.environ[var]
|
||||
|
||||
args = parse_args()
|
||||
# Only allow one instance of the program to run at a time
|
||||
pm = hifi_vcpkg.VcpkgRepo(args)
|
||||
with hifi_singleton.Singleton(pm.lockFile) as lock:
|
||||
if not pm.upToDate():
|
||||
pm.bootstrap()
|
||||
|
||||
# Always write the tag, even if we changed nothing. This
|
||||
# allows vcpkg to reclaim disk space by identifying directories with
|
||||
# tags that haven't been touched in a long time
|
||||
pm.writeTag()
|
||||
|
||||
from argparse import ArgumentParser
|
||||
parser = ArgumentParser(description='Prepare build dependencies.')
|
||||
parser.add_argument('--android', action='store_true')
|
||||
parser.add_argument('--debug', action='store_true')
|
||||
parser.add_argument('--force-bootstrap', action='store_true')
|
||||
parser.add_argument('--force-build', action='store_true')
|
||||
parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
|
||||
parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
|
||||
# Grab our required dependencies:
|
||||
# * build host tools, like spirv-cross and scribe
|
||||
# * build client dependencies like openssl and nvtt
|
||||
pm.setupDependencies()
|
||||
|
||||
args = parser.parse_args()
|
||||
# wipe out the build directories (after writing the tag, since failure
|
||||
# here shouldn't invalidte the vcpkg install)
|
||||
pm.cleanBuilds()
|
||||
|
||||
# Write the vcpkg config to the build directory last
|
||||
pm.writeConfig()
|
||||
|
||||
print(sys.argv)
|
||||
main()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue