Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-07 20:22:27 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

🚧 Replace vcpkg with conan

Browse source
This commit is contained in:
Edgar 2023-08-13 18:38:39 +02:00
parent c4fa5bf186
commit fc1d4323ba
No known key found for this signature in database
GPG key ID: 3C2E1F2C1C353131
8 changed files with 42 additions and 1597 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
CMakeLists.txt
View file

@ -18,39 +18,6 @@ cmake_minimum_required(VERSION 3.14)
# This should allow using long paths on Windows
SET(CMAKE_NINJA_FORCE_RESPONSE_FILE 1 CACHE INTERNAL "")
# Passing of variables to vcpkg
#
# vcpkg runs cmake scripts in an isolated environment, see this for details:
# https://github.com/Microsoft/vcpkg/issues/3712
#
# Here's how this works and how we work around this issue:
#
# 1. This file (CMakeLists.txt) runs first and is authoritative. It is the one
# that reads the environment, sets variables and sets a default value.
# 2. It writes the contents of the variables to
# $CMAKE_CURRENT_BINARY_DIR/_env/$VARNAME
# 3. hifi_vcpkg.py takes the _env directory, and copies it to the vcpkg dir.
# This solves the issue of CMakeLists.txt not knowing where the vcpkg dir is.
# 4. cmake/ports/*/portfile.cmake does know where the vcpkg dir is, and can
# read the _env that was copied there to obtain the variable's name.
#
# To ensure no old data could be accidentally read, the _env directories are
# deleted on each execution and fully recreated.
# Ensure nothing is kept from any previous run
file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/_env")
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/_env")
# Base URL for externally downloaded files
set(EXTERNAL_BUILD_ASSETS "https://build-deps.overte.org")
if( DEFINED ENV{EXTERNAL_BUILD_ASSETS} )
set(EXTERNAL_BUILD_ASSETS "$ENV{EXTERNAL_BUILD_ASSETS}")
endif()
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/EXTERNAL_BUILD_ASSETS.txt" "${EXTERNAL_BUILD_ASSETS}")
MESSAGE(STATUS "EXTERNAL_BUILD_ASSETS: ${EXTERNAL_BUILD_ASSETS}")
# read USE_GLES enviroment variable and sets it as GLES option
# TODO still gets overwritten by "use GLES on linux aarch64"
set(GLES_OPTION "$ENV{USE_GLES}")
@ -60,18 +27,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarc
set(GLES_OPTION ON)
endif()
# Will affect VCPKG dependencies
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/USE_GLES.txt" "${GLES_OPTION}")
MESSAGE(STATUS "GLES_OPTION: ${GLES_OPTION}")
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros/TargetPython.cmake")
target_python()
if (WIN32 AND NOT HIFI_ANDROID AND NOT (CMAKE_GENERATOR STREQUAL "Ninja"))
# Force x64 toolset
set(CMAKE_GENERATOR_TOOLSET "host=x64" CACHE STRING "64-bit toolset" FORCE)
endif()
# set our OS X deployment target
# (needs to be set before first project() call and before prebuild.py)
# Will affect VCPKG dependencies
@ -194,50 +149,9 @@ if(OVERTE_WARNINGS_AS_ERRORS)
endif()
if (HIFI_ANDROID)
execute_process(
COMMAND ${HIFI_PYTHON_EXEC} ${CMAKE_CURRENT_SOURCE_DIR}/prebuild.py --release-type ${RELEASE_TYPE} --android ${HIFI_ANDROID_APP} --build-root ${CMAKE_BINARY_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULTS_VARIABLE PREBUILD_RET
)
else()
set(VCPKG_BUILD_TYPE_PARAM "")
if (VCPKG_BUILD_TYPE)
set(VCPKG_BUILD_TYPE_PARAM --vcpkg-build-type ${VCPKG_BUILD_TYPE})
endif()
execute_process(
COMMAND ${HIFI_PYTHON_EXEC} ${CMAKE_CURRENT_SOURCE_DIR}/prebuild.py --release-type ${RELEASE_TYPE} --build-root ${CMAKE_BINARY_DIR} ${VCPKG_BUILD_TYPE_PARAM}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULTS_VARIABLE PREBUILD_RET
)
endif()
if (PREBUILD_RET GREATER 0)
message(FATAL_ERROR "prebuild.py failed with error ${PREBUILD_RET}")
endif()
if(NOT EXISTS "${CMAKE_BINARY_DIR}/vcpkg.cmake")
message(FATAL_ERROR "vcpkg configuration missing.")
endif()
include("${CMAKE_BINARY_DIR}/vcpkg.cmake")
if (HIFI_ANDROID)
set(QT_CMAKE_PREFIX_PATH "$ENV{HIFI_ANDROID_PRECOMPILED}/qt/lib/cmake")
else()
if ("$ENV{OVERTE_USE_SYSTEM_QT}" STREQUAL "")
if(NOT EXISTS "${CMAKE_BINARY_DIR}/qt.cmake")
message(FATAL_ERROR "qt configuration missing.")
endif()
include("${CMAKE_BINARY_DIR}/qt.cmake")
message(STATUS "${CMAKE_BINARY_DIR}/qt.cmake included!")
else()
message(STATUS "System Qt in use, not including qt.cmake!")
endif()
endif()
option(VCPKG_APPLOCAL_DEPS OFF)
project(overte)
include("cmake/init.cmake")
include("cmake/compiler.cmake")
option(VCPKG_APPLOCAL_DEPS OFF)
add_paths_to_fixup_libs(${VCPKG_INSTALL_ROOT}/bin)
add_paths_to_fixup_libs(${VCPKG_INSTALL_ROOT}/debug/bin)

42
conanfile.py Normal file
View file

@ -0,0 +1,42 @@
import os
from conan import ConanFile
from conan.tools.files import copy
class Overte(ConanFile):
name = "Overte"
settings = "os", "compiler", "build_type", "arch"
generators = "CMakeToolchain", "CMakeDeps"
def requirements(self):
self.requires("bullet3/3.25")
self.requires("draco/1.5.6")
self.requires("etc2comp/cci.20170424")
self.requires("glad/0.1.36")
self.requires("glm/cci.20230113")
self.requires("nodejs/18.15.0")
self.requires("openexr/3.1.9")
self.requires("openssl/1.1.1t")
self.requires("opus/1.3.1")
# self.requires("tbb/2020.3")
self.requires("zlib/1.2.13")
self.requires("glslang/11.7.0")
self.requires("nlohmann_json/3.11.2")
# self.requires("openvr/1.16.8")
self.requires("quazip/1.4")
self.requires("sdl/2.26.5")
self.requires("spirv-cross/cci.20211113")
self.requires("spirv-tools/2021.4")
self.requires("vulkan-memory-allocator/3.0.1")
def generate(self):
for dep in self.dependencies.values():
for f in dep.cpp_info.bindirs:
self.cp_data(f)
for f in dep.cpp_info.libdirs:
self.cp_data(f)
def cp_data(self, src):
bindir = os.path.join(self.build_folder, "bin")
copy(self, "*.dll", src, bindir, False)
copy(self, "*.so*", src, bindir, False)

337
hifi_android.py
View file

@ -1,337 +0,0 @@
import hifi_utils
import json
import os
import platform
import re
import shutil
import xml.etree.ElementTree as ET
import functools
import zipfile
print = functools.partial(print, flush=True)
ANDROID_PACKAGE_URL = 'https://build-deps.overte.org/dependencies/android/'
ANDROID_PACKAGES = {
'qt' : {
'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz',
'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df',
},
'bullet': {
'file': 'bullet-2.88_armv8-libcpp.tgz',
'checksum': '81642779ccb110f8c7338e8739ac38a0',
},
'draco': {
'file': 'draco_armv8-libcpp.tgz',
'checksum': '617a80d213a5ec69fbfa21a1f2f738cd',
},
'glad': {
'file': 'glad_armv8-libcpp.zip',
'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449',
},
'gvr': {
'file': 'gvrsdk_v1.101.0.tgz',
'checksum': '57fd02baa069176ba18597a29b6b4fc7',
},
'nvtt': {
'file': 'nvtt_armv8-libcpp.zip',
'checksum': 'eb46d0b683e66987190ed124aabf8910',
'sharedLibFolder': 'lib',
'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so']
},
'ovr_sdk_mobile_1.37.0': {
'file': 'ovr_sdk_mobile_1.37.0.zip',
'checksum': '6040e1966f335a3e5015295154cd7383',
'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release',
'includeLibs': ['libvrapi.so']
},
'ovr_platform_sdk_23.0.0': {
'file': 'ovr_platform_sdk_23.0.0.zip',
'checksum': '29d02b560f60d0fa7b8a64cd965dd55b',
'sharedLibFolder': 'Android/libs/arm64-v8a',
'includeLibs': ['libovrplatformloader.so']
},
'openssl': {
'file': 'openssl-1.1.0g_armv8.tgz',
'checksum': 'cabb681fbccd79594f65fcc266e02f32'
},
'polyvox': {
'file': 'polyvox_armv8-libcpp.tgz',
'checksum': 'dba88b3a098747af4bb169e9eb9af57e',
'sharedLibFolder': 'lib',
'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'],
},
'tbb': {
'file': 'tbb-2018_U1_armv8_libcpp.tgz',
'checksum': '20768f298f53b195e71b414b0ae240c4',
'sharedLibFolder': 'lib/release',
'includeLibs': ['libtbb.so', 'libtbbmalloc.so'],
},
'etc2comp': {
'file': 'etc2comp-patched-armv8-libcpp.tgz',
'checksum': '14b02795d774457a33bbc60e00a786bc'
},
'breakpad': {
'file': 'breakpad.tgz',
'checksum': 'ddcb23df336b08017042ba4786db1d9e',
'sharedLibFolder': 'lib',
'includeLibs': {'libbreakpad_client.a'}
},
'webrtc': {
'file': 'webrtc-20190626-android.tar.gz',
'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53'
}
}
ANDROID_PLATFORM_PACKAGES = {
'Darwin' : {
'qt': {
'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz',
'checksum': 'c83cc477c08a892e00c71764dca051a0'
},
},
'Windows' : {
'qt': {
'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz',
'checksum': '0582191cc55431aa4f660848a542883e'
},
}
}
QT5_DEPS = [
'Qt5Concurrent',
'Qt5Core',
'Qt5Gui',
'Qt5Multimedia',
'Qt5Network',
'Qt5OpenGL',
'Qt5Qml',
'Qt5Quick',
'Qt5QuickControls2',
'Qt5QuickTemplates2',
'Qt5Script',
'Qt5ScriptTools',
'Qt5Svg',
'Qt5WebChannel',
'Qt5WebSockets',
'Qt5Widgets',
'Qt5XmlPatterns',
# Android specific
'Qt5AndroidExtras',
'Qt5WebView',
]
def getPlatformPackages():
result = ANDROID_PACKAGES.copy()
system = platform.system()
if system in ANDROID_PLATFORM_PACKAGES:
platformPackages = ANDROID_PLATFORM_PACKAGES[system]
result = { **result, **platformPackages }
return result
def getPackageUrl(package):
url = ANDROID_PACKAGE_URL
if 'baseUrl' in package:
url = package['baseUrl']
url += package['file']
if 'versionId' in package:
url += '?versionId=' + package['versionId']
return url
def copyAndroidLibs(packagePath, appPath):
androidPackages = getPlatformPackages()
jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a')
if not os.path.isdir(jniPath):
os.makedirs(jniPath)
for packageName in androidPackages:
package = androidPackages[packageName]
if 'sharedLibFolder' in package:
sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder'])
if 'includeLibs' in package:
for lib in package['includeLibs']:
sourceFile = os.path.join(sharedLibFolder, lib)
destFile = os.path.join(jniPath, os.path.split(lib)[1])
if not os.path.isfile(destFile):
print("Copying {}".format(lib))
shutil.copy(sourceFile, destFile)
gvrLibFolder = os.path.join(packagePath, 'gvr/gvr-android-sdk-1.101.0/libraries')
audioSoOut = os.path.join(gvrLibFolder, 'libgvr_audio.so')
if not os.path.isfile(audioSoOut):
audioAar = os.path.join(gvrLibFolder, 'sdk-audio-1.101.0.aar')
with zipfile.ZipFile(audioAar) as z:
with z.open('jni/arm64-v8a/libgvr_audio.so') as f:
with open(audioSoOut, 'wb') as of:
shutil.copyfileobj(f, of)
audioSoOut2 = os.path.join(jniPath, 'libgvr_audio.so')
if not os.path.isfile(audioSoOut2):
shutil.copy(audioSoOut, audioSoOut2)
baseSoOut = os.path.join(gvrLibFolder, 'libgvr.so')
if not os.path.isfile(baseSoOut):
baseAar = os.path.join(gvrLibFolder, 'sdk-base-1.101.0.aar')
with zipfile.ZipFile(baseAar) as z:
with z.open('jni/arm64-v8a/libgvr.so') as f:
with open(baseSoOut, 'wb') as of:
shutil.copyfileobj(f, of)
baseSoOut2 = os.path.join(jniPath, 'libgvr.so')
if not os.path.isfile(baseSoOut2):
shutil.copy(baseSoOut, baseSoOut2)
class QtPackager:
def __init__(self, appPath, qtRootPath):
self.appPath = appPath
self.qtRootPath = qtRootPath
self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a')
self.assetPath = os.path.join(self.appPath, 'src/main/assets')
self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--')
self.qtAssetCacheList = os.path.join(self.qtAssetPath, 'qt_cache_pregenerated_file_list')
# Jars go into the qt library
self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs'))
self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml')
self.files = []
self.features = []
self.permissions = []
def copyQtDeps(self):
for lib in QT5_DEPS:
libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib))
if not os.path.exists(libfile):
continue
self.files.append(libfile)
androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib))
if not os.path.exists(androidDeps):
continue
tree = ET.parse(androidDeps)
root = tree.getroot()
for item in root.findall('./dependencies/lib/depends/*'):
if (item.tag == 'lib') or (item.tag == 'bundled'):
relativeFilename = item.attrib['file']
if (relativeFilename.startswith('qml')):
continue
filename = os.path.join(self.qtRootPath, relativeFilename)
self.files.extend(hifi_utils.recursiveFileList(filename, excludeNamePattern=r"^\."))
elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1":
self.files.append(os.path.join(self.qtRootPath, item.attrib['file']))
elif item.tag == 'permission':
self.permissions.append(item.attrib['name'])
elif item.tag == 'feature':
self.features.append(item.attrib['name'])
def scanQmlImports(self):
qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner')
system = platform.system()
if 'Windows' == system:
qmlImportCommandFile += ".exe"
if not os.path.isfile(qmlImportCommandFile):
raise RuntimeError("Couldn't find qml import scanner")
qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml')
qmlImportPath = os.path.join(self.qtRootPath, 'qml')
commandResult = hifi_utils.executeSubprocessCapture([
qmlImportCommandFile,
'-rootPath', qmlRootPath,
'-importPath', qmlImportPath
])
qmlImportResults = json.loads(commandResult)
for item in qmlImportResults:
if 'path' not in item:
continue
path = os.path.realpath(item['path'])
if not os.path.exists(path):
continue
basePath = path
if os.path.isfile(basePath):
basePath = os.path.dirname(basePath)
basePath = os.path.normcase(basePath)
if basePath.startswith(qmlRootPath):
continue
self.files.extend(hifi_utils.recursiveFileList(path, excludeNamePattern=r"^\."))
def processFiles(self):
self.files = list(set(self.files))
self.files.sort()
libsXmlRoot = ET.Element('resources')
qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'})
bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'})
bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'})
libPrefix = 'lib'
for sourceFile in self.files:
if not os.path.isfile(sourceFile):
raise RuntimeError("Unable to find dependency file " + sourceFile)
relativePath = os.path.relpath(sourceFile, self.qtRootPath).replace('\\', '/')
destinationFile = None
if relativePath.endswith('.so'):
garbledFileName = None
if relativePath.startswith(libPrefix):
garbledFileName = relativePath[4:]
p = re.compile(r'lib(Qt5.*).so')
m = p.search(garbledFileName)
if not m:
raise RuntimeError("Huh?")
libName = m.group(1)
ET.SubElement(qtLibsNode, 'item').text = libName
else:
garbledFileName = 'lib' + relativePath.replace('/', '_'[0])
value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/')
ET.SubElement(bundledLibsNode, 'item').text = value
destinationFile = os.path.join(self.jniPath, garbledFileName)
elif relativePath.startswith('jar'):
destinationFile = os.path.join(self.jarPath, relativePath[4:])
else:
value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/')
ET.SubElement(bundledAssetsNode, 'item').text = value
destinationFile = os.path.join(self.qtAssetPath, relativePath)
destinationParent = os.path.realpath(os.path.dirname(destinationFile))
if not os.path.isdir(destinationParent):
os.makedirs(destinationParent)
if not os.path.isfile(destinationFile):
shutil.copy(sourceFile, destinationFile)
tree = ET.ElementTree(libsXmlRoot)
tree.write(self.xmlFile, 'UTF-8', True)
def generateAssetsFileList(self):
print("Implement asset file list")
# outputFilename = os.path.join(self.qtAssetPath, "qt_cache_pregenerated_file_list")
# fileList = hifi_utils.recursiveFileList(self.qtAssetPath)
# fileMap = {}
# for fileName in fileList:
# relativeFileName = os.path.relpath(fileName, self.assetPath)
# dirName, localFileName = os.path.split(relativeFileName)
# if not dirName in fileMap:
# fileMap[dirName] = []
# fileMap[dirName].append(localFileName)
# for dirName in fileMap:
# for localFileName in fileMap[dirName]:
# ????
#
# Gradle version
#
# DataOutputStream fos = new DataOutputStream(new FileOutputStream(outputFile));
# for (Map.Entry<String, List<String>> e: directoryContents.entrySet()) {
# def entryList = e.getValue()
# fos.writeInt(e.key.length()*2); // 2 bytes per char
# fos.writeChars(e.key);
# fos.writeInt(entryList.size());
# for (String entry: entryList) {
# fos.writeInt(entry.length()*2);
# fos.writeChars(entry);
# }
# }
def bundle(self):
if not os.path.isfile(self.xmlFile):
print("Bundling Qt info into {}".format(self.xmlFile))
self.copyQtDeps()
self.scanQmlImports()
self.processFiles()
# if not os.path.isfile(self.qtAssetCacheList):
# self.generateAssetsFileList()

257
hifi_qt.py
View file

@ -1,257 +0,0 @@
# Copyright 2013-2019 High Fidelity, Inc.
# Copyright 2020-2022 Vircadia contributors.
# Copyright 2020-2022 Overte e.V.
# SPDX-License-Identifier: Apache-2.0
import hifi_utils
import hifi_android
import hashlib
import os
import platform
import re
import shutil
import tempfile
import json
import xml.etree.ElementTree as ET
import functools
# The way Qt is handled is a bit complicated, so I'm documenting it here.
#
# 1. User runs cmake
# 2. cmake calls prebuild.py, which is referenced in /CMakeLists.txt
# 3. prebuild.py calls this code.
# 4. hifi_qt.py determines how to handle cmake: do we need to download a package, and which?
# 4.a - Using system Qt
# No download, most special paths are turned off.
# We build in the same way a normal Qt program would.
# 4.b - Using an user-provided Qt build in a custom directory.
# We just need to set the cmakePath to the right dir (qt5-install/lib/cmake)
# 4.c - Using a premade package.
# We check the OS and distro and set qtUrl to the URL to download.
# After this, it works on the same pathway as 4.b.
# 5. We write /qt.cmake, which contains paths that are passed down to SetupQt.cmake
# The template for this file is in CMAKE_TEMPLATE just below this comment
# and it sets the QT_CMAKE_PREFIX_PATH variable used by SetupQt.cmake.
# 6. cmake includes /qt.cmake receiving our information
# In the case of system Qt, this step is skipped.
# 7. cmake runs SetupQt.cmake which takes care of the cmake parts of the Qt configuration.
# In the case of system Qt, SetupQt.cmake is a no-op. It runs but exits immediately.
#
# The format for a prebuilt qt is a package containing a top-level directory named
# 'qt5-install', which contains the result of a "make install" from a build of the Qt source.
print = functools.partial(print, flush=True)
# Encapsulates the vcpkg system
class QtDownloader:
CMAKE_TEMPLATE = """
# this file auto-generated by hifi_qt.py
get_filename_component(QT_CMAKE_PREFIX_PATH "{}" ABSOLUTE CACHE)
get_filename_component(QT_CMAKE_PREFIX_PATH_UNCACHED "{}" ABSOLUTE)
# If the cached cmake toolchain path is different from the computed one, exit
if(NOT (QT_CMAKE_PREFIX_PATH_UNCACHED STREQUAL QT_CMAKE_PREFIX_PATH))
message(FATAL_ERROR "QT_CMAKE_PREFIX_PATH has changed, please wipe the build directory and rerun cmake")
endif()
"""
def __init__(self, args):
self.args = args
self.configFilePath = os.path.join(args.build_root, 'qt.cmake')
self.version = os.getenv('OVERTE_USE_QT_VERSION', '5.15.2')
self.assets_url = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_BUILD_ASSETS')
# OS dependent information
system = platform.system()
qt_found = False
system_qt = False
# Here we handle the 3 possible cases of dealing with Qt:
if bool(os.getenv('OVERTE_USE_SYSTEM_QT', False)):
# 1. Using the system provided Qt. This is only recommended for Qt 5.15.0 and above,
# as it includes a required fix on Linux.
#
# This path only works on Linux as neither Windows nor OSX ship Qt.
if system != "Linux":
raise Exception("Using the system Qt is only supported on Linux")
self.path = None
self.cmakePath = None
qt_found = True
system_qt = True
if not self.args.quiet:
print("Using system Qt")
elif os.getenv('OVERTE_QT_PATH', "") != "":
# 2. Using an user-provided directory.
# OVERTE_QT_PATH must point to a directory with a Qt install in it.
self.path = os.getenv('OVERTE_QT_PATH')
self.fullPath = self.path
self.cmakePath = os.path.join(self.fullPath, 'lib', 'cmake')
qt_found = True
if not self.args.quiet:
print("Using Qt from " + self.fullPath)
else:
# 3. Using a pre-built Qt.
#
# This works somewhat differently from above, notice how path and fullPath are
# used differently in this case.
#
# In the case of an user-provided directory, we just use the user-supplied directory.
#
# For a pre-built qt, however, we have to unpack it. The archive is required to contain
# a qt5-install directory in it.
self.path = os.path.expanduser("~/overte-files/qt")
self.fullPath = os.path.join(self.path, 'qt5-install')
self.cmakePath = os.path.join(self.fullPath, 'lib', 'cmake')
if (not os.path.isdir(self.path)):
os.makedirs(self.path)
qt_found = os.path.isdir(self.fullPath)
print("Using a packaged Qt")
if not system_qt:
if qt_found:
# Sanity check, ensure we have a good cmake directory
qt5_dir = os.path.join(self.cmakePath, "Qt5")
if not os.path.isdir(qt5_dir):
raise Exception("Failed to find Qt5 directory under " + self.cmakePath + ". There should be a " + qt5_dir)
else:
print("Qt5 check passed, found " + qt5_dir)
# I'm not sure why this is needed. It's used by hifi_singleton.
# Perhaps it stops multiple build processes from interferring?
lockDir, lockName = os.path.split(self.path)
lockName += '.lock'
if not os.path.isdir(lockDir):
os.makedirs(lockDir)
self.lockFile = os.path.join(lockDir, lockName)
if qt_found:
if not self.args.quiet:
print("Found pre-built Qt5")
return
if 'Windows' == system:
self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.10-2023.10.02-windows-x86_64.tar.xz'
elif 'Darwin' == system:
self.qtUrl = self.assets_url + '/dependencies/vcpkg/qt5-install-5.15.2-macos.tar.gz'
elif 'Linux' == system:
import distro
cpu_architecture = platform.machine()
if 'x86_64' == cpu_architecture:
# `major_version()` can return blank string on rolling release distros like arch
# The `or 0` conditional assignment prevents the int parsing error from hiding the useful Qt package error
u_major = int( distro.major_version() or '0' )
if distro.id() == 'ubuntu' or distro.id() == 'linuxmint':
if (distro.id() == 'ubuntu' and u_major == 20) or distro.id() == 'linuxmint' and u_major == 20:
self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.16-2024.12.14-kde_32be154325bfba3ad2ba8bf75dad702f3588e8d3-ubuntu-20.04-amd64.tar.xz'
elif (distro.id() == 'ubuntu' and u_major > 20) or (distro.id() == 'linuxmint' and u_major > 20):
self.__no_qt_package_error()
else:
self.__unsupported_error()
else:
self.__no_qt_package_error()
elif 'aarch64' == cpu_architecture:
if distro.id() == 'ubuntu':
u_major = int( distro.major_version() )
if u_major == 20:
self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.9-2023.05.21-kde_fb3ec282151b1ee281a24f0545a40ac6438537c2-ubuntu-20.04-aarch64.tar.xz'
elif u_major > 20:
self.__no_qt_package_error()
else:
self.__unsupported_error()
elif distro.id() == 'debian':
u_major = int( distro.major_version() )
if u_major > 10:
self.__no_qt_package_error()
else:
self.__unsupported_error()
else:
self.__no_qt_package_error()
else:
raise Exception('UNKNOWN CPU ARCHITECTURE!!!')
else:
print("System : " + platform.system())
print("Architecture: " + platform.architecture())
print("Machine : " + platform.machine())
raise Exception('UNKNOWN OPERATING SYSTEM!!!')
def showQtBuildInfo(self):
print("")
print("It's also possible to build Qt for your distribution, please see the documentation at:")
print("https://github.com/overte-org/overte/tree/master/tools/qt-builder")
print("")
print("Alternatively, you can try building against the system Qt by setting the OVERTE_USE_SYSTEM_QT environment variable.")
print("You'll need to install the development packages, and to have Qt 5.15.0 or later.")
def writeConfig(self):
print("Writing cmake config to {}".format(self.configFilePath))
# Write out the configuration for use by CMake
cmakeConfig = QtDownloader.CMAKE_TEMPLATE.format(self.cmakePath, self.cmakePath).replace('\\', '/')
with open(self.configFilePath, 'w') as f:
f.write(cmakeConfig)
def installQt(self):
if not os.path.isdir(self.fullPath):
print ('Downloading Qt package')
print('Extracting ' + self.qtUrl + ' to ' + self.path)
hifi_utils.downloadAndExtract(self.qtUrl, self.path)
else:
print ('Qt has already been downloaded')
def __unsupported_error(self):
import distro
cpu_architecture = platform.machine()
print('')
hifi_utils.color('red')
print("Sorry, " + distro.name(pretty=True) + " on " + cpu_architecture + " is too old and won't be officially supported.")
hifi_utils.color('white')
print("Please upgrade to a more recent Linux distribution.")
hifi_utils.color('clear')
print('')
raise hifi_utils.SilentFatalError(3)
def __no_qt_package_error(self):
import distro
cpu_architecture = platform.machine()
print('')
hifi_utils.color('red')
print("Sorry, we don't have a prebuilt Qt package for " + distro.name(pretty=True) + " on " + cpu_architecture + ".")
hifi_utils.color('white')
print('')
print("If this is a recent distribution, dating from 2021 or so, you can try building")
print("against the system Qt by running this command, and trying again:")
print(" export OVERTE_USE_SYSTEM_QT=1")
print("")
hifi_utils.color('clear')
print("If you'd like to try to build Qt from source either for building Overte, or")
print("to contribute a prebuilt package for your distribution, please see the")
print("documentation at: ", end='')
hifi_utils.color('blue')
print("https://github.com/overte-org/overte/tree/master/tools/qt-builder")
hifi_utils.color('clear')
print('')
raise hifi_utils.SilentFatalError(2)

160
hifi_singleton.py
View file

@ -1,160 +0,0 @@
import json
import logging
import os
import platform
import time
try:
import fcntl
except ImportError:
fcntl = None
try:
import msvcrt
except ImportError:
msvcrt = None
logger = logging.getLogger(__name__)
# Used to ensure only one instance of the script runs at a time
class Singleton:
def __init__(self, path):
self.fh = None
self.windows = 'Windows' == platform.system()
self.path = path
def __enter__(self):
success = False
while not success:
try:
if self.windows:
if os.path.exists(self.path):
os.unlink(self.path)
self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
else:
self.fh = open(self.path, 'x')
fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
success = True
except EnvironmentError as err:
if self.fh is not None:
if self.windows:
os.close(self.fh)
else:
self.fh.close()
self.fh = None
# print is horked here so write directly to stdout.
with open(1, mode="w", closefd=False) as _stdout:
_stdout.write(f"Couldn't aquire lock {self.path}, retrying in 10 seconds\n")
_stdout.flush()
time.sleep(10)
return self
def __exit__(self, type, value, traceback):
if self.windows:
os.close(self.fh)
else:
fcntl.lockf(self.fh, fcntl.LOCK_UN)
self.fh.close()
os.unlink(self.path)
class FLock:
"""
File locking context manager
>> with FLock("/tmp/foo.lock"):
>> do_something_that_must_be_synced()
The lock file must stick around forever. The author is not aware of a no cross platform way to clean it up w/o introducting race conditions.
"""
def __init__(self, path):
self.fh = os.open(path, os.O_CREAT | os.O_RDWR)
self.path = path
def _lock_posix(self):
try:
fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
# Windows sleeps for 10 seconds before giving up on a lock.
# Lets mimic that behavior.
time.sleep(10)
return False
else:
return True
def _lock_windows(self):
try:
msvcrt.locking(self.fh, msvcrt.LK_LOCK, 1)
except OSError:
return False
else:
return True
if fcntl is not None:
_lock = _lock_posix
elif msvcrt is not None:
_lock = _lock_windows
else:
raise RuntimeError("No locking library found")
def read_stats(self):
data = {}
with open(self.fh, mode="r", closefd=False) as stats_file:
stats_file.seek(0)
try:
data = json.loads(stats_file.read())
except json.decoder.JSONDecodeError:
logger.warning("couldn't decode json in lock file")
except PermissionError:
# Can't read a locked file on Windows :(
pass
lock_age = time.time() - os.fstat(self.fh).st_mtime
if lock_age > 0:
data["Age"] = "%0.2f" % lock_age
with open(1, mode="w", closefd=False) as _stdout:
_stdout.write("Lock stats:\n")
for key, value in sorted(data.items()):
_stdout.write("* %s: %s\n" % (key, value))
_stdout.flush()
def write_stats(self):
stats = {
"Owner PID": os.getpid(),
}
flock_env_vars = os.getenv("FLOCK_ENV_VARS")
if flock_env_vars:
for env_var_name in flock_env_vars.split(":"):
stats[env_var_name] = os.getenv(env_var_name)
with open(self.fh, mode="w", closefd=False) as stats_file:
stats_file.truncate()
return stats_file.write(json.dumps(stats, indent=2))
def __enter__(self):
while not self._lock():
try:
self.read_stats()
except (IOError, ValueError) as exc:
logger.exception("couldn't read stats")
time.sleep(3.33) # don't hammer the file
self.write_stats()
return self
def __exit__(self, type, value, traceback):
os.close(self.fh)
# WARNING: `os.close` gives up the lock on `fh` then we attempt the `os.unlink`. On posix platforms this can lead to us deleting a lock file that another process owns. This step is required to maintain compatablity with Singleton. When and if FLock is completely rolled out to the build fleet this unlink should be removed.
try:
os.unlink(self.path)
except (FileNotFoundError, PermissionError):
logger.exception("couldn't unlink lock file")
if os.getenv("USE_FLOCK_CLS") is not None:
logger.warning("Using FLock locker")
Singleton = FLock

172
hifi_utils.py
View file

@ -1,172 +0,0 @@
import os
import hashlib
import platform
import shutil
import ssl
import subprocess
import sys
import tarfile
import re
import urllib
import urllib.request
import zipfile
import tempfile
import time
import functools
print = functools.partial(print, flush=True)
ansi_colors = {
'black' : 30,
'red': 31,
'green': 32,
'yellow': 33,
'blue': 34,
'magenta': 35,
'cyan': 36,
'white': 37,
'clear': 0
}
def scriptRelative(*paths):
scriptdir = os.path.dirname(os.path.realpath(sys.argv[0]))
result = os.path.join(scriptdir, *paths)
result = os.path.realpath(result)
result = os.path.normcase(result)
return result
def recursiveFileList(startPath, excludeNamePattern=None ):
result = []
if os.path.isfile(startPath):
result.append(startPath)
elif os.path.isdir(startPath):
for dirName, subdirList, fileList in os.walk(startPath):
for fname in fileList:
if excludeNamePattern and re.match(excludeNamePattern, fname):
continue
result.append(os.path.realpath(os.path.join(startPath, dirName, fname)))
result.sort()
return result
def executeSubprocessCapture(processArgs):
processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if (0 != processResult.returncode):
raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format(
processArgs[0],
' '.join(processArgs[1:]),
processResult.stdout.decode('utf-8'),
processResult.stderr.decode('utf-8')))
return processResult.stdout.decode('utf-8')
def executeSubprocess(processArgs, folder=None, env=None):
restoreDir = None
if folder != None:
restoreDir = os.getcwd()
os.chdir(folder)
process = subprocess.Popen(
processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env)
process.wait()
if (0 != process.returncode):
raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format(
processArgs[0],
' '.join(processArgs[1:]),
))
if restoreDir != None:
os.chdir(restoreDir)
def hashFile(file, hasher = hashlib.sha512()):
with open(file, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hasher.update(chunk)
return hasher.hexdigest()
# Assumes input files are in deterministic order
def hashFiles(filenames):
hasher = hashlib.sha256()
for filename in filenames:
with open(filename, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hasher.update(chunk)
return hasher.hexdigest()
def hashFolder(folder):
filenames = recursiveFileList(folder)
return hashFiles(filenames)
def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3):
for i in range(retries):
tempFileName = None
# OSX Python doesn't support SSL, so we need to bypass it.
# However, we still validate the downloaded file's sha512 hash
if 'Darwin' == platform.system():
tempFileDescriptor, tempFileName = tempfile.mkstemp()
context = ssl._create_unverified_context()
with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile:
shutil.copyfileobj(response, tempFile)
else:
tempFileName, headers = urllib.request.urlretrieve(url)
downloadHash = hashFile(tempFileName, hasher)
# Verify the hash
if hash is not None and hash != downloadHash:
print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url))
os.remove(tempFileName)
continue
return tempFileName
raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url))
def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False):
tempFileName = downloadFile(url, hash, hasher)
if isZip or ".zip" in url:
with zipfile.ZipFile(tempFileName) as zip:
zip.extractall(destPath)
else:
# Extract the archive
with tarfile.open(tempFileName, 'r:*') as tgz:
def is_within_directory(directory, target):
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
prefix = os.path.commonprefix([abs_directory, abs_target])
return prefix == abs_directory
def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
for member in tar.getmembers():
member_path = os.path.join(path, member.name)
if not is_within_directory(path, member_path):
raise Exception("Attempted Path Traversal in Tar File")
tar.extractall(path, members, numeric_owner=numeric_owner)
safe_extract(tgz, destPath)
os.remove(tempFileName)
def readEnviromentVariableFromFile(buildRootDir, var):
with open(os.path.join(buildRootDir, '_env', var + ".txt")) as fp:
return fp.read()
class SilentFatalError(Exception):
"""Thrown when some sort of fatal condition happened, and we already reported it to the user.
This excecption exists to give a chance to run any cleanup needed before exiting.
It should be handled at the bottom of the call stack, where the only action is to call
sys.exit(ex.exit_code)
"""
def __init__(self, exit_code):
self.exit_code = exit_code
def color(color_name):
# Ideally we'd use the termcolor module, but this avoids adding it as a dependency.
print("\033[1;{}m".format(ansi_colors[color_name]), end='')

365
hifi_vcpkg.py
View file

@ -1,365 +0,0 @@
import hifi_utils
import hifi_android
import hashlib
import os
import platform
import re
import shutil
import tempfile
import json
import xml.etree.ElementTree as ET
import functools
from os import path
print = functools.partial(print, flush=True)
# Encapsulates the vcpkg system
class VcpkgRepo:
CMAKE_TEMPLATE = """
# this file auto-generated by hifi_vcpkg.py
get_filename_component(CMAKE_TOOLCHAIN_FILE "{}" ABSOLUTE CACHE)
get_filename_component(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}" ABSOLUTE)
set(VCPKG_INSTALL_ROOT "{}")
set(VCPKG_TOOLS_DIR "{}")
set(VCPKG_TARGET_TRIPLET "{}")
"""
CMAKE_TEMPLATE_NON_ANDROID = """
# If the cached cmake toolchain path is different from the computed one, exit
if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE))
message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake")
endif()
"""
def __init__(self, args):
self.args = args
# our custom ports, relative to the script location
self.sourcePortsPath = args.ports_path
self.vcpkgBuildType = args.vcpkg_build_type
if (self.vcpkgBuildType):
self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8] + "-" + self.vcpkgBuildType
else:
self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8]
self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake')
if args.get_vcpkg_id or args.get_vcpkg_path:
# With these arguments no assets will be downloaded, and they may be used in conditions
# where the _env hack doesn't work.
self.assets_url = "http://no_assets.invalid"
else:
self.assets_url = self.readVar('EXTERNAL_BUILD_ASSETS')
# The noClean flag indicates we're doing weird dependency maintenance stuff
# i.e. we've got an explicit checkout of vcpkg and we don't want the script to
# do stuff it might otherwise do. It typically indicates that we're using our
# own git checkout of vcpkg and manually managing it
self.noClean = False
# OS dependent information
system = platform.system()
machine = platform.machine()
if 'HIFI_VCPKG_PATH' in os.environ:
self.path = os.environ['HIFI_VCPKG_PATH']
self.noClean = True
elif self.args.vcpkg_root is not None:
self.path = args.vcpkg_root
self.noClean = True
else:
defaultBasePath = os.path.expanduser('~/overte-files/vcpkg')
if 'CI_WORKSPACE' in os.environ:
self.basePath = os.path.join(os.getenv('CI_WORKSPACE'), 'overte-files/vcpkg')
else:
self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath)
if self.args.android:
self.basePath = os.path.join(self.basePath, 'android')
if (not os.path.isdir(self.basePath)):
os.makedirs(self.basePath)
self.path = os.path.join(self.basePath, self.id)
if not self.args.quiet:
print("Using vcpkg path {}".format(self.path))
lockDir, lockName = os.path.split(self.path)
lockName += '.lock'
if not os.path.isdir(lockDir):
os.makedirs(lockDir)
self.lockFile = os.path.join(lockDir, lockName)
self.tagFile = os.path.join(self.path, '.id')
self.prebuildTagFile = os.path.join(self.path, '.prebuild')
# A format version attached to the tag file... increment when you want to force the build systems to rebuild
# without the contents of the ports changing
self.version = 1
self.tagContents = "{}_{}".format(self.id, self.version)
self.bootstrapEnv = os.environ.copy()
self.buildEnv = os.environ.copy()
self.prebuiltArchive = None
usePrebuilt = False
# usePrebuild Disabled, to re-enabled using the prebuilt archives for GitHub action builds uncomment the following line:
# usePrebuilt = ('CI_BUILD' in os.environ) and os.environ["CI_BUILD"] == "Github" and (not self.noClean)
if 'Windows' == system:
self.exe = os.path.join(self.path, 'vcpkg.exe')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.bat'), '-disableMetrics' ]
self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-windows_x86_64_2024.06.15.zip'
self.vcpkgHash = 'f335234f0722c15376fb10747f558c18c83a3e1e3b6565cf0dabfb18c9625a99234d054457fd05190c0ecd7a59ca43305bc93b50dbf764a4e1f567a15168d051'
self.hostTriplet = 'x64-windows'
if usePrebuilt:
self.prebuiltArchive = self.assets_url + "/dependencies/vcpkg/builds/vcpkg-win32.zip%3FversionId=3SF3mDC8dkQH1JP041m88xnYmWNzZflx"
elif 'Darwin' == system:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-osx-x86_64-2022.06.16.1.tar.xz'
self.vcpkgHash = '4dfdb3d8c40440330d50b54073e59218334379d0cb3ca437252a29d3c8674c0eeeec9acd3927488a5ce96cbcdf411632d3576baf1e279181233209ec01761d1d'
self.hostTriplet = 'x64-osx'
elif 'Linux' == system and 'aarch64' == machine:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-linux_aarch64_2023.11.20.tar.xz'
self.vcpkgHash = 'f38efba40bd4b0b6df47986e373d5535d3e787e257cf19d66ee8ee00e670a6fb95b3e824020024f3edbdcf86a0548e5bbddcc0ac7bd2ff6352a245efac8402fe'
self.hostTriplet = 'arm64-linux'
else:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-linux_amd64_2023.10.19.tar.xz'
self.vcpkgHash = '6c26ff73d6348e121cca47e90d5358587bf83ba22852acb195b76fbf0473070b24512c8fdd3216d26f03515a79c085f239272ef87c7020cc578cc79abbbd338d'
self.hostTriplet = 'x64-linux'
if self.args.android:
self.triplet = 'arm64-android'
self.androidPackagePath = os.getenv('HIFI_ANDROID_PRECOMPILED', os.path.join(self.path, 'android'))
else:
self.triplet = self.hostTriplet
def readVar(self, var):
with open(os.path.join(self.args.build_root, '_env', var + ".txt")) as fp:
return fp.read()
def writeVar(self, var, value):
with open(os.path.join(self.args.build_root, '_env', var + ".txt"), 'w') as fp:
fp.write(value)
def upToDate(self):
# Prevent doing a clean if we've explcitly set a directory for vcpkg
if self.noClean:
return True
if self.args.force_build:
print("Force build, out of date")
return False
if not os.path.isfile(self.exe):
print("Exe file {} not found, out of date".format(self.exe))
return False
if not os.path.isfile(self.tagFile):
print("Tag file {} not found, out of date".format(self.tagFile))
return False
with open(self.tagFile, 'r') as f:
storedTag = f.read()
if storedTag != self.tagContents:
print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents))
return False
return True
def copyEnv(self):
print("Passing on variables to vcpkg")
srcEnv = os.path.join(self.args.build_root, "_env")
destEnv = os.path.join(self.path, "_env")
if path.exists(destEnv):
shutil.rmtree(destEnv)
shutil.copytree(srcEnv, destEnv)
def clean(self):
print("Cleaning vcpkg installation at {}".format(self.path))
if os.path.isdir(self.path):
print("Removing {}".format(self.path))
shutil.rmtree(self.path, ignore_errors=True)
# Make sure the VCPKG prerequisites are all there.
def bootstrap(self):
if self.upToDate():
self.copyEnv()
return
if self.prebuiltArchive is not None:
return
self.clean()
downloadVcpkg = False
if self.args.force_bootstrap:
print("Forcing bootstrap")
downloadVcpkg = True
if not downloadVcpkg and not os.path.isfile(self.exe):
print("Missing executable, boot-strapping")
downloadVcpkg = True
# Make sure we have a vcpkg executable
testFile = os.path.join(self.path, '.vcpkg-root')
if not downloadVcpkg and not os.path.isfile(testFile):
print("Missing {}, bootstrapping".format(testFile))
downloadVcpkg = True
if downloadVcpkg:
if "HIFI_VCPKG_BOOTSTRAP" in os.environ:
print("Cloning vcpkg from github to {}".format(self.path))
hifi_utils.executeSubprocess(['git', 'clone', 'https://github.com/microsoft/vcpkg', self.path])
print("Bootstrapping vcpkg")
hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv)
else:
print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path)
print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
if (os.path.islink(portsPath)):
os.unlink(portsPath)
if (os.path.isdir(portsPath)):
shutil.rmtree(portsPath, ignore_errors=True)
shutil.copytree(self.sourcePortsPath, portsPath)
self.copyEnv()
def run(self, commands):
actualCommands = [self.exe, '--vcpkg-root', self.path]
actualCommands.extend(commands)
print("Running command")
print(actualCommands)
hifi_utils.executeSubprocess(actualCommands, folder=self.path, env=self.buildEnv)
def copyTripletForBuildType(self, triplet):
print('Copying triplet ' + triplet + ' to have build type ' + self.vcpkgBuildType)
tripletPath = os.path.join(self.path, 'triplets', triplet + '.cmake')
tripletForBuildTypePath = os.path.join(self.path, 'triplets', self.getTripletWithBuildType(triplet) + '.cmake')
shutil.copy(tripletPath, tripletForBuildTypePath)
with open(tripletForBuildTypePath, "a") as tripletForBuildTypeFile:
tripletForBuildTypeFile.write("set(VCPKG_BUILD_TYPE " + self.vcpkgBuildType + ")\n")
def getTripletWithBuildType(self, triplet):
if (not self.vcpkgBuildType):
return triplet
return triplet + '-' + self.vcpkgBuildType
def setupDependencies(self, qt=None):
if self.prebuiltArchive:
if not os.path.isfile(self.prebuildTagFile):
print('Extracting ' + self.prebuiltArchive + ' to ' + self.path)
hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path)
self.writePrebuildTag()
return
if qt is not None:
self.buildEnv['QT_CMAKE_PREFIX_PATH'] = qt
# Special case for android, grab a bunch of binaries
# FIXME remove special casing for android builds eventually
if self.args.android:
print("Installing Android binaries")
self.setupAndroidDependencies()
print("Installing host tools")
if (self.vcpkgBuildType):
self.copyTripletForBuildType(self.hostTriplet)
self.run(['install', '--triplet', self.getTripletWithBuildType(self.hostTriplet), 'hifi-host-tools'])
# If not android, install the hifi-client-deps libraries
if not self.args.android:
print("Installing build dependencies")
if (self.vcpkgBuildType):
self.copyTripletForBuildType(self.triplet)
self.run(['install', '--triplet', self.getTripletWithBuildType(self.triplet), 'hifi-client-deps'])
def cleanBuilds(self):
if self.noClean:
return
# Remove temporary build artifacts
builddir = os.path.join(self.path, 'buildtrees')
if os.path.isdir(builddir):
print("Wiping build trees")
shutil.rmtree(builddir, ignore_errors=True)
# Removes large files used to build the vcpkg, for CI purposes.
def cleanupDevelopmentFiles(self):
shutil.rmtree(os.path.join(self.path, "downloads"), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, "packages"), ignore_errors=True)
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = self.assets_url + "/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
print("Installing additional android archives")
androidPackages = hifi_android.getPlatformPackages()
for packageName in androidPackages:
package = androidPackages[packageName]
dest = os.path.join(self.androidPackagePath, packageName)
if os.path.isdir(dest):
continue
url = hifi_android.getPackageUrl(package)
zipFile = package['file'].endswith('.zip')
print("Android archive {}".format(package['file']))
hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5())
def writeTag(self):
if self.noClean:
return
print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
if not os.path.isdir(self.path):
os.makedirs(self.path)
with open(self.tagFile, 'w') as f:
f.write(self.tagContents)
def writePrebuildTag(self):
print("Writing tag {} to {}".format(self.tagContents, self.tagFile))
with open(self.prebuildTagFile, 'w') as f:
f.write(self.tagContents)
def fixupCmakeScript(self):
cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
newCmakeScript = cmakeScript + '.new'
isFileChanged = False
removalPrefix = "set(VCPKG_TARGET_TRIPLET "
# Open original file in read only mode and dummy file in write mode
with open(cmakeScript, 'r') as read_obj, open(newCmakeScript, 'w') as write_obj:
# Line by line copy data from original file to dummy file
for line in read_obj:
if not line.startswith(removalPrefix):
write_obj.write(line)
else:
isFileChanged = True
if isFileChanged:
shutil.move(newCmakeScript, cmakeScript)
else:
os.remove(newCmakeScript)
def writeConfig(self):
print("Writing cmake config to {}".format(self.configFilePath))
# Write out the configuration for use by CMake
cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake')
installPath = os.path.join(self.path, 'installed', self.getTripletWithBuildType(self.triplet))
toolsPath = os.path.join(self.path, 'installed', self.getTripletWithBuildType(self.hostTriplet), 'tools')
cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE
if self.args.android:
precompiled = os.path.realpath(self.androidPackagePath)
cmakeTemplate += 'set(HIFI_ANDROID_PRECOMPILED "{}")\n'.format(precompiled)
else:
cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID
cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath, self.getTripletWithBuildType(self.hostTriplet)).replace('\\', '/')
with open(self.configFilePath, 'w') as f:
f.write(cmakeConfig)
def cleanOldBuilds(self):
# FIXME because we have the base directory, and because a build will
# update the tag file on every run, we can scan the base dir for sub directories containing
# a tag file that is older than N days, and if found, delete the directory, recovering space
print("Not implemented")

220
prebuild.py
View file

@ -1,220 +0,0 @@
#!python
# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring
# tools required by the build as well as dependencies on which we rely.
#
# By using this script, we can reduce the requirements for a developer getting started to:
#
# * A working C++ dev environment like visual studio, xcode, gcc, or clang
# * Qt
# * CMake
# * Python 3.x
#
# The function of the build script is to acquire, if not already present, all the other build requirements
# The build script should be idempotent. If you run it with the same arguments multiple times, that should
# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not
# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should
# execute quickly, determining that no work is to be done
import hifi_singleton
import hifi_utils
import hifi_android
import hifi_vcpkg
import hifi_qt
import argparse
import concurrent
import hashlib
import importlib
import json
import os
import platform
import shutil
import ssl
import sys
import re
import tempfile
import time
import functools
import subprocess
import logging
from uuid import uuid4
from contextlib import contextmanager
print = functools.partial(print, flush=True)
class TrackableLogger(logging.Logger):
guid = str(uuid4())
def _log(self, msg, *args, **kwargs):
x = {'guid': self.guid}
if 'extra' in kwargs:
kwargs['extra'].update(x)
else:
kwargs['extra'] = x
super()._log(msg, *args, **kwargs)
logging.setLoggerClass(TrackableLogger)
logger = logging.getLogger('prebuild')
@contextmanager
def timer(name):
''' Print the elapsed time a context's execution takes to execute '''
start = time.time()
yield
# Please take care when modifiying this print statement.
# Log parsing logic may depend on it.
logger.info('%s took %.3f secs' % (name, time.time() - start))
def parse_args():
# our custom ports, relative to the script location
defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports')
from argparse import ArgumentParser
parser = ArgumentParser(description='Prepare build dependencies.')
parser.add_argument('--android', type=str)
parser.add_argument('--debug', action='store_true')
parser.add_argument('--force-bootstrap', action='store_true')
parser.add_argument('--force-build', action='store_true')
parser.add_argument('--release-type', type=str, default="DEV", help="DEV, PR, or PRODUCTION")
parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution')
parser.add_argument('--vcpkg-build-type', type=str, help='Could be `release` or `debug`. By default it doesn`t set the build-type')
parser.add_argument('--vcpkg-skip-clean', action='store_true', help='Skip the cleanup of vcpkg downloads and packages folders after vcpkg build completition.')
parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build')
parser.add_argument('--ports-path', type=str, default=defaultPortsPath)
parser.add_argument('--ci-build', action='store_true', default=os.getenv('CI_BUILD') is not None)
parser.add_argument('--get-vcpkg-id', action='store_true', help='Get the VCPKG ID, the hash path of the full VCPKG path')
parser.add_argument('--get-vcpkg-path', action='store_true', help='Get the full VCPKG path, ID included.')
parser.add_argument('--quiet', action='store_true', default=False, help='Quiet mode with less output')
if True:
args = parser.parse_args()
else:
args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/overte/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a'])
return args
def main():
# Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build
# VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for
# building OpenSSL
removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE']
for var in removeEnvVars:
if var in os.environ:
del os.environ[var]
args = parse_args()
if args.get_vcpkg_id or args.get_vcpkg_path:
# These arguments need quiet mode to avoid confusing scripts that use them.
args.quiet = True
if not args.quiet:
print(sys.argv)
if args.ci_build:
logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO)
logger.info('start')
pm = hifi_vcpkg.VcpkgRepo(args)
if args.get_vcpkg_id:
print(pm.id)
exit(0)
if args.get_vcpkg_path:
print(pm.path)
exit(0)
assets_url = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_BUILD_ASSETS')
# OS dependent information
system = platform.system()
if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github":
logger.info("Downloading NSIS")
with timer('NSIS'):
hifi_utils.downloadAndExtract(assets_url + '/dependencies/NSIS-hifi-plugins-1.0.tgz', "C:/Program Files (x86)")
qtInstallPath = None
# If not android, install our Qt build
if not args.android:
qt = hifi_qt.QtDownloader(args)
qtInstallPath = qt.cmakePath
if qtInstallPath is not None:
# qtInstallPath is None when we're doing a system Qt build
print("cmake path: " + qtInstallPath)
with hifi_singleton.Singleton(qt.lockFile) as lock:
with timer('Qt'):
qt.installQt()
qt.writeConfig()
else:
if (os.environ["OVERTE_USE_SYSTEM_QT"]):
if not args.quiet:
print("System Qt selected")
else:
raise Exception("Internal error: System Qt not selected, but hifi_qt.py failed to return a cmake path")
if qtInstallPath is not None:
pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath)
# Only allow one instance of the program to run at a time
if qtInstallPath is not None:
pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath)
# Only allow one instance of the program to run at a time
with hifi_singleton.Singleton(pm.lockFile) as lock:
with timer('Bootstraping'):
if not pm.upToDate():
pm.bootstrap()
# Always write the tag, even if we changed nothing. This
# allows vcpkg to reclaim disk space by identifying directories with
# tags that haven't been touched in a long time
pm.writeTag()
# Grab our required dependencies:
# * build host tools, like spirv-cross and scribe
# * build client dependencies like openssl and nvtt
with timer('Setting up dependencies'):
pm.setupDependencies(qt=qtInstallPath)
# wipe out the build directories (after writing the tag, since failure
# here shouldn't invalidate the vcpkg install)
if not args.vcpkg_skip_clean:
with timer('Cleaning builds'):
pm.cleanBuilds()
# If we're running in android mode, we also need to grab a bunch of additional binaries
# (this logic is all migrated from the old setupDependencies tasks in gradle)
if args.android:
# Find the target location
appPath = hifi_utils.scriptRelative('android/apps/' + args.android)
# Copy the non-Qt libraries specified in the config in hifi_android.py
hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath)
# Determine the Qt package path
qtPath = os.path.join(pm.androidPackagePath, 'qt')
hifi_android.QtPackager(appPath, qtPath).bundle()
# Fixup the vcpkg cmake to not reset VCPKG_TARGET_TRIPLET
pm.fixupCmakeScript()
if not args.vcpkg_skip_clean:
# Cleanup downloads and packages folders in vcpkg to make it smaller for CI
pm.cleanupDevelopmentFiles()
# Write the vcpkg config to the build directory last
with timer('Writing configuration'):
pm.writeConfig()
logger.info('end')
try:
main()
except hifi_utils.SilentFatalError as fatal_ex:
sys.exit(fatal_ex.exit_code)