From e3f148e83b0596f91c7bb54caf399984f425c092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gro=C3=9F?= Date: Sun, 31 Jul 2022 22:45:23 +0200 Subject: [PATCH] Revert "Update dump.yml, master_build.yml, and 3 more files..." This reverts commit 5ad38742d99b4df8599b15284cc729ce57f1afda. --- .github/workflows/dump.yml | 46 ++ .github/workflows/master_build.yml | 364 ++++++++++++++++ .github/workflows/master_deploy_apidocs.yml | 40 ++ .github/workflows/master_deploy_doxygen.yml | 38 ++ .github/workflows/pr_build.yml | 444 ++++++++++++++++++++ 5 files changed, 932 insertions(+) create mode 100644 .github/workflows/dump.yml create mode 100644 .github/workflows/master_build.yml create mode 100644 .github/workflows/master_deploy_apidocs.yml create mode 100644 .github/workflows/master_deploy_doxygen.yml create mode 100644 .github/workflows/pr_build.yml diff --git a/.github/workflows/dump.yml b/.github/workflows/dump.yml new file mode 100644 index 0000000000..dc7dd686d0 --- /dev/null +++ b/.github/workflows/dump.yml @@ -0,0 +1,46 @@ +name: Dump Contexts + +on: + push: + branches: + - master + + +jobs: + one: + strategy: + matrix: + os: [windows-latest, macOS-latest] + + runs-on: ${{ matrix.os }} + steps: + - name: Dump GitHub context + shell: bash + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + shell: bash + env: + JOB_CONTEXT: ${{ toJson(job) }} + run: echo "$JOB_CONTEXT" + - name: Dump steps context + shell: bash + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + - name: Dump runner context + shell: bash + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + run: echo "$RUNNER_CONTEXT" + - name: Dump strategy context + shell: bash + env: + STRATEGY_CONTEXT: ${{ toJson(strategy) }} + run: echo "$STRATEGY_CONTEXT" + - name: Dump matrix context + shell: bash + env: + MATRIX_CONTEXT: ${{ toJson(matrix) }} + run: echo "$MATRIX_CONTEXT" diff --git a/.github/workflows/master_build.yml b/.github/workflows/master_build.yml new file mode 100644 index 0000000000..1d83e31297 --- /dev/null +++ b/.github/workflows/master_build.yml @@ -0,0 +1,364 @@ +name: Master CI Build + +on: + push: + branches: + - master + +env: + APP_NAME: interface + BUILD_TYPE: Release + BUILD_NUMBER: ${{ github.run_number }} + CI_BUILD: Github + GIT_COMMIT: ${{ github.sha }} + # VCPKG did not build well on OSX disabling HIFI_VCPKG_BOOTSTRAP, which invokes a download to a working version of vcpkg + # HIFI_VCPKG_BOOTSTRAP: true + RELEASE_TYPE: PRODUCTION + RELEASE_NUMBER: ${{ github.run_number }} + STABLE_BUILD: 0 + UPLOAD_BUCKET: overte-public + UPLOAD_REGION: fra1 + UPLOAD_ENDPOINT: "https://fra1.digitaloceanspaces.com" + + # OSX-specific variables + DEVELOPER_DIR: /Applications/Xcode_11.2.app/Contents/Developer + # We are using macOS SDK 10.12 to work around a lot of OpenGL issues in newer SDKs. + # There are crashes on startup, crashes on window resize, etc. on newer macOS SDKs. + MACOSX_DEPLOYMENT_TARGET: '10.12' + + # WIN-specific variables + PreferredToolArchitecture: X64 + +jobs: + build: + strategy: + matrix: + include: + - os: windows-2019 + build_type: full + - os: macOS-10.15 + build_type: full + - os: ubuntu-18.04 + build_type: full + apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 python3-github python3-distro + fail-fast: false + runs-on: ${{matrix.os}} + steps: + - name: Report Build Number + shell: bash + run: | + + echo "Build number: $BUILD_NUMBER" + + - name: Configure build environment 1 + shell: bash + id: buildenv1 + run: | + + echo "UPLOAD_PREFIX=build/overte/master" >> $GITHUB_ENV + echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7` + echo "JOB_NAME=build (${{matrix.os}}, ${{matrix.build_type}})" >> $GITHUB_ENV + echo "APP_TARGET_NAME=$APP_NAME" >> $GITHUB_ENV + # Linux build variables + if [[ "${{ matrix.os }}" = "ubuntu-"* ]]; then + echo "PYTHON_EXEC=python3" >> $GITHUB_ENV + echo "INSTALLER_EXT=tgz" >> $GITHUB_ENV + echo "CMAKE_BUILD_EXTRA=-- -j$(nproc)" >> $GITHUB_ENV + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE=-msse3 -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV + fi + # Mac build variables + if [ "${{ matrix.os }}" = "macOS-10.15" ]; then + echo "PYTHON_EXEC=python3" >> $GITHUB_ENV + echo "ZIP_COMMAND=zip" >> $GITHUB_ENV + echo "ZIP_ARGS=-r" >> $GITHUB_ENV + echo "INSTALLER_EXT=dmg" >> $GITHUB_ENV + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DCMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.12.sdk -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=OFF -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DOPENSSL_LIBRARIES=/usr/local/opt/openssl/lib -G Xcode" >> $GITHUB_ENV + echo "::set-output name=symbols_archive::${BUILD_NUMBER}-${{ matrix.build_type }}-mac-symbols.zip" + echo "APP_TARGET_NAME=Vircadia" >> $GITHUB_ENV + fi + # Windows build variables + if [ "${{ matrix.os }}" = "windows-2019" ]; then + echo "PYTHON_EXEC=python" >> $GITHUB_ENV + echo "ZIP_COMMAND=7z" >> $GITHUB_ENV + echo "ZIP_ARGS=a" >> $GITHUB_ENV + echo "INSTALLER_EXT=exe" >> $GITHUB_ENV + echo "CMAKE_EXTRA=-A x64" >> $GITHUB_ENV + echo "SYMBOL_REGEX=\(exe\|dll\|pdb\)" >> $GITHUB_ENV + echo "symbols_archive=${BUILD_NUMBER}-${{ matrix.build_type }}-win-symbols.zip" >> $GITHUB_ENV + # echo "HF_PFX_PASSPHRASE=${{secrets.pfx_key}}" >> $GITHUB_ENV + # echo "HF_PFX_FILE=${{runner.workspace}}\build\codesign.pfx" >> $GITHUB_ENV + fi + # Configuration is broken into two steps because you can't set an env var and also reference it in the same step + - name: Configure build environment 2 + shell: bash + run: | + + echo "${{ steps.buildenv1.outputs.symbols_archive }}" + echo "ARTIFACT_PATTERN=Overte-*.$INSTALLER_EXT" >> $GITHUB_ENV + echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV + # Build type variables + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "CLIENT_ONLY=FALSE" >> $GITHUB_ENV + echo "INSTALLER=Overte-master$BUILD_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + else + echo "CLIENT_ONLY=TRUE" >> $GITHUB_ENV + echo "INSTALLER=Overte-Interface-master$BUILD_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + fi + - name: Clear working directory + if: startsWith(matrix.os, 'windows') + shell: bash + working-directory: ${{runner.workspace}} + run: rm -rf ./* + - uses: actions/checkout@v1 + with: + submodules: false + fetch-depth: 1 + + - name: Install dependencies + shell: bash + if: startsWith(matrix.os, 'ubuntu') || contains(matrix.os, 'debian') || startsWith(matrix.os, 'macOS') + run: | + if [[ "${{ matrix.os }}" =~ "ubuntu" || "${{ matrix.os }}" =~ "debian" ]]; then + + echo "Updating apt repository index" + sudo apt update || exit 1 + + echo "Installing apt packages" + sudo apt install -y ${{ matrix.apt-dependencies }} || exit 1 + + echo "Installing Python Modules" + pip3 install boto3 || exit 1 + + else # macOS + echo "Downloading MacOSX10.12 SDK.." + curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://data.moto9000.moe/overte_packages/macOS_SDK10.12.4.tar.xz" || exit 1 + echo " done\n" + echo "Checking MacOSX10.12 SDK.." + echo "c00e03cb64bed8d0d75d7a01c93de094b6ffaa3bc3318e7b29a9bdc354f0e29e363c454b7db89db8a3b78e9702668d1c661f1af279807de8db11accecb0ca498 macOS_SDK10.12.4.tar.xz" >> macOS_SDK10.12.4.tar.xz.checksum + shasum -a 512 -w -c macOS_SDK10.12.4.tar.xz.checksum + echo "Extracting MacOSX10.12 SDK.." + tar -xf macOS_SDK10.12.4.tar.xz || exit 1 + echo " done\n" + echo "Installing MacOSX10.12 SDK.." + # cp always sets an error code while copying the SDK so we return 0 when that happens. + sudo cp -rp MacOSX10.12.sdk /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ || exit 0 + echo " done" + fi + + - name: Override NSIS + shell: pwsh + if: startsWith(matrix.os, 'windows') + run: choco install nsis --version=3.06.1 + + - name: Install Python modules + if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + shell: bash + run: $PYTHON_EXEC -m pip install boto3 PyGithub + + - name: Create build environment + shell: bash + run: cmake -E make_directory "${{runner.workspace}}/build" + + - name: Configure CMake + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DVCPKG_BUILD_TYPE=release -DCLIENT_ONLY:BOOLEAN=$CLIENT_ONLY -DBYPASS_SIGNING:BOOLEAN=TRUE $CMAKE_EXTRA + + - name: Build application + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target $APP_TARGET_NAME $CMAKE_BUILD_EXTRA + + - name: Build domain server + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target domain-server $CMAKE_BUILD_EXTRA + + - name: Build assignment client + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target assignment-client $CMAKE_BUILD_EXTRA + + - name: Build console + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target packaged-server-console $CMAKE_BUILD_EXTRA + + - name: Build installer + working-directory: ${{runner.workspace}}/build + shell: bash + run: | + echo "Retry code from https://unix.stackexchange.com/a/137639" + function fail { + echo $1 >&2 + exit 1 + } + function retry { + local n=1 + local max=5 + local delay=15 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + sleep $delay; + else + fail "The command has failed after $n attempts." + fi + } + done + } + retry cmake --build . --config $BUILD_TYPE --target package $CMAKE_BUILD_EXTRA + + #- name: Sign installer (Windows) + # if: startsWith(matrix.os, 'windows') + # shell: powershell + # working-directory: C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64 + # run: .\signtool.exe sign /fd sha256 /f ${{runner.workspace}}\build\codesign.pfx /p ${{secrets.pfx_key}} /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /td SHA256 ${{runner.workspace}}\build\${env:INSTALLER} + + - name: Output system stats + if: ${{ always() }} + working-directory: ${{runner.workspace}}/build + shell: bash + run: | + echo "Disk usage:" + df -h + + - name: Output installer logs + if: failure() && startsWith(matrix.os, 'windows') + shell: bash + working-directory: ${{runner.workspace}}/build + run: cat ./_CPack_Packages/win64/NSIS/NSISOutput.log + + - name: Upload artifact + # Linux archives are currently dummies + if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') || matrix.build_type == 'android' + shell: bash + working-directory: ${{runner.workspace}}/build + env: + AWS_ACCESS_KEY_ID: ${{ secrets.s3_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.s3_secret_access_key }} + run: $PYTHON_EXEC $GITHUB_WORKSPACE/tools/ci-scripts/upload.py + + #- name: Archive symbols + # if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + # working-directory: ${{runner.workspace}} + # shell: bash + # run: | + # SYMBOLS_TEMP="symbols-temp" + # mkdir $SYMBOLS_TEMP + # find "./build" \( -path '*/tools/gpu-frame-player/*' -or -path '*/interface/*' -or -path '*/plugins/*' \) -regex ".*\.$SYMBOL_REGEX" -exec cp -r {} $SYMBOLS_TEMP \; + # cd $SYMBOLS_TEMP + # $ZIP_COMMAND $ZIP_ARGS ../${{ steps.buildenv1.outputs.symbols_archive }} . + #- name: Upload symbols + # if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + # working-directory: ${{runner.workspace}} + # shell: bash + # run: | + # curl --data-binary @${{ steps.buildenv1.outputs.symbols_archive }} "$CMAKE_BACKTRACE_URL/post?format=symbols&token=$CMAKE_BACKTRACE_SYMBOLS_TOKEN&upload_file=${{steps.buildenv1.outputs.symbols_archive}}&tag=$BUILD_NUMBER" + #- name: Debug list symbols + # if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + # working-directory: ${{runner.workspace}} + # shell: bash + # run: | + # unzip -v "${{runner.workspace}}/${{ steps.buildenv1.outputs.symbols_archive }}" + #- name: Upload debug list symbols + # if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + # uses: actions/upload-artifact@v1 + # with: + # name: symbols + # path: ${{runner.workspace}}/${{ steps.buildenv1.outputs.symbols_archive }} + + build_docker_server: + name: "${{matrix.os}}, Debian Docker" + container: docker.io/juliangro/overte-server-build:0.1 + strategy: + matrix: + include: + - os: ubuntu-20.04 + # Do not change the names of self-hosted runners without knowing what you are doing, as they correspond to labels that have to be set on the runner. + - os: self-hosted_debian-11_aarch64 + fail-fast: false + runs-on: ${{matrix.os}} + steps: + - name: Report Build Number + shell: bash + run: | + echo "Build number: $BUILD_NUMBER" + + - uses: actions/checkout@v1 + with: + submodules: false + fetch-depth: 1 + + - name: Configure Build Environment + shell: bash + id: env1 + run: | + echo "UPLOAD_PREFIX=build/overte/master" >> $GITHUB_ENV + echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7` + echo "JOB_NAME=${{matrix.os}}, ${{matrix.build_type}}" >> $GITHUB_ENV + echo "VIRCADIA_USE_SYSTEM_QT=true" >> $GITHUB_ENV + if [[ "${{ matrix.os }}" = *"aarch64" ]]; then + echo "VCPKG_FORCE_SYSTEM_BINARIES=true" >> $GITHUB_ENV + fi + + - name: Create build environment + shell: bash + run: | + mkdir build + + - name: Configure CMake + working-directory: ./build + run: | + if [ "$(uname -m)" = "x86_64" ]; then + VIRCADIA_USE_SYSTEM_QT=1 STABLE_BUILD=0 cmake .. -DSERVER_ONLY=1 -DBUILD_TOOLS=0 -DVIRCADIA_CPU_ARCHITECTURE=-msse3 -DCMAKE_BUILD_TYPE=Release -DVCPKG_BUILD_TYPE=release + else + VIRCADIA_USE_SYSTEM_QT=1 STABLE_BUILD=0 cmake .. -DSERVER_ONLY=1 -DBUILD_TOOLS=0 -DVIRCADIA_CPU_ARCHITECTURE= -DCMAKE_BUILD_TYPE=Release -DVCPKG_BUILD_TYPE=release + fi + + - name: Build Domain Server + working-directory: ./build + shell: bash + run: make -j$(nproc) domain-server + + - name: Build Assignment Client + working-directory: ./build + shell: bash + run: make -j$(nproc) assignment-client + + - name: Create Docker runtime image + run: docker build -t overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" -f tools/ci-scripts/docker_package/Dockerfile_runtime . + + - name: Export Docker runtime image + run : | + if [ "$(uname -m)" = "x86_64" ]; then + echo "Exporting image.." + docker image save -o overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" + echo "Compressing with xz.. + xz -z -T0 overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar + echo "Done." + echo "ARTIFACT_PATTERN=overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar.xz" >> $GITHUB_ENV + else + echo "Exporting image.." + docker image save -o overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" + echo "Compressing with xz.. + xz -z -T0 overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar + echo "Done." + echo "ARTIFACT_PATTERN=overte-server-master${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar.xz" >> $GITHUB_ENV + fi + + - name: Output system stats + if: ${{ always() }} + shell: bash + run: | + echo "Disk usage:" + df -h + + - name: Upload artifact + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.s3_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.s3_secret_access_key }} + run: python3 ./tools/ci-scripts/upload.py diff --git a/.github/workflows/master_deploy_apidocs.yml b/.github/workflows/master_deploy_apidocs.yml new file mode 100644 index 0000000000..a29ae62475 --- /dev/null +++ b/.github/workflows/master_deploy_apidocs.yml @@ -0,0 +1,40 @@ +name: Master API-docs CI Build and Deploy + +on: + push: + branches: + - master + +jobs: + build_site: + runs-on: ubuntu-20.04 + + name: Build and deploy API-docs + steps: + - uses: actions/checkout@v2 + + - name: Install dependencies + working-directory: tools/jsdoc + run: | + sudo apt update + sudo apt install npm nodejs + npm install jsdoc -g + npm install + + - name: Build API-docs + working-directory: tools/jsdoc + shell: bash + run: | + jsdoc root.js -r api-mainpage.md -c config.json -d output + + - name: Deploy API-docs + uses: SamKirkland/FTP-Deploy-Action@4.3.0 + with: + server: ftp.tuxfamily.org + username: ${{ secrets.GHA_JSDOC_FTP_USER }} + password: ${{ secrets.GHA_JSDOC_FTP_PASS }} + local-dir: ./tools/jsdoc/output/ + server-dir: overte/apidocs.overte.org-web/htdocs/ + exclude: | + **/staging/** + .htaccess diff --git a/.github/workflows/master_deploy_doxygen.yml b/.github/workflows/master_deploy_doxygen.yml new file mode 100644 index 0000000000..b6ba363af3 --- /dev/null +++ b/.github/workflows/master_deploy_doxygen.yml @@ -0,0 +1,38 @@ +name: Master Doxygen CI Build and Deploy + +on: + push: + branches: + - master + +jobs: + build_doxygen: + runs-on: ubuntu-20.04 + + name: Build and deploy Doxygen documentation + steps: + - uses: actions/checkout@v2 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install doxygen + + - name: Build Doxygen + working-directory: tools/doxygen + shell: bash + run: | + mkdir -p ../../build/docs/html + doxygen Doxyfile + + - name: Deploy Doxygen + uses: SamKirkland/FTP-Deploy-Action@4.3.0 + with: + server: ftp.tuxfamily.org + username: ${{ secrets.GHA_JSDOC_FTP_USER }} + password: ${{ secrets.GHA_JSDOC_FTP_PASS }} + local-dir: ./build/docs/html/ + server-dir: overte/doxygen.overte.org-web/htdocs/ + exclude: | + **/staging/** + .htaccess diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 0000000000..54211d1061 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,444 @@ +name: Pull Request CI Build +# Keep in mind that GitHub Actions does not allow reading secrets during PR builds. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + +env: + APP_NAME: interface + BUILD_TYPE: Release + CI_BUILD: Github + GIT_COMMIT: ${{ github.sha }} + # VCPKG did not build well on OSX disabling HIFI_VCPKG_BOOTSTRAP, which invokes a download to a working version of vcpkg + # HIFI_VCPKG_BOOTSTRAP: true + RELEASE_TYPE: PR + RELEASE_NUMBER: ${{ github.event.number }} + VERSION_CODE: ${{ github.event.number }} + # Sentry Crash Reporting + CMAKE_BACKTRACE_URL: + CMAKE_BACKTRACE_TOKEN: PR_${{ github.event.number }}_${{ github.sha }} + + UPLOAD_BUCKET: overte-public + UPLOAD_REGION: fra1 + UPLOAD_ENDPOINT: "https://fra1.digitaloceanspaces.com" + + # OSX specific variables + DEVELOPER_DIR: /Applications/Xcode_11.2.app/Contents/Developer + # We are using macOS SDK 10.12 to work around a lot of OpenGL issues in newer SDKs. + # There are crashes on startup, crashes on window resize, etc. on newer macOS SDKs. + MACOSX_DEPLOYMENT_TARGET: '10.12' + + # WIN32 specific variables + PreferredToolArchitecture: X64 + + # Ensure XZ always runs with muticore support + XZ_OPT: -T0 + +jobs: + build_pr: + name: "${{matrix.os}}, ${{matrix.build_type}}" + strategy: + matrix: + include: + - os: windows-2019 + build_type: full + - os: macOS-10.15 + build_type: full + - os: ubuntu-18.04 + build_type: full + apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 python3-distro + - os: ubuntu-18.04 + build_type: android + apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 python3-github python3-distro + # Do not change the names of self-hosted runners without knowing what you are doing, as they correspond to labels that have to be set on the runner. + - os: self-hosted_debian-11_aarch64 + build_type: full + apt-dependencies: qtbase5-dev qtbase5-private-dev qtwebengine5-dev qtwebengine5-dev-tools qtmultimedia5-dev libqt5opengl5-dev qtscript5-dev libqt5scripttools5 libqt5webchannel5-dev libqt5websockets5-dev qtxmlpatterns5-dev-tools qttools5-dev libqt5xmlpatterns5-dev libqt5svg5-dev qml-module-qtwebchannel build-essential cmake curl freeglut3-dev git libasound2 libasound2-dev libdouble-conversion-dev libdrm-dev libfontconfig1 libgl1-mesa-dev libglvnd-dev libharfbuzz-dev libjack-dev libjack0 libnspr4 libnss3 libpcre2-16-0 libpulse0 libsdl2-dev libssl-dev libudev-dev libxcb-xinerama0-dev libxcb-xinput0 libxcomposite1 libxcursor1 libxi-dev libxmu-dev libxrandr-dev libxslt1.1 libxtst6 make mesa-common-dev mesa-utils nodejs npm patchelf python2 python3 python3-distro xdg-user-dirs zlib1g-dev ninja-build zip python3-distro + fail-fast: false + runs-on: ${{matrix.os}} + if: github.event.action != 'labeled' || github.event.label.name == 'rebuild' + steps: + - name: Configure Build Environment 1 + shell: bash + id: buildenv1 + run: | + + echo "UPLOAD_PREFIX=build/overte/PR" >> $GITHUB_ENV + echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7` + echo "JOB_NAME=${{matrix.os}}, ${{matrix.build_type}}" >> $GITHUB_ENV + echo "APP_TARGET_NAME=$APP_NAME" >> $GITHUB_ENV + + # Linux build variables + if [[ "${{ matrix.os }}" = "ubuntu-"* || "${{ matrix.os }}" = *"debian"* ]]; then + echo "PYTHON_EXEC=python3" >> $GITHUB_ENV + echo "INSTALLER_EXT=*" >> $GITHUB_ENV + echo "CMAKE_BUILD_EXTRA=-- -j$(nproc)" >> $GITHUB_ENV + # Variables specific to our aarch64 runner + if [ "${{ matrix.os }}" = "self-hosted_debian-11_aarch64" ]; then + echo "VIRCADIA_USE_SYSTEM_QT=true" >> $GITHUB_ENV + echo "CI_WORKSPACE=${{runner.workspace}}" >> $GITHUB_ENV + # Don't optimize builds to save build time. + echo "VIRCADIA_OPTIMIZE=false" >> $GITHUB_ENV + fi + + if [[ "${{ matrix.os }}" = *"aarch64" ]]; then + echo "VCPKG_FORCE_SYSTEM_BINARIES=true" >> $GITHUB_ENV + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV + else + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DCLIENT_ONLY=1 -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV + fi + + fi + + if [[ "${{ matrix.os }}" != *"aarch64" ]]; then + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE=-msse3 -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV + else + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE=-msse3 -DCLIENT_ONLY=1 -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV + fi + fi + fi + + # Mac build variables + if [ "${{ matrix.os }}" = "macOS-10.15" ]; then + echo "PYTHON_EXEC=python3" >> $GITHUB_ENV + echo "INSTALLER_EXT=dmg" >> $GITHUB_ENV + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DCMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.12.sdk -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=OFF -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DOPENSSL_LIBRARIES=/usr/local/opt/openssl/lib -G Xcode" >> $GITHUB_ENV + else + echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DCMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.12.sdk -DCLIENT_ONLY=1 -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=OFF -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DOPENSSL_LIBRARIES=/usr/local/opt/openssl/lib -G Xcode" >> $GITHUB_ENV + fi + echo "APP_TARGET_NAME=Vircadia" >> $GITHUB_ENV + fi + # Windows build variables + if [ "${{ matrix.os }}" = "windows-2019" ]; then + echo "PYTHON_EXEC=python" >> $GITHUB_ENV + echo "INSTALLER_EXT=exe" >> $GITHUB_ENV + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "CMAKE_EXTRA=-A x64" >> $GITHUB_ENV + else + echo "CMAKE_EXTRA=-A x64 -DCLIENT_ONLY=1" >> $GITHUB_ENV + fi + fi + # Android + Quest build variables + if [[ "${{ matrix.build_type }}" == "android" ]]; then + HIFI_ANDROID_PRECOMPILED="${{runner.workspace}}/dependencies" + echo "HIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED" >> $GITHUB_ENV + mkdir $HIFI_ANDROID_PRECOMPILED + echo "INSTALLER_EXT=apk" >> $GITHUB_ENV + fi + + # Configuration is broken into two steps because you can't set an env var and also reference it in the same step + - name: Configure Build Environment 2 + shell: bash + run: | + + echo "${{ steps.buildenv1.outputs.symbols_archive }}" + # Setting short commit SHA for use as RELEASE_NAME in generated installers. + echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV + if [[ "${{ matrix.build_type }}" != "android" ]]; then + if [ "${{ matrix.build_type }}" = "full" ]; then + echo "ARTIFACT_PATTERN=Overte-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + echo "INSTALLER=Overte-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + else + echo "ARTIFACT_PATTERN=Overte-Interface-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + echo "INSTALLER=Overte-Interface-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + fi + else + echo "ARTIFACT_PATTERN=*.$INSTALLER_EXT" >> $GITHUB_ENV + fi + + - name: Clear Working Directory + if: startsWith(matrix.os, 'windows') || contains(matrix.os, 'self-hosted') + shell: bash + working-directory: ${{runner.workspace}} + run: rm -rf ./* + + - uses: actions/checkout@v1 + with: + submodules: false + fetch-depth: 1 + + - name: Install dependencies + shell: bash + if: startsWith(matrix.os, 'ubuntu') || contains(matrix.os, 'debian') || startsWith(matrix.os, 'macOS') + run: | + if [[ "${{ matrix.os }}" =~ "ubuntu" || "${{ matrix.os }}" =~ "debian" ]]; then + + echo "Updating apt repository index" + sudo apt update || exit 1 + + echo "Installing apt packages" + sudo apt install -y ${{ matrix.apt-dependencies }} || exit 1 + + else # macOS + echo "Downloading MacOSX10.12 SDK.." + curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://data.moto9000.moe/overte_packages/macOS_SDK10.12.4.tar.xz" || exit 1 + echo " done\n" + echo "Checking MacOSX10.12 SDK.." + echo "c00e03cb64bed8d0d75d7a01c93de094b6ffaa3bc3318e7b29a9bdc354f0e29e363c454b7db89db8a3b78e9702668d1c661f1af279807de8db11accecb0ca498 macOS_SDK10.12.4.tar.xz" >> macOS_SDK10.12.4.tar.xz.checksum + shasum -a 512 -w -c macOS_SDK10.12.4.tar.xz.checksum + echo "Extracting MacOSX10.12 SDK.." + tar -xf macOS_SDK10.12.4.tar.xz || exit 1 + echo " done\n" + echo "Installing MacOSX10.12 SDK.." + # cp always sets an error code while copying the SDK so we return 0 when that happens. + sudo cp -rp MacOSX10.12.sdk /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ || exit 0 + echo " done" + fi + + - name: Override NSIS + shell: pwsh + if: startsWith(matrix.os, 'windows') + run: choco install nsis --version=3.06.1 + + - name: Install Python modules + if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + shell: bash + run: $PYTHON_EXEC -m pip install boto3 PyGithub + + - name: Create Build Environment + shell: bash + run: cmake -E make_directory "${{runner.workspace}}/build" + + - name: Configure CMake + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DVCPKG_BUILD_TYPE=release $CMAKE_EXTRA + + - name: Compress cmake logs + if: always() + shell: bash + run: | + if [ "${{ matrix.os }}" == "macOS-10.15" ]; then + TAR=gtar + else + TAR=tar + fi + + if [ ${{ env.CI_WORKSPACE }} ]; then + find "$CI_WORKSPACE/vircadia-files/vcpkg" -name '*log' -type f -print0 | $TAR --null --force-local -T - -c --xz -v -f "${{ runner.workspace }}/cmake-logs-${{ matrix.os }}-${{ github.event.number }}.tar.xz" + else + find "$HOME/vircadia-files/vcpkg" -name '*log' -type f -print0 | $TAR --null --force-local -T - -c --xz -v -f "${{ runner.workspace }}/cmake-logs-${{ matrix.os }}-${{ github.event.number }}.tar.xz" + fi + + - name: Archive cmake logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: cmake-logs-${{ matrix.os }}-${{ github.event.number }}.tar.xz + path: ${{ runner.workspace }}/cmake-logs-${{ matrix.os }}-${{ github.event.number }}.tar.xz + if-no-files-found: error + + - name: Build Application + if: matrix.build_type == 'full' || matrix.build_type == 'client' + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target $APP_TARGET_NAME $CMAKE_BUILD_EXTRA + + - name: Build Domain Server + if: matrix.build_type == 'full' + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target domain-server $CMAKE_BUILD_EXTRA + + - name: Build Assignment Client + if: matrix.build_type == 'full' + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target assignment-client $CMAKE_BUILD_EXTRA + + - name: Build Console + if: matrix.build_type == 'full' || startsWith(matrix.os, 'windows') + working-directory: ${{runner.workspace}}/build + shell: bash + run: cmake --build . --config $BUILD_TYPE --target packaged-server-console $CMAKE_BUILD_EXTRA + + - name: Build Installer + if: matrix.build_type != 'android' + working-directory: ${{runner.workspace}}/build + shell: bash + run: | + echo "Retry code from https://unix.stackexchange.com/a/137639" + function fail { + echo $1 >&2 + exit 1 + } + + function retry { + local n=1 + local max=5 + local delay=15 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + sleep $delay; + else + fail "The command has failed after $n attempts." + fi + } + done + } + retry cmake --build . --config $BUILD_TYPE --target package $CMAKE_BUILD_EXTRA + # As of 05/17/21 GitHub Virtual Environments changed their "Ubuntu 18.04.5 LTS" image to include two versions of CMake for Android + # https://github.com/actions/virtual-environments/blob/ubuntu18/20210517.1/images/linux/Ubuntu1804-README.md + # Removing 3.18.1 version of CMake as its not compatible with our Android build. + # It will fall back to 3.10.2 which is already installed + - name: Nuke CMake 3.18.1-g262b901 + if: matrix.build_type == 'android' + shell: bash + run: | + /usr/local/lib/android/sdk/tools/bin/sdkmanager --uninstall 'cmake;3.18.1' + + - name: Build for Android + Quest + if: matrix.build_type == 'android' + shell: bash + working-directory: ${{runner.workspace}}/overte + run: | + echo "Pre-cache the vcpkg managed dependencies" + $PYTHON_EXEC prebuild.py --build-root ${{runner.workspace}}/build --android interface + cd android + # Pre-cache the gradle dependencies + ./gradlew -m tasks -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED + # Build! + ./build_android.sh + + - name: Output system stats + if: ${{ always() }} + working-directory: ${{runner.workspace}}/build + shell: bash + run: | + echo "Disk usage:" + df -h + + - name: Output Installer Logs + if: failure() && startsWith(matrix.os, 'windows') + shell: bash + working-directory: ${{runner.workspace}}/build + run: cat ./_CPack_Packages/win64/NSIS/NSISOutput.log + + - name: Upload Artifact + # Linux archives are currently dummies + if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') || matrix.build_type == 'android' + shell: bash + working-directory: ${{runner.workspace}}/build + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + if [[ "${{ matrix.build_type }}" == "android" ]]; then + cd $GITHUB_WORKSPACE/android + fi + $PYTHON_EXEC "$GITHUB_WORKSPACE/tools/ci-scripts/upload_to_publish_server.py" + + # Frees up memory on the self-hosted runner + - name: Clear Working Directory + if: contains(matrix.os, 'self-hosted') + shell: bash + working-directory: ${{runner.workspace}} + run: rm -rf ./* + + + build_docker_server: + name: "${{matrix.os}}, Debian Docker" + container: docker.io/juliangro/overte-server-build:0.1 + strategy: + matrix: + include: + - os: ubuntu-20.04 + # Do not change the names of self-hosted runners without knowing what you are doing, as they correspond to labels that have to be set on the runner. + - os: self-hosted_debian-11_aarch64 + fail-fast: false + runs-on: ${{matrix.os}} + if: github.event.action != 'labeled' || github.event.label.name == 'rebuild' + steps: + - name: Report Build Number + shell: bash + run: | + echo "Build number: $BUILD_NUMBER" + + - uses: actions/checkout@v1 + with: + submodules: false + fetch-depth: 1 + + - name: Configure Build Environment + shell: bash + id: env1 + run: | + echo "UPLOAD_PREFIX=build/overte/PR" >> $GITHUB_ENV + echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7` + echo "JOB_NAME=${{matrix.os}}, ${{matrix.build_type}}" >> $GITHUB_ENV + echo "VIRCADIA_USE_SYSTEM_QT=true" >> $GITHUB_ENV + if [[ "${{ matrix.os }}" = *"aarch64" ]]; then + echo "VCPKG_FORCE_SYSTEM_BINARIES=true" >> $GITHUB_ENV + fi + + - name: Create build environment + shell: bash + run: | + mkdir build + + - name: Configure CMake + working-directory: ./build + run: | + if [ "$(uname -m)" = "x86_64" ]; then + VIRCADIA_USE_SYSTEM_QT=1 STABLE_BUILD=0 cmake .. -DSERVER_ONLY=1 -DBUILD_TOOLS=0 -DVIRCADIA_CPU_ARCHITECTURE=-msse3 -DCMAKE_BUILD_TYPE=Release -DVCPKG_BUILD_TYPE=release + else + VIRCADIA_USE_SYSTEM_QT=1 STABLE_BUILD=0 cmake .. -DSERVER_ONLY=1 -DBUILD_TOOLS=0 -DVIRCADIA_CPU_ARCHITECTURE= -DCMAKE_BUILD_TYPE=Release -DVCPKG_BUILD_TYPE=release + fi + + - name: Build Domain Server + working-directory: ./build + shell: bash + run: make -j$(nproc) domain-server + + - name: Build Assignment Client + working-directory: ./build + shell: bash + run: make -j$(nproc) assignment-client + + - name: Create Docker runtime image + run: docker build -t overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" -f tools/ci-scripts/docker_package/Dockerfile_runtime . + + - name: Export Docker runtime image + run : | + if [ "$(uname -m)" = "x86_64" ]; then + echo "Exporting image.." + docker image save -o overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" + echo "Compressing with xz.." + xz -z -T0 overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar + echo "Done." + echo "ARTIFACT_PATTERN=overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-amd64.tar.xz" >> $GITHUB_ENV + else + echo "Exporting image.." + docker image save -o overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar overte-org/overte-server:"$GIT_COMMIT"_"$BUILD_NUMBER" + echo "Compressing with xz.." + xz -z -T0 overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar + echo "Done." + echo "ARTIFACT_PATTERN=overte-server-PR${{ github.event.number }}-${{ steps.env1.outputs.github_sha_short }}-aarch64.tar.xz" >> $GITHUB_ENV + fi + + - name: Output system stats + if: ${{ always() }} + shell: bash + run: | + echo "Disk usage:" + df -h + + - name: Upload Artifact + shell: bash + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + python3 "$GITHUB_WORKSPACE/tools/ci-scripts/upload_to_publish_server.py" + + # Frees up memory on the self-hosted runner + - name: Clear Working Directory + if: contains(matrix.os, 'self-hosted') + shell: bash + working-directory: ${{runner.workspace}} + run: rm -rf ./*