From 37d21425824718bbc00b719fa2e4d523af699bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gro=C3=9F?= Date: Fri, 25 Mar 2022 14:37:19 +0100 Subject: [PATCH] Update PR CI builds --- .github/workflows/pr_build.yml | 61 ++++++++++++-------- tools/ci-scripts/upload_to_publish_server.py | 3 +- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 55dec1737d..4a6120669b 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -1,8 +1,9 @@ name: Pull Request CI Build +# Keep in mind that GitHub Actions does not allow reading secrets during PR builds. on: pull_request: - types: [opened, synchronize, reopened, labeled] + types: [opened, synchronize, reopened] env: APP_NAME: interface @@ -15,9 +16,13 @@ env: RELEASE_NUMBER: ${{ github.event.number }} VERSION_CODE: ${{ github.event.number }} # Sentry Crash Reporting - CMAKE_BACKTRACE_URL: ${{ secrets.MINIDUMP_TOKEN }} + CMAKE_BACKTRACE_URL: CMAKE_BACKTRACE_TOKEN: PR_${{ github.event.number }}_${{ github.sha }} + UPLOAD_BUCKET: overte-public + UPLOAD_REGION: fra1 + UPLOAD_ENDPOINT: "https://fra1.digitaloceanspaces.com" + # OSX specific variables DEVELOPER_DIR: /Applications/Xcode_11.2.app/Contents/Developer # We are using macOS SDK 10.12 to work around a lot of OpenGL issues in newer SDKs. @@ -31,43 +36,42 @@ env: XZ_OPT: -T0 jobs: - build: - name: "build (${{matrix.os}}, ${{matrix.build_type}})" + build_pr: + name: "${{matrix.os}}, ${{matrix.build_type}}" strategy: matrix: include: - - os: windows-latest + - os: windows-2019 build_type: full - os: macOS-10.15 build_type: full - - os: macOS-10.15 - build_type: client - os: ubuntu-18.04 build_type: full - apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 + apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 python3-distro - os: ubuntu-18.04 build_type: android - apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 + apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 python3-github python3-distro # Do not change the names of self-hosted runners without knowing what you are doing, as they correspond to labels that have to be set on the runner. - os: self-hosted_debian-11_aarch64 build_type: full - apt-dependencies: qtbase5-dev qtbase5-private-dev qtwebengine5-dev qtwebengine5-dev-tools qtmultimedia5-dev libqt5opengl5-dev qtscript5-dev libqt5scripttools5 libqt5webchannel5-dev libqt5websockets5-dev qtxmlpatterns5-dev-tools qttools5-dev libqt5xmlpatterns5-dev libqt5svg5-dev qml-module-qtwebchannel build-essential cmake curl freeglut3-dev git libasound2 libasound2-dev libdouble-conversion-dev libdrm-dev libfontconfig1 libgl1-mesa-dev libglvnd-dev libharfbuzz-dev libjack-dev libjack0 libnspr4 libnss3 libpcre2-16-0 libpulse0 libsdl2-dev libssl-dev libudev-dev libxcb-xinerama0-dev libxcb-xinput0 libxcomposite1 libxcursor1 libxi-dev libxmu-dev libxrandr-dev libxslt1.1 libxtst6 make mesa-common-dev mesa-utils nodejs npm patchelf python2 python3 python3-distro xdg-user-dirs zlib1g-dev ninja-build zip + apt-dependencies: qtbase5-dev qtbase5-private-dev qtwebengine5-dev qtwebengine5-dev-tools qtmultimedia5-dev libqt5opengl5-dev qtscript5-dev libqt5scripttools5 libqt5webchannel5-dev libqt5websockets5-dev qtxmlpatterns5-dev-tools qttools5-dev libqt5xmlpatterns5-dev libqt5svg5-dev qml-module-qtwebchannel build-essential cmake curl freeglut3-dev git libasound2 libasound2-dev libdouble-conversion-dev libdrm-dev libfontconfig1 libgl1-mesa-dev libglvnd-dev libharfbuzz-dev libjack-dev libjack0 libnspr4 libnss3 libpcre2-16-0 libpulse0 libsdl2-dev libssl-dev libudev-dev libxcb-xinerama0-dev libxcb-xinput0 libxcomposite1 libxcursor1 libxi-dev libxmu-dev libxrandr-dev libxslt1.1 libxtst6 make mesa-common-dev mesa-utils nodejs npm patchelf python2 python3 python3-distro xdg-user-dirs zlib1g-dev ninja-build zip python3-distro fail-fast: false runs-on: ${{matrix.os}} - if: github.event.action != 'labeled' || github.event.label.name == 'rebuild' steps: - name: Configure Build Environment 1 shell: bash id: buildenv1 run: | + + echo "UPLOAD_PREFIX=build/overte/PR" >> $GITHUB_ENV echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7` - echo "JOB_NAME=build (${{matrix.os}}, ${{matrix.build_type}})" >> $GITHUB_ENV + echo "JOB_NAME=${{matrix.os}}, ${{matrix.build_type}}" >> $GITHUB_ENV echo "APP_TARGET_NAME=$APP_NAME" >> $GITHUB_ENV # Linux build variables if [[ "${{ matrix.os }}" = "ubuntu-"* || "${{ matrix.os }}" = *"debian"* ]]; then echo "PYTHON_EXEC=python3" >> $GITHUB_ENV - echo "INSTALLER_EXT=*" >> $GITHUB_ENV + echo "INSTALLER_EXT=test" >> $GITHUB_ENV echo "CMAKE_BUILD_EXTRA=-- -j$(nproc)" >> $GITHUB_ENV if [ "${{ matrix.os }}" = "self-hosted_debian-11_aarch64" ]; then echo "VIRCADIA_USE_SYSTEM_QT=true" >> $GITHUB_ENV @@ -96,7 +100,7 @@ jobs: echo "APP_TARGET_NAME=Vircadia" >> $GITHUB_ENV fi # Windows build variables - if [ "${{ matrix.os }}" = "windows-latest" ]; then + if [ "${{ matrix.os }}" = "windows-2019" ]; then echo "PYTHON_EXEC=python" >> $GITHUB_ENV echo "INSTALLER_EXT=exe" >> $GITHUB_ENV if [ "${{ matrix.build_type }}" = "full" ]; then @@ -117,15 +121,17 @@ jobs: - name: Configure Build Environment 2 shell: bash run: | + echo "${{ steps.buildenv1.outputs.symbols_archive }}" + # Setting short commit SHA for use as RELEASE_NAME in generated installers. echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV if [[ "${{ matrix.build_type }}" != "android" ]]; then if [ "${{ matrix.build_type }}" = "full" ]; then - echo "ARTIFACT_PATTERN=Vircadia-PR${{ github.event.number }}-*.$INSTALLER_EXT" >> $GITHUB_ENV - echo "INSTALLER=Vircadia-$RELEASE_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV + echo "ARTIFACT_PATTERN=Overte-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + echo "INSTALLER=Overte-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV else - echo "ARTIFACT_PATTERN=Vircadia-Interface-PR${{ github.event.number }}-*.$INSTALLER_EXT" >> $GITHUB_ENV - echo "INSTALLER=Vircadia-Interface-$RELEASE_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV + echo "ARTIFACT_PATTERN=Overte-Interface-PR${{ github.event.number }}-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV + echo "INSTALLER=Overte-Interface-$RELEASE_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV fi else echo "ARTIFACT_PATTERN=*.$INSTALLER_EXT" >> $GITHUB_ENV @@ -136,6 +142,7 @@ jobs: shell: bash working-directory: ${{runner.workspace}} run: rm -rf ./* + - uses: actions/checkout@v1 with: submodules: false @@ -146,17 +153,16 @@ jobs: if: startsWith(matrix.os, 'ubuntu') || contains(matrix.os, 'debian') || startsWith(matrix.os, 'macOS') run: | if [[ "${{ matrix.os }}" =~ "ubuntu" || "${{ matrix.os }}" =~ "debian" ]]; then - echo "Installing Python Modules:" - pip3 install distro || exit 1 echo "Updating apt repository index" sudo apt update || exit 1 echo "Installing apt packages" sudo apt install -y ${{ matrix.apt-dependencies }} || exit 1 + else # macOS echo "Downloading MacOSX10.12 SDK.." - curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://cloudflare-ipfs.com/ipfs/QmTDLEYdo616YgZKSYC2mySSw8EKQWTz7CH8UyMupMBpzK?filename=macOS_SDK10.12.4.tar.xz" || exit 1 + curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://data.moto9000.moe/overte_packages/macOS_SDK10.12.4.tar.xz" || exit 1 echo " done\n" echo "Checking MacOSX10.12 SDK.." echo "c00e03cb64bed8d0d75d7a01c93de094b6ffaa3bc3318e7b29a9bdc354f0e29e363c454b7db89db8a3b78e9702668d1c661f1af279807de8db11accecb0ca498 macOS_SDK10.12.4.tar.xz" >> macOS_SDK10.12.4.tar.xz.checksum @@ -175,6 +181,11 @@ jobs: if: startsWith(matrix.os, 'windows') run: choco install nsis --version=3.06.1 + - name: Install Python modules + if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') + shell: bash + run: $PYTHON_EXEC -m pip install boto3 PyGithub + - name: Create Build Environment shell: bash run: cmake -E make_directory "${{runner.workspace}}/build" @@ -227,7 +238,7 @@ jobs: run: cmake --build . --config $BUILD_TYPE --target assignment-client $CMAKE_BUILD_EXTRA - name: Build Console - if: matrix.build_type == 'full' || matrix.os == 'windows-latest' + if: matrix.build_type == 'full' || startsWith(matrix.os, 'windows') working-directory: ${{runner.workspace}}/build shell: bash run: cmake --build . --config $BUILD_TYPE --target packaged-server-console $CMAKE_BUILD_EXTRA @@ -267,14 +278,13 @@ jobs: - name: Nuke CMake 3.18.1-g262b901 if: matrix.build_type == 'android' shell: bash - working-directory: ${{runner.workspace}}/vircadia run: | /usr/local/lib/android/sdk/tools/bin/sdkmanager --uninstall 'cmake;3.18.1' - name: Build for Android + Quest if: matrix.build_type == 'android' shell: bash - working-directory: ${{runner.workspace}}/vircadia + working-directory: ${{runner.workspace}}/overte run: | echo "Pre-cache the vcpkg managed dependencies" $PYTHON_EXEC prebuild.py --build-root ${{runner.workspace}}/build --android interface @@ -293,7 +303,7 @@ jobs: df -h - name: Output Installer Logs - if: failure() && matrix.os == 'windows-latest' + if: failure() && startsWith(matrix.os, 'windows') shell: bash working-directory: ${{runner.workspace}}/build run: cat ./_CPack_Packages/win64/NSIS/NSISOutput.log @@ -304,6 +314,7 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: | + tree if [[ "${{ matrix.build_type }}" == "android" ]]; then cd $GITHUB_WORKSPACE/android fi diff --git a/tools/ci-scripts/upload_to_publish_server.py b/tools/ci-scripts/upload_to_publish_server.py index 5e1a9b24f2..8d5886a9bc 100644 --- a/tools/ci-scripts/upload_to_publish_server.py +++ b/tools/ci-scripts/upload_to_publish_server.py @@ -41,7 +41,8 @@ for archiveFile in files: file_contents.append(file_data) file.close() -conn = http.client.HTTPSConnection("build-uploader.vircadia.com") +# Connect to an instance of https://github.com/JulianGro/gha-artifact-uploader +conn = http.client.HTTPConnection("artifact-uploader.overte.org:3000") context = json.loads(os.environ['GITHUB_CONTEXT'])