Merge pull request #48 from JulianGro/s3-master-builds

GitHub Actions master builds

.github/workflows/master_build.yml

@@ -16,7 +16,9 @@ env:
   RELEASE_TYPE: PRODUCTION
   RELEASE_NUMBER: ${{ github.run_number }}
   STABLE_BUILD: 0
-  UPLOAD_BUCKET: athena-public
+  UPLOAD_BUCKET: overte-public
+  UPLOAD_REGION: fra1
+  UPLOAD_ENDPOINT: "https://fra1.digitaloceanspaces.com"
 
   # OSX-specific variables
   DEVELOPER_DIR: /Applications/Xcode_11.2.app/Contents/Developer
@@ -31,25 +33,29 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [windows-latest, macOS-10.15, ubuntu-18.04]
-        # build_type: [full, client]
-        build_type: [full]
         include:
+          - os: windows-2019
+            build_type: full
+          - os: macOS-10.15
+            build_type: full
           - os: ubuntu-18.04
             build_type: full
-            apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3
+            apt-dependencies: mesa-common-dev libegl1 libglvnd-dev libdouble-conversion1 libpulse0 libsnappy1v5 libwebpdemux2 libwebpmux3 python3-github python3-distro
       fail-fast: false
     runs-on: ${{matrix.os}}
     steps:
     - name: Report Build Number
       shell: bash
       run: |
+
         echo "Build number: $BUILD_NUMBER"
+
     - name: Configure build environment 1
       shell: bash
       id: buildenv1
       run: |
-        echo "UPLOAD_PREFIX=master" >> $GITHUB_ENV
+
+        echo "UPLOAD_PREFIX=build/overte/master" >> $GITHUB_ENV
         echo ::set-output name=github_sha_short::`echo $GIT_COMMIT | cut -c1-7`
         echo "JOB_NAME=build (${{matrix.os}}, ${{matrix.build_type}})" >> $GITHUB_ENV
         echo "APP_TARGET_NAME=$APP_NAME" >> $GITHUB_ENV
@@ -57,7 +63,7 @@ jobs:
         if [[ "${{ matrix.os }}" = "ubuntu-"* ]]; then
           echo "PYTHON_EXEC=python3" >> $GITHUB_ENV
           echo "INSTALLER_EXT=tgz" >> $GITHUB_ENV
-          echo "CMAKE_BUILD_EXTRA=-- -j3" >> $GITHUB_ENV
+          echo "CMAKE_BUILD_EXTRA=-- -j$(nproc)" >> $GITHUB_ENV
           echo "CMAKE_EXTRA=-DVIRCADIA_CPU_ARCHITECTURE= -DBUILD_TOOLS:BOOLEAN=FALSE -DHIFI_PYTHON_EXEC:FILEPATH=$(which python3)" >> $GITHUB_ENV
         fi
         # Mac build variables
@@ -71,7 +77,7 @@ jobs:
           echo "APP_TARGET_NAME=Vircadia" >> $GITHUB_ENV
         fi
         # Windows build variables
-        if [ "${{ matrix.os }}" = "windows-latest" ]; then
+        if [ "${{ matrix.os }}" = "windows-2019" ]; then
           echo "PYTHON_EXEC=python" >> $GITHUB_ENV
           echo "ZIP_COMMAND=7z" >> $GITHUB_ENV
           echo "ZIP_ARGS=a" >> $GITHUB_ENV
@@ -86,16 +92,18 @@ jobs:
     - name: Configure build environment 2
       shell: bash
       run: |
+
         echo "${{ steps.buildenv1.outputs.symbols_archive }}"
-        echo "ARTIFACT_PATTERN=Vircadia-*.$INSTALLER_EXT" >> $GITHUB_ENV
+        echo "ARTIFACT_PATTERN=Overte-*.$INSTALLER_EXT" >> $GITHUB_ENV
+        # Setting short commit SHA for use as RELEASE_NAME in generated installers.
+        echo "RELEASE_NAME=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV
         # Build type variables
-        echo "GIT_COMMIT_SHORT=${{ steps.buildenv1.outputs.github_sha_short }}" >> $GITHUB_ENV
         if [ "${{ matrix.build_type }}" = "full" ]; then
           echo "CLIENT_ONLY=FALSE" >> $GITHUB_ENV
-          echo "INSTALLER=Vircadia-$BUILD_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV
+          echo "INSTALLER=Overte-master$BUILD_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
         else
           echo "CLIENT_ONLY=TRUE" >> $GITHUB_ENV
-          echo "INSTALLER=Vircadia-Interface-$BUILD_NUMBER-$GIT_COMMIT_SHORT.$INSTALLER_EXT" >> $GITHUB_ENV
+          echo "INSTALLER=Overte-Interface-master$BUILD_NUMBER-${{ steps.buildenv1.outputs.github_sha_short }}.$INSTALLER_EXT" >> $GITHUB_ENV
         fi
     - name: Clear working directory
       if: startsWith(matrix.os, 'windows')
@@ -112,17 +120,19 @@ jobs:
       if: startsWith(matrix.os, 'ubuntu') || contains(matrix.os, 'debian') || startsWith(matrix.os, 'macOS')
       run: |
         if [[ "${{ matrix.os }}" =~ "ubuntu" || "${{ matrix.os }}" =~ "debian" ]]; then
-          echo "Installing Python Modules:"
-          pip3 install distro || exit 1
 
           echo "Updating apt repository index"
           sudo apt update || exit 1
 
           echo "Installing apt packages"
           sudo apt install -y ${{ matrix.apt-dependencies }} || exit 1
+
+          echo "Installing Python Modules"
+          pip3 install boto3 || exit 1
+
         else # macOS
           echo "Downloading MacOSX10.12 SDK.."
-          curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://cloudflare-ipfs.com/ipfs/QmTDLEYdo616YgZKSYC2mySSw8EKQWTz7CH8UyMupMBpzK?filename=macOS_SDK10.12.4.tar.xz" || exit 1
+          curl --progress-bar -L -o macOS_SDK10.12.4.tar.xz "https://data.moto9000.moe/overte_packages/macOS_SDK10.12.4.tar.xz" || exit 1
           echo "  done\n"
           echo "Checking MacOSX10.12 SDK.."
           echo "c00e03cb64bed8d0d75d7a01c93de094b6ffaa3bc3318e7b29a9bdc354f0e29e363c454b7db89db8a3b78e9702668d1c661f1af279807de8db11accecb0ca498  macOS_SDK10.12.4.tar.xz" >> macOS_SDK10.12.4.tar.xz.checksum
@@ -223,12 +233,11 @@ jobs:
       run: cat ./_CPack_Packages/win64/NSIS/NSISOutput.log
 
     - name: Upload artifact
-      if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS')
       shell: bash
       working-directory: ${{runner.workspace}}/build
       env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.s3_access_key_id }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.s3_secret_access_key }}
       run: $PYTHON_EXEC $GITHUB_WORKSPACE/tools/ci-scripts/upload.py
 
     #- name: Archive symbols

.gitignore

@@ -111,4 +111,7 @@ tools/unity-avatar-exporter
 server-console/package-lock.json
 vcpkg/
 /tools/nitpick/compiledResources
-qt/
+qt/
+
+# Act local GitHub Actions
+.secret

README.md

@@ -29,6 +29,7 @@ Overte is a 3D social software project seeking to incrementally bring about a tr
 ### Releases
 
 [View Releases here](https://github.com/overte-org/overte/releases/)
+[Pre-releases for testing are available here](https://public.overte.org/index.html#build/overte/)
 
 ### How to deploy a Server
 

cmake/macros/GenerateInstallers.cmake

@@ -5,6 +5,7 @@
 #  Created by Leonardo Murillo on 12/16/2015.
 #  Copyright 2015 High Fidelity, Inc.
 #  Copyright 2021 Vircadia contributors.
+#  Copyright 2022 Overte e.V.
 #
 #  Distributed under the Apache License, Version 2.0.
 #  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
@@ -19,25 +20,24 @@ macro(GENERATE_INSTALLERS)
   if (CLIENT_ONLY)
     set(_PACKAGE_NAME_EXTRA "-Interface")
     set(INSTALLER_TYPE "client_only")
-    string(REGEX REPLACE "Vircadia" "Vircadia Interface" _DISPLAY_NAME ${BUILD_ORGANIZATION})
+    string(REGEX REPLACE "Overte" "Overte Interface" _DISPLAY_NAME ${BUILD_ORGANIZATION})
   elseif (SERVER_ONLY)
     set(_PACKAGE_NAME_EXTRA "-Server")
     set(INSTALLER_TYPE "server_only")
-    string(REGEX REPLACE "Vircadia" "Vircadia Server" _DISPLAY_NAME ${BUILD_ORGANIZATION})
+    string(REGEX REPLACE "Overte" "Overte Server" _DISPLAY_NAME ${BUILD_ORGANIZATION})
   else ()
     set(_DISPLAY_NAME ${BUILD_ORGANIZATION})
     set(INSTALLER_TYPE "full")
   endif ()
 
   set(CPACK_PACKAGE_NAME ${_DISPLAY_NAME})
-  set(CPACK_PACKAGE_VENDOR "Vircadia")
+  set(CPACK_PACKAGE_VENDOR "Overte")
   set(CPACK_PACKAGE_VERSION ${BUILD_VERSION})
-  set(CPACK_PACKAGE_FILE_NAME "Vircadia${_PACKAGE_NAME_EXTRA}-${BUILD_VERSION}-${RELEASE_NAME}")
+  # There is some sort of bug which adds a "-" between the BUILD_VERSION and the RELEASE_NAME.
+  set(CPACK_PACKAGE_FILE_NAME "Overte${_PACKAGE_NAME_EXTRA}-${BUILD_VERSION}${RELEASE_NAME}")
   set(CPACK_NSIS_DISPLAY_NAME ${_DISPLAY_NAME})
   set(CPACK_NSIS_PACKAGE_NAME ${_DISPLAY_NAME})
-  if (PR_BUILD)
+  set(CPACK_NSIS_COMPRESSOR "LZMA")
-    set(CPACK_NSIS_COMPRESSOR "bzip2")
-  endif ()
   set(CPACK_PACKAGE_INSTALL_DIRECTORY ${_DISPLAY_NAME})
 
 
@@ -123,11 +123,11 @@ macro(GENERATE_INSTALLERS)
   set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE")
 
   if (BUILD_CLIENT)
-    cpack_add_component(${CLIENT_COMPONENT} DISPLAY_NAME "Vircadia Interface")
+    cpack_add_component(${CLIENT_COMPONENT} DISPLAY_NAME "Overte Interface")
   endif ()
 
   if (BUILD_SERVER)
-    cpack_add_component(${SERVER_COMPONENT} DISPLAY_NAME "Vircadia Server")
+    cpack_add_component(${SERVER_COMPONENT} DISPLAY_NAME "Overte Server")
   endif ()
 
   include(CPack)

tools/ci-scripts/upload.py

@@ -7,17 +7,21 @@ import glob
 from github import Github
 
 def main():
+    print("Starting upload..")
     bucket_name = os.environ['UPLOAD_BUCKET']
     upload_prefix = os.environ['UPLOAD_PREFIX']
     release_number = os.environ['RELEASE_NUMBER']
-    full_prefix = upload_prefix + '/' + release_number
+    full_prefix = upload_prefix + release_number
-    S3 = boto3.client('s3')
+    S3 = boto3.client('s3', region_name=os.environ['UPLOAD_REGION'], endpoint_url=os.environ['UPLOAD_ENDPOINT'])
     path = os.path.join(os.getcwd(), os.environ['ARTIFACT_PATTERN'])
+    print("Checking for files to upload in {}..".format(path))
     files = glob.glob(path, recursive=False)
     for archiveFile in files:
         filePath, fileName = os.path.split(archiveFile)
+        print("Uploading {}/{}..".format(full_prefix, fileName))
         S3.upload_file(os.path.join(filePath, fileName), bucket_name, full_prefix + '/' + fileName)
-        print("Uploaded Artifact to S3: https://{}.s3-eu-west-3.amazonaws.com/{}/{}".format(bucket_name, full_prefix, fileName))
+        S3.put_object_acl(ACL='public-read', Bucket=bucket_name, Key=full_prefix + '/' + fileName)
+        print("Uploaded Artifact to S3: https://public.overte.org/{}/{}".format(full_prefix, fileName))
     print("Finished")
 
 main()