diff --git a/.gitignore b/.gitignore index ef1a7b215e..09b58d71ef 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,11 @@ Makefile local.properties android/gradle* android/.gradle -android/app/src/main/jniLibs -android/app/libs -android/app/src/main/res/values/libs.xml -android/app/src/main/assets/bundled +android/**/src/main/jniLibs +android/**/libs +android/**/src/main/res/values/libs.xml +android/**/src/main/assets +android/**/gradle* # VSCode # List taken from Github Global Ignores master@435c4d92 @@ -83,9 +84,6 @@ npm-debug.log # Android studio files *___jb_old___ -# Generated assets for Android -android/app/src/main/assets - # Resource binary file interface/compiledResources @@ -95,6 +93,9 @@ interface/resources/GPUCache/* # package lock file for JSDoc tool tools/jsdoc/package-lock.json +# Python compile artifacts +**/__pycache__ + # ignore unneeded unity project files for avatar exporter tools/unity-avatar-exporter/Library tools/unity-avatar-exporter/Packages diff --git a/BUILD_ANDROID.md b/BUILD_ANDROID.md index 8c9263b6e7..c111e758b5 100644 --- a/BUILD_ANDROID.md +++ b/BUILD_ANDROID.md @@ -6,13 +6,8 @@ Building is currently supported on OSX, Windows and Linux platforms, but develop You will need the following tools to build Android targets. -* [Gradle](https://gradle.org/install/) * [Android Studio](https://developer.android.com/studio/index.html) -### Gradle - -Install gradle version 4.1 or higher. Following the instructions to install via [SDKMAN!](http://sdkman.io/install.html) is recommended. - ### Android Studio Download the Android Studio installer and run it. Once installed, at the welcome screen, click configure in the lower right corner and select SDK manager @@ -29,6 +24,8 @@ From the SDK Tools tab select the following * Android SDK Tools * NDK (even if you have the NDK installed separately) +Make sure the NDK installed version is 18 (or higher) + # Environment Setting up the environment for android builds requires some additional steps @@ -51,17 +48,17 @@ Enter the repository `android` directory `cd hifi/android` -Execute a gradle pre-build setup. This step should only need to be done once +Execute two gradle pre-build steps. This steps should only need to be done once, unless you're working on the Android dependencies -`gradle setupDependencies` +`./gradlew extractDependencies` +`./gradlew setupDependencies` # Building & Running * Open Android Studio * Choose _Open Existing Android Studio Project_ * Navigate to the `hifi` repository and choose the `android` folder and select _OK_ -* If Android Studio asks you if you want to use the Gradle wrapper, select cancel and tell it where your local gradle installation is. If you used SDKMAN to install gradle it will be located in `$HOME/.sdkman/candidates/gradle/current/` * From the _Build_ menu select _Make Project_ * Once the build completes, from the _Run_ menu select _Run App_ diff --git a/android/app/build.gradle b/android/app/build.gradle index cbc31479ec..e3c6989baf 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -139,23 +139,23 @@ dependencies { implementation 'com.android.support.constraint:constraint-layout:1.0.2' implementation 'com.android.support:design:26.1.0' - compile 'com.android.support:support-v4:26.1.0' - compile 'com.android.support:appcompat-v7:26.1.0' - compile 'com.android.support:support-vector-drawable:26.1.0' + api 'com.android.support:support-v4:26.1.0' + api 'com.android.support:appcompat-v7:26.1.0' + api 'com.android.support:support-vector-drawable:26.1.0' implementation 'com.android.support:appcompat-v7:26.1.0' - compile 'com.android.support:recyclerview-v7:26.1.0' - compile 'com.android.support:cardview-v7:26.1.0' + api 'com.android.support:recyclerview-v7:26.1.0' + api 'com.android.support:cardview-v7:26.1.0' - compile 'com.squareup.retrofit2:retrofit:2.4.0' - compile 'com.squareup.retrofit2:converter-gson:2.4.0' + api 'com.squareup.retrofit2:retrofit:2.4.0' + api 'com.squareup.retrofit2:converter-gson:2.4.0' implementation 'com.squareup.picasso:picasso:2.71828' - compile 'com.squareup.retrofit2:retrofit:2.4.0' - compile 'com.squareup.retrofit2:converter-gson:2.4.0' + api 'com.squareup.retrofit2:retrofit:2.4.0' + api 'com.squareup.retrofit2:converter-gson:2.4.0' implementation 'com.squareup.picasso:picasso:2.71828' - compile 'com.sothree.slidinguppanel:library:3.4.0' + api 'com.sothree.slidinguppanel:library:3.4.0' implementation fileTree(include: ['*.jar'], dir: 'libs') } diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 57e708068f..2ff35b6c3e 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -1,7 +1,6 @@ - diff --git a/android/build.gradle b/android/build.gradle index e22c2d877f..8d03b9f6b3 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -14,7 +14,7 @@ buildscript { google() } dependencies { - classpath 'com.android.tools.build:gradle:3.0.1' + classpath 'com.android.tools.build:gradle:3.2.1' } } @@ -392,7 +392,7 @@ task extractDependencies(dependsOn: verifyDependencyDownloads) { } // Copies the non Qt dependencies. Qt dependencies (primary libraries and plugins) are handled by the qtBundle task -task copyDependencies(dependsOn: [ extractDependencies ]) { +task copyDependencies() { doLast { packages.each { entry -> def packageName = entry.key @@ -414,7 +414,7 @@ task copyDependencies(dependsOn: [ extractDependencies ]) { } } -task extractGvrBinaries(dependsOn: extractDependencies) { +task extractGvrBinaries() { doLast { def gvrLibFolder = new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries'); zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-audio-1.101.0.aar')).visit { element -> diff --git a/android/build_android.sh b/android/build_android.sh new file mode 100755 index 0000000000..f98bd1a4b2 --- /dev/null +++ b/android/build_android.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -xeuo pipefail +./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} setupDependencies +./gradlew -PHIFI_ANDROID_PRECOMPILED=${HIFI_ANDROID_PRECOMPILED} -PRELEASE_NUMBER=${RELEASE_NUMBER} -PRELEASE_TYPE=${RELEASE_TYPE} app:${ANDROID_BUILD_TARGET} \ No newline at end of file diff --git a/android/containerized_build.sh b/android/containerized_build.sh new file mode 100755 index 0000000000..cd6f15a92e --- /dev/null +++ b/android/containerized_build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +DOCKER_IMAGE_NAME="hifi_androidbuild" + +docker build --build-arg BUILD_UID=`id -u` -t "${DOCKER_IMAGE_NAME}" -f docker/Dockerfile docker + +docker run \ + --rm \ + --security-opt seccomp:unconfined \ + -v "${WORKSPACE}":/home/jenkins/hifi \ + -e "RELEASE_NUMBER=${RELEASE_NUMBER}" \ + -e "RELEASE_TYPE=${RELEASE_TYPE}" \ + -e "ANDROID_BUILD_TARGET=assembleDebug" \ + -e "CMAKE_BACKTRACE_URL=${CMAKE_BACKTRACE_URL}" \ + -e "CMAKE_BACKTRACE_TOKEN=${CMAKE_BACKTRACE_TOKEN}" \ + -e "CMAKE_BACKTRACE_SYMBOLS_TOKEN=${CMAKE_BACKTRACE_SYMBOLS_TOKEN}" \ + -e "GA_TRACKING_ID=${GA_TRACKING_ID}" \ + -e "GIT_PR_COMMIT=${GIT_PR_COMMIT}" \ + -e "VERSION_CODE=${VERSION_CODE}" \ + "${DOCKER_IMAGE_NAME}" \ + sh -c "./build_android.sh" diff --git a/android/docker/Dockerfile b/android/docker/Dockerfile new file mode 100644 index 0000000000..2a6943cbc2 --- /dev/null +++ b/android/docker/Dockerfile @@ -0,0 +1,92 @@ +FROM openjdk:8 + +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +RUN apt-get update && apt-get -y install \ + curl \ + gnupg \ + software-properties-common \ + unzip \ + - + +# --- Versions and Download paths +ENV ANDROID_HOME="/usr/local/android-sdk" \ + ANDROID_NDK_HOME="/usr/local/android-ndk" \ + ANDROID_SDK_HOME="/usr/local/android-sdk-home" \ + ANDROID_VERSION=26 \ + ANDROID_BUILD_TOOLS_VERSION=28.0.3 \ + ANDROID_NDK_VERSION=r18 + +ENV SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip" \ + NDK_URL="https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip" + +# --- Android SDK +RUN mkdir -p "$ANDROID_HOME" "$ANDROID_SDK_HOME" && \ + cd "$ANDROID_HOME" && \ + curl -s -S -o sdk.zip -L "${SDK_URL}" && \ + unzip sdk.zip && \ + rm sdk.zip && \ + yes | $ANDROID_HOME/tools/bin/sdkmanager --licenses + +# Install Android Build Tool and Libraries +RUN $ANDROID_HOME/tools/bin/sdkmanager --update +RUN $ANDROID_HOME/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" \ + "platforms;android-${ANDROID_VERSION}" \ + "platform-tools" + +RUN chmod -R a+w "${ANDROID_HOME}" +RUN chmod -R a+w "${ANDROID_SDK_HOME}" + +# --- Android NDK +# download +RUN mkdir /usr/local/android-ndk-tmp && \ + cd /usr/local/android-ndk-tmp && \ + curl -s -S -o ndk.zip -L "${NDK_URL}" && \ + unzip -q ndk.zip && \ + mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \ + cd ${ANDROID_NDK_HOME} && \ + rm -rf /usr/local/android-ndk-tmp + +ENV PATH ${PATH}:${ANDROID_NDK_HOME} + +RUN apt-get -y install \ + g++ \ + gcc \ + - + +# --- Gradle +ARG BUILD_UID=1001 +RUN useradd -ms /bin/bash -u $BUILD_UID jenkins +USER jenkins +WORKDIR /home/jenkins + +# Hifi dependencies +ENV HIFI_BASE="/home/jenkins/hifi_android" +ENV HIFI_ANDROID_PRECOMPILED="$HIFI_BASE/dependencies" +ENV HIFI_VCPKG_BASE="$HIFI_BASE/vcpkg" + +RUN mkdir "$HIFI_BASE" && \ + mkdir "$HIFI_VCPKG_BASE" && \ + mkdir "$HIFI_ANDROID_PRECOMPILED" + +RUN git clone https://github.com/jherico/hifi.git && \ + cd ~/hifi && \ + git checkout feature/build/gradle-wrapper + + +WORKDIR /home/jenkins/hifi + +RUN touch .test4 && \ + git fetch && git reset origin/feature/build/gradle-wrapper --hard + +RUN mkdir build + +# Pre-cache the vcpkg managed dependencies +WORKDIR /home/jenkins/hifi/build +RUN python3 ../prebuild.py --build-root `pwd` --android + +# Pre-cache the gradle dependencies +WORKDIR /home/jenkins/hifi/android +RUN ./gradlew -m tasks -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED +RUN ./gradlew extractDependencies -PHIFI_ANDROID_PRECOMPILED=$HIFI_ANDROID_PRECOMPILED + diff --git a/android/gradle/wrapper/gradle-wrapper.jar b/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..f6b961fd5a Binary files /dev/null and b/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/android/gradle/wrapper/gradle-wrapper.properties b/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..36dba9b2f5 --- /dev/null +++ b/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Sat Dec 01 08:32:47 PST 2018 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-all.zip diff --git a/android/gradlew b/android/gradlew new file mode 100755 index 0000000000..cccdd3d517 --- /dev/null +++ b/android/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/android/gradlew.bat b/android/gradlew.bat new file mode 100755 index 0000000000..f9553162f1 --- /dev/null +++ b/android/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/hifi_android.py b/hifi_android.py new file mode 100644 index 0000000000..e3944cda9a --- /dev/null +++ b/hifi_android.py @@ -0,0 +1,286 @@ +import hifi_utils +import json +import os +import platform +import re +import shutil +import xml.etree.ElementTree as ET +import functools + +print = functools.partial(print, flush=True) + +ANDROID_PACKAGE_URL = 'https://hifi-public.s3.amazonaws.com/dependencies/android/' + +ANDROID_PACKAGES = { + 'qt' : { + 'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz', + 'versionId': '3S97HBM5G5Xw9EfE52sikmgdN3t6C2MN', + 'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df', + }, + 'bullet': { + 'file': 'bullet-2.88_armv8-libcpp.tgz', + 'versionId': 'S8YaoED0Cl8sSb8fSV7Q2G1lQJSNDxqg', + 'checksum': '81642779ccb110f8c7338e8739ac38a0', + }, + 'draco': { + 'file': 'draco_armv8-libcpp.tgz', + 'versionId': '3.B.uBj31kWlgND3_R2xwQzT_TP6Dz_8', + 'checksum': '617a80d213a5ec69fbfa21a1f2f738cd', + }, + 'glad': { + 'file': 'glad_armv8-libcpp.zip', + 'versionId': 'r5Zran.JSCtvrrB6Q4KaqfIoALPw3lYY', + 'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449', + }, + 'gvr': { + 'file': 'gvrsdk_v1.101.0.tgz', + 'versionId': 'nqBV_j81Uc31rC7bKIrlya_Hah4v3y5r', + 'checksum': '57fd02baa069176ba18597a29b6b4fc7', + }, + 'nvtt': { + 'file': 'nvtt_armv8-libcpp.zip', + 'versionId': 'lmkBVR5t4UF1UUwMwEirnk9H_8Nt90IO', + 'checksum': 'eb46d0b683e66987190ed124aabf8910', + 'sharedLibFolder': 'lib', + 'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so'] + }, + 'oculus': { + 'file': 'ovr_sdk_mobile_1.19.0.zip', + 'versionId': 's_RN1vlEvUi3pnT7WPxUC4pQ0RJBs27y', + 'checksum': '98f0afb62861f1f02dd8110b31ed30eb', + 'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release', + 'includeLibs': ['libvrapi.so'] + }, + 'openssl': { + 'file': 'openssl-1.1.0g_armv8.tgz', + 'versionId': 'AiiPjmgUZTgNj7YV1EEx2lL47aDvvvAW', + 'checksum': 'cabb681fbccd79594f65fcc266e02f32' + }, + 'polyvox': { + 'file': 'polyvox_armv8-libcpp.tgz', + 'versionId': 'A2kbKiNhpIenGq23bKRRzg7IMAI5BI92', + 'checksum': 'dba88b3a098747af4bb169e9eb9af57e', + 'sharedLibFolder': 'lib', + 'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'], + }, + 'tbb': { + 'file': 'tbb-2018_U1_armv8_libcpp.tgz', + 'versionId': 'mrRbWnv4O4evcM1quRH43RJqimlRtaKB', + 'checksum': '20768f298f53b195e71b414b0ae240c4', + 'sharedLibFolder': 'lib/release', + 'includeLibs': ['libtbb.so', 'libtbbmalloc.so'], + }, + 'hifiAC': { + 'baseUrl': 'http://s3.amazonaws.com/hifi-public/dependencies/', + 'file': 'codecSDK-android_armv8-2.0.zip', + 'checksum': '1cbef929675818fc64c4101b72f84a6a' + }, + 'etc2comp': { + 'file': 'etc2comp-patched-armv8-libcpp.tgz', + 'versionId': 'bHhGECRAQR1vkpshBcK6ByNc1BQIM8gU', + 'checksum': '14b02795d774457a33bbc60e00a786bc' + }, + 'breakpad': { + 'file': 'breakpad.tgz', + 'versionId': '8VrYXz7oyc.QBxNia0BVJOUBvrFO61jI', + 'checksum': 'ddcb23df336b08017042ba4786db1d9e', + 'sharedLibFolder': 'lib', + 'includeLibs': {'libbreakpad_client.a'} + } +} + +ANDROID_PLATFORM_PACKAGES = { + 'Darwin' : { + 'qt': { + 'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz', + 'versionId': 'OxBD7iKINv1HbyOXmAmDrBb8AF3N.Kup', + 'checksum': 'c83cc477c08a892e00c71764dca051a0' + }, + }, + 'Windows' : { + 'qt': { + 'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz', + 'versionId': 'JfWM0P_Mz5Qp0LwpzhrsRwN3fqlLSFeT', + 'checksum': '0582191cc55431aa4f660848a542883e' + }, + } +} + +QT5_DEPS = [ + 'Qt5Concurrent', + 'Qt5Core', + 'Qt5Gui', + 'Qt5Multimedia', + 'Qt5Network', + 'Qt5OpenGL', + 'Qt5Qml', + 'Qt5Quick', + 'Qt5QuickControls2', + 'Qt5QuickTemplates2', + 'Qt5Script', + 'Qt5ScriptTools', + 'Qt5Svg', + 'Qt5WebChannel', + 'Qt5WebSockets', + 'Qt5Widgets', + 'Qt5XmlPatterns', + # Android specific + 'Qt5AndroidExtras', + 'Qt5WebView', +] + +def getPlatformPackages(): + result = ANDROID_PACKAGES.copy() + system = platform.system() + if system in ANDROID_PLATFORM_PACKAGES: + platformPackages = ANDROID_PLATFORM_PACKAGES[system] + result = { **result, **platformPackages } + return result + +def getPackageUrl(package): + url = ANDROID_PACKAGE_URL + if 'baseUrl' in package: + url = package['baseUrl'] + url += package['file'] + if 'versionId' in package: + url += '?versionId=' + package['versionId'] + return url + +def copyAndroidLibs(packagePath, appPath): + androidPackages = getPlatformPackages() + jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a') + if not os.path.isdir(jniPath): + os.makedirs(jniPath) + for packageName in androidPackages: + package = androidPackages[packageName] + if 'sharedLibFolder' in package: + sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder']) + if 'includeLibs' in package: + for lib in package['includeLibs']: + sourceFile = os.path.join(sharedLibFolder, lib) + destFile = os.path.join(jniPath, os.path.split(lib)[1]) + if not os.path.isfile(destFile): + print("Copying {}".format(lib)) + shutil.copy(sourceFile, destFile) + +class QtPackager: + def __init__(self, appPath, qtRootPath): + self.appPath = appPath + self.qtRootPath = qtRootPath + self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a') + self.assetPath = os.path.join(self.appPath, 'src/main/assets') + self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--') + # Jars go into the qt library + self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs')) + self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml') + self.files = [] + self.features = [] + self.permissions = [] + + def copyQtDeps(self): + for lib in QT5_DEPS: + libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib)) + if not os.path.exists(libfile): + continue + self.files.append(libfile) + androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib)) + if not os.path.exists(androidDeps): + continue + + tree = ET.parse(androidDeps) + root = tree.getroot() + for item in root.findall('./dependencies/lib/depends/*'): + if (item.tag == 'lib') or (item.tag == 'bundled'): + relativeFilename = item.attrib['file'] + if (relativeFilename.startswith('qml')): + continue + filename = os.path.join(self.qtRootPath, relativeFilename) + self.files.extend(hifi_utils.recursiveFileList(filename)) + elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1": + self.files.append(os.path.join(self.qtRootPath, item.attrib['file'])) + elif item.tag == 'permission': + self.permissions.append(item.attrib['name']) + elif item.tag == 'feature': + self.features.append(item.attrib['name']) + + def scanQmlImports(self): + qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner') + system = platform.system() + if 'Windows' == system: + qmlImportCommandFile += ".exe" + if not os.path.isfile(qmlImportCommandFile): + raise RuntimeError("Couldn't find qml import scanner") + qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml') + qmlImportPath = os.path.join(self.qtRootPath, 'qml') + commandResult = hifi_utils.executeSubprocessCapture([ + qmlImportCommandFile, + '-rootPath', qmlRootPath, + '-importPath', qmlImportPath + ]) + qmlImportResults = json.loads(commandResult) + for item in qmlImportResults: + if 'path' not in item: + print("Warning: QML import could not be resolved in any of the import paths: {}".format(item['name'])) + continue + path = os.path.realpath(item['path']) + if not os.path.exists(path): + continue + basePath = path + if os.path.isfile(basePath): + basePath = os.path.dirname(basePath) + basePath = os.path.normcase(basePath) + if basePath.startswith(qmlRootPath): + continue + self.files.extend(hifi_utils.recursiveFileList(path)) + + def processFiles(self): + self.files = list(set(self.files)) + self.files.sort() + libsXmlRoot = ET.Element('resources') + qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'}) + bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'}) + bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'}) + libPrefix = 'lib' + for sourceFile in self.files: + if not os.path.isfile(sourceFile): + raise RuntimeError("Unable to find dependency file " + sourceFile) + relativePath = os.path.relpath(sourceFile, self.qtRootPath) + destinationFile = None + if relativePath.endswith('.so'): + garbledFileName = None + if relativePath.startswith(libPrefix): + garbledFileName = relativePath[4:] + p = re.compile(r'lib(Qt5.*).so') + m = p.search(garbledFileName) + if not m: + raise RuntimeError("Huh?") + libName = m.group(1) + ET.SubElement(qtLibsNode, 'item').text = libName + else: + garbledFileName = 'lib' + relativePath.replace('\\', '_'[0]) + value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/') + ET.SubElement(bundledLibsNode, 'item').text = value + destinationFile = os.path.join(self.jniPath, garbledFileName) + elif relativePath.startswith('jar'): + destinationFile = os.path.join(self.jarPath, relativePath[4:]) + else: + value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/') + ET.SubElement(bundledAssetsNode, 'item').text = value + destinationFile = os.path.join(self.qtAssetPath, relativePath) + + destinationParent = os.path.realpath(os.path.dirname(destinationFile)) + if not os.path.isdir(destinationParent): + os.makedirs(destinationParent) + if not os.path.isfile(destinationFile): + shutil.copy(sourceFile, destinationFile) + + tree = ET.ElementTree(libsXmlRoot) + tree.write(self.xmlFile, 'UTF-8', True) + + def bundle(self): + if not os.path.isfile(self.xmlFile) or True: + self.copyQtDeps() + self.scanQmlImports() + self.processFiles() + + diff --git a/hifi_singleton.py b/hifi_singleton.py new file mode 100644 index 0000000000..692948c80b --- /dev/null +++ b/hifi_singleton.py @@ -0,0 +1,46 @@ +import os +import platform +import time + +try: + import fcntl +except ImportError: + fcntl = None + +# Used to ensure only one instance of the script runs at a time +class Singleton: + def __init__(self, path): + self.fh = None + self.windows = 'Windows' == platform.system() + self.path = path + + def __enter__(self): + success = False + while not success: + try: + if self.windows: + if os.path.exists(self.path): + os.unlink(self.path) + self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR) + else: + self.fh = open(self.path, 'x') + fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + success = True + except EnvironmentError as err: + if self.fh is not None: + if self.windows: + os.close(self.fh) + else: + self.fh.close() + self.fh = None + print("Couldn't aquire lock, retrying in 10 seconds") + time.sleep(10) + return self + + def __exit__(self, type, value, traceback): + if self.windows: + os.close(self.fh) + else: + fcntl.lockf(self.fh, fcntl.LOCK_UN) + self.fh.close() + os.unlink(self.path) \ No newline at end of file diff --git a/hifi_utils.py b/hifi_utils.py new file mode 100644 index 0000000000..f53258d4f6 --- /dev/null +++ b/hifi_utils.py @@ -0,0 +1,124 @@ +import os +import hashlib +import platform +import shutil +import ssl +import subprocess +import sys +import tarfile +import urllib +import urllib.request +import zipfile +import tempfile +import time +import functools + +print = functools.partial(print, flush=True) + +def scriptRelative(*paths): + scriptdir = os.path.dirname(os.path.realpath(sys.argv[0])) + result = os.path.join(scriptdir, *paths) + result = os.path.realpath(result) + result = os.path.normcase(result) + return result + + +def recursiveFileList(startPath): + result = [] + if os.path.isfile(startPath): + result.append(startPath) + elif os.path.isdir(startPath): + for dirName, subdirList, fileList in os.walk(startPath): + for fname in fileList: + result.append(os.path.realpath(os.path.join(startPath, dirName, fname))) + result.sort() + return result + + +def executeSubprocessCapture(processArgs): + processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if (0 != processResult.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format( + processArgs[0], + ' '.join(processArgs[1:]), + processResult.stdout.decode('utf-8'), + processResult.stderr.decode('utf-8'))) + return processResult.stdout.decode('utf-8') + +def executeSubprocess(processArgs, folder=None, env=None): + restoreDir = None + if folder != None: + restoreDir = os.getcwd() + os.chdir(folder) + + process = subprocess.Popen( + processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env) + process.wait() + + if (0 != process.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format( + processArgs[0], + ' '.join(processArgs[1:]), + )) + + if restoreDir != None: + os.chdir(restoreDir) + + +def hashFile(file, hasher = hashlib.sha512()): + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +# Assumes input files are in deterministic order +def hashFiles(filenames): + hasher = hashlib.sha256() + for filename in filenames: + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def hashFolder(folder): + filenames = recursiveFileList(folder) + return hashFiles(filenames) + +def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3): + for i in range(retries): + tempFileName = None + # OSX Python doesn't support SSL, so we need to bypass it. + # However, we still validate the downloaded file's sha512 hash + if 'Darwin' == platform.system(): + tempFileDescriptor, tempFileName = tempfile.mkstemp() + context = ssl._create_unverified_context() + with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile: + shutil.copyfileobj(response, tempFile) + else: + tempFileName, headers = urllib.request.urlretrieve(url) + + # for some reason the hash we get back from the downloaded file is sometimes wrong if we check it right away + # but if we examine the file later, it is correct. + time.sleep(3) + downloadHash = hashFile(tempFileName, hasher) + # Verify the hash + if hash is not None and hash != downloadHash: + print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url)) + os.remove(tempFileName) + continue + + return tempFileName + + raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url)) + + +def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False): + tempFileName = downloadFile(url, hash, hasher) + if isZip: + with zipfile.ZipFile(tempFileName) as zip: + zip.extractall(destPath) + else: + # Extract the archive + with tarfile.open(tempFileName, 'r:gz') as tgz: + tgz.extractall(destPath) + os.remove(tempFileName) diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py new file mode 100644 index 0000000000..5492109864 --- /dev/null +++ b/hifi_vcpkg.py @@ -0,0 +1,216 @@ +import hifi_utils +import hifi_android +import hashlib +import os +import platform +import re +import shutil +import tempfile +import json +import xml.etree.ElementTree as ET +import functools + +print = functools.partial(print, flush=True) + +# Encapsulates the vcpkg system +class VcpkgRepo: + CMAKE_TEMPLATE = """ +set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file") +set(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}") +set(VCPKG_INSTALL_ROOT "{}") +set(VCPKG_TOOLS_DIR "{}") +""" + + CMAKE_TEMPLATE_NON_ANDROID = """ +# If the cached cmake toolchain path is different from the computed one, exit +if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE)) + message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake") +endif() +""" + + def __init__(self, args): + self.args = args + # our custom ports, relative to the script location + self.sourcePortsPath = args.ports_path + self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8] + self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake') + + # OS dependent information + system = platform.system() + + if self.args.vcpkg_root is not None: + self.path = args.vcpkg_root + else: + if 'Darwin' == system: + defaultBasePath = os.path.expanduser('~/hifi/vcpkg') + else: + defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg') + self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath) + if self.basePath == defaultBasePath: + print("Warning: Environment variable HIFI_VCPKG_BASE not set, using {}".format(defaultBasePath)) + if self.args.android: + self.basePath = os.path.join(self.basePath, 'android') + if (not os.path.isdir(self.basePath)): + os.makedirs(self.basePath) + self.path = os.path.join(self.basePath, self.id) + + print("Using vcpkg path {}".format(self.path)) + lockDir, lockName = os.path.split(self.path) + lockName += '.lock' + if not os.path.isdir(lockDir): + os.makedirs(lockDir) + + self.lockFile = os.path.join(lockDir, lockName) + self.tagFile = os.path.join(self.path, '.id') + # A format version attached to the tag file... increment when you want to force the build systems to rebuild + # without the contents of the ports changing + self.version = 1 + self.tagContents = "{}_{}".format(self.id, self.version) + + if 'Windows' == system: + self.exe = os.path.join(self.path, 'vcpkg.exe') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ' + self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e' + self.hostTriplet = 'x64-windows' + elif 'Darwin' == system: + self.exe = os.path.join(self.path, 'vcpkg') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc' + self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d' + self.hostTriplet = 'x64-osx' + else: + self.exe = os.path.join(self.path, 'vcpkg') + self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU' + self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d' + self.hostTriplet = 'x64-linux' + + if self.args.android: + self.triplet = 'arm64-android' + self.androidPackagePath = os.path.join(self.path, 'android') + else: + self.triplet = self.hostTriplet + + def upToDate(self): + # Prevent doing a clean if we've explcitly set a directory for vcpkg + if self.args.vcpkg_root is not None: + return True + + if self.args.force_build: + print("Force build, out of date") + return False + if not os.path.isfile(self.exe): + print("Exe file {} not found, out of date".format(self.exe)) + return False + if not os.path.isfile(self.tagFile): + print("Tag file {} not found, out of date".format(self.tagFile)) + return False + with open(self.tagFile, 'r') as f: + storedTag = f.read() + if storedTag != self.tagContents: + print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents)) + return False + return True + + def clean(self): + print("Cleaning vcpkg installation at {}".format(self.path)) + if os.path.isdir(self.path): + print("Removing {}".format(self.path)) + shutil.rmtree(self.path, ignore_errors=True) + + # Make sure the VCPKG prerequisites are all there. + def bootstrap(self): + if self.upToDate(): + return + + self.clean() + + downloadVcpkg = False + if self.args.force_bootstrap: + print("Forcing bootstrap") + downloadVcpkg = True + + if not downloadVcpkg and not os.path.isfile(self.exe): + print("Missing executable, boostrapping") + downloadVcpkg = True + + # Make sure we have a vcpkg executable + testFile = os.path.join(self.path, '.vcpkg-root') + if not downloadVcpkg and not os.path.isfile(testFile): + print("Missing {}, bootstrapping".format(testFile)) + downloadVcpkg = True + + if downloadVcpkg: + print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) + hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) + + print("Replacing port files") + portsPath = os.path.join(self.path, 'ports') + if (os.path.islink(portsPath)): + os.unlink(portsPath) + if (os.path.isdir(portsPath)): + shutil.rmtree(portsPath, ignore_errors=True) + shutil.copytree(self.sourcePortsPath, portsPath) + + def run(self, commands): + actualCommands = [self.exe, '--vcpkg-root', self.path] + actualCommands.extend(commands) + print("Running command") + print(actualCommands) + hifi_utils.executeSubprocess(actualCommands, folder=self.path) + + def setupDependencies(self): + # Special case for android, grab a bunch of binaries + # FIXME remove special casing for android builds eventually + if self.args.android: + print("Installing Android binaries") + self.setupAndroidDependencies() + + print("Installing host tools") + self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools']) + + # If not android, install the hifi-client-deps libraries + if not self.args.android: + print("Installing build dependencies") + self.run(['install', '--triplet', self.triplet, 'hifi-client-deps']) + + def cleanBuilds(self): + # Remove temporary build artifacts + builddir = os.path.join(self.path, 'buildtrees') + if os.path.isdir(builddir): + print("Wiping build trees") + shutil.rmtree(builddir, ignore_errors=True) + + def setupAndroidDependencies(self): + # vcpkg prebuilt + if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')): + dest = os.path.join(self.path, 'installed') + url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" + # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash + #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" + #hifi_utils.downloadAndExtract(url, dest, hash) + hifi_utils.downloadAndExtract(url, dest) + + def writeTag(self): + print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) + with open(self.tagFile, 'w') as f: + f.write(self.tagContents) + + def writeConfig(self): + print("Writing cmake config to {}".format(self.configFilePath)) + # Write out the configuration for use by CMake + cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') + installPath = os.path.join(self.path, 'installed', self.triplet) + toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools') + cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE + if not self.args.android: + cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID + cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath).replace('\\', '/') + with open(self.configFilePath, 'w') as f: + f.write(cmakeConfig) + + def cleanOldBuilds(self): + # FIXME because we have the base directory, and because a build will + # update the tag file on every run, we can scan the base dir for sub directories containing + # a tag file that is older than N days, and if found, delete the directory, recovering space + print("Not implemented") + + diff --git a/interface/resources/qml/dialogs/preferences/CheckBoxPreference.qml b/interface/resources/qml/dialogs/preferences/CheckBoxPreference.qml index f6f840bbe8..c4d5ec7272 100644 --- a/interface/resources/qml/dialogs/preferences/CheckBoxPreference.qml +++ b/interface/resources/qml/dialogs/preferences/CheckBoxPreference.qml @@ -16,10 +16,11 @@ import controlsUit 1.0 Preference { id: root height: spacer.height + Math.max(hifi.dimensions.controlLineHeight, checkBox.implicitHeight) - + property bool value: false Component.onCompleted: { checkBox.checked = preference.value; preference.value = Qt.binding(function(){ return checkBox.checked; }); + value = checkBox.checked; } function save() { @@ -47,6 +48,7 @@ Preference { onClicked: { Tablet.playSound(TabletEnums.ButtonClick); + value = checked; } anchors { diff --git a/interface/resources/qml/hifi/commerce/wallet/Wallet.qml b/interface/resources/qml/hifi/commerce/wallet/Wallet.qml index eb4e1f7aa8..f8e2c9115b 100644 --- a/interface/resources/qml/hifi/commerce/wallet/Wallet.qml +++ b/interface/resources/qml/hifi/commerce/wallet/Wallet.qml @@ -100,6 +100,9 @@ Rectangle { console.log("Failed to get Available Updates", result.data.message); } else { exchangeMoneyButtonContainer.messagesWaiting = result.data.updates.length > 0; + if (!exchangeMoneyButtonContainer.messagesWaiting) { + sendToScript({method: 'clearShouldShowDotUpdates'}); + } } } } diff --git a/interface/resources/qml/hifi/tablet/tabletWindows/TabletPreferencesDialog.qml b/interface/resources/qml/hifi/tablet/tabletWindows/TabletPreferencesDialog.qml index a5d7b23df6..846c0600e4 100644 --- a/interface/resources/qml/hifi/tablet/tabletWindows/TabletPreferencesDialog.qml +++ b/interface/resources/qml/hifi/tablet/tabletWindows/TabletPreferencesDialog.qml @@ -123,12 +123,12 @@ Item { } // Runtime customization of preferences. - var locomotionPreference = findPreference("VR Movement", "Teleporting only / Walking and teleporting"); + var locomotionPreference = findPreference("VR Movement", "Walking"); var flyingPreference = findPreference("VR Movement", "Jumping and flying"); if (locomotionPreference && flyingPreference) { - flyingPreference.visible = (locomotionPreference.value === 1); + flyingPreference.visible = locomotionPreference.value; locomotionPreference.valueChanged.connect(function () { - flyingPreference.visible = (locomotionPreference.value === 1); + flyingPreference.visible = locomotionPreference.value; }); } if (HMD.isHeadControllerAvailable("Oculus")) { diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index fbad66bf55..bf5fcc5b20 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -100,6 +100,7 @@ #include #include #include +#include #include #include #include @@ -830,6 +831,7 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(NodeType::Agent, listenPort); DependencyManager::set(); DependencyManager::set(); + DependencyManager::set(); // ModelFormatRegistry must be defined before ModelCache. See the ModelCache constructor. DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -2690,6 +2692,7 @@ Application::~Application() { DependencyManager::destroy(); DependencyManager::destroy(); DependencyManager::destroy(); + DependencyManager::destroy(); DependencyManager::destroy(); DependencyManager::destroy(); DependencyManager::destroy(); diff --git a/interface/src/Menu.cpp b/interface/src/Menu.cpp index e9a44b1e87..bbbd8db89f 100644 --- a/interface/src/Menu.cpp +++ b/interface/src/Menu.cpp @@ -802,7 +802,7 @@ Menu::Menu() { connect(action, &QAction::triggered, qApp, []() { std::thread(crash::newFault).join(); }); // Developer > Show Statistics - addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats); + addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::Stats, 0, true); // Developer > Show Animation Statistics addCheckableActionToQMenuAndActionHash(developerMenu, MenuOption::AnimStats); diff --git a/interface/src/avatar/MyAvatar.h b/interface/src/avatar/MyAvatar.h index 848107917a..b2381366bb 100644 --- a/interface/src/avatar/MyAvatar.h +++ b/interface/src/avatar/MyAvatar.h @@ -247,6 +247,7 @@ class MyAvatar : public Avatar { Q_PROPERTY(bool isInSittingState READ getIsInSittingState WRITE setIsInSittingState); Q_PROPERTY(MyAvatar::SitStandModelType userRecenterModel READ getUserRecenterModel WRITE setUserRecenterModel); Q_PROPERTY(bool isSitStandStateLocked READ getIsSitStandStateLocked WRITE setIsSitStandStateLocked); + Q_PROPERTY(bool allowTeleporting READ getAllowTeleporting) const QString DOMINANT_LEFT_HAND = "left"; const QString DOMINANT_RIGHT_HAND = "right"; @@ -559,6 +560,9 @@ public: void setUseAdvancedMovementControls(bool useAdvancedMovementControls) { _useAdvancedMovementControls.set(useAdvancedMovementControls); } + bool getAllowTeleporting() { return _allowTeleportingSetting.get(); } + void setAllowTeleporting(bool allowTeleporting) { _allowTeleportingSetting.set(allowTeleporting); } + bool getShowPlayArea() const { return _showPlayArea.get(); } void setShowPlayArea(bool showPlayArea) { _showPlayArea.set(showPlayArea); } @@ -1889,6 +1893,7 @@ private: Setting::Handle _userHeightSetting; Setting::Handle _flyingHMDSetting; Setting::Handle _avatarEntityCountSetting; + Setting::Handle _allowTeleportingSetting { "allowTeleporting", true }; std::vector> _avatarEntityIDSettings; std::vector> _avatarEntityDataSettings; }; diff --git a/interface/src/graphics/GraphicsEngine.cpp b/interface/src/graphics/GraphicsEngine.cpp index 36bf3a1b97..c2137d3d97 100644 --- a/interface/src/graphics/GraphicsEngine.cpp +++ b/interface/src/graphics/GraphicsEngine.cpp @@ -56,9 +56,11 @@ void GraphicsEngine::initializeGPU(GLWidget* glwidget) { glwidget->makeCurrent(); _gpuContext = std::make_shared(); +#ifndef Q_OS_ANDROID _gpuContext->pushProgramsToSync(shader::allPrograms(), [this] { _programsCompiled.store(true); }, 1); +#endif DependencyManager::get()->setGPUContext(_gpuContext); } diff --git a/interface/src/graphics/GraphicsEngine.h b/interface/src/graphics/GraphicsEngine.h index 83e774a64f..f0b88d2459 100644 --- a/interface/src/graphics/GraphicsEngine.h +++ b/interface/src/graphics/GraphicsEngine.h @@ -86,7 +86,11 @@ protected: FrameTimingsScriptingInterface _frameTimingsScriptingInterface; std::shared_ptr _splashScreen { std::make_shared() }; +#ifndef Q_OS_ANDROID std::atomic _programsCompiled { false }; +#else + std::atomic _programsCompiled { true }; +#endif friend class Application; }; diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index d1fbe02759..27e51c3654 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -238,14 +238,15 @@ void setupPreferences() { static const QString VR_MOVEMENT{ "VR Movement" }; { - auto getter = [myAvatar]()->int { return myAvatar->useAdvancedMovementControls() ? 1 : 0; }; - auto setter = [myAvatar](int value) { myAvatar->setUseAdvancedMovementControls(value == 1); }; - auto preference = - new RadioButtonsPreference(VR_MOVEMENT, "Teleporting only / Walking and teleporting", getter, setter); - QStringList items; - items << "Teleporting only" << "Walking and teleporting"; - preference->setHeading("Movement mode"); - preference->setItems(items); + auto getter = [myAvatar]()->bool { return myAvatar->getAllowTeleporting(); }; + auto setter = [myAvatar](bool value) { myAvatar->setAllowTeleporting(value); }; + auto preference = new CheckPreference(VR_MOVEMENT, "Teleporting", getter, setter); + preferences->addPreference(preference); + } + { + auto getter = [myAvatar]()->bool { return myAvatar->useAdvancedMovementControls(); }; + auto setter = [myAvatar](bool value) { myAvatar->setUseAdvancedMovementControls(value); }; + auto preference = new CheckPreference(VR_MOVEMENT, "Walking", getter, setter); preferences->addPreference(preference); } { diff --git a/libraries/animation/src/AnimationCache.cpp b/libraries/animation/src/AnimationCache.cpp index ec26782d0e..f7a7dd861a 100644 --- a/libraries/animation/src/AnimationCache.cpp +++ b/libraries/animation/src/AnimationCache.cpp @@ -20,6 +20,7 @@ #include #include "AnimationLogging.h" +#include int animationPointerMetaTypeId = qRegisterMetaType(); diff --git a/libraries/animation/src/AnimationCache.h b/libraries/animation/src/AnimationCache.h index d4574d9d3b..2f8168625e 100644 --- a/libraries/animation/src/AnimationCache.h +++ b/libraries/animation/src/AnimationCache.h @@ -17,7 +17,7 @@ #include #include -#include +#include #include class Animation; diff --git a/libraries/fbx/src/FBXSerializer.cpp b/libraries/fbx/src/FBXSerializer.cpp index 67220342b8..b425b6795d 100644 --- a/libraries/fbx/src/FBXSerializer.cpp +++ b/libraries/fbx/src/FBXSerializer.cpp @@ -1854,6 +1854,17 @@ HFMModel* FBXSerializer::extractHFMModel(const QVariantHash& mapping, const QStr return hfmModelPtr; } +MediaType FBXSerializer::getMediaType() const { + MediaType mediaType("fbx"); + mediaType.extensions.push_back("fbx"); + mediaType.fileSignatures.emplace_back("Kaydara FBX Binary \x00", 0); + return mediaType; +} + +std::unique_ptr FBXSerializer::getFactory() const { + return std::make_unique>(); +} + HFMModel::Pointer FBXSerializer::read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url) { QBuffer buffer(const_cast(&data)); buffer.open(QIODevice::ReadOnly); diff --git a/libraries/fbx/src/FBXSerializer.h b/libraries/fbx/src/FBXSerializer.h index c69f75cc5c..a76fb8f9bf 100644 --- a/libraries/fbx/src/FBXSerializer.h +++ b/libraries/fbx/src/FBXSerializer.h @@ -96,6 +96,9 @@ class ExtractedMesh; class FBXSerializer : public HFMSerializer { public: + MediaType getMediaType() const override; + std::unique_ptr getFactory() const override; + HFMModel* _hfmModel; /// Reads HFMModel from the supplied model and mapping data. /// \exception QString if an error occurs in parsing diff --git a/libraries/fbx/src/GLTFSerializer.cpp b/libraries/fbx/src/GLTFSerializer.cpp index 28d377c605..e254a91eb0 100644 --- a/libraries/fbx/src/GLTFSerializer.cpp +++ b/libraries/fbx/src/GLTFSerializer.cpp @@ -35,11 +35,6 @@ #include "FBXSerializer.h" - -GLTFSerializer::GLTFSerializer() { - -} - bool GLTFSerializer::getStringVal(const QJsonObject& object, const QString& fieldname, QString& value, QMap& defined) { bool _defined = (object.contains(fieldname) && object[fieldname].isString()); @@ -910,6 +905,17 @@ bool GLTFSerializer::buildGeometry(HFMModel& hfmModel, const QUrl& url) { return true; } +MediaType GLTFSerializer::getMediaType() const { + MediaType mediaType("gltf"); + mediaType.extensions.push_back("gltf"); + mediaType.webMediaTypes.push_back("model/gltf+json"); + return mediaType; +} + +std::unique_ptr GLTFSerializer::getFactory() const { + return std::make_unique>(); +} + HFMModel::Pointer GLTFSerializer::read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url) { _url = url; diff --git a/libraries/fbx/src/GLTFSerializer.h b/libraries/fbx/src/GLTFSerializer.h index 1ec1183e36..5fca77c4fd 100644 --- a/libraries/fbx/src/GLTFSerializer.h +++ b/libraries/fbx/src/GLTFSerializer.h @@ -16,7 +16,6 @@ #include #include #include -#include "FBXSerializer.h" struct GLTFAsset { @@ -703,7 +702,9 @@ struct GLTFFile { class GLTFSerializer : public QObject, public HFMSerializer { Q_OBJECT public: - GLTFSerializer(); + MediaType getMediaType() const override; + std::unique_ptr getFactory() const override; + HFMModel::Pointer read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url = QUrl()) override; private: GLTFFile _file; diff --git a/libraries/fbx/src/OBJSerializer.cpp b/libraries/fbx/src/OBJSerializer.cpp index af8dfb5562..9c92466565 100644 --- a/libraries/fbx/src/OBJSerializer.cpp +++ b/libraries/fbx/src/OBJSerializer.cpp @@ -651,6 +651,15 @@ done: return result; } +MediaType OBJSerializer::getMediaType() const { + MediaType mediaType("obj"); + mediaType.extensions.push_back("obj"); + return mediaType; +} + +std::unique_ptr OBJSerializer::getFactory() const { + return std::make_unique>(); +} HFMModel::Pointer OBJSerializer::read(const QByteArray& data, const QVariantHash& mapping, const QUrl& url) { PROFILE_RANGE_EX(resource_parse, __FUNCTION__, 0xffff0000, nullptr); diff --git a/libraries/fbx/src/OBJSerializer.h b/libraries/fbx/src/OBJSerializer.h index a6fe3817ca..c4f8025e66 100644 --- a/libraries/fbx/src/OBJSerializer.h +++ b/libraries/fbx/src/OBJSerializer.h @@ -14,7 +14,6 @@ #include #include -#include "FBXSerializer.h" class OBJTokenizer { public: @@ -92,6 +91,9 @@ public: class OBJSerializer: public QObject, public HFMSerializer { // QObject so we can make network requests. Q_OBJECT public: + MediaType getMediaType() const override; + std::unique_ptr getFactory() const override; + typedef QVector FaceGroup; QVector vertices; QVector vertexColors; diff --git a/libraries/hfm/src/hfm/HFMFormatRegistry.cpp b/libraries/hfm/src/hfm/HFMFormatRegistry.cpp new file mode 100644 index 0000000000..1328369cff --- /dev/null +++ b/libraries/hfm/src/hfm/HFMFormatRegistry.cpp @@ -0,0 +1,65 @@ +// +// HFMFormatRegistry.cpp +// libraries/hfm/src/hfm +// +// Created by Sabrina Shanman on 2018/11/29. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "HFMFormatRegistry.h" + +namespace hfm { + +FormatRegistry::MediaTypeID FormatRegistry::registerMediaType(const MediaType& mediaType, std::unique_ptr supportedFactory) { + std::lock_guard lock(_libraryLock); + + MediaTypeID id = _mediaTypeLibrary.registerMediaType(mediaType); + _supportedFormats.emplace_back(id, supportedFactory); + return id; +} + +void FormatRegistry::unregisterMediaType(const MediaTypeID& mediaTypeID) { + std::lock_guard lock(_libraryLock); + + for (auto it = _supportedFormats.begin(); it != _supportedFormats.end(); it++) { + if ((*it).mediaTypeID == mediaTypeID) { + _supportedFormats.erase(it); + break; + } + } + _mediaTypeLibrary.unregisterMediaType(mediaTypeID); +} + +std::shared_ptr FormatRegistry::getSerializerForMediaTypeID(FormatRegistry::MediaTypeID mediaTypeID) const { + // TODO: shared_lock in C++14 + std::lock_guard lock(*const_cast(&_libraryLock)); + + for (auto it = _supportedFormats.begin(); it != _supportedFormats.end(); it++) { + if ((*it).mediaTypeID == mediaTypeID) { + return (*it).factory->get(); + } + } + return std::shared_ptr(); +} + +std::shared_ptr FormatRegistry::getSerializerForMediaType(const hifi::ByteArray& data, const hifi::URL& url, const std::string& webMediaType) const { + MediaTypeID id; + { + // TODO: shared_lock in C++14 + std::lock_guard lock(*const_cast(&_libraryLock)); + + id = _mediaTypeLibrary.findMediaTypeForData(data); + if (id == INVALID_MEDIA_TYPE_ID) { + id = _mediaTypeLibrary.findMediaTypeForURL(url); + if (id == INVALID_MEDIA_TYPE_ID) { + id = _mediaTypeLibrary.findMediaTypeForWebID(webMediaType); + } + } + } + return getSerializerForMediaTypeID(id); +} + +}; diff --git a/libraries/hfm/src/hfm/HFMFormatRegistry.h b/libraries/hfm/src/hfm/HFMFormatRegistry.h new file mode 100644 index 0000000000..a437e9ac37 --- /dev/null +++ b/libraries/hfm/src/hfm/HFMFormatRegistry.h @@ -0,0 +1,50 @@ +// +// HFMFormatRegistry.h +// libraries/hfm/src/hfm +// +// Created by Sabrina Shanman on 2018/11/28. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_HFMFormatRegistry_h +#define hifi_HFMFormatRegistry_h + +#include "HFMSerializer.h" +#include +#include + +namespace hfm { + +class FormatRegistry { +public: + using MediaTypeID = MediaTypeLibrary::ID; + static const MediaTypeID INVALID_MEDIA_TYPE_ID { MediaTypeLibrary::INVALID_ID }; + + MediaTypeID registerMediaType(const MediaType& mediaType, std::unique_ptr supportedFactory); + void unregisterMediaType(const MediaTypeID& id); + + std::shared_ptr getSerializerForMediaType(const hifi::ByteArray& data, const hifi::URL& url, const std::string& webMediaType) const; + +protected: + std::shared_ptr getSerializerForMediaTypeID(MediaTypeID id) const; + + MediaTypeLibrary _mediaTypeLibrary; + std::mutex _libraryLock; + class SupportedFormat { + public: + SupportedFormat(const MediaTypeID& mediaTypeID, std::unique_ptr& factory) : + mediaTypeID(mediaTypeID), + factory(std::move(factory)) { + } + MediaTypeID mediaTypeID; + std::unique_ptr factory; + }; + std::vector _supportedFormats; +}; + +}; + +#endif // hifi_HFMFormatRegistry_h diff --git a/libraries/hfm/src/hfm/HFMSerializer.h b/libraries/hfm/src/hfm/HFMSerializer.h index db18f21e06..868ec3dd45 100644 --- a/libraries/hfm/src/hfm/HFMSerializer.h +++ b/libraries/hfm/src/hfm/HFMSerializer.h @@ -15,10 +15,27 @@ #include #include "HFM.h" +#include namespace hfm { class Serializer { +public: + class Factory { + public: + virtual std::shared_ptr get() = 0; + }; + + template + class SimpleFactory : public Factory { + std::shared_ptr get() override { + return std::make_shared(); + } + }; + + virtual MediaType getMediaType() const = 0; + virtual std::unique_ptr getFactory() const = 0; + virtual Model::Pointer read(const hifi::ByteArray& data, const hifi::VariantHash& mapping, const hifi::URL& url = hifi::URL()) = 0; }; diff --git a/libraries/hfm/src/hfm/ModelFormatRegistry.cpp b/libraries/hfm/src/hfm/ModelFormatRegistry.cpp new file mode 100644 index 0000000000..d95453161a --- /dev/null +++ b/libraries/hfm/src/hfm/ModelFormatRegistry.cpp @@ -0,0 +1,20 @@ +// +// ModelFormatRegistry.cpp +// libraries/model-networking/src/model-networking +// +// Created by Sabrina Shanman on 2018/11/30. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ModelFormatRegistry.h" + +void ModelFormatRegistry::addFormat(const hfm::Serializer& serializer) { + _hfmFormatRegistry.registerMediaType(serializer.getMediaType(), serializer.getFactory()); +} + +std::shared_ptr ModelFormatRegistry::getSerializerForMediaType(const hifi::ByteArray& data, const hifi::URL& url, const std::string& webMediaType) const { + return _hfmFormatRegistry.getSerializerForMediaType(data, url, webMediaType); +} diff --git a/libraries/hfm/src/hfm/ModelFormatRegistry.h b/libraries/hfm/src/hfm/ModelFormatRegistry.h new file mode 100644 index 0000000000..1228465298 --- /dev/null +++ b/libraries/hfm/src/hfm/ModelFormatRegistry.h @@ -0,0 +1,28 @@ +// +// ModelFormatRegistry.h +// libraries/hfm/src/hfm +// +// Created by Sabrina Shanman on 2018/11/30. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ModelFormatRegistry_h +#define hifi_ModelFormatRegistry_h + +#include "HFMFormatRegistry.h" + +#include + +class ModelFormatRegistry : public Dependency { +public: + void addFormat(const hfm::Serializer& serializer); + std::shared_ptr getSerializerForMediaType(const hifi::ByteArray& data, const hifi::URL& url, const std::string& webMediaType) const; + +protected: + hfm::FormatRegistry _hfmFormatRegistry; +}; + +#endif // hifi_ModelFormatRegistry_h diff --git a/libraries/model-networking/src/model-networking/ModelCache.cpp b/libraries/model-networking/src/model-networking/ModelCache.cpp index 254f61eba9..dfee4750f5 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.cpp +++ b/libraries/model-networking/src/model-networking/ModelCache.cpp @@ -12,9 +12,6 @@ #include "ModelCache.h" #include #include -#include "FBXSerializer.h" -#include "OBJSerializer.h" -#include "GLTFSerializer.h" #include #include @@ -26,6 +23,10 @@ #include "ModelNetworkingLogging.h" #include #include +#include +#include +#include +#include Q_LOGGING_CATEGORY(trace_resource_parse_geometry, "trace.resource.parse.geometry") @@ -144,9 +145,9 @@ void GeometryMappingResource::onGeometryMappingLoaded(bool success) { class GeometryReader : public QRunnable { public: - GeometryReader(QWeakPointer& resource, const QUrl& url, const QVariantHash& mapping, - const QByteArray& data, bool combineParts) : - _resource(resource), _url(url), _mapping(mapping), _data(data), _combineParts(combineParts) { + GeometryReader(const ModelLoader& modelLoader, QWeakPointer& resource, const QUrl& url, const QVariantHash& mapping, + const QByteArray& data, bool combineParts, const QString& webMediaType) : + _modelLoader(modelLoader), _resource(resource), _url(url), _mapping(mapping), _data(data), _combineParts(combineParts), _webMediaType(webMediaType) { DependencyManager::get()->incrementStat("PendingProcessing"); } @@ -154,11 +155,13 @@ public: virtual void run() override; private: + ModelLoader _modelLoader; QWeakPointer _resource; QUrl _url; QVariantHash _mapping; QByteArray _data; bool _combineParts; + QString _webMediaType; }; void GeometryReader::run() { @@ -183,62 +186,53 @@ void GeometryReader::run() { throw QString("reply is NULL"); } - QString urlname = _url.path().toLower(); - if (!urlname.isEmpty() && !_url.path().isEmpty() && + // Ensure the resource has not been deleted + auto resource = _resource.toStrongRef(); + if (!resource) { + qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; + return; + } - (_url.path().toLower().endsWith(".fbx") || - _url.path().toLower().endsWith(".obj") || - _url.path().toLower().endsWith(".obj.gz") || - _url.path().toLower().endsWith(".gltf"))) { - - HFMModel::Pointer hfmModel; - - QVariantHash serializerMapping = _mapping; - serializerMapping["combineParts"] = _combineParts; - - if (_url.path().toLower().endsWith(".fbx")) { - hfmModel = FBXSerializer().read(_data, serializerMapping, _url); - if (hfmModel->meshes.size() == 0 && hfmModel->joints.size() == 0) { - throw QString("empty geometry, possibly due to an unsupported FBX version"); - } - } else if (_url.path().toLower().endsWith(".obj")) { - hfmModel = OBJSerializer().read(_data, serializerMapping, _url); - } else if (_url.path().toLower().endsWith(".obj.gz")) { - QByteArray uncompressedData; - if (gunzip(_data, uncompressedData)){ - hfmModel = OBJSerializer().read(uncompressedData, serializerMapping, _url); - } else { - throw QString("failed to decompress .obj.gz"); - } - - } else if (_url.path().toLower().endsWith(".gltf")) { - hfmModel = GLTFSerializer().read(_data, serializerMapping, _url); - if (hfmModel->meshes.size() == 0 && hfmModel->joints.size() == 0) { - throw QString("empty geometry, possibly due to an unsupported GLTF version"); - } - } else { - throw QString("unsupported format"); - } - - // Add scripts to hfmModel - if (!_mapping.value(SCRIPT_FIELD).isNull()) { - QVariantList scripts = _mapping.values(SCRIPT_FIELD); - for (auto &script : scripts) { - hfmModel->scripts.push_back(script.toString()); - } - } - - // Ensure the resource has not been deleted - auto resource = _resource.toStrongRef(); - if (!resource) { - qCWarning(modelnetworking) << "Abandoning load of" << _url << "; could not get strong ref"; - } else { - QMetaObject::invokeMethod(resource.data(), "setGeometryDefinition", - Q_ARG(HFMModel::Pointer, hfmModel)); - } - } else { + if (_url.path().isEmpty()) { throw QString("url is invalid"); } + + HFMModel::Pointer hfmModel; + QVariantHash serializerMapping = _mapping; + serializerMapping["combineParts"] = _combineParts; + + if (_url.path().toLower().endsWith(".gz")) { + QByteArray uncompressedData; + if (!gunzip(_data, uncompressedData)) { + throw QString("failed to decompress .gz model"); + } + // Strip the compression extension from the path, so the loader can infer the file type from what remains. + // This is okay because we don't expect the serializer to be able to read the contents of a compressed model file. + auto strippedUrl = _url; + strippedUrl.setPath(_url.path().left(_url.path().size() - 3)); + hfmModel = _modelLoader.load(uncompressedData, serializerMapping, strippedUrl, ""); + } else { + hfmModel = _modelLoader.load(_data, serializerMapping, _url, _webMediaType.toStdString()); + } + + if (!hfmModel) { + throw QString("unsupported format"); + } + + if (hfmModel->meshes.empty() || hfmModel->joints.empty()) { + throw QString("empty geometry, possibly due to an unsupported model version"); + } + + // Add scripts to hfmModel + if (!_mapping.value(SCRIPT_FIELD).isNull()) { + QVariantList scripts = _mapping.values(SCRIPT_FIELD); + for (auto &script : scripts) { + hfmModel->scripts.push_back(script.toString()); + } + } + + QMetaObject::invokeMethod(resource.data(), "setGeometryDefinition", + Q_ARG(HFMModel::Pointer, hfmModel)); } catch (const std::exception&) { auto resource = _resource.toStrongRef(); if (resource) { @@ -258,8 +252,8 @@ void GeometryReader::run() { class GeometryDefinitionResource : public GeometryResource { Q_OBJECT public: - GeometryDefinitionResource(const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl, bool combineParts) : - GeometryResource(url, resolveTextureBaseUrl(url, textureBaseUrl)), _mapping(mapping), _combineParts(combineParts) {} + GeometryDefinitionResource(const ModelLoader& modelLoader, const QUrl& url, const QVariantHash& mapping, const QUrl& textureBaseUrl, bool combineParts) : + GeometryResource(url, resolveTextureBaseUrl(url, textureBaseUrl)), _modelLoader(modelLoader), _mapping(mapping), _combineParts(combineParts) {} QString getType() const override { return "GeometryDefinition"; } @@ -269,6 +263,7 @@ protected: Q_INVOKABLE void setGeometryDefinition(HFMModel::Pointer hfmModel); private: + ModelLoader _modelLoader; QVariantHash _mapping; bool _combineParts; }; @@ -278,7 +273,7 @@ void GeometryDefinitionResource::downloadFinished(const QByteArray& data) { _url = _effectiveBaseURL; _textureBaseUrl = _effectiveBaseURL; } - QThreadPool::globalInstance()->start(new GeometryReader(_self, _effectiveBaseURL, _mapping, data, _combineParts)); + QThreadPool::globalInstance()->start(new GeometryReader(_modelLoader, _self, _effectiveBaseURL, _mapping, data, _combineParts, _request->getWebMediaType())); } void GeometryDefinitionResource::setGeometryDefinition(HFMModel::Pointer hfmModel) { @@ -316,6 +311,11 @@ ModelCache::ModelCache() { const qint64 GEOMETRY_DEFAULT_UNUSED_MAX_SIZE = DEFAULT_UNUSED_MAX_SIZE; setUnusedResourceCacheSize(GEOMETRY_DEFAULT_UNUSED_MAX_SIZE); setObjectName("ModelCache"); + + auto modelFormatRegistry = DependencyManager::get(); + modelFormatRegistry->addFormat(FBXSerializer()); + modelFormatRegistry->addFormat(OBJSerializer()); + modelFormatRegistry->addFormat(GLTFSerializer()); } QSharedPointer ModelCache::createResource(const QUrl& url, const QSharedPointer& fallback, @@ -328,7 +328,7 @@ QSharedPointer ModelCache::createResource(const QUrl& url, const QShar auto mapping = geometryExtra ? geometryExtra->mapping : QVariantHash(); auto textureBaseUrl = geometryExtra ? geometryExtra->textureBaseUrl : QUrl(); bool combineParts = geometryExtra ? geometryExtra->combineParts : true; - resource = new GeometryDefinitionResource(url, mapping, textureBaseUrl, combineParts); + resource = new GeometryDefinitionResource(_modelLoader, url, mapping, textureBaseUrl, combineParts); } return QSharedPointer(resource, &Resource::deleter); diff --git a/libraries/model-networking/src/model-networking/ModelCache.h b/libraries/model-networking/src/model-networking/ModelCache.h index 9d458e7512..1018bdecd5 100644 --- a/libraries/model-networking/src/model-networking/ModelCache.h +++ b/libraries/model-networking/src/model-networking/ModelCache.h @@ -20,6 +20,7 @@ #include "FBXSerializer.h" #include "TextureCache.h" +#include "ModelLoader.h" // Alias instead of derive to avoid copying @@ -158,6 +159,7 @@ protected: private: ModelCache(); virtual ~ModelCache() = default; + ModelLoader _modelLoader; }; class NetworkMaterial : public graphics::Material { diff --git a/libraries/model-networking/src/model-networking/ModelLoader.cpp b/libraries/model-networking/src/model-networking/ModelLoader.cpp new file mode 100644 index 0000000000..65314633c9 --- /dev/null +++ b/libraries/model-networking/src/model-networking/ModelLoader.cpp @@ -0,0 +1,24 @@ +// +// ModelLoader.cpp +// libraries/model-networking/src/model-networking +// +// Created by Sabrina Shanman on 2018/11/14. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ModelLoader.h" + +#include +#include + + +hfm::Model::Pointer ModelLoader::load(const hifi::ByteArray& data, const hifi::VariantHash& mapping, const hifi::URL& url, const std::string& webMediaType) const { + auto serializer = DependencyManager::get()->getSerializerForMediaType(data, url, webMediaType); + if (!serializer) { + return hfm::Model::Pointer(); + } + return serializer->read(data, mapping, url); +} diff --git a/libraries/model-networking/src/model-networking/ModelLoader.h b/libraries/model-networking/src/model-networking/ModelLoader.h new file mode 100644 index 0000000000..5fbab4fb65 --- /dev/null +++ b/libraries/model-networking/src/model-networking/ModelLoader.h @@ -0,0 +1,26 @@ +// +// ModelLoader.h +// libraries/model-networking/src/model-networking +// +// Created by Sabrina Shanman on 2018/11/13. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ModelLoader_h +#define hifi_ModelLoader_h + +#include +#include + +class ModelLoader { +public: + // Given the currently stored list of supported file formats, determine how to load a model from the given parameters. + // If successful, return an owned reference to the newly loaded model. + // If failed, return an empty reference. + hfm::Model::Pointer load(const hifi::ByteArray& data, const hifi::VariantHash& mapping, const hifi::URL& url, const std::string& webMediaType) const; +}; + +#endif // hifi_ModelLoader_h diff --git a/libraries/networking/src/HTTPResourceRequest.cpp b/libraries/networking/src/HTTPResourceRequest.cpp index e26f27adcf..50221a136a 100644 --- a/libraries/networking/src/HTTPResourceRequest.cpp +++ b/libraries/networking/src/HTTPResourceRequest.cpp @@ -94,7 +94,7 @@ void HTTPResourceRequest::onRequestFinished() { // Content-Range: -/* // Content-Range: */ // - auto parseContentRangeHeader = [](QString contentRangeHeader) -> std::pair { + static auto parseContentRangeHeader = [](QString contentRangeHeader) -> std::pair { auto unitRangeParts = contentRangeHeader.split(' '); if (unitRangeParts.size() != 2) { return { false, 0 }; @@ -115,6 +115,15 @@ void HTTPResourceRequest::onRequestFinished() { } }; + static auto parseMediaType = [](QString contentTypeHeader) -> std::pair { + auto contentTypeParts = contentTypeHeader.split(';'); + if (contentTypeParts.size() < 1) { + return { false, "" }; + } + + return { true, contentTypeParts[0] }; + }; + switch(_reply->error()) { case QNetworkReply::NoError: _data = _reply->readAll(); @@ -141,6 +150,16 @@ void HTTPResourceRequest::onRequestFinished() { } } + { + auto contentTypeHeader = _reply->rawHeader("Content-Type"); + bool success; + QString mediaType; + std::tie(success, mediaType) = parseMediaType(contentTypeHeader); + if (success) { + _webMediaType = mediaType; + } + } + recordBytesDownloadedInStats(STAT_HTTP_RESOURCE_TOTAL_BYTES, _data.size()); break; diff --git a/libraries/networking/src/ResourceRequest.h b/libraries/networking/src/ResourceRequest.h index eb306ca5be..550294d79b 100644 --- a/libraries/networking/src/ResourceRequest.h +++ b/libraries/networking/src/ResourceRequest.h @@ -84,6 +84,7 @@ public: bool loadedFromCache() const { return _loadedFromCache; } bool getRangeRequestSuccessful() const { return _rangeRequestSuccessful; } bool getTotalSizeOfResource() const { return _totalSizeOfResource; } + QString getWebMediaType() const { return _webMediaType; } void setFailOnRedirect(bool failOnRedirect) { _failOnRedirect = failOnRedirect; } void setCacheEnabled(bool value) { _cacheEnabled = value; } @@ -111,6 +112,7 @@ protected: ByteRange _byteRange; bool _rangeRequestSuccessful { false }; uint64_t _totalSizeOfResource { 0 }; + QString _webMediaType; int64_t _lastRecordedBytesDownloaded { 0 }; bool _isObservable; qint64 _callerId; diff --git a/libraries/shared/src/SpatiallyNestable.cpp b/libraries/shared/src/SpatiallyNestable.cpp index 97e20f5627..d704498143 100644 --- a/libraries/shared/src/SpatiallyNestable.cpp +++ b/libraries/shared/src/SpatiallyNestable.cpp @@ -748,6 +748,18 @@ const Transform SpatiallyNestable::getTransform() const { return result; } +void SpatiallyNestable::breakParentingLoop() const { + // someone created a loop. break it... + qCDebug(shared) << "Parenting loop detected: " << getID(); + SpatiallyNestablePointer _this = getThisPointer(); + _this->setParentID(QUuid()); + bool setPositionSuccess; + AACube aaCube = getQueryAACube(setPositionSuccess); + if (setPositionSuccess) { + _this->setWorldPosition(aaCube.calcCenter()); + } +} + const Transform SpatiallyNestable::getTransform(int jointIndex, bool& success, int depth) const { // this returns the world-space transform for this object. It finds its parent's transform (which may // cause this object's parent to query its parent, etc) and multiplies this object's local transform onto it. @@ -755,15 +767,7 @@ const Transform SpatiallyNestable::getTransform(int jointIndex, bool& success, i if (depth > MAX_PARENTING_CHAIN_SIZE) { success = false; - // someone created a loop. break it... - qCDebug(shared) << "Parenting loop detected: " << getID(); - SpatiallyNestablePointer _this = getThisPointer(); - _this->setParentID(QUuid()); - bool setPositionSuccess; - AACube aaCube = getQueryAACube(setPositionSuccess); - if (setPositionSuccess) { - _this->setWorldPosition(aaCube.calcCenter()); - } + breakParentingLoop(); return jointInWorldFrame; } @@ -1208,8 +1212,12 @@ AACube SpatiallyNestable::getQueryAACube() const { return result; } -bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { - bool success; +bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType, int depth) const { + if (depth > MAX_PARENTING_CHAIN_SIZE) { + breakParentingLoop(); + return false; + } + if (nestableType == NestableType::Avatar) { QUuid parentID = getParentID(); if (parentID == AVATAR_SELF_ID) { @@ -1217,6 +1225,7 @@ bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { } } + bool success; SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return false; @@ -1226,11 +1235,14 @@ bool SpatiallyNestable::hasAncestorOfType(NestableType nestableType) const { return true; } - return parent->hasAncestorOfType(nestableType); + return parent->hasAncestorOfType(nestableType, depth + 1); } -const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType) const { - bool success; +const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType, int depth) const { + if (depth > MAX_PARENTING_CHAIN_SIZE) { + breakParentingLoop(); + return QUuid(); + } if (nestableType == NestableType::Avatar) { QUuid parentID = getParentID(); @@ -1239,6 +1251,7 @@ const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType) con } } + bool success; SpatiallyNestablePointer parent = getParentPointer(success); if (!success || !parent) { return QUuid(); @@ -1248,7 +1261,7 @@ const QUuid SpatiallyNestable::findAncestorOfType(NestableType nestableType) con return parent->getID(); } - return parent->findAncestorOfType(nestableType); + return parent->findAncestorOfType(nestableType, depth + 1); } void SpatiallyNestable::getLocalTransformAndVelocities( @@ -1336,7 +1349,12 @@ void SpatiallyNestable::dump(const QString& prefix) const { } } -bool SpatiallyNestable::isParentPathComplete() const { +bool SpatiallyNestable::isParentPathComplete(int depth) const { + if (depth > MAX_PARENTING_CHAIN_SIZE) { + breakParentingLoop(); + return false; + } + static const QUuid IDENTITY; QUuid parentID = getParentID(); if (parentID.isNull() || parentID == IDENTITY) { @@ -1349,5 +1367,5 @@ bool SpatiallyNestable::isParentPathComplete() const { return false; } - return parent->isParentPathComplete(); + return parent->isParentPathComplete(depth + 1); } diff --git a/libraries/shared/src/SpatiallyNestable.h b/libraries/shared/src/SpatiallyNestable.h index f8658f73d4..cf2e304d19 100644 --- a/libraries/shared/src/SpatiallyNestable.h +++ b/libraries/shared/src/SpatiallyNestable.h @@ -75,7 +75,7 @@ public: static QString nestableTypeToString(NestableType nestableType); - virtual bool isParentPathComplete() const; + virtual bool isParentPathComplete(int depth = 0) const; // world frame @@ -189,8 +189,8 @@ public: bool isParentIDValid() const { bool success = false; getParentPointer(success); return success; } virtual SpatialParentTree* getParentTree() const { return nullptr; } - bool hasAncestorOfType(NestableType nestableType) const; - const QUuid findAncestorOfType(NestableType nestableType) const; + bool hasAncestorOfType(NestableType nestableType, int depth = 0) const; + const QUuid findAncestorOfType(NestableType nestableType, int depth = 0) const; SpatiallyNestablePointer getParentPointer(bool& success) const; static SpatiallyNestablePointer findByID(QUuid id, bool& success); @@ -248,6 +248,8 @@ private: mutable bool _parentKnowsMe { false }; bool _isDead { false }; bool _queryAACubeIsPuffed { false }; + + void breakParentingLoop() const; }; diff --git a/libraries/shared/src/shared/MediaTypeLibrary.cpp b/libraries/shared/src/shared/MediaTypeLibrary.cpp new file mode 100644 index 0000000000..790897c3e2 --- /dev/null +++ b/libraries/shared/src/shared/MediaTypeLibrary.cpp @@ -0,0 +1,85 @@ +// +// MediaTypeLibrary.cpp +// libraries/shared/src/shared +// +// Created by Sabrina Shanman on 2018/11/29. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "MediaTypeLibrary.h" + +MediaType MediaType::NONE = MediaType(""); + +MediaTypeLibrary::ID MediaTypeLibrary::registerMediaType(const MediaType& mediaType) { + ID id = nextID++; + _mediaTypes.emplace_back(id, mediaType); + return id; +} + +void MediaTypeLibrary::unregisterMediaType(const MediaTypeLibrary::ID& id) { + for (auto it = _mediaTypes.begin(); it != _mediaTypes.end(); it++) { + if ((*it).id == id) { + _mediaTypes.erase(it); + break; + } + } +} + +MediaType MediaTypeLibrary::getMediaType(const MediaTypeLibrary::ID& id) const { + for (auto& supportedFormat : _mediaTypes) { + if (supportedFormat.id == id) { + return supportedFormat.mediaType; + } + } + return MediaType::NONE; +} + +MediaTypeLibrary::ID MediaTypeLibrary::findMediaTypeForData(const hifi::ByteArray& data) const { + // Check file contents + for (auto& mediaType : _mediaTypes) { + for (auto& fileSignature : mediaType.mediaType.fileSignatures) { + auto testBytes = data.mid(fileSignature.byteOffset, (int)fileSignature.bytes.size()).toStdString(); + if (testBytes == fileSignature.bytes) { + return mediaType.id; + } + } + } + + return INVALID_ID; +} + +MediaTypeLibrary::ID MediaTypeLibrary::findMediaTypeForURL(const hifi::URL& url) const { + // Check file extension + std::string urlString = url.path().toStdString(); + std::size_t extensionSeparator = urlString.rfind('.'); + if (extensionSeparator != std::string::npos) { + std::string detectedExtension = urlString.substr(extensionSeparator + 1); + for (auto& supportedFormat : _mediaTypes) { + for (auto& extension : supportedFormat.mediaType.extensions) { + if (extension == detectedExtension) { + return supportedFormat.id; + } + } + } + } + + return INVALID_ID; +} + +MediaTypeLibrary::ID MediaTypeLibrary::findMediaTypeForWebID(const std::string& webMediaType) const { + // Check web media type + if (webMediaType != "") { + for (auto& supportedFormat : _mediaTypes) { + for (auto& candidateWebMediaType : supportedFormat.mediaType.webMediaTypes) { + if (candidateWebMediaType == webMediaType) { + return supportedFormat.id; + } + } + } + } + + return INVALID_ID; +} diff --git a/libraries/shared/src/shared/MediaTypeLibrary.h b/libraries/shared/src/shared/MediaTypeLibrary.h new file mode 100644 index 0000000000..c87da01fa1 --- /dev/null +++ b/libraries/shared/src/shared/MediaTypeLibrary.h @@ -0,0 +1,90 @@ +// +// MediaTypeLibrary.h +// libraries/shared/src/shared +// +// Created by Sabrina Shanman on 2018/11/28. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_MediaTypeLibrary_h +#define hifi_MediaTypeLibrary_h + +#include +#include +#include +#include + +#include "HifiTypes.h" + +// A short sequence of bytes, typically at the beginning of the file, which identifies the file format +class FileSignature { +public: + FileSignature(const std::string& bytes, int byteOffset) : + bytes(bytes), + byteOffset(byteOffset) { + } + FileSignature(const FileSignature& fileSignature) : + bytes(fileSignature.bytes), + byteOffset(fileSignature.byteOffset) { + } + + std::string bytes; + int byteOffset; +}; + +// A named file extension with a list of known ways to positively identify the file type +class MediaType { +public: + MediaType(const std::string& name) : + name(name) { + } + MediaType() {}; + MediaType(const MediaType& mediaType) : + name(mediaType.name), + extensions(mediaType.extensions), + webMediaTypes(mediaType.webMediaTypes), + fileSignatures(mediaType.fileSignatures) { + } + + static MediaType NONE; + + std::string name; + std::vector extensions; + std::vector webMediaTypes; + std::vector fileSignatures; +}; + +class MediaTypeLibrary { +public: + using ID = unsigned int; + static const ID INVALID_ID { 0 }; + + ID registerMediaType(const MediaType& mediaType); + void unregisterMediaType(const ID& id); + + MediaType getMediaType(const ID& id) const; + + ID findMediaTypeForData(const hifi::ByteArray& data) const; + ID findMediaTypeForURL(const hifi::URL& url) const; + ID findMediaTypeForWebID(const std::string& webMediaType) const; + +protected: + ID nextID { 1 }; + + class Entry { + public: + Entry(const ID& id, const MediaType& mediaType) : + id(id), + mediaType(mediaType) { + } + ID id; + MediaType mediaType; + }; + + std::vector _mediaTypes; +}; + +#endif // hifi_MeidaTypeLibrary_h diff --git a/prebuild.py b/prebuild.py index b8cb2c96ae..a758dcbea2 100644 --- a/prebuild.py +++ b/prebuild.py @@ -1,5 +1,26 @@ #!python +# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring +# tools required by the build as well as dependencies on which we rely. +# +# By using this script, we can reduce the requirements for a developer getting started to: +# +# * A working C++ dev environment like visual studio, xcode, gcc, or clang +# * Qt +# * CMake +# * Python 3.x +# +# The function of the build script is to acquire, if not already present, all the other build requirements +# The build script should be idempotent. If you run it with the same arguments multiple times, that should +# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not +# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should +# execute quickly, determining that no work is to be done + +import hifi_singleton +import hifi_utils +import hifi_android +import hifi_vcpkg + import argparse import concurrent import hashlib @@ -9,252 +30,65 @@ import os import platform import shutil import ssl -import subprocess import sys -import tarfile +import re import tempfile import time -import urllib.request import functools print = functools.partial(print, flush=True) -def executeSubprocess(processArgs, folder=None, env=None): - restoreDir = None - if folder != None: - restoreDir = os.getcwd() - os.chdir(folder) - - process = subprocess.Popen( - processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env) - process.wait() - - if (0 != process.returncode): - raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format( - processArgs[0], - ' '.join(processArgs[1:]), - )) - - if restoreDir != None: - os.chdir(restoreDir) - - -def hashFile(file): - hasher = hashlib.sha512() - with open(file, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def hashFolder(folder): - hasher = hashlib.sha256() - for dirName, subdirList, fileList in os.walk(folder): - for fname in fileList: - with open(os.path.join(folder, dirName, fname), "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def downloadAndExtract(url, destPath, hash=None): - tempFileDescriptor, tempFileName = tempfile.mkstemp() - # OSX Python doesn't support SSL, so we need to bypass it. - # However, we still validate the downloaded file's sha512 hash - context = ssl._create_unverified_context() - with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile: - shutil.copyfileobj(response, tempFile) - - # Verify the hash - if hash and hash != hashFile(tempFileName): - raise RuntimeError("Downloaded file does not match hash") - - # Extract the archive - with tarfile.open(tempFileName, 'r:gz') as tgz: - tgz.extractall(destPath) - os.remove(tempFileName) - - -class VcpkgRepo: - def __init__(self): - global args - scriptPath = os.path.dirname(os.path.realpath(sys.argv[0])) - # our custom ports, relative to the script location - self.sourcePortsPath = os.path.join(scriptPath, 'cmake', 'ports') - # FIXME Revert to ports hash before release - self.id = hashFolder(self.sourcePortsPath)[:8] - # OS dependent information - system = platform.system() - - if args.vcpkg_root is not None: - print("override vcpkg path with " + args.vcpkg_root) - self.path = args.vcpkg_root - else: - if 'Darwin' == system: - defaultBasePath = os.path.expanduser('~/hifi/vcpkg') - else: - defaultBasePath = os.path.join(tempfile.gettempdir(), 'hifi', 'vcpkg') - basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath) - if (not os.path.isdir(basePath)): - os.makedirs(basePath) - self.path = os.path.join(basePath, self.id) - - self.tagFile = os.path.join(self.path, '.id') - # A format version attached to the tag file... increment when you want to force the build systems to rebuild - # without the contents of the ports changing - self.version = 1 - self.tagContents = "{}_{}".format(self.id, self.version) - - print("prebuild path: " + self.path) - if 'Windows' == system: - self.exe = os.path.join(self.path, 'vcpkg.exe') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-win32.tar.gz?versionId=YZYkDejDRk7L_hrK_WVFthWvisAhbDzZ' - self.vcpkgHash = '3e0ff829a74956491d57666109b3e6b5ce4ed0735c24093884317102387b2cb1b2cd1ff38af9ed9173501f6e32ffa05cc6fe6d470b77a71ca1ffc3e0aa46ab9e' - self.hostTriplet = 'x64-windows' - elif 'Darwin' == system: - self.exe = os.path.join(self.path, 'vcpkg') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-osx.tar.gz?versionId=_fhqSxjfrtDJBvEsQ8L_ODcdUjlpX9cc' - self.vcpkgHash = '519d666d02ef22b87c793f016ca412e70f92e1d55953c8f9bd4ee40f6d9f78c1df01a6ee293907718f3bbf24075cc35492fb216326dfc50712a95858e9cbcb4d' - self.hostTriplet = 'x64-osx' - else: - self.exe = os.path.join(self.path, 'vcpkg') - self.vcpkgUrl = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-linux.tar.gz?versionId=97Nazh24etEVKWz33XwgLY0bvxEfZgMU' - self.vcpkgHash = '6a1ce47ef6621e699a4627e8821ad32528c82fce62a6939d35b205da2d299aaa405b5f392df4a9e5343dd6a296516e341105fbb2dd8b48864781d129d7fba10d' - self.hostTriplet = 'x64-linux' - - if args.android: - self.triplet = 'arm64-android' - else: - self.triplet = self.hostTriplet - - def outOfDate(self): - global args - # Prevent doing a clean if we've explcitly set a directory for vcpkg - if args.vcpkg_root is not None: - return False - if args.force_build: - return True - print("Looking for tag file {}".format(self.tagFile)) - if not os.path.isfile(self.tagFile): - return True - with open(self.tagFile, 'r') as f: - storedTag = f.read() - print("Found stored tag {}".format(storedTag)) - if storedTag != self.tagContents: - print("Doesn't match computed tag {}".format(self.tagContents)) - return True - return False - - def clean(self): - cleanPath = self.path - print("Cleaning vcpkg installation at {}".format(cleanPath)) - if os.path.isdir(self.path): - print("Removing {}".format(cleanPath)) - shutil.rmtree(cleanPath, ignore_errors=True) - - def bootstrap(self): - global args - if self.outOfDate(): - self.clean() - - # don't download the vcpkg binaries if we're working with an explicit - # vcpkg directory (possibly a git checkout) - if args.vcpkg_root is None: - downloadVcpkg = False - if args.force_bootstrap: - print("Forcing bootstrap") - downloadVcpkg = True - - if not downloadVcpkg and not os.path.isfile(self.exe): - print("Missing executable, boostrapping") - downloadVcpkg = True - - # Make sure we have a vcpkg executable - testFile = os.path.join(self.path, '.vcpkg-root') - if not downloadVcpkg and not os.path.isfile(testFile): - print("Missing {}, bootstrapping".format(testFile)) - downloadVcpkg = True - - if downloadVcpkg: - print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) - downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) - - print("Replacing port files") - portsPath = os.path.join(self.path, 'ports') - if (os.path.islink(portsPath)): - os.unlink(portsPath) - if (os.path.isdir(portsPath)): - shutil.rmtree(portsPath, ignore_errors=True) - shutil.copytree(self.sourcePortsPath, portsPath) - - def downloadAndroidDependencies(self): - url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" - hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" - dest = os.path.join(self.path, 'installed') - downloadAndExtract(url, dest, hash) - - def run(self, commands): - actualCommands = [self.exe, '--vcpkg-root', self.path] - actualCommands.extend(commands) - print("Running command") - print(actualCommands) - executeSubprocess(actualCommands, folder=self.path) - - def buildDependencies(self): - global args - print("Installing host tools") - self.run(['install', '--triplet', self.hostTriplet, 'hifi-host-tools']) - # Special case for android, grab a bunch of binaries - if args.android: - self.downloadAndroidDependencies() - return - - print("Installing build dependencies") - self.run(['install', '--triplet', self.triplet, 'hifi-client-deps']) - # Remove temporary build artifacts - builddir = os.path.join(self.path, 'buildtrees') - if os.path.isdir(builddir): - print("Wiping build trees") - shutil.rmtree(builddir, ignore_errors=True) - - def writeConfig(self): - global args - configFilePath = os.path.join(args.build_root, 'vcpkg.cmake') - print("Writing cmake config to {}".format(configFilePath)) - # Write out the configuration for use by CMake - cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') - installPath = os.path.join(self.path, 'installed', self.triplet) - toolsPath = os.path.join(self.path, 'installed', self.hostTriplet, 'tools') - cmakeTemplate = 'set(CMAKE_TOOLCHAIN_FILE "{}" CACHE FILEPATH "Toolchain file")\n' - cmakeTemplate += 'set(VCPKG_INSTALL_ROOT "{}" CACHE FILEPATH "vcpkg installed packages path")\n' - cmakeTemplate += 'set(VCPKG_TOOLS_DIR "{}" CACHE FILEPATH "vcpkg installed packages path")\n' - cmakeConfig = cmakeTemplate.format(cmakeScript, installPath, toolsPath).replace('\\', '/') - with open(configFilePath, 'w') as f: - f.write(cmakeConfig) - - def writeTag(self): - print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) - with open(self.tagFile, 'w') as f: - f.write(self.tagContents) +def parse_args(): + # our custom ports, relative to the script location + defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports') + from argparse import ArgumentParser + parser = ArgumentParser(description='Prepare build dependencies.') + parser.add_argument('--android', action='store_true') + #parser.add_argument('--android', type=str) + parser.add_argument('--debug', action='store_true') + parser.add_argument('--force-bootstrap', action='store_true') + parser.add_argument('--force-build', action='store_true') + parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') + parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') + parser.add_argument('--ports-path', type=str, default=defaultPortsPath) + if True: + args = parser.parse_args() + else: + args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/hifi/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a']) + return args def main(): - vcpkg = VcpkgRepo() - vcpkg.bootstrap() - vcpkg.buildDependencies() - vcpkg.writeConfig() - vcpkg.writeTag() + # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build + # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for + # building OpenSSL + removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE'] + for var in removeEnvVars: + if var in os.environ: + del os.environ[var] + args = parse_args() + # Only allow one instance of the program to run at a time + pm = hifi_vcpkg.VcpkgRepo(args) + with hifi_singleton.Singleton(pm.lockFile) as lock: + if not pm.upToDate(): + pm.bootstrap() + # Always write the tag, even if we changed nothing. This + # allows vcpkg to reclaim disk space by identifying directories with + # tags that haven't been touched in a long time + pm.writeTag() -from argparse import ArgumentParser -parser = ArgumentParser(description='Prepare build dependencies.') -parser.add_argument('--android', action='store_true') -parser.add_argument('--debug', action='store_true') -parser.add_argument('--force-bootstrap', action='store_true') -parser.add_argument('--force-build', action='store_true') -parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') -parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') + # Grab our required dependencies: + # * build host tools, like spirv-cross and scribe + # * build client dependencies like openssl and nvtt + pm.setupDependencies() -args = parser.parse_args() + # wipe out the build directories (after writing the tag, since failure + # here shouldn't invalidte the vcpkg install) + pm.cleanBuilds() + + # Write the vcpkg config to the build directory last + pm.writeConfig() + +print(sys.argv) main() - diff --git a/scripts/system/commerce/wallet.js b/scripts/system/commerce/wallet.js index 3a8462c5cb..5c312b0ec5 100644 --- a/scripts/system/commerce/wallet.js +++ b/scripts/system/commerce/wallet.js @@ -536,6 +536,10 @@ function fromQml(message) { shouldShowDotHistory = false; ui.messagesWaiting(shouldShowDotUpdates || shouldShowDotHistory); break; + case 'clearShouldShowDotUpdates': + shouldShowDotUpdates = false; + ui.messagesWaiting(shouldShowDotUpdates || shouldShowDotHistory); + break; case 'http.request': // Handled elsewhere, don't log. break; @@ -578,7 +582,7 @@ function notificationDataProcessPageHistory(data) { var shouldShowDotUpdates = false; function notificationPollCallbackUpdates(updatesArray) { - shouldShowDotUpdates = shouldShowDotUpdates || updatesArray.length > 0; + shouldShowDotUpdates = updatesArray.length > 0; ui.messagesWaiting(shouldShowDotUpdates || shouldShowDotHistory); if (updatesArray.length > 0) { diff --git a/scripts/system/controllers/controllerModules/teleport.js b/scripts/system/controllers/controllerModules/teleport.js index 44aa04b497..8770ae8dde 100644 --- a/scripts/system/controllers/controllerModules/teleport.js +++ b/scripts/system/controllers/controllerModules/teleport.js @@ -701,7 +701,7 @@ Script.include("/~/system/libraries/controllers.js"); }; this.isReady = function(controllerData, deltaTime) { - if (Window.interstitialModeEnabled && !Window.isPhysicsEnabled()) { + if ((Window.interstitialModeEnabled && !Window.isPhysicsEnabled()) || !MyAvatar.allowTeleporting) { return makeRunningValues(false, [], []); } diff --git a/scripts/system/makeUserConnection.js b/scripts/system/makeUserConnection.js index 4b59f050c9..37d7d898bc 100644 --- a/scripts/system/makeUserConnection.js +++ b/scripts/system/makeUserConnection.js @@ -51,6 +51,7 @@ "emitterShouldTrail": 1, "isEmitting": 1, "lifespan": 3, + "lifetime": 5, "maxParticles": 1000, "particleRadius": 0.003, "polarStart": Math.PI / 2, @@ -82,6 +83,7 @@ "emitterShouldTrail": 1, "isEmitting": 1, "lifespan": 3.6, + "lifetime": 5, "maxParticles": 4000, "particleRadius": 0.048, "polarStart": 0, @@ -287,6 +289,11 @@ } function updateMakingConnection() { + if (!makingConnectionParticleEffect) { + particleEffectUpdateTimer = null; + return; + } + makingConnectionEmitRate = Math.max(makingConnectionEmitRate * MAKING_CONNECTION_DECAY_RATE, MAKING_CONNECTION_MINIMUM_EMIT_RATE); isMakingConnectionEmitting = true; @@ -302,6 +309,11 @@ } function updateParticleEffect() { + if (!particleEffect) { + particleEffectUpdateTimer = null; + return; + } + particleEmitRate = Math.max(PARTICLE_MINIMUM_EMIT_RATE, particleEmitRate * PARTICLE_DECAY_RATE); Entities.editEntity(particleEffect, { emitRate: particleEmitRate diff --git a/tools/nitpick/README.md b/tools/nitpick/README.md index 7d75d660d7..23105a0e02 100644 --- a/tools/nitpick/README.md +++ b/tools/nitpick/README.md @@ -5,7 +5,7 @@ Nitpick is a stand alone application that provides a mechanism for regression te * The snapshots are compared to a 'canonical' set of images that have been produced beforehand. * The result, if any test failed, is a zipped folder describing the failure. -Nitpick has 5 functions, separated into 4 tabs: +Nitpick has 5 functions, separated into separate tabs: 1. Creating tests, MD files and recursive scripts 1. Windows task bar utility (Windows only) 1. Running tests @@ -22,9 +22,9 @@ Nitpick is built as part of the High Fidelity build. 1. Select all, right-click and select 7-Zip->Add to archive... 1. Set Archive format to 7z 1. Check "Create SFX archive -1. Enter installer name (i.e. `nitpick-installer-v1.1.exe`) +1. Enter installer name (i.e. `nitpick-installer-v1.2.exe`) 1. Click "OK" -1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe: aws s3 cp nitpick-installer-v1.1.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.exe +1. Copy created installer to https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe: aws s3 cp nitpick-installer-v1.2.exe s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.exe #### Mac These steps assume the hifi repository has been cloned to `~/hifi`. 1. (first time) Install brew @@ -37,12 +37,12 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Change the loader instruction to find the dynamic library locally In a terminal: `install_name_tool -change ~/hifi/build/ext/Xcode/quazip/project/lib/libquazip5.1.dylib libquazip5.1.dylib nitpick` 1. Delete any existing disk images. In a terminal: `rm *.dmg` -1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.1 nitpick-installer-v1.1.dmg .` +1. Create installer (note final period).In a terminal: `create-dmg --volname nitpick-installer-v1.2 nitpick-installer-v1.2.dmg .` Make sure to wait for completion. -1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.1.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.1.dmg` +1. Copy created installer to AWS: `~/Library/Python/3.7/bin/aws s3 cp nitpick-installer-v1.2.dmg s3://hifi-qa/nitpick/Mac/nitpick-installer-v1.2.dmg` ### Installation #### Windows -1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.1.exe) +1. (First time) download and install vc_redist.x64.exe (available at https://hifi-qa.s3.amazonaws.com/nitpick/Windows/nitpick-installer-v1.2.exe) 1. (First time) download and install Python 3 from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/python-3.7.0-amd64.exe (also located at https://www.python.org/downloads/) 1. After installation - create an environment variable called PYTHON_PATH and set it to the folder containing the Python executable. 1. (First time) download and install AWS CLI from https://hifi-qa.s3.amazonaws.com/nitpick/Windows/AWSCLI64PY3.msi (also available at https://aws.amazon.com/cli/ @@ -52,7 +52,7 @@ These steps assume the hifi repository has been cloned to `~/hifi`. 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: `pip install boto3` -1. Download the installer by browsing to [here]() +1. Download the installer by browsing to [here]() 1. Double click on the installer and install to a convenient location ![](./setup_7z.PNG) @@ -76,14 +76,14 @@ In a terminal: `python3 get-pip.py --user` 1. Enter the secret key 1. Leave region name and ouput format as default [None] 1. Install the latest release of Boto3 via pip: pip3 install boto3 -1. Download the installer by browsing to [here](). +1. Download the installer by browsing to [here](). 1. Double-click on the downloaded image to mount it 1. Create a folder for the nitpick files (e.g. ~/nitpick) If this folder exists then delete all it's contents. 1. Copy the downloaded files to the folder In a terminal: `cd ~/nitpick` - `cp -r /Volumes/nitpick-installer-v1.1/* .` + `cp -r /Volumes/nitpick-installer-v1.2/* .` 1. __To run nitpick, cd to the folder that you copied to and run `./nitpick`__ # Usage diff --git a/tools/nitpick/src/AWSInterface.cpp b/tools/nitpick/src/AWSInterface.cpp index e43ef8dc75..0b93ce44e5 100644 --- a/tools/nitpick/src/AWSInterface.cpp +++ b/tools/nitpick/src/AWSInterface.cpp @@ -10,6 +10,8 @@ #include "AWSInterface.h" #include +#include +#include #include #include @@ -22,12 +24,12 @@ AWSInterface::AWSInterface(QObject* parent) : QObject(parent) { } void AWSInterface::createWebPageFromResults(const QString& testResults, - const QString& snapshotDirectory, + const QString& workingDirectory, QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) { _testResults = testResults; - _snapshotDirectory = snapshotDirectory; - + _workingDirectory = workingDirectory; + _urlLineEdit = urlLineEdit; _urlLineEdit->setEnabled(false); @@ -36,6 +38,9 @@ void AWSInterface::createWebPageFromResults(const QString& testResults, if (updateAWSCheckBox->isChecked()) { updateAWS(); + QMessageBox::information(0, "Success", "HTML file has been created and copied to AWS"); + } else { + QMessageBox::information(0, "Success", "HTML file has been created"); } } @@ -43,14 +48,14 @@ void AWSInterface::extractTestFailuresFromZippedFolder() { // For a test results zip file called `D:/tt/TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ].zip` // the folder will be called `TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]` // and, this folder will be in the working directory - QStringList parts =_testResults.split('/'); - QString zipFolderName = _snapshotDirectory + "/" + parts[parts.length() - 1].split('.')[0]; + QStringList parts = _testResults.split('/'); + QString zipFolderName = _workingDirectory + "/" + parts[parts.length() - 1].split('.')[0]; if (QDir(zipFolderName).exists()) { QDir dir = zipFolderName; dir.removeRecursively(); } - JlCompress::extractDir(_testResults, _snapshotDirectory); + JlCompress::extractDir(_testResults, _workingDirectory); } void AWSInterface::createHTMLFile() { @@ -60,7 +65,7 @@ void AWSInterface::createHTMLFile() { QString filename = pathComponents[pathComponents.length() - 1]; _resultsFolder = filename.left(filename.length() - 4); - QString resultsPath = _snapshotDirectory + "/" + _resultsFolder + "/"; + QString resultsPath = _workingDirectory + "/" + _resultsFolder + "/"; QDir().mkdir(resultsPath); _htmlFilename = resultsPath + HTML_FILENAME; @@ -98,65 +103,11 @@ void AWSInterface::writeHead(QTextStream& stream) { void AWSInterface::writeBody(QTextStream& stream) { stream << "\t" << "\n"; - writeTitle(stream); - writeTable(stream); - stream << "\t" << "\n"; -} -void AWSInterface::finishHTMLpage(QTextStream& stream) { - stream << "\n"; -} - -void AWSInterface::writeTitle(QTextStream& stream) { - // Separate relevant components from the results name - // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` - QStringList tokens = _testResults.split('/'); - - // date_buildorPR_hostName will be 2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ] - QString date_buildorPR_hostName = tokens[tokens.length() - 1].split("--")[1].split(".")[0]; - - QString buildorPR = date_buildorPR_hostName.split('(')[1].split(')')[0]; - QString hostName = date_buildorPR_hostName.split('[')[1].split(']')[0]; - - QStringList dateList = date_buildorPR_hostName.split('(')[0].split('_')[0].split('-'); - QString year = dateList[0]; - QString month = dateList[1]; - QString day = dateList[2]; - - QStringList timeList = date_buildorPR_hostName.split('(')[0].split('_')[1].split('-'); - QString hour = timeList[0]; - QString minute = timeList[1]; - QString second = timeList[2]; - - const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - - stream << "\t" << "\t" << "\n"; - stream << "\t" << "\t" << "

Failures for "; - stream << months[month.toInt() - 1] << " " << day << ", " << year << ", "; - stream << hour << ":" << minute << ":" << second << ", "; - - if (buildorPR.left(2) == "PR") { - stream << "PR " << buildorPR.right(buildorPR.length() - 2) << ", "; - } else { - stream << "build " << buildorPR << ", "; - } - - stream << "run on " << hostName << "

\n"; -} - -void AWSInterface::writeTable(QTextStream& stream) { - QString previousTestName{ "" }; - - // Loop over all entries in directory. This is done in stages, as the names are not in the order of the tests - // The first stage reads the directory names into a list - // The second stage renames the tests by removing everything up to "--tests." - // The third stage renames the directories - // The fourth and lasts stage creates the HTML entries - // - // Note that failures are processed first, then successes + // The results are read here as they are used both in the title (for the summary) and for table QStringList originalNamesFailures; QStringList originalNamesSuccesses; - QDirIterator it1(_snapshotDirectory.toStdString().c_str()); + QDirIterator it1(_workingDirectory); while (it1.hasNext()) { QString nextDirectory = it1.next(); @@ -165,7 +116,7 @@ void AWSInterface::writeTable(QTextStream& stream) { continue; } - // Only process failure folders + // Only process result folders if (!nextDirectory.contains("--tests.")) { continue; } @@ -180,6 +131,71 @@ void AWSInterface::writeTable(QTextStream& stream) { } } + writeTitle(stream, originalNamesFailures, originalNamesSuccesses); + writeTable(stream, originalNamesFailures, originalNamesSuccesses); + stream << "\t" << "\n"; +} + +void AWSInterface::finishHTMLpage(QTextStream& stream) { + stream << "\n"; +} + +void AWSInterface::writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { + // Separate relevant components from the results name + // The expected format is as follows: `D:/tt/snapshots/TestResults--2018-10-04_11-09-41(PR14128)[DESKTOP-PMKNLSQ].zip` + QStringList tokens = _testResults.split('/'); + + // date_buildorPR_hostName will be 2018-10-03_15-35-28(9433)[DESKTOP-PMKNLSQ] + QString date_buildorPR_hostName = tokens[tokens.length() - 1].split("--")[1].split(".")[0]; + + QString buildorPR = date_buildorPR_hostName.split('(')[1].split(')')[0]; + QString hostName = date_buildorPR_hostName.split('[')[1].split(']')[0]; + + QStringList dateList = date_buildorPR_hostName.split('(')[0].split('_')[0].split('-'); + QString year = dateList[0]; + QString month = dateList[1]; + QString day = dateList[2]; + + QStringList timeList = date_buildorPR_hostName.split('(')[0].split('_')[1].split('-'); + QString hour = timeList[0]; + QString minute = timeList[1]; + QString second = timeList[2]; + + const QString months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

Results for "; + stream << months[month.toInt() - 1] << " " << day << ", " << year << ", "; + stream << hour << ":" << minute << ":" << second << ", "; + + if (buildorPR.left(2) == "PR") { + stream << "PR " << buildorPR.right(buildorPR.length() - 2) << ", "; + } else { + stream << "build " << buildorPR << ", "; + } + + stream << "run on " << hostName << "

\n"; + + int numberOfFailures = originalNamesFailures.length(); + int numberOfSuccesses = originalNamesSuccesses.length(); + + stream << "

" << QString::number(numberOfFailures) << " failed, out of a total of " << QString::number(numberOfSuccesses) << " tests

\n"; + + stream << "\t" << "\t" << "\n"; + stream << "\t" << "\t" << "

The following tests failed:

"; +} + +void AWSInterface::writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses) { + QString previousTestName{ "" }; + + // Loop over all entries in directory. This is done in stages, as the names are not in the order of the tests + // The first stage reads the directory names into a list + // The second stage renames the tests by removing everything up to "--tests." + // The third stage renames the directories + // The fourth and lasts stage creates the HTML entries + // + // Note that failures are processed first, then successes + QStringList newNamesFailures; for (int i = 0; i < originalNamesFailures.length(); ++i) { newNamesFailures.append(originalNamesFailures[i].split("--tests.")[1]); @@ -189,11 +205,11 @@ void AWSInterface::writeTable(QTextStream& stream) { for (int i = 0; i < originalNamesSuccesses.length(); ++i) { newNamesSuccesses.append(originalNamesSuccesses[i].split("--tests.")[1]); } - - _htmlFailuresFolder = _snapshotDirectory + "/" + _resultsFolder + "/" + FAILURES_FOLDER; + + _htmlFailuresFolder = _workingDirectory + "/" + _resultsFolder + "/" + FAILURES_FOLDER; QDir().mkdir(_htmlFailuresFolder); - _htmlSuccessesFolder = _snapshotDirectory + "/" + _resultsFolder + "/" + SUCCESSES_FOLDER; + _htmlSuccessesFolder = _workingDirectory + "/" + _resultsFolder + "/" + SUCCESSES_FOLDER; QDir().mkdir(_htmlSuccessesFolder); for (int i = 0; i < newNamesFailures.length(); ++i) { @@ -204,7 +220,11 @@ void AWSInterface::writeTable(QTextStream& stream) { QDir().rename(originalNamesSuccesses[i], _htmlSuccessesFolder + "/" + newNamesSuccesses[i]); } - QDirIterator it2((_htmlFailuresFolder).toStdString().c_str()); + // Mac does not read folders in lexicographic order, so this step is divided into 2 + // Each test consists of the test name and its index. + QStringList folderNames; + + QDirIterator it2(_htmlFailuresFolder); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -214,10 +234,17 @@ void AWSInterface::writeTable(QTextStream& stream) { } QStringList pathComponents = nextDirectory.split('/'); - QString filename = pathComponents[pathComponents.length() - 1]; - int splitIndex = filename.lastIndexOf("."); - QString testName = filename.left(splitIndex).replace(".", " / "); - QString testNumber = filename.right(filename.length() - (splitIndex + 1)); + QString folderName = pathComponents[pathComponents.length() - 1]; + + folderNames << folderName; + } + + folderNames.sort(); + for (const auto& folderName : folderNames) { + int splitIndex = folderName.lastIndexOf("."); + QString testName = folderName.left(splitIndex).replace('.', " / "); + + int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt(); // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table if (testName != previousTestName) { @@ -228,18 +255,20 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, true); } - createEntry(testNumber.toInt(), filename, stream, true); + createEntry(testNumber, folderName, stream, true); } closeTable(stream); stream << "\t" << "\t" << "\n"; stream << "\t" << "\t" << "

The following tests passed:

"; - QDirIterator it3((_htmlSuccessesFolder).toStdString().c_str()); + // Now do the same for passes + folderNames.clear(); + + QDirIterator it3(_htmlSuccessesFolder); while (it3.hasNext()) { QString nextDirectory = it3.next(); @@ -249,10 +278,17 @@ void AWSInterface::writeTable(QTextStream& stream) { } QStringList pathComponents = nextDirectory.split('/'); - QString filename = pathComponents[pathComponents.length() - 1]; - int splitIndex = filename.lastIndexOf("."); - QString testName = filename.left(splitIndex).replace(".", " / "); - QString testNumber = filename.right(filename.length() - (splitIndex + 1)); + QString folderName = pathComponents[pathComponents.length() - 1]; + + folderNames << folderName; + } + + folderNames.sort(); + for (const auto& folderName : folderNames) { + int splitIndex = folderName.lastIndexOf("."); + QString testName = folderName.left(splitIndex).replace('.', " / "); + + int testNumber = folderName.right(folderName.length() - (splitIndex + 1)).toInt(); // The failures are ordered lexicographically, so we know that we can rely on the testName changing to create a new table if (testName != previousTestName) { @@ -263,64 +299,153 @@ void AWSInterface::writeTable(QTextStream& stream) { previousTestName = testName; stream << "\t\t

" << testName << "

\n"; - - openTable(stream); + openTable(stream, folderName, false); } - createEntry(testNumber.toInt(), filename, stream, false); + createEntry(testNumber, folderName, stream, false); } closeTable(stream); } -void AWSInterface::openTable(QTextStream& stream) { - stream << "\t\t\n"; - stream << "\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; - stream << "\t\t\t\n"; +void AWSInterface::openTable(QTextStream& stream, const QString& testResult, const bool isFailure) { + QStringList resultNameComponents = testResult.split('/'); + QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + + bool textResultsFileFound; + if (isFailure) { + textResultsFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Result.txt"); + } else { + textResultsFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Result.txt"); + } + + if (textResultsFileFound) { + if (isFailure) { + stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } else { + stream << "\t\t

No errors found

\n\n"; + stream << "\t\t

===============

\n\n"; + } + } else { + stream << "\t\t

Element

Actual Value

Expected Value

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + stream << "\t\t\t\n"; + } } void AWSInterface::closeTable(QTextStream& stream) { stream << "\t\t

Test

Actual Image

Expected Image

Difference Image

\n"; } -void AWSInterface::createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure) { - stream << "\t\t\t\n"; - stream << "\t\t\t\t

" << QString::number(index) << "

\n"; - +void AWSInterface::createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure) { // For a test named `D:/t/fgadhcUDHSFaidsfh3478JJJFSDFIUSOEIrf/Failure_1--tests.engine.interaction.pick.collision.many.00000` // we need `Failure_1--tests.engine.interaction.pick.collision.many.00000` QStringList resultNameComponents = testResult.split('/'); QString resultName = resultNameComponents[resultNameComponents.length() - 1]; + QString textResultFilename; + if (isFailure) { + textResultFilename = _htmlFailuresFolder + "/" + resultName + "/Result.txt"; + } else { + textResultFilename = _htmlSuccessesFolder + "/" + resultName + "/Result.txt"; + } + bool textResultsFileFound{ QFile::exists(textResultFilename) }; + QString folder; bool differenceFileFound; if (isFailure) { folder = FAILURES_FOLDER; differenceFileFound = QFile::exists(_htmlFailuresFolder + "/" + resultName + "/Difference Image.png"); } else { - folder = SUCCESSES_FOLDER; + folder = SUCCESSES_FOLDER; differenceFileFound = QFile::exists(_htmlSuccessesFolder + "/" + resultName + "/Difference Image.png"); } - - stream << "\t\t\t\t\n"; - stream << "\t\t\t\t\n"; + if (textResultsFileFound) { + // Parse the JSON file + QFile file; + file.setFileName(textResultFilename); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to open file " + textResultFilename); + } - if (differenceFileFound) { - stream << "\t\t\t\t\n"; + QString value = file.readAll(); + file.close(); + + // The Result.txt file is an object containing elements such as the following: + // "angularDamping": { + // "actual": 0.3938899040222168, + // "expected" : 0.3938899, + // "result" : "pass" + // }, + // + // Failures are thos element that have "fail for the result + + QJsonDocument document = QJsonDocument::fromJson(value.toUtf8()); + QJsonObject json = document.object(); + foreach(const QString& key, json.keys()) { + QJsonValue value = json.value(key); + QJsonObject object = value.toObject(); + + QJsonValue actualValue = object.value("actual"); + QString actualValueString; + if (actualValue.isString()) { + actualValueString = actualValue.toString(); + } else if (actualValue.isBool()) { + actualValueString = actualValue.toBool() ? "true" : "false"; + } else if (actualValue.isDouble()) { + actualValueString = QString::number(actualValue.toDouble()); + } + + QJsonValue expectedValue = object.value("expected"); + QString expectedValueString; + if (expectedValue.isString()) { + expectedValueString = expectedValue.toString(); + } else if (expectedValue.isBool()) { + expectedValueString = expectedValue.toBool() ? "true" : "false"; + } else if (expectedValue.isDouble()) { + expectedValueString = QString::number(expectedValue.toDouble()); + } + QString result = object.value("result").toString(); + + if (result == "fail") { + stream << "\t\t\t\n"; + stream << "\t\t\t\t" + key + "\n"; + stream << "\t\t\t\t" + actualValueString + "\n"; + stream << "\t\t\t\t" + expectedValueString + "\n"; + stream << "\t\t\t\n"; + } + } } else { - stream << "\t\t\t\t

No Image Found

\n"; + stream << "\t\t\t\n"; + stream << "\t\t\t\t

" << QString::number(index) << "

\n"; + + stream << "\t\t\t\t\n"; + stream << "\t\t\t\t\n"; + + if (differenceFileFound) { + stream << "\t\t\t\t\n"; + } else { + stream << "\t\t\t\t

No Image Found

\n"; + } + + stream << "\t\t\t\n"; } - stream << "\t\t\t\n"; } void AWSInterface::updateAWS() { - QString filename = _snapshotDirectory + "/updateAWS.py"; + QString filename = _workingDirectory + "/updateAWS.py"; if (QFile::exists(filename)) { QFile::remove(filename); } @@ -337,7 +462,7 @@ void AWSInterface::updateAWS() { stream << "import boto3\n"; stream << "s3 = boto3.resource('s3')\n\n"; - QDirIterator it1(_htmlFailuresFolder.toStdString().c_str()); + QDirIterator it1(_htmlFailuresFolder); while (it1.hasNext()) { QString nextDirectory = it1.next(); @@ -345,26 +470,26 @@ void AWSInterface::updateAWS() { if (nextDirectory.right(1) == ".") { continue; } - + // nextDirectory looks like `D:/t/TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` // We need to concatenate the last 3 components, to get `TestResults--2018-10-02_16-54-11(9426)[DESKTOP-PMKNLSQ]/failures/engine.render.effect.bloom.00000` QStringList parts = nextDirectory.split('/'); QString filename = parts[parts.length() - 3] + "/" + parts[parts.length() - 2] + "/" + parts[parts.length() - 1]; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Actual Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Actual Image.png" << "', Body=data)\n\n"; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Expected Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n"; if (QFile::exists(_htmlFailuresFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) { - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Difference Image.png" << "', 'rb')\n"; @@ -372,7 +497,7 @@ void AWSInterface::updateAWS() { } } - QDirIterator it2(_htmlSuccessesFolder.toStdString().c_str()); + QDirIterator it2(_htmlSuccessesFolder); while (it2.hasNext()) { QString nextDirectory = it2.next(); @@ -386,20 +511,20 @@ void AWSInterface::updateAWS() { QStringList parts = nextDirectory.split('/'); QString filename = parts[parts.length() - 3] + "/" + parts[parts.length() - 2] + "/" + parts[parts.length() - 1]; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Actual Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Actual Image.png" << "', Body=data)\n\n"; - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Expected Image.png" << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << filename << "/" << "Expected Image.png" << "', Body=data)\n\n"; if (QFile::exists(_htmlSuccessesFolder + "/" + parts[parts.length() - 1] + "/Difference Image.png")) { - stream << "data = open('" << _snapshotDirectory << "/" << filename << "/" + stream << "data = open('" << _workingDirectory << "/" << filename << "/" << "Difference Image.png" << "', 'rb')\n"; @@ -407,7 +532,7 @@ void AWSInterface::updateAWS() { } } - stream << "data = open('" << _snapshotDirectory << "/" << _resultsFolder << "/" << HTML_FILENAME << "', 'rb')\n"; + stream << "data = open('" << _workingDirectory << "/" << _resultsFolder << "/" << HTML_FILENAME << "', 'rb')\n"; stream << "s3.Bucket('hifi-content').put_object(Bucket='" << AWS_BUCKET << "', Key='" << _resultsFolder << "/" << HTML_FILENAME << "', Body=data, ContentType='text/html')\n"; @@ -426,10 +551,10 @@ void AWSInterface::updateAWS() { [=](int exitCode, QProcess::ExitStatus exitStatus) { _busyWindow.hide(); }); #ifdef Q_OS_WIN - QStringList parameters = QStringList() << filename ; + QStringList parameters = QStringList() << filename; process->start(_pythonCommand, parameters); #elif defined Q_OS_MAC - QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; + QStringList parameters = QStringList() << "-c" << _pythonCommand + " " + filename; process->start("sh", parameters); #endif -} +} \ No newline at end of file diff --git a/tools/nitpick/src/AWSInterface.h b/tools/nitpick/src/AWSInterface.h index f4084f1a14..fda250b115 100644 --- a/tools/nitpick/src/AWSInterface.h +++ b/tools/nitpick/src/AWSInterface.h @@ -26,7 +26,7 @@ public: explicit AWSInterface(QObject* parent = 0); void createWebPageFromResults(const QString& testResults, - const QString& snapshotDirectory, + const QString& workingDirectory, QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit); @@ -38,18 +38,18 @@ public: void writeBody(QTextStream& stream); void finishHTMLpage(QTextStream& stream); - void writeTitle(QTextStream& stream); - void writeTable(QTextStream& stream); - void openTable(QTextStream& stream); + void writeTitle(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void writeTable(QTextStream& stream, const QStringList& originalNamesFailures, const QStringList& originalNamesSuccesses); + void openTable(QTextStream& stream, const QString& testResult, const bool isFailure); void closeTable(QTextStream& stream); - void createEntry(int index, const QString& testResult, QTextStream& stream, const bool isFailure); + void createEntry(const int index, const QString& testResult, QTextStream& stream, const bool isFailure); void updateAWS(); private: QString _testResults; - QString _snapshotDirectory; + QString _workingDirectory; QString _resultsFolder; QString _htmlFailuresFolder; QString _htmlSuccessesFolder; diff --git a/tools/nitpick/src/Test.cpp b/tools/nitpick/src/Test.cpp index e17978d9d0..19c49eac42 100644 --- a/tools/nitpick/src/Test.cpp +++ b/tools/nitpick/src/Test.cpp @@ -105,7 +105,7 @@ int Test::compareImageLists() { ++numberOfFailures; if (!isInteractiveMode) { - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); } else { _mismatchWindow.exec(); @@ -113,7 +113,7 @@ int Test::compareImageLists() { case USER_RESPONSE_PASS: break; case USE_RESPONSE_FAIL: - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), true); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), true); break; case USER_RESPONSE_ABORT: keepOn = false; @@ -124,7 +124,7 @@ int Test::compareImageLists() { } } } else { - appendTestResultsToFile(_testResultsFolderPath, testResult, _mismatchWindow.getComparisonImage(), false); + appendTestResultsToFile(testResult, _mismatchWindow.getComparisonImage(), false); } _progressBar->setValue(i); @@ -134,12 +134,36 @@ int Test::compareImageLists() { return numberOfFailures; } -void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed) { +int Test::checkTextResults() { + // Create lists of failed and passed tests + QStringList nameFilterFailed; + nameFilterFailed << "*.failed.txt"; + QStringList testsFailed = QDir(_snapshotDirectory).entryList(nameFilterFailed, QDir::Files, QDir::Name); + + QStringList nameFilterPassed; + nameFilterPassed << "*.passed.txt"; + QStringList testsPassed = QDir(_snapshotDirectory).entryList(nameFilterPassed, QDir::Files, QDir::Name); + + // Add results to Test Results folder + foreach(QString currentFilename, testsFailed) { + appendTestResultsToFile(currentFilename, true); + } + + foreach(QString currentFilename, testsPassed) { + appendTestResultsToFile(currentFilename, false); + } + + return testsFailed.length(); +} + +void Test::appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed) { + // Critical error if Test Results folder does not exist if (!QDir().exists(_testResultsFolderPath)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Folder " + _testResultsFolderPath + " not found"); exit(-1); } + // There are separate subfolders for failures and passes QString resultFolderPath; if (hasFailed) { resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + @@ -195,6 +219,33 @@ void Test::appendTestResultsToFile(const QString& _testResultsFolderPath, TestRe comparisonImage.save(resultFolderPath + "/" + "Difference Image.png"); } +void::Test::appendTestResultsToFile(QString testResultFilename, bool hasFailed) { + // The test name includes everything until the penultimate period + QString testNameTemp = testResultFilename.left(testResultFilename.lastIndexOf('.')); + QString testName = testResultFilename.left(testNameTemp.lastIndexOf('.')); + QString resultFolderPath; + if (hasFailed) { + resultFolderPath = _testResultsFolderPath + "/Failure_" + QString::number(_failureIndex) + "--" + testName; + ++_failureIndex; + } else { + resultFolderPath = _testResultsFolderPath + "/Success_" + QString::number(_successIndex) + "--" + testName; + ++_successIndex; + } + + if (!QDir().mkdir(resultFolderPath)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), + "Failed to create folder " + resultFolderPath); + exit(-1); + } + + QString source = _snapshotDirectory + "/" + testResultFilename; + QString destination = resultFolderPath + "/Result.txt"; + if (!QFile::copy(source, destination)) { + QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), "Failed to copy " + testResultFilename + " to " + resultFolderPath); + exit(-1); + } +} + void Test::startTestsEvaluation(const bool isRunningFromCommandLine, const bool isRunningInAutomaticTestRun, const QString& snapshotDirectory, @@ -211,7 +262,7 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, if (!parent.isNull() && parent.right(1) != "/") { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return @@ -270,9 +321,14 @@ void Test::startTestsEvaluation(const bool isRunningFromCommandLine, nitpick->downloadFiles(expectedImagesURLs, _snapshotDirectory, _expectedImagesFilenames, (void *)this); } + void Test::finishTestsEvaluation() { + // First - compare the pairs of images int numberOfFailures = compareImageLists(); - + + // Next - check text results + numberOfFailures += checkTextResults(); + if (!_isRunningFromCommandLine && !_isRunningInAutomaticTestRun) { if (numberOfFailures == 0) { QMessageBox::information(0, "Success", "All images are as expected"); @@ -344,7 +400,7 @@ void Test::createTests() { parent += "/"; } - _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the test images", parent, + _snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select folder containing the snapshots", parent, QFileDialog::ShowDirsOnly); // If user canceled then restore previous selection and return @@ -542,7 +598,7 @@ void Test::createAllMDFiles() { createMDFile(_testsRootDirectory); } - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -636,7 +692,7 @@ void Test::createAllTestAutoScripts() { createTestAutoScript(_testsRootDirectory); } - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -653,7 +709,7 @@ void Test::createAllTestAutoScripts() { } } - QMessageBox::information(0, "Success", "'nitpick.js' scripts have been created"); + QMessageBox::information(0, "Success", "All 'testAuto.js' scripts have been created"); } bool Test::createTestAutoScript(const QString& directory) { @@ -704,7 +760,7 @@ void Test::createAllRecursiveScripts() { createRecursiveScript(_testsRootDirectory, false); - QDirIterator it(_testsRootDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testsRootDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -716,7 +772,7 @@ void Test::createAllRecursiveScripts() { // Only process directories that have sub-directories bool hasNoSubDirectories{ true }; - QDirIterator it2(directory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it2(directory, QDirIterator::Subdirectories); while (it2.hasNext()) { QString directory2 = it2.next(); @@ -737,16 +793,17 @@ void Test::createAllRecursiveScripts() { } void Test::createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode) { - const QString recursiveTestsFilename("testRecursive.js"); - QFile allTestsFilename(topLevelDirectory + "/" + recursiveTestsFilename); - if (!allTestsFilename.open(QIODevice::WriteOnly | QIODevice::Text)) { + const QString recursiveTestsScriptName("testRecursive.js"); + const QString recursiveTestsFilename(topLevelDirectory + "/" + recursiveTestsScriptName); + QFile recursiveTestsFile(recursiveTestsFilename); + if (!recursiveTestsFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QMessageBox::critical(0, "Internal error: " + QString(__FILE__) + ":" + QString::number(__LINE__), - "Failed to create \"" + recursiveTestsFilename + "\" in directory \"" + topLevelDirectory + "\""); + "Failed to create \"" + recursiveTestsScriptName + "\" in directory \"" + topLevelDirectory + "\""); exit(-1); } - QTextStream textStream(&allTestsFilename); + QTextStream textStream(&recursiveTestsFile); textStream << "// This is an automatically generated file, created by nitpick" << endl; @@ -787,7 +844,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact testFound = true; } - QDirIterator it(topLevelDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(topLevelDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -809,7 +866,15 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact if (interactiveMode && !testFound) { QMessageBox::information(0, "Failure", "No \"" + TEST_FILENAME + "\" files found"); - allTestsFilename.close(); + recursiveTestsFile.close(); + return; + } + + // If 'directories' is empty, this means that this recursive script has no tests to call, so it is redundant + // The script will be closed and deleted + if (directories.length() == 0) { + recursiveTestsFile.close(); + QFile::remove(recursiveTestsFilename); return; } @@ -821,7 +886,7 @@ void Test::createRecursiveScript(const QString& topLevelDirectory, bool interact textStream << endl; textStream << "nitpick.runRecursive();" << endl; - allTestsFilename.close(); + recursiveTestsFile.close(); } void Test::createTestsOutline() { @@ -858,7 +923,7 @@ void Test::createTestsOutline() { int rootDepth { _testDirectory.count('/') }; // Each test is shown as the folder name linking to the matching GitHub URL, and the path to the associated test.md file - QDirIterator it(_testDirectory.toStdString().c_str(), QDirIterator::Subdirectories); + QDirIterator it(_testDirectory, QDirIterator::Subdirectories); while (it.hasNext()) { QString directory = it.next(); @@ -1052,11 +1117,11 @@ void Test::createWebPage(QCheckBox* updateAWSCheckBox, QLineEdit* urlLineEdit) { return; } - QString snapshotDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", + QString workingDirectory = QFileDialog::getExistingDirectory(nullptr, "Please select a folder to store temporary files in", nullptr, QFileDialog::ShowDirsOnly); - if (snapshotDirectory.isNull()) { + if (workingDirectory.isNull()) { return; } - _awsInterface.createWebPageFromResults(testResults, snapshotDirectory, updateAWSCheckBox, urlLineEdit); + _awsInterface.createWebPageFromResults(testResults, workingDirectory, updateAWSCheckBox, urlLineEdit); } \ No newline at end of file diff --git a/tools/nitpick/src/Test.h b/tools/nitpick/src/Test.h index a79252b92a..9ef7c5627a 100644 --- a/tools/nitpick/src/Test.h +++ b/tools/nitpick/src/Test.h @@ -77,6 +77,7 @@ public: void createRecursiveScript(const QString& topLevelDirectory, bool interactiveMode); int compareImageLists(); + int checkTextResults(); QStringList createListOfAll_imagesInDirectory(const QString& imageFormat, const QString& pathToImageDirectory); @@ -84,7 +85,8 @@ public: void includeTest(QTextStream& textStream, const QString& testPathname); - void appendTestResultsToFile(const QString& testResultsFolderPath, TestResult testResult, QPixmap comparisonImage, bool hasFailed); + void appendTestResultsToFile(TestResult testResult, QPixmap comparisonImage, bool hasFailed); + void appendTestResultsToFile(QString testResultFilename, bool hasFailed); bool createTestResultsFolderPath(const QString& directory); QString zipAndDeleteTestResultsFolder(); diff --git a/tools/nitpick/src/TestRailInterface.cpp b/tools/nitpick/src/TestRailInterface.cpp index a0c0d74526..1d7aa0a32f 100644 --- a/tools/nitpick/src/TestRailInterface.cpp +++ b/tools/nitpick/src/TestRailInterface.cpp @@ -275,7 +275,7 @@ void TestRailInterface::processDirectoryPython(const QString& directory, const QString& userGitHub, const QString& branchGitHub) { // Loop over all entries in directory - QDirIterator it(directory.toStdString().c_str()); + QDirIterator it(directory); while (it.hasNext()) { QString nextDirectory = it.next(); @@ -855,7 +855,7 @@ QDomElement TestRailInterface::processDirectoryXML(const QString& directory, QDomElement result = element; // Loop over all entries in directory - QDirIterator it(directory.toStdString().c_str()); + QDirIterator it(directory); while (it.hasNext()) { QString nextDirectory = it.next(); diff --git a/tools/nitpick/src/TestRunner.cpp b/tools/nitpick/src/TestRunner.cpp index 12bdf87495..9b99e114a7 100644 --- a/tools/nitpick/src/TestRunner.cpp +++ b/tools/nitpick/src/TestRunner.cpp @@ -366,7 +366,7 @@ void TestRunner::createSnapshotFolder() { // Note that we cannot use just a `png` filter, as the filenames include periods // Also, delete any `jpg` and `txt` files // The idea is to leave only previous zipped result folders - QDirIterator it(_snapshotFolder.toStdString().c_str()); + QDirIterator it(_snapshotFolder); while (it.hasNext()) { QString filename = it.next(); if (filename.right(4) == ".png" || filename.right(4) == ".jpg" || filename.right(4) == ".txt") { diff --git a/tools/nitpick/src/ui/Nitpick.cpp b/tools/nitpick/src/ui/Nitpick.cpp index 201d6e562d..cdd2ff89d9 100644 --- a/tools/nitpick/src/ui/Nitpick.cpp +++ b/tools/nitpick/src/ui/Nitpick.cpp @@ -36,7 +36,7 @@ Nitpick::Nitpick(QWidget* parent) : QMainWindow(parent) { _ui.statusLabel->setText(""); _ui.plainTextEdit->setReadOnly(true); - setWindowTitle("Nitpick - v1.1"); + setWindowTitle("Nitpick - v1.2"); // Coming soon to a nitpick near you... //// _helpWindow.textBrowser->setText()