From ddf1543d689f0e021b1961e38c80a35063bdceea Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 24 May 2019 09:21:30 -0700 Subject: [PATCH] Update mac qt build * Reformat whitespace in README.md * hifi_vckpg.py: Remove macos var/temp hack for vcpkg * hifi_vckpg.py: Add ability to override HIFI_QT_INSTALL_ROOT dir with environment variable. * hifi_vckpg.py: Update macos url for pre-built qt 5.12.3 on S3. * cmake/ports/bullet3/portfile.cmake: add comment to force change in vcpkg hash. * tools/qt-builder/README.md: updated formatting and added details about how to generate symbols for macos and pc * tools/qt-builder: added prepare python scripts for symbols * cmake/macros/SetupQt.cmake: Remove hardcoded var/temp qt cmake path --- cmake/macros/SetupQt.cmake | 5 +- cmake/ports/bullet3/portfile.cmake | 2 + hifi_vcpkg.py | 11 +- tools/qt-builder/README.md | 382 +++++++++--------- .../prepare-mac-symbols-for-backtrace.py | 193 +++++++++ .../prepare-windows-symbols-for-backtrace.py | 132 ++++++ 6 files changed, 528 insertions(+), 197 deletions(-) create mode 100644 tools/qt-builder/prepare-mac-symbols-for-backtrace.py create mode 100644 tools/qt-builder/prepare-windows-symbols-for-backtrace.py diff --git a/cmake/macros/SetupQt.cmake b/cmake/macros/SetupQt.cmake index b9ee425169..82e59cfb71 100644 --- a/cmake/macros/SetupQt.cmake +++ b/cmake/macros/SetupQt.cmake @@ -52,10 +52,7 @@ macro(setup_qt) message(FATAL_ERROR "VCPKG_QT_CMAKE_PREFIX_PATH should have been set by hifi_vcpkg.py") endif() if (NOT DEV_BUILD) - if (APPLE) - # HACK: manually set the QT_CMAKE_PREFIX_PATH so that hard-coded paths find new QT libs where we'll put them - set(QT_CMAKE_PREFIX_PATH "/var/tmp/qt5-install/lib/cmake") - elseif (UNIX AND DEFINED ENV{QT_CMAKE_PREFIX_PATH}) + if (UNIX AND DEFINED ENV{QT_CMAKE_PREFIX_PATH}) # HACK: obey QT_CMAKE_PREFIX_PATH to allow UNIX to use older QT libs set(QT_CMAKE_PREFIX_PATH $ENV{QT_CMAKE_PREFIX_PATH}) else() diff --git a/cmake/ports/bullet3/portfile.cmake b/cmake/ports/bullet3/portfile.cmake index 9318385de0..1faf7d983b 100644 --- a/cmake/ports/bullet3/portfile.cmake +++ b/cmake/ports/bullet3/portfile.cmake @@ -9,6 +9,8 @@ # VCPKG_ROOT_DIR = # VCPKG_TARGET_ARCHITECTURE = target architecture (x64, x86, arm) # +# + include(vcpkg_common_functions) if (VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py index cdd44d304e..56e904b795 100644 --- a/hifi_vcpkg.py +++ b/hifi_vcpkg.py @@ -214,11 +214,10 @@ endif() f.write(self.tagContents) def getQt5InstallPath(self): - qt5InstallPath = os.path.join(self.path, 'installed', 'qt5-install') - if platform.system() == "Darwin" and self.args.release_type != "DEV": - # HACK for MacOS Jenkins PRODUCTION and PR builds during Qt-5.12.3 transition - # we always supply /var/tmp/qt5-install for QT_CMAKE_PREFIX_PATH - qt5InstallPath = "/var/tmp/qt5-install" + qt5InstallPath = os.path.join(self.path, 'installed', 'qt5-install') + if os.getenv('HIFI_QT_INSTALL_ROOT'): + # Allow qt install path to be overriden by an environment variable + qt5InstallPath = os.getenv('HIFI_QT_INSTALL_ROOT') elif self.args.android: precompiled = os.path.realpath(self.androidPackagePath) qt5InstallPath = os.path.realpath(os.path.join(precompiled, 'qt')) @@ -260,7 +259,7 @@ endif() if platform.system() == 'Windows': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-windows2.tar.gz' elif platform.system() == 'Darwin': - url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos.tar.gz' + url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos2.tar.gz' elif platform.system() == 'Linux': if platform.linux_distribution()[1][:3] == '16.': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-16.04.tar.gz' diff --git a/tools/qt-builder/README.md b/tools/qt-builder/README.md index b7861a5e53..5b4084e61e 100644 --- a/tools/qt-builder/README.md +++ b/tools/qt-builder/README.md @@ -1,235 +1,243 @@ # General This document describes the process to build Qt 5.12.3. -Note that there are two patches. The first (to qfloat16.h) is needed to compile QT 5.12.3 on Visual Studio 2017 due to a bug in Visual Studio (*bitset* will not compile. Note that there is a change in CMakeLists.txt to support this. +Note that there are two patches. The first (to qfloat16.h) is needed to compile QT 5.12.3 on Visual Studio 2017 due to a bug in Visual Studio (*bitset* will not compile. Note that there is a change in CMakeLists.txt to support this. The second patch is to OpenSL ES audio. ## Requirements ### Windows -1. Visual Studio 2017 - If you don’t have Community or Professional edition of Visual Studio 2017, download [Visual Studio Community 2017](https://www.visualstudio.com/downloads/). -Install with defaults +1. Visual Studio 2017 + If you don’t have Community or Professional edition of Visual Studio 2017, download [Visual Studio Community 2017](https://www.visualstudio.com/downloads/). +Install with C++ support. 1. python 2.7.16 -Check if needed running `python --version` - should return python 2.7.x -Install from https://www.python.org/ftp/python/2.7.16/python-2.7.16.amd64.msi +Check if needed running `python --version` - should return python 2.7.x +Install from https://www.python.org/ftp/python/2.7.16/python-2.7.16.amd64.msi Add path to python executable to PATH. NOTE: REMOVE python 3 from PATH. Our regular build uses python 3. This will still work, because HIFI_PYTHON_EXEC points to the python 3 executable. -Verify that python runs python 2.7 (run “python --version”) -1. git >= 1.6 -Check if needed `git --version` -Download from https://git-scm.com/download/win -Verify by entering `git --version` -1. perl >= 5.14 -Install from Strawberry Perl - http://strawberryperl.com/ - 5.28.1.1 64 bit to C:\Strawberry\ -Verify by running `perl --version` -1. flex and bison -Download from https://sourceforge.net/projects/winflexbison/files/latest/download -Uncompress in C:\flex_bison -Rename win-bison.exe to bison.exe and win-flex.exe to flex.exe -Add C:\flex_bison to PATH -Verify by running `flex --version` -Verify by running `bison --version` -1. gperf -Install from http://gnuwin32.sourceforge.net/downlinks/gperf.php -Add C:\Program Files (x86)\GnuWin32\bin to PATH -Verify by running `gperf --version` -1. 7-zip -Download from https://www.7-zip.org/download.html -1. Bash shell -From *Settings* select *Update & Security* -Select *For Developers* -Enable *Developer mode* -Restart PC -Open Control Panel and select *Programs and Features* -Select *Turn Windows features on or off* -Check *Windows Subsystem for Linux* -Click *Restart now* -Download from the Microsoft Store - Search for *bash* and choose the latest Ubuntu version -[First run will take a few minutes] -Enter a user name - all small letters (this is used for *sudo* commands) +Verify that python runs python 2.7 (run “python --version”) +1. git >= 1.6 +Check if needed `git --version` +Download from https://git-scm.com/download/win +Verify by entering `git --version` +1. perl >= 5.14 +Install from Strawberry Perl - http://strawberryperl.com/ - 5.28.1.1 64 bit to C:\Strawberry\ +Verify by running `perl --version` +1. flex and bison +Download from https://sourceforge.net/projects/winflexbison/files/latest/download +Uncompress in C:\flex_bison +Rename win-bison.exe to bison.exe and win-flex.exe to flex.exe +Add C:\flex_bison to PATH +Verify by running `flex --version` +Verify by running `bison --version` +1. gperf +Install from http://gnuwin32.sourceforge.net/downlinks/gperf.php +Add C:\Program Files (x86)\GnuWin32\bin to PATH +Verify by running `gperf --version` +1. 7-zip +Download from https://www.7-zip.org/download.html +1. Bash shell +From *Settings* select *Update & Security* +Select *For Developers* +Enable *Developer mode* +Restart PC +Open Control Panel and select *Programs and Features* +Select *Turn Windows features on or off* +Check *Windows Subsystem for Linux* +Click *Restart now* +Download from the Microsoft Store - Search for *bash* and choose the latest Ubuntu version +[First run will take a few minutes] +Enter a user name - all small letters (this is used for *sudo* commands) Choose a password +1. Jom +jom is a clone of nmake to support the execution of multiple independent commands in parallel. +https://wiki.qt.io/Jom ### Linux -Tested on Ubuntu 16.04 and 18.04. +Tested on Ubuntu 16.04 and 18.04. **16.04 NEEDED FOR JENKINS~~ ** -1. qt5 requirements -edit /etc/apt/sources.list (edit as root) -replace all *# deb-src* with *deb-src* (in vi `1,$s/# deb-src/deb-src/`) -`sudo apt-get update -y` -`sudo apt-get upgrade -y` -`sudo apt-get build-dep qt5-default -y` -1. git >= 1.6 -Check if needed `git --version` -`sudo apt-get install git -y` -Verify again -1. python -Check if needed `python --version` - should return python 2.7.x -`sudo apt-get install python -y` (not python 3!) -Verify again -1. gperf -Check if needed `gperf --version` -`sudo apt-get install gperf -y` -Verify again -1. bison and flex -Check if needed `flex --version` and `bison --version` -`sudo apt-get install flex bison -y` -Verify again -1. pkg-config (needed for qtwebengine) -Check if needed `pkg-config --version` -`sudo apt-get install pkg-config -y` -Verify again -1. OpenGL -Verify (first install mesa-utils - `sudo apt install mesa-utils -y`) by `glxinfo | grep "OpenGL version"` -`sudo apt-get install libgl1-mesa-dev -y` -`sudo ln -s /usr/lib/x86_64-linux-gnu/libGL.so.346.35 /usr/lib/x86_64-linux-gnu/libGL.so.1.2.0` -Verify again -1. make -Check if needed `make --version` -`sudo apt-get install make -y` -Verify again -1. g++ -Check if needed - `g++ --version` -`sudo apt-get install g++ -y` -Verify again -1. dbus-1 (needed for qtwebengine) -`sudo apt-get install libdbus-glib-1-dev -y` -1. nss (needed for qtwebengine) -`sudo apt-get install libnss3-dev -y` +1. qt5 requirements +edit /etc/apt/sources.list (edit as root) +replace all *# deb-src* with *deb-src* (in vi `1,$s/# deb-src/deb-src/`) +`sudo apt-get update -y` +`sudo apt-get upgrade -y` +`sudo apt-get build-dep qt5-default -y` +1. git >= 1.6 +Check if needed `git --version` +`sudo apt-get install git -y` +Verify again +1. python +Check if needed `python --version` - should return python 2.7.x +`sudo apt-get install python -y` (not python 3!) +Verify again +1. gperf +Check if needed `gperf --version` +`sudo apt-get install gperf -y` +Verify again +1. bison and flex +Check if needed `flex --version` and `bison --version` +`sudo apt-get install flex bison -y` +Verify again +1. pkg-config (needed for qtwebengine) +Check if needed `pkg-config --version` +`sudo apt-get install pkg-config -y` +Verify again +1. OpenGL +Verify (first install mesa-utils - `sudo apt install mesa-utils -y`) by `glxinfo | grep "OpenGL version"` +`sudo apt-get install libgl1-mesa-dev -y` +`sudo ln -s /usr/lib/x86_64-linux-gnu/libGL.so.346.35 /usr/lib/x86_64-linux-gnu/libGL.so.1.2.0` +Verify again +1. make +Check if needed `make --version` +`sudo apt-get install make -y` +Verify again +1. g++ +Check if needed + `g++ --version` +`sudo apt-get install g++ -y` +Verify again +1. dbus-1 (needed for qtwebengine) +`sudo apt-get install libdbus-glib-1-dev -y` +1. nss (needed for qtwebengine) +`sudo apt-get install libnss3-dev -y` ### Mac -1. git >= 1.6 -Check if needed `git --version` -Install from https://git-scm.com/download/mac -Verify again -1. pkg-config -brew fontconfig dbus-glib stall pkg-config -1. dbus-1 -brew install dbus-glib +1. git >= 1.6 +Check if needed `git --version` +Install from https://git-scm.com/download/mac +Verify again +1. pkg-config +brew fontconfig dbus-glib stall pkg-config +1. dbus-1 +brew install dbus-glib ## Build Process ### General -qt is cloned to the qt5 folder. -The build is performed in the qt5-build folder. -Build products are installed to the qt5-install folder. -Before running configure, make sure that the qt5-build folder is empty. +qt is cloned to the qt5 folder. +The build is performed in the qt5-build folder. +Build products are installed to the qt5-install folder. +Before running configure, make sure that the qt5-build folder is empty. -**Only run the git patches once!!!** +**Only run the git patches once!!!** ### Windows -Before building, verify that **HIFI_VCPKG_BASE_VERSION** points to a *vcpkg* folder containing *packages\openssl-windows_x64-windows*. +Before building, verify that **HIFI_VCPKG_BASE_VERSION** points to a *vcpkg* folder containing *packages\openssl-windows_x64-windows*. If not, follow https://github.com/highfidelity/vcpkg to install *vcpkg* and then *openssl*. +Also, make sure the directory that you are using to build qt is not deeply nested. It is quite possible to run into the windows MAX_PATH limit when building chromium. For example: `c:\msys64\home\ajt\code\hifi\tools\qt-builder\qt5-build` is too long. `c:\q\qt5-build\` is a better choice. #### Preparing source files -`git clone --recursive https://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch` +`git clone --recursive https://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch` + +* Copy the **patches** folder to qt5 +* Copy the **qt5vars.bat** file to qt5 +* Apply the two patches to Qt -* Copy the **patches** folder to qt5 -* Copy the **qt5vars.bat** file to qt5 -* Apply the two patches to Qt +`cd qt5` +`git apply --ignore-space-change --ignore-whitespace patches/qfloat16.patch` +`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` +`cd ..` +#### Configuring +`mkdir qt5-install` +`mkdir qt5-build` +`cd qt5-build` -`cd qt5` -`git apply --ignore-space-change --ignore-whitespace patches/qfloat16.patch` -`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` -`cd ..` -#### Configuring -`mkdir qt5-install` -`mkdir qt5-build` -`cd qt5-build` +run `..\qt5\qt5vars.bat` +`cd ..\..\qt5-build` -run `..\qt5\qt5vars.bat` -`cd ..\..\qt5-build` - -`..\qt5\configure -force-debug-info -opensource -confirm-license -opengl desktop -platform win32-msvc -openssl-linked OPENSSL_LIBS="-lssleay32 -llibeay32" -I %HIFI_VCPKG_BASE_VERSION%\packages\openssl-windows_x64-windows\include -L %HIFI_VCPKG_BASE_VERSION%\packages\openssl-windows_x64-windows\lib -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ..\qt5-install` +`..\qt5\configure -force-debug-info -opensource -confirm-license -opengl desktop -platform win32-msvc -openssl-linked OPENSSL_LIBS="-lssleay32 -llibeay32" -I %HIFI_VCPKG_BASE_VERSION%\packages\openssl-windows_x64-windows\include -L %HIFI_VCPKG_BASE_VERSION%\packages\openssl-windows_x64-windows\lib -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ..\qt5-install` #### Make -`nmake` -`nmake install` +`jom` +`jom install` #### Fixing -The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) -1. Open a bash terminal -1. `cd` to the *qt5-install* folder (e.g. `cd /mnt/d/qt5-install/`) -1. Run the following command -`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` -1. Copy *qt.conf* to *qt5-install\bin* +The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) +1. Open a bash terminal +1. `cd` to the *qt5-install* folder (e.g. `cd /mnt/d/qt5-install/`) +1. Run the following command +`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` +1. Copy *qt.conf* to *qt5-install\bin* #### Uploading -Create a tar file called qt5-install.tar from the qt5-install folder (e.g. using 7-zip) -Create a gzip file called qt5-install.tar.gz from the qt5-install.tar file just created (e.g. using 7-zip) -Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Windows/ +Create a tar file called qt5-install-5.12.3-windows.tar.gz from the qt5-install folder +Upload qt5-install-5.12.3-windows.tar.gz to our Amazon S3 hifi-public bucket, under the dependencies/vckpg directory +Update hifi_vcpkg.py to use this new URL. Additionally, you should make a small change to any file in the hifi/cmake/ports directory to force the re-download of the qt-install.tar.gz during the build process for hifi. +#### Preparing Symbols +Run `python3 prepare-windows-symbols-for-backtrace.py qt5-install` to scan the qt5-install directory for any dlls and pdbs. After running this command the backtrace directory will be created. Zip this directory up, but make sure that all dlls and pdbs are in the root of the zip file, not under a sub-directory. This file can then be uploaded to backtrace here: https://highfidelity.sp.backtrace.io/p/Interface/settings/symbol/upload ### Linux #### Preparing source files -`git clone --recursive git://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch` - -* Copy the **patches** folder to qt5 -* Apply one patch to Qt -`cd qt5` -`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` +`git clone --recursive git://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch` + +* Copy the **patches** folder to qt5 +* Apply one patch to Qt +`cd qt5` +`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` `cd ..` #### Configuring -`mkdir qt5-install` -`mkdir qt5-build` -`cd qt5-build` +`mkdir qt5-install` +`mkdir qt5-build` +`cd qt5-build` -*Ubuntu 16.04* -`../qt5/configure -opensource -confirm-license -platform linux-g++-64 -qt-zlib -qt-libjpeg -qt-libpng -qt-xcb -qt-freetype -qt-pcre -qt-harfbuzz -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -no-egl -no-icu -prefix ../qt5-install` +*Ubuntu 16.04* +`../qt5/configure -opensource -confirm-license -platform linux-g++-64 -qt-zlib -qt-libjpeg -qt-libpng -qt-xcb -qt-freetype -qt-pcre -qt-harfbuzz -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -no-egl -no-icu -prefix ../qt5-install` -*Ubuntu 18.04* -`../qt5/configure -force-debug-info -release -opensource -confirm-license -gdb-index -recheck-all -nomake tests -nomake examples -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -c++std c++14 -prefix ../qt5-install` +*Ubuntu 18.04* +`../qt5/configure -force-debug-info -release -opensource -confirm-license -gdb-index -recheck-all -nomake tests -nomake examples -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -c++std c++14 -prefix ../qt5-install` -???`../qt5/configure -opensource -confirm-license -gdb-index -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ../qt5-install` -#### Make -`make` +???`../qt5/configure -opensource -confirm-license -gdb-index -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ../qt5-install` +#### Make +`make` -????*Ubuntu 18.04 only* -????`make module-qtwebengine` -????`make module-qtscript` +????*Ubuntu 18.04 only* +????`make module-qtwebengine` +????`make module-qtscript` -*Both* -`make install` +*Both* +`make install` #### Fixing -1. The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) -`cd ../qt5-install` -`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` -1. Copy *qt.conf* to *qt5-install\bin* -#### Uploading -*Ubuntu 16.04* -1. Return to the home folder -`cd ..` -1. Open a python 3 shell -`python3` -1. Run the following snippet: -`import os` -`import tarfile` -`filename=tarfile.open("qt5-install.tar.gz", "w:gz")` -`filename.add("qt5-install", os.path.basename("qt5-install"))` -`exit()` -1. Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Ubuntu/16.04 +1. The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) +`cd ../qt5-install` +`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` +1. Copy *qt.conf* to *qt5-install\bin* +#### Uploading +*Ubuntu 16.04* +1. Return to the home folder +`cd ..` +1. Open a python 3 shell +`python3` +1. Run the following snippet: +`import os` +`import tarfile` +`filename=tarfile.open("qt5-install.tar.gz", "w:gz")` +`filename.add("qt5-install", os.path.basename("qt5-install"))` +`exit()` +1. Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Ubuntu/16.04 -*Ubuntu 18.04* -``tar -zcvf qt5-install.tar.gz qt5-install` -1. Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Ubuntu/18.04 +*Ubuntu 18.04* +``tar -zcvf qt5-install.tar.gz qt5-install` +1. Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Ubuntu/18.04 -1. ### Mac -#### Preparing source files -git clone --recursive git://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch - -* Copy the **patches** folder to qt5 -* Apply one patch to Qt -`cd qt5` -`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` -`cd ..` +### Mac +#### Preparing source files +git clone --recursive git://code.qt.io/qt/qt5.git -b 5.12.3 --single-branch + +* Copy the **patches** folder to qt5 +* Apply one patch to Qt +`cd qt5` +`git apply --ignore-space-change --ignore-whitespace patches/aec.patch` +`cd ..` #### Configuring -`mkdir qt5-install` -`mkdir qt5-build` -`cd ../qt5-build` +`mkdir qt5-install` +`mkdir qt5-build` +`cd ../qt5-build` -`../qt5/configure -force-debug-info -opensource -confirm-license -qt-zlib -qt-libjpeg -qt-libpng -qt-freetype -qt-pcre -qt-harfbuzz -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ../qt5-install` +`../qt5/configure -force-debug-info -opensource -confirm-license -qt-zlib -qt-libjpeg -qt-libpng -qt-freetype -qt-pcre -qt-harfbuzz -nomake examples -nomake tests -skip qttranslations -skip qtserialport -skip qt3d -skip qtlocation -skip qtwayland -skip qtsensors -skip qtgamepad -skip qtspeech -skip qtcharts -skip qtx11extras -skip qtmacextras -skip qtvirtualkeyboard -skip qtpurchasing -skip qtdatavis3d -no-warnings-are-errors -no-pch -prefix ../qt5-install` #### Make -`make` -`make install` +`make` +`make install` #### Fixing -1. The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) -`cd ../qt5-install` -`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` -`cd ..` +1. The *.prl* files have an absolute path that needs to be removed (see http://www.linuxfromscratch.org/blfs/view/stable-systemd/x/qtwebengine.html) +`cd ../qt5-install` +`find . -name \*.prl -exec sed -i -e '/^QMAKE_PRL_BUILD_DIR/d' {} \;` +`cd ..` 1. Copy *qt.conf* to *qt5-install\bin* #### Uploading -`tar -zcvf qt5-install.tar.gz qt5-install` -Upload qt5-install.tar.gz to https://hifi-qa.s3.amazonaws.com/qt5/Mac/ +`tar -zcvf qt5-install-5.13.2-macos.tar.gz qt5-install` +Upload qt5-install-5.13.2-macos.tar.gz to our Amazon S3 hifi-public bucket, under the dependencies/vckpg directory +#### Creating symbols +Run `python3 prepare-mac-symbols-for-backtrace.py qt5-install` to scan the qt5-build directory for any dylibs and execute dsymutil to create dSYM bundles. After running this command the backtrace directory will be created. Zip this directory up, but make sure that all dylibs and dSYM fiels are in the root of the zip file, not under a sub-directory. This file can then be uploaded to backtrace here: https://highfidelity.sp.backtrace.io/p/Interface/settings/symbol/upload ## Problems *configure* errors, if any, may be viewed in **config.log** and **config.summary** diff --git a/tools/qt-builder/prepare-mac-symbols-for-backtrace.py b/tools/qt-builder/prepare-mac-symbols-for-backtrace.py new file mode 100644 index 0000000000..2a27bf3939 --- /dev/null +++ b/tools/qt-builder/prepare-mac-symbols-for-backtrace.py @@ -0,0 +1,193 @@ +# Scans the Mac qt-install directory for all known qt dylibs used by high fidelity. +# Then copies any DWARF symbols into a coorespoding dSYM file. +# +# usage +# python prepare-mac-symbols-for-backtace.py QT_BUILD_DIR +# +# QT_BUILD_DIR should be the directory where qt was built, because +# it requires that the temporary .o files associated with the dylibs +# are present. See hifi/tools/qt-builder/README.md for more info +# + +import sys + +# python 3 script +if (not sys.version_info > (3, 5)): + print("ERROR: REQUIRES Python 3") + quit() + +import pathlib +import shutil +import os +import tarfile +import subprocess +import re + +def cp(a, b): + print("cp {} {}".format(a, b)) + shutil.copyfile(a, b) + +# list of all qt dylibs used by hifi +# generarted by installing hifi and running these commands +# > find interface.app -exec file {} \; > files.txt +# > awk '$2 == "Mach-O"' files.txt > dylibs.txt +# then removing any non-qt dylibs from the list. +hifi_dylibs = [ + "Contents/PlugIns/mediaservice/libqavfmediaplayer.dylib", + "Contents/PlugIns/mediaservice/libqtmedia_audioengine.dylib", + "Contents/PlugIns/mediaservice/libqavfcamera.dylib", + "Contents/PlugIns/quick/libdeclarative_multimedia.dylib", + "Contents/PlugIns/quick/libqtquickextrasflatplugin.dylib", + "Contents/PlugIns/quick/libqmlxmllistmodelplugin.dylib", + "Contents/PlugIns/quick/libqtgraphicaleffectsprivate.dylib", + "Contents/PlugIns/quick/libqtgraphicaleffectsplugin.dylib", + "Contents/PlugIns/quick/libdeclarative_webchannel.dylib", + "Contents/PlugIns/quick/libqtqmlremoteobjects.dylib", + "Contents/PlugIns/quick/libmodelsplugin.dylib", + "Contents/PlugIns/quick/libqtwebengineplugin.dylib", + "Contents/PlugIns/quick/libqtquickextrasplugin.dylib", + "Contents/PlugIns/quick/libqmlfolderlistmodelplugin.dylib", + "Contents/PlugIns/quick/libqquicklayoutsplugin.dylib", + "Contents/PlugIns/quick/libqtquickcontrols2materialstyleplugin.dylib", + "Contents/PlugIns/quick/libqtquick2plugin.dylib", + "Contents/PlugIns/quick/libwindowplugin.dylib", + "Contents/PlugIns/quick/libqtquickcontrols2imaginestyleplugin.dylib", + "Contents/PlugIns/quick/libdialogplugin.dylib", + "Contents/PlugIns/quick/libqtquickcontrolsplugin.dylib", + "Contents/PlugIns/quick/libwidgetsplugin.dylib", + "Contents/PlugIns/quick/libqtquickcontrols2universalstyleplugin.dylib", + "Contents/PlugIns/quick/libdialogsprivateplugin.dylib", + "Contents/PlugIns/quick/libqtquickcontrols2fusionstyleplugin.dylib", + "Contents/PlugIns/quick/libqtquicktemplates2plugin.dylib", + "Contents/PlugIns/quick/libqmlsettingsplugin.dylib", + "Contents/PlugIns/quick/libqtqmlstatemachine.dylib", + "Contents/PlugIns/quick/libqtquickcontrols2plugin.dylib", + "Contents/PlugIns/styles/libqmacstyle.dylib", + "Contents/PlugIns/audio/libqtaudio_coreaudio.dylib", + "Contents/PlugIns/bearer/libqgenericbearer.dylib", + "Contents/PlugIns/iconengines/libqsvgicon.dylib", + "Contents/PlugIns/imageformats/libqgif.dylib", + "Contents/PlugIns/imageformats/libqwbmp.dylib", + "Contents/PlugIns/imageformats/libqwebp.dylib", + "Contents/PlugIns/imageformats/libqico.dylib", + "Contents/PlugIns/imageformats/libqmacheif.dylib", + "Contents/PlugIns/imageformats/libqjpeg.dylib", + "Contents/PlugIns/imageformats/libqtiff.dylib", + "Contents/PlugIns/imageformats/libqsvg.dylib", + "Contents/PlugIns/imageformats/libqicns.dylib", + "Contents/PlugIns/imageformats/libqtga.dylib", + "Contents/PlugIns/imageformats/libqmacjp2.dylib", + "Contents/Frameworks/QtGui.framework/Versions/5/QtGui", + "Contents/Frameworks/QtGui.framework/QtGui", + "Contents/Frameworks/QtDBus.framework/Versions/5/QtDBus", + "Contents/Frameworks/QtDBus.framework/QtDBus", + "Contents/Frameworks/QtNetwork.framework/Versions/5/QtNetwork", + "Contents/Frameworks/QtNetwork.framework/QtNetwork", + "Contents/Frameworks/QtMultimedia.framework/Versions/5/QtMultimedia", + "Contents/Frameworks/QtMultimedia.framework/QtMultimedia", + "Contents/Frameworks/QtQml.framework/QtQml", + "Contents/Frameworks/QtQml.framework/Versions/5/QtQml", + "Contents/Frameworks/QtXml.framework/Versions/5/QtXml", + "Contents/Frameworks/QtXml.framework/QtXml", + "Contents/Frameworks/QtWidgets.framework/Versions/5/QtWidgets", + "Contents/Frameworks/QtWidgets.framework/QtWidgets", + "Contents/Frameworks/QtMultimediaQuick.framework/Versions/5/QtMultimediaQuick", + "Contents/Frameworks/QtMultimediaQuick.framework/QtMultimediaQuick", + "Contents/Frameworks/QtWebChannel.framework/Versions/5/QtWebChannel", + "Contents/Frameworks/QtWebChannel.framework/QtWebChannel", + "Contents/Frameworks/QtScriptTools.framework/Versions/5/QtScriptTools", + "Contents/Frameworks/QtScriptTools.framework/QtScriptTools", + "Contents/Frameworks/libPolyVoxCore.dylib", + "Contents/Frameworks/QtWebSockets.framework/QtWebSockets", + "Contents/Frameworks/QtWebSockets.framework/Versions/5/QtWebSockets", + "Contents/Frameworks/QtQuickTemplates2.framework/QtQuickTemplates2", + "Contents/Frameworks/QtQuickTemplates2.framework/Versions/5/QtQuickTemplates2", + "Contents/Frameworks/QtCore.framework/Versions/5/QtCore", + "Contents/Frameworks/QtCore.framework/QtCore", + "Contents/Frameworks/QtWebEngine.framework/Versions/5/QtWebEngine", + "Contents/Frameworks/QtWebEngine.framework/QtWebEngine", + "Contents/Frameworks/QtOpenGL.framework/QtOpenGL", + "Contents/Frameworks/QtOpenGL.framework/Versions/5/QtOpenGL", + "Contents/Frameworks/QtMultimediaWidgets.framework/Versions/5/QtMultimediaWidgets", + "Contents/Frameworks/QtMultimediaWidgets.framework/QtMultimediaWidgets", + "Contents/Frameworks/QtQuickControls2.framework/Versions/5/QtQuickControls2", + "Contents/Frameworks/QtQuickControls2.framework/QtQuickControls2", + "Contents/Frameworks/QtRemoteObjects.framework/QtRemoteObjects", + "Contents/Frameworks/QtRemoteObjects.framework/Versions/5/QtRemoteObjects", + "Contents/Frameworks/QtConcurrent.framework/Versions/5/QtConcurrent", + "Contents/Frameworks/QtConcurrent.framework/QtConcurrent", + "Contents/Frameworks/QtScript.framework/QtScript", + "Contents/Frameworks/QtScript.framework/Versions/5/QtScript", + "Contents/Frameworks/QtQuick.framework/Versions/5/QtQuick", + "Contents/Frameworks/QtQuick.framework/QtQuick", + "Contents/Frameworks/QtPrintSupport.framework/Versions/5/QtPrintSupport", + "Contents/Frameworks/QtPrintSupport.framework/QtPrintSupport", + "Contents/Frameworks/QtSvg.framework/Versions/5/QtSvg", + "Contents/Frameworks/QtSvg.framework/QtSvg", + "Contents/Frameworks/QtWebEngineCore.framework/Versions/5/QtWebEngineCore", + "Contents/Frameworks/QtWebEngineCore.framework/Versions/5/Helpers/QtWebEngineProcess.app/Contents/MacOS/QtWebEngineProcess", + "Contents/Frameworks/QtWebEngineCore.framework/QtWebEngineCore", + "Contents/Frameworks/QtXmlPatterns.framework/Versions/5/QtXmlPatterns", + "Contents/Frameworks/QtXmlPatterns.framework/QtXmlPatterns", + "Contents/Resources/qml/QtGraphicalEffects/libqtgraphicaleffectsplugin.dylib", + "Contents/Resources/qml/QtGraphicalEffects/private/libqtgraphicaleffectsprivate.dylib", + "Contents/Resources/qml/QtQml/StateMachine/libqtqmlstatemachine.dylib", + "Contents/Resources/qml/QtQml/Models.2/libmodelsplugin.dylib", + "Contents/Resources/qml/QtQml/RemoteObjects/libqtqmlremoteobjects.dylib", + "Contents/Resources/qml/Qt/labs/settings/libqmlsettingsplugin.dylib", + "Contents/Resources/qml/Qt/labs/folderlistmodel/libqmlfolderlistmodelplugin.dylib", + "Contents/Resources/qml/QtQuick.2/libqtquick2plugin.dylib", + "Contents/Resources/qml/QtWebEngine/libqtwebengineplugin.dylib", + "Contents/Resources/qml/QtWebChannel/libdeclarative_webchannel.dylib", + "Contents/Resources/qml/QtMultimedia/libdeclarative_multimedia.dylib", + "Contents/Resources/qml/QtQuick/Extras/libqtquickextrasplugin.dylib", + "Contents/Resources/qml/QtQuick/XmlListModel/libqmlxmllistmodelplugin.dylib", + "Contents/Resources/qml/QtQuick/PrivateWidgets/libwidgetsplugin.dylib", + "Contents/Resources/qml/QtQuick/Layouts/libqquicklayoutsplugin.dylib", + "Contents/Resources/qml/QtQuick/Window.2/libwindowplugin.dylib", + "Contents/Resources/qml/QtQuick/Dialogs/Private/libdialogsprivateplugin.dylib", + "Contents/Resources/qml/QtQuick/Dialogs/libdialogplugin.dylib", + "Contents/Resources/qml/QtQuick/Templates.2/libqtquicktemplates2plugin.dylib", + "Contents/Resources/qml/QtQuick/Controls.2/Material/libqtquickcontrols2materialstyleplugin.dylib", + "Contents/Resources/qml/QtQuick/Controls.2/Universal/libqtquickcontrols2universalstyleplugin.dylib", + "Contents/Resources/qml/QtQuick/Controls.2/Imagine/libqtquickcontrols2imaginestyleplugin.dylib", + "Contents/Resources/qml/QtQuick/Controls.2/Fusion/libqtquickcontrols2fusionstyleplugin.dylib", + "Contents/Resources/qml/QtQuick/Controls.2/libqtquickcontrols2plugin.dylib", + "Contents/Resources/qml/QtQuick/Controls/Styles/Flat/libqtquickextrasflatplugin.dylib", + "Contents/Resources/qml/QtQuick/Controls/libqtquickcontrolsplugin.dylib", +] + +# strip off the path +desired_dylibs = [pathlib.PurePath(i).name for i in hifi_dylibs] + +QT_BUILD_DIRECTORY = sys.argv[1] + +# all files in the qt build directory +found_files = pathlib.Path(QT_BUILD_DIRECTORY).glob('**/*') + +# create temp directory +TEMP_DIR_NAME = "backtrace" +if os.path.exists(TEMP_DIR_NAME): + shutil.rmtree(TEMP_DIR_NAME) +os.mkdir(TEMP_DIR_NAME) + +# for each file in the build directory (this might take a while) +for f in found_files: + + # if this file is desired + if f.name in desired_dylibs: + + # run the file command on this file to determine what kind of file it is. + result = subprocess.run(['file', f], stdout=subprocess.PIPE) + + # if this file is a dylib + if re.match(".*Mach-O 64-bit dynamically linked shared library x86_64", str(result.stdout)): + + dst_dylib = pathlib.PurePath(TEMP_DIR_NAME, f.name) + dst_dsym = pathlib.PurePath(TEMP_DIR_NAME, f.stem + ".dSYM") + + # generate a dSYM file for this dylib in the temp folder + subprocess.run(['dsymutil', f, '-o', dst_dsym]) + + # copy dylib into the temp folder + cp(f, dst_dylib) diff --git a/tools/qt-builder/prepare-windows-symbols-for-backtrace.py b/tools/qt-builder/prepare-windows-symbols-for-backtrace.py new file mode 100644 index 0000000000..5738907a2d --- /dev/null +++ b/tools/qt-builder/prepare-windows-symbols-for-backtrace.py @@ -0,0 +1,132 @@ +# Scans the Windows qt-install directory for all known qt dlls used by high fidelity. +# Then copies all matching dlls and pdbs into folder named backtrace. +# +# usage +# python prepare-windows-symbols-for-backrace.py QT_INSTALL_DIR +# +# QT_BUILD_DIR should be the directory where qt is installed after running `jom install` or `nmake install` +# see hifi/tools/qt-builder/README.md for more info +# + +import sys + +# python 3 script +if (not sys.version_info > (3, 2)): + print("ERROR: REQUIRES Python 3") + quit() + +import pathlib +import shutil +import os +import tarfile + +def cp(a, b): + print("cp {} {}".format(a, b)) + shutil.copyfile(a, b) + +# list of all qt dlls used in hifi +hifi_dlls = [ + "audioWin7/audio/qtaudio_windows.dll", + "audioWin8/audio/qtaudio_wasapi.dll", + "bearer/qgenericbearer.dll", + "iconengines/qsvgicon.dll", + "imageformats/qgif.dll", + "imageformats/qicns.dll", + "imageformats/qico.dll", + "imageformats/qjpeg.dll", + "imageformats/qsvg.dll", + "imageformats/qtga.dll", + "imageformats/qtiff.dll", + "imageformats/qwbmp.dll", + "imageformats/qwebp.dll", + "mediaservice/dsengine.dll", + "mediaservice/qtmedia_audioengine.dll", + "mediaservice/wmfengine.dll", + "platforminputcontexts/qtvirtualkeyboardplugin.dll", + "platforms/qwindows.dll", + "playlistformats/qtmultimedia_m3u.dll", + "position/qtposition_geoclue.dll", + "position/qtposition_positionpoll.dll", + "position/qtposition_serialnmea.dll", + "qmltooling/qmldbg_debugger.dll", + "qmltooling/qmldbg_inspector.dll", + "qmltooling/qmldbg_local.dll", + "qmltooling/qmldbg_messages.dll", + "qmltooling/qmldbg_native.dll", + "qmltooling/qmldbg_nativedebugger.dll", + "qmltooling/qmldbg_preview.dll", + "qmltooling/qmldbg_profiler.dll", + "qmltooling/qmldbg_quickprofiler.dll", + "qmltooling/qmldbg_server.dll", + "qmltooling/qmldbg_tcp.dll", + "Qt/labs/folderlistmodel/qmlfolderlistmodelplugin.dll", + "Qt/labs/settings/qmlsettingsplugin.dll", + "Qt5Core.dll", + "Qt5Gui.dll", + "Qt5Multimedia.dll", + "Qt5MultimediaQuick.dll", + "Qt5Network.dll", + "Qt5Positioning.dll", + "Qt5Qml.dll", + "Qt5Quick.dll", + "Qt5QuickControls2.dll", + "Qt5QuickTemplates2.dll", + "Qt5RemoteObjects.dll", + "Qt5Script.dll", + "Qt5ScriptTools.dll", + "Qt5SerialPort.dll", + "Qt5Svg.dll", + "Qt5WebChannel.dll", + "Qt5WebEngine.dll", + "Qt5WebEngineCore.dll", + "Qt5WebSockets.dll", + "Qt5Widgets.dll", + "Qt5XmlPatterns.dll", + "QtGraphicalEffects/private/qtgraphicaleffectsprivate.dll", + "QtGraphicalEffects/qtgraphicaleffectsplugin.dll", + "QtMultimedia/declarative_multimedia.dll", + "QtQml/Models.2/modelsplugin.dll", + "QtQml/RemoteObjects/qtqmlremoteobjects.dll", + "QtQml/StateMachine/qtqmlstatemachine.dll", + "QtQuick/Controls/qtquickcontrolsplugin.dll", + "QtQuick/Controls/Styles/Flat/qtquickextrasflatplugin.dll", + "QtQuick/Controls.2/Fusion/qtquickcontrols2fusionstyleplugin.dll", + "QtQuick/Controls.2/Imagine/qtquickcontrols2imaginestyleplugin.dll", + "QtQuick/Controls.2/Material/qtquickcontrols2materialstyleplugin.dll", + "QtQuick/Controls.2/qtquickcontrols2plugin.dll", + "QtQuick/Controls.2/Universal/qtquickcontrols2universalstyleplugin.dll", + "QtQuick/Dialogs/dialogplugin.dll", + "QtQuick/Dialogs/Private/dialogsprivateplugin.dll", + "QtQuick/Extras/qtquickextrasplugin.dll", + "QtQuick/Layouts/qquicklayoutsplugin.dll", + "QtQuick/PrivateWidgets/widgetsplugin.dll", + "QtQuick/Templates.2/qtquicktemplates2plugin.dll", + "QtQuick/Window.2/windowplugin.dll", + "QtQuick/XmlListModel/qmlxmllistmodelplugin.dll", + "QtQuick.2/qtquick2plugin.dll", + "QtWebChannel/declarative_webchannel.dll", + "QtWebEngine/qtwebengineplugin.dll", + "scenegraph/qsgd3d12backend.dll", + "styles/qwindowsvistastyle.dll", +] + +# list of desired pdbs, created from hifi_dlls +desired_pdbs = [pathlib.PurePath(i).stem + ".pdb" for i in hifi_dlls] + +QT_INSTALL_DIRECTORY = sys.argv[1] + +# find all pdb files in the qt build directory +found_pdbs = pathlib.Path(QT_INSTALL_DIRECTORY).glob('**/*.pdb') + +# create temp directory +TEMP_DIR_NAME = "backtrace" +if os.path.exists(TEMP_DIR_NAME): + shutil.rmtree(TEMP_DIR_NAME) +os.mkdir(TEMP_DIR_NAME) + +# copy all found dlls and pdbs into the temp directory +for pdb in found_pdbs: + if pdb.name in desired_pdbs: + dll = pathlib.PurePath(pdb.parent, pdb.stem + ".dll") + cp(pdb, pathlib.PurePath(TEMP_DIR_NAME, pdb.name)) + cp(dll, pathlib.PurePath(TEMP_DIR_NAME, dll.name))