import de.undercouch.gradle.tasks.download.Download import de.undercouch.gradle.tasks.download.Verify import groovy.io.FileType import groovy.json.JsonSlurper import groovy.xml.XmlUtil import org.apache.tools.ant.taskdefs.condition.Os import java.util.regex.Matcher import java.util.regex.Pattern buildscript { repositories { jcenter() google() } dependencies { classpath 'com.android.tools.build:gradle:3.0.1' } } plugins { id 'de.undercouch.download' version '3.3.0' id "cz.malohlava" version "1.0.3" } allprojects { repositories { jcenter() google() } } task clean(type: Delete) { delete rootProject.buildDir } ext { RELEASE_NUMBER = project.hasProperty('RELEASE_NUMBER') ? project.getProperty('RELEASE_NUMBER') : '0' RELEASE_TYPE = project.hasProperty('RELEASE_TYPE') ? project.getProperty('RELEASE_TYPE') : 'DEV' BUILD_BRANCH = project.hasProperty('BUILD_BRANCH') ? project.getProperty('BUILD_BRANCH') : '' EXEC_SUFFIX = Os.isFamily(Os.FAMILY_WINDOWS) ? '.exe' : '' QT5_DEPS = [ 'Qt5Core', 'Qt5Gui', 'Qt5Multimedia', 'Qt5Network', 'Qt5OpenGL', 'Qt5Qml', 'Qt5Quick', 'Qt5Script', 'Qt5ScriptTools', 'Qt5WebChannel', 'Qt5WebSockets', 'Qt5Widgets', 'Qt5XmlPatterns', // Android specific 'Qt5AndroidExtras', 'Qt5WebView', ] } def baseFolder = new File(HIFI_ANDROID_PRECOMPILED) def appDir = new File(projectDir, 'app') def jniFolder = new File(appDir, 'src/main/jniLibs/arm64-v8a') def baseUrl = 'https://hifi-public.s3.amazonaws.com/austin/android/' def qtFile='qt-5.9.3_linux_armv8-libcpp_openssl.tgz' def qtChecksum='04599670ccca84bd2b15f6915568eb2d' def qtVersionId='PeoqzN31n.YvLfs9JE2SgHgZ4.IaKAlt' if (Os.isFamily(Os.FAMILY_MAC)) { qtFile = 'qt-5.9.3_osx_armv8-libcpp_openssl.tgz' qtChecksum='4b02de9d67d6bfb202355a808d2d9c59' qtVersionId='HygCmtMLPYioyil0DfXckGVzhw2SXZA9' } else if (Os.isFamily(Os.FAMILY_WINDOWS)) { qtFile = 'qt-5.9.3_win_armv8-libcpp_openssl.tgz' qtChecksum='a93d22c0c59aa112fda18c4c6d157d17' qtVersionId='0Bl9NSUWb5CBKLT_NXaxTt75SNBBZ9sB' } def packages = [ qt: [ file: qtFile, versionId: qtVersionId, checksum: qtChecksum, ], bullet: [ file: 'bullet-2.83_armv8-libcpp.tgz', versionId: 'ljb7v.1IjVRqyopUKVDbVnLA4z88J8Eo', checksum: '2c558d604fce337f5eba3eb7ec1252fd', ], draco: [ file: 'draco_armv8-libcpp.tgz', versionId: 'cA3tVJSmkvb1naA3l6D_Jv2Noh.4yc4m', checksum: '617a80d213a5ec69fbfa21a1f2f738cd', ], glm: [ file: 'glm-0.9.8.tgz', versionId: 'BlkJNwaYV2Gfy5XwMeU7K0uzPDRKFMt2', checksum: 'd2b42cee31d2bc17bab6ce69e6b3f30a', ], gvr: [ file: 'gvrsdk_v1.101.0.tgz', versionId: 'UTberAIFraEfF9IVjoV66u1DTPTopgeY', checksum: '57fd02baa069176ba18597a29b6b4fc7', ], openssl: [ file: 'openssl-1.1.0g_armv8.tgz', versionId: 'DmahmSGFS4ltpHyTdyQvv35WOeUOiib9', checksum: 'cabb681fbccd79594f65fcc266e02f32', ], polyvox: [ file: 'polyvox_armv8-libcpp.tgz', versionId: 'LDJtzMTvdm4SAc2KYg8Cg6uwWk4Vq3e3', checksum: '349ad5b72aaf2749ca95d847e60c5314', sharedLibFolder: 'lib', includeLibs: ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'], ], tbb: [ file: 'tbb-2018_U1_armv8_libcpp.tgz', versionId: 'YZliDD8.Menh1IVXKEuLPeO3xAjJ1UdF', checksum: '20768f298f53b195e71b414b0ae240c4', sharedLibFolder: 'lib/release', includeLibs: ['libtbb.so', 'libtbbmalloc.so'], ] ] def scribeLocalFile='scribe' + EXEC_SUFFIX def scribeFile='scribe_linux_x86_64' def scribeChecksum='c98678d9726bd8bbf1bab792acf3ff6c' if (Os.isFamily(Os.FAMILY_MAC)) { scribeFile = 'scribe_osx_x86_64' scribeChecksum='a137ad62c1bf7cca739da219544a9a16' } else if (Os.isFamily(Os.FAMILY_WINDOWS)) { scribeFile = 'scribe_win32_x86_64.exe' scribeChecksum='75c2ce9ed45d17de375e3988bfaba816' } def options = [ files: new TreeSet(), features: new HashSet(), permissions: new HashSet() ] def qmlRoot = new File(HIFI_ANDROID_PRECOMPILED, 'qt') def captureOutput = { String command -> def proc = command.execute() def sout = new StringBuilder(), serr = new StringBuilder() proc.consumeProcessOutput(sout, serr) proc.waitForOrKill(10000) def errorOutput = serr.toString() if (!errorOutput.isEmpty()) { throw new GradleException("Command '${command}' failed with error ${errorOutput}") } return sout.toString() } def relativize = { File root, File absolute -> def relativeURI = root.toURI().relativize(absolute.toURI()) return new File(relativeURI.toString()) } def scanQmlImports = { File qmlRootPath -> def qmlImportCommandFile = new File(qmlRoot, 'bin/qmlimportscanner' + EXEC_SUFFIX) if (!qmlImportCommandFile.exists()) { throw new GradleException('Unable to find required qmlimportscanner executable at ' + qmlImportCommandFile.parent.toString()) } def command = qmlImportCommandFile.absolutePath + " -rootPath ${qmlRootPath.absolutePath}" + " -importPath ${qmlRoot.absolutePath}/qml" def commandResult = captureOutput(command) new JsonSlurper().parseText(commandResult).each { if (!it.containsKey('path')) { println "Warning: QML import could not be resolved in any of the import paths: ${it.name}" return } def file = new File(it.path) // Ignore non-existent files if (!file.exists()) { return } // Ignore files in the import path if (file.canonicalPath.startsWith(qmlRootPath.canonicalPath)) { return } if (file.isFile()) { options.files.add(file) } else { file.eachFileRecurse(FileType.FILES, { options.files.add(it) }) } } } def parseQtDependencies = { List qtLibs -> qtLibs.each({ def libFile = new File(qmlRoot, "lib/lib${it}.so") options.files.add(libFile) def androidDeps = new File(qmlRoot, "lib/${it}-android-dependencies.xml") if (!libFile.exists()) return if (!androidDeps.exists()) return new XmlSlurper().parse(androidDeps).dependencies.lib.depends.'*'.each{ node -> switch (node.name()) { case 'lib': case 'bundled': def relativeFilename = node.@file.toString() // Special case, since this is handled by qmlimportscanner instead if (relativeFilename.startsWith('qml')) return def file = new File(qmlRoot, relativeFilename) if (!file.exists()) return if (file.isFile()) { options.files.add(file) } else { file.eachFileRecurse(FileType.FILES, { options.files.add(it) }) } break case 'jar': if (node.@bundling == "1") { def jar = new File(qmlRoot, node.@file.toString()) if (!jar.exists()) { throw new GradleException('Unable to find required JAR ' + jar.path) } options.files.add(jar) } break case 'permission': options.permissions.add(node.@name) break case 'feature': options.features.add(node.@name) break default: throw new GradleException('Unhandled Android Dependency node ' + node.name()) } } }) } def generateLibsXml = { def libDestinationDirectory = jniFolder def jarDestinationDirectory = new File(appDir, 'libs') def assetDestinationDirectory = new File(appDir, 'src/main/assets/bundled'); def libsXmlFile = new File(appDir, 'src/main/res/values/libs.xml') def libPrefix = 'lib' + File.separator def jarPrefix = 'jar' + File.separator def xmlParser = new XmlParser() def libsXmlRoot = xmlParser.parseText('') def qtLibsNode = xmlParser.createNode(libsXmlRoot, 'array', [name: 'qt_libs']) def bundledLibsNode = xmlParser.createNode(libsXmlRoot, 'array', [name: 'bundled_in_lib']) def bundledAssetsNode = xmlParser.createNode(libsXmlRoot, 'array', [name: 'bundled_in_assets']) options.files.each { def sourceFile = it if (!sourceFile.exists()) { throw new GradleException("Unable to find dependency file " + sourceFile.toString()) } def relativePath = relativize( qmlRoot, sourceFile ).toString() def destinationFile if (relativePath.endsWith('.so')) { def garbledFileName if (relativePath.startsWith(libPrefix)) { garbledFileName = relativePath.substring(libPrefix.size()) Pattern p = ~/lib(Qt5.*).so/ Matcher m = p.matcher(garbledFileName) assert m.matches() def libName = m.group(1) xmlParser.createNode(qtLibsNode, 'item', [:]).setValue(libName) } else { garbledFileName = 'lib' + relativePath.replace(File.separator, '_'[0]) xmlParser.createNode(bundledLibsNode, 'item', [:]).setValue("${garbledFileName}:${relativePath}".replace(File.separator, '/')) } destinationFile = new File(libDestinationDirectory, garbledFileName) } else if (relativePath.startsWith('jar')) { destinationFile = new File(jarDestinationDirectory, relativePath.substring(jarPrefix.size())) } else { xmlParser.createNode(bundledAssetsNode, 'item', [:]).setValue("bundled/${relativePath}:${relativePath}".replace(File.separator, '/')) destinationFile = new File(assetDestinationDirectory, relativePath) } copy { from sourceFile; into destinationFile.parent; rename(sourceFile.name, destinationFile.name) } assert destinationFile.exists() && destinationFile.isFile() } def xml = XmlUtil.serialize(libsXmlRoot) new FileWriter(libsXmlFile).withPrintWriter { writer -> writer.write(xml) } } task downloadDependencies { doLast { packages.each { entry -> def filename = entry.value['file']; def url = baseUrl + filename; if (entry.value.containsKey('versionId')) { url = url + '?versionId=' + entry.value['versionId'] } download { src url dest new File(baseFolder, filename) onlyIfNewer true } } } } task verifyQt(type: Verify) { def p = packages['qt']; src new File(baseFolder, p['file']); checksum p['checksum']; } task verifyBullet(type: Verify) { def p = packages['bullet']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyDraco(type: Verify) { def p = packages['draco']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyGvr(type: Verify) { def p = packages['gvr']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyOpenSSL(type: Verify) { def p = packages['openssl']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyPolyvox(type: Verify) { def p = packages['polyvox']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyTBB(type: Verify) { def p = packages['tbb']; src new File(baseFolder, p['file']); checksum p['checksum'] } task verifyDependencyDownloads(dependsOn: downloadDependencies) { } verifyDependencyDownloads.dependsOn verifyQt verifyDependencyDownloads.dependsOn verifyBullet verifyDependencyDownloads.dependsOn verifyDraco verifyDependencyDownloads.dependsOn verifyGvr verifyDependencyDownloads.dependsOn verifyOpenSSL verifyDependencyDownloads.dependsOn verifyPolyvox verifyDependencyDownloads.dependsOn verifyTBB task extractDependencies(dependsOn: verifyDependencyDownloads) { doLast { packages.each { entry -> def folder = entry.key def filename = entry.value['file'] def localFile = new File(HIFI_ANDROID_PRECOMPILED, filename) def localFolder = new File(HIFI_ANDROID_PRECOMPILED, folder) def fileTree; if (filename.endsWith('zip')) { fileTree = zipTree(localFile) } else { fileTree = tarTree(resources.gzip(localFile)) } copy { from fileTree into localFolder } } } } // Copies the non Qt dependencies. Qt dependencies (primary libraries and plugins) are handled by the qtBundle task task copyDependencies(dependsOn: [ extractDependencies ]) { doLast { packages.each { entry -> def packageName = entry.key def currentPackage = entry.value; if (currentPackage.containsKey('sharedLibFolder')) { def localFolder = new File(baseFolder, packageName + '/' + currentPackage['sharedLibFolder']) def tree = fileTree(localFolder); if (currentPackage.containsKey('includeLibs')) { currentPackage['includeLibs'].each { includeSpec -> tree.include includeSpec } } tree.visit { element -> if (!element.file.isDirectory()) { println "Copying " + element.file + " to " + jniFolder copy { from element.file; into jniFolder } } } } } } } task downloadScribe(type: Download) { src baseUrl + scribeFile dest new File(baseFolder, scribeLocalFile) onlyIfNewer true } task verifyScribe (type: Verify, dependsOn: downloadScribe) { src new File(baseFolder, scribeLocalFile); checksum scribeChecksum } task fixScribePermissions(type: Exec, dependsOn: verifyScribe) { commandLine 'chmod', 'a+x', HIFI_ANDROID_PRECOMPILED + '/' + scribeLocalFile } task setupScribe(dependsOn: verifyScribe) { } // On Windows, we don't need to set the executable bit, but on OSX and Unix we do if (!Os.isFamily(Os.FAMILY_WINDOWS)) { setupScribe.dependsOn fixScribePermissions } task extractGvrBinaries(dependsOn: extractDependencies) { doLast { def gvrLibFolder = new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries'); zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-audio-1.101.0.aar')).visit { element -> def fileName = element.file.toString(); if (fileName.endsWith('libgvr_audio.so') && fileName.contains('arm64-v8a')) { copy { from element.file; into gvrLibFolder } } } zipTree(new File(HIFI_ANDROID_PRECOMPILED, 'gvr/gvr-android-sdk-1.101.0/libraries/sdk-base-1.101.0.aar')).visit { element -> def fileName = element.file.toString(); if (fileName.endsWith('libgvr.so') && fileName.contains('arm64-v8a')) { copy { from element.file; into gvrLibFolder } } } fileTree(gvrLibFolder).visit { element -> if (element.file.toString().endsWith('.so')) { copy { from element.file; into jniFolder } } } } } // Copy required Qt main libraries and required plugins based on the predefined list here // FIXME eventually we would like to use the readelf functionality to automatically detect dependencies // from our built applications and use that during the full build process. However doing so would mean // hooking existing Android build tasks since the output from the qtBundle logic adds JNI libs, asset // files and resources files and potentially modifies the AndroidManifest.xml task qtBundle { doLast { parseQtDependencies(QT5_DEPS) //def qmlImportFolder = new File("${appDir}/../../interface/resources/qml/") def qmlImportFolder = new File("${projectDir}/app/src/main/cpp") scanQmlImports(qmlImportFolder) generateLibsXml() } } task setupDependencies(dependsOn: [setupScribe, copyDependencies, extractGvrBinaries, qtBundle]) { } task cleanDependencies(type: Delete) { delete HIFI_ANDROID_PRECOMPILED delete 'app/src/main/jniLibs/arm64-v8a' delete 'app/src/main/assets/bundled' delete 'app/src/main/res/values/libs.xml' } // FIXME this code is prototyping the desired functionality for doing build time binary dependency resolution. // See the comment on the qtBundle task above /* // FIXME derive the path from the gradle environment def toolchain = [ version: '4.9', prefix: 'aarch64-linux-android', // FIXME derive from the host OS ndkHost: 'windows-x86_64', ] def findDependentLibrary = { String name -> def libFolders = [ new File(qmlRoot, 'lib'), new File("${HIFI_ANDROID_PRECOMPILED}/tbb/lib/release"), new File("${HIFI_ANDROID_PRECOMPILED}/polyvox/lib/Release"), new File("${HIFI_ANDROID_PRECOMPILED}/polyvox/lib/"), new File("${HIFI_ANDROID_PRECOMPILED}/gvr/gvr-android-sdk-1.101.0/libraries"), ] } def readElfBinary = new File(android.ndkDirectory, "/toolchains/${toolchain.prefix}-${toolchain.version}/prebuilt/${toolchain.ndkHost}/bin/${toolchain.prefix}-readelf${EXEC_SUFFIX}") def getDependencies = { File elfBinary -> Set result = [] Queue pending = new LinkedList<>() pending.add(elfBinary) Set scanned = [] Pattern p = ~/.*\(NEEDED\).*Shared library: \[(.*\.so)\]/ while (!pending.isEmpty()) { File current = pending.remove() if (scanned.contains(current)) { continue } scanned.add(current) def command = "${readElfBinary} -d -W ${current.absolutePath}" captureOutput(command).split('[\r\n]').each { line -> Matcher m = p.matcher(line) if (!m.matches()) { return } def libName = m.group(1) def file = new File(qmlRoot, "lib/${libName}") if (file.exists()) { result.add(file) pending.add(file) } } } return result } task testElf (dependsOn: 'externalNativeBuildDebug') { doLast { def appLibraries = new HashSet() def qtDependencies = new HashSet() externalNativeBuildDebug.nativeBuildConfigurationsJsons.each { File file -> def json = new JsonSlurper().parse(file) json.libraries.each { node -> def outputFile = new File(node.value.output) if (outputFile.canonicalPath.startsWith(projectDir.canonicalPath)) { appLibraries.add(outputFile) } } } appLibraries.each { File file -> println getDependencies(file) } } } */