diff --git a/.gitignore b/.gitignore index 41862b53b..1db0f10ec 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,9 @@ Release32/ Release64/ build/ build-*/ -dist/ +/dist/ +/package/ +util/buildscripts/support Resources/*.png qrenderdoc/Makefile* *.exe diff --git a/util/buildscripts/README.md b/util/buildscripts/README.md new file mode 100644 index 000000000..91ce1a4f7 --- /dev/null +++ b/util/buildscripts/README.md @@ -0,0 +1,40 @@ +# Build scripts + +The build.sh in this folder is used for building packaged builds of renderdoc on both windows and linux. + +On windows the only supported environment is a MSYS2 bash shell, other shells may work but modifications to support them are unlikely to be accepted. + +Running build.sh will print the usage instructions. It can be run from anywhere. + +By default it will compile everything and package, which will work out of a fresh checkout. You can also pass --skipcompile to package already built binaries, but in this case ensure that everything is compiled as expected: + +* On windows, Win32 and x64 Release targets must be built. +* On windows, the `htmlhelp` docs target must be built, and on Linux the `html` target. +* On all platforms, an arm32 android build must exist in the `build-android-arm32` subfolder under the root, and likewise for arm64. +* On linux, the cmake build must be `make installed` to a `dist` folder in the root. + +As above, running build.sh by default will compile all this. + +# Running + +Running build.sh apart from the optional arguments must pass `--snapshot `. The name is used in packaging, i.e. the resulting zips will be `RenderDoc_name.zip`. + +The output files will be placed in a `package` subfolder under the root. + +`gpg` is used to generate signatures for the file, so a public/private key should be configured. + +# Extras + +Some extra files can be used, see the `support` subfolder. Also https://renderdoc.org/plugins.zip contains the windows plugins, and https://renderdoc.org/plugins.tgz contains the linux plugins. Extracting these in the root of the repository will be included in the package builds. + +On windows PySide2 can be included with the Qt build by extracting https://renderdoc.org/qrenderdoc_3rdparty.zip in the root. + +For windows to build installers, all these extras must be present. + +# Requirements + +In addition to the usual build requirements for native and android builds, windows requires the WiX toolset to generate installer files. + +Linux requires docker to build the tool in an isolated container for maximum compatibility, although using the above mentioned `--skipcompile` you could get around this by building locally. + +Both platforms require s phinx to be available in python, and in addition windows requires the HTML help workshop. diff --git a/util/buildscripts/build.sh b/util/buildscripts/build.sh new file mode 100644 index 000000000..0584a275a --- /dev/null +++ b/util/buildscripts/build.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +TYPE="" +SNAPNAME="" +SYMSTORE="" +ERRORMAIL="" +SKIPCOMPILE="" +LLVM_ARM32=$(readlink -f $(dirname $0)/support/llvm_arm32) +LLVM_ARM64=$(readlink -f $(dirname $0)/support/llvm_arm64) + +usage() { + echo "Usage: $0 --official|--snapshot [options...]"; + echo; + echo " --symstore [Windows only] Specify a path to a symbol store."; + echo " --llvm_arm32 Give the path to an ARM32 build of LLVM, for android."; + echo " --llvm_arm64 Give the path to an ARM64 build of LLVM, for android."; + echo " --errormail Send emails on error"; + echo " --skipcompile Skip compile steps, package already compiled binaries."; +} + +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --official) + TYPE="official" + shift + ;; + --snapshot) + TYPE="snapshot" + SNAPNAME="$2" + shift + shift + ;; + + -s|--symstore) + SYMSTORE="$2" + shift + shift + ;; + + --llvm_arm32) + LLVM_ARM32="$(readlink -f "$2")" + shift + shift + ;; + --llvm_arm64) + LLVM_ARM64="$(readlink -f "$2")" + shift + shift + ;; + + --errormail) + ERRORMAIL="$2" + shift + shift + ;; + + --skipcompile) + SKIPCOMPILE="yes" + shift + ;; + + -h|-?|-help|--help) + usage; + exit 0; + ;; + + *) # unknown option + echo "Unknown option '$1'"; + echo; + usage; + exit 1; + ;; + esac +done + +if [[ "$TYPE" != "official" ]] && [[ "$TYPE" != "snapshot" ]]; then + echo "Must specify either --official or --snapshot as build type."; + echo; + usage; + exit 1; +fi + +if [[ "$TYPE" == "snapshot" ]] && [[ "$SNAPNAME" == "" ]]; then + echo "Must give name for snapshot."; + echo; + usage; + exit 1; +fi + +echo "Building $TYPE"; + +if [[ "$SYMSTORE" != "" ]]; then + echo "Storing symbols in $SYMSTORE"; + export SYMSTORE; +fi + +export ERRORMAIL; + +echo "Using ARM32 LLVM from '$LLVM_ARM32' and ARM64 from '$LLVM_ARM64'" + +export LLVM_ARM32; +export LLVM_ARM64; + +# Ensure we're in the root directory where this script is +export BUILD_ROOT=$(dirname $(readlink -f $0)) + +cd "${BUILD_ROOT}" + +export REPO_ROOT=$(readlink -f "$(pwd)/../..") + +if [ ! -f "${REPO_ROOT}"/renderdoc.sln ]; then + echo "Script misconfiguration - expected root of repository in '$REPO_ROOT'"; + exit 1; +fi + +echo "Repository root $REPO_ROOT"; + +if [[ "$TYPE" == "official" ]]; then + + sed -i.bak "s%RENDERDOC_OFFICIAL_BUILD 0%RENDERDOC_OFFICIAL_BUILD 1%" "${REPO_ROOT}"/renderdoc/api/replay/version.h + sed -i.bak "s%RENDERDOC_STABLE_BUILD 0%RENDERDOC_STABLE_BUILD 1%" "${REPO_ROOT}"/renderdoc/api/replay/version.h + + export GITTAG=v$(egrep "#define RENDERDOC_VERSION_(MAJOR|MINOR)" "${REPO_ROOT}"/renderdoc/api/replay/version.h | tr -dc '[0-9\n]' | tr '\n' '.' | egrep -o '[0-9]+\.[0-9]+') + +else # snapshot + + export GITTAG=$(cd "${REPO_ROOT}" && git rev-parse HEAD) + +fi; + +cd "${BUILD_ROOT}" + +if [[ "$SKIPCOMPILE" == "" ]]; then + + ./scripts/compile.sh + +fi + +if [ $? -ne 0 ]; then + exit 1; +fi + +cd "${BUILD_ROOT}" + +# Do some windows-specific build steps, like binary signing and symbol server processing +if [ "$(uname)" != "Linux" ]; then + + cd "${BUILD_ROOT}" + + ./scripts/sign_files.sh + + cd "${BUILD_ROOT}" + + ./scripts/prepare_symbols.sh + +fi + +cd "${BUILD_ROOT}" + +if [[ "$TYPE" == "official" ]]; then + + FILENAME=RenderDoc_$(echo $GITTAG | tr -d 'v') + +else # snapshot + + FILENAME=RenderDoc_${SNAPNAME} + +fi + +./scripts/make_package.sh "${FILENAME}" + diff --git a/util/buildscripts/scripts/compile.sh b/util/buildscripts/scripts/compile.sh new file mode 100644 index 000000000..cbe425986 --- /dev/null +++ b/util/buildscripts/scripts/compile.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +## Compilation is platform-specific, dispatch to helper + +if [ "$(uname)" == "Linux" ]; then + + ./scripts/compile_linux.sh + +else + + ./scripts/compile_win32.sh + +fi diff --git a/util/buildscripts/scripts/compile_docker.sh b/util/buildscripts/scripts/compile_docker.sh new file mode 100644 index 000000000..dc9a63737 --- /dev/null +++ b/util/buildscripts/scripts/compile_docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e +set -x + +rm -rf /io/* + +cd / +mkdir renderdoc_build +cd renderdoc_build +CC=clang CXX=clang++ CFLAGS="-fPIC -fvisibility=hidden" LDFLAGS="-static-libstdc++" cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/io/dist/ -DVULKAN_LAYER_FOLDER=/io/dist/etc/vulkan/implicit_layer.d -DSTATIC_QRENDERDOC=ON -DQRENDERDOC_NO_CXX11_REGEX=ON /renderdoc +make -j8 +make install + +# Copy python modules +mkdir /io/pymodules +cp -R bin/*.so /io/pymodules + +# Copy python lib folder, and trim +mkdir -p /io/dist/share/renderdoc/pylibs/lib +cd /io/dist/share/renderdoc/pylibs/lib +cp -R /usr/lib/python3.6/ . +cd python3.6 +# remove cache files +rm -rf $(find -iname __pycache__) +# remove unwanted modules +rm -rf test site-packages ensurepip distutils idlelib config-* diff --git a/util/buildscripts/scripts/compile_linux.sh b/util/buildscripts/scripts/compile_linux.sh new file mode 100644 index 000000000..51463d4e4 --- /dev/null +++ b/util/buildscripts/scripts/compile_linux.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +rm -f /tmp/compile_errors /tmp/error_email + +# Store the path to the error-mail script +ERROR_SCRIPT=$(readlink -f "${BUILD_ROOT}"/scripts/errormail.sh) + +# Ensure the docker image is prepared +pushd "${BUILD_ROOT}"/scripts/docker +docker build -t renderdoc-build . +popd + +# Run the docker compilation script inside the container above to build the main renderdoc project +mkdir -p /tmp/rdoc_docker +cp "${BUILD_ROOT}"/scripts/compile_docker.sh /tmp/rdoc_docker +docker run --rm -v /tmp/rdoc_docker:/io -v $(readlink -f "${REPO_ROOT}"):/renderdoc:ro renderdoc-build bash /io/compile_docker.sh 2>&1 | tee /tmp/compile_errors + +if [ -d /tmp/rdoc_docker/dist ]; then + echo "No error."; +else + echo "Error encountered."; + iconv -f UTF-8 -t ASCII//translit /tmp/compile_errors > /tmp/error_email + $ERROR_SCRIPT /tmp/error_email + exit 1; +fi + +# pushd into the git checkout +pushd "${REPO_ROOT}" + +# Copy the dist folder structure to the git checkout +cp -R /tmp/rdoc_docker/dist . + +# TODO - here we could copy off the build with symbols? + +# Strip the binaries +strip --strip-unneeded dist/bin/* +strip --strip-unneeded dist/lib/* + +# Copy python modules to where they'd be built natively, for documentation build +mkdir build +cp -R /tmp/rdoc_docker/pymodules build/bin + +# Step into the docs folder and build +pushd docs +make clean +make html > /tmp/sphinx.log + +if [ $? -ne 0 ]; then + $ERROR_SCRIPT /tmp/sphinx.log + exit 1; +fi + +popd; # docs + +# if we didn't produce an html file, bail out even if sphinx didn't return an error code above +if [ ! -f ./Documentation/html/index.html ]; then + echo >> /tmp/sphinx.log + echo "Didn't auto-build html docs." >> /tmp/sphinx.log + $ERROR_SCRIPT /tmp/sphinx.log + exit 1; +fi + +# Build android libraries and apks +export PATH=$PATH:$ANDROID_SDK/tools/ + +# Check that we're set up to build for android +if [ ! -d $ANDROID_SDK/tools ] ; then + echo "\$ANDROID_SDK is not correctly configured: '$ANDROID_SDK'" >> /tmp/android.log + $ERROR_SCRIPT /tmp/android.log + # Don't return an error code, consider android errors non-fatal other than emailing + exit 0; +fi + +if [ ! -d $LLVM_ARM32 ] || [ ! -d $LLVM_ARM64 ] ; then + echo "llvm is not available, expected $LLVM_ARM32 and $LLVM_ARM64 respectively." >> /tmp/android.log + $ERROR_SCRIPT /tmp/android.log + # Don't return an error code, consider android errors non-fatal other than emailing + exit 0; +fi + +# Build the arm32 variant +mkdir build-android-arm32 +pushd build-android-arm32 + +cmake -DBUILD_ANDROID=1 -DANDROID_ABI=armeabi-v7a -DANDROID_NATIVE_API_LEVEL=23 -DCMAKE_BUILD_TYPE=Release -DSTRIP_ANDROID_LIBRARY=On -DLLVM_DIR=$LLVM_ARM32/lib/cmake/llvm -DUSE_INTERCEPTOR_LIB=On -DCMAKE_MAKE_PROGRAM=make .. +make -j8 + +if ! ls bin/*.apk; then + echo >> /tmp/cmake.log + echo "Failed to build android?" >> /tmp/cmake.log + $ERROR_SCRIPT /tmp/cmake.log +fi + +popd # build-android-arm32 + +mkdir build-android-arm64 +pushd build-android-arm64 + +cmake -DBUILD_ANDROID=1 -DANDROID_ABI=arm64-v8a -DANDROID_NATIVE_API_LEVEL=23 -DCMAKE_BUILD_TYPE=Release -DSTRIP_ANDROID_LIBRARY=On -DLLVM_DIR=$LLVM_ARM64/lib/cmake/llvm -DUSE_INTERCEPTOR_LIB=On -DCMAKE_MAKE_PROGRAM=make .. +make -j8 + +if ! ls bin/*.apk; then + echo >> /tmp/cmake.log + echo "Failed to build android?" >> /tmp/cmake.log + $ERROR_SCRIPT /tmp/cmake.log +fi + +popd # build-android-arm64 + +popd # $REPO_ROOT diff --git a/util/buildscripts/scripts/compile_win32.sh b/util/buildscripts/scripts/compile_win32.sh new file mode 100644 index 000000000..f6727494e --- /dev/null +++ b/util/buildscripts/scripts/compile_win32.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Get the logfile as a windows path +LOGFILE=$(cd /tmp && pwd -W)/MSBuild.log + +# Store the path to the error-mail script +ERROR_SCRIPT=$(readlink -f "${BUILD_ROOT}"/scripts/errormail.sh) + +# pushd into the git checkout +pushd "${REPO_ROOT}" + +# Build 32-bit Release +msbuild.exe //nologo //fl4 /flp4':Verbosity=minimal;Encoding=ASCII;logfile='$LOGFILE renderdoc.sln //t:Rebuild //p:'Configuration=Release;Platform=x86' +if [ $? -ne 0 ]; then + $ERROR_SCRIPT /tmp/MSbuild.log + exit 1; +fi + +# Build 64-bit Release +msbuild.exe //nologo //fl4 /flp4':Verbosity=minimal;Encoding=ASCII;logfile='$LOGFILE renderdoc.sln //t:Rebuild //p:'Configuration=Release;Platform=x64' +if [ $? -ne 0 ]; then + $ERROR_SCRIPT /tmp/MSbuild.log + exit 1; +fi + +# Step into the docs folder and build +pushd docs +make clean +make htmlhelp > /tmp/sphinx.log + +if [ $? -ne 0 ]; then + $ERROR_SCRIPT /tmp/sphinx.log + exit 1; +fi + +popd; # docs + +# if we didn't produce a chm file, bail out even if sphinx didn't return an error code above +if [ ! -f ./Documentation/htmlhelp/renderdoc.chm ]; then + echo >> /tmp/sphinx.log + echo "Didn't auto-build chm file. Missing HTML Help Workshop?" >> /tmp/sphinx.log + $ERROR_SCRIPT /tmp/sphinx.log + exit 1; +fi + +export PATH=$PATH:$(cygpath -u $ANDROID_SDK/tools/) + +# Check that we're set up to build for android +if [ ! -d $ANDROID_SDK/tools ] ; then + echo "\$ANDROID_SDK is not correctly configured: '$ANDROID_SDK'" >> /tmp/android.log + $ERROR_SCRIPT /tmp/android.log + # Don't return an error code, consider android errors non-fatal other than emailing + exit 0; +fi + +if [ ! -d $LLVM_ARM32 ] || [ ! -d $LLVM_ARM64 ] ; then + echo "llvm is not available, expected $LLVM_ARM32 and $LLVM_ARM64 respectively." >> /tmp/android.log + $ERROR_SCRIPT /tmp/android.log + # Don't return an error code, consider android errors non-fatal other than emailing + exit 0; +fi + +# Build the arm32 variant +mkdir build-android-arm32 +pushd build-android-arm32 + +cmake -G "MSYS Makefiles" -DBUILD_ANDROID=1 -DANDROID_ABI=armeabi-v7a -DCMAKE_BUILD_TYPE=Release -DSTRIP_ANDROID_LIBRARY=On -DLLVM_DIR=$LLVM_ARM32/lib/cmake/llvm -DUSE_INTERCEPTOR_LIB=On .. 2>&1 | tee /tmp/cmake.log +make -j8 2>&1 | tee -a /tmp/cmake.log + +if ! ls bin/*.apk; then + echo >> /tmp/cmake.log + echo "Failed to build android?" >> /tmp/cmake.log + $ERROR_SCRIPT /tmp/cmake.log +fi + +popd # build-android-arm32 + +mkdir build-android-arm64 +pushd build-android-arm64 + +cmake -G "MSYS Makefiles" -DBUILD_ANDROID=1 -DANDROID_ABI=arm64-v8a -DCMAKE_BUILD_TYPE=Release -DSTRIP_ANDROID_LIBRARY=On -DLLVM_DIR=$LLVM_ARM64/lib/cmake/llvm -DUSE_INTERCEPTOR_LIB=On .. | tee /tmp/cmake.log +make -j8 2>&1 | tee -a /tmp/cmake.log + +if ! ls bin/*.apk; then + echo >> /tmp/cmake.log + echo "Failed to build android?" >> /tmp/cmake.log + $ERROR_SCRIPT /tmp/cmake.log +fi + +popd # build-android-arm64 + +popd # $REPO_ROOT + diff --git a/util/buildscripts/scripts/docker/Dockerfile b/util/buildscripts/scripts/docker/Dockerfile new file mode 100644 index 000000000..75d64cbaa --- /dev/null +++ b/util/buildscripts/scripts/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:precise +LABEL maintainer baldurk@baldurk.org +COPY prepare.sh / +COPY static_tagging.patch / +RUN /prepare.sh diff --git a/util/buildscripts/scripts/docker/README.md b/util/buildscripts/scripts/docker/README.md new file mode 100644 index 000000000..55170db0b --- /dev/null +++ b/util/buildscripts/scripts/docker/README.md @@ -0,0 +1,3 @@ +This folder creates a docker container pre-configured for compiling RenderDoc with the oldest possible distribution, for maximum compatibility of the binary build. + +It doesn't contain anything RenderDoc-specific, apart from being tailored to its dependencies. diff --git a/util/buildscripts/scripts/docker/prepare.sh b/util/buildscripts/scripts/docker/prepare.sh new file mode 100644 index 000000000..bf99926c2 --- /dev/null +++ b/util/buildscripts/scripts/docker/prepare.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -e +set -x + +# initial update +apt-get update + +# for add-apt-repository +apt-get install -y software-properties-common python-software-properties wget + +# for newer libstdc++ +add-apt-repository -y ppa:ubuntu-toolchain-r/test +# for clang +add-apt-repository 'deb http://apt.llvm.org/precise/ llvm-toolchain-precise-3.8 main' +wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +apt-get update + +# install dependencies +apt-get install --force-yes -y libx11-dev libx11-xcb-dev mesa-common-dev libgl1-mesa-dev gcc g++ clang-3.8 clang++-3.8 make pkg-config git libcurl4-openssl-dev libpcre3-dev + +# install dependencies for building qt +apt-get install --force-yes -y libproxy-dev autoconf autogen libtool xutils-dev bison + +update-alternatives --install /usr/bin/clang clang /usr/bin/clang-3.8 380 +update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-3.8 380 + +# build xcb-keysyms for static linking +wget https://xcb.freedesktop.org/dist/xcb-util-keysyms-0.4.0.tar.gz +tar xzf xcb-util-keysyms-0.4.0.tar.gz +cd xcb-util-keysyms-0.4.0/ +CFLAGS="-fPIC -fvisibility=hidden" ./configure --prefix=/usr --disable-shared --enable-static +make +make install +cd .. + +# xcb-proto +wget http://xcb.freedesktop.org/dist/xcb-proto-1.10.tar.gz +tar -xf xcb-proto-1.10.tar.gz +cd xcb-proto-1.10 +CFLAGS="-fPIC -fvisibility=hidden" ./configure --prefix=/usr +make +make install +cd .. + +# libxcb +wget https://xcb.freedesktop.org/dist/libxcb-1.10.tar.gz +tar -xf libxcb-1.10.tar.gz +cd libxcb-1.10 +CFLAGS="-fPIC -fvisibility=hidden" ./configure --prefix=/usr +make +make install +cd .. + +# libxkbcommon +wget https://github.com/xkbcommon/libxkbcommon/archive/xkbcommon-0.7.0.tar.gz +tar -xf xkbcommon-0.7.0.tar.gz +cd libxkbcommon-xkbcommon-0.7.0 +CFLAGS="-fPIC -fvisibility=hidden" ./autogen.sh --disable-shared --prefix=/usr --enable-static +make +make install +cd .. + +# libfontconfig static linking seems to break, so dynamic link against libfreetype and libfontconfig +apt-get install --force-yes -y libfontconfig1-dev + +# build qt for static linking +wget http://download.qt.io/archive/qt/5.6/5.6.2/single/qt-everywhere-opensource-src-5.6.2.tar.xz +tar -xf qt-everywhere-opensource-src-5.6.2.tar.xz +cd qt-everywhere-opensource-src-5.6.2 + +# Fix for linking static qt into a shared library: +# https://bugreports.qt.io/browse/QTBUG-52605 +# https://codereview.qt-project.org/171007 +cd qtbase +git apply < /static_tagging.patch +cd .. + +./configure -prefix /usr -release -opensource -confirm-license -static -platform linux-clang -no-qml-debug -qt-zlib -no-mtdev -no-journald -no-syslog -qt-libpng -qt-libjpeg -system-xkbcommon-x11 -fontconfig -no-icu -qt-harfbuzz -openssl -libproxy -qt-pcre -qt-xcb -no-xinput2 -no-pulseaudio -no-alsa -v -no-cups -no-linuxfb -no-opengl -no-gstreamer -no-libinput -no-sse3 -no-ssse3 -no-sse4.1 -no-sse4.2 -no-avx -no-avx2 -skip qtdeclarative -skip qtsensors -skip qtconnectivity -skip qtmultimedia -skip qtscript -skip qtserialbus -skip qtserialport -skip qtcanvas3d -skip qtenginio -skip qtlocation -skip qtgraphicaleffects -skip qtxmlpatterns -skip qtwebview -skip qt3d -skip qttools -nomake examples -nomake tools -nomake tests +make -j4 +make install +cd .. + +# build cmake locally, to get ssl support for external projects +wget http://www.cmake.org/files/v3.2/cmake-3.2.2.tar.gz +tar xf cmake-3.2.2.tar.gz +cd cmake-3.2.2 +./configure --prefix=/usr --system-curl --parallel=4 +make -j4 +make install +cd .. + +# build python locally to static link against +wget https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tgz +tar xf Python-3.6.1.tgz +cd Python-3.6.1 +./configure --prefix=/usr +make -j4 +make install +cd .. + diff --git a/util/buildscripts/scripts/docker/static_tagging.patch b/util/buildscripts/scripts/docker/static_tagging.patch new file mode 100644 index 000000000..857a714e7 --- /dev/null +++ b/util/buildscripts/scripts/docker/static_tagging.patch @@ -0,0 +1,13 @@ +diff --git a/src/corelib/global/qversiontagging.h b/src/corelib/global/qversiontagging.h +index fa824d1..b46bb7a 100644 (file) +--- a/src/corelib/global/qversiontagging.h ++++ b/src/corelib/global/qversiontagging.h +@@ -54,7 +54,7 @@ QT_BEGIN_NAMESPACE + * There will only be one copy of the section in the output library or application. + */ + +-#if defined(QT_BUILD_CORE_LIB) || defined(QT_BOOTSTRAPPED) || defined(QT_NO_VERSION_TAGGING) ++#if defined(QT_BUILD_CORE_LIB) || defined(QT_BOOTSTRAPPED) || defined(QT_NO_VERSION_TAGGING) || defined(QT_STATIC) + // don't make tags in QtCore, bootstrapped systems or if the user asked not to + #elif defined(Q_CC_GNU) && !defined(Q_OS_ANDROID) + # if defined(Q_PROCESSOR_X86) && (defined(Q_OS_LINUX) || defined(Q_OS_FREEBSD_KERNEL)) diff --git a/util/buildscripts/scripts/errormail.sh b/util/buildscripts/scripts/errormail.sh new file mode 100644 index 000000000..0cad0a164 --- /dev/null +++ b/util/buildscripts/scripts/errormail.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +EMAIL_BODY_FILE=$1 + +PLATFORM=Windows + +if [[ "$ERRORMAIL" == "" ]]; then + exit 0; +fi + +if [ "$(uname)" == "Linux" ]; then + PLATFORM=Linux +fi + +SUBJECT="[renderdoc] Compile error on $PLATFORM" + +EMAILHOST="${BUILD_ROOT}"/support/emailhost + +if [ -f "${EMAILHOST}" ]; then + ssh $(cat "${EMAILHOST}") "mail -s \"$SUBJECT\" $ERRORMAIL" < $EMAIL_BODY_FILE + exit; +else + mail -s $SUBJECT $ERRORMAIL < $EMAIL_BODY_FILE +fi diff --git a/util/buildscripts/scripts/make_package.sh b/util/buildscripts/scripts/make_package.sh new file mode 100644 index 000000000..beb57967c --- /dev/null +++ b/util/buildscripts/scripts/make_package.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +## Packaging is platform-specific, dispatch to helper + +if [ "$(uname)" == "Linux" ]; then + + ./scripts/make_package_linux.sh $1 + +else + + ./scripts/make_package_win32.sh $1 + +fi diff --git a/util/buildscripts/scripts/make_package_linux.sh b/util/buildscripts/scripts/make_package_linux.sh new file mode 100644 index 000000000..2da260b60 --- /dev/null +++ b/util/buildscripts/scripts/make_package_linux.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +FILENAME="$1" + +if [ $# -ne 1 ]; then + echo "Usage: $0 FILENAME"; + exit 1; +fi + +if [ ! -d "${REPO_ROOT}"/dist/bin ]; then + echo "Expected 'dist' folder in renderdoc. When building, set CMAKE_INSTALL_PREFIX to dist and make install"; + exit 1; +fi + +FILENAME="$(echo $FILENAME | tr A-Z a-z)"; + +rm -rf "${REPO_ROOT}"/package +mkdir "${REPO_ROOT}"/package +pushd "${REPO_ROOT}"/package + +mkdir -p ${FILENAME}/ + +# copy in the target directory tree, cmake will have done most of our packaging for us +cp -R "${REPO_ROOT}"/dist/* "./${FILENAME}/" + +# copy readme and license to the package root +cp "./${FILENAME}"/share/doc/renderdoc/{LICENSE.md,README} "./${FILENAME}" + +# copy in html documentation +if [ -d "${REPO_ROOT}"/Documentation/html ]; then + cp -R "${REPO_ROOT}"/Documentation/html "./${FILENAME}/share/doc/renderdoc/html" +else + echo "WARNING: Documentation not built! run 'make html' in docs folder"; +fi + +# copy in plugins +if [ -d "${REPO_ROOT}"/plugins-linux64 ]; then + cp -R "${REPO_ROOT}"/plugins-linux64 "./${FILENAME}/share/renderdoc/plugins" + chmod +x -R "./${FILENAME}/share/renderdoc/plugins"/* +else + echo "WARNING: Plugins not present. Download and extract https://renderdoc.org/plugins.tgz in root folder"; +fi + +# copy in all of the android files. +mkdir -p "./${FILENAME}/share/renderdoc/plugins/android/" + +if ls "${REPO_ROOT}"/build-android*/bin/*.apk; then + cp "${REPO_ROOT}"/build-android*/bin/*.apk "./${FILENAME}/share/renderdoc/plugins/android/" +else + echo "WARNING: Android build not present. Build arm32 and arm64 apks in build-android-arm{32,64} folders"; +fi + +# compress the folder and GPG sign it +tar -zcf ${FILENAME}.tar.gz "./${FILENAME}/"* --xform 's#^./##g' +gpg -o ${FILENAME}.tar.gz.sig --detach-sign --armor ${FILENAME}.tar.gz + +rm -rf "./${FILENAME}" + +popd # $REPO_ROOT/package diff --git a/util/dist.sh b/util/buildscripts/scripts/make_package_win32.sh similarity index 62% rename from util/dist.sh rename to util/buildscripts/scripts/make_package_win32.sh index 26e9015eb..d27e854c7 100644 --- a/util/dist.sh +++ b/util/buildscripts/scripts/make_package_win32.sh @@ -1,26 +1,19 @@ #!/bin/bash -AUTOBUILD=1 +FILENAME="$1" -if [ ! -f LICENSE.md ]; then - echo "This script should be run from the root of the checkout."; - echo "e.g. ./util/dist.sh"; +if [ $# -ne 1 ]; then + echo "Usage: $0 FILENAME"; exit; fi -if [ $# -ne 1 ] || [ $1 != "autobuild" ]; then - AUTOBUILD=0 - echo "=== Building standalone folders. Hit enter when each prompt is satisfied" - - echo "Have you rebuilt the documentation? (cd docs/ && ./make.sh htmlhelp)" - read; - - echo "Have you built 32-bit and 64-bit Release builds?" - read; - - echo "=== Building folders" +if [ ! -d "${REPO_ROOT}"/x64/Release ] || [ ! -d "${REPO_ROOT}"/Win32/Release ]; then + echo "ERROR: Missing one of Win32 or x64 Release builds"; + exit 1; fi +pushd "${REPO_ROOT}" + # clean any old files lying around and make new structure rm -rf dist mkdir -p dist/Release{32,64} @@ -43,8 +36,17 @@ cp LICENSE.md Documentation/htmlhelp/*.chm dist/Release64/ cp LICENSE.md Documentation/htmlhelp/*.chm dist/Release32/ # Copy in appropriate plugins folder if they exist -cp -R plugins-win64/ dist/Release64/plugins -cp -R plugins-win32/ dist/Release32/plugins +if [ -d plugins-win64 ]; then + cp -R plugins-win64/ dist/Release64/plugins +else + echo "WARNING: x64 plugins missing, download and extract https://renderdoc.org/plugins.zip in root"; +fi; + +if [ -d plugins-win32 ]; then + cp -R plugins-win32/ dist/Release32/plugins +else + echo "WARNING: x86 plugins missing, download and extract https://renderdoc.org/plugins.zip in root"; +fi # Delete new VS2015 incremental pdb files, these are just build artifacts # and aren't needed for later symbol resolution etc @@ -53,16 +55,20 @@ find dist/Release{32,64}/ -iname '*.iobj' -exec rm '{}' \; # Copy in any android APKs that were built mkdir -p dist/Release64/plugins/android/ -find build-android-* -iname 'org.renderdoc.renderdoccmd.*.apk' -exec cp '{}' dist/Release64/plugins/android ';' +if ls build-android-* > /dev/null; then + find build-android-* -iname 'org.renderdoc.renderdoccmd.*.apk' -exec cp '{}' dist/Release64/plugins/android ';' +else + echo "WARNING: No android builds found, expected build-android-arm32 and build-android-arm64"; +fi # Copy in android adb and patching requirements if [ -f plugins-android/adb.exe ]; then cp -R plugins-android/* dist/Release64/plugins/android/; else - echo "ERROR: Couldn't find android dependency (adb.exe and friends)"; - echo " These files must be in plugins-android/ in the root and"; - echo " MUST be built locally from an AOSP checkout, not from a"; - echo " distributed android SDK due to licensing concerns."; + echo "WARNING: Couldn't find android dependency (adb.exe and friends)"; + echo " These files must be in plugins-android/ in the root and"; + echo " MUST be built locally from an AOSP checkout, not from a"; + echo " distributed android SDK due to licensing concerns."; fi # Generate a debug key for signing purposes @@ -93,13 +99,6 @@ cp -R dist/Release32/{d3dcompiler_47.dll,renderdoc.dll,renderdoc.json,renderdocs mkdir -p dist/ReleasePDBs64/x86 cp -R dist/ReleasePDBs32/{d3dcompiler_47.dll,renderdoc.dll,renderdoc.json,renderdoc.pdb,renderdocshim32.dll,renderdocshim32.pdb,renderdoccmd.exe,renderdoccmd.pdb,dbghelp.dll,symsrv.dll,symsrv.yes} dist/ReleasePDBs64/x86/ -if [[ $AUTOBUILD -eq 0 ]]; then - echo "=== Folders built. Ready to make installer MSIs." - echo - echo "Hit enter when ready" - read; -fi - VERSION=`grep -E "#define RENDERDOC_VERSION_(MAJOR|MINOR)" renderdoc/api/replay/version.h | tr -dc '[0-9\n]' | tr '\n' '.' | grep -Eo '[0-9]+\.[0-9]+'` export RENDERDOC_VERSION="${VERSION}" @@ -110,4 +109,37 @@ export RENDERDOC_VERSION="${VERSION}" "$WIX/bin/candle.exe" -o dist/Installer64.wixobj util/installer/Installer64.wxs "$WIX/bin/light.exe" -ext WixUIExtension -sw1076 -loc util/installer/customtext.wxl -o dist/Installer64.msi dist/Installer64.wixobj -rm dist/*.wixobj dist/*.wixpdb +rm -f dist/*.wixobj dist/*.wixpdb + +popd # $REPO_ROOT + +rm -rf "${REPO_ROOT}"/package +mkdir "${REPO_ROOT}"/package +pushd "${REPO_ROOT}"/package + +cp -R "${REPO_ROOT}"/dist/Release32 ${FILENAME}_32 +cp -R "${REPO_ROOT}"/dist/Release64 ${FILENAME}_64 +zip -r ${FILENAME}_32.zip ${FILENAME}_32/ +zip -r ${FILENAME}_64.zip ${FILENAME}_64/ +gpg -o ${FILENAME}_32.zip.sig --detach-sign --armor ${FILENAME}_32.zip +gpg -o ${FILENAME}_64.zip.sig --detach-sign --armor ${FILENAME}_64.zip + +if [ -f "${REPO_ROOT}"/dist/Installer32.msi ]; then + + cp "${REPO_ROOT}"/dist/Installer32.msi ${FILENAME}_32.msi + cp "${REPO_ROOT}"/dist/Installer64.msi ${FILENAME}_64.msi + + gpg -o ${FILENAME}_32.msi.sig --detach-sign --armor ${FILENAME}_32.msi + gpg -o ${FILENAME}_64.msi.sig --detach-sign --armor ${FILENAME}_64.msi + + # On windows, also sign the installers + if [ "$(uname)" != "Linux" ]; then + "${BUILD_ROOT}"/scripts/sign.sh ${FILENAME}_32.msi + "${BUILD_ROOT}"/scripts/sign.sh ${FILENAME}_64.msi + fi + +fi; + +rm -rf ${FILENAME}_32/ ${FILENAME}_64/ + +popd # $REPO_ROOT/package diff --git a/util/buildscripts/scripts/prepare_symbols.sh b/util/buildscripts/scripts/prepare_symbols.sh new file mode 100644 index 000000000..cd1d06dd2 --- /dev/null +++ b/util/buildscripts/scripts/prepare_symbols.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# We don't have symbol servers on linux +if [ "$(uname)" == "Linux" ]; then + exit; +fi + +if [[ "$GITTAG" == "" ]]; then + echo "Git tag is not valid."; + exit 0; +fi + +if [ ! -d "${REPO_ROOT}"/Win32/Release ] || [ ! -d "${REPO_ROOT}"/x64/Release ]; then + echo "WARNING: No compiled binaries found in release folders." + exit 0 +fi + +if [ ! -f "${BUILD_ROOT}"/support/pdbstr.exe ]; then + echo "WARNING: Need pdbstr.exe from Windows Debugger folder in build root to set up source server information in symbol files." + echo "e.g. C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\srcsrv\pdbstr.exe"; + exit 0 +fi + +########################################################## +# Create a source-mapping file to embed into PDBs + +cat << EOF > /tmp/pdbstr.txt +SRCSRV: ini ------------------------------------------------ +VERSION=2 +VERCTRL=http +SRCSRV: variables ------------------------------------------ +HTTP_ALIAS=https://raw.githubusercontent.com/baldurk/renderdoc/$GITTAG/ +HTTP_EXTRACT_TARGET=%HTTP_ALIAS%%var2% +SRCSRVTRG=%HTTP_EXTRACT_TARGET% +SRCSRV: source files --------------------------------------- +EOF + +for I in $(find "${REPO_ROOT}" \( -path '*/3rdparty' -o -path '*/build-android*' -o -path '*/generated' \) -prune -o \( -iname '*.cpp' -o -iname '*.c' -o -iname '*.h' -o -iname '*.inl' \) -print); do + echo ${I}*${I}; +done | + sed -e '{s#\*'"${REPO_ROOT}"'/\?#\*#g}' | + sed -e '{s#^/\(.\)/#\1:/#g}' | + awk -F"*" '{gsub("/","\\",$1); print $1 "*" $2}' >> /tmp/pdbstr.txt + +echo "SRCSRV: end ------------------------------------------------" >> /tmp/pdbstr.txt + +########################################################## + +# Apply the source-indexing mapping into every pdb file +for PDB in "${REPO_ROOT}"/Win32/Release/*.pdb "${REPO_ROOT}"/x64/Release/*.pdb; do + "${BUILD_ROOT}"/support/pdbstr.exe -w -p:$PDB -s:srcsrv -i:/tmp/pdbstr.txt +done + +if [ ! -f "${BUILD_ROOT}"/support/symstore.exe ]; then + echo "Need symstore.exe from Windows Debugger folder in build root." + echo "e.g. C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\symstore.exe"; + exit 0 +fi + +# check if a symbol store is specified +if [[ "$SYMSTORE" == "" ]]; then + exit 0; +fi + +SYMSTORE=$(readlink -f "${SYMSTORE}") + +echo "Storing symbols for $GITTAG in symbol store $SYMSTORE" + +mkdir -p "${SYMSTORE}" + +# Store the pdbs in a symbol store +# Process Win32 and x64 separately since they'll overwrite each other otherwise. +for ARCH in Win32 x64; do + + rm -rf /tmp/symstore + + # Add to local symbol store (which is rsync'd to public symbol server) + for PDB in "${REPO_ROOT}"/$ARCH/Release/*.pdb; do + mkdir -p /tmp/symstore + cp $PDB /tmp/symstore/ + + EXE=${PDB%.pdb}.exe + DLL=${PDB%.pdb}.dll + if [ -f "$EXE" ]; then + cp $EXE /tmp/symstore/ + fi + if [ -f "$DLL" ]; then + cp $DLL /tmp/symstore/ + fi + done + + if [ -d /tmp/symstore ]; then + "${BUILD_ROOT}"/support/symstore.exe add //s "${SYMSTORE}" //compress //r //f /tmp/symstore //t RenderDoc //v $GITTAG + fi + +done + +rm -rf /tmp/symstore diff --git a/util/buildscripts/scripts/sign.sh b/util/buildscripts/scripts/sign.sh new file mode 100644 index 000000000..85a763eb1 --- /dev/null +++ b/util/buildscripts/scripts/sign.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Script to sign a file using the key.pfx certificate +if [ $# -ne 1 ] ; then + echo Usage: $0 file + exit 1 +fi + +# Make sure the file exists +if [ ! -f $1 ] ; then + echo File $1 does not exist + exit 1 +fi + +if [ ! -f "${BUILD_ROOT}"/support/key.pass ] || [ ! -f "${BUILD_ROOT}"/support/key.pfx ] ; then + echo Key key.pfx / key.pass does not exist + exit 1 +fi + +PASS=$(cat "${BUILD_ROOT}"/support/key.pass) + +# First check to see if it is already signed. +# An exit value of 1 from signtool indicates it is not signed. +signtool verify //pa $1 >/dev/null 2>&1 +if [ $? -eq 1 ] ; then + + # This is the list of timestamp servers to try. + # Sometime the signing operation fails because we can't contact the + # timestamp server, so we try several servers. + TSSLIST=( + http://timestamp.comodoca.com/rfc3161 + http://timestamp.digicert.com + http://tsa.starfieldtech.com + http://timestamp.geotrust.com/tsa) + + TSS=${TSSLIST[0]} + echo Signing $1 using timestamp server $TSS ... + sleep 1 + signtool sign //f "${BUILD_ROOT}"/support/key.pfx //fd sha256 //p $PASS //tr $TSS //td sha256 $1 + if [ $? -eq 0 ] ; then + # Successfully signed, return success + exit 0 + fi + + for RETRY in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ; do + + # Sometimes signtool returns failure, but the file was already signed. + # Not sure why that happens. Since the file is now signed, return successs. + sleep 1 + signtool verify //pa $1 >/dev/null 2>&1 + if [ $? -eq 0 ] ; then + echo Signing returned failure, but file was signed. Returning success. + exit 0 + fi + + # Retry with a different timestamp server. + TSS=${TSSLIST[`expr $RETRY % ${#TSSLIST[@]}`]} + echo Signing failed, retry $RETRY. Using timestamp server $TSS ... + sleep 4 + echo Retrying signing of $1 + signtool sign //f "${BUILD_ROOT}"/support/key.pfx //p $PASS //tr $TSS $1 + if [ $? -eq 0 ] ; then + # Successfully signed, return success + exit 0 + fi + done + # We didn't sign the file succesfully + exit 1 +else + echo Signing of $1 skipped, already signed... + exit 0 +fi diff --git a/util/buildscripts/scripts/sign_files.sh b/util/buildscripts/scripts/sign_files.sh new file mode 100644 index 000000000..17a343acc --- /dev/null +++ b/util/buildscripts/scripts/sign_files.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# We don't sign files on linux +if [ "$(uname)" == "Linux" ]; then + exit; +fi + +# Skip silently if no key is present +if [ ! -f "${BUILD_ROOT}"/support/key.pass ] || [ ! -f "${BUILD_ROOT}"/support/key.pfx ] ; then + exit; +fi + +# sign all of our files +for PDB in "${REPO_ROOT}"/Win32/Release/*.pdb "${REPO_ROOT}"/x64/Release/*.pdb; do + EXE=${PDB%.pdb}.exe + DLL=${PDB%.pdb}.dll + + if [ -f "$EXE" ]; then + bash "${BUILD_ROOT}"/scripts/sign.sh $EXE + fi + if [ -f "$DLL" ]; then + bash "${BUILD_ROOT}"/scripts/sign.sh $DLL + fi +done + diff --git a/util/buildscripts/support/README.md b/util/buildscripts/support/README.md new file mode 100644 index 000000000..582cbdc34 --- /dev/null +++ b/util/buildscripts/support/README.md @@ -0,0 +1,6 @@ +This folder contains various files not included with the build scripts, but which can be used. + +* dbghelp.dll, pdbstr.exe, symstore.exe, symsrv.dll, symsrv.yes - used for setting up source server and symbol store with built symbols. Taken from the Debuggers/x64 folder in the Windows 10 SDK +* key.pfx, key.pass - used for signing resulting binaries +* llvm_arm32, llvm_arm64 - used for building android with interceptor-lib. +* emailhost - an optional file containing an ssh-compatible user@host string, which is used to send error emails (the email is sent via running 'mail' on that remote host)