de-Archify some aspects of builds and -scripts; allow to initiate separate steps manually; first attempt to build tarball on ubuntu 16.04

This commit is contained in:
ohfp 2020-03-28 23:15:41 +01:00
parent 1402bdd5e7
commit b409944fad
11 changed files with 301 additions and 104 deletions

View file

@ -1,7 +1,20 @@
stages:
- build
- paks
# - deploy
# variables:
# GENERIC: $GENERIC
# ARCH: $ARCH
# FLATPAK: $FLATPAK
# APPIMAGE: $FLATPAK
# pkgver: $PKGVER
# pkgrel: $PKGREL
# AARCH64: $AARCH64
# X86_64: $X86_64
# download tarball if none available as artifact,
# ie. not built in the same pipeline run
# TARBALL_URL: $TARBALL_URL
# $RELEASE, to trigger deploy, maybe?
.carch_aarch64: &carch_aarch64
variables:
@ -11,18 +24,23 @@ stages:
variables:
CARCH: 'x86_64'
.main_build_config: &main_build_config
.tarball_build_config: &tarball_build_config
stage: build
script:
- ./scripts/01_arch_build.sh
- ./scripts/02_configure_tarball.sh
- ./binary_tarball/build_tarball.sh "${CI_PROJECT_DIR}/LibreWolf.${CARCH}.tar.bz2"
artifacts:
name: "Librewolf-${CI_COMMIT_TAG}-${CARCH}"
paths:
- "librewolf*pkg.tar*"
- "LibreWolf.${CARCH}.tar.bz2"
only:
- tags
.arch_build_config: &arch_build_config
stage: build
script:
- ./arch/01_build.sh
artifacts:
name: "Librewolf-Arch-${CI_COMMIT_TAG}-${CARCH}"
paths:
- "librewolf*pkg.tar*"
.flatpak_config: &flatpak_config
stage: paks
@ -31,8 +49,6 @@ stages:
paths:
- "LibreWolf.${CARCH}.flatpak"
- "${CI_PROJECT_DIR}/librewolf-${CARCH}-flatpak-repo"
only:
- tags
script:
- ./flatpak/build_flatpak.sh "${CI_PROJECT_DIR}/LibreWolf.${CARCH}.tar.bz2" "${CI_PROJECT_DIR}/librewolf-${CARCH}-flatpak-repo" "${CI_PROJECT_DIR}/LibreWolf.${CARCH}.flatpak"
@ -42,56 +58,79 @@ stages:
name: "Librewolf-${CI_COMMIT_TAG}-AppImage-${CARCH}"
paths:
- "LibreWolf.${CARCH}.AppImage"
only:
- tags
script:
- ./appimage/build_appimage.sh "${CI_PROJECT_DIR}/LibreWolf.${CARCH}.tar.bz2" "${CI_PROJECT_DIR}/LibreWolf.${CARCH}.AppImage"
build_x86_64:
arch_x86_64:
image: archlinux/base
tags: [x86_64b]
<<: *carch_x86_64
<<: *main_build_config
<<: *arch_build_config
only:
variables:
- $ARCH && $X86_64
build_aarch64:
arch_aarch64:
image: registry.gitlab.com/ohfp/manjaro-arm-docker
tags: [aarch64b]
<<: *carch_aarch64
<<: *main_build_config
<<: *arch_build_config
only:
variables:
- $ARCH && $AARCH64
tarball_x86_64:
image: ubuntu:16.04
tags: [x86_64b]
<<: *carch_x86_64
<<: *tarball_build_config
only:
variables:
- $TARBALL && $X86_64
tarball_aarch64:
image: arm64v8/ubuntu:16.04
tags: [aarch64b]
<<: *carch_aarch64
<<: *tarball_build_config
only:
variables:
- $TARBALL && AARCH64
flatpak_x86_64:
image: ubuntu:18.04
image: ubuntu:16.04
tags: [flat_runner]
<<: *carch_x86_64
<<: *flatpak_config
only:
variables:
- $FLATPAK && $X86_64
flatpak_aarch64:
image: arm64v8/ubuntu:18.04
image: arm64v8/ubuntu:16.04
tags: [flat_runner_aarch64]
<<: *carch_aarch64
<<: *flatpak_config
only:
variables:
- $FLATPAK && $AARCH64
appimage_x86_64:
image: ubuntu:18.04
image: ubuntu:16.04
tags: [x86_64b]
<<: *carch_x86_64
<<: *appimage_config
only:
variables:
- $APPIMAGE && $X86_64
appimage_aarch64:
image: arm64v8/ubuntu:18.04
image: arm64v8/ubuntu:16.04
tags:
- aarch64b
- arm64 # can also be run on any other less powerful aarch64 runner
<<: *carch_aarch64
<<: *appimage_config
# Release to Gitlab:
# stage: deploy
# image: python:latest
# script:
# - pip3 install gitlab-release
# - tar czf librewolf-aarch64-flatpak-repo.tar.gz librewolf-aarch64-flatpak-repo
# - tar czf librewolf-x86_64-flatpak-repo.tar.gz librewolf-x86_64-flatpak-repo
# - "gitlab-release librewolf*.pkg.tar.* LibreWolf.*.AppImage librewolf-*-flatpak-repo.tar.gz LibreWolf.*.flatpak"
# only:
# - tags
only:
variables:
- $APPIMAGE && $AARCH64

View file

@ -5,8 +5,11 @@
pkgname=librewolf
_pkgname=LibreWolf
pkgver=74.0
pkgrel=3
# pkgver=74.0
# pkgrel=3
# now provided as ci variables
pkgver=$pkgver
pkgrel=$pkgrel
pkgdesc="Community-maintained fork of Firefox, focused on privacy, security and freedom."
arch=(x86_64 aarch64)
license=(MPL GPL LGPL)
@ -94,7 +97,7 @@ mk_add_options MOZ_TELEMETRY_REPORTING=0
END
if [[ $CARCH == 'aarch64' ]]; then
cat >>../mozconfig <<END
cat >>../mozconfig <<END
# taken from manjaro build:
ac_add_options --enable-optimize="-g0 -O2"
export MOZ_DEBUG_FLAGS=" "
@ -131,7 +134,6 @@ fi
build() {
cd firefox-$pkgver
export MOZ_SOURCE_REPO="$_repo"
export MOZ_NOSPAM=1
export MOZBUILD_STATE_PATH="$srcdir/mozbuild"
@ -159,6 +161,7 @@ ac_add_options --enable-profile-generate=cross
END
fi
./mach build
echo "Profiling instrumented browser..."
@ -221,19 +224,13 @@ package() {
local vendorjs="$pkgdir/usr/lib/$pkgname/browser/defaults/preferences/vendor.js"
# move the following part to librewolf.cfg instead?
install -Dvm644 /dev/stdin "$vendorjs" <<END
// Use LANG environment variable to choose locale
pref("intl.locale.requested", "");
// Use system-provided dictionaries
pref("spellchecker.dictionary_path", "/usr/share/hunspell");
// Disable default browser checking.
pref("browser.shell.checkDefaultBrowser", false);
// Don't disable extensions in the application directory
pref("extensions.autoDisableScopes", 11);
// done in librewolf.cf
// pref("extensions.autoDisableScopes", 11);
END
cp -r ${srcdir}/settings/* ${pkgdir}/usr/lib/${pkgname}/
@ -243,12 +240,12 @@ END
[Global]
id=io.gitlab.${pkgname}
version=1.0
about=LibreWolf Arch Linux
about=LibreWolf
[Preferences]
app.distributor=archlinux
app.distributor="LibreWolf Community"
app.distributor.channel=$pkgname
app.partner.archlinux=archlinux
app.partner.librewolf=$pkgname
END
for i in 16 32 48 64 128; do

View file

@ -1,9 +1,4 @@
#!/bin/bash
"""
"""
printf "\n\n------------------------------------ APPIMAGE BUILD -----------------------------------------\n";
# Aborts the script upon any faliure
@ -21,7 +16,16 @@ _APPIMAGETOOL_FILE=$_SCRIPT_FOLDER/appimagetool;
_APPIMAGE_CONTENT_FOLDER=$_SCRIPT_FOLDER/content
# Installs needed dependencies
apt-get update && apt-get -y install file;
apt-get update && apt-get -y install file wget;
if [[ -z "${TARBALL_URL}" ]];then
wget "${TARBALL_URL}"
fi
if [[ ! -f "${BINARY_TARBALL}" ]];then
echo "Tarball not provided via pipeline or download."
exit 1
fi
if [[ $CARCH == 'aarch64' ]]; then
apt install -y zlib1g-dev

View file

@ -1,4 +1,5 @@
#!/bin/bash
pacman --noconfirm -Syu --needed base-devel
# this is a very ugly fix for recent makepkg-5.1-chmod-shenanigans, which mess up the build process in docker
sed -E -i 's/^chmod a-s \"\$BUILDDIR\"$/# chmod a-s \"\$BUILDDIR\"/' `which makepkg`
@ -9,4 +10,4 @@ usermod -d /home/nobody nobody
usermod -e '' nobody
chown -R nobody .
# makepkg will not run as root
sudo -u nobody -E -H makepkg --noconfirm --nosign --syncdeps --cleanbuild --skippgpcheck
sudo -u nobody -E -H makepkg --noconfirm --nosign --syncdeps --cleanbuild

View file

@ -2,7 +2,8 @@
printf "\n\n-------------------------------------- DEPENDENCY INSTALLATION ---------------------------------------------\n";
# Setup Script Variables
_DEPENDENCIES="mercurial wget git flatpak flatpak-builder";
# _DEPENDENCIES="mercurial wget git flatpak flatpak-builder";
_DEPENDENCIES="wget git xvfb";
# Installs Dependencies
printf "\nInstalling dependencies: $_DEPENDENCIES\n";

View file

@ -1,10 +1,12 @@
#!/bin/bash
printf "\n\n--------------------------------- SOURCE CODE DOWNLOAD --------------------------------------\n";
if [[ -z ${pkgver} || -z ${pkgrel} ]]; then
echo '$pkgrel and/or $pkgver not provided'
exit 1
fi
# Setup Script Variables
SOURCE_FOLDER=$1;
# hardcoded vor now, we'll parse the _pkgver later on, probably from tags
pkgver="73.0.1"
_SOURCE_CODE_URL="https://archive.mozilla.org/pub/firefox/releases/$pkgver/source/firefox-$pkgver.source.tar.xz";
_SOURCE_TAR="firefox-${pkgver}.tar.xz"

View file

@ -3,9 +3,11 @@ printf "\n\n------------------------------ FINAL PREBUILD CONFIGURATION --------
# Setup Script Variables
SOURCE_CODE_FOLDER=$1;
CI_PROJECT_DIR=${CI_PROJECT_DIR:-$(realpath $(dirname $0)/../../)}
_COMMON_REPO='https://gitlab.com/librewolf-community/browser/common.git';
_POCKET_SED_STRING="s/'pocket'/#'pocket'/g";
_POCKET_FILE=$SOURCE_CODE_FOLDER/browser/components/moz.build;
_MOZBUILD=$SOURCE_CODE_FOLDER/../mozbuild
mkdir -p ${_MOZBUILD}
# Copy Source Code Changes to Source Code
printf "\nCopying branding and source code changes to firefox source code\n";
@ -13,7 +15,87 @@ git clone $_COMMON_REPO common;
cp -r common/source_files/* $SOURCE_CODE_FOLDER/;
rm -rf common;
# Disables Pocket
printf "\nDisabling Pocket\n";
sed -i $_POCKET_SED_STRING $_POCKET_FILE;
cd $SOURCE_CODE_FOLDER
cat >../mozconfig <<END
ac_add_options --enable-application=browser
# This supposedly speeds up compilation (We test through dogfooding anyway)
ac_add_options --disable-tests
ac_add_options --disable-debug
ac_add_options --prefix=/usr
ac_add_options --enable-release
ac_add_options --enable-hardening
ac_add_options --enable-rust-simd
export CC='clang'
export CXX='clang++'
export AR=llvm-ar
export NM=llvm-nm
export RANLIB=llvm-ranlib
# Branding
ac_add_options --enable-update-channel=release
ac_add_options --with-app-name=${pkgname}
ac_add_options --with-app-basename=${_pkgname}
ac_add_options --with-branding=browser/branding/${pkgname}
ac_add_options --with-distribution-id=io.gitlab.${pkgname}
ac_add_options --with-unsigned-addon-scopes=app,system
ac_add_options --allow-addon-sideload
export MOZ_REQUIRE_SIGNING=0
# System libraries
ac_add_options --with-system-nspr
ac_add_options --with-system-nss
# Features
ac_add_options --enable-alsa
ac_add_options --enable-jack
ac_add_options --enable-startup-notification
ac_add_options --disable-crashreporter
ac_add_options --disable-gconf
ac_add_options --disable-updater
ac_add_options --disable-tests
# Disables crash reporting, telemetry and other data gathering tools
mk_add_options MOZ_CRASHREPORTER=0
mk_add_options MOZ_DATA_REPORTING=0
mk_add_options MOZ_SERVICES_HEALTHREPORT=0
mk_add_options MOZ_TELEMETRY_REPORTING=0
# options for ci / weaker build systems
# mk_add_options MOZ_MAKE_FLAGS="-j4"
# ac_add_options --enable-linker=gold
END
if [[ $CARCH == 'aarch64' ]]; then
cat >>../mozconfig <<END
# taken from manjaro build:
ac_add_options --enable-optimize="-g0 -O2"
export MOZ_DEBUG_FLAGS=" "
export CFLAGS+=" -g0"
export CXXFLAGS+=" -g0"
export RUSTFLAGS="-Cdebuginfo=0"
# from ALARM
ac_add_options --disable-webrtc
END
LDFLAGS+=" -Wl,--no-keep-memory -Wl,--reduce-memory-overheads"
patch -p1 -i ${$CI_PROJECT_DIR}/arm.patch
wget https://raw.githubusercontent.com/archlinuxarm/PKGBUILDs/master/extra/firefox/build-arm-libopus.patch -O ${$CI_PROJECT_DIR}/build-arm-libopus.patch
patch -p1 -i ${$CI_PROJECT_DIR}/build-arm-libopus.patch
fi
# Disabling Pocket
printf "\nDisabling Pocket\n";
sed -i "s/'pocket'/#'pocket'/g" browser/components/moz.build
# this one only to remove an annoying error message:
sed -i 's#SaveToPocket.init();#// SaveToPocket.init();#g' browser/components/BrowserGlue.jsm
# allow SearchEngines option in non-ESR builds
sed -i 's#"enterprise_only": true,#"enterprise_only": false,#g' browser/components/enterprisepolicies/schemas/policies-schema.json
rm -f common/source_files/mozconfig

View file

@ -4,14 +4,42 @@ printf "\n\n--------------------------------------- BUILD ----------------------
# Setup Script Variables
SOURCE_FOLDER=$1;
OUTPUT_TARBALL=$2;
CI_PROJECT_DIR=${CI_PROJECT_DIR:-$(realpath $(dirname $0)/../../)}
_SOURCE_CODE_BINARY_TARBALL_LOCATION="./obj*/dist/librewolf*.tar.bz2";
export MOZ_NOSPAM=1
export MOZBUILD_STATE_PATH="$srcdir/mozbuild"
# LTO needs more open files
ulimit -n 4096
# -fno-plt with cross-LTO causes obscure LLVM errors
# LLVM ERROR: Function Import: link error
CFLAGS="${CFLAGS/-fno-plt/}"
CXXFLAGS="${CXXFLAGS/-fno-plt/}"
# Prevents build from breaking in CI/CD environments
export SHELL=/bin/bash;
# Changes current folder to the source code folder
cd $SOURCE_FOLDER;
# Do 3-tier PGO
echo "Building instrumented browser..."
if [[ $CARCH == 'aarch64' ]]; then
cat >.mozconfig ../mozconfig - <<END
ac_add_options --enable-profile-generate
END
else
cat >.mozconfig ../mozconfig - <<END
ac_add_options --enable-profile-generate=cross
END
fi
# Runs bootstrapper to install dependencies
printf "\nRunning bootstrapper to install build dependencies (using ./mach script within source code)\n";
./mach bootstrap --application-choice=browser --no-interactive;
@ -20,6 +48,58 @@ printf "\nRunning bootstrapper to install build dependencies (using ./mach scrip
printf "\nBuilding LibreWolf\n";
./mach build;
echo "Profiling instrumented browser..."
./mach package
LLVM_PROFDATA=llvm-profdata \
JARLOG_FILE="$PWD/jarlog" \
xvfb-run -s "-screen 0 1920x1080x24 -nolisten local" \
./mach python build/pgo/profileserver.py
if [[ ! -s merged.profdata ]]; then
echo "No profile data produced."
return 1
fi
if [[ ! -s jarlog ]]; then
echo "No jar log produced."
return 1
fi
echo "Removing instrumented browser..."
./mach clobber
echo "Building optimized browser..."
if [[ $CARCH == 'aarch64' ]]; then
cat >.mozconfig ../mozconfig - <<END
ac_add_options --enable-lto
ac_add_options --enable-profile-use
ac_add_options --with-pgo-profile-path=${PWD@Q}/merged.profdata
ac_add_options --with-pgo-jarlog=${PWD@Q}/jarlog
# seems to break on arm
# ac_add_options --enable-linker=gold
END
else
cat >.mozconfig ../mozconfig - <<END
ac_add_options --enable-lto=cross
ac_add_options --enable-profile-use=cross
ac_add_options --with-pgo-profile-path=${PWD@Q}/merged.profdata
ac_add_options --with-pgo-jarlog=${PWD@Q}/jarlog
ac_add_options --enable-linker=gold
END
fi
./mach build
echo "Building symbol archive..."
./mach buildsymbols
# End "build()" equivalent.
# Packages the build into a binary tarball
printf "\nPackaging LibreWolf\n";
./mach package;

View file

@ -5,13 +5,19 @@ printf "\n\n--------------------------------- SETTINGS INTEGRATION -------------
BINARY_TARBALL=$1;
TOGGLE_SETTINGS_SCRIPT=$2;
LAUNCHER_SCRIPT=$3;
CI_PROJECT_DIR=${CI_PROJECT_DIR:-$(realpath $(dirname $0)/../../)}
OUTPUT_TARBALL=$CI_PROJECT_DIR/LibreWolf.${CARCH}.tar.bz2
_SCRIPT_FOLDER=$(realpath $(dirname $0));
_EXTRACTED_TARBALL_FOLDER=$_SCRIPT_FOLDER/librewolf;
_SETTINGS_REPO='https://gitlab.com/librewolf-community/settings.git';
# Moves the packaged tarball to the specified location
printf "\nMoving Binary Tarball to output location\n"
mv $BINARY_TARBALL $OUTPUT_TARBALL
# Extracts the binary tarball
printf "\nExtracting librewolf binary tarball\n";
tar -xf $BINARY_TARBALL -C $_EXTRACTED_TARBALL_FOLDER/..;
tar -xf $OUTPUT_TARBALL -C $_EXTRACTED_TARBALL_FOLDER/..;
# Adds the librefox config files to the packaged tarball
printf "\nCopying librewolf settings to extracted binary tarball\n";
@ -25,8 +31,24 @@ cp $LAUNCHER_SCRIPT $_EXTRACTED_TARBALL_FOLDER/launch_librewolf.sh;
# Somewhat crude workaround to use settings per default
# until we've worked out how to use `--install-settings` with links
# in all major packages instead
printf "\nWorkaround: auto-enable Settings\n"
cp $_EXTRACTED_TARBALL_FOLDER/settings/* $_EXTRACTED_TARBALL_FOLDER;
# Add distribution.ini
distini="$_EXTRACTED_TARBALL_FOLDER/distribution/distribution.ini"
install -Dvm644 /dev/stdin "$distini" <<END
[Global]
id=io.gitlab.${pkgname}
version=1.0
about=LibreWolf
[Preferences]
app.distributor="LibreWolf Community"
app.distributor.channel=librewolf
app.partner.librewolf=librewolf
END
# Repacks the binary tarball
printf "\nRecompressing binary tarball\n";
tar -jvcf $BINARY_TARBALL -C $_EXTRACTED_TARBALL_FOLDER .;
tar -jvcf $OUTPUT_TARBALL -C $_EXTRACTED_TARBALL_FOLDER .;

View file

@ -16,6 +16,17 @@ _FLATPAK_JSON_FILE=$_SCRIPT_FOLDER/content/io.gitlab.LibreWolf.json;
_FLATPAK_BUILD_SOURCE_FOLDER=$_SCRIPT_FOLDER/source;
_FLATPAK_BUILD_FOLDER=build-dir;
if [[ -z "${TARBALL_URL}" ]];then
apt update && apt-install -y wget
wget "${TARBALL_URL}"
fi
if [[ ! -f "${BINARY_TARBALL}" ]];then
echo "Tarball not provided via pipeline or download."
exit 1
fi
# Install build dependencies
printf "\nInstalling flatpak build dependencies\n";

View file

@ -1,42 +0,0 @@
#!/bin/bash
set -e
printf "\n\n---------------- prepare package for other distros ----------------\n"
# Setup Script Variables
# use $CI_PROJECT_DIR unless not in CI, then assign script path
CI_PROJECT_DIR=${CI_PROJECT_DIR:-$(realpath $(dirname $0)/../)}
OUTPUT_TARBALL=$CI_PROJECT_DIR/LibreWolf.${CARCH}.tar.bz2
SOURCE_CODE_BINARY_TARBALL_LOCATION="$CI_PROJECT_DIR/src/firefox-*/obj*/dist/librewolf*.tar.bz2"
EXTRACTED_TARBALL_FOLDER=$CI_PROJECT_DIR/librewolf_unpacked/librewolf
# Prevents build from breaking in CI/CD environments
export SHELL=/bin/bash
# Moves the packaged tarball to the specified location
printf "\nMoving Binary Tarball to output location\n"
mv $SOURCE_CODE_BINARY_TARBALL_LOCATION $OUTPUT_TARBALL
# Extracts the binary tarball
printf "\nExtracting librewolf binary tarball\n"
mkdir librewolf_unpacked
tar -xf $OUTPUT_TARBALL -C librewolf_unpacked
# Adds the librefox config files to the packaged tarball
printf "\nCopying librewolf settings to extracted binary tarball\n"
cp -r $CI_PROJECT_DIR/src/settings $EXTRACTED_TARBALL_FOLDER/settings
# no need to keep that in there
rm -rf "${EXTRACTED_TARBALL_FOLDER}/settings/.git";
cp $CI_PROJECT_DIR/content/toggle-settings.sh $EXTRACTED_TARBALL_FOLDER/settings
cp $CI_PROJECT_DIR/content/launch_librewolf.sh $EXTRACTED_TARBALL_FOLDER/launch_librewolf.sh
# Somewhat crude workaround to use settings per default
# until we've worked out how to use `--install-settings` with links
# in all major packages instead
printf "\nWorkaround: auto-enable Settings\n"
cp -r $EXTRACTED_TARBALL_FOLDER/settings/* $EXTRACTED_TARBALL_FOLDER;
# Repacks the binary tarball
printf "\nRecompressing binary tarball\n"
tar -jvcf $OUTPUT_TARBALL -C $EXTRACTED_TARBALL_FOLDER .