diff options
Diffstat (limited to 'gitstatus/mbuild')
| -rwxr-xr-x | gitstatus/mbuild | 365 |
1 files changed, 365 insertions, 0 deletions
diff --git a/gitstatus/mbuild b/gitstatus/mbuild new file mode 100755 index 00000000..608e3aba --- /dev/null +++ b/gitstatus/mbuild @@ -0,0 +1,365 @@ +#!/usr/bin/env zsh +# +# This script does not have a stable API. +# +# Usage: mbuild [-b git-ref] [kernel-arch]... +# +# Builds a bunch of gitstatusd-* binaries. Without arguments builds binaries +# for all platforms. git-ref defaults to master. +# +# Before using this script you need to set up build servers and list them +# in ~/.ssh/config. There should be a Host entry for every value of `assets` +# association defined below. VMs and cloud instances work as well as physical +# machines, including localhost. As long as the machine has been set up as +# described below and you can SSH to it without password, it should work. +# +# ===[ Build Server Setup ]=== +# +# Linux +# +# - Install docker. +# $ apt install docker.io # adjust appropriately if there is no `apt` +# $ usermod -aG docker $USER # not needed if going to build as root +# - Install git. +# $ apt install git # adjust appropriately if there is no `apt` +# +# macOS +# +# - Install compiler tools: +# $ xcode-select --install +# - Install homebrew: https://brew.sh/. +# $ bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" +# +# FreeBSD +# +# - Install git. +# $ pkg install git +# +# Windows +# +# - Disable Windows Defender (optional). +# ps> Set-MpPreference -DisableRealtimeMonitoring $true +# - Install 64-bit and 32-bit msys2: https://www.msys2.org/wiki/MSYS2-installation/. +# - Open each of them after installation, type `pacman -Syu --noconfirm` and close the window. +# - Then run in powershell while having no msys2 or cygwin windows open: +# ps> C:\msys32\autorebase.bat +# ps> C:\msys64\autorebase.bat +# - Install 64-bit and 32-bit cygwin: https://cygwin.com/install.html. +# - Choose to install 32-bit to c:/cygwin32 instead of the default c:/cygwin. +# - Select these packages: binutils, cmake, gcc-core, gcc-g++, git, make, perl, wget. +# +# IMPORTANT: Install msys2 and cygwin one at a time. +# +# IMPORTANT: msys2 builder can reboot the build machine. +# +# Option 1: OpenSSH for Windows +# +# - Install OpenSSH: https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse. +# ps> Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +# ps> Start-Service sshd +# ps> Set-Service -Name sshd -StartupType 'Automatic' +# - Enable publickey authentication: https://stackoverflow.com/a/50502015/1095235. +# ps> cd $env:USERPROFILE +# ps> mkdir .ssh +# ps> notepad.exe .ssh/authorized_keys +# - Paste your public key, save, close. +# ps> icacls .ssh/authorized_keys /inheritance:r +# ps> notepad.exe C:\ProgramData\ssh\sshd_config +# - Comment out these two lines, save, close: +# # Match Group administrators +# # AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys +# ps> Restart-Service sshd +# +# Option 2: OpenSSH from WSL +# +# - Install WSL. +# - Install Ubuntu. +# - Install sshd. +# $ apt install openssh-server +# $ dpkg-reconfigure openssh-server +# $ cat >/etc/ssh/sshd_config <<\END +# ClientAliveInterval 60 +# AcceptEnv TERM LANG LC_* +# PermitRootLogin no +# AllowTcpForwarding no +# AllowAgentForwarding no +# AllowStreamLocalForwarding no +# AuthenticationMethods publickey +# END +# service ssh --full-restart +# - Add your public ssh key to ~/.ssh/authorized_keys. +# - Make `sshd` start when Windows boots. + +'emulate' '-L' 'zsh' '-o' 'no_aliases' '-o' 'err_return' +setopt no_unset extended_glob pipe_fail prompt_percent typeset_silent \ + no_prompt_subst no_prompt_bang pushd_silent warn_create_global + +autoload -Uz is-at-least + +if ! is-at-least 5.1 || [[ $ZSH_VERSION == 5.4.* ]]; then + print -ru2 -- "[error] unsupported zsh version: $ZSH_VERSION" + return 1 +fi + +zmodload zsh/system + +local -r git_url='https://github.com/romkatv/gitstatus.git' + +local -rA assets=( + # target kernel-arch hostname of the build machine + cygwin_nt-10.0-i686 build-windows-x86_64 + cygwin_nt-10.0-x86_64 build-windows-x86_64 + msys_nt-10.0-i686 build-windows-x86_64 + msys_nt-10.0-x86_64 build-windows-x86_64 + darwin-arm64 build-macos-arm64 + darwin-x86_64 build-macos-x86_64 + freebsd-amd64 build-freebsd-amd64 + linux-aarch64 build-linux-aarch64 + linux-armv6l build-linux-armv7l + linux-armv7l build-linux-armv7l + linux-i686 build-linux-x86_64 + linux-ppc64le build-linux-ppc64le + linux-x86_64 build-linux-x86_64 +) + +local -rA protocol=( + 'cygwin_nt-10.0-*' windows + 'msys_nt-10.0-*' windows + 'darwin-*' unix + 'freebsd-*' unix + 'linux-*' unix +) + +local -r rootdir=${ZSH_SCRIPT:h} +local -r logs=$rootdir/logs +local -r locks=$rootdir/locks +local -r binaries=$rootdir/usrbin + +function usage() { + print -r -- 'usage: mbuild [-b REF] [KERNEL-ARCH]...' +} + +local OPTARG opt git_ref=master +local -i OPTIND +while getopts ":b:h" opt; do + case $opt in + h) usage; return 0;; + b) [[ -n $OPTARG ]]; git_ref=$OPTARG;; + \?) print -ru2 -- "mbuild: invalid option: -$OPTARG" ; return 1;; + :) print -ru2 -- "mbuild: missing required argument: -$OPTARG"; return 1;; + *) print -ru2 -- "mbuild: invalid option: -$opt" ; return 1;; + esac +done + +shift $((OPTIND - 1)) + +(( $# )) || set -- ${(ko)assets} +set -- ${(u)@} + +local platform +for platform; do + if (( ! $+assets[$platform] )); then + print -ru2 -- "mbuild: invalid platform: $platform" + return 1 + fi +done + +local build=' + rm -rf gitstatus + git clone --recursive --shallow-submodules --depth=1 -b '$git_ref' '$git_url' + cd gitstatus + if command -v zsh >/dev/null 2>&1; then + sh=zsh + elif command -v dash >/dev/null 2>&1; then + sh=dash + elif command -v ash >/dev/null 2>&1; then + sh=ash + else + sh=sh + fi + $sh -x ./build -m ' + +function build-unix() { + local intro flags=(-sw) + case $2 in + linux-ppc64le) ;; + linux-*) flags+=(-d docker);; + darwin-arm64) intro='PATH="/opt/local/bin:$PATH"';; + darwin-*) intro='PATH="/usr/local/bin:$PATH"';; + esac + ssh $1 -- /bin/sh -uex <<<" + $intro + cd /tmp + $build ${2##*-} ${(j: :)${(@q)flags}}" + scp $1:/tmp/gitstatus/usrbin/gitstatusd $binaries/gitstatusd-$2 +} + +function build-windows() { + local shell=$(ssh $1 'echo $0') + if [[ $shell == '$0'* ]]; then + local c='c:' + else + local c='/mnt/c' + fi + + local tmp env bin intro flags=(-w) + case $2 in + cygwin_nt-10.0-i686) bin='cygwin32/bin' ;| + cygwin_nt-10.0-x86_64) bin='cygwin64/bin' ;| + msys_nt-10.0-i686) bin='msys32/usr/bin';| + msys_nt-10.0-x86_64) bin='msys64/usr/bin';| + cygwin_nt-10.0-*) + tmp='/cygdrive/c/tmp' + ;| + msys_nt-10.0-*) + flags+=(-s) + tmp='/c/tmp' + env='MSYSTEM=MSYS' + intro='pacman -Syu --noconfirm; pacman -S --needed --noconfirm git; ' + intro+='PATH="$PATH:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl"' + while true; do + # TODO: run autorebase only when getting an error that can be fixed by autorebasing. + break + local out + out="$(ssh $1 cmd.exe "$c/${bin%%/*}/autorebase.bat" 2>&1)" + [[ $out == *"The following DLLs couldn't be rebased"* ]] || break + # Reboot to get rid of whatever is using those DLLs. + ssh $1 powershell.exe <<<'Restart-Computer -Force' || true + sleep 30 + while ! ssh $1 <<<''; do sleep 5; done + done + () { + while true; do + local -i fd + exec {fd}< <( + ssh $1 $c/$bin/env.exe $env c:/$bin/bash.exe -l 2>&1 <<<" + pacman -Syu --noconfirm + exit") + { + local line + while true; do + IFS= read -u $fd -r line || return 0 + if [[ $line == *"warning: terminate MSYS2"* ]]; then + # At this point the machine is hosed. A rogue process with a corrupted name + # is eating all CPU. The top SSH connection won't terminate on its own. + ssh $1 powershell.exe <<<'Restart-Computer -Force' || true + sleep 30 + while ! ssh $1 <<<''; do sleep 5; done + break + fi + done + } always { + exec {fd}<&- + kill -- -$sysparams[procsubstpid] 2>/dev/null || true + } + done + } "$@" + ;| + esac + + ssh $1 $c/$bin/env.exe $env c:/$bin/bash.exe -l <<<" + set -uex + $intro + mkdir -p -- $tmp + cd -- $tmp + $build ${2##*-} ${(j: :)${(@q)flags}} + exit" + scp $1:$c/tmp/gitstatus/usrbin/gitstatusd $binaries/gitstatusd-$2 + chmod +x $binaries/gitstatusd-$2 +} + +function build() ( + setopt xtrace + local platform=$1 + local machine=$assets[$platform] + print -n >>$locks/$machine + zsystem flock $locks/$machine + build-${protocol[(k)$platform]} $machine $platform + local tmp=gitstatusd-$platform.tmp.$$.tar.gz + ( cd -q -- $binaries; tar --owner=0 --group=0 -I 'gzip -9' -cf $tmp gitstatusd-$platform ) + mv -f -- $binaries/$tmp $binaries/gitstatusd-$platform.tar.gz +) + +function mbuild() { + local platform pid pids=() + for platform; do + build $platform &>$logs/$platform & + print -r -- "starting build for $platform on $assets[$platform] (pid $!)" + pids+=($platform $!) + done + local failed=() + for platform pid in $pids; do + print -rn -- "$platform => " + if wait $pid; then + print -r -- "ok" + else + print -r -- "error" + failed+=$platform + fi + done + (( $#failed )) || return 0 + print + print -r -- "Error logs:" + print + for platform in $failed; do + print -r -- " $platform => $logs/$platform" + done + return 1 +} + +# Copied from https://github.com/romkatv/run-process-tree. +function run-process-tree() { + zmodload zsh/parameter zsh/param/private || return + local -P opt=(${(kv)options[@]}) || return + local -P pat=(${patchars[@]}) || return + local -P dis_pat=(${dis_patchars[@]}) || return + emulate -L zsh -o err_return || return + setopt monitor traps_async pipe_fail no_unset + zmodload zsh/system + + if (( $# == 0 )); then + print -ru2 -- 'usage: run-process-tree command [arg]...' + return 1 + fi + + local -P stdout REPLY + exec {stdout}>&1 + { + { + local -Pi pipe + local -P gid=$sysparams[pid] + local -P sig=(ABRT EXIT HUP ILL INT PIPE QUIT TERM ZERR) + local -P trap=(trap "trap - $sig; kill -- -$sysparams[pid]" $sig) + + exec {pipe}>&1 1>&$stdout + $trap + + { + $trap + while sleep 1 && print -u $pipe .; do; done + } 2>/dev/null & + local -Pi watchdog=$! + + { + trap - ZERR + exec {pipe}>&- + enable -p -- $pat + disable -p -- $dis_pat + options=($opt zle off monitor off) + "$@" + } & + local -Pi ret + wait $! || ret=$? + + trap "exit $ret" TERM + kill $watchdog + wait $watchdog + return ret + } | while read; do; done || return + } always { + exec {stdout}>&- + } +} + +mkdir -p -- $logs $locks $binaries +run-process-tree mbuild $@ |