#!/usr/bin/env bash # # Copyright 2022-23 Bryan C. Roessler # # Build and deploy OpenWRT images # # Apache 2.0 License # Set default release : "${RELEASE:="22.03.3"}" printHelp() { debug "${FUNCNAME[0]}" cat <<-'EOF' Build and deploy OpenWRT images USAGE: openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]... OPTIONS --profile,-p PROFILE --release,-r,--version,-v RELEASE ("snapshot", "22.03.3") --buildroot,-b PATH Defaults to location of openwrtbuilder script --source Build image from source, not from Image Builder --ssh-upgrade HOST Example: root@192.168.1.1 --ssh-backup SSH_PATH (Enabled by default for --ssh-upgrade) --flash,-f DEVICE Example: /dev/sdX --reset Cleanup all source and output files --debug,-d --help,-h EOF } init() { debug "${FUNCNAME[0]}" declare -g ID RPM_MGR SCRIPTDIR DL_TOOL debug || echo "To enable debugging output, use --debug or -d" # Save the script directory # https://stackoverflow.com/a/4774063 SCRIPTDIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)" if [[ -e "/etc/os-release" ]]; then source "/etc/os-release" else err "/etc/os-release not found" err "Your OS is unsupported" printHelp exit 1 fi debug "Detected host platform: $ID" # normalize distro ID case "$ID" in debian|arch) ;; centos|fedora) if hash dnf &>/dev/null; then RPM_MGR="dnf" elif hash yum &>/dev/null; then RPM_MGR="yum" fi ;; rhel) ID="centos" ;; linuxmint|neon|*ubuntu*) ID="ubuntu" ;; *suse*) ID="suse" ;; raspbian) ID="debian" ;; *) echo "Autodetecting distro, this may be unreliable" if hash dnf &>/dev/null; then ID="fedora" RPM_MGR="dnf" elif hash yum &>/dev/null; then ID="centos" RPM_MGR="yum" elif hash apt &>/dev/null; then ID="ubuntu" elif hash pacman &>/dev/null; then ID="arch" else return 1 fi ;; esac debug "Using host platform: $ID" # Set distro-specific functions case "$ID" in fedora|centos) pkg_install(){ sudo "$RPM_MGR" install -y "$@"; } ;; debian|ubuntu) pkg_install(){ sudo apt-get install -y -q0 "$@"; } ;; suse) pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; } ;; arch) pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; } ;; esac if hash axel &>/dev/null; then DL_TOOL="axel" elif hash curl &>/dev/null; then DL_TOOL="curl" else echo "Downloading the Image Builder requires axel or curl!" return 1 fi } readInput() { debug "${FUNCNAME[0]}" unset RESET declare -ga PROFILES declare long_opts='release:,version:,profile:,buildroot:,source,' long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,debug,help' if _input=$(getopt -o +r:v:p:b:sf:dh -l $long_opts -- "$@"); then eval set -- "$_input" while true; do case "$1" in --release|-r|--version|-v) shift && declare -g USER_RELEASE="$1" ;; --profile|-p) shift && PROFILES+=("$1") ;; --buildroot|-b) shift && BUILDROOT="$1" ;; --source|-s) FROM_SOURCE=1 ;; --ssh-upgrade) shift && SSH_UPGRADE_PATH="$1" ;; --ssh-backup) shift && SSH_BACKUP_PATH="$1" ;; --flash|-f) shift && FLASH_DEV="$1" ;; --reset) RESET=1 ;; --debug|-d) echo "Debugging on" DEBUG=1 ;; --help|-h) printHelp && exit 0 ;; --) shift break ;; esac shift done else echo "Incorrect options provided" printHelp && exit 1 fi } installDependencies() { debug "${FUNCNAME[0]}" declare -a pkg_list # TODO please contribute your platform here if (( FROM_SOURCE )); then # For building from source with make # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem case "$ID" in fedora|centos) pkg_list+=( "bash-completion" "bzip2" "gcc" "gcc-c++" "git" "make" "ncurses-devel" "patch" "rsync" "tar" "unzip" "wget" "which" "diffutils" "python2" "python3" "perl-base" "perl-Data-Dumper" "perl-File-Compare" "perl-File-Copy" "perl-FindBin" "perl-Thread-Queue" "clang" # for qosify ) ;; debian|ubuntu) pkg_list+=( "build-essential" "clang" "flex" "g++" "gawk" "gcc-multilib" "gettext" "git" "libncurses5-dev" "libssl-dev" "python3-distutils" "rsync" "unzip" "zlib1g-dev" "file" "wget" ) ;; arch) pkg_list+=( "base-devel" "autoconf" "automake" "bash" "binutils" "bison" "bzip2" "clang" "fakeroot" "file" "findutils" "flex" "gawk" "gcc" "gettext" "git" "grep" "groff" "gzip" "libelf" "libtool" "libxslt" "m4" "make" "ncurses" "openssl" "patch" "pkgconf" "python" "rsync" "sed" "texinfo" "time" "unzip" "util-linux" "wget" "which" "zlib" ) ;; esac else # For Imagebuilder case "$ID" in fedora|centos) pkg_list+=( "@c-development" "@development-tools" "@development-libs" "perl-FindBin" "zlib-static" "elfutils-libelf-devel" "gawk" "unzip" "file" "wget" "python3" "python2" "axel" ) ;; debian|ubuntu) pkg_list+=( "build-essential" "libncurses5-dev" "libncursesw5-dev" "zlib1g-dev" "gawk" "git" "gettext" "libssl-dev" "xsltproc" "wget" "unzip" "python" "axel" ) ;; esac fi pkg_install "${pkg_list[@]}" } getImageBuilder() { debug "${FUNCNAME[0]}" if [[ -f "$IB_ARCHIVE" ]]; then if askOK "$IB_ARCHIVE exists. Re-download?"; then rm -f "$IB_ARCHIVE" else return 0 fi fi echo "Downloading Image Builder archive using $DL_TOOL" debug "$DL_TOOL -o $IB_ARCHIVE $IB_URL" "$DL_TOOL" -o "$IB_ARCHIVE" "$IB_URL" } getImageBuilderChecksum() { debug "${FUNCNAME[0]}" if [[ -f $IB_SHA256_FILE ]]; then if askOk "$IB_SHA256_FILE exists. Re-download?"; then rm -f "$IB_SHA256_FILE" else return 0 fi fi "$DL_TOOL" -o "$IB_SHA256_FILE" "$IB_SHA256_URL" } extractImageBuilder() { debug "${FUNCNAME[0]}" echo "Extracting Image Builder archive" [[ ! -d "$BUILDDIR" ]] && mkdir -p "$BUILDDIR" debug "tar -xf $IB_ARCHIVE -C $BUILDDIR --strip-components 1" if ! tar -xf "$IB_ARCHIVE" -C "$BUILDDIR" --strip-components 1; then echo "Extraction failed" return 1 fi } addRepos() { debug "${FUNCNAME[0]}" if [[ -v P_ARR[repo] ]]; then if ! grep -q "${P_ARR[repo]}" "$BUILDDIR/repositories.conf"; then echo "${P_ARR[repo]}" >> "$BUILDDIR/repositories.conf" fi sed -i '/option check_signature/d' "$BUILDDIR/repositories.conf" fi } sshBackup() { debug "${FUNCNAME[0]}" local _date _hostname _backup_fname [[ -d "$FILESDIR" ]] || mkdir -p "$FILESDIR" printf -v _date '%(%Y-%m-%d-%H-%M-%S)T' _hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME) _backup_fname="backup-$_hostname-$_date.tar.gz" # Make backup archive on remote debug "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$_backup_fname" if ! ssh -t "$SSH_BACKUP_PATH" "sysupgrade -b /tmp/$_backup_fname"; then echo "SSH backup failed" exit 1 fi # Move backup archive locally debug "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$_backup_fname $BUILDDIR/" if ! rsync -avz --remove-source-files \ "$SSH_BACKUP_PATH":"/tmp/$_backup_fname" "$BUILDDIR/"; then echo "Could not copy SSH backup" exit 1 fi # Extract backup archive debug "tar -C $FILESDIR -xzf $BUILDDIR/$_backup_fname" if ! tar -C "$FILESDIR" -xzf "$BUILDDIR/$_backup_fname"; then echo "Could not extract SSH backup" exit 1 fi rm "$BUILDDIR/$_backup_fname" } makeImages() { debug "${FUNCNAME[0]}" # Reuse the existing output if [[ -d "$THIS_BINDIR" ]]; then if askOk "$THIS_BINDIR exists. Rebuild?"; then rm -rf "$THIS_BINDIR" else return 0 fi fi [[ -d "$BUILDDIR" ]] || mkdir -p "$BUILDDIR" make image \ BIN_DIR="$THIS_BINDIR" \ PROFILE="${P_ARR[profile]}" \ PACKAGES="${P_ARR[packages]:+"${P_ARR[packages]}"}" \ FILES="${FILESDIR}" \ --directory="$BUILDDIR" \ --jobs="$(nproc)" \ > "$BUILDDIR/make.log" } verifyImages() { debug "${FUNCNAME[0]}" declare outfile for outfile in "$THIS_BINDIR"/*.img.gz; do verify "$outfile" "$IB_OUT_SHA256_FILE" || return 1 done } flashImage() { debug "${FUNCNAME[0]}" declare img img_gz partitions if [[ ! -e "$FLASH_DEV" ]]; then echo "The device specified by --flash could not be found" exit 1 fi # TODO Roughly choose the correct image if [[ -f "$FACTORYIMGGZ" ]]; then img_gz="$FACTORYIMGGZ" img="$FACTORYIMG" elif [[ -f "$SYSUPGRADEIMGGZ" ]]; then img_gz="$SYSUPGRADEIMGGZ" img="$SYSUPGRADEIMG" else return 1 fi debug "$img_gz $img" debug "gunzip -qfk $img_gz" gunzip -qfk "$img_gz" echo "Unmounting target device $FLASH_DEV partitions" partitions=( "$FLASH_DEV"?* ) debug "umount ${partitions[*]}" sudo umount "${partitions[@]}" debug "sudo dd if=\"$img\" of=\"$FLASH_DEV\" bs=2M conv=fsync" if sudo dd if="$img" of="$FLASH_DEV" bs=2M conv=fsync; then sync echo "Image flashed sucessfully!" else echo "dd failed!" exit 1 fi } sshUpgrade() { debug "${FUNCNAME[0]}" echo "Copying '$SYSUPGRADEIMGGZ' to $SSH_UPGRADE_PATH/tmp/" debug "scp \"$SYSUPGRADEIMGGZ\" \"$SSH_UPGRADE_PATH\":\"/tmp/$SYSUPGRADEIMGGZFNAME\"" if ! scp "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH":"/tmp/$SYSUPGRADEIMGGZFNAME"; then echo "Could not access the --ssh-upgrade PATH" exit 1 fi echo "Executing remote sysupgrade" debug "ssh \"$SSH_UPGRADE_PATH\" \"sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME\"" # shellcheck disable=SC2029 ssh "$SSH_UPGRADE_PATH" "sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME" } fromSource() { debug "${FUNCNAME[0]}" declare src_url="https://github.com/openwrt/openwrt.git" declare pkg kopt declare -a make_opts echo "Building from source is under development" if [[ ! -d "$GITSRCDIR" ]]; then mkdir -p "$GITSRCDIR" git clone "$src_url" "$GITSRCDIR" fi pushd "$GITSRCDIR" &>/dev/null || return 1 if [[ ${P_ARR[release]} == "snapshot" ]]; then git checkout master else git checkout "v$RELEASE" fi git pull # Update package feed ./scripts/feeds update -a && ./scripts/feeds install -a # Grab the release seed config if [[ -f "$SEED_FILE" ]]; then if askOk "$SEED_FILE exists. Re-download?"; then rm -f "$SEED_FILE" fi fi if [[ -f "$SEED_FILE" ]]; then echo "Reusing existing $SEED_FILE seed config" else if ! curl -so "$SEED_FILE" "$SEED_URL"; then echo "Could not obtain seed config" fi fi make defconfig # normalize .config and remove dupes for pkg in ${P_ARR[packages]:+${P_ARR[packages]}}; do if [[ $pkg == -* ]]; then make_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package else make_opts+=("CONFIG_PACKAGE_$pkg=y") # add package fi done for kopt in ${P_ARR[kopts]:+${P_ARR[kopts]}}; do make_opts+=("$kopt") done make_opts+=("CONFIG_TARGET_MULTI_PROFILE=n") make_opts+=("CONFIG_TARGET_PROFILE=DEVICE_${P_ARR[profile]}") make_opts+=("CONFIG_TARGET_DEVICE_${P_ARR[target]//\//_}_DEVICE_${P_ARR[profile]}=y") make_opts+=("CONFIG_BINARY_FOLDER=\"$THIS_BINDIR\"") # Override default .config seed with profile packages # for pkg in ${P_ARR[packages]:+${P_ARR[packages]}}; do # if [[ $pkg == -* ]]; then # echo "CONFIG_PACKAGE_${pkg#-}=n" >> "$SEED_FILE" # remove package # else # echo "CONFIG_PACKAGE_$pkg=y" >> "$SEED_FILE" # add package # fi # done # Add custom kernel config options # for kopt in ${P_ARR[kopts]:+${P_ARR[kopts]}}; do # echo "$kopt" >> "$SEED_FILE" # done # Only compile our target # TODO causes a "configuration is out of sync" error w/o defconfig # sed -i '/CONFIG_TARGET_DEVICE_/d' "$SEED_FILE" # echo "CONFIG_TARGET_DEVICE_${P_ARR[target]//\//_}_DEVICE_${P_ARR[profile]}=y" >> "$SEED_FILE" # output to bindir instead of builddir # echo "CONFIG_BINARY_FOLDER=\"$THIS_BINDIR\"" >> "$SEED_FILE" # TODO symlink clang for qosify [[ -d "$GITSRCDIR/staging_dir/host/llvm-bpf/bin" ]] || mkdir -p "$GITSRCDIR/staging_dir/host/llvm-bpf/bin" ln -fs "$(which clang)" "$GITSRCDIR/staging_dir/host/llvm-bpf/bin/clang" make targetclean make "${make_opts[@]}" download && debug "${make_opts[*]} -j$(nproc) world" && make "${make_opts[@]}" -j"$(nproc)" world popd &>/dev/null || return 1 exit # TODO exit here for fromSource() testing } debug() { (( DEBUG )) && echo "Debug: $*"; } askOk() { local _response read -r -p "$* [y/N]" _response _response=${_response,,} [[ "$_response" =~ ^(yes|y)$ ]] } resetAll() { debug "${FUNCNAME[0]}" askOk "Remove $SRCDIR and $BINDIR?" || exit $? debug "rm -rf $SRCDIR $BINDIR" rm -rf "$SRCDIR" "$BINDIR" } resetProfile() { debug "${FUNCNAME[0]}" askOk "Remove $BUILDDIR and $THIS_BINDIR?" || exit $? debug "rm -rf $BUILDDIR $THIS_BINDIR" rm -rf "$BUILDDIR" "$THIS_BINDIR" } loadProfiles() { debug "${FUNCNAME[0]}" declare -g PFILE PFILE="$SCRIPTDIR/profiles" # shellcheck source=./profiles ! source "$PFILE" && echo "profiles file missing!" && return 1 } verify() { debug "${FUNCNAME[0]}" declare file_to_check="$1" declare sumfile="$2" declare checksum hash sha256sum &>/dev/null || return 1 [[ -f $sumfile && -f $file_to_check ]] || return 1 checksum=$(grep "${file_to_check##*/}" "$sumfile" | cut -f1 -d' ') echo -n "$checksum $file_to_check" | sha256sum --check --status } main() { debug "${FUNCNAME[0]}" init loadProfiles readInput "$@" # Fallback to SCRIPTDIR if BUILDROOT has not been set declare -g BUILDROOT="${BUILDROOT:=$SCRIPTDIR}" [[ $BUILDROOT == "/" ]] && echo "Invalid --buildroot" && exit 1 declare -g FILESDIR="${FILESDIR:=$BUILDROOT/src/files}" declare -g SRCDIR="$BUILDROOT/src" # input/build declare -g BINDIR="$BUILDROOT/bin" # output declare -g GITSRCDIR="$SRCDIR/openwrt" for dir in "$SRCDIR" "$BINDIR"; do [[ -d "$dir" ]] || mkdir -p "$dir" done # Allow --reset without a profile if [[ ${#PROFILES} -lt 1 ]]; then if (( RESET )); then resetAll exit else echo "No profile supplied" && return 1 fi fi installDependencies for profile in "${PROFILES[@]}"; do debug "Starting profile: $profile" if [[ ! ${!profile@a} = A ]]; then echo "Profile '$profile' does not exist" return 1 fi # Store profile settings in P_ARR declare -gn P_ARR="$profile" # release precedence: user input>profile>env>hardcode declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}" declare -g BUILDDIR="$SRCDIR/${P_ARR[profile]}-$RELEASE" declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}" declare -g THIS_BINDIR="$BINDIR/${P_ARR[profile]}-$RELEASE" if [[ "$RELEASE" == "snapshot" ]]; then declare url_prefix="https://downloads.openwrt.org/snapshots/targets/${P_ARR[target]}" declare url_filename="openwrt-imagebuilder-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz" declare img_prefix="$BUILDDIR/openwrt-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM" else declare url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/${P_ARR[target]}" declare url_filename="openwrt-imagebuilder-$RELEASE-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz" declare img_prefix="$BUILDDIR/openwrt-$RELEASE-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM" fi declare -g IB_URL="$url_prefix/$url_filename" declare -g IB_ARCHIVE="$SRCDIR/$url_filename" declare -g IB_SHA256_URL="$url_prefix/sha256sums" declare -g IB_SHA256_FILE="$BUILDDIR/sha256sums" declare -g IB_OUT_SHA256_FILE="$THIS_BINDIR/sha256sums" declare -g SEED_URL="$url_prefix/config.buildinfo" declare -g SEED_FILE="$GITSRCDIR/.config" declare -g FACTORYIMG="$img_prefix-factory.img" declare -g FACTORYIMGGZ="$img_prefix-factory.img.gz" declare -g FACTORYIMGGZFNAME="${FACTORYIMGGZ##*/}" declare -g SYSUPGRADEIMG="$img_prefix-sysupgrade.img" declare -g SYSUPGRADEIMGGZ="$img_prefix-sysupgrade.img.gz" declare -g SYSUPGRADEIMGGZFNAME="${SYSUPGRADEIMGGZ##*/}" if (( DEBUG )); then echo "Profile settings:" for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done echo "Build settings:" cat <<- EOF ALIAS=$profile BUILDROOT=$BUILDROOT BUILDDIR=$BUILDDIR SRCDIR=$SRCDIR BINDIR=$BINDIR GITSRCDIR=$GITSRCDIR THIS_BINDIR=$THIS_BINDIR TARGET=${P_ARR[target]} PROFILE=${P_ARR[profile]} RELEASE=$RELEASE FILESYSTEM=$FILESYSTEM IB_URL=$IB_URL IB_ARCHIVE=$IB_ARCHIVE SEED_URL=$SEED_URL SEED_FILE=$SEED_FILE IB_SHA256_URL=$IB_SHA256_URL IB_SHA256_FILE=$IB_SHA256_FILE IB_OUT_SHA256_FILE=$IB_OUT_SHA256_FILE FACTORYIMGGZ: $FACTORYIMGGZ FACTORYIMGGZFNAME: $FACTORYIMGGZFNAME SYSUPGRADEIMGGZ: $SYSUPGRADEIMGGZ SYSUPGRADEIMGGZFNAME: $SYSUPGRADEIMGGZFNAME EOF fi (( RESET )) && resetProfile (( FROM_SOURCE )) && fromSource # Acquire and verify Image Builder getImageBuilder && getImageBuilderChecksum && verify "$IB_ARCHIVE" "$IB_SHA256_FILE" || return $? extractImageBuilder || return $? addRepos #copyFiles [[ -v SSH_BACKUP_PATH ]] && sshBackup if makeImages && verifyImages; then [[ -v SSH_UPGRADE_PATH ]] && sshUpgrade [[ -v FLASH_DEV ]] && flashImage fi done } main "$@" exit # VM setup (for testing) # sudo sgdisk -N 0 /dev/vda && # sudo mkfs.ext4 /dev/vda1 # mkdir ~/mnt # sudo mount /dev/vda1 ~/mnt # sudo chown liveuser:liveuser -R ~/mnt