#!/usr/bin/env bash # Builds and deploys OpenWRT images # Copyright 2022-24 Bryan C. Roessler # Apache 2.0 License # See README.md and ./profiles for device configuration # Set default release : "${RELEASE:="23.05.5"}" # @internal print_help() { debug "${FUNCNAME[0]}" cat <<-'EOF' Build and deploy OpenWRT images USAGE: openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]... OPTIONS --profile,-p PROFILE --release,-r,--version,-v RELEASE ("snapshot", "22.03.5") --buildroot,-b PATH Default: location of openwrtbuilder script --source Build image from source, not from Image Builder --ssh-upgrade HOST Examples: root@192.168.1.1, root@router.lan --ssh-backup SSH_PATH Enabled by default for --ssh-upgrade --flash,-f DEVICE Example: /dev/sdX --reset Cleanup all source and output files --yes,-y Assume yes for all questions (automatic mode) --verbose Make make or imagebuilder noisier --debug,-d --help,-h EXAMPLES ./openwrtbuilder -p r4s -r snapshot ./openwrtbuilder -p ax6000 -r 23.05.0-rc3 --source --debug ./openwrtbuilder -p rpi4 -r 22.03.3 --flash /dev/sdX ./openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1 EOF } # @internal init() { debug "${FUNCNAME[0]}" declare -g ID RPM_MGR SCRIPT_DIR DL_TOOL ((DEBUG)) || echo "To enable debugging output, use --debug or -d" # Save the script directory # https://stackoverflow.com/a/4774063 SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)" if [[ -e "/etc/os-release" ]]; then source "/etc/os-release" else echo "/etc/os-release not found" echo "Your OS is unsupported" print_help exit 1 fi debug "Detected host platform: $ID" # normalize distro ID case "$ID" in debian|arch) ;; centos|fedora) if command -v dnf &>/dev/null; then RPM_MGR="dnf" elif command -v yum &>/dev/null; then RPM_MGR="yum" fi ;; rhel) ID="centos" ;; linuxmint|neon|*ubuntu*) ID="ubuntu" ;; *suse*) ID="suse" ;; raspbian) ID="debian" ;; *) echo "Autodetecting distro, this may be unreliable" for cmd in dnf yum apt pacman; do if command -v "$cmd" &>/dev/null; then case "$cmd" in dnf) ID="fedora"; RPM_MGR="dnf" ;; yum) ID="centos"; RPM_MGR="yum" ;; apt) ID="ubuntu" ;; pacman) ID="arch" ;; esac break fi done [[ -z $ID ]] && return 1 ;; esac debug "Using host platform: $ID" # Set distro-specific functions case "$ID" in fedora|centos) pkg_install(){ sudo "$RPM_MGR" install -y "$@"; } ;; debian|ubuntu) pkg_install(){ sudo apt-get install --ignore-missing -y -q0 "$@"; } ;; suse) pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; } ;; arch) pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; } ;; esac if command -v axel &>/dev/null; then DL_TOOL="axel" elif command -v curl &>/dev/null; then DL_TOOL="curl" else echo "Downloading the Image Builder requires axel or curl" return 1 fi } # @description Arguments parse_input() { debug "${FUNCNAME[0]}" "$*" declare -ga PROFILES declare -g RESET=0 FROM_SOURCE=0 YES=0 VERBOSE=0 DEBUG=0 declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV local long_opts='release:,version:,profile:,buildroot:,source,' long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,yes,verbose,debug,help' if _input=$(getopt -o +r:v:p:b:sf:ydh -l $long_opts -- "$@"); then eval set -- "$_input" while true; do case "$1" in --release|-r|--version|-v) shift; USER_RELEASE="$1" ;; --profile|-p) shift; PROFILES+=("$1") ;; --buildroot|-b) shift; BUILD_ROOT="$1" ;; --source|-s) FROM_SOURCE=1 ;; --ssh-upgrade) shift; SSH_UPGRADE_PATH="$1" ;; --ssh-backup) shift; SSH_BACKUP_PATH="$1" ;; --flash|-f) shift; FLASH_DEV="$1" ;; --reset) RESET=1 ;; --yes|-y) YES=1 ;; --verbose) VERBOSE=1 ;; --debug|-d) echo "Debugging on"; DEBUG=1 ;; --help|-h) print_help; exit 0 ;; --) shift; break ;; esac shift done else echo "Incorrect options provided" print_help; exit 1 fi } # @description Install build dependencies on major distros install_dependencies() { debug "${FUNCNAME[0]}" local -a pkg_list local lock_file if ((FROM_SOURCE)); then lock_file="$BUILD_ROOT/.dependencies_source" else lock_file="$BUILD_ROOT/.dependencies_ib" fi if [[ ! -f $lock_file ]]; then if ((FROM_SOURCE)); then # For building from source code # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem case "$ID" in fedora|centos) pkg_list+=( bash-completion bzip2 gcc gcc-c++ git make ncurses-devel patch rsync tar unzip wget which diffutils python3 python3-devel python3-setuptools python3-pyelftools perl-base perl-Data-Dumper perl-File-Compare perl-File-Copy perl-FindBin perl-IPC-Cmd perl-Thread-Queue perl-Time-Piece perl-JSON-PP swig clang # for qosify llvm15-libs patch) ;; debian|ubuntu) pkg_list+=( build-essential clang flex g++ gawk gcc-multilib gettext git libncurses5-dev libssl-dev python3-distutils rsync unzip zlib1g-dev file wget patch) ;; arch) pkg_list+=( base-devel autoconf automake bash binutils bison bzip2 clang fakeroot file findutils flex gawk gcc gettext git grep groff gzip libelf libtool libxslt m4 make ncurses openssl patch pkgconf python rsync sed texinfo time unzip util-linux wget which zlib patch) ;; *) debug "Skipping dependency install, your OS is unsupported" return 1 ;; esac else # For Imagebuilder case "$ID" in fedora|centos) pkg_list+=( @c-development @development-tools @development-libs perl-FindBin zlib-static elfutils-libelf-devel gawk unzip file wget python3 axel perl-IPC-Cmd) ;; debian|ubuntu) pkg_list+=( build-essential libncurses5-dev libncursesw5-dev zlib1g-dev gawk git gettext libssl-dev xsltproc wget unzip python axel) ;; *) debug "Skipping dependency install, your OS is unsupported" return 1 ;; esac fi pkg_install "${pkg_list[@]}" && echo "${pkg_list[@]}" > "$lock_file" fi } # @description Acquires the OpenWRT Image Builder get_imagebuilder() { debug "${FUNCNAME[0]}" "$*" local -a url_file_pairs=("$@") for ((i=0; i<${#url_file_pairs[@]}; i+=2)); do local url="${url_file_pairs[i]}" local file="${url_file_pairs[i+1]}" # Check if file exists and ask user to remove and redownload if [[ -f $file ]] && ! ask_ok "Use existing $file?"; then execute rm -f "$file" fi # Download the file if it doesn't exist if [[ ! -f "$file" ]]; then echo "Downloading $url to $file using $DL_TOOL" execute "$DL_TOOL" "-o" "$file" "$url" fi done } add_repos() { debug "${FUNCNAME[0]}" if [[ -v P_ARR[repo] ]]; then if ! grep -q "${P_ARR[repo]}" "$BUILD_DIR/repositories.conf"; then echo "${P_ARR[repo]}" >> "$BUILD_DIR/repositories.conf" fi sed -i '/option check_signature/d' "$BUILD_DIR/repositories.conf" fi } ssh_backup() { debug "${FUNCNAME[0]}" local date hostname backup_fname printf -v date '%(%Y-%m-%d-%H-%M-%S)T' hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME) backup_fname="backup-$hostname-$date.tar.gz" [[ -d "$FILES_DIR" ]] || execute mkdir -p "$FILES_DIR" # Make backup archive on remote if ! execute "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$backup_fname"; then echo "SSH backup failed" exit 1 fi # Move backup archive locally if ! execute "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$backup_fname $BUILD_DIR/"; then echo "Could not copy SSH backup" exit 1 fi # Extract backup archive if ! execute "tar -C $FILES_DIR -xzf $BUILD_DIR/$backup_fname"; then echo "Could not extract SSH backup" exit 1 fi execute "rm $BUILD_DIR/$backup_fname" } make_images() { debug "${FUNCNAME[0]}" local -a make_opts # Reuse the existing output # if [[ -d "$BIN_DIR" ]]; then # if ask_ok "$BIN_DIR exists. Rebuild?"; then # execute rm -rf "$BIN_DIR" # else # return 0 # fi # fi ((VERBOSE)) && make_opts+=("V=s") debug make "${make_opts[@]}" image BIN_DIR="$BIN_DIR" \ PROFILE="$DEVICE" PACKAGES="$PACKAGES" \ FILES="$FILES_DIR" --directory="$BUILD_DIR" \ --jobs="$(($(nproc) - 1))" make "${make_opts[@]}" image \ BIN_DIR="$BINDIR" \ PROFILE="$DEVICE" \ PACKAGES="$PACKAGES" \ FILES="$FILES_DIR" \ --directory="$BUILD_DIR" \ --jobs="$(($(nproc) - 1))" \ > "$BUILD_DIR/make.log" } flash_images() { debug "${FUNCNAME[0]}" local img_gz="$1" local dev="$2" local img="${img_gz%.gz}" local partitions if [[ ! -e "$dev" ]]; then echo "The device specified by --flash could not be found" return 1 fi [[ -f $img_gz ]] || { echo "$img_gz does not exist"; return 1; } execute gunzip -qfk "$img_gz" echo "Unmounting target device $dev partitions" partitions=("$dev"?*) execute sudo umount "${partitions[@]}" if execute sudo dd if="$img" of="$dev" bs=2M conv=fsync; then sync echo "Image flashed successfully!" else echo "dd failed!" return 1 fi } ssh_upgrade() { debug "${FUNCNAME[0]}" local img_gz="$1" local ssh_path="$2" local img_fname="${img_gz##*/}" [[ -f $img_gz ]] || { echo "$img_gz is missing, check build output"; return 1; } echo "Copying '$img_gz' to $ssh_path/tmp/$img_fname" if ! execute scp "$img_gz" "$ssh_path:/tmp/$img_fname"; then echo "Could not copy $img_gz to $ssh_path:/tmp/$img_fname" return 1 fi echo "Executing remote sysupgrade" # This may result in weird exit code from closing the ssh connection # shellcheck disable=SC2029 ssh "$ssh_path" "sysupgrade -F /tmp/$img_fname" return 0 } # @description Builds OpenWRT from source code using the the default buildbot as base # This enables the use of kernel config options in profiles # @arg $1 string .config seed URL from_source() { debug "${FUNCNAME[0]}" "$*" local seed_url="$1" local src_url="https://github.com/openwrt/openwrt.git" local seed_file="$WORKTREE_DIR/.config" local pkg config commit seed_file wt_commit description local -a make_opts config_opts echo "Building from source is under development" # Update source code if [[ ! -d "$SRC_DIR" ]]; then execute mkdir -p "$SRC_DIR" execute git clone "$src_url" "$SRC_DIR" fi git -C "$SRC_DIR" pull # Generate commitish for git worktree case "$RELEASE" in snapshot) wt_commit="origin/main" ;; [0-9][0-9].[0-9][0-9].*) local branch="openwrt-${RELEASE%.*}" local tag="v$RELEASE" if ask_ok "Use $branch branch HEAD (y, recommended) or $tag tag (N)?"; then wt_commit="origin/$branch" else wt_commit="$tag" fi ;; *) debug "Passing '$RELEASE' commit-ish to git worktree" wt_commit="$RELEASE" ;; esac # TODO There's a bug in the make clean functions that seem to invoke a full make if [[ -d "$WORKTREE_DIR" ]]; then execute git -C "$WORKTREE_DIR" checkout "$wt_commit" execute git -C "$WORKTREE_DIR" pull else execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit" fi # To workaround bug, don't use make *clean, blow it away and start fresh # [[ -d "$WORKTREE_DIR" ]] && execute rm -rf "$WORKTREE_DIR" # execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit" # Print commit information commit=$(git -C "$WORKTREE_DIR" rev-parse HEAD) description=$(git -C "$WORKTREE_DIR" describe) echo "Current commit hash: $commit" echo "Git worktree description: $description" ((DEBUG)) && git --no-pager -C "$WORKTREE_DIR" log -1 # Enter worktree execute pushd "$WORKTREE_DIR" || return 1 # Cleanup build environment ((VERBOSE)) && make_opts+=("V=s") execute make "${make_opts[@]}" "-j1" dirclean # make clean # compiled output # make targetclean # compiled output, toolchain # make dirclean # compiled output, toolchain, build tools # make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache # Update package feed ./scripts/feeds update -i -f && ./scripts/feeds update -a -f && ./scripts/feeds install -a -f # Grab the release seed config if ! execute "$DL_TOOL" "-o" "$seed_file" "$seed_url"; then echo "Could not obtain $seed_file from $seed_url" return 1 fi # Set compilation output dir config_opts+=("CONFIG_BINARY_FOLDER=\"$BIN_DIR\"") # Add custom packages for pkg in $PACKAGES; do if [[ $pkg == -* ]]; then config_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package else config_opts+=("CONFIG_PACKAGE_$pkg=y") # add package fi done # Add config options from profile for config in ${P_ARR[config]}; do config_opts+=("$config") done # Only compile selected fs execute sed -i '/CONFIG_TARGET_ROOTFS_/d' "$seed_file" config_opts+=("CONFIG_TARGET_PER_DEVICE_ROOTFS=n") if [[ $FILESYSTEM == "squashfs" ]]; then config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=n") config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=y") elif [[ $FILESYSTEM == "ext4" ]]; then config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=n") config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=y") fi # Only compile selected target image execute sed -i '/CONFIG_TARGET_DEVICE_/d' "$seed_file" config_opts+=("CONFIG_TARGET_MULTI_PROFILE=n") config_opts+=("CONFIG_TARGET_PROFILE=DEVICE_$DEVICE") config_opts+=("CONFIG_TARGET_${TARGET//\//_}_DEVICE_$DEVICE=y") config_opts+=("CONFIG_SDK=n") config_opts+=("CONFIG_SDK_LLVM_BPF=n") config_opts+=("CONFIG_IB=n") config_opts+=("CONFIG_MAKE_TOOLCHAIN=n") # Write options to config seed file for config in "${config_opts[@]}"; do debug "Writing $config to $seed_file" echo "$config" >> "$seed_file" done # Make prep execute make "${make_opts[@]}" "-j1" defconfig execute make "${make_opts[@]}" "-j1" download # ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)+1))") make_opts+=("-j$(($(nproc)+1))") # Make image if ! execute ionice -c 3 chrt --idle 0 nice -n19 make "${make_opts[@]}" world; then echo "Error: make failed" return 1 fi execute popd || return 1 # Symlink output images to root of BIN_DIR (match Image Builder) shopt -s nullglob for image in "$BIN_DIR/targets/${TARGET}/"*.{img,img.gz,ubi}; do execute ln -fs "$image" "$BIN_DIR/${image##*/}" done shopt -u nullglob return 0 } # @description Backs up a file to a chosen directory using its timestamp # @arg $1 string File to backup # @arg $2 string Directory to backup to backup() { debug "${FUNCNAME[0]}" "$*" local file="$1" dir="$2" local creation_date base_name backup_file [[ -f $file ]] || { debug "File not found: $file"; return 1; } [[ -d $dir ]] || execute mkdir -p "$dir" || { debug "Failed to create directory: $dir"; return 1; } if creation_date=$(stat -c %w "$file" 2>/dev/null || stat -c %y "$file" 2>/dev/null) && \ [[ $creation_date != "-" && -n $creation_date ]] && \ creation_date=$(date -d "$creation_date" +%y%m%d%H%M 2>/dev/null); then debug "Creation date: $creation_date" else creation_date="unknown" debug "Unable to determine creation date, using 'unknown'" fi base_name="${file##*/}" backup_file="$dir/$creation_date-$base_name" [[ -f $backup_file ]] || execute cp --archive "$file" "$backup_file" } # @section Helper functions # @internal debug() { ((DEBUG)) && echo "Debug: $*"; } ask_ok() { ((YES)) && return local r read -r -p "$* [y/N]: " r r=${r,,} [[ "$r" =~ ^(yes|y)$ ]] } extract() { local archive="$1" local out_dir="$2" if ! execute tar -axf "$archive" -C "$out_dir" --strip-components 1; then echo "Extraction failed" return 1 fi } verify() { local file_to_check="$1" sum_file="$2" local checksum command -v sha256sum &>/dev/null || return 1 [[ -f $sum_file && -f $file_to_check ]] || return 1 checksum=$(grep "${file_to_check##*/}" "$sum_file" | cut -f1 -d' ') echo -n "$checksum $file_to_check" | sha256sum --check --status } load() { local source_file="$1" # shellcheck disable=SC1090 [[ -f $source_file ]] && source "$source_file" } execute() { if debug "$*"; then "$@" else "$@" &>/dev/null fi } # @description The openwrtbuilder main function # @internal main() { debug "${FUNCNAME[0]}" init load "$SCRIPT_DIR/profiles" parse_input "$@" # Fallback to SCRIPT_DIR if BUILD_ROOT has not been set declare -g BUILD_ROOT="${BUILD_ROOT:=$SCRIPT_DIR}" declare -g FILES_DIR="${FILES_DIR:=$BUILD_ROOT/src/files}" declare -g BACKUP_DIR="$SCRIPT_DIR/backups" # This could be dangerous if [[ $BUILD_ROOT == "/" ]]; then echo "Invalid --buildroot" exit 1 fi for dir in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do [[ -d "$dir" ]] || execute mkdir -p "$dir" done # Allow --reset without a profile if ((RESET)) && [[ ${#PROFILES} -lt 1 ]]; then for d in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do ask_ok "Remove $d?" && execute rm -rf "$d" done exit $? fi install_dependencies for profile in "${PROFILES[@]}"; do debug "Running profile: $profile" if [[ ! ${!profile@a} = A ]]; then echo "Profile '$profile' does not exist" return 1 fi # Store profile in P_ARR nameref declare -gn P_ARR="$profile" declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}" declare -g TARGET="${P_ARR[target]}" declare -g DEVICE="${P_ARR[device]}" declare -g PACKAGES="${P_ARR[packages]:-}" declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}" # normalize RELEASE case "$RELEASE" in snapshot|latest|main|master) RELEASE="snapshot" ;; v[0-9][0-9].[0-9][0-9].*) RELEASE="${RELEASE#v}" ;; [0-9][0-9].[0-9][0-9].*) ;; *) if ! ((FROM_SOURCE)); then echo "Error: Invalid release version format" echo "Use semantic version, tag, or 'snapshot'" exit 1 fi ;; esac declare -g SRC_DIR="$BUILD_ROOT/src/openwrt" declare -g WORKTREE_DIR="$BUILD_ROOT/src/$profile/$RELEASE-src" declare -g BUILD_DIR="$BUILD_ROOT/src/$profile/$RELEASE" declare -g BIN_DIR="$BUILD_ROOT/bin/$profile/$RELEASE" if [[ "$RELEASE" == "snapshot" ]]; then local url_prefix="https://downloads.openwrt.org/snapshots/targets/$TARGET" local url_filename="openwrt-imagebuilder-${TARGET//\//-}.Linux-x86_64.tar.zst" local img_fname="openwrt-${TARGET//\//-}-$DEVICE-$FILESYSTEM" else local url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/$TARGET" local url_filename="openwrt-imagebuilder-$RELEASE-${TARGET//\//-}.Linux-x86_64.tar.xz" local img_fname="openwrt-$RELEASE-${TARGET//\//-}-$DEVICE-$FILESYSTEM" fi local ib_url="$url_prefix/$url_filename" local ib_file="$BUILD_DIR/$url_filename" local ib_sha256_url="$url_prefix/sha256sums" local ib_sha256_file="$BUILD_DIR/sha256sums" local seed_url="$url_prefix/config.buildinfo" if ((FROM_SOURCE)); then declare -g SYSUPGRADEIMGGZ="$BIN_DIR/targets/$TARGET/$img_fname-sysupgrade.img.gz" else declare -g SYSUPGRADEIMGGZ="$BUILD_DIR/$img_fname-sysupgrade.img.gz" fi backup "$SYSUPGRADEIMGGZ" "$BACKUP_DIR/$profile/$RELEASE" if ((RESET)); then if ((FROM_SOURCE)); then [[ -d $WORKTREE_DIR ]] && ask_ok "Remove $WORKTREE_DIR?" execute git worktree remove --force "$WORKTREE_DIR" execute rm -rf "$WORKTREE_DIR" elif [[ -d $BUILD_DIR ]] && ask_ok "Remove $BUILD_DIR?"; then execute rm -rf "$BUILD_DIR" fi fi if ((DEBUG)); then echo "Profile settings:" for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done echo "Environment variables:" declare -p fi if ((FROM_SOURCE)); then from_source "$seed_url" || return $? else [[ -d $BUILD_DIR ]] || mkdir -p "$BUILD_DIR" get_imagebuilder "$ib_url" "$ib_file" "$ib_sha256_url" "$ib_sha256_file" && verify "$ib_file" "$ib_sha256_file" && extract "$ib_file" "$BUILD_DIR" || return $? add_repos make_images # Verify output image for stock builds (in testing) if [[ ! -v P_ARR[packages] || -z ${P_ARR[packages]} ]]; then shopt -s nullglob local -a outfiles=("$BIN_DIR"/*.img.gz "$BIN_DIR"/*.img) shopt -u nullglob for outfile in "${outfiles[@]}"; do verify "$outfile" "$ib_sha256_file" || return 1 done fi #copyFiles fi [[ -v SSH_BACKUP_PATH ]] && ssh_backup [[ -v SSH_UPGRADE_PATH ]] && ssh_upgrade "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH" [[ -v FLASH_DEV ]] && flash_images "$SYSUPGRADEIMGGZ" "$FLASH_DEV" done } main "$@" exit