openwrtbuilder/openwrtbuilder

#!/usr/bin/env bash
#
# Copyright 2022-23 Bryan C. Roessler
#
# Build and deploy OpenWRT images
#
# Apache 2.0 License

# Set default release
: "${RELEASE:="22.03.3"}"

printHelp() {
    debug "${FUNCNAME[0]}"

    cat <<-'EOF'
		Build and deploy OpenWRT images

		USAGE:
		    openwrtbuilder [OPTION [VALUE]] -p PROFILE [-p PROFILE2]...

		OPTIONS
		    --profile, -p PROFILE
		    --info, -i (print profile info)
		    --list-profiles, -l
		    --release, --version, -r, -v RELEASE ("snapshot", "22.03.3") 
		    --buildroot, -b PATH
		    --ssh-upgrade HOST
		        Example: root@192.168.1.1
		    --ssh-backup SSH_PATH
		        (Enabled by default for --ssh-upgrade)
		    --flash, -f DEVICE
		        Example: /dev/sdX
		    --reset
		        Cleanup all source and output files
		    --debug, -d
		    --help, -h
	EOF
}


init() {
    debug "${FUNCNAME[0]}"

    declare -g ID RPM_MGR SCRIPTDIR DL_TOOL

    debug || echo "To enable debugging output, use --debug or -d"

    # Save the script directory
    SCRIPTDIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)"

    if [[ -e "/etc/os-release" ]]; then
        source "/etc/os-release"
    else
        err "/etc/os-release not found"
        err "Your OS is unsupported"
        printHelp
        exit 1
    fi

    debug "Detected host platform: $ID"

    # normalize distro ID
    case "$ID" in
        debian|arch)
            ;;
        centos|fedora)
            if hash dnf &>/dev/null; then
                RPM_MGR="dnf"
            elif hash yum &>/dev/null; then
                RPM_MGR="yum"
            fi
            ;;
        rhel)
            ID="centos"
            ;;
        linuxmint|neon|*ubuntu*)
            ID="ubuntu"
            ;;
        *suse*)
            ID="suse"
            ;;
        raspbian)
            ID="debian"
            ;;
        *)
            echo "Autodetecting distro, this may be unreliable"
            if hash dnf &>/dev/null; then
                ID="fedora"
                RPM_MGR="dnf"
            elif hash yum &>/dev/null; then
                ID="centos"
                RPM_MGR="yum"
            elif hash apt &>/dev/null; then
                ID="ubuntu"
            elif hash pacman &>/dev/null; then
                ID="arch"
            else
                return 1
            fi
            ;;
    esac

    debug "Using host platform: $ID"

    # Set distro-specific functions
    case "$ID" in
        fedora|centos)
            pkg_install(){ sudo "$RPM_MGR" install -y "$@"; }
            ;;
        debian|ubuntu)
            pkg_install(){ sudo apt-get install -y -q0 "$@"; }
            ;;
        suse)
            pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; }
            ;;
        arch)
            pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; }
            ;;
    esac

    if hash axel &>/dev/null; then
        DL_TOOL="axel"
    elif hash curl &>/dev/null; then
        DL_TOOL="curl"
    else
        echo "Downloading the Image Builder requires axel or curl!"
        return 1
    fi
}


readInput() {
    debug "${FUNCNAME[0]}"

    unset RESET

    declare -ga PROFILES
    declare -g PROFILE_INFO
    declare long_opts
    long_opts='release:,version:,profile:,info:,list-profiles,buildroot:,'
    long_opts+='source,ssh-upgrade:,ssh-backup:,flash:,reset,debug,help'

    if _input=$(getopt -o +r:v:p:i:lb:sf:dh -l $long_opts -- "$@"); then
        eval set -- "$_input"
        while true; do
            case "$1" in
                --release|-r|--version|-v)
                    shift && declare -g USER_RELEASE="$1"
                    ;;
                --profile|-p)
                    shift && PROFILES+=("$1")
                    ;;
                --info|-i)
                    PROFILE_INFO=1
                    ;;
                --list-profiles|-l)
                    listProfiles && exit $?
                    ;;
                --buildroot|-b)
                    shift && BUILDROOT="$1"
                    ;;
                --source|-s)
                    FROM_SOURCE=1
                    ;;
                --ssh-upgrade)
                    shift && SSH_UPGRADE_PATH="$1"
                    ;;
                --ssh-backup)
                    shift && SSH_BACKUP_PATH="$1"
                    ;;
                --flash|-f)
                    shift && FLASH_DEV="$1"
                    ;;
                --reset)
                    RESET=1
                    ;;
                --debug|-d)
                    echo "Debugging on"
                    DEBUG=1
                    ;;
                --help|-h)
                    printHelp && exit 0
                    ;;
                --)
                    shift
                    break
                    ;;
            esac
            shift
        done
    else
        echo "Incorrect options provided"
        printHelp && exit 1
    fi
}


listProfiles() {
    debug "${FUNCNAME[0]}"
    grep "declare -Ag" "$PFILE" | cut -d" " -f3
}


installDependencies() {
    debug "${FUNCNAME[0]}"

    declare -a pkg_list

    # TODO please contribute your platform here
    if (( FROM_SOURCE )); then
        # For building from source with make
        # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
        case "$ID" in
            fedora|centos)
                pkg_list+=(
                    "bash-completion"
                    "bzip2"
                    "gcc"
                    "gcc-c++"
                    "git"
                    "make"
                    "ncurses-devel"
                    "patch"
                    "rsync"
                    "tar"
                    "unzip"
                    "wget"
                    "which"
                    "diffutils"
                    "python2"
                    "python3"
                    "perl-base"
                    "perl-Data-Dumper"
                    "perl-File-Compare"
                    "perl-File-Copy"
                    "perl-FindBin"
                    "perl-Thread-Queue"
                )
                ;;
            debian|ubuntu)
                pkg_list+=(
                    "build-essential"
                    "clang"
                    "flex"
                    "g++"
                    "gawk"
                    "gcc-multilib"
                    "gettext"
                    "git"
                    "libncurses5-dev"
                    "libssl-dev"
                    "python3-distutils"
                    "rsync"
                    "unzip"
                    "zlib1g-dev"
                    "file"
                    "wget"
                )
                ;;
            arch)
                pkg_list+=(
                    "base-devel"
                    "autoconf"
                    "automake"
                    "bash"
                    "binutils"
                    "bison"
                    "bzip2"
                    "fakeroot"
                    "file"
                    "findutils"
                    "flex"
                    "gawk"
                    "gcc"
                    "gettext"
                    "git"
                    "grep"
                    "groff"
                    "gzip"
                    "libelf"
                    "libtool"
                    "libxslt"
                    "m4"
                    "make"
                    "ncurses"
                    "openssl"
                    "patch"
                    "pkgconf"
                    "python"
                    "rsync"
                    "sed"
                    "texinfo"
                    "time"
                    "unzip"
                    "util-linux"
                    "wget"
                    "which"
                    "zlib"
                )
                ;;
        esac
    else
        # For Imagebuilder
        case "$ID" in
            fedora|centos)
                pkg_list+=(
                    "@c-development" 
                    "@development-tools"
                    "@development-libs"
                    "perl-FindBin"
                    "zlib-static"
                    "elfutils-libelf-devel"
                    "gawk"
                    "unzip"
                    "file"
                    "wget"
                    "python3"
                    "python2"
                    "axel"
                )
                ;;
            debian|ubuntu)
                pkg_list+=(
                    "build-essential"
                    "libncurses5-dev"
                    "libncursesw5-dev"
                    "zlib1g-dev"
                    "gawk"
                    "git"
                    "gettext"
                    "libssl-dev"
                    "xsltproc"
                    "wget"
                    "unzip"
                    "python"
                    "axel"
                )
                ;;
        esac
    fi

    pkg_install "${pkg_list[@]}"
}


getImageBuilder() {
    debug "${FUNCNAME[0]}"

    if [[ -f "$IB_ARCHIVE" ]]; then
        if askOK "$IB_ARCHIVE exists. Re-download?"; then
            rm -f "$IB_ARCHIVE"
        else
            return 0
        fi
    fi

    echo "Downloading Image Builder archive using $DL_TOOL"
    debug "$DL_TOOL -o $IB_ARCHIVE $IB_URL"
    "$DL_TOOL" -o "$IB_ARCHIVE" "$IB_URL"
}


getImageBuilderChecksum() {
    debug "${FUNCNAME[0]}"

    if [[ -f $IB_SHA256_FILE ]]; then
        if askOk "$IB_SHA256_FILE exists. Re-download?"; then
            rm -f "$IB_SHA256_FILE"
        else
            return 0
        fi
    fi

    "$DL_TOOL" -o "$IB_SHA256_FILE" "$IB_SHA256_URL"
}


extractImageBuilder() {
    debug "${FUNCNAME[0]}"

    echo "Extracting Image Builder archive"
    [[ ! -d "$BUILDDIR" ]] && mkdir -p "$BUILDDIR"
    debug "tar -xf $IB_ARCHIVE -C $BUILDDIR --strip-components 1"
    if ! tar -xf "$IB_ARCHIVE" -C "$BUILDDIR" --strip-components 1; then
        echo "Extraction failed"
        return 1
    fi
}


addRepos() {
    debug "${FUNCNAME[0]}"

    if [[ -v P_ARR[repo] ]]; then
        if ! grep -q "${P_ARR[repo]}" "$BUILDDIR/repositories.conf"; then
            echo "${P_ARR[repo]}" >> "$BUILDDIR/repositories.conf"
        fi
        sed -i '/option check_signature/d' "$BUILDDIR/repositories.conf"
    fi
}


sshBackup() {
    debug "${FUNCNAME[0]}"

    local _date _hostname _backup_fname

    [[ -d "$FILESDIR" ]] || mkdir -p "$FILESDIR"

    printf -v _date '%(%Y-%m-%d-%H-%M-%S)T'
    _hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME)
    _backup_fname="backup-$_hostname-$_date.tar.gz"

    # Make backup archive on remote
    debug "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$_backup_fname"
    if ! ssh -t "$SSH_BACKUP_PATH" "sysupgrade -b /tmp/$_backup_fname"; then
        echo "SSH backup failed"
        exit 1
    fi

    # Move backup archive locally
    debug "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$_backup_fname $BUILDDIR/"
    if ! rsync -avz --remove-source-files \
        "$SSH_BACKUP_PATH":"/tmp/$_backup_fname" "$BUILDDIR/"; then
        echo "Could not copy SSH backup"
        exit 1
    fi

    # Extract backup archive
    debug "tar -C $FILESDIR -xzf $BUILDDIR/$_backup_fname"
    if ! tar -C "$FILESDIR" -xzf "$BUILDDIR/$_backup_fname"; then
        echo "Could not extract SSH backup"
        exit 1
    fi

    rm "$BUILDDIR/$_backup_fname"
}


makeImages() {
    debug "${FUNCNAME[0]}"

    # Reuse the existing output
    if [[ -d "$THIS_BINDIR" ]]; then
        if askOk "$THIS_BINDIR exists. Rebuild?"; then
            rm -rf "$THIS_BINDIR"
        else
            return 0
        fi
    fi

    [[ -d "$BUILDDIR" ]] || mkdir -p "$BUILDDIR"

    make image \
        BIN_DIR="$THIS_BINDIR" \
        PROFILE="${P_ARR[profile]}" \
        PACKAGES="${P_ARR[packages]:+"${P_ARR[packages]}"}" \
        FILES="${FILESDIR}" \
        --directory="$BUILDDIR" \
        --jobs="$(nproc)"  \
        > "$BUILDDIR/make.log"
}


verifyImages() {
    debug "${FUNCNAME[0]}"

    declare outfile

    for outfile in "$THIS_BINDIR"/*.img.gz; do
        verify "$outfile" "$IB_OUT_SHA256_FILE" || return 1
    done
}


flashImage() {
    debug "${FUNCNAME[0]}"

    declare img img_gz partitions

    if [[ ! -e "$FLASH_DEV" ]]; then
        echo "The device specified by --flash could not be found"
        exit 1
    fi

    # TODO Roughly choose the correct image
    if [[ -f "$FACTORYIMGGZ" ]]; then
        img_gz="$FACTORYIMGGZ"
        img="$FACTORYIMG"
    elif [[ -f "$SYSUPGRADEIMGGZ" ]]; then
        img_gz="$SYSUPGRADEIMGGZ"
        img="$SYSUPGRADEIMG"
    else
        return 1
    fi

    debug "$img_gz $img"

    debug "gunzip -qfk $img_gz"
    gunzip -qfk "$img_gz"
        
    echo "Unmounting target device $FLASH_DEV partitions"
    partitions=( "$FLASH_DEV"?* )
    debug "umount ${partitions[*]}"
    sudo umount "${partitions[@]}"

    debug "sudo dd if=\"$img\" of=\"$FLASH_DEV\" bs=2M conv=fsync"
    if sudo dd if="$img" of="$FLASH_DEV" bs=2M conv=fsync; then
        sync
        echo "Image flashed sucessfully!"
    else
        echo "dd failed!"
        exit 1
    fi
}


sshUpgrade() {
    debug "${FUNCNAME[0]}"

    echo "Copying '$SYSUPGRADEIMGGZ' to $SSH_UPGRADE_PATH/tmp/"
    debug "scp \"$SYSUPGRADEIMGGZ\" \"$SSH_UPGRADE_PATH\":\"/tmp/$SYSUPGRADEIMGGZFNAME\""
    
    if ! scp "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH":"/tmp/$SYSUPGRADEIMGGZFNAME"; then
        echo "Could not access the --ssh-upgrade PATH"
        exit 1
    fi

    echo "Executing remote sysupgrade"
    debug "ssh \"$SSH_UPGRADE_PATH\" \"sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME\""
    # shellcheck disable=SC2029
    ssh "$SSH_UPGRADE_PATH" "sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME"
}


fromSource() {
    debug "${FUNCNAME[0]}"

    declare src_url="https://github.com/openwrt/openwrt.git"
    declare pkg kopt

    echo "Building from source is under development"

    if [[ ! -d "$GITSRCDIR" ]]; then
        mkdir -p "$GITSRCDIR"
        git clone "$src_url" "$GITSRCDIR"
    fi

    pushd "$GITSRCDIR" &>/dev/null || return 1

    if [[ ${P_ARR[release]} == "snapshot" ]]; then
        git checkout master
    else
        git checkout "v$RELEASE"
    fi

    # Update package feed
    ./scripts/feeds update -a &&
    ./scripts/feeds install -a

    # Grab the release seed config
    if [[ -f "$SEED_FILE" ]]; then
        if askOk "$SEED_FILE exists. Re-download?"; then
            rm -f "$SEED_FILE"
        fi
    fi
    if [[ -f "$SEED_FILE" ]]; then
        echo "Reusing existing $SEED_FILE"
    else
        if ! curl -so "$SEED_FILE" "$SEED_URL"; then
            echo "Could not obtain seed config"
        fi
    fi

    # Override default .config seed with profile packages
    for pkg in ${P_ARR[packages]:+${P_ARR[packages]}}; do
        if [[ $pkg == -* ]]; then
            echo "CONFIG_PACKAGE_${pkg#-}=n" >> "$SEED_FILE" # remove package
        else
            echo "CONFIG_PACKAGE_$pkg=y" >> "$SEED_FILE" # add package
        fi
    done

    # Add custom kernel config options
    for kopt in ${P_ARR[kopts]:+${P_ARR[kopts]}}; do
        echo "$kopt" >> "$SEED_FILE"
    done

    make defconfig # normalize .config and remove dupes

    # Only compile our target
    # TODO causes a "configuration is out of sync" error w/o defconfig
    echo "CONFIG_TARGET_MULTI_PROFILE=n" >> "$SEED_FILE"
    sed -i '/CONFIG_TARGET_DEVICE_/d' "$SEED_FILE"
    echo "CONFIG_TARGET_DEVICE_${P_ARR[target]//\//_}_DEVICE_${P_ARR[profile]}=y" >> "$SEED_FILE"
    # output to bindir instead of builddir
    echo "CONFIG_BINARY_FOLDER=\"$THIS_BINDIR\"" >> "$SEED_FILE"
    make targetclean
    make download &&
    make -j"$(nproc)" world

    popd &>/dev/null || return 1
    exit # TODO exit here for fromSource() testing
}


debug() { (( DEBUG )) && echo "Debug: $*"; }


askOk() {
    local _response
    read -r -p "$* [y/N]" _response
    _response=${_response,,}
    [[ "$_response" =~ ^(yes|y)$ ]]
}


resetAll() {
    debug "${FUNCNAME[0]}"
    askOk "Remove $SRCDIR and $BINDIR?" || exit $?
    debug "rm -rf $SRCDIR $BINDIR"
    rm -rf "$SRCDIR" "$BINDIR"
}


resetProfile() {
    debug "${FUNCNAME[0]}"
    askOk "Remove $BUILDDIR and $THIS_BINDIR?" || exit $?
    debug "rm -rf $BUILDDIR $THIS_BINDIR"
    rm -rf "$BUILDDIR" "$THIS_BINDIR"
}


loadProfiles() {
    debug "${FUNCNAME[0]}"
    declare -g PFILE
    # https://stackoverflow.com/a/4774063
    PFILE="$SCRIPTDIR/profiles"
    # shellcheck source=./profiles
    ! source "$PFILE" && echo "profiles file missing!" && return 1
}


verify() {
    debug "${FUNCNAME[0]}"
    declare file_to_check="$1"
    declare sumfile="$2"
    declare checksum

    hash sha256sum &>/dev/null || return 1
    [[ -f $sumfile && -f $file_to_check ]] || return 1
    checksum=$(grep "${file_to_check##*/}" "$sumfile" | cut -f1 -d' ')
    echo -n "$checksum $file_to_check" | sha256sum --check --status
}


main() {
    debug "${FUNCNAME[0]}"

    init

    loadProfiles
   
    readInput "$@"

    # Fallback to SCRIPTDIR if BUILDROOT has not been set
    declare -g BUILDROOT="${BUILDROOT:=$SCRIPTDIR}"
    [[ $BUILDROOT == "/" ]] && echo "Invalid --buildroot" && exit 1
    declare -g FILESDIR="${FILESDIR:=$BUILDROOT/src/files}"
    declare -g SRCDIR="$BUILDROOT/src" # input/build
    declare -g BINDIR="$BUILDROOT/bin" # output
    declare -g GITSRCDIR="$SRCDIR/openwrt"

    for dir in "$SRCDIR" "$BINDIR"; do
        [[ -d "$dir" ]] || mkdir -p "$dir"
    done

    # Allow --reset without a profile
    if [[ ${#PROFILES} -lt 1 ]]; then
        if (( RESET )); then
            resetAll
            exit
        else
            echo "No profile supplied" && return 1
        fi
    fi

    installDependencies

    for profile in "${PROFILES[@]}"; do
        debug "Starting profile: $profile"

        if [[ ! ${!profile@a} = A ]]; then
            echo "Profile '$profile' does not exist"
            return 1
        fi

        # Store profile settings in P_ARR
        declare -gn P_ARR="$profile"

        # release precedence: user input>profile>env>hardcode
        declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"
        declare -g BUILDDIR="$SRCDIR/${P_ARR[profile]}-$RELEASE"
        declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
        declare -g THIS_BINDIR="$BINDIR/${P_ARR[profile]}-$RELEASE"

        if [[ "$RELEASE" == "snapshot" ]]; then
            declare url_prefix="https://downloads.openwrt.org/snapshots/targets/${P_ARR[target]}"
            declare url_filename="openwrt-imagebuilder-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
            declare img_prefix="$BUILDDIR/openwrt-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM" 
        else
            declare url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/${P_ARR[target]}"
            declare url_filename="openwrt-imagebuilder-$RELEASE-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
            declare img_prefix="$BUILDDIR/openwrt-$RELEASE-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM"
        fi

        declare -g IB_URL="$url_prefix/$url_filename"
        declare -g IB_ARCHIVE="$SRCDIR/$url_filename"
        declare -g IB_SHA256_URL="$url_prefix/sha256sums"
        declare -g IB_SHA256_FILE="$BUILDDIR/sha256sums"
        declare -g IB_OUT_SHA256_FILE="$THIS_BINDIR/sha256sums"
        declare -g SEED_URL="$url_prefix/config.buildinfo"
        declare -g SEED_FILE="$GITSRCDIR/.config"

        declare -g FACTORYIMG="$img_prefix-factory.img"
        declare -g FACTORYIMGGZ="$img_prefix-factory.img.gz"
        declare -g FACTORYIMGGZFNAME="${FACTORYIMGGZ##*/}"
        declare -g SYSUPGRADEIMG="$img_prefix-sysupgrade.img"
        declare -g SYSUPGRADEIMGGZ="$img_prefix-sysupgrade.img.gz"
        declare -g SYSUPGRADEIMGGZFNAME="${SYSUPGRADEIMGGZ##*/}"

        if (( DEBUG )) || (( PROFILE_INFO )); then
            echo "Profile settings:"
            for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
            echo "Build settings:"
            cat <<- EOF
				ALIAS=$profile
				BUILDROOT=$BUILDROOT
				BUILDDIR=$BUILDDIR
				SRCDIR=$SRCDIR
				BINDIR=$BINDIR
				GITSRCDIR=$GITSRCDIR
				THIS_BINDIR=$THIS_BINDIR
				TARGET=${P_ARR[target]}
				PROFILE=${P_ARR[profile]}
				RELEASE=$RELEASE
				FILESYSTEM=$FILESYSTEM
				IB_URL=$IB_URL
				IB_ARCHIVE=$IB_ARCHIVE
				SEED_URL=$SEED_URL
				SEED_FILE=$SEED_FILE
				IB_SHA256_URL=$IB_SHA256_URL
				IB_SHA256_FILE=$IB_SHA256_FILE
				IB_OUT_SHA256_FILE=$IB_OUT_SHA256_FILE
				FACTORYIMGGZ: $FACTORYIMGGZ
				FACTORYIMGGZFNAME: $FACTORYIMGGZFNAME
				SYSUPGRADEIMGGZ: $SYSUPGRADEIMGGZ
				SYSUPGRADEIMGGZFNAME: $SYSUPGRADEIMGGZFNAME
			EOF
        fi

        (( RESET )) && resetProfile

        (( FROM_SOURCE )) && fromSource

        # Acquire and verify Image Builder
        getImageBuilder && 
        getImageBuilderChecksum &&
        verify "$IB_ARCHIVE" "$IB_SHA256_FILE" || 
        return $?
        
        extractImageBuilder || return $?

        addRepos

        #copyFiles

        [[ -v SSH_BACKUP_PATH ]] && sshBackup

        if makeImages && verifyImages; then
            [[ -v SSH_UPGRADE_PATH ]] && sshUpgrade
            [[ -v FLASH_DEV ]] && flashImage
        fi
    done
}

main "$@"
exit