Files
openwrtbuilder/openwrtbuilder
2023-02-14 10:45:11 -05:00

847 lines
23 KiB
Bash
Executable File

#!/usr/bin/env bash
#
# Copyright 2022-23 Bryan C. Roessler
#
# Build and deploy OpenWRT images
#
# Apache 2.0 License
#
# See README.md and ./profiles
#
# This will install a lot of system dependencies by default so I
# recommend running it in `toolbox`, container, or VM
#
# Some PROFILE options are incompatible with Image Builder mode (kopts) and
# will be ignored
#
# Set default release
: "${RELEASE:="22.03.3"}"
printHelp() {
debug "${FUNCNAME[0]}"
cat <<-'EOF'
Build and deploy OpenWRT images
USAGE:
openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]...
OPTIONS
--profile,-p PROFILE
--release,-r,--version,-v RELEASE ("snapshot", "22.03.3")
--buildroot,-b PATH
Default: location of openwrtbuilder script
--source
Build image from source, not from Image Builder
--ssh-upgrade HOST
Examples: root@192.168.1.1, root@router.lan
--ssh-backup SSH_PATH
Enabled by default for --ssh-upgrade
--flash,-f DEVICE
Example: /dev/sdX
--reset
Cleanup all source and output files
--debug,-d
--help,-h
EXAMPLES
./openwrtbuilder -p r4s -r snapshot
./openwrtbuilder -p ax6000_stock -r 22.03.3 --source --debug
./openwrtbuilder -p rpi4 -r 22.03.3 --flash /dev/sdX
./openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1
EOF
}
init() {
debug "${FUNCNAME[0]}"
declare -g ID RPM_MGR SCRIPTDIR DL_TOOL
(( DEBUG )) || echo "To enable debugging output, use --debug or -d"
# Save the script directory
# https://stackoverflow.com/a/4774063
SCRIPTDIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)"
if [[ -e "/etc/os-release" ]]; then
source "/etc/os-release"
else
err "/etc/os-release not found"
err "Your OS is unsupported"
printHelp
exit 1
fi
debug "Detected host platform: $ID"
# normalize distro ID
case "$ID" in
debian|arch)
;;
centos|fedora)
if hash dnf &>/dev/null; then
RPM_MGR="dnf"
elif hash yum &>/dev/null; then
RPM_MGR="yum"
fi
;;
rhel)
ID="centos"
;;
linuxmint|neon|*ubuntu*)
ID="ubuntu"
;;
*suse*)
ID="suse"
;;
raspbian)
ID="debian"
;;
*)
echo "Autodetecting distro, this may be unreliable"
if hash dnf &>/dev/null; then
ID="fedora"
RPM_MGR="dnf"
elif hash yum &>/dev/null; then
ID="centos"
RPM_MGR="yum"
elif hash apt &>/dev/null; then
ID="ubuntu"
elif hash pacman &>/dev/null; then
ID="arch"
else
return 1
fi
;;
esac
debug "Using host platform: $ID"
# Set distro-specific functions
case "$ID" in
fedora|centos)
pkg_install(){ sudo "$RPM_MGR" install -y "$@"; }
;;
debian|ubuntu)
pkg_install(){ sudo apt-get install -y -q0 "$@"; }
;;
suse)
pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; }
;;
arch)
pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; }
;;
esac
if hash axel &>/dev/null; then
DL_TOOL="axel"
elif hash curl &>/dev/null; then
DL_TOOL="curl"
else
echo "Downloading the Image Builder requires axel or curl"
return 1
fi
}
readInput() {
debug "${FUNCNAME[0]}"
unset RESET
declare -ga PROFILES
declare long_opts='release:,version:,profile:,buildroot:,source,'
long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,debug,help'
if _input=$(getopt -o +r:v:p:b:sf:dh -l $long_opts -- "$@"); then
eval set -- "$_input"
while true; do
case "$1" in
--release|-r|--version|-v)
shift && declare -g USER_RELEASE="$1"
;;
--profile|-p)
shift && PROFILES+=("$1")
;;
--buildroot|-b)
shift && BUILDROOT="$1"
;;
--source|-s)
FROM_SOURCE=1
;;
--ssh-upgrade)
shift && SSH_UPGRADE_PATH="$1"
;;
--ssh-backup)
shift && SSH_BACKUP_PATH="$1"
;;
--flash|-f)
shift && FLASH_DEV="$1"
;;
--reset)
RESET=1
;;
--debug|-d)
echo "Debugging on"
DEBUG=1
;;
--help|-h)
printHelp && exit 0
;;
--)
shift
break
;;
esac
shift
done
else
echo "Incorrect options provided"
printHelp && exit 1
fi
}
installDependencies() {
debug "${FUNCNAME[0]}"
declare -a pkg_list
# TODO please contribute your platform here
if (( FROM_SOURCE )); then
# For building from source with make
# https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
case "$ID" in
fedora|centos)
pkg_list+=(
"bash-completion"
"bzip2"
"gcc"
"gcc-c++"
"git"
"make"
"ncurses-devel"
"patch"
"rsync"
"tar"
"unzip"
"wget"
"which"
"diffutils"
"python2"
"python3"
"perl-base"
"perl-Data-Dumper"
"perl-File-Compare"
"perl-File-Copy"
"perl-FindBin"
"perl-Thread-Queue"
"clang" # for qosify
)
;;
debian|ubuntu)
pkg_list+=(
"build-essential"
"clang"
"flex"
"g++"
"gawk"
"gcc-multilib"
"gettext"
"git"
"libncurses5-dev"
"libssl-dev"
"python3-distutils"
"rsync"
"unzip"
"zlib1g-dev"
"file"
"wget"
)
;;
arch)
pkg_list+=(
"base-devel"
"autoconf"
"automake"
"bash"
"binutils"
"bison"
"bzip2"
"clang"
"fakeroot"
"file"
"findutils"
"flex"
"gawk"
"gcc"
"gettext"
"git"
"grep"
"groff"
"gzip"
"libelf"
"libtool"
"libxslt"
"m4"
"make"
"ncurses"
"openssl"
"patch"
"pkgconf"
"python"
"rsync"
"sed"
"texinfo"
"time"
"unzip"
"util-linux"
"wget"
"which"
"zlib"
)
;;
*)
debug "Skipping dependency install, your OS is unsupported"
return 1
;;
esac
else
# For Imagebuilder
case "$ID" in
fedora|centos)
pkg_list+=(
"@c-development"
"@development-tools"
"@development-libs"
"perl-FindBin"
"zlib-static"
"elfutils-libelf-devel"
"gawk"
"unzip"
"file"
"wget"
"python3"
"python2"
"axel"
)
;;
debian|ubuntu)
pkg_list+=(
"build-essential"
"libncurses5-dev"
"libncursesw5-dev"
"zlib1g-dev"
"gawk"
"git"
"gettext"
"libssl-dev"
"xsltproc"
"wget"
"unzip"
"python"
"axel"
)
;;
*)
debug "Skipping dependency install, your OS is unsupported"
return 1
;;
esac
fi
pkg_install "${pkg_list[@]}"
}
getImageBuilder() {
debug "${FUNCNAME[0]}"
declare url="$1"
if [[ -f "$IB_ARCHIVE" ]]; then
if askOK "$IB_ARCHIVE exists. Re-download?"; then
rm -f "$IB_ARCHIVE"
else
return 0
fi
fi
echo "Downloading Image Builder archive using $DL_TOOL"
debug "$DL_TOOL -o $IB_ARCHIVE $url"
"$DL_TOOL" -o "$IB_ARCHIVE" "$url"
}
getImageBuilderChecksum() {
debug "${FUNCNAME[0]}"
if [[ -f $IB_SHA256_FILE ]]; then
if askOk "$IB_SHA256_FILE exists. Re-download?"; then
rm -f "$IB_SHA256_FILE"
else
return 0
fi
fi
execute "$DL_TOOL -o $IB_SHA256_FILE $IB_SHA256_URL"
}
addRepos() {
debug "${FUNCNAME[0]}"
if [[ -v P_ARR[repo] ]]; then
if ! grep -q "${P_ARR[repo]}" "$BUILDDIR/repositories.conf"; then
echo "${P_ARR[repo]}" >> "$BUILDDIR/repositories.conf"
fi
sed -i '/option check_signature/d' "$BUILDDIR/repositories.conf"
fi
}
sshBackup() {
debug "${FUNCNAME[0]}"
declare date hostname backup_fname
[[ -d "$FILESDIR" ]] || mkdir -p "$FILESDIR"
printf -v date '%(%Y-%m-%d-%H-%M-%S)T'
hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME)
backup_fname="backup-$hostname-$date.tar.gz"
# Make backup archive on remote
if ! execute "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$backup_fname"; then
echo "SSH backup failed"
exit 1
fi
# Move backup archive locally
if ! execute "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$backup_fname $BUILDDIR/"; then
echo "Could not copy SSH backup"
exit 1
fi
# Extract backup archive
if ! execute "tar -C $FILESDIR -xzf $BUILDDIR/$backup_fname"; then
echo "Could not extract SSH backup"
exit 1
fi
execute "rm $BUILDDIR/$backup_fname"
}
makeImages() {
debug "${FUNCNAME[0]}"
# Reuse the existing output
if [[ -d "$BINDIR" ]]; then
if askOk "$BINDIR exists. Rebuild?"; then
rm -rf "$BINDIR"
else
return 0
fi
fi
make image \
BIN_DIR="$BINDIR" \
PROFILE="$PROFILE" \
PACKAGES="$PACKAGES" \
FILES="${FILESDIR}" \
--directory="$BUILDDIR" \
--jobs="$(nproc)" \
> "$BUILDDIR/make.log"
}
verifyImages() {
debug "${FUNCNAME[0]}"
declare outfile
for outfile in "$BINDIR"/*.img.gz; do
verify "$outfile" "$IB_OUT_SHA256_FILE" || return 1
done
}
flashImage() {
debug "${FUNCNAME[0]}"
declare img_gz="$1"
declare dev="$2"
declare img="${img_gz%.gz}"
declare partitions
if [[ ! -e "$dev" ]]; then
echo "The device specified by --flash could not be found"
return 1
fi
if [[ ! -f $img_gz ]]; then
echo "$img_gz does not exist"
echo "Check your build output"
return 1
fi
debug "gunzip -qfk $img_gz"
gunzip -qfk "$img_gz"
echo "Unmounting target device $dev partitions"
partitions=( "$dev"?* )
debug "umount ${partitions[*]}"
sudo umount "${partitions[@]}"
debug "sudo dd if=$img of=$dev bs=2M conv=fsync"
if sudo dd if="$img" of="$dev" bs=2M conv=fsync; then
sync
echo "Image flashed sucessfully!"
else
echo "dd failed!"
exit 1
fi
}
sshUpgrade() {
debug "${FUNCNAME[0]}"
declare img_gz="$1"
declare ssh_path="$2"
declare img_fname="${img_gz##*/}"
if ! [[ -f $img_gz ]]; then
echo "$img_gz is missing, check build output"
return 1
fi
echo "Copying '$img_gz' to $ssh_path/tmp/$img_fname"
debug "scp $img_gz $ssh_path:/tmp/$img_fname"
if ! scp "$img_gz" "$ssh_path:/tmp/$img_fname"; then
echo "Could not copy $img_gz to $ssh_path:/tmp/$img_fname"
return 1
fi
echo "Executing remote sysupgrade"
debug "ssh $ssh_path sysupgrade -F /tmp/$img_fname"
# shellcheck disable=SC2029
# execute remotely
# this will probably be a weird exit code from closed connection
ssh "$ssh_path" "sysupgrade -F /tmp/$img_fname"
}
fromSource() {
debug "${FUNCNAME[0]}"
declare src_dir="$BUILDROOT/src/openwrt"
declare src_url="https://github.com/openwrt/openwrt.git"
declare pkg kopt opt commit gitworktreedir seed_file
declare -a make_opts config_opts
echo "Building from source is under development"
# Update source code
if [[ ! -d "$src_dir" ]]; then
mkdir -p "$src_dir"
git clone "$src_url" "$src_dir"
fi
git -C "$src_dir" pull
debug "Current commit:"
(( DEBUG )) && git -C "$src_dir" log -1
(( DEBUG )) && git -C "$src_dir" describe
commit=$(git -C "$src_dir" rev-parse HEAD)
debug "Current commit hash: $commit"
if [[ $RELEASE == "snapshot" ]]; then
gitworktreedir="$BUILDROOT/src/$profile/$RELEASE"
git -C "$src_dir" \
worktree add \
--force \
--detach \
"$gitworktreedir" \
"master"
else
gitworktreedir="$BUILDROOT/src/$profile/$RELEASE"
git -C "$src_dir" \
worktree add \
--force \
--detach \
"$gitworktreedir" \
"origin/openwrt-${RELEASE%.*}"
fi
seed_file="$gitworktreedir/.config"
debug "seed_file: $seed_file"
pushd "$gitworktreedir" &>/dev/null || return 1
# Update package feed
./scripts/feeds update -a &&
./scripts/feeds install -a
# Grab the release seed config
if ! curl -so "$seed_file" "$SEED_URL"; then
echo "Could not obtain $seed_file from $SEED_URL"
return 1
fi
# Set compilation output dir
config_opts+=("CONFIG_BINARY_FOLDER=\"$BINDIR\"")
# Add custom packages
for pkg in $PACKAGES; do
if [[ $pkg == -* ]]; then
config_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package
else
config_opts+=("CONFIG_PACKAGE_$pkg=y") # add package
fi
done
# Add kopts from profile
for kopt in ${P_ARR[kopts]}; do
config_opts+=("$kopt")
done
# Only compile selected fs
# sed -i '/CONFIG_TARGET_ROOTFS_/d' "$seed_file"
# config_opts+=("CONFIG_TARGET_PER_DEVICE_ROOTFS=n")
# if [[ $FILESYSTEM == "squashfs" ]]; then
# config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=n")
# config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=y")
# elif [[ $FILESYSTEM == "ext4" ]]; then
# config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=n")
# config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=y")
# fi
# Only compile selected target image
sed -i '/CONFIG_TARGET_DEVICE_/d' "$seed_file"
config_opts+=("CONFIG_TARGET_MULTI_PROFILE=n")
config_opts+=("CONFIG_TARGET_PROFILE=DEVICE_$PROFILE")
config_opts+=("CONFIG_TARGET_${TARGET//\//_}_DEVICE_$PROFILE=y")
config_opts+=("CONFIG_SDK=n")
config_opts+=("CONFIG_SDK_LLVM_BPF=n")
config_opts+=("CONFIG_IB=n")
config_opts+=("CONFIG_MAKE_TOOLCHAIN=n")
# Write options to config seed file
for opt in "${config_opts[@]}"; do
debug "Writing $opt to $seed_file"
echo "$opt" >> "$seed_file"
done
# [[ -v P_ARR[nested_kopts] ]] &&
# echo "${P_ARR[nested_kopts]}" >> "$seed_file"
# Cleaning modes
# make clean # compiled output
# make targetclean # compiled output, toolchain
# make dirclean # compiled output, toolchain, build tools
# make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache
# Make image
(( DEBUG )) && make_opts+=("V=s")
debug "make ${make_opts[*]} defconfig"
make "${make_opts[@]}" defconfig
debug "make ${make_opts[*]} targetclean"
make "${make_opts[@]}" targetclean
debug "make ${make_opts[*]} download"
make "${make_opts[@]}" download
debug "make ${make_opts[*]} -j$(nproc) world"
make "${make_opts[@]}" -j"$(nproc)" world
popd &>/dev/null || return 1
# Provide symlinks to images in root of BINDIR (to match Image Builder)
for image in "$BINDIR/targets/${TARGET}/"*.{img,img.gz,ubi}; do
ln -fs "$image" "$BINDIR/${image##*/}"
done
return 0
}
# Generic helpers
debug() { (( DEBUG )) && echo "Debug: $*"; }
askOk() {
local _response
read -r -p "$* [y/N]" _response
_response=${_response,,}
[[ "$_response" =~ ^(yes|y)$ ]]
}
extract() {
debug "${FUNCNAME[0]}"
declare archive="$1"
declare out_dir="$2"
debug "tar -xf $archive -C $out_dir --strip-components 1"
if ! tar -xf "$archive" -C "$out_dir" --strip-components 1; then
echo "Extraction failed"
return 1
fi
}
verify() {
debug "${FUNCNAME[0]}"
declare file_to_check="$1"
declare sumfile="$2"
declare checksum
hash sha256sum &>/dev/null || return 1
[[ -f $sumfile && -f $file_to_check ]] || return 1
checksum=$(grep "${file_to_check##*/}" "$sumfile" | cut -f1 -d' ')
echo -n "$checksum $file_to_check" | sha256sum --check --status
}
load() {
debug "${FUNCNAME[0]}"
declare source_file="$1"
# shellcheck disable=SC1090
[[ -f $source_file ]] && source "$source_file"
}
execute() {
declare cmd="$*"
debug "$cmd" || cmd+=" &>/dev/null"
eval "${cmd[*]}"
}
main() {
debug "${FUNCNAME[0]}"
init
load "$SCRIPTDIR/profiles"
readInput "$@"
# Fallback to SCRIPTDIR if BUILDROOT has not been set
declare -g BUILDROOT="${BUILDROOT:=$SCRIPTDIR}"
declare -g FILESDIR="${FILESDIR:=$BUILDROOT/src/files}"
# This could be dangerous
if [[ $BUILDROOT == "/" ]]; then
echo "Invalid --buildroot"
exit 1
fi
for dir in "$BUILDROOT/src" "$BUILDROOT/bin"; do
[[ -d "$dir" ]] || mkdir -p "$dir"
done
# Allow --reset without a profile
if (( RESET )) && [[ ${#PROFILES} -lt 1 ]]; then
askOk "Remove $BUILDROOT/src and $BUILDROOT/bin?" || exit $?
debug "rm -rf $BUILDROOT/src $BUILDROOT/bin"
rm -rf "${BUILDROOT:?}/src" "${BUILDROOT:?}/bin"
exit $?
fi
installDependencies
for profile in "${PROFILES[@]}"; do
debug "Starting profile: $profile"
if [[ ! ${!profile@a} = A ]]; then
echo "Profile '$profile' does not exist"
return 1
fi
# Store profile settings in P_ARR
declare -gn P_ARR="$profile"
# release precedence: user input>profile>env>hardcode
declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"
declare -g BUILDDIR="$BUILDROOT/src/$profile/$RELEASE"
declare -g BINDIR="$BUILDROOT/bin/$profile/$RELEASE"
declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
declare -g TARGET="${P_ARR[target]}"
declare -g PROFILE="${P_ARR[profile]}"
declare -g PACKAGES="${P_ARR[packages]:-}"
if (( RESET )); then
askOk "Remove $BUILDDIR and $BINDIR?" || exit $?
debug "rm -rf $BUILDDIR $BINDIR"
rm -rf "$BUILDDIR" "$BINDIR"
fi
[[ -d $BUILDDIR ]] || mkdir -p "$BUILDDIR"
if [[ "$RELEASE" == "snapshot" ]]; then
declare url_prefix="https://downloads.openwrt.org/snapshots/targets/$TARGET"
declare url_filename="openwrt-imagebuilder-${TARGET//\//-}.Linux-x86_64.tar.xz"
declare img_fname="openwrt-${TARGET//\//-}-$PROFILE-$FILESYSTEM"
else
declare url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/$TARGET"
declare url_filename="openwrt-imagebuilder-$RELEASE-${TARGET//\//-}.Linux-x86_64.tar.xz"
declare img_fname="openwrt-$RELEASE-${TARGET//\//-}-$PROFILE-$FILESYSTEM"
fi
declare ib_url="$url_prefix/$url_filename"
if (( FROM_SOURCE )); then
declare -g SYSUPGRADEIMGGZ="$BINDIR/targets/$img_fname-sysupgrade.img.gz"
declare -g SEED_URL="$url_prefix/config.buildinfo"
else
declare -g SYSUPGRADEIMGGZ="$BUILDDIR/$img_fname-sysupgrade.img.gz"
declare -g IB_ARCHIVE="$BUILDDIR/$url_filename"
declare -g IB_SHA256_URL="$url_prefix/sha256sums"
declare -g IB_SHA256_FILE="$IB_ARCHIVE.sha256sums"
declare -g IB_OUT_SHA256_FILE="$BINDIR/sha256sums"
fi
if (( DEBUG )); then
echo "Profile settings:"
for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
echo "Build settings:"
cat <<- EOF
ALIAS (\$profile, \$P_ARR -- should match)=$profile, $P_ARR
BUILDROOT=$BUILDROOT
BUILDDIR=$BUILDDIR
BINDIR=$BINDIR
TARGET=$TARGET
PROFILE=$PROFILE
RELEASE=$RELEASE
FILESYSTEM=$FILESYSTEM
SYSUPGRADEIMGGZ=$SYSUPGRADEIMGGZ
ib_url=$ib_url
EOF
fi
if (( FROM_SOURCE )); then
fromSource || return $?
else
getImageBuilder "$ib_url" &&
getImageBuilderChecksum &&
verify "$IB_ARCHIVE" "$IB_SHA256_FILE" &&
extract "$IB_ARCHIVE" "$BUILDDIR" || return $?
addRepos
makeImages &&
verifyImages
#copyFiles
fi
[[ -v SSH_BACKUP_PATH ]] &&
sshBackup
[[ -v SSH_UPGRADE_PATH ]] &&
sshUpgrade "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH"
[[ -v FLASH_DEV ]] &&
flashImage "$SYSUPGRADEIMGGZ" "$FLASH_DEV"
done
}
main "$@"
exit
# VM setup (for testing)
# sudo sgdisk -N 0 /dev/vda &&
# sudo mkfs.ext4 /dev/vda1
# mkdir ~/mnt
# sudo mount /dev/vda1 ~/mnt
# sudo chown liveuser:liveuser -R ~/mnt