Files
openwrtbuilder/openwrtbuilder
2023-01-20 19:34:18 -05:00

813 lines
22 KiB
Bash
Executable File

#!/usr/bin/env bash
#
# Copyright 2022-23 Bryan C. Roessler
#
# Build and deploy OpenWRT images
#
# Apache 2.0 License
# Set default release
: "${RELEASE:="22.03.3"}"
printHelp() {
debug "${FUNCNAME[0]}"
cat <<-'EOF'
Build and deploy OpenWRT images
USAGE:
openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]...
OPTIONS
--profile,-p PROFILE
--release,-r,--version,-v RELEASE ("snapshot", "22.03.3")
--buildroot,-b PATH
Defaults to location of openwrtbuilder script
--source
Build image from source, not from Image Builder
--ssh-upgrade HOST
Example: root@192.168.1.1
--ssh-backup SSH_PATH
(Enabled by default for --ssh-upgrade)
--flash,-f DEVICE
Example: /dev/sdX
--reset
Cleanup all source and output files
--debug,-d
--help,-h
EOF
}
init() {
debug "${FUNCNAME[0]}"
declare -g ID RPM_MGR SCRIPTDIR DL_TOOL
debug || echo "To enable debugging output, use --debug or -d"
# Save the script directory
# https://stackoverflow.com/a/4774063
SCRIPTDIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)"
if [[ -e "/etc/os-release" ]]; then
source "/etc/os-release"
else
err "/etc/os-release not found"
err "Your OS is unsupported"
printHelp
exit 1
fi
debug "Detected host platform: $ID"
# normalize distro ID
case "$ID" in
debian|arch)
;;
centos|fedora)
if hash dnf &>/dev/null; then
RPM_MGR="dnf"
elif hash yum &>/dev/null; then
RPM_MGR="yum"
fi
;;
rhel)
ID="centos"
;;
linuxmint|neon|*ubuntu*)
ID="ubuntu"
;;
*suse*)
ID="suse"
;;
raspbian)
ID="debian"
;;
*)
echo "Autodetecting distro, this may be unreliable"
if hash dnf &>/dev/null; then
ID="fedora"
RPM_MGR="dnf"
elif hash yum &>/dev/null; then
ID="centos"
RPM_MGR="yum"
elif hash apt &>/dev/null; then
ID="ubuntu"
elif hash pacman &>/dev/null; then
ID="arch"
else
return 1
fi
;;
esac
debug "Using host platform: $ID"
# Set distro-specific functions
case "$ID" in
fedora|centos)
pkg_install(){ sudo "$RPM_MGR" install -y "$@"; }
;;
debian|ubuntu)
pkg_install(){ sudo apt-get install -y -q0 "$@"; }
;;
suse)
pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; }
;;
arch)
pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; }
;;
esac
if hash axel &>/dev/null; then
DL_TOOL="axel"
elif hash curl &>/dev/null; then
DL_TOOL="curl"
else
echo "Downloading the Image Builder requires axel or curl!"
return 1
fi
}
readInput() {
debug "${FUNCNAME[0]}"
unset RESET
declare -ga PROFILES
declare long_opts='release:,version:,profile:,buildroot:,source,'
long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,debug,help'
if _input=$(getopt -o +r:v:p:b:sf:dh -l $long_opts -- "$@"); then
eval set -- "$_input"
while true; do
case "$1" in
--release|-r|--version|-v)
shift && declare -g USER_RELEASE="$1"
;;
--profile|-p)
shift && PROFILES+=("$1")
;;
--buildroot|-b)
shift && BUILDROOT="$1"
;;
--source|-s)
FROM_SOURCE=1
;;
--ssh-upgrade)
shift && SSH_UPGRADE_PATH="$1"
;;
--ssh-backup)
shift && SSH_BACKUP_PATH="$1"
;;
--flash|-f)
shift && FLASH_DEV="$1"
;;
--reset)
RESET=1
;;
--debug|-d)
echo "Debugging on"
DEBUG=1
;;
--help|-h)
printHelp && exit 0
;;
--)
shift
break
;;
esac
shift
done
else
echo "Incorrect options provided"
printHelp && exit 1
fi
}
installDependencies() {
debug "${FUNCNAME[0]}"
declare -a pkg_list
# TODO please contribute your platform here
if (( FROM_SOURCE )); then
# For building from source with make
# https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
case "$ID" in
fedora|centos)
pkg_list+=(
"bash-completion"
"bzip2"
"gcc"
"gcc-c++"
"git"
"make"
"ncurses-devel"
"patch"
"rsync"
"tar"
"unzip"
"wget"
"which"
"diffutils"
"python2"
"python3"
"perl-base"
"perl-Data-Dumper"
"perl-File-Compare"
"perl-File-Copy"
"perl-FindBin"
"perl-Thread-Queue"
"clang" # for qosify
)
;;
debian|ubuntu)
pkg_list+=(
"build-essential"
"clang"
"flex"
"g++"
"gawk"
"gcc-multilib"
"gettext"
"git"
"libncurses5-dev"
"libssl-dev"
"python3-distutils"
"rsync"
"unzip"
"zlib1g-dev"
"file"
"wget"
)
;;
arch)
pkg_list+=(
"base-devel"
"autoconf"
"automake"
"bash"
"binutils"
"bison"
"bzip2"
"clang"
"fakeroot"
"file"
"findutils"
"flex"
"gawk"
"gcc"
"gettext"
"git"
"grep"
"groff"
"gzip"
"libelf"
"libtool"
"libxslt"
"m4"
"make"
"ncurses"
"openssl"
"patch"
"pkgconf"
"python"
"rsync"
"sed"
"texinfo"
"time"
"unzip"
"util-linux"
"wget"
"which"
"zlib"
)
;;
esac
else
# For Imagebuilder
case "$ID" in
fedora|centos)
pkg_list+=(
"@c-development"
"@development-tools"
"@development-libs"
"perl-FindBin"
"zlib-static"
"elfutils-libelf-devel"
"gawk"
"unzip"
"file"
"wget"
"python3"
"python2"
"axel"
)
;;
debian|ubuntu)
pkg_list+=(
"build-essential"
"libncurses5-dev"
"libncursesw5-dev"
"zlib1g-dev"
"gawk"
"git"
"gettext"
"libssl-dev"
"xsltproc"
"wget"
"unzip"
"python"
"axel"
)
;;
esac
fi
pkg_install "${pkg_list[@]}"
}
getImageBuilder() {
debug "${FUNCNAME[0]}"
if [[ -f "$IB_ARCHIVE" ]]; then
if askOK "$IB_ARCHIVE exists. Re-download?"; then
rm -f "$IB_ARCHIVE"
else
return 0
fi
fi
echo "Downloading Image Builder archive using $DL_TOOL"
debug "$DL_TOOL -o $IB_ARCHIVE $IB_URL"
"$DL_TOOL" -o "$IB_ARCHIVE" "$IB_URL"
}
getImageBuilderChecksum() {
debug "${FUNCNAME[0]}"
if [[ -f $IB_SHA256_FILE ]]; then
if askOk "$IB_SHA256_FILE exists. Re-download?"; then
rm -f "$IB_SHA256_FILE"
else
return 0
fi
fi
"$DL_TOOL" -o "$IB_SHA256_FILE" "$IB_SHA256_URL"
}
extractImageBuilder() {
debug "${FUNCNAME[0]}"
echo "Extracting Image Builder archive"
[[ ! -d "$BUILDDIR" ]] && mkdir -p "$BUILDDIR"
debug "tar -xf $IB_ARCHIVE -C $BUILDDIR --strip-components 1"
if ! tar -xf "$IB_ARCHIVE" -C "$BUILDDIR" --strip-components 1; then
echo "Extraction failed"
return 1
fi
}
addRepos() {
debug "${FUNCNAME[0]}"
if [[ -v P_ARR[repo] ]]; then
if ! grep -q "${P_ARR[repo]}" "$BUILDDIR/repositories.conf"; then
echo "${P_ARR[repo]}" >> "$BUILDDIR/repositories.conf"
fi
sed -i '/option check_signature/d' "$BUILDDIR/repositories.conf"
fi
}
sshBackup() {
debug "${FUNCNAME[0]}"
local _date _hostname _backup_fname
[[ -d "$FILESDIR" ]] || mkdir -p "$FILESDIR"
printf -v _date '%(%Y-%m-%d-%H-%M-%S)T'
_hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME)
_backup_fname="backup-$_hostname-$_date.tar.gz"
# Make backup archive on remote
debug "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$_backup_fname"
if ! ssh -t "$SSH_BACKUP_PATH" "sysupgrade -b /tmp/$_backup_fname"; then
echo "SSH backup failed"
exit 1
fi
# Move backup archive locally
debug "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$_backup_fname $BUILDDIR/"
if ! rsync -avz --remove-source-files \
"$SSH_BACKUP_PATH":"/tmp/$_backup_fname" "$BUILDDIR/"; then
echo "Could not copy SSH backup"
exit 1
fi
# Extract backup archive
debug "tar -C $FILESDIR -xzf $BUILDDIR/$_backup_fname"
if ! tar -C "$FILESDIR" -xzf "$BUILDDIR/$_backup_fname"; then
echo "Could not extract SSH backup"
exit 1
fi
rm "$BUILDDIR/$_backup_fname"
}
makeImages() {
debug "${FUNCNAME[0]}"
# Reuse the existing output
if [[ -d "$THIS_BINDIR" ]]; then
if askOk "$THIS_BINDIR exists. Rebuild?"; then
rm -rf "$THIS_BINDIR"
else
return 0
fi
fi
[[ -d "$BUILDDIR" ]] || mkdir -p "$BUILDDIR"
make image \
BIN_DIR="$THIS_BINDIR" \
PROFILE="${P_ARR[profile]}" \
PACKAGES="${P_ARR[packages]:+"${P_ARR[packages]}"}" \
FILES="${FILESDIR}" \
--directory="$BUILDDIR" \
--jobs="$(nproc)" \
> "$BUILDDIR/make.log"
}
verifyImages() {
debug "${FUNCNAME[0]}"
declare outfile
for outfile in "$THIS_BINDIR"/*.img.gz; do
verify "$outfile" "$IB_OUT_SHA256_FILE" || return 1
done
}
flashImage() {
debug "${FUNCNAME[0]}"
declare img img_gz partitions
if [[ ! -e "$FLASH_DEV" ]]; then
echo "The device specified by --flash could not be found"
exit 1
fi
# TODO Roughly choose the correct image
if [[ -f "$FACTORYIMGGZ" ]]; then
img_gz="$FACTORYIMGGZ"
img="$FACTORYIMG"
elif [[ -f "$SYSUPGRADEIMGGZ" ]]; then
img_gz="$SYSUPGRADEIMGGZ"
img="$SYSUPGRADEIMG"
else
return 1
fi
debug "$img_gz $img"
debug "gunzip -qfk $img_gz"
gunzip -qfk "$img_gz"
echo "Unmounting target device $FLASH_DEV partitions"
partitions=( "$FLASH_DEV"?* )
debug "umount ${partitions[*]}"
sudo umount "${partitions[@]}"
debug "sudo dd if=\"$img\" of=\"$FLASH_DEV\" bs=2M conv=fsync"
if sudo dd if="$img" of="$FLASH_DEV" bs=2M conv=fsync; then
sync
echo "Image flashed sucessfully!"
else
echo "dd failed!"
exit 1
fi
}
sshUpgrade() {
debug "${FUNCNAME[0]}"
echo "Copying '$SYSUPGRADEIMGGZ' to $SSH_UPGRADE_PATH/tmp/"
debug "scp \"$SYSUPGRADEIMGGZ\" \"$SSH_UPGRADE_PATH\":\"/tmp/$SYSUPGRADEIMGGZFNAME\""
if ! scp "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH":"/tmp/$SYSUPGRADEIMGGZFNAME"; then
echo "Could not access the --ssh-upgrade PATH"
exit 1
fi
echo "Executing remote sysupgrade"
debug "ssh \"$SSH_UPGRADE_PATH\" \"sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME\""
# shellcheck disable=SC2029
ssh "$SSH_UPGRADE_PATH" "sysupgrade -F /tmp/$SYSUPGRADEIMGGZFNAME"
}
fromSource() {
debug "${FUNCNAME[0]}"
declare src_url="https://github.com/openwrt/openwrt.git"
declare pkg kopt
declare -a make_opts
echo "Building from source is under development"
if [[ ! -d "$GITSRCDIR" ]]; then
mkdir -p "$GITSRCDIR"
git clone "$src_url" "$GITSRCDIR"
fi
pushd "$GITSRCDIR" &>/dev/null || return 1
if [[ ${P_ARR[release]} == "snapshot" ]]; then
git checkout master
else
git checkout "v$RELEASE"
fi
git pull
# Update package feed
./scripts/feeds update -a &&
./scripts/feeds install -a
# Grab the release seed config
if [[ -f "$SEED_FILE" ]]; then
if askOk "$SEED_FILE exists. Re-download?"; then
rm -f "$SEED_FILE"
fi
fi
if [[ -f "$SEED_FILE" ]]; then
echo "Reusing existing $SEED_FILE seed config"
else
if ! curl -so "$SEED_FILE" "$SEED_URL"; then
echo "Could not obtain seed config"
fi
fi
make defconfig # normalize .config and remove dupes
for pkg in ${P_ARR[packages]:+${P_ARR[packages]}}; do
if [[ $pkg == -* ]]; then
make_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package
else
make_opts+=("CONFIG_PACKAGE_$pkg=y") # add package
fi
done
for kopt in ${P_ARR[kopts]:+${P_ARR[kopts]}}; do
make_opts+=("$kopt")
done
make_opts+=("CONFIG_TARGET_MULTI_PROFILE=n")
make_opts+=("CONFIG_TARGET_PROFILE=DEVICE_${P_ARR[profile]}")
make_opts+=("CONFIG_TARGET_DEVICE_${P_ARR[target]//\//_}_DEVICE_${P_ARR[profile]}=y")
make_opts+=("CONFIG_BINARY_FOLDER=\"$THIS_BINDIR\"")
# Override default .config seed with profile packages
# for pkg in ${P_ARR[packages]:+${P_ARR[packages]}}; do
# if [[ $pkg == -* ]]; then
# echo "CONFIG_PACKAGE_${pkg#-}=n" >> "$SEED_FILE" # remove package
# else
# echo "CONFIG_PACKAGE_$pkg=y" >> "$SEED_FILE" # add package
# fi
# done
# Add custom kernel config options
# for kopt in ${P_ARR[kopts]:+${P_ARR[kopts]}}; do
# echo "$kopt" >> "$SEED_FILE"
# done
# Only compile our target
# TODO causes a "configuration is out of sync" error w/o defconfig
# sed -i '/CONFIG_TARGET_DEVICE_/d' "$SEED_FILE"
# echo "CONFIG_TARGET_DEVICE_${P_ARR[target]//\//_}_DEVICE_${P_ARR[profile]}=y" >> "$SEED_FILE"
# output to bindir instead of builddir
# echo "CONFIG_BINARY_FOLDER=\"$THIS_BINDIR\"" >> "$SEED_FILE"
# TODO symlink clang for qosify
[[ -d "$GITSRCDIR/staging_dir/host/llvm-bpf/bin" ]] || mkdir -p "$GITSRCDIR/staging_dir/host/llvm-bpf/bin"
ln -fs "$(which clang)" "$GITSRCDIR/staging_dir/host/llvm-bpf/bin/clang"
make targetclean
make "${make_opts[@]}" download &&
debug "${make_opts[*]} -j$(nproc) world" &&
make "${make_opts[@]}" -j"$(nproc)" world
popd &>/dev/null || return 1
exit # TODO exit here for fromSource() testing
}
debug() { (( DEBUG )) && echo "Debug: $*"; }
askOk() {
local _response
read -r -p "$* [y/N]" _response
_response=${_response,,}
[[ "$_response" =~ ^(yes|y)$ ]]
}
resetAll() {
debug "${FUNCNAME[0]}"
askOk "Remove $SRCDIR and $BINDIR?" || exit $?
debug "rm -rf $SRCDIR $BINDIR"
rm -rf "$SRCDIR" "$BINDIR"
}
resetProfile() {
debug "${FUNCNAME[0]}"
askOk "Remove $BUILDDIR and $THIS_BINDIR?" || exit $?
debug "rm -rf $BUILDDIR $THIS_BINDIR"
rm -rf "$BUILDDIR" "$THIS_BINDIR"
}
loadProfiles() {
debug "${FUNCNAME[0]}"
declare -g PFILE
PFILE="$SCRIPTDIR/profiles"
# shellcheck source=./profiles
! source "$PFILE" && echo "profiles file missing!" && return 1
}
verify() {
debug "${FUNCNAME[0]}"
declare file_to_check="$1"
declare sumfile="$2"
declare checksum
hash sha256sum &>/dev/null || return 1
[[ -f $sumfile && -f $file_to_check ]] || return 1
checksum=$(grep "${file_to_check##*/}" "$sumfile" | cut -f1 -d' ')
echo -n "$checksum $file_to_check" | sha256sum --check --status
}
main() {
debug "${FUNCNAME[0]}"
init
loadProfiles
readInput "$@"
# Fallback to SCRIPTDIR if BUILDROOT has not been set
declare -g BUILDROOT="${BUILDROOT:=$SCRIPTDIR}"
[[ $BUILDROOT == "/" ]] && echo "Invalid --buildroot" && exit 1
declare -g FILESDIR="${FILESDIR:=$BUILDROOT/src/files}"
declare -g SRCDIR="$BUILDROOT/src" # input/build
declare -g BINDIR="$BUILDROOT/bin" # output
declare -g GITSRCDIR="$SRCDIR/openwrt"
for dir in "$SRCDIR" "$BINDIR"; do
[[ -d "$dir" ]] || mkdir -p "$dir"
done
# Allow --reset without a profile
if [[ ${#PROFILES} -lt 1 ]]; then
if (( RESET )); then
resetAll
exit
else
echo "No profile supplied" && return 1
fi
fi
installDependencies
for profile in "${PROFILES[@]}"; do
debug "Starting profile: $profile"
if [[ ! ${!profile@a} = A ]]; then
echo "Profile '$profile' does not exist"
return 1
fi
# Store profile settings in P_ARR
declare -gn P_ARR="$profile"
# release precedence: user input>profile>env>hardcode
declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"
declare -g BUILDDIR="$SRCDIR/${P_ARR[profile]}-$RELEASE"
declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
declare -g THIS_BINDIR="$BINDIR/${P_ARR[profile]}-$RELEASE"
if [[ "$RELEASE" == "snapshot" ]]; then
declare url_prefix="https://downloads.openwrt.org/snapshots/targets/${P_ARR[target]}"
declare url_filename="openwrt-imagebuilder-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
declare img_prefix="$BUILDDIR/openwrt-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM"
else
declare url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/${P_ARR[target]}"
declare url_filename="openwrt-imagebuilder-$RELEASE-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
declare img_prefix="$BUILDDIR/openwrt-$RELEASE-${P_ARR[target]//\//-}-${P_ARR[profile]}-$FILESYSTEM"
fi
declare -g IB_URL="$url_prefix/$url_filename"
declare -g IB_ARCHIVE="$SRCDIR/$url_filename"
declare -g IB_SHA256_URL="$url_prefix/sha256sums"
declare -g IB_SHA256_FILE="$BUILDDIR/sha256sums"
declare -g IB_OUT_SHA256_FILE="$THIS_BINDIR/sha256sums"
declare -g SEED_URL="$url_prefix/config.buildinfo"
declare -g SEED_FILE="$GITSRCDIR/.config"
declare -g FACTORYIMG="$img_prefix-factory.img"
declare -g FACTORYIMGGZ="$img_prefix-factory.img.gz"
declare -g FACTORYIMGGZFNAME="${FACTORYIMGGZ##*/}"
declare -g SYSUPGRADEIMG="$img_prefix-sysupgrade.img"
declare -g SYSUPGRADEIMGGZ="$img_prefix-sysupgrade.img.gz"
declare -g SYSUPGRADEIMGGZFNAME="${SYSUPGRADEIMGGZ##*/}"
if (( DEBUG )); then
echo "Profile settings:"
for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
echo "Build settings:"
cat <<- EOF
ALIAS=$profile
BUILDROOT=$BUILDROOT
BUILDDIR=$BUILDDIR
SRCDIR=$SRCDIR
BINDIR=$BINDIR
GITSRCDIR=$GITSRCDIR
THIS_BINDIR=$THIS_BINDIR
TARGET=${P_ARR[target]}
PROFILE=${P_ARR[profile]}
RELEASE=$RELEASE
FILESYSTEM=$FILESYSTEM
IB_URL=$IB_URL
IB_ARCHIVE=$IB_ARCHIVE
SEED_URL=$SEED_URL
SEED_FILE=$SEED_FILE
IB_SHA256_URL=$IB_SHA256_URL
IB_SHA256_FILE=$IB_SHA256_FILE
IB_OUT_SHA256_FILE=$IB_OUT_SHA256_FILE
FACTORYIMGGZ: $FACTORYIMGGZ
FACTORYIMGGZFNAME: $FACTORYIMGGZFNAME
SYSUPGRADEIMGGZ: $SYSUPGRADEIMGGZ
SYSUPGRADEIMGGZFNAME: $SYSUPGRADEIMGGZFNAME
EOF
fi
(( RESET )) && resetProfile
(( FROM_SOURCE )) && fromSource
# Acquire and verify Image Builder
getImageBuilder &&
getImageBuilderChecksum &&
verify "$IB_ARCHIVE" "$IB_SHA256_FILE" ||
return $?
extractImageBuilder || return $?
addRepos
#copyFiles
[[ -v SSH_BACKUP_PATH ]] && sshBackup
if makeImages && verifyImages; then
[[ -v SSH_UPGRADE_PATH ]] && sshUpgrade
[[ -v FLASH_DEV ]] && flashImage
fi
done
}
main "$@"
exit
# VM setup (for testing)
# sudo sgdisk -N 0 /dev/vda &&
# sudo mkfs.ext4 /dev/vda1
# mkdir ~/mnt
# sudo mount /dev/vda1 ~/mnt
# sudo chown liveuser:liveuser -R ~/mnt