openwrtbuilder/openwrtbuilder

#!/usr/bin/env bash
# Builds and deploys OpenWRT images
# Copyright 2022-24 Bryan C. Roessler
# Apache 2.0 License
# See README.md and ./profiles for device configuration

# Set default release
: "${RELEASE:="23.05.5"}"

print_help() {
  debug "${FUNCNAME[0]}"

  cat <<-'EOF'
	Build and deploy OpenWRT images

	USAGE:
	  openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]...

	OPTIONS
	  --profile,-p PROFILE
	  --release,-r,--version,-v RELEASE ("snapshot", "22.03.5") 
	  --buildroot,-b PATH
	    Default: location of openwrtbuilder script
	  --source
	    Build image from source, not from Image Builder
	  --ssh-upgrade HOST
	    Examples: root@192.168.1.1, root@router.lan
	  --ssh-backup SSH_PATH
	    Enabled by default for --ssh-upgrade
	  --flash,-f DEVICE
	    Example: /dev/sdX
	  --reset
	    Cleanup all source and output files
	  --yes,-y
	    Assume yes for all questions (automatic mode)
	  --verbose
	    Make make or imagebuilder noisier
	  --debug,-d
	  --help,-h

	EXAMPLES
	  ./openwrtbuilder -p r4s -r snapshot
	  ./openwrtbuilder -p ax6000 -r 23.05.0-rc3 --source --debug
	  ./openwrtbuilder -p rpi4 -r 22.03.3 --flash /dev/sdX
	  ./openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1		  
	EOF
}

init() {
  debug "${FUNCNAME[0]}"
  declare -g ID RPM_MGR SCRIPT_DIR DL_TOOL

  ((DEBUG)) || echo "To enable debugging output, use --debug or -d"

  # Save the script directory
  # https://stackoverflow.com/a/4774063
  SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)"

  if [[ -e "/etc/os-release" ]]; then
    source "/etc/os-release"
  else
    echo "/etc/os-release not found"
    echo "Your OS is unsupported"
    print_help
    exit 1
  fi

  debug "Detected host platform: $ID"

  # normalize distro ID
  case "$ID" in
    debian|arch) ;;
    centos|fedora)
      if command -v dnf &>/dev/null; then
        RPM_MGR="dnf"
      elif command -v yum &>/dev/null; then
        RPM_MGR="yum"
      fi
      ;;
    rhel) ID="centos" ;;
    linuxmint|neon|*ubuntu*) ID="ubuntu" ;;
    *suse*) ID="suse" ;;
    raspbian) ID="debian" ;;
    *)
      echo "Autodetecting distro, this may be unreliable"
      if command -v dnf &>/dev/null; then
        ID="fedora"
        RPM_MGR="dnf"
      elif command -v yum &>/dev/null; then
        ID="centos"
        RPM_MGR="yum"
      elif command -v apt &>/dev/null; then
        ID="ubuntu"
      elif command -v pacman &>/dev/null; then
        ID="arch"
      else
        return 1
      fi
      ;;
  esac

  debug "Using host platform: $ID"

  # Set distro-specific functions
  case "$ID" in
    fedora|centos) pkg_install(){ sudo "$RPM_MGR" install -y "$@"; } ;;
    debian|ubuntu) pkg_install(){ sudo apt-get install --ignore-missing -y -q0 "$@"; } ;;
    suse) pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; } ;;
    arch) pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; } ;;
  esac

  if command -v axel &>/dev/null; then
    DL_TOOL="axel"
  elif command -v curl &>/dev/null; then
    DL_TOOL="curl"
  else
    echo "Downloading the Image Builder requires axel or curl"
    return 1
  fi
}

parse_input() {
  debug "${FUNCNAME[0]}" "$*"
  declare -ga PROFILES
  declare -g RESET=0 FROM_SOURCE=0 YES=0 VERBOSE=0 DEBUG=0
  declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV
  local long_opts='release:,version:,profile:,buildroot:,source,'
  long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,yes,verbose,debug,help'

  if _input=$(getopt -o +r:v:p:b:sf:ydh -l $long_opts -- "$@"); then
    eval set -- "$_input"
    while true; do
      case "$1" in
        --release|-r|--version|-v) shift; USER_RELEASE="$1" ;;
        --profile|-p) shift; PROFILES+=("$1") ;;
        --buildroot|-b) shift; BUILD_ROOT="$1" ;;
        --source|-s) FROM_SOURCE=1 ;;
        --ssh-upgrade) shift; SSH_UPGRADE_PATH="$1" ;;
        --ssh-backup) shift; SSH_BACKUP_PATH="$1" ;;
        --flash|-f) shift; FLASH_DEV="$1" ;;
        --reset) RESET=1 ;;
        --yes|-y) YES=1 ;;
        --verbose) VERBOSE=1 ;;
        --debug|-d) echo "Debugging on"; DEBUG=1 ;;
        --help|-h) print_help; exit 0 ;;
        --)
          shift
          break
          ;;
      esac
      shift
    done
  else
    echo "Incorrect options provided"
    print_help; exit 1
  fi
}

install_dependencies() {
  debug "${FUNCNAME[0]}"
  local -a pkg_list
  local lock_file
  if ((FROM_SOURCE)); then
    lock_file="$BUILD_ROOT/.dependencies_source"
  else
    lock_file="$BUILD_ROOT/.dependencies_ib"
  fi

  if [[ ! -f $lock_file ]]; then
    if ((FROM_SOURCE)); then
      # For building from source code
      # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
      case "$ID" in
        fedora|centos)
          pkg_list+=(
            bash-completion
            bzip2
            gcc
            gcc-c++
            git
            make
            ncurses-devel
            patch
            rsync
            tar
            unzip
            wget
            which
            diffutils
            python3
            python3-devel
            python3-setuptools
            python3-pyelftools
            perl-base
            perl-Data-Dumper
            perl-File-Compare
            perl-File-Copy
            perl-FindBin
            perl-IPC-Cmd
            perl-Thread-Queue
            perl-Time-Piece
            perl-JSON-PP
            swig
            clang # for qosify
            llvm15-libs
            patch)
          ;;
        debian|ubuntu)
          pkg_list+=(
            build-essential
            clang
            flex
            g++
            gawk
            gcc-multilib
            gettext
            git
            libncurses5-dev
            libssl-dev
            python3-distutils
            rsync
            unzip
            zlib1g-dev
            file
            wget
            patch)
          ;;
        arch)
          pkg_list+=(
            base-devel
            autoconf
            automake
            bash
            binutils
            bison
            bzip2
            clang
            fakeroot
            file
            findutils
            flex
            gawk
            gcc
            gettext
            git
            grep
            groff
            gzip
            libelf
            libtool
            libxslt
            m4
            make
            ncurses
            openssl
            patch
            pkgconf
            python
            rsync
            sed
            texinfo
            time
            unzip
            util-linux
            wget
            which
            zlib
            patch)
          ;;
        *)
          debug "Skipping dependency install, your OS is unsupported"
          return 1
          ;;
      esac
    else
      # For Imagebuilder
      case "$ID" in
        fedora|centos)
          pkg_list+=(
            @c-development 
            @development-tools
            @development-libs
            perl-FindBin
            zlib-static
            elfutils-libelf-devel
            gawk
            unzip
            file
            wget
            python3
            python2
            axel
            perl-IPC-Cmd)
          ;;
        debian|ubuntu)
          pkg_list+=(
            build-essential
            libncurses5-dev
            libncursesw5-dev
            zlib1g-dev
            gawk
            git
            gettext
            libssl-dev
            xsltproc
            wget
            unzip
            python
            axel)
          ;;
        *)
          debug "Skipping dependency install, your OS is unsupported"
          return 1
          ;;
      esac
    fi

    pkg_install "${pkg_list[@]}" && echo "${pkg_list[@]}" > "$lock_file"
  fi
}

get_imagebuilder() {
  debug "${FUNCNAME[0]}" "$*"
  local -a url_file_pairs=("$@")

  for ((i=0; i<${#url_file_pairs[@]}; i+=2)); do
    local url="${url_file_pairs[i]}"
    local file="${url_file_pairs[i+1]}"

    # Check if file exists and ask user to remove and redownload
    if [[ -f $file ]] && ! ask_ok "Use existing $file?"; then
      execute rm -f "$file"
    fi

    # Download the file if it doesn't exist
    if [[ ! -f "$file" ]]; then
      echo "Downloading $url to $file using $DL_TOOL"
      execute "$DL_TOOL" "-o" "$file" "$url"
    fi
  done
}

add_repos() {
  debug "${FUNCNAME[0]}"

  if [[ -v P_ARR[repo] ]]; then
    if ! grep -q "${P_ARR[repo]}" "$BUILD_DIR/repositories.conf"; then
      echo "${P_ARR[repo]}" >> "$BUILD_DIR/repositories.conf"
    fi
    sed -i '/option check_signature/d' "$BUILD_DIR/repositories.conf"
  fi
}

ssh_backup() {
  debug "${FUNCNAME[0]}"
  local date hostname backup_fname
  printf -v date '%(%Y-%m-%d-%H-%M-%S)T'
  hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME)
  backup_fname="backup-$hostname-$date.tar.gz"

  [[ -d "$FILES_DIR" ]] || execute mkdir -p "$FILES_DIR"

  # Make backup archive on remote
  if ! execute "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$backup_fname"; then
    echo "SSH backup failed"
    exit 1
  fi

  # Move backup archive locally
  if ! execute "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$backup_fname $BUILD_DIR/"; then
    echo "Could not copy SSH backup"
    exit 1
  fi

  # Extract backup archive
  if ! execute "tar -C $FILES_DIR -xzf $BUILD_DIR/$backup_fname"; then
    echo "Could not extract SSH backup"
    exit 1
  fi

  execute "rm $BUILD_DIR/$backup_fname"
}

make_images() {
  debug "${FUNCNAME[0]}"
  local -a make_opts

  # Reuse the existing output
  # if [[ -d "$BIN_DIR" ]]; then
  #   if ask_ok "$BIN_DIR exists. Rebuild?"; then
  #     execute rm -rf "$BIN_DIR"
  #   else
  #     return 0
  #   fi
  # fi

  ((VERBOSE)) && make_opts+=("V=s")

  debug make "${make_opts[@]}" image BIN_DIR="$BIN_DIR" \
    PROFILE="$DEVICE" PACKAGES="$PACKAGES" \
    FILES="$FILES_DIR" --directory="$BUILD_DIR" \
    --jobs="$(($(nproc) - 1))"

  make "${make_opts[@]}" image \
    BIN_DIR="$BINDIR" \
    PROFILE="$DEVICE" \
    PACKAGES="$PACKAGES" \
    FILES="$FILES_DIR" \
    --directory="$BUILD_DIR" \
    --jobs="$(($(nproc) - 1))"  \
    > "$BUILD_DIR/make.log"
}

flash_images() {
  debug "${FUNCNAME[0]}"
  local img_gz="$1"
  local dev="$2"
  local img="${img_gz%.gz}"
  local partitions

  if [[ ! -e "$dev" ]]; then
    echo "The device specified by --flash could not be found"
    return 1
  fi

  [[ -f $img_gz ]] || { echo "$img_gz does not exist"; return 1; }

  execute gunzip -qfk "$img_gz"
    
  echo "Unmounting target device $dev partitions"
  partitions=("$dev"?*)
  execute sudo umount "${partitions[@]}"

  if execute sudo dd if="$img" of="$dev" bs=2M conv=fsync; then
    sync
    echo "Image flashed successfully!"
  else
    echo "dd failed!"
    return 1
  fi
}

ssh_upgrade() {
  debug "${FUNCNAME[0]}"
  local img_gz="$1"
  local ssh_path="$2"
  local img_fname="${img_gz##*/}"

  [[ -f $img_gz ]] || { echo "$img_gz is missing, check build output"; return 1; }

  echo "Copying '$img_gz' to $ssh_path/tmp/$img_fname"
  if ! execute scp "$img_gz" "$ssh_path:/tmp/$img_fname"; then
    echo "Could not copy $img_gz to $ssh_path:/tmp/$img_fname"
    return 1
  fi
      
  echo "Executing remote sysupgrade"
  # This may result in weird exit code from closing the ssh connection
  # shellcheck disable=SC2029
  ssh "$ssh_path" "sysupgrade -F /tmp/$img_fname"
  return 0
}

from_source() {
  debug "${FUNCNAME[0]}" "$*"
  local seed_url="$1"
  local src_url="https://github.com/openwrt/openwrt.git"
  local seed_file="$WORKTREE_DIR/.config"
  local pkg config commit seed_file wt_commit description
  local -a make_opts config_opts

  echo "Building from source is under development"

  # Update source code
  if [[ ! -d "$SRC_DIR" ]]; then
    execute mkdir -p "$SRC_DIR"
    execute git clone "$src_url" "$SRC_DIR"
  fi

  git -C "$SRC_DIR" pull
 
  # Generate commitish for git worktree
  case "$RELEASE" in
    snapshot) wt_commit="origin/main" ;;
    [0-9][0-9].[0-9][0-9].*)
      local branch="openwrt-${RELEASE%.*}"
      local tag="v$RELEASE"
      if ask_ok "Use $branch branch HEAD (y, recommended) or $tag tag (N)?"; then
        wt_commit="origin/$branch"
      else
        wt_commit="$tag"
      fi
      ;;
    *)
      debug "Passing '$RELEASE' commit-ish to git worktree"
      wt_commit="$RELEASE"
      ;;
  esac

  # TODO There's a bug in the make clean functions that seem to invoke a full make
  if [[ -d "$WORKTREE_DIR" ]]; then
    execute git -C "$WORKTREE_DIR" checkout "$wt_commit"
    execute git -C "$WORKTREE_DIR" pull
  else
    execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit"
  fi

  # To workaround bug, don't use make *clean, blow it away and start fresh
  # [[ -d "$WORKTREE_DIR" ]] && execute rm -rf "$WORKTREE_DIR"
  # execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit"

  # Print commit information
  commit=$(git -C "$WORKTREE_DIR" rev-parse HEAD)
  description=$(git -C "$WORKTREE_DIR" describe)
  echo "Current commit hash: $commit"
  echo "Git worktree description: $description"
  
  ((DEBUG)) && git --no-pager -C "$WORKTREE_DIR" log -1

  # Enter worktree
  execute pushd "$WORKTREE_DIR" || return 1

  # Update package feed
  ./scripts/feeds update -i -f &&
  ./scripts/feeds update -a -f &&
  ./scripts/feeds install -a -f

  # Cleanup build environment
  ((VERBOSE)) && make_opts+=("V=s")
  execute make "${make_opts[@]}" "-j1" distclean # TODO 'dirclean' has a bug that triggers menuconfig

  # Grab the release seed config
  if ! execute "$DL_TOOL" "-o" "$seed_file" "$seed_url"; then
    echo "Could not obtain $seed_file from $seed_url"
    return 1
  fi

  # Set compilation output dir
  config_opts+=("CONFIG_BINARY_FOLDER=\"$BIN_DIR\"")

  # Add custom packages
  for pkg in $PACKAGES; do
    if [[ $pkg == -* ]]; then
      config_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package
    else
      config_opts+=("CONFIG_PACKAGE_$pkg=y") # add package
    fi
  done

  # Add config options from profile
  for config in ${P_ARR[config]}; do
    config_opts+=("$config")
  done

  # Only compile selected fs
  execute sed -i '/CONFIG_TARGET_ROOTFS_/d' "$seed_file"
  config_opts+=("CONFIG_TARGET_PER_DEVICE_ROOTFS=n")
  if [[ $FILESYSTEM == "squashfs" ]]; then
    config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=n")
    config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=y")
  elif [[ $FILESYSTEM == "ext4" ]]; then
    config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=n")
    config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=y")
  fi

  # Only compile selected target image
  execute sed -i '/CONFIG_TARGET_DEVICE_/d' "$seed_file"
  config_opts+=("CONFIG_TARGET_MULTI_PROFILE=n")
  config_opts+=("CONFIG_TARGET_PROFILE=DEVICE_$DEVICE")
  config_opts+=("CONFIG_TARGET_${TARGET//\//_}_DEVICE_$DEVICE=y")
  config_opts+=("CONFIG_SDK=n")
  config_opts+=("CONFIG_SDK_LLVM_BPF=n")
  config_opts+=("CONFIG_IB=n")
  config_opts+=("CONFIG_MAKE_TOOLCHAIN=n")
  
  # Write options to config seed file
  for config in "${config_opts[@]}"; do
    debug "Writing $config to $seed_file"
    echo "$config" >> "$seed_file"
  done

  # Cleaning modes
  # make clean # compiled output
  # make targetclean # compiled output, toolchain
  # make dirclean # compiled output, toolchain, build tools
  # make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache

  # Make prep
  execute make "${make_opts[@]}" "-j1" defconfig
  execute make "${make_opts[@]}" "-j1" download
  # ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)+1))")
  make_opts+=("-j$(($(nproc)+1))")

  # Make image
  if ! execute ionice -c 3 chrt --idle 0 nice -n19 make "${make_opts[@]}" world; then
    echo "Error: make failed"
    return 1
  fi

  popd || return 1

  # Symlink output images to root of BIN_DIR (match Image Builder)
  shopt -s nullglob
  for image in "$BIN_DIR/targets/${TARGET}/"*.{img,img.gz,ubi}; do
    execute ln -fs "$image" "$BIN_DIR/${image##*/}"
  done
  shopt -u nullglob

  return 0
}

# Generic helpers
debug() { ((DEBUG)) && echo "Debug: $*"; }
ask_ok() {
  ((YES)) && return
  local r
  read -r -p "$* [y/N]: " r
  r=${r,,}
  [[ "$r" =~ ^(yes|y)$ ]]
}
extract() {
  debug "${FUNCNAME[0]}" "$*"
  local archive="$1"
  local out_dir="$2"
  if ! execute tar -axf "$archive" -C "$out_dir" --strip-components 1; then
    echo "Extraction failed"
    return 1
  fi
}
backup_image() {
  debug "${FUNCNAME[0]} $*"
  local file="$1" dir="$2" count=1
  [[ -f $file ]] || return 1
  local creation_date
  creation_date=$(stat -c %w "$file" 2>/dev/null || stat -c %y "$file" | cut -d' ' -f1)
  creation_date=${creation_date:-unknown} # Default to "unknown" if no creation date
  [[ "$creation_date" == "-" ]] && creation_date="unknown"
  local base_name; base_name=$(basename "$file")
  execute mkdir -p "$dir"
  while [[ -e "$dir/$creation_date-$base_name.bk.$count" ]]; do ((count++)); done
  execute mv "$file" "$dir/$creation_date-$base_name.bk.$count"
}
verify() {
  debug "${FUNCNAME[0]}" "$*"
  local file_to_check="$1"
  local sumfile="$2"
  local checksum

  command -v sha256sum &>/dev/null || return 1
  [[ -f $sumfile && -f $file_to_check ]] || return 1
  checksum=$(grep "${file_to_check##*/}" "$sumfile" | cut -f1 -d' ')
  echo -n "$checksum $file_to_check" | sha256sum --check --status
}
load() {
  debug "${FUNCNAME[0]}" "$*"
  local source_file="$1"
  # shellcheck disable=SC1090
  [[ -f $source_file ]] && source "$source_file"
}
execute() {
  if debug "$*"; then
    "$@"
  else
    "$@" &>/dev/null
  fi
}

main() {
  debug "${FUNCNAME[0]}"

  init
  load "$SCRIPT_DIR/profiles"
  parse_input "$@"

  # Fallback to SCRIPT_DIR if BUILD_ROOT has not been set
  declare -g BUILD_ROOT="${BUILD_ROOT:=$SCRIPT_DIR}"
  declare -g FILES_DIR="${FILES_DIR:=$BUILD_ROOT/src/files}"
  declare -g BACKUP_DIR="$SCRIPT_DIR/backups"

  # This could be dangerous
  if [[ $BUILD_ROOT == "/" ]]; then
    echo "Invalid --buildroot"
    exit 1
  fi

  for dir in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do
    [[ -d "$dir" ]] || execute mkdir -p "$dir"
  done

  # Allow --reset without a profile
  if ((RESET)) && [[ ${#PROFILES} -lt 1 ]]; then
    for d in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do
      ask_ok "Remove $d?" && execute rm -rf "$d"
    done
    exit $?
  fi

  install_dependencies

  for profile in "${PROFILES[@]}"; do
    debug "Running profile: $profile"

    if [[ ! ${!profile@a} = A ]]; then
      echo "Profile '$profile' does not exist"
      return 1
    fi

    # Store profile in P_ARR nameref
    declare -gn P_ARR="$profile"
    declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
    declare -g TARGET="${P_ARR[target]}"
    declare -g DEVICE="${P_ARR[device]}"
    declare -g PACKAGES="${P_ARR[packages]:-}"
    declare -g RELEASE="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"

    # normalize RELEASE
    case "$RELEASE" in
      snapshot|latest|main|master) RELEASE="snapshot" ;;
      v[0-9][0-9].[0-9][0-9].*) RELEASE="${RELEASE#v}" ;;
      [0-9][0-9].[0-9][0-9].*) ;;
      *)
        if ! ((FROM_SOURCE)); then
          echo "Error: Invalid release version format"
          echo "Use semantic version, tag, or 'snapshot'"
          exit 1
        fi
        ;;
    esac

    declare -g SRC_DIR="$BUILD_ROOT/src/openwrt"
    declare -g WORKTREE_DIR="$BUILD_ROOT/src/$profile/$RELEASE-src"
    declare -g BUILD_DIR="$BUILD_ROOT/src/$profile/$RELEASE"
    declare -g BIN_DIR="$BUILD_ROOT/bin/$profile/$RELEASE"

    if [[ "$RELEASE" == "snapshot" ]]; then
      local url_prefix="https://downloads.openwrt.org/snapshots/targets/$TARGET"
      local url_filename="openwrt-imagebuilder-${TARGET//\//-}.Linux-x86_64.tar.zst"
      local img_fname="openwrt-${TARGET//\//-}-$DEVICE-$FILESYSTEM" 
    else
      local url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/$TARGET"
      local url_filename="openwrt-imagebuilder-$RELEASE-${TARGET//\//-}.Linux-x86_64.tar.xz"
      local img_fname="openwrt-$RELEASE-${TARGET//\//-}-$DEVICE-$FILESYSTEM"
    fi

    local ib_url="$url_prefix/$url_filename"
    local ib_file="$BUILD_DIR/$url_filename"
    local ib_sha256_url="$url_prefix/sha256sums"
    local ib_sha256_file="$BUILD_DIR/sha256sums"
    local seed_url="$url_prefix/config.buildinfo"

    if ((FROM_SOURCE)); then
      declare -g SYSUPGRADEIMGGZ="$BIN_DIR/targets/$img_fname-sysupgrade.img.gz"
    else
      declare -g SYSUPGRADEIMGGZ="$BUILD_DIR/$img_fname-sysupgrade.img.gz"
    fi

    backup_image "$SYSUPGRADEIMGGZ" "$BACKUP_DIR/$profile/$RELEASE"

    if ((RESET)); then
      if ((FROM_SOURCE)); then
        [[ -d $WORKTREE_DIR ]] && ask_ok "Remove $WORKTREE_DIR?"
          execute git worktree remove --force "$WORKTREE_DIR"
          execute rm -rf "$WORKTREE_DIR"
      elif [[ -d $BUILD_DIR ]] && ask_ok "Remove $BUILD_DIR?"; then
        execute rm -rf "$BUILD_DIR"
      fi
    fi

    if ((DEBUG)); then
      echo "Profile settings:"
      for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
      echo "Environment variables:"
      declare -p
    fi

    if ((FROM_SOURCE)); then
      from_source "$seed_url" || return $?
    else
      [[ -d $BUILD_DIR ]] || mkdir -p "$BUILD_DIR"
      get_imagebuilder "$ib_url" "$ib_file" "$ib_sha256_url" "$ib_sha256_file" && 
      verify "$ib_file" "$ib_sha256_file" &&
      extract "$ib_file" "$BUILD_DIR" || return $?
      add_repos
      make_images
      # Verify output image for stock builds (in testing)
      if [[ ! -v P_ARR[packages] || -z ${P_ARR[packages]} ]]; then
        shopt -s nullglob
        local -a outfiles=("$BIN_DIR"/*.img.gz "$BIN_DIR"/*.img)
        shopt -u nullglob
        for outfile in "${outfiles[@]}"; do
          verify "$outfile" "$ib_sha256_file" || return 1
        done
      fi
      #copyFiles
    fi

    [[ -v SSH_BACKUP_PATH ]] && ssh_backup
    [[ -v SSH_UPGRADE_PATH ]] && ssh_upgrade "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH"
    [[ -v FLASH_DEV ]] && flash_images "$SYSUPGRADEIMGGZ" "$FLASH_DEV"
  done
}

main "$@"
exit