openwrtbuilder/openwrtbuilder

#!/usr/bin/env bash
# Build and deploy OpenWRT images using convenient shell-style device profiles.
# Copyright 2022-25 Bryan C. Roessler
# Apache 2.0 License
# See README and profiles for device configuration

# Set default release
: "${RELEASE:="24.10.2"}"

# @internal
print_help() {
  debug "${FUNCNAME[0]}"

  cat <<-'EOF'
	Build and deploy OpenWRT images using convenient profiles.

	USAGE:
	  openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]...

	OPTIONS
	  --profile,-p PROFILE
	  --release,-r,--version,-v RELEASE ("snapshot", "22.03.5") 
	  --buildroot,-b PATH
	    Default: location of openwrtbuilder script
	  --source
	    Build image from source, not from Image Builder
	    Allows make config options to be passed in profile
	    Uses git worktree for multi-profile deduplication
	  --ssh-upgrade HOST
	    Examples: root@192.168.1.1, root@router.lan
	  --ssh-backup SSH_PATH
	    Enabled by default for --ssh-upgrade
	  --flash,-f DEVICE
	    Example: /dev/sdX
	  --reset
	    Cleanup all source and output files
	  --depends
	    Force dependency installation
	  --yes,-y
	    Assume yes for all questions (automatic mode)
	  --debug,-d
	  --help,-h

	EXAMPLES
	  openwrtbuilder -p r4s -r snapshot
	  openwrtbuilder -p ax6000 -r 23.05.0-rc3 --source --debug
	  openwrtbuilder -p rpi4 -r 24.10.0 --flash /dev/sdX
	  openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1		  
	EOF
}

# @internal
init() {
  debug "${FUNCNAME[0]}"
  declare -g ID RPM_MGR SCRIPT_DIR DL_TOOL

  ((DEBUG)) || echo "To enable debugging output, use --debug or -d"

  # Save the script directory
  # https://stackoverflow.com/a/4774063
  SCRIPT_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit $? ; pwd -P)"

  if [[ -e "/etc/os-release" ]]; then
    source "/etc/os-release"
  else
    echo "/etc/os-release not found"
    echo "Your OS is unsupported"
    print_help
    exit 1
  fi

  debug "Detected host platform: $ID"

  # normalize distro ID
  case "$ID" in
    debian|arch) ;;
    centos|fedora)
      if command -v dnf &>/dev/null; then
        RPM_MGR="dnf"
      elif command -v yum &>/dev/null; then
        RPM_MGR="yum"
      fi
      ;;
    rhel) ID="centos" ;;
    linuxmint|neon|*ubuntu*) ID="ubuntu" ;;
    *suse*) ID="suse" ;;
    raspbian) ID="debian" ;;
    *)
      echo "Autodetecting distro, this may be unreliable"
      for cmd in dnf yum apt pacman; do
        if command -v "$cmd" &>/dev/null; then
          case "$cmd" in
            dnf) ID="fedora"; RPM_MGR="dnf" ;;
            yum) ID="centos"; RPM_MGR="yum" ;;
            apt) ID="ubuntu" ;;
            pacman) ID="arch" ;;
          esac
          break
        fi
      done
      [[ -z $ID ]] && return 1
      ;;
  esac

  debug "Using host platform: $ID"

  # Set distro-specific functions
  case "$ID" in
    fedora|centos) pkg_install(){ sudo "$RPM_MGR" install -y "$@"; } ;;
    debian|ubuntu) pkg_install(){ sudo apt-get install --ignore-missing -y -q0 "$@"; } ;;
    suse) pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; } ;;
    arch) pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; } ;;
  esac

  if command -v axel &>/dev/null; then
    DL_TOOL="axel"
  elif command -v curl &>/dev/null; then
    DL_TOOL="curl"
  else
    echo "Downloading the Image Builder requires axel or curl"
    return 1
  fi
}

# @description Arguments
parse_input() {
  debug "${FUNCNAME[0]}" "$*"
  declare -ga PROFILES
  declare -gi RESET=0 FROM_SOURCE=0 YES=0 DEBUG=0 FORCE_DEPENDS=0
  declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV
  local long_opts='release:,version:,profile:,buildroot:,source,'
  long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,depends,yes,debug,help'

  if _input=$(getopt -o +r:v:p:b:sf:ydh -l $long_opts -- "$@"); then
    eval set -- "$_input"
    while true; do
      case "$1" in
        --release|-r|--version|-v) shift; USER_RELEASE="$1" ;;
        --profile|-p) shift; PROFILES+=("$1") ;;
        --buildroot|-b) shift; BUILD_ROOT="$1" ;;
        --source|-s) FROM_SOURCE=1 ;;
        --ssh-upgrade) shift; SSH_UPGRADE_PATH="$1" ;;
        --ssh-backup) shift; SSH_BACKUP_PATH="$1" ;;
        --flash|-f) shift; FLASH_DEV="$1" ;;
        --reset) RESET=1 ;;
        --depends) FORCE_DEPENDS=1 ;;
        --yes|-y) YES=1 ;;
        --debug|-d) echo "Debugging on"; DEBUG=1 ;;
        --help|-h) print_help; exit 0 ;;
        --) shift; break ;;
      esac
      shift
    done
  else
    echo "Incorrect options provided"
    print_help; exit 1
  fi
}

# @description Install build dependencies on major distros
# @arg $1 string Build mode ("source" or "imagebuilder")
install_dependencies() {
  debug "${FUNCNAME[0]}"
  local mode="$1"
  local -a pkg_list
  local lock_file

  # Set appropriate lock file based on mode
  if [[ "$mode" == "source" ]]; then
    lock_file="$BUILD_ROOT/.dependencies_source.lock"
  elif [[ "$mode" == "imagebuilder" ]]; then
    lock_file="$BUILD_ROOT/.dependencies_ib.lock"
  fi

  [[ -f $lock_file ]] && debug "$lock_file lock file exists" && return 0

  if [[ "$mode" == "source" ]]; then
    # For building from source code see:
    # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
    case "$ID" in
      fedora|centos)
        pkg_list+=(
          bzip2
          clang # for qosify
          diffutils
          gcc
          gcc-c++
          git
          llvm15-libs # for qosify
          make
          ncurses-devel
          patch
          perl
          perl-Data-Dumper
          perl-File-Compare
          perl-File-Copy
          perl-FindBin
          perl-IPC-Cmd
          perl-JSON-PP
          perl-Thread-Queue
          perl-Time-Piece
          perl-base
          python3
          python3-devel
          python3-pyelftools
          python3-setuptools
          rsync
          swig
          tar
          unzip
          wget
          which
        ) ;;
      debian|ubuntu)
        pkg_list+=(
          build-essential
          clang
          file
          flex
          g++
          gawk
          gcc-multilib
          gettext
          git
          liblzma-dev
          libncurses5-dev
          libssl-dev
          python3-distutils
          rsync
          patch
          unzip
          wget
          zlib1g-dev
        ) ;;
      arch)
        pkg_list+=(
          autoconf
          automake
          base-devel
          bash
          binutils
          bison
          bzip2
          clang
          fakeroot
          file
          findutils
          flex
          gawk
          gcc
          gettext
          git
          grep
          groff
          gzip
          libelf
          libtool
          libxslt
          m4
          make
          ncurses
          net-snmp
          openssl
          patch
          pkgconf
          python
          rsync
          sed
          texinfo
          time
          unzip
          util-linux
          wget
          which
          xz
          zlib
        ) ;;
      *) debug "Unsupported OS, skipping dependency install"; return 1 ;;
    esac
  elif [[ "$mode" == "imagebuilder" ]]; then
    # For Imagebuilder
    case "$ID" in
      fedora|centos)
        pkg_list+=(
          @c-development 
          @development-tools
          @development-libs
          perl-FindBin
          zlib-static
          elfutils-libelf-devel
          gawk
          unzip
          file
          wget
          python3
          axel
          perl-IPC-Cmd
          zstd) ;;
      debian|ubuntu)
        pkg_list+=(
          build-essential
          libncurses5-dev
          libncursesw5-dev
          zlib1g-dev
          gawk
          git
          gettext
          libssl-dev
          xsltproc
          wget
          unzip
          python
          axel
          zstd) ;;
      *) debug "Unsupported OS, skipping dependency install"; return 1 ;;
    esac
  fi

  pkg_install "${pkg_list[@]}" && echo "${pkg_list[@]}" > "$lock_file"
}

# @description Normalize release and set worktree reference
# @arg $1 string Raw release input
# @arg $2 string Build mode ("source" or "imagebuilder")
# @returns string Normalized release and reference
normalize_and_ref() {
  local input="$1" mode="$2"
  local rel ref branch tag

  case "$input" in
    snapshot|latest|main|master)
      rel="snapshot"
      ref="main"
      ;;
    v[0-9][0-9].[0-9][0-9].*|[0-9][0-9].[0-9][0-9].*)
      # strip optional leading “v”
      rel="${input#v}"
      if [[ "$mode" == "source" ]]; then
        branch="openwrt-${rel%.*}"
        tag="v$rel"
        if ask_ok "Use branch $branch HEAD (y) or tag $tag (n)?"; then
          ref="$branch"
        else
          ref="$tag"
        fi
      else
        ref="$rel"
      fi
      ;;
    *)
      if [[ "$mode" == "source" ]]; then
        # arbitrary commit-ish allowed
        rel="$input"
        ref="$input"
      else
        echo "Error: invalid release '$input'" >&2
        exit 1
      fi
      ;;
  esac

  printf '%s %s' "$rel" "$ref"
}

# @description Acquires the OpenWRT Image Builder
get_imagebuilder() {
  debug "${FUNCNAME[0]}" "$*"
  local -a url_file_pairs=("$@")

  for ((i=0; i<${#url_file_pairs[@]}; i+=2)); do
    local url="${url_file_pairs[i]}"
    local file="${url_file_pairs[i+1]}"

    # Check if file exists and ask user to remove and redownload
    if [[ -f $file ]] && ! ask_ok "Use existing $file?"; then
      execute rm -f "$file"
    fi

    # Download the file if it doesn't exist
    if [[ ! -f "$file" ]]; then
      echo "Downloading $url to $file using $DL_TOOL"
      execute "$DL_TOOL" "-o" "$file" "$url"
    fi
  done
}

ssh_backup() {
  debug "${FUNCNAME[0]}"
  local date hostname backup_fname
  printf -v date '%(%Y-%m-%d-%H-%M-%S)T'
  hostname=$(ssh -qt "$SSH_BACKUP_PATH" echo -n \$HOSTNAME)
  backup_fname="backup-$hostname-$date.tar.gz"

  [[ -d "$FILES_DIR" ]] || execute mkdir -p "$FILES_DIR"

  # Make backup archive on remote
  if ! execute "ssh -t $SSH_BACKUP_PATH sysupgrade -b /tmp/$backup_fname"; then
    echo "SSH backup failed"
    exit 1
  fi

  # Move backup archive locally
  if ! execute "rsync -avz --remove-source-files $SSH_BACKUP_PATH:/tmp/$backup_fname $BUILD_DIR/"; then
    echo "Could not copy SSH backup"
    exit 1
  fi

  # Extract backup archive
  if ! execute "tar -C $FILES_DIR -xzf $BUILD_DIR/$backup_fname"; then
    echo "Could not extract SSH backup"
    exit 1
  fi

  execute "rm $BUILD_DIR/$backup_fname"
}

make_images() {
  debug "${FUNCNAME[0]}"
  local -a make_opts

  # Reuse the existing output
  # if [[ -d "$BIN_DIR" ]]; then
  #   if ask_ok "$BIN_DIR exists. Rebuild?"; then
  #     execute rm -rf "$BIN_DIR"
  #   else
  #     return 0
  #   fi
  # fi

  ((DEBUG)) && make_opts+=("V=sc")

  debug make "${make_opts[@]}" image BIN_DIR="$BIN_DIR" \
    PROFILE="$DEVICE" PACKAGES="$PACKAGES" \
    FILES="$FILES_DIR" --directory="$BUILD_DIR" \
    --jobs="$(($(nproc) - 1))"

  make "${make_opts[@]}" image \
    BIN_DIR="$BIN_DIR" \
    PROFILE="$DEVICE" \
    PACKAGES="$PACKAGES" \
    FILES="$FILES_DIR" \
    --directory="$BUILD_DIR" \
    --jobs="$(($(nproc) - 1))"  \
    > "$BUILD_DIR/make.log"
}

flash_images() {
  debug "${FUNCNAME[0]}"
  local img_gz="$1"
  local dev="$2"
  local img="${img_gz%.gz}"
  local partitions

  if [[ ! -e "$dev" ]]; then
    echo "The device specified by --flash could not be found"
    return 1
  fi

  [[ -f $img_gz ]] || { echo "$img_gz does not exist"; return 1; }

  execute gunzip -qfk "$img_gz"
    
  echo "Unmounting target device $dev partitions"
  partitions=("$dev"?*)
  execute sudo umount "${partitions[@]}"

  if execute sudo dd if="$img" of="$dev" bs=2M conv=fsync; then
    sync
    echo "Image flashed successfully!"
  else
    echo "dd failed!"
    return 1
  fi
}

ssh_upgrade() {
  debug "${FUNCNAME[0]}"
  local img_gz="$1"
  local ssh_path="$2"
  local img_fname="${img_gz##*/}"

  [[ -f $img_gz ]] || { echo "$img_gz is missing, check build output"; return 1; }

  echo "Copying '$img_gz' to $ssh_path/tmp/$img_fname"
  if ! execute scp "$img_gz" "$ssh_path:/tmp/$img_fname"; then
    echo "Could not copy $img_gz to $ssh_path:/tmp/$img_fname"
    return 1
  fi
      
  echo "Executing remote sysupgrade"
  # This may result in weird exit code from closing the ssh connection
  # shellcheck disable=SC2029
  ssh "$ssh_path" "sysupgrade -F /tmp/$img_fname"
  return 0
}

# @description Builds OpenWRT from source code using the the default buildbot as base
# This enables the use of kernel config options in profiles
# @arg $1 string .config seed URL
# @arg $2 string Profile name
# @arg $3 string Worktree ref (commit-ish or branch name)
from_source() {
  debug "${FUNCNAME[0]}" "$*"
  local seed_url="$1"
  local profile="$2"
  local ref="$3"
  local src_url="https://github.com/openwrt/openwrt.git"
  local seed_file="$BUILD_DIR/.config"
  local worktree_meta="$SRC_DIR/.git/worktrees/source-$ref"
  local pkg config commit seed_file description
  local -a make_opts config_opts

  echo "Building from source is under development"

  # Remove all build directories and worktrees
  if ((RESET)); then
    if [[ -d "$BUILD_DIR" || -d "$worktree_meta" ]]; then
      execute git -C "$SRC_DIR" worktree remove --force --force "$BUILD_DIR"
      [[ -d "$BUILD_DIR" ]] && execute rm -rf "$BUILD_DIR"
      [[ -d "$worktree_meta" ]] && execute rm -rf "$worktree_meta"
    fi
    [[ -d "$BUILD_DIR" ]] && execute rm -rf "$BUILD_DIR"
  fi

  # Pull or clone source repo
  if [[ -d "$SRC_DIR" ]]; then
    execute git -C "$SRC_DIR" pull
  else
    execute mkdir -p "$SRC_DIR"
    execute git clone "$src_url" "$SRC_DIR"
  fi

  # Remove existing build dir and add new worktree
  if [[ -d "$BUILD_DIR" ]]; then
    execute rm -rf "$BUILD_DIR"
  fi
  execute git -C "$SRC_DIR" worktree prune --verbose
  execute git -C "$SRC_DIR" worktree add "$BUILD_DIR" "$ref"

  # Print commit info
  commit=$(git -C "$BUILD_DIR" rev-parse HEAD)
  description=$(git -C "$BUILD_DIR" describe)
  echo "Current commit hash: $commit"
  echo "Git worktree description: $description"

  ((DEBUG)) && git --no-pager -C "$BUILD_DIR" log -1

  # Enter worktree
  execute pushd "$BUILD_DIR" || return 1
  
  # Begin OpenWRT build process
  ((DEBUG)) && make_opts+=("V=sc")
  
  # Cleanup build environment
  execute make "${make_opts[@]}" "-j1" distclean
  # make clean # compiled output
  # make targetclean # compiled output, toolchain
  # make dirclean # compiled output, toolchain, build tools
  # make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache

  # Use a custom (faster) mirror
  # execute sed -i -E 's;git.openwrt.org/(feed|project);github.com/openwrt;' feeds.conf.default

  # Update package feed
  ./scripts/feeds update -a -f &&
  ./scripts/feeds install -a -f

  # Grab the release seed config
  if ! execute "$DL_TOOL" "-o" "$seed_file" "$seed_url"; then
    echo "Could not obtain $seed_file from $seed_url"
    return 1
  fi

  # Set compilation output dir
  config_opts+=("CONFIG_BINARY_FOLDER=\"$BIN_DIR\"")

  # Add custom packages
  for pkg in $PACKAGES; do
    if [[ $pkg == -* ]]; then
      config_opts+=("CONFIG_PACKAGE_${pkg#-}=n") # remove package
    else
      config_opts+=("CONFIG_PACKAGE_$pkg=y") # add package
    fi
  done

  # Add config options from profile
  for config in ${P_ARR[config]}; do
    config_opts+=("$config")
  done

  # Only compile selected fs
  execute sed -i '/CONFIG_TARGET_ROOTFS_/d' "$seed_file"
  config_opts+=("CONFIG_TARGET_PER_DEVICE_ROOTFS=n")
  if [[ $FILESYSTEM == "squashfs" ]]; then
    config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=n")
    config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=y")
  elif [[ $FILESYSTEM == "ext4" ]]; then
    config_opts+=("CONFIG_TARGET_ROOTFS_SQUASHFS=n")
    config_opts+=("CONFIG_TARGET_ROOTFS_EXT4FS=y")
  fi

  # Only compile selected target image
  execute sed -i '/CONFIG_TARGET_DEVICE_/d' "$seed_file"
  config_opts+=("CONFIG_TARGET_MULTI_PROFILE=n")
  config_opts+=("CONFIG_TARGET_PROFILE=DEVICE_$DEVICE")
  config_opts+=("CONFIG_TARGET_${TARGET//\//_}_DEVICE_$DEVICE=y")
  config_opts+=("CONFIG_SDK=n")
  config_opts+=("CONFIG_SDK_LLVM_BPF=n")
  config_opts+=("CONFIG_IB=n")
  config_opts+=("CONFIG_MAKE_TOOLCHAIN=n")
  
  # Write options to config seed file
  for config in "${config_opts[@]}"; do
    debug "Writing $config to $seed_file"
    echo "$config" >> "$seed_file"
  done

  # Serial make prep is more reliable
  execute make "${make_opts[@]}" "-j1" defconfig
  execute make "${make_opts[@]}" "-j1" download

  # make_opts+=("-j$(($(nproc)-1))")
  ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)-1))")

  # Make image
  if ! execute ionice -c 3 chrt --idle 0 nice -n19 make "${make_opts[@]}" world; then
    echo "Error: make failed"
    return 1
  fi

  execute popd || return 1

  # Symlink output images to root of BIN_DIR (match Image Builder)
  shopt -s nullglob
  for image in "$BIN_DIR/targets/${TARGET}/"*.{img,img.gz,ubi}; do
    execute ln -fs "$image" "$BIN_DIR/${image##*/}"
  done
  shopt -u nullglob

  return 0
}

# @description Backs up a file to a chosen directory using its timestamp
# @arg $1 string File to backup
# @arg $2 string Directory to backup to
backup() {
  debug "${FUNCNAME[0]}" "$*"
  local file="$1" dir="$2"
  local creation_date base_name backup_file

  [[ -f $file ]] || return 1
  [[ -d $dir ]] || execute mkdir -p "$dir" || { debug "Failed to create directory: $dir"; return 1; }

  if creation_date=$(stat -c %w "$file" 2>/dev/null || stat -c %y "$file" 2>/dev/null) && \
     [[ $creation_date != "-" && -n $creation_date ]] && \
     creation_date=$(date -d "$creation_date" +%y%m%d%H%M 2>/dev/null); then
    debug "Creation date: $creation_date"
  else
    creation_date="unknown"
    debug "Unable to determine creation date, using 'unknown'"
  fi

  base_name="${file##*/}"
  backup_file="$dir/$creation_date-$base_name"

  [[ -f $backup_file ]] || execute cp --archive "$file" "$backup_file"
}

# @section Helper functions
# @internal
debug() { ((DEBUG)) && echo "Debug: $*"; }
ask_ok() {
  ((YES)) && return
  local r
  read -r -p "$* [y/N]: " r
  r=${r,,}
  [[ "$r" =~ ^(yes|y)$ ]]
}
extract() {
  local archive="$1"
  local out_dir="$2"
  if ! execute tar -axf "$archive" -C "$out_dir" --strip-components 1; then
    echo "Extraction failed"
    return 1
  fi
}
verify() {
  local file_to_check="$1" sum_file="$2"
  local checksum
  command -v sha256sum &>/dev/null || return 1
  [[ -f $sum_file && -f $file_to_check ]] || return 1
  checksum=$(grep "${file_to_check##*/}" "$sum_file" | cut -f1 -d' ')
  echo -n "$checksum $file_to_check" | sha256sum --check --status
}
load() {
  local source_file="$1"
  # shellcheck disable=SC1090
  [[ -f $source_file ]] && source "$source_file"
}
execute() {
  if debug "$*"; then
    "$@"
  else
    "$@" &>/dev/null
  fi
}

# @description The openwrtbuilder main function
# @internal
main() {
  debug "${FUNCNAME[0]}"

  init
  load "$SCRIPT_DIR/profiles"
  parse_input "$@"

  # Fallback to SCRIPT_DIR if BUILD_ROOT has not been set
  declare -g BUILD_ROOT="${BUILD_ROOT:=$SCRIPT_DIR}"
  declare -g FILES_DIR="${FILES_DIR:=$BUILD_ROOT/src/files}"
  declare -g BACKUP_DIR="$SCRIPT_DIR/backups"

  # This could be dangerous
  if [[ $BUILD_ROOT == "/" ]]; then
    echo "Invalid --buildroot"
    exit 1
  fi

  for dir in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do
    [[ -d "$dir" ]] || execute mkdir -p "$dir"
  done

  # Allow --reset without a profile
  if ((RESET)) && [[ ${#PROFILES} -lt 1 ]]; then
    for d in "$BUILD_ROOT/src" "$BUILD_ROOT/bin"; do
      ask_ok "Remove $d?" && execute rm -rf "$d"
    done
    exit $?
  fi

  # Remove dependency lock files for --depends
  if ((FORCE_DEPENDS)); then
    [[ -f "$BUILD_ROOT/.dependencies_source.lock" ]] && rm -f "$BUILD_ROOT/.dependencies_source.lock"
    [[ -f "$BUILD_ROOT/.dependencies_ib.lock" ]] && rm -f "$BUILD_ROOT/.dependencies_ib.lock"
  fi

  # Run selected profiles
  for profile in "${PROFILES[@]}"; do
    debug "Running profile: $profile"

    if [[ ! ${!profile@a} = A ]]; then
      echo "Profile '$profile' does not exist"
      return 1
    fi

    # Store profile in P_ARR nameref
    local -n P_ARR="$profile"
    local mode="${P_ARR[mode]:="imagebuilder"}"
    ((FROM_SOURCE)) && mode="source" # allow cli override
    install_dependencies "$mode"
    local repo="${P_ARR[repo]:-}"
    declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
    declare -g TARGET="${P_ARR[target]}"
    declare -g DEVICE="${P_ARR[device]}"
    declare -g PACKAGES="${P_ARR[packages]:-}"

    # pull in USER_RELEASE from args or profile default
    local raw_release="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"

    # single call to normalize+ref
    read -r release ref < <(normalize_and_ref "$raw_release" "$mode")

    declare -g SRC_DIR="$BUILD_ROOT/src/.openwrt"
    declare -g BUILD_DIR="$BUILD_ROOT/src/$profile/$mode-$ref"
    declare -g BIN_DIR="$BUILD_ROOT/bin/$profile/$mode-$ref"

    if [[ "$release" == "snapshot" ]]; then
      local url_prefix="https://downloads.openwrt.org/snapshots/targets/$TARGET"
      local url_filename="openwrt-imagebuilder-${TARGET//\//-}.Linux-x86_64.tar.zst"
      local img_fname="openwrt-${TARGET//\//-}-$DEVICE-$FILESYSTEM" 
    else
      local url_prefix="https://downloads.openwrt.org/releases/$release/targets/$TARGET"
      local url_filename="openwrt-imagebuilder-$release-${TARGET//\//-}.Linux-x86_64.tar.zst"
      local img_fname="openwrt-$release-${TARGET//\//-}-$DEVICE-$FILESYSTEM"
    fi

    local ib_url="$url_prefix/$url_filename"
    local ib_file="$BUILD_DIR/$url_filename"
    local ib_sha256_url="$url_prefix/sha256sums"
    local ib_sha256_file="$BUILD_DIR/sha256sums"
    local seed_url="$url_prefix/config.buildinfo"

    if [[ "$mode" == "source" ]]; then
      declare -g SYSUPGRADEIMGGZ="$BIN_DIR/targets/$TARGET/$img_fname-sysupgrade.img.gz"
    else
      declare -g SYSUPGRADEIMGGZ="$BUILD_DIR/$img_fname-sysupgrade.img.gz"
    fi

    backup "$SYSUPGRADEIMGGZ" "$BACKUP_DIR/$profile/$mode-$ref"

    if [[ "$mode" == "source" ]]; then
      from_source "$seed_url" "$profile" "$ref" || return $?
    elif [[ "$mode" == "imagebuilder" ]]; then
      [[ -d $BUILD_DIR ]] || mkdir -p "$BUILD_DIR"
      get_imagebuilder "$ib_url" "$ib_file" "$ib_sha256_url" "$ib_sha256_file" && 
      verify "$ib_file" "$ib_sha256_file" &&
      extract "$ib_file" "$BUILD_DIR" || return $?
      if [[ -v $repo ]]; then
        if ! grep -q "$repo" "$BUILD_DIR/repositories.conf"; then
          echo "$repo" >> "$BUILD_DIR/repositories.conf"
        fi
        sed -i '/option check_signature/d' "$BUILD_DIR/repositories.conf"
      fi
      make_images
      # Verify output image for stock builds (in testing)
      if [[ ! -v P_ARR[packages] || -z ${P_ARR[packages]} ]]; then
        shopt -s nullglob
        local -a outfiles=("$BIN_DIR"/*.img.gz "$BIN_DIR"/*.img)
        shopt -u nullglob
        for outfile in "${outfiles[@]}"; do
          verify "$outfile" "$ib_sha256_file" || return 1
        done
      fi
      #copyFiles
    fi

    [[ -v SSH_BACKUP_PATH ]] && ssh_backup
    [[ -v SSH_UPGRADE_PATH ]] && ssh_upgrade "$SYSUPGRADEIMGGZ" "$SSH_UPGRADE_PATH"
    [[ -v FLASH_DEV ]] && flash_images "$SYSUPGRADEIMGGZ" "$FLASH_DEV"
  done
}

main "$@"
exit