Update README

README.md

@@ -12,10 +12,12 @@ Build and deploy OpenWRT images using shell-style device profiles, via source co
 --profile,-p PROFILE
 --release,-r,--version,-v RELEASE ("snapshot", "22.03.3")
 --buildroot,-b PATH (Default: script directory)
---source
+--cpus,-c NUM
-  Build image from source code, not from Image Builder.
+	Default: # of host CPUS minus 1
-  Allows make config options to be passed in profile.
+--mode,-m imagebuilder|source
-  Uses git worktree for multi-profile deduplication.
+  Default: imagebuilder
+--clean clean|targetclean|dirclean|distclean
+  Optional clean step for source mode
 --ssh-upgrade HOST
   Example: root@192.168.1.1
 --ssh-backup SSH_PATH
@@ -37,13 +39,13 @@ Build and deploy OpenWRT images using shell-style device profiles, via source co
 
 See `profiles` for example device profile definitions. Multiple `--profile` can be passed at once.
 
-The default build mode is  `imagebuilder` unless `--source` is passed. Default profile modes can be set individually in `profiles`.
+The default build mode is  `imagebuilder` unless `--mode=source` is passed. Default profile modes can be set individually in `profiles`.
 
 ## Examples
 
 * `openwrtbuilder -p r4s -p ax6000`
 * `openwrtbuilder -p r4s -r snapshot --debug`
-* `openwrtbuilder -p ax6000 -r 23.05.5 --source --debug`
+* `openwrtbuilder -p ax6000 -r 23.05.5 --mode source --debug`
 * `openwrtbuilder -p rpi4 -r 23.05.5 --flash /dev/sdX`
 * `openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1`
 

openwrtbuilder

@@ -5,7 +5,7 @@
 # See README and ./profiles for device configuration
 
 # Set default release
-: "${RELEASE:="24.10.5"}"
+: "${DEFAULT_RELEASE:=${RELEASE:="24.10.5"}}"
 
 # @internal
 print_help() {
@@ -19,13 +19,16 @@ print_help() {
 
 	OPTIONS
 	  --profile,-p PROFILE
-	  --release,-r,--version,-v RELEASE ("snapshot", "22.03.5") 
+	  --release,-r,--version,-v RELEASE ("snapshot", "24.10.5")
+	    Default: From profile or hardcoded RELEASE    
 	  --buildroot,-b PATH
 	    Default: location of openwrtbuilder script
-	  --source[=CLEAN]
+	  --cpus,-c NUM
-	    Build image from source, not from Image Builder
+	    Default: # of host CPUS minus 1
-	    Optional CLEAN runs the given clean mode before building
+	  --mode,-m imagebuilder|source
-	      (clean|targetclean|dirclean|distclean)
+	    Default: imagebuilder
+	  --clean clean|targetclean|dirclean|distclean
+	    Optional clean step for source mode
 	  --ssh-upgrade HOST
 	    Examples: root@192.168.1.1, root@router.lan
 	  --ssh-backup SSH_PATH
@@ -35,7 +38,7 @@ print_help() {
 	  --depends
 	    Force dependency installation
 	  --yes,-y
-	    Assume yes for all questions (automatic mode)
+	    Assume yes for all questions (non-interactive)
 	  --debug,-d
 	  --help,-h
 
@@ -124,24 +127,21 @@ init() {
 parse_input() {
   debug "${FUNCNAME[0]}" "$*"
   declare -ga PROFILES
-  declare -gi RESET=0 FROM_SOURCE=0 YES=0 DEBUG=0 FORCE_DEPENDS=0
+  declare -gi RESET=0 YES=0 DEBUG=0 FORCE_DEPENDS=0 CPUS=0
-  declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV SOURCE_CLEAN 
+  declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV USER_MODE USER_CLEAN 
-  local long_opts='release:,version:,profile:,buildroot:,source::,'
+  local long_opts='release:,version:,profile:,buildroot:,cpus:,mode:,clean:'
   long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,depends,yes,debug,help'
 
-  if _input=$(getopt -o +r:v:p:b:sf:ydh -l $long_opts -- "$@"); then
+  if _input=$(getopt -o +r:v:p:b:m:c:f:ydh -l $long_opts -- "$@"); then
     eval set -- "$_input"
     while true; do
       case "$1" in
         --release|-r|--version|-v) shift; USER_RELEASE="$1" ;;
         --profile|-p) shift; PROFILES+=("$1") ;;
         --buildroot|-b) shift; BUILD_ROOT="$1" ;;
-        --source|-s) FROM_SOURCE=1
+        --mode|-m) shift; USER_MODE="$1" ;;
-          case "$1" in
+        --clean) shift; USER_CLEAN="$1" ;;
-            -*|"") ;; # if empty
+        --cpus|-c) shift; CPUS="$1" ;;
-            *) SOURCE_CLEAN="$1"; shift ;;
-          esac
-          ;;
         --ssh-upgrade) shift; SSH_UPGRADE_PATH="$1" ;;
         --ssh-backup) shift; SSH_BACKUP_PATH="$1" ;;
         --flash|-f) shift; FLASH_DEV="$1" ;;
@@ -172,7 +172,7 @@ install_dependencies() {
   if [[ "$mode" == "source" ]]; then
     lock_file="$BUILD_ROOT/.dependencies_source.lock"
   elif [[ "$mode" == "imagebuilder" ]]; then
-    lock_file="$BUILD_ROOT/.dependencies_ib.lock"
+    lock_file="$BUILD_ROOT/.dependencies_imagebuilder.lock"
   fi
 
   [[ -f $lock_file ]] && debug "$lock_file lock file exists but skipping for --debug" && return 0
@@ -441,7 +441,7 @@ make_images() {
   debug make "${make_opts[@]}" image BIN_DIR="$BIN_DIR" \
     PROFILE="$DEVICE" PACKAGES="$PACKAGES" \
     FILES="$FILES_DIR" --directory="$BUILD_DIR" \
-    --jobs="$(($(nproc) - 1))"
+    --jobs="$JOBS"
 
   make "${make_opts[@]}" image \
     BIN_DIR="$BIN_DIR" \
@@ -449,7 +449,7 @@ make_images() {
     PACKAGES="$PACKAGES" \
     FILES="$FILES_DIR" \
     --directory="$BUILD_DIR" \
-    --jobs="$(($(nproc) - 1))"  \
+    --jobs="$JOBS"  \
     > "$BUILD_DIR/make.log"
 }
 
@@ -505,18 +505,12 @@ ssh_upgrade() {
 
 # @description Builds OpenWRT from source code using the the default buildbot as base
 # This enables the use of kernel config options in profiles
-# @arg $1 string .config seed URL
-# @arg $2 string Profile name
-# @arg $3 string Worktree ref (commit-ish or branch name)
 from_source() {
-  debug "${FUNCNAME[0]}" "$*"
+  debug "${FUNCNAME[0]}"
-  local seed_url="$1"
-  local profile="$2"
-  local ref="$3"
   local src_url="https://github.com/openwrt/openwrt.git"
   local seed_file="$BUILD_DIR/.config"
-  local worktree_meta="$SRC_DIR/.git/worktrees/source-$ref"
+  local worktree_meta="$SRC_DIR/.git/worktrees/source-$REF"
-  local pkg config commit seed_file description
+  local pkg config commit description
   local -a make_opts
   local -a config_opts=(
     "CONFIG_TARGET_${TARGET%%/*}=y"
@@ -560,22 +554,21 @@ from_source() {
   # Reuse worktree if present; otherwise create it (support branches and tags)
   if git -C "$BUILD_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
     execute git -C "$BUILD_DIR" fetch origin --tags --prune
-    execute git -C "$BUILD_DIR" reset --hard "origin/$ref" || \
+    execute git -C "$BUILD_DIR" reset --hard "origin/$REF" || \
-      execute git -C "$BUILD_DIR" reset --hard "$ref" || \
+      execute git -C "$BUILD_DIR" reset --hard "$REF" || \
-      execute git -C "$BUILD_DIR" checkout --detach "$ref"
+      execute git -C "$BUILD_DIR" checkout --detach "$REF"
   else
     execute git -C "$SRC_DIR" worktree prune --verbose
     # Prefer local tag/branch if present, otherwise use remote-tracking branch
-    if ! execute git -C "$SRC_DIR" worktree add --detach "$BUILD_DIR" "$ref"; then
+    if ! execute git -C "$SRC_DIR" worktree add --detach "$BUILD_DIR" "$REF"; then
-      execute git -C "$SRC_DIR" worktree add --detach "$BUILD_DIR" "origin/$ref"
+      execute git -C "$SRC_DIR" worktree add --detach "$BUILD_DIR" "origin/$REF"
     fi
   fi
 
   # Add cherrypicks
-  for entry in ${P_ARR[cherrypicks]}; do
+  for cherrypick in $CHERRYPICKS; do
-    url_branch="${entry%:*}"
+    url_branch="${cherrypick%:*}"
-    commit="${entry##*:}"
+    commit="${cherrypick##*:}"
-
     branch=""
     url="$url_branch"
     if [[ "$url_branch" == *"@"* ]]; then
@@ -611,8 +604,8 @@ from_source() {
   done
 
   # Merge entire branches
-  for entry in ${P_ARR[branches]}; do
+  for branch in $BRANCHES; do
-    url_branch="$entry"
+    url_branch="$branch"
     branch=""
     url="$url_branch"
     if [[ "$url_branch" == *"@"* ]]; then
@@ -637,7 +630,7 @@ from_source() {
       execute git -C "$BUILD_DIR" merge --allow-unrelated-histories -m "Merge $remote/$branch" "$remote/$branch" || \
         { debug "Merge conflict or failed for $remote/$branch"; return 1; }
     else
-      debug "Cannot merge: no branch specified in entry '$entry'"
+      debug "Cannot merge: no branch specified in entry '$branch'"
       continue
     fi
   done
@@ -661,8 +654,8 @@ from_source() {
   # make targetclean # compiled output, toolchain
   # make dirclean # compiled output, toolchain, build tools
   # make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache
-  if [[ -n $SOURCE_CLEAN ]]; then
+  if [[ -n $CLEAN ]]; then
-    execute make "${make_opts[@]}" "-j1" "$SOURCE_CLEAN"
+    execute make "${make_opts[@]}" "-j1" "$CLEAN"
   else
     debug "Skipping cleanup step"
   fi
@@ -684,7 +677,7 @@ from_source() {
   done
 
   # Add profile config options
-  for config in ${P_ARR[config]}; do
+  for config in $CONFIGS; do
     config_opts+=("$config")
   done
 
@@ -702,8 +695,8 @@ from_source() {
   execute make "${make_opts[@]}" "-j1" download
 
   # (Optional) Disable multicore make world
-  # ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)-2))")
+  # ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$JOBS)")
-  make_opts+=("-j$(($(nproc)-2))")
+  make_opts+=("-j$JOBS")
 
   # Make image
   if ! execute ionice -c2 -n7 nice -n19 make "${make_opts[@]}" BIN_DIR="$BIN_DIR" world; then
@@ -723,32 +716,6 @@ from_source() {
   return 0
 }
 
-# @description Backs up a file to a chosen directory using its timestamp
-# @arg $1 string File to backup
-# @arg $2 string Directory to backup to
-backup() {
-  debug "${FUNCNAME[0]}" "$*"
-  local file="$1" dir="$2"
-  local creation_date base_name backup_file
-
-  [[ -f $file ]] || return 1
-  [[ -d $dir ]] || execute mkdir -p "$dir" || { debug "Failed to create directory: $dir"; return 1; }
-
-  if creation_date=$(stat -c %w "$file" 2>/dev/null || stat -c %y "$file" 2>/dev/null) && \
-     [[ $creation_date != "-" && -n $creation_date ]] && \
-     creation_date=$(date -d "$creation_date" +%y%m%d%H%M 2>/dev/null); then
-    debug "Creation date: $creation_date"
-  else
-    creation_date="unknown"
-    debug "Unable to determine creation date, using 'unknown'"
-  fi
-
-  base_name="${file##*/}"
-  backup_file="$dir/$creation_date-$base_name"
-
-  [[ -f $backup_file ]] || execute cp --archive "$file" "$backup_file"
-}
-
 # @section Helper functions
 # @internal
 debug() { ((DEBUG)) && echo "Debug: $*"; }
@@ -823,73 +790,92 @@ main() {
   # Remove dependency lock files for --depends
   if ((FORCE_DEPENDS)); then
     [[ -f "$BUILD_ROOT/.dependencies_source.lock" ]] && rm -f "$BUILD_ROOT/.dependencies_source.lock"
-    [[ -f "$BUILD_ROOT/.dependencies_ib.lock" ]] && rm -f "$BUILD_ROOT/.dependencies_ib.lock"
+    [[ -f "$BUILD_ROOT/.dependencies_imagebuilder.lock" ]] && rm -f "$BUILD_ROOT/.dependencies_imagebuilder.lock"
   fi
 
   # Run selected profiles
-  for profile in "${PROFILES[@]}"; do
+  for PROFILE in "${PROFILES[@]}"; do
-    debug "Running profile: $profile"
+    debug "Running profile: $PROFILE"
 
-    if [[ ! ${!profile@a} = A ]]; then
+    if [[ ! ${!PROFILE@a} = A ]]; then
-      echo "Profile '$profile' does not exist"
+      echo "Profile '$PROFILE' does not exist"
       return 1
     fi
 
-    # Store profile in P_ARR nameref
+    # Store profile in P_ARR nameref and set global profile vars
-    local -n P_ARR="$profile"
+    local -n P_ARR="$PROFILE"
-    local mode="${P_ARR[mode]:-"imagebuilder"}"
+    declare -g REPO="${P_ARR[repo]:-}"
-    ((FROM_SOURCE)) && mode="source" # allow cli override
-    install_dependencies "$mode"
-    local repo="${P_ARR[repo]:-}"
     declare -g FILESYSTEM="${P_ARR[filesystem]:="squashfs"}"
     declare -g TARGET="${P_ARR[target]}"
     declare -g DEVICE="${P_ARR[device]}"
-    declare -g PACKAGES="${P_ARR[packages]:-}"
+    declare -g MODE="${USER_MODE:-${P_ARR[mode]:-imagebuilder}}"
+    declare -g CLEAN="${USER_CLEAN:-${P_ARR[clean]:-}}"
+    declare -g PACKAGES="${P_ARR[packages]:-}" # scalar
+    declare -g CHERRYPICKS="${P_ARR[cherrypicks]:-}" # scalar
+    declare -g BRANCHES="${P_ARR[branches]:-}" # scalar
+    declare -g CONFIGS="${P_ARR[configs]:-}" # scalar
 
-    # pull in USER_RELEASE from args or profile default
+    install_dependencies "$MODE"
-    local raw_release="${USER_RELEASE:=${P_ARR[release]:=$RELEASE}}"
 
-    # single call to normalize+ref
+    # Set number of parallel jobs for make and imagebuilder
-    read -r release ref < <(normalize_and_ref "$raw_release" "$mode")
+    declare -gi JOBS
+    if ((CPUS)); then
+      JOBS="$CPUS" # user overide (--cpus)
+    else
+      JOBS=$(nproc || echo 4) # fallback to quad-core if nproc fails
+      ((JOBS > 1)) && JOBS=$((JOBS - 1)) # leave one CPU free
+    fi
+
+    # Normalize RELEASE and set REF committish
+    local raw_release="${USER_RELEASE:=${P_ARR[release]:=$DEFAULT_RELEASE}}"
+    declare -g RELEASE REF
+    read -r RELEASE REF < <(normalize_and_ref "$raw_release" "$MODE")
 
     declare -g SRC_DIR="$BUILD_ROOT/src/.openwrt"
-    declare -g BUILD_DIR="$BUILD_ROOT/src/$profile/$ref-$mode"
+    declare -g BUILD_DIR="$BUILD_ROOT/src/$PROFILE/$REF-$MODE"
-    declare -g BIN_DIR="$BUILD_ROOT/bin/$profile/$ref-$mode"
+    declare -g BIN_DIR="$BUILD_ROOT/bin/$PROFILE/$REF-$MODE"
 
-    if [[ "$release" == "snapshot" ]]; then
+    if [[ "$RELEASE" == "snapshot" ]]; then
       local url_prefix="https://downloads.openwrt.org/snapshots/targets/$TARGET"
       local url_filename="openwrt-imagebuilder-${TARGET//\//-}.Linux-x86_64.tar.zst"
       local img_fname="openwrt-${TARGET//\//-}-$DEVICE-$FILESYSTEM" 
     else
-      local url_prefix="https://downloads.openwrt.org/releases/$release/targets/$TARGET"
+      local url_prefix="https://downloads.openwrt.org/releases/$RELEASE/targets/$TARGET"
-      local url_filename="openwrt-imagebuilder-$release-${TARGET//\//-}.Linux-x86_64.tar.zst"
+      local url_filename="openwrt-imagebuilder-$RELEASE-${TARGET//\//-}.Linux-x86_64.tar.zst"
-      local img_fname="openwrt-$release-${TARGET//\//-}-$DEVICE-$FILESYSTEM"
+      local img_fname="openwrt-$RELEASE-${TARGET//\//-}-$DEVICE-$FILESYSTEM"
     fi
 
-    local ib_url="$url_prefix/$url_filename"
+    local imagebuilder_url="$url_prefix/$url_filename"
-    local ib_file="$BUILD_DIR/$url_filename"
+    local imagebuilder_file="$BUILD_DIR/$url_filename"
-    local ib_sha256_url="$url_prefix/sha256sums"
+    local imagebuilder_sha256_url="$url_prefix/sha256sums"
-    local ib_sha256_file="$BUILD_DIR/sha256sums"
+    local imagebuilder_sha256_file="$BUILD_DIR/sha256sums"
-    local seed_url="$url_prefix/config.buildinfo"
 
-    if [[ "$mode" == "source" ]]; then
+    if [[ "$MODE" == "source" ]]; then
       declare -g SYSUPGRADEIMGGZ="$BIN_DIR/targets/$TARGET/$img_fname-sysupgrade.img.gz"
     else
       declare -g SYSUPGRADEIMGGZ="$BUILD_DIR/$img_fname-sysupgrade.img.gz"
     fi
 
-    backup "$SYSUPGRADEIMGGZ" "$BACKUP_DIR/$profile/$ref-$mode"
+    # Backup existing output directory
+    if [[ -d "$BIN_DIR" ]]; then
+      local timestamp
+      timestamp=$(date +%y%m%d%H%M)
+      execute mkdir -p "$BACKUP_DIR/$PROFILE/$REF-$MODE-$timestamp"
+      execute rsync -a --delete --exclude 'packages/' "$BIN_DIR/" "$BACKUP_DIR/$PROFILE/$REF-$MODE-$timestamp/"
+    fi
 
-    if [[ "$mode" == "source" ]]; then
+    if [[ "$MODE" == "source" ]]; then
-      from_source "$seed_url" "$profile" "$ref" || return $?
+      from_source || return $?
-    elif [[ "$mode" == "imagebuilder" ]]; then
+    elif [[ "$MODE" == "imagebuilder" ]]; then
       [[ -d $BUILD_DIR ]] || mkdir -p "$BUILD_DIR"
-      get_imagebuilder "$ib_url" "$ib_file" "$ib_sha256_url" "$ib_sha256_file" && 
+      get_imagebuilder "$imagebuilder_url" "$imagebuilder_file" "$imagebuilder_sha256_url" "$imagebuilder_sha256_file" && 
-      verify "$ib_file" "$ib_sha256_file" &&
+      verify "$imagebuilder_file" "$imagebuilder_sha256_file" &&
-      extract "$ib_file" "$BUILD_DIR" || return $?
+      extract "$imagebuilder_file" "$BUILD_DIR" || return $?
-      if [[ -v $repo ]]; then
+
-        if ! grep -q "$repo" "$BUILD_DIR/repositories.conf"; then
+      # Add external repositories for the Image Builder build
-          echo "$repo" >> "$BUILD_DIR/repositories.conf"
+      if [[ -n $REPO ]]; then
+        if ! grep -q "$REPO" "$BUILD_DIR/repositories.conf"; then
+          echo "$REPO" >> "$BUILD_DIR/repositories.conf"
         fi
         sed -i '/option check_signature/d' "$BUILD_DIR/repositories.conf"
       fi
@@ -900,7 +886,7 @@ main() {
         local -a outfiles=("$BIN_DIR"/*.img.gz "$BIN_DIR"/*.img)
         shopt -u nullglob
         for outfile in "${outfiles[@]}"; do
-          verify "$outfile" "$ib_sha256_file" || return 1
+          verify "$outfile" "$imagebuilder_sha256_file" || return 1
         done
       fi
       #copyFiles

profiles

@@ -2,15 +2,14 @@
 # Device profiles for openwrtbuilder
 # shellcheck disable=SC2034
 
-RELEASE="25.12.0-rc4" # overrides default release in openwrtbuilder
+DEFAULT_RELEASE="25.12.0-rc4" # overrides default release in openwrtbuilder
 
 # Default packages (precede with "-" to exclude)
 default_packages=(
-  nano vim htop diffutils tar iperf3 zsh rsync tcpdump
+  ca-bundle nano vim htop diffutils tar iperf3 zsh rsync tcpdump ethtool
   openssh-sftp-server 
   luci luci-ssl luci-proto-wireguard luci-app-statistics luci-app-filemanager
   collectd-mod-sensors collectd-mod-thermal collectd-mod-conntrack collectd-mod-cpu
-  ca-bundle ethtool
 )
 
 # Default kernel configs
@@ -29,7 +28,7 @@ declare -Ag r4s=(
     collectd-mod-df usbutils kmod-usb-storage kmod-usb-storage-uas \
     kmod-fs-btrfs btrfs-progs block-mount smcroute avahi-daemon \
     ethtool ca-bundle tailscale"
-  [config]="${default_configs[*]} \
+  [configs]="${default_configs[*]} \
     CONFIG_KERNEL_BTRFS_FS_POSIX_ACL=y CONFIG_BTRFS_PROGS_ZSTD=y \
     CONFIG_TARGET_ROOTFS_PARTSIZE=512 CONFIG_TARGET_KERNEL_PARTSIZE=32"
   [files]="/mnt/backup"
@@ -67,7 +66,7 @@ declare -Ag n5100=(
     btrfs-progs block-mount cryptsetup kmod-crypto-xts smcroute \
     avahi-daemon ethtool ca-bundle smartmontools intel-microcode \
     lm-sensors samba4-server luci-app-samba4 tailscale shadow-useradd"
-  [config]="${default_configs[*]} \
+  [configs]="${default_configs[*]} \
     CONFIG_KERNEL_BTRFS_FS_POSIX_ACL=y CONFIG_BTRFS_PROGS_ZSTD=y \
     CONFIG_TARGET_ROOTFS_PARTSIZE=512 CONFIG_TARGET_KERNEL_PARTSIZE=32"
   # [files]="/mnt/backup"