Remove index update

README.md

@@ -1,6 +1,6 @@
 # openwrtbuilder
 
-Sanely build and deploy OpenWRT images using the Image Builder or from source code.
+Build and deploy OpenWRT images using convenient profiles.
 
 ## Usage
 
@@ -15,6 +15,7 @@ Sanely build and deploy OpenWRT images using the Image Builder or from source co
 --source
   Build image from source, not from Image Builder
   Allows make config options to be passed in profile
+  Uses git worktree for multi-profile deduplication
 --ssh-upgrade HOST
   Example: root@192.168.1.1
 --ssh-backup SSH_PATH
@@ -24,6 +25,9 @@ Sanely build and deploy OpenWRT images using the Image Builder or from source co
 --reset
   Cleanup all source and output files
   Can be combined with -p to reset a specific profile
+--depends
+  Force dependency installation
+  Ignores .dependencies files
 --yes,-y
   Assume yes for all questions (automatic mode)
 --debug,-d

openwrtbuilder

@@ -1,18 +1,18 @@
 #!/usr/bin/env bash
 # Builds and deploys OpenWRT images
-# Copyright 2022-24 Bryan C. Roessler
+# Copyright 2022-25 Bryan C. Roessler
 # Apache 2.0 License
-# See README.md and ./profiles for device configuration
+# See README and profiles for device configuration
 
 # Set default release
-: "${RELEASE:="24.10.0-rc5"}"
+: "${RELEASE:="24.10.2"}"
 
 # @internal
 print_help() {
   debug "${FUNCNAME[0]}"
 
   cat <<-'EOF'
-	Build and deploy OpenWRT images
+	Build and deploy OpenWRT images using convenient profiles.
 
 	USAGE:
 	  openwrtbuilder [OPTION [VALUE]]... -p PROFILE [-p PROFILE]...
@@ -24,6 +24,8 @@ print_help() {
 	    Default: location of openwrtbuilder script
 	  --source
 	    Build image from source, not from Image Builder
+	    Allows make config options to be passed in profile
+	    Uses git worktree for multi-profile deduplication
 	  --ssh-upgrade HOST
 	    Examples: root@192.168.1.1, root@router.lan
 	  --ssh-backup SSH_PATH
@@ -32,17 +34,17 @@ print_help() {
 	    Example: /dev/sdX
 	  --reset
 	    Cleanup all source and output files
+	  --depends
+	    Force dependency installation
 	  --yes,-y
 	    Assume yes for all questions (automatic mode)
-	  --verbose
-	    Make make or imagebuilder noisier
 	  --debug,-d
 	  --help,-h
 
 	EXAMPLES
 	  ./openwrtbuilder -p r4s -r snapshot
 	  ./openwrtbuilder -p ax6000 -r 23.05.0-rc3 --source --debug
-	  ./openwrtbuilder -p rpi4 -r 22.03.3 --flash /dev/sdX
+	  ./openwrtbuilder -p rpi4 -r 24.10.0 --flash /dev/sdX
 	  ./openwrtbuilder -p linksys -r snapshot --ssh-upgrade root@192.168.1.1		  
 	EOF
 }
@@ -124,10 +126,10 @@ init() {
 parse_input() {
   debug "${FUNCNAME[0]}" "$*"
   declare -ga PROFILES
-  declare -g RESET=0 FROM_SOURCE=0 YES=0 VERBOSE=0 DEBUG=0
+  declare -gi RESET=0 FROM_SOURCE=0 YES=0 DEBUG=0 FORCE_DEPENDS=0
   declare -g USER_RELEASE SSH_UPGRADE_PATH SSH_BACKUP_PATH FLASH_DEV
   local long_opts='release:,version:,profile:,buildroot:,source,'
-  long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,yes,verbose,debug,help'
+  long_opts+='ssh-upgrade:,ssh-backup:,flash:,reset,depends,yes,debug,help'
 
   if _input=$(getopt -o +r:v:p:b:sf:ydh -l $long_opts -- "$@"); then
     eval set -- "$_input"
@@ -141,8 +143,8 @@ parse_input() {
         --ssh-backup) shift; SSH_BACKUP_PATH="$1" ;;
         --flash|-f) shift; FLASH_DEV="$1" ;;
         --reset) RESET=1 ;;
+        --depends) FORCE_DEPENDS=1 ;;
         --yes|-y) YES=1 ;;
-        --verbose) VERBOSE=1 ;;
         --debug|-d) echo "Debugging on"; DEBUG=1 ;;
         --help|-h) print_help; exit 0 ;;
         --) shift; break ;;
@@ -166,70 +168,70 @@ install_dependencies() {
     lock_file="$BUILD_ROOT/.dependencies_ib"
   fi
 
-  if [[ ! -f $lock_file ]]; then
+  if ((FORCE_DEPENDS)) || [[ ! -f $lock_file ]]; then
     if ((FROM_SOURCE)); then
-      # For building from source code
+      # For building from source code see:
       # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
       case "$ID" in
         fedora|centos)
           pkg_list+=(
-            bash-completion
             bzip2
+            clang # for qosify
+            diffutils
             gcc
             gcc-c++
             git
+            llvm15-libs # for qosify
             make
             ncurses-devel
             patch
-            rsync
-            tar
-            unzip
-            wget
-            which
-            diffutils
-            python3
-            python3-devel
-            python3-setuptools
-            python3-pyelftools
-            perl-base
+            perl
             perl-Data-Dumper
             perl-File-Compare
             perl-File-Copy
             perl-FindBin
             perl-IPC-Cmd
+            perl-JSON-PP
             perl-Thread-Queue
             perl-Time-Piece
-            perl-JSON-PP
+            perl-base
+            python3
+            python3-devel
+            python3-pyelftools
+            python3-setuptools
+            rsync
             swig
-            clang # for qosify
-            llvm15-libs
-            patch)
-          ;;
+            tar
+            unzip
+            wget
+            which
+          ) ;;
         debian|ubuntu)
           pkg_list+=(
             build-essential
             clang
+            file
             flex
             g++
             gawk
             gcc-multilib
             gettext
             git
+            liblzma-dev
             libncurses5-dev
             libssl-dev
             python3-distutils
             rsync
+            patch
             unzip
-            zlib1g-dev
-            file
             wget
-            patch)
-          ;;
+            zlib1g-dev
+          ) ;;
         arch)
           pkg_list+=(
-            base-devel
             autoconf
             automake
+            base-devel
             bash
             binutils
             bison
@@ -252,6 +254,7 @@ install_dependencies() {
             m4
             make
             ncurses
+            net-snmp
             openssl
             patch
             pkgconf
@@ -264,13 +267,10 @@ install_dependencies() {
             util-linux
             wget
             which
+            xz
             zlib
-            patch)
-          ;;
-        *)
-          debug "Skipping dependency install, your OS is unsupported"
-          return 1
-          ;;
+          ) ;;
+        *) debug "Unsupported OS, skipping dependencies"; return 1 ;;
       esac
     else
       # For Imagebuilder
@@ -391,7 +391,7 @@ make_images() {
   #   fi
   # fi
 
-  ((VERBOSE)) && make_opts+=("V=s")
+  ((DEBUG)) && make_opts+=("V=sc")
 
   debug make "${make_opts[@]}" image BIN_DIR="$BIN_DIR" \
     PROFILE="$DEVICE" PACKAGES="$PACKAGES" \
@@ -466,18 +466,37 @@ from_source() {
   local seed_url="$1"
   local src_url="https://github.com/openwrt/openwrt.git"
   local seed_file="$WORKTREE_DIR/.config"
+  local worktree_name; worktree_name="$(basename "$WORKTREE_DIR")"
+  local worktree_meta="$SRC_DIR/.git/worktrees/$worktree_name"
   local pkg config commit seed_file wt_commit description
   local -a make_opts config_opts
 
   echo "Building from source is under development"
 
+  if ((DEBUG)); then
+    echo "Profile settings:"
+    for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
+    echo "Environment variables:"
+    declare -p
+  fi
+
+  if ((RESET)); then
+    if [[ -d "$WORKTREE_DIR" || -d "$worktree_meta" ]]; then
+      execute git -C "$SRC_DIR" worktree remove --force --force "$WORKTREE_DIR"
+      [[ -d "$WORKTREE_DIR" ]] && execute rm -rf "$WORKTREE_DIR"
+      [[ -d "$worktree_meta" ]] && execute rm -rf "$worktree_meta"
+    fi
+    [[ -d "$BUILD_DIR" ]] && execute rm -rf "$BUILD_DIR"
+  fi
+
   # Update source code
   if [[ ! -d "$SRC_DIR" ]]; then
     execute mkdir -p "$SRC_DIR"
     execute git clone "$src_url" "$SRC_DIR"
   fi
 
-  git -C "$SRC_DIR" pull
+  execute git -C "$SRC_DIR" worktree prune --verbose
+  execute git -C "$SRC_DIR" pull
  
   # Generate commitish for git worktree
   case "$RELEASE" in
@@ -497,18 +516,16 @@ from_source() {
       ;;
   esac
 
-  # TODO There's a bug in the make clean functions that seem to invoke a full make
-  if [[ -d "$WORKTREE_DIR" ]]; then
+  # Pull existing or add new git worktree
+  if [[ -d "$WORKTREE_DIR" && -d "$worktree_meta" ]]; then
     execute git -C "$WORKTREE_DIR" checkout "$wt_commit"
     execute git -C "$WORKTREE_DIR" pull
   else
-    execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit"
+    [[ -d "$WORKTREE_DIR" ]] && execute rm -rf "$WORKTREE_DIR"
+    [[ -d "$worktree_meta" ]] && execute rm -rf "$worktree_meta"
+    execute git -C "$SRC_DIR" worktree add --force --force --detach "$WORKTREE_DIR" "$wt_commit"
   fi
 
-  # To workaround bug, don't use make *clean, blow it away and start fresh
-  # [[ -d "$WORKTREE_DIR" ]] && execute rm -rf "$WORKTREE_DIR"
-  # execute git -C "$SRC_DIR" worktree add --force --detach "$WORKTREE_DIR" "$wt_commit"
-
   # Print commit information
   commit=$(git -C "$WORKTREE_DIR" rev-parse HEAD)
   description=$(git -C "$WORKTREE_DIR" describe)
@@ -521,15 +538,17 @@ from_source() {
   execute pushd "$WORKTREE_DIR" || return 1
   
   # Cleanup build environment
-  ((VERBOSE)) && make_opts+=("V=s")
+  ((DEBUG)) && make_opts+=("V=sc")
   execute make "${make_opts[@]}" "-j1" distclean
   # make clean # compiled output
   # make targetclean # compiled output, toolchain
   # make dirclean # compiled output, toolchain, build tools
   # make distclean # compiled output, toolchain, build tools, .config, feeds, .ccache
 
+  # Use a custom (faster) mirror
+  # execute sed -i -E 's;git.openwrt.org/(feed|project);github.com/openwrt;' feeds.conf.default
+
   # Update package feed
-  ./scripts/feeds update -i -f &&
   ./scripts/feeds update -a -f &&
   ./scripts/feeds install -a -f
 
@@ -586,8 +605,7 @@ from_source() {
   # Make prep
   execute make "${make_opts[@]}" "-j1" defconfig
   execute make "${make_opts[@]}" "-j1" download
-  # ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)+1))")
-  make_opts+=("-j$(($(nproc)+1))")
+  ((DEBUG)) && make_opts+=("-j1") || make_opts+=("-j$(($(nproc)-1))")
 
   # Make image
   if ! execute ionice -c 3 chrt --idle 0 nice -n19 make "${make_opts[@]}" world; then
@@ -765,23 +783,6 @@ main() {
 
     backup "$SYSUPGRADEIMGGZ" "$BACKUP_DIR/$profile/$RELEASE"
 
-    if ((RESET)); then
-      if ((FROM_SOURCE)); then
-        [[ -d $WORKTREE_DIR ]] && ask_ok "Remove $WORKTREE_DIR?"
-          execute git worktree remove --force "$WORKTREE_DIR"
-          execute rm -rf "$WORKTREE_DIR"
-      elif [[ -d $BUILD_DIR ]] && ask_ok "Remove $BUILD_DIR?"; then
-        execute rm -rf "$BUILD_DIR"
-      fi
-    fi
-
-    if ((DEBUG)); then
-      echo "Profile settings:"
-      for x in "${!P_ARR[@]}"; do printf "%s=%s\n" "$x" "${P_ARR[$x]}"; done
-      echo "Environment variables:"
-      declare -p
-    fi
-
     if ((FROM_SOURCE)); then
       from_source "$seed_url" || return $?
     else

profiles

@@ -16,7 +16,7 @@ declare -Ag r4s=(
   [packages]="$default_packages luci-app-ddns luci-app-sqm irqbalance \
     collectd-mod-df usbutils kmod-usb-storage kmod-usb-storage-uas \
     kmod-fs-btrfs btrfs-progs block-mount smcroute avahi-daemon \
-    curl ethtool ca-bundle"
+    curl ethtool ca-bundle tailscale"
   [config]="CONFIG_KERNEL_BTRFS_FS_POSIX_ACL=y CONFIG_BTRFS_PROGS_ZSTD=y \
     CONFIG_TARGET_ROOTFS_PARTSIZE=512 CONFIG_TARGET_KERNEL_PARTSIZE=32 \
     CONFIG_BUILDBOT=n"
@@ -28,7 +28,7 @@ declare -Ag ax6000=(
   [target]="mediatek/filogic"
   [release]="snapshot"
   [filesystem]="squashfs"
-  [packages]="$default_packages -dnsmasq -odhcpd-ipv6only -nftables -firewall4"
+  [packages]="$default_packages -dnsmasq -odhcpd-ipv6only -nftables -firewall4 tailscale"
 )
 
 declare -Ag ax6000_uboot=(
@@ -50,7 +50,7 @@ declare -Ag n5100=(
     lm-sensors samba4-server luci-app-samba4 tailscale shadow-useradd"
   [config]="CONFIG_KERNEL_BTRFS_FS_POSIX_ACL=y CONFIG_BTRFS_PROGS_ZSTD=y \
     CONFIG_TARGET_ROOTFS_PARTSIZE=512 CONFIG_TARGET_KERNEL_PARTSIZE=32"
-  [files]="/mnt/backup"
+  # [files]="/mnt/backup"
 )
 
 declare -Ag rpi4=(