Rewrite dependency logic for --from-source

README.md

@@ -1,3 +1,3 @@
 # openwrtbuilder
 
-See `openwrtbuilder --help` for help options.
+See `openwrtbuilder --help` for usage.

openwrtbuilder

@@ -6,25 +6,28 @@
 #
 # Apache 2.0 License
 
+# Set default release
+: "${RELEASE:="22.03.2"}"
+
 printHelp() {
     debug "${FUNCNAME[0]}"
 
     cat <<-'EOF'
-USAGE:
+Run and deploy OpenWRT images using the Image Builder.
-openwrtbuilder [[OPTION] [VALUE]]...
 
-Run and deploy OpenWRT imagebuilder.
+USAGE:
+openwrtbuilder [OPTION [VALUE]]...
 
 OPTIONS
     --profile, -p PROFILE
     --info, -i PROFILE
     --list-profiles, -l
-    --release, -r RELEASE
+    --release, -r, --version, -v RELEASE_VERSION ("snapshot", "22.03.3", etc.)
     --builddir, -b PATH
     --ssh-upgrade HOST
         Example: root@192.168.1.1
     --ssh-backup SSH_PATH
-        (For testing, enabled by default for --ssh-upgrade)
+        (Enabled by default for --ssh-upgrade)
     --flash, -f DEVICE
         Example: /dev/sdX
     --reset
@@ -40,11 +43,11 @@ readInput() {
 
     unset RESET
 
-    if _input=$(getopt -o +v:p:i:lb:f:dh -l release:,profile:,info:,list-profiles,builddir:,ssh-upgrade:,ssh-backup:,flash:,reset,debug,help -- "$@"); then
+    if _input=$(getopt -o +r:v:p:i:lb:sf:dh -l release:,version:,profile:,info:,list-profiles,builddir:,from-source,ssh-upgrade:,ssh-backup:,flash:,reset,debug,help -- "$@"); then
         eval set -- "$_input"
         while true; do
             case "$1" in
-                --release|-r)
+                --release|-r|--version|-v)
                     shift && RELEASE="$1"
                     ;;
                 --profile|-p)
@@ -59,6 +62,9 @@ readInput() {
                 --builddir|-b)
                     shift && BUILDDIR="$1"
                     ;;
+                --from-source|-s)
+                    FROM_SOURCE=1
+                    ;;
                 --ssh-upgrade)
                     shift && SSH_UPGRADE_PATH="$1"
                     ;;
@@ -98,98 +104,187 @@ listProfiles() {
 }
 
 
-installHostDependencies() {
+installDependencies() {
     debug "${FUNCNAME[0]}"
 
-    local -a _pkg_list
+    declare -a pkg_list
-    local _pkg_cmd
 
-    source /etc/os-release
+    # TODO please contribute your platform here
-
+    if (( FROM_SOURCE )); then
-    if [[ "$ID" =~ ^(fedora)$ ]]; then
+        # For building from source with make
-        _pkg_list=(\
+        # https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem
-            "@c-development" \
+        case "$ID" in
-            "@development-tools" \
+            fedora|centos)
-            "@development-libs" \
+                pkg_list+=(
-            "perl-FindBin" \
+                    "bash-completion"
-            "zlib-static" \
+                    "bzip2"
-            "elfutils-libelf-devel" \
+                    "gcc"
-            "gawk" \
+                    "gcc-c++"
-            "unzip" \
+                    "git"
-            "file" \
+                    "make"
-            "wget" \
+                    "ncurses-devel"
-            "python3" \
+                    "patch"
-            "python2" \
+                    "rsync"
-            "axel" \
+                    "tar"
+                    "unzip"
+                    "wget"
+                    "which"
+                    "diffutils"
+                    "python2"
+                    "python3"
+                    "perl-base"
+                    "perl-Data-Dumper"
+                    "perl-File-Compare"
+                    "perl-File-Copy"
+                    "perl-FindBin"
+                    "perl-Thread-Queue"
                 )
-        _pkg_cmd="dnf"
+                ;;
-    elif [[ "$ID" =~ ^(debian|ubuntu)$ ]]; then
+            debian|ubuntu)
-        _pkg_list=(\
+                pkg_list+=(
-            "build-essential" \
+                    "build-essential"
-            "libncurses5-dev" \
+                    "clang"
-            "libncursesw5-dev" \
+                    "flex"
-            "zlib1g-dev" \
+                    "g++"
-            "gawk" \
+                    "gawk"
-            "git" \
+                    "gcc-multilib"
-            "gettext" \
+                    "gettext"
-            "libssl-dev" \
+                    "git"
-            "xsltproc" \
+                    "libncurses5-dev"
-            "wget" \
+                    "libssl-dev"
-            "unzip" \
+                    "python3-distutils"
-            "python" \
+                    "rsync"
-            "axel" \
+                    "unzip"
+                    "zlib1g-dev"
+                    "file"
+                    "wget"
                 )
-        _pkg_cmd="apt-get"
+                ;;
+            arch)
+                pkg_list+=(
+                    "base-devel"
+                    "autoconf"
+                    "automake"
+                    "bash"
+                    "binutils"
+                    "bison"
+                    "bzip2"
+                    "fakeroot"
+                    "file"
+                    "findutils"
+                    "flex"
+                    "gawk"
+                    "gcc"
+                    "gettext"
+                    "git"
+                    "grep"
+                    "groff"
+                    "gzip"
+                    "libelf"
+                    "libtool"
+                    "libxslt"
+                    "m4"
+                    "make"
+                    "ncurses"
+                    "openssl"
+                    "patch"
+                    "pkgconf"
+                    "python"
+                    "rsync"
+                    "sed"
+                    "texinfo"
+                    "time"
+                    "unzip"
+                    "util-linux"
+                    "wget"
+                    "which"
+                    "zlib"
+                )
+        esac
+    else
+        # For Imagebuilder
+        case "$ID" in
+            fedora|centos)
+                pkg_list+=(
+                    "@c-development" 
+                    "@development-tools"
+                    "@development-libs"
+                    "perl-FindBin"
+                    "zlib-static"
+                    "elfutils-libelf-devel"
+                    "gawk"
+                    "unzip"
+                    "file"
+                    "wget"
+                    "python3"
+                    "python2"
+                    "axel"
+                )
+                ;;
+            debian|ubuntu)
+                pkg_list+=(
+                    "build-essential"
+                    "libncurses5-dev"
+                    "libncursesw5-dev"
+                    "zlib1g-dev"
+                    "gawk"
+                    "git"
+                    "gettext"
+                    "libssl-dev"
+                    "xsltproc"
+                    "wget"
+                    "unzip"
+                    "python"
+                    "axel"
+                )
+                ;;
+        esac
     fi
 
-    echo "Installing dependencies"
+    pkg_install "${pkg_list[@]}"
-    debug "sudo $_pkg_cmd -y install ${_pkg_list[*]}"
-    if ! sudo "$_pkg_cmd" -y install "${_pkg_list[@]}" > /dev/null 2>&1; then
-        echo "Warning: Problem installing prerequisites"
-        return 1
-    fi
 }
 
 
 getImageBuilder() {
     debug "${FUNCNAME[0]}"
 
-    local _url _filename _dl_tool
+    declare url_prefix filename url dl_tool sha256sum
 
     if [[ "${P_ARR[release]}" == "snapshot" ]]; then
-        _filename="openwrt-imagebuilder-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
+        url_prefix="https://downloads.openwrt.org/snapshots/targets/${P_ARR[target]}"
-        _url="https://downloads.openwrt.org/snapshots/targets/${P_ARR[target]}/$_filename"
+        filename="openwrt-imagebuilder-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
+        url="${url_prefix}/$filename"
+    else
+        url_prefix="https://downloads.openwrt.org/releases/${P_ARR[release]}/targets/${P_ARR[target]}"
+        filename="openwrt-imagebuilder-${P_ARR[release]}-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
+        url="${url_prefix}/$filename"
+        [[ -f "${P_ARR[source_archive]}" ]] && return 0 # Reuse existing ImageBuilders
+    fi
+
     if [[ -f "${P_ARR[source_archive]}" ]]; then
-            if askOk "Update ImageBuilder snapshot?"; then
+        if askOk "Update ImageBuilder ?"; then
             rm -f "${P_ARR[source_archive]}"
         else
             return 0
         fi
     fi
-    else
-        _filename="openwrt-imagebuilder-${P_ARR[release]}-${P_ARR[target]//\//-}.Linux-x86_64.tar.xz"
-        _url="https://downloads.openwrt.org/releases/${P_ARR[release]}/targets/${P_ARR[target]}/$_filename"
-        [[ -f "${P_ARR[source_archive]}" ]] && return 0 # Reuse existing ImageBuilders
-    fi
     
     # Make sources directory if it does not exist
     [[ ! -d "$BUILDDIR/sources" ]] && mkdir -p "$BUILDDIR/sources"
 
     if hash axel &>/dev/null; then
-        _dl_tool="axel"
+        dl_tool="axel"
     elif hash curl &>/dev/null; then
-        _dl_tool="curl"
+        dl_tool="curl"
     else
         echo "Downloading the ImageBuilder requires axel or curl!"
         return 1
     fi
     
-    _dl_tool="curl" # TODO remove
+    echo "Downloading imagebuilder archive using $dl_tool"
 
-    echo "Downloading imagebuilder archive using $_dl_tool"
+    debug "$dl_tool -o ${P_ARR[source_archive]} $url"
-
+    if ! "$dl_tool" -o "${P_ARR[source_archive]}" "$url"; then
-    debug "$_dl_tool -o ${P_ARR[source_archive]} $_url"
-    if ! "$_dl_tool" -o "${P_ARR[source_archive]}" "$_url" > /dev/null 2>&1; then
         echo "Could not download imagebuilder archive"
         exit 1
     fi
@@ -199,6 +294,16 @@ getImageBuilder() {
         exit 1
     fi
 
+    
+    if hash sha256sum &>/dev/null; then
+        echo "Verifying checksums"
+        debug "$dl_tool -s $url_prefix/sha256sums | grep $filename | cut -f1 -d' '"
+        sha256sum=$($dl_tool -s "$url_prefix"/sha256sums |grep "$filename" |cut -f1 -d' ')
+        debug "Downloaded sha256sum: $sha256sum"
+
+    fi
+
+
     echo "Extracting image archive"
     [[ ! -d "${P_ARR[source_dir]}" ]] && mkdir -p "${P_ARR[source_dir]}"
     debug "tar -xf ${P_ARR[source_archive]} -C ${P_ARR[source_dir]} --strip-components 1"
@@ -271,9 +376,14 @@ makeImage() {
 
     [[ ! -d "${P_ARR[out_bin_dir]}" ]] && mkdir -p "${P_ARR[out_bin_dir]}"
 
-    # build image
+    if ! make image \
-    debug "make -j4 image BIN_DIR=${P_ARR[out_bin_dir]} PROFILE=${P_ARR[profile]} PACKAGES=${P_ARR[packages]} FILES=$FILESDIR --directory=${P_ARR[source_dir]} > make.log"
+            BIN_DIR="${P_ARR[out_bin_dir]}" \
-    if ! make image BIN_DIR="${P_ARR[out_bin_dir]}" PROFILE="${P_ARR[profile]}" PACKAGES="${P_ARR[packages]}" FILES="$FILESDIR" --directory="${P_ARR[source_dir]}" > make.log; then
+            PROFILE="${P_ARR[profile]}" \
+            PACKAGES="${P_ARR[packages]}" \
+            FILES="$FILESDIR" \
+            --directory="${P_ARR[source_dir]}" \
+            --jobs=$(( $(nproc) - 1 )) \
+            > make.log; then
         echo "Make image failed!"
         exit 1
     fi
@@ -340,6 +450,35 @@ sshUpgrade() {
 }
 
 
+fromSource() {
+    debug "${FUNCNAME[0]}"
+
+    declare src_url="https://github.com/openwrt/openwrt.git"
+    declare -a pkg_list
+
+    echo "Building from source is under development"
+
+
+    git clone --depth=1 "$src_url" "$BUILDDIR/sources"
+    pushd "$BUILDDIR/sources/$(basename "$src_url" .git)" || return 1
+
+    if [[ ${P_ARR[release]} == "snapshot" ]]; then
+        git checkout master
+    else
+        git checkout "${P_ARR[release]}"
+    fi
+
+    ./scripts/feeds update -a
+    ./scripts/feeds install -a
+
+    make distclean
+    make download
+    make -j"$(nproc)" world
+
+    popd || return 1
+}
+
+
 debug() { (( DEBUG )) && echo "Running: $*"; }
 
 
@@ -371,37 +510,118 @@ loadProfiles() {
 }
 
 
+init() {
+    debug "${FUNCNAME[0]}"
+
+    declare -g ID RPM_MGR
+
+    echo "Starting openwrtbuilder"
+    debug || echo "To enable debugging output, use --debug or -d"
+
+    if [[ -e "/etc/os-release" ]]; then
+        source "/etc/os-release"
+    else
+        err "/etc/os-release not found"
+        err "Your OS is unsupported"
+        printHelp
+        exit 1
+    fi
+
+    debug "Detected host platform: $ID $VERSION_ID"
+
+    # normalize distro ID
+    case "$ID" in
+        debian|arch)
+            ;;
+        centos|fedora)
+            if hash dnf &>/dev/null; then
+                RPM_MGR="dnf"
+            elif hash yum &>/dev/null; then
+                RPM_MGR="yum"
+            fi
+            ;;
+        rhel)
+            ID="centos"
+            ;;
+        linuxmint|neon|*ubuntu*)
+            ID="ubuntu"
+            ;;
+        *suse*)
+            ID="suse"
+            ;;
+        raspbian)
+            ID="debian"
+            ;;
+        *)
+            echo "Autodetecting distro, this may be unreliable and --compat may also be required"
+            if hash dnf &>/dev/null; then
+                ID="fedora"
+                RPM_MGR="dnf"
+            elif hash yum &>/dev/null; then
+                ID="centos"
+                RPM_MGR="yum"
+            elif hash apt &>/dev/null; then
+                ID="ubuntu"
+            elif hash pacman &>/dev/null; then
+                ID="arch"
+            else
+                return 1
+            fi
+            ;;
+    esac
+
+    debug "Using host platform: $ID $VERSION_ID"
+
+    # Set distro-specific functions
+    case "$ID" in
+        fedora|centos)
+            pkg_install(){ sudo "$RPM_MGR" install -y "$@"; }
+            ;;
+        debian|ubuntu)
+            pkg_install(){ sudo apt-get install -y -q0 "$@"; }
+            ;;
+        suse)
+            pkg_install(){ sudo zypper --non-interactive -q install --force --no-confirm "$@"; }
+            ;;
+        arch)
+            pkg_install(){ sudo pacman -S --noconfirm --needed "$@"; }
+            ;;
+    esac
+}
+
+
 main() {
     debug "${FUNCNAME[0]}"
 
+    init
+
     loadProfiles
    
     readInput "$@"
 
-    [[ ! -v "$PROFILE"[@] ]] && echo "Profile does not exist" && return 1
+    [[ ! ${!PROFILE@a} = A ]] && echo "Profile does not exist" && return 1
     declare -gn P_ARR="$PROFILE"
-    declare _out_prefix
     
     : "${BUILDDIR:=$SCRIPTDIR}"
     : "${FILESDIR:=$BUILDDIR/files}"
-    : "${RELEASE:="21.02.1"}"
 
-    : "${P_ARR[release]:=$RELEASE}"
+    : "${P_ARR[release]:=$RELEASE}" # profiles overrides user input and hardcoded versions
     : "${P_ARR[source_archive]:=$BUILDDIR/sources/${P_ARR[profile]}-${P_ARR[release]}.tar.xz}"
     : "${P_ARR[source_dir]:=${P_ARR[source_archive]%.tar.xz}}"
     : "${P_ARR[out_bin_dir]:=$BUILDDIR/bin/${P_ARR[profile]}-${P_ARR[release]}}"
 
+    declare out_prefix
     if [[ "${P_ARR[release]}" == "snapshot" ]]; then
-        _out_prefix="${P_ARR[out_bin_dir]}/openwrt-${P_ARR[target]//\//-}-${P_ARR[profile]}"
+        out_prefix="${P_ARR[out_bin_dir]}/openwrt-${P_ARR[target]//\//-}-${P_ARR[profile]}"
     else
-        _out_prefix="${P_ARR[out_bin_dir]}/openwrt-${P_ARR[release]}-${P_ARR[target]//\//-}-${P_ARR[profile]}"
+        out_prefix="${P_ARR[out_bin_dir]}/openwrt-${P_ARR[release]}-${P_ARR[target]//\//-}-${P_ARR[profile]}"
     fi
 
-    : "${P_ARR[factory_img]:=$_out_prefix-${P_ARR[filesystem]}-factory.img}"
+    : "${P_ARR[factory_img]:=$out_prefix-${P_ARR[filesystem]}-factory.img}"
     : "${P_ARR[factory_img_gz]:=${P_ARR[factory_img]}.gz}"
-    : "${P_ARR[sysupgrade_img]:=$_out_prefix-${P_ARR[filesystem]}-sysupgrade.img}"
+    : "${P_ARR[sysupgrade_img]:=$out_prefix-${P_ARR[filesystem]}-sysupgrade.img}"
     : "${P_ARR[sysupgrade_img_gz]:=${P_ARR[sysupgrade_img]}.gz}"
-    : "${P_ARR[sysupgrade_bin]:=$_out_prefix-${P_ARR[filesystem]}-sysupgrade.img}"
+    : "${P_ARR[sysupgrade_bin]:=$out_prefix-${P_ARR[filesystem]}-sysupgrade.img}"
     : "${P_ARR[sysupgrade_bin_fname]:=${P_ARR[sysupgrade_bin]##*/}}"
     : "${P_ARR[sysupgrade_bin_gz]:=${P_ARR[sysupgrade_bin]}.gz}"
     : "${P_ARR[sysupgrade_bin_gz_fname]:=${P_ARR[sysupgrade_bin_gz]##*/}}"
@@ -412,7 +632,10 @@ main() {
         for x in "${!P_ARR[@]}"; do printf "[%s]=%s\n" "$x" "${P_ARR[$x]}"; done
     fi
 
-    installHostDependencies
+    installDependencies
+
+    # Experimental
+    (( FROM_SOURCE )) && fromSource
 
     getImageBuilder
 

profiles

@@ -13,7 +13,13 @@ default_packages="\
 	tar \
 	iperf \
 	bash \
-	rsync " # Leave trailing whitespace
+	rsync \
+	openssh-sftp-server \
+	luci-app-statistics \
+	collectd-mod-sensors \
+	collectd-mod-thermal \
+	collectd-mod-conntrack \
+	collectd-mod-cpu " # Leave trailing whitespace
 
 
 declare -Ag archer
@@ -76,91 +82,68 @@ r2s['packages']="\
 	ethtool"
 
 declare -Ag r4s
-r4s['release']="snapshot"
 r4s['profile']="friendlyarm_nanopi-r4s"
 r4s['target']="rockchip/armv8"
 r4s['filesystem']="ext4"
-r4s['repo']="src/gz stangri_repo https://repo.openwrt.melmac.net"
-# fw3 + vpn-policy-routing
-# r4s['packages']="\
-# 	$default_packages \
-# 	luci-app-upnp \
-# 	luci-app-wireguard \
-# 	luci-app-vpn-policy-routing \
-#   vpn-policy-routing \
-# 	-dnsmasq \
-# 	dnsmasq-full \
-# 	luci-app-ddns \
-# 	luci-app-sqm \
-# 	luci-app-statistics \
-# 	collectd-mod-sensors \
-# 	collectd-mod-thermal \
-# 	collectd-mod-conntrack \
-# 	smcroute \
-# 	curl \
-# 	ethtool \
-# 	kmod-ipt-nat6 \
-# 	-firewall4 \
-# 	-nftables \
-# 	-kmod-nft-offload \
-# 	firewall \
-# 	ip6tables \
-# 	iptables \
-# 	kmod-ipt-offload"
-
-# fw3 + pbr
 r4s['packages']="\
 	$default_packages \
 	luci-app-upnp \
 	luci-app-wireguard \
-	luci-app-pbr \
-	pbr-ipt \
-	-dnsmasq \
-	dnsmasq-full \
 	luci-app-ddns \
 	luci-app-sqm \
-	luci-app-statistics \
+	irqbalance \
-	collectd-mod-sensors \
+	collectd-mod-sqm \
-	collectd-mod-thermal \
+	collectd-mod-df \
-	collectd-mod-conntrack \
+	pbr \
+	luci-app-pbr \
+	usbutils \
+	kmod-usb-storage \
+	kmod-usb-storage-uas \
+	kmod-fs-btrfs \
+	btrfs-progs \
+	block-mount \
 	smcroute \
 	curl \
 	ethtool \
-	kmod-ipt-nat6 \
+	ca-bundle"
-	-firewall4 \
-	-nftables \
-	-kmod-nft-offload \
-	firewall \
-	ip6tables \
-	iptables \
-	kmod-ipt-offload"
 
-# fw4 + pbr
+declare -Ag ax6000_stock
-# r4s['packages']="\
+ax6000_stock['profile']="xiaomi_redmi-router-ax6000-stock"
-# 	$default_packages \
+ax6000_stock['target']="mediatek/filogic"
-# 	luci-app-upnp \
+ax6000_stock['release']="snapshot"
-# 	luci-app-wireguard \
+ax6000_stock['filesystem']="squashfs"
-# 	luci-app-pbr \
+ax6000_stock['packages']="\
-# 	pbr-netifd \
-# 	-dnsmasq \
-# 	dnsmasq-full \
-# 	luci-app-ddns \
-# 	luci-app-sqm \
-# 	luci-app-statistics \
-# 	collectd-mod-sensors \
-# 	collectd-mod-thermal \
-# 	collectd-mod-conntrack \
-# 	smcroute \
-# 	curl \
-# 	ethtool \
-# 	kmod-nft-nat6"
-
-declare -Ag x5000r
-x5000r['profile']="totolink_x5000r"
-x5000r['target']="ramips/mt7621"
-x5000r['filesystem']="squashfs"
-x5000r['packages']="\
 	$default_packages \
 	-dnsmasq \
-	-odhcpd \
+	-odhcpd-ipv6only \
-	-iptables"
+	-nftables \
+	-firewall4 \
+	-kmod-nft-offload \
+	collectd-mod-iwinfo"
+
+declare -Ag ax6000_uboot
+ax6000_uboot['profile']="xiaomi_redmi-router-ax6000-ubootmod"
+ax6000_uboot['target']="mediatek/filogic"
+ax6000_uboot['release']="snapshot"
+ax6000_uboot['filesystem']="squashfs"
+ax6000_uboot['packages']="\
+	$default_packages \
+	-dnsmasq \
+	-odhcpd-ipv6only \
+	-nftables \
+	-firewall4 \
+	-kmod-nft-offload \
+	collectd-mod-iwinfo"
+
+declare -Ag totolink
+totolink['profile']="totolink_x5000r"
+totolink['target']="ramips/mt7621"
+totolink['filesystem']="squashfs"
+totolink['packages']="\
+	$default_packages \
+	-dnsmasq \
+	-odhcpd-ipv6only \
+	-nftables \
+	-firewall4 \
+	-kmod-nft-offload \
+	collectd-mod-iwinfo"