Squashed initial commit

2024-09-10 13:47:29 -04:00
commit 8ebb6ad265
6221 changed files with 2512206 additions and 0 deletions
--- a/server-scripts/script-drives-fix-btrfs-full
+++ b/server-scripts/script-drives-fix-btrfs-full
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Fix the btrfs out of space error
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+fs=( "/mnt/array" "/mnt/backup" )
+
+# Discard empty blocks
+for f in "${fs[@]}"; do
+    btrfs balance start -dusage=0 "$f"
+    btrfs balance start -musage=0 "$f"
+done
+
+exit $?
--- a/server-scripts/script-files-permissions-reset
+++ b/server-scripts/script-files-permissions-reset
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Smartly change permissions on selected directories
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+[[ $# -eq 0 ]] && DIRS=("/mnt/data") || DIRS=("$@")
+
+ask_ok "Reset permissions on ${DIRS[*]}?"
+
+chgrp smbgrp -R "${DIRS[@]}" && \
+chmod 6775 -R "${DIRS[@]}"
+
+exit $?
--- a/server-scripts/script-files-permissions-set
+++ b/server-scripts/script-files-permissions-set
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Smartly change permissions on selected directories
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+if [[ $# -eq 0 ]]; then
+    echo "No arguments provided, using autodetection"
+    paths=("$PWD")
+    user=$(stat -c "%U" "$PWD")
+    group=$(stat -c "%G" "$PWD")
+elif [[ $# -eq 1 ]]; then
+    user="$1"
+    group="$1"
+    paths=("$PWD")
+elif [[ $# -eq 2 ]]; then
+    user="$1"
+    group="$2"
+    paths=("$PWD")
+elif [[ $# -gt 2 ]]; then
+    user="$1"
+    group="$2"
+    paths=("${@:3}")
+fi
+
+for path in "${paths[@]}"; do 
+    if [[ "$path" == "/" ]]; then
+        echo "You are trying to operate on the root partition!"
+        echo "This seems highly unusual!"
+        ask_ok "Continue?" || exit $?
+    fi
+    og_user=$(stat -c "%U" "$path")
+    og_group=$(stat -c "%G" "$path")
+    echo -e "PATH\tUSER\tGROUP"
+    echo -e "$path\t$og_user\t$og_group"
+    if [[ "$group" != "smbgrp" || "$og_group" != "smbgrp" ]]; then
+        echo "$path is not world accessible by the smbgrp group"
+        ask_ok "Change $path group $og_group to smbgrp?" && group="smbgrp"
+    fi
+done
+
+
+ask_ok "Apply user: $user and group: $group to ${paths[*]} and all subdirs?" && \
+    chown -R "$user":"$group" "${paths[@]}"
+
+[[ "$group" == "smbgrp" ]] && mode=6775 || mode=755
+    
+ask_ok "Apply chmod $mode to ${paths[*]} and all subdirs?" && \
+    chmod -R $mode "${paths[@]}"
+
+# Let's do it in less steps (see above) for now unless it becomes a problem
+# echo "Apply setuid/setgid bits to ${paths[*]} and all subdirs?"
+# ask_ok "Files/dirs will inherit their " && \
+#     chmod -R g+s,u+s "${paths[@]}"    
+
+
+exit $?
--- a/server-scripts/script-functions
+++ b/server-scripts/script-functions
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Common functions for the lab scripts
+# Copyright Bryan C. Roessler
+
+# Don't run this script directly
+[[ "${BASH_SOURCE[0]}" == "${0}" ]] && exit 0
+
+
+### VARS ###
+export INSTALL_DIR=/usr/local/bin
+
+
+### FUNCTIONS ###
+prompt() { read -r -p "Enter $1: " "$1"; }
+
+ask_ok() {
+    declare response
+    (( YES_SWITCH )) && return 0
+    read -r -p "$* [y/N]: " response
+    [[ ${response,,} =~ ^(yes|y)$ ]]
+}
+
+is_root() {
+    [[ $EUID -gt 0 ]] && echo "Script must be run with sudo" && exit 1
+}
+
+copy_manual() {
+    cat <<-EOF > "$1/manual.desktop"
+    [Desktop Entry]
+    Encoding=UTF-8
+    Name=Hartman Lab Server Manual
+    Type=Link
+    URL=https://docs.google.com/document/d/1K_KwAlv8Zljmy-enwmhT6gMTFutlAFglixvpLGBx0VY
+    Icon=text-html
+EOF
+}
--- a/server-scripts/script-install-btrfsmaintenance
+++ b/server-scripts/script-install-btrfsmaintenance
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Generic btrfsmaintenance install script
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+# Optionally provide the config directory manually
+if [[ $# -lt 1 ]]; then
+    if [[ -d /etc/sysconfig ]]; then
+        CONFDIR=/etc/sysconfig
+    elif [[ -d /etc/default ]]; then
+        CONFDIR=/etc/default
+    else
+        echo "Cannot detect sysconfig directory, please specify manually"
+        exit 1
+    fi
+else
+    CONFDIR="$1"
+fi
+
+# This is hardcoded by the btrfs maintenance scripts, change at your peril
+INSTALLDIR="/usr/share/btrfsmaintenance"
+
+# Backup existing installation
+if [[ -d "$INSTALLDIR" ]]; then
+    TEMPDIR="/tmp/btrfs-maintenance.bk"
+    echo "Moving existing $INSTALLDIR to $TEMPDIR"
+    [[ -d "$TEMPDIR" ]] && rm -rf "$TEMPDIR"
+    mv "$INSTALLDIR" "$TEMPDIR"
+fi
+
+git clone "https://github.com/kdave/btrfsmaintenance.git" "$INSTALLDIR"
+
+# Quirks
+if [[ -e "/etc/os-release" ]]; then
+    source "/etc/os-release"
+    if [[ "$ID" == "centos" && "$VERSION_ID" == "7" ]]; then
+        sed -i 's/flock --verbose/flock' "$INSTALLDIR"/btrfsmaintenance-functions
+    fi
+fi
+
+chmod 755 "$INSTALLDIR"/*.sh
+
+# Copy config file
+[[ ! -f "$CONFDIR"/btrfsmaintenance ]] && install -oroot -groot -m644 "$INSTALLDIR"/sysconfig.btrfsmaintenance "$CONFDIR"/btrfsmaintenance
+
+# Copy systemd files and reload
+for f in "$INSTALLDIR"/btrfs-*.{service,timer}; do
+    cp "$f" /usr/lib/systemd/system/
+done
+systemctl daemon-reload
+
+# Optionally, start and enable the services
+# systemctl enable --now btrfs-scrub
+
+exit $?
--- a/server-scripts/script-install-manual-scripts
+++ b/server-scripts/script-install-manual-scripts
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# This script will add scripts-* to the PATH and the manual to each user's desktop
+# Copyright 2021-2023 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+reload=0
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || reload=1
+
+sourcedir="/home/roessler/shared/hartmanlab"
+original_dir="$PWD"
+
+if [[ "$original_dir" != "$sourcedir" ]]; then
+    pushd "$sourcedir" || exit $?
+fi
+
+(( reload )) && [[ -f $parent/script-functions ]] && . "$parent"/script-functions
+
+is_root
+
+target=/usr/local/bin
+for script in script-*; do
+    echo "Installing $script to $target"
+    [[ $script == "script-functions" ]] && install -m 644 "$script" "$target"
+    cp -u "$script" "$target/"
+done
+
+# Install manual link
+remove=("manual.pdf" "manual.odt" "Notes.pdf" "Notes.odt" 
+"README.html" "Link to Manual.desktop" "manual-images" 
+"manual.html" "manual-images" "Manual.desktop" "manual.desktop")
+for homedir in /home/*; do
+    desktop="$homedir/Desktop"
+    [[ -d $desktop ]] || continue
+    echo "Scanning $desktop for old manuals"
+    for f in "${remove[@]}"; do
+        [[ -e $desktop/$f || -L $desktop/$f ]] &&
+        echo "Removing $desktop/$f" &&
+        rm -f "${desktop:?}/$f"
+    done
+    echo "Installing manual to $desktop/manual.desktop"
+    cat <<-EOF > "$desktop/manual.desktop"
+		[Desktop Entry]
+		Encoding=UTF-8
+		Name=Hartman Lab Server Manual
+		Type=Link
+		URL=https://docs.google.com/document/d/1K_KwAlv8Zljmy-enwmhT6gMTFutlAFglixvpLGBx0VY
+		Icon=text-html
+	EOF
+done
+
--- a/server-scripts/script-install-motd
+++ b/server-scripts/script-install-motd
@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+# Install and generate motd
+# Bryan C. Roessler
+
+main() {
+	if [[ " $1 " == " --motd " ]]; then
+		print_motd
+	else
+		parent="${BASH_SOURCE[0]}"
+		parent=${parent%/*}
+
+		[[ -f "$parent"/script-functions ]] && . "$parent"/script-functions || exit 1
+
+		is_root
+
+		script="/usr/local/bin/${0##*/}"
+		service="/usr/lib/systemd/system/motd.service"
+		timer="/usr/lib/systemd/system/motd.timer"
+
+		[[ -f $script ]] || cp "$0" /usr/local/bin/
+
+		install_services
+	fi
+}
+
+print_motd() {
+	# colors
+	default='\e[0m'
+	green='\e[32m'
+	red='\e[31m'
+	dim='\e[2m'
+	undim='\e[0m'
+
+# shellcheck disable=SC2016
+	echo -e \
+' _    _            _                           _           _     
+| |  | |          | |                         | |         | |    
+| |__| | __ _ _ __| |_ _ __ ___   __ _ _ __   | |     __ _| |__  
+|  __  |/ _` | `__| __| `_ ` _ \ / _` | `_ \  | |    / _` | `_ \ 
+| |  | | (_| | |  | |_| | | | | | (_| | | | | | |___| (_| | |_) |
+|_|  |_|\__,_|_|   \__|_| |_| |_|\__,_|_| |_| |______\__,_|_.__/ '
+
+	# System info
+	# get load averages
+	IFS=" " read -r LOAD1 LOAD5 LOAD15 <<<"$(awk '{ print $1,$2,$3 }' /proc/loadavg)"
+	# get free memory
+	IFS=" " read -r USED AVAIL TOTAL <<<"$(free -htm | grep "Mem" | awk '{print $3,$7,$2}')"
+	# get processes
+	PROCESS=$(ps -eo user=|sort|uniq -c | awk '{ print $2 " " $1 }')
+	PROCESS_ALL=$(echo "$PROCESS"| awk '{print $2}' | awk '{ SUM += $1} END { print SUM }')
+	PROCESS_ROOT=$(echo "$PROCESS"| grep root | awk '{print $2}')
+	PROCESS_USER=$(echo "$PROCESS"| grep -v root | awk '{print $2}' | awk '{ SUM += $1} END { print SUM }')
+	# get processors
+	PROCESSOR_NAME=$(grep "model name" /proc/cpuinfo | cut -d ' ' -f3- | awk '{print $0}' | head -1)
+	PROCESSOR_COUNT=$(grep -ioP 'processor\t:' /proc/cpuinfo | wc -l)
+
+	echo -e "
+  ${default}Distro......: $default$(cat /etc/*release | grep "PRETTY_NAME" | cut -d "=" -f 2- | sed 's/"//g')
+  ${default}Kernel......: $default$(uname -sr)
+  ${default}Uptime......: $default$(uptime -p)
+  ${default}Load........: $green$LOAD1$default (1m), $green$LOAD5$default (5m), $green$LOAD15$default (15m)
+  ${default}Processes...: $default$green$PROCESS_ROOT$default (root), $green$PROCESS_USER$default (user), $green$PROCESS_ALL$default (total)
+  ${default}CPU.........: $default$PROCESSOR_NAME ($green$PROCESSOR_COUNT$default vCPU)
+  ${default}Memory......: $green$USED$default used, $green$AVAIL$default avail, $green$TOTAL$default total$default"
+
+	# Disk usage
+	# config
+	max_usage=90
+	bar_width=50
+
+	# disk usage: ignore zfs, squashfs & tmpfs
+	while IFS= read -r line; do dfs+=("$line"); done < <(df -H -x zfs -x squashfs -x tmpfs -x devtmpfs -x overlay --output=target,pcent,size | tail -n+2)
+	printf "\nDisk usage\n"
+
+	for line in "${dfs[@]}"; do
+		# get disk usage
+		usage=$(echo "$line" | awk '{print $2}' | sed 's/%//')
+		used_width=$(((usage*bar_width)/100))
+		# color is green if usage < max_usage, else red
+		if [ "${usage}" -ge "${max_usage}" ]; then
+			color=$red
+		else
+			color=$green
+		fi
+		# print green/red bar until used_width
+		bar="[${color}"
+		for ((i=0; i<used_width; i++)); do
+			bar+="="
+		done
+		# print dimmmed bar until end
+		bar+="${default}${dim}"
+		for ((i=used_width; i<bar_width; i++)); do
+			bar+="="
+		done
+		bar+="${undim}]"
+		# print usage line & bar
+		echo "${line}" | awk '{ printf("%-31s%+3s used out of %+4s\n", $1, $2, $3); }' | sed -e 's/^/  /'
+		echo -e "${bar}" | sed -e 's/^/  /'
+	done
+
+	# # Disk health
+	# cat <<- 'EOF' >> "$script"
+	# # config
+	# MAX_TEMP=40
+	# # set column width
+	# COLUMNS=2
+	# # colors
+
+	# # disks to check
+	# disks=(sda sdb sdc sdd sde sdf sdg sdi)
+	# disknames=(sda sdb sdc sdd sde sdf sdg sdi)
+
+	# # hddtemp
+	# hddtemp_host=localhost
+	# hddtemp_port=7634
+
+	# # logfiles to check
+	# logfiles='/var/log/syslog /var/log/syslog.1'
+
+	# # get all lines with smartd entries from syslog
+	# lines=$(tac $logfiles | grep -hiP 'smartd\[[[:digit:]]+\]:' | grep -iP "previous self-test")
+	# # use nc to query temps from hddtemp daemon
+	# hddtemp=$(timeout 0.01 nc $hddtemp_host $hddtemp_port | sed 's/|//m' | sed 's/||/ \n/g')
+
+	# out=""
+	# for i in "${!disks[@]}"; do
+	#     disk=${disks[$i]}
+	#     # use disknames if given
+	#     diskname=${disknames[$i]}
+	#     if [ -z "${diskname}" ]; then
+	#         diskname=$disk
+	#     fi
+	#     uuid=$(blkid -s UUID -o value "/dev/${disk}")
+	#     status=$( (grep "${uuid}" <<< "${lines}") | grep -m 1 -oP "previous self-test.*" | awk '{ print $4 " " $5 }')
+	#     temp=$( (grep "${disk}" <<< "${hddtemp}") | awk -F'|' '{ print $3 }')
+
+	#     # color green if temp <= MAX_TEMP, else red
+	#     if [[ "${temp}" -gt "${MAX_TEMP}" ]]; then
+	#         color=$red
+	#     else
+	#         color=$green
+	#     fi
+	#     # add "C" if temp is numeric
+	#     if [[ "$temp" =~ ^[0-9]+$ ]]; then
+	#         temp="${temp}C"
+	#     fi
+	#     # color green if status is "without error", else red
+	#     if [[ "${status}" == "without error" ]]; then
+	#         status_color=$green
+	#     else
+	#         status_color=$red
+	#     fi
+
+	#     # print temp & smartd error
+	#     out+="${diskname}:,${color}${temp}${undim} | ${status_color}${status}${undim},"
+	#     # insert \n every $COLUMNS column
+	#     if [ $((($i+1) % $COLUMNS)) -eq 0 ]; then
+	#         out+="\n"
+	#     fi
+	# done
+	# out+="\n"
+
+	# printf "\ndisk status:\n"
+	# printf "$out" | column -ts $',' | sed -e 's/^/  /'
+	# EOF
+
+
+	# Services
+	COLUMNS=2
+
+	services=(
+		btrfs-balance.timer btrfs-scrub.timer backup.timer btrbk.timer fstrim.timer
+		fail2ban firewalld smb nmb motion smartd cockpit.socket
+		dnf-automatic.timer motd.timer
+	)
+
+	service_status=()
+	# get status of all services
+	for service in "${services[@]}"; do
+		service_status+=("$(systemctl is-active "$service")")
+	done
+
+	out=""
+	for i in "${!services[@]}"; do
+		# color green if service is active, else red
+		if [[ "${service_status[$i]}" == "active" ]]; then
+			out+="${services[$i]%.*}:,${green}${service_status[$i]}${undim},"
+		else
+			out+="${services[$i]%.*}:,${red}${service_status[$i]}${undim},"
+		fi
+		# insert \n every $COLUMNS column
+		if [[ $(((i+1) % COLUMNS)) -eq 0 ]]; then
+			out+="\n"
+		fi
+	done
+
+	printf "\nServices\n"
+	printf "%b\n" "$out" | column -ts $',' | sed -e 's/^/  /'
+
+
+	# Fail2Ban
+	# fail2ban-client status to get all jails, takes about ~70ms
+	read -r -a jails <<< "$(fail2ban-client status | grep "Jail list:" | sed "s/ //g" | awk '{split($2,a,",");for(i in a) print a[i]}')"
+
+	out="jail,failed,total,banned,total\n"
+
+	for jail in "${jails[@]}"; do
+		# slow because fail2ban-client has to be called for every jail (~70ms per jail)
+		status=$(fail2ban-client status "$jail")
+		failed=$(echo "$status" | grep -ioP '(?<=Currently failed:\t)[[:digit:]]+')
+		totalfailed=$(echo "$status" | grep -ioP '(?<=Total failed:\t)[[:digit:]]+')
+		banned=$(echo "$status" | grep -ioP '(?<=Currently banned:\t)[[:digit:]]+')
+		totalbanned=$(echo "$status" | grep -ioP '(?<=Total banned:\t)[[:digit:]]+')
+		out+="$jail,$failed,$totalfailed,$banned,$totalbanned\n"
+	done
+
+	printf "\nFail2ban\n"
+	printf "%b\n" "$out" | column -ts $',' | sed -e 's/^/  /'
+
+	# Help links
+	echo -e '
+Links (ctrl+click to follow)
+  Server Manual.........: https://tinyurl.com/jjz9h6fr
+  Cockpit (for admins)..: http://localhost:9090
+  Robot Camera..........: http://localhost:9999
+  JupyterLab............: http://localhost:8888
+  RStudio Server........: http://localhost:8787
+  Robot Computer........: vnc://192.168.16.101:5900
+  Windows 10 VM.........: vnc://localhost:5900 (pw: hartman)
+	'
+
+	# Scheduled reboot
+	if systemctl is-active scheduled-reboot.timer &>/dev/null; then
+		echo -n "Next scheduled reboot: "
+		time=$(systemctl cat scheduled-reboot.timer | grep OnCalendar=)
+		time=${time#*=}
+		echo "$time"
+	fi
+}
+
+install_services() {
+	cat <<-EOF > "$service"
+		[Unit]
+		Description=Generate MoTD
+
+		[Service]
+		Type=simple
+		ExecStart=/usr/bin/bash -c '$script --motd > /etc/motd'
+
+		[Install]
+		WantedBy=default.target
+	EOF
+	cat <<-'EOF' > "$timer"
+		[Unit]
+		Description=Generate MoTD every minute on a timer
+
+		[Timer]
+		OnCalendar=*:0/1
+		OnBootSec=10s
+
+		[Install]
+		WantedBy=timers.target
+	EOF
+
+	chmod +x "$script" &&
+	systemctl daemon-reload &&
+	systemctl enable --now "${timer##*/}"
+}
+
+main "$@"
+
+exit $?
--- a/server-scripts/script-install-reverse-proxy
+++ b/server-scripts/script-install-reverse-proxy
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Adds a reverse proxy for local system services
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+dnf install -y nginx || exit $?
+
+cat <<- 'EOF' > /etc/nginx/conf.d/hartmanlab.conf
+server {
+	listen 80 default_server;
+	server_name localhost;
+	location /cockpit {
+		proxy_pass http://localhost:9090;
+	}
+}
+EOF
+
+	# location /robot {
+    #     proxy_pass http://127.0.0.1:8888;
+    #     proxy_redirect http://127.0.0.1:8888/;
+    # }
+
+systemctl enable --now nginx
+
+exit $?
--- a/server-scripts/script-qhtcp-new-project
+++ b/server-scripts/script-qhtcp-new-project
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Generate a new QHTCP experiment directory
+# Copyright 2024 Bryan C. Roessler
+
+echo "This script supports one optional argument, a project name"
+
+PROJECTS_DIR="/mnt/data/StudiesQHTCP"
+TEMPLATE_DIR="/mnt/data/StudiesQHTCP/_TEMPLATE_2copy_rename_4every_new_QHTCPstudy_23_1001"
+PROJECT_PREFIX="$(whoami)-$(date +%y-%m-%d)"
+
+ask_project_name() { read -r -p "Enter a new project name: " PROJECT_NAME; }
+
+if [[ $# == 1 ]]; then
+  PROJECT_NAME="$1"
+else
+  ask_project_name
+fi
+
+PROJECT_DIR="$PROJECTS_DIR/$PROJECT_PREFIX-$PROJECT_NAME"
+
+while [[ -d $PROJECT_DIR ]]; do
+  echo "A project already exists at $PROJECT_DIR"
+  ask_project_name
+  PROJECT_DIR="$PROJECTS_DIR/$PROJECT_PREFIX-$PROJECT_NAME"
+done
+
+if mkdir "$PROJECT_DIR" &&
+cp -a "$TEMPLATE_DIR"/* "$PROJECT_DIR"; then
+  echo "New project created at $PROJECT_DIR"
+fi
+
+
--- a/server-scripts/script-system-scheduled-restart
+++ b/server-scripts/script-system-scheduled-restart
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Update and restart the system
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+[[ $# -eq 0 ]] && time='*-*-* 01:30:00' # 1:30AM
+[[ $# -gt 1 ]] && time="$*"
+
+script-system-update
+
+ask_ok "Set a scheduled reboot for $time?" || exit 1
+
+cat <<- EOF > "/usr/lib/systemd/system/scheduled-reboot.timer"
+[Unit]
+Description=Scheduled reboot
+
+[Timer]
+OnCalendar=$time
+Unit=reboot.target
+
+[Install]
+WantedBy=timers.target
+EOF
+
+systemctl daemon-reload
+systemctl start scheduled-reboot.timer
+
+# Current date
+dt=$(date '+%d/%m/%Y %H:%M:%S');
+
+message="System restart scheduled for $time. The current time is $dt. Make sure all changes are saved."
+
+# Graphical notification
+IFS=$'\n'
+for LINE in $(w -hs); do
+    USER=$(echo "$LINE" | awk '{print $1}')
+    USER_ID=$(id -u "$USER")
+    DISP_ID=$(echo "$LINE" | awk '{print $8}')
+    sudo -u "$USER" DISPLAY="$DISP_ID" DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/"$USER_ID"/bus notify-send "$message" --icon=dialog-warning
+done
+
+# Wall notification
+wall -n "$message"
+
+exit $?
--- a/server-scripts/script-system-update
+++ b/server-scripts/script-system-update
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Update the system
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+ask_ok "Security updates are automatically installed, perform a full system update?" || exit $?
+
+dnf update --refresh
--- a/server-scripts/script-tree-to-markdown
+++ b/server-scripts/script-tree-to-markdown
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Make a nice markdown file from a dir tree
+# Copyright 2021 Bryan C. Roessler
+
+tree=$(tree -f --noreport --charset ascii "$1" |
+sed -e 's/| \+/  /g' -e 's/[|`]-\+/ */g' -e 's:\(* \)\(\(.*/\)\([^/]\+\)\):\1[\4](\2):g')
+printf "# Code/Directory Structure:\n\n%s" "$tree"
--- a/server-scripts/script-user-add
+++ b/server-scripts/script-user-add
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Add a user to the Hartman Lab server
+# Copyright Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+echo "This script supports two optional arguments, a username and password"
+
+if [[ $# -eq 0 ]]; then
+    prompt user
+    prompt password
+elif [[ $# -eq 1 ]]; then
+    user="$1"
+    prompt password
+elif [[ $# -eq 2 ]]; then
+    user="$1"
+    password="$2"
+elif [[ $# -gt 2 ]]; then
+    echo "Too many arguments provided"
+    exit 1
+fi
+
+useradd_cmd=(useradd -m -U)
+
+if id -u "$user" &>/dev/null; then
+    ask_ok "User $user exists. Run script-user-remove first?" || exit $?
+    "$parent"/script-user-remove "$user" || exit $?
+fi
+
+ask_ok "Create user $user with password $password?" || exit $?
+
+restore=0
+if [[ -d /mnt/array/home-retired/$user ]]; then
+    ask_ok "Restore user $user's files from /mnt/array/home-retired/$user?" && restore=1
+fi
+
+samba=0
+ask_ok "Enable shared file access for user $user?" && group_str="smbgrp" && samba=1
+
+ask_ok "Make $user an admin?" && \
+    group_str+=",wheel"
+
+useradd_cmd+=("-G" "$group_str")
+useradd_cmd+=("$user")
+
+if (( restore )); then
+    if rsync -av --progress=info2 /mnt/array/home-retired/"$user" /home/"$user"; then
+        ask_ok "User $user's files successfully restored, remove backup at /mnt/array/home-retired/$user?" && \
+            rm -rf /mnt/array/home-retired/"$user"
+    fi
+fi
+
+# echo "Running: ${useradd_cmd[*]}"
+"${useradd_cmd[@]}"
+echo "$user":"$password" | chpasswd
+
+
+if (( samba )); then
+    (echo "$password"; echo "$password") | smbpasswd -a -s "$user"
+fi
+
+ask_ok "Prompt user to reset password on next login?" &&
+passwd --expire "$user" &&
+echo "NOTE: The file sharing (smbpasswd) will not be changed"
+
+# TODO check if centos 9 does by default
+# Add subuids & subgids for container namespace
+# id_offset=100000
+# id_num=65536
+# last_uid=$(tail -1 /etc/subuid | cut -d':' -f2)
+# last_gid=$(tail -1 /etc/subgid | cut -d':' -f2)
+# start_uid=$(( last_uid + id_offset ))
+# start_gid=$(( last_gid + id_offset ))
+# echo "$user:$start_uid:$id_num" >> /etc/subuid
+# echo "$user:$start_gid:$id_num" >> /etc/subgid
+
+# Copy manual to user desktop
+desktop="/home/$user/Desktop"
+[[ -d $desktop ]] || sudo -u "$user" mkdir -p "$desktop"
+copy_manual "$desktop"
+
+exit 0
--- a/server-scripts/script-user-remove
+++ b/server-scripts/script-user-remove
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Remove a user from the server
+# Copyright 2021-2023 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+echo "This script supports one optional argument, a username"
+
+if [[ $# -eq 1 ]]; then
+    user="$1"
+else
+    prompt user
+fi
+
+if ! id -u "$user" &>/dev/null; then
+    echo "User $user does not exist"
+    exit 1
+fi
+
+ask_ok "Remove user $user?" || exit 1
+
+killall -u "$user"
+
+if ask_ok "Backup /home/$user?" && [[ -d /home/$user ]]; then
+    if [[ ! -d /mnt/array/home-retired/$user ]]; then 
+        btrfs subvolume create /mnt/array/home-retired/"$user" || exit 1
+    fi
+    rsync -av /home/"$user"/ /mnt/array/home-retired/"$user" || exit 1
+fi
+
+smbpasswd -x "$user"
+
+sed -i "/$user/d" /etc/subuid
+sed -i "/$user/d" /etc/subgid
+
+if ! userdel -fr "$user"; then
+  ask_ok "Userdel failed, kill processes again and retry?" || exit 1
+  killall -u "$user" -s SIGKILL
+  userdel -fr "$user"
+fi
+
+exit $?
--- a/server-scripts/script-user-reset-desktop
+++ b/server-scripts/script-user-reset-desktop
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Reset default desktop preferences
+# Copyright Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+echo "This script will only work for the current user"
+
+[[ $EUID -eq 0 ]] && echo "Do not run as root (do not use sudo)" && exit 1
+
+to_reset=(
+  /org/mate/panel/
+  /org/mate/panel/objects/
+  /org/mate/desktop/background/
+  /org/mate/desktop/font-rendering/
+  /org/mate/desktop/interface/
+  /org/mate/desktop/screensaver/
+  /org/mate/desktop/media-handling/
+  /org/mate/desktop/screensaver/
+  /org/mate/mate-menu/
+  /org/mate/marco/general/
+  /org/mate/caja/desktop/
+  /org/mate/caja/preferences/
+  /org/mate/notification-daemon/
+)
+
+echo "Resetting desktop for user $(whoami)"
+for p in "${to_reset[@]}"; do
+  dconf reset -f "$p"
+done
+
+mate-panel --reset
--- a/server-scripts/script-user-reset-password
+++ b/server-scripts/script-user-reset-password
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# This script will reset a user password on the server
+# Copyright 2021-24 Bryan C. Roessler
+
+unset user password
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+echo "This script supports two optional arguments, a username and password"
+
+if [[ $# -eq 0 ]]; then
+    prompt user
+    prompt password
+elif [[ $# -eq 1 ]]; then
+    user="$1"
+    prompt password
+elif [[ $# -eq 2 ]]; then
+    user="$1"
+    password="$2"
+elif [[ $# -gt 2 ]]; then
+    echo "Too many arguments provided"
+    exit 1
+fi
+
+if ! id -u "$user" &>/dev/null; then
+    echo "User $user does not exist"
+    exit 1
+fi
+
+if ask_ok "Change user $user's password to $password?"; then
+    echo "$user":"$password" | chpasswd
+    (echo "$password"; echo "$password") | smbpasswd -a -s "$user"
+fi
+
+ask_ok "Prompt user to reset password on next login?" &&
+passwd --expire "$user" &&
+echo "NOTE: The file sharing (smbpasswd) will not be changed"
+   
+exit 0
--- a/server-scripts/script-user-reset-x2go
+++ b/server-scripts/script-user-reset-x2go
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# This script will reset x2go sessions to a working state
+# Use --all to reset all user X2Go sessions
+# Copyright Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+echo "This script supports one optional argument, a username (or --all for all users)"
+
+USER_MODE=0
+USER_LIST=()
+if [[ $EUID -gt 0 ]]; then
+    echo "This script should normally be run as root (with sudo)"
+    echo "Attempting to run in user mode"
+    USER_MODE=1
+    USER_LIST+=($(whoami))
+else
+    if [[ $# -eq 1 ]]; then
+        if [[ $1 == '--all' ]]; then
+            for i in /home/*; do
+                USER_LIST+=("${i##*/}")
+            done
+        else
+            USER_LIST+=("$1")
+        fi
+    else
+        prompt user
+        USER_LIST+=("$user")
+        unset user
+    fi
+fi
+
+for user in "${USER_LIST[@]}"; do
+    # Clean local user cache
+    shopt -s nullglob
+    caches=(/home/"$user"/.x2go/C-"$user"-* /home/"$user"/.xsession-x2go-*)
+    shopt -u nullglob
+    if [[ ${#caches} -gt 0 ]]; then
+        ask_ok "Remove X2Go cache files for user $user?" &&
+        rm -rf /home/"$user"/.x2go/C-"$user"-* &&
+        echo "Removed: ${caches[*]} for user $user"
+    fi
+    # Clean X2Go sessions
+    if (( USER_MODE )); then
+        mapfile -t sessions < <(x2golistsessions | grep "$user"| cut -f2 -d'|')
+    else
+        mapfile -t sessions < <(x2golistsessions_root | grep "$user"| cut -f2 -d'|')
+    fi
+    if [[ ${#sessions} -gt 0 ]]; then
+        ask_ok "Terminate X2Go sessions for user $user?" &&
+        for session in "${sessions[@]}"; do
+            x2goterminate-session "$session" &&
+            echo "Terminated: $session for user $user"
+        done
+    fi
+done
--- a/server-scripts/script-user-unban
+++ b/server-scripts/script-user-unban
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Unbans a fail2ban IP
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+echo "This script supports one optional argument, an IP address"
+
+if [[ $# -eq 1 ]]; then
+    ip_address="$1"
+else
+    prompt ip_address
+fi
+
+if fail2ban-client set sshd unbanip "$ip_address"; then
+  echo "IP address $ip_address unbanned"
+fi
+
+exit $?
--- a/server-scripts/smartd-notify-all
+++ b/server-scripts/smartd-notify-all
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Notify all users on S.M.A.R.T errors
+# Place in /usr/share/smartmontools/smartd_warning.d/ or use "DEVICESCAN -m @smartd-notify-all" in /etc/smartd.conf
+# Copyright 2021 Bryan C. Roessler
+
+parent="${BASH_SOURCE[0]}"
+parent=${parent%/*}
+
+[[ -f $parent/script-functions ]] && . "$parent"/script-functions || exit 1
+
+is_root
+
+IFS=$'\n'
+for LINE in $(w -hs); do
+    USER=$(echo "$LINE" | awk '{print $1}')
+    USER_ID=$(id -u "$USER")
+    DISP_ID=$(echo "$LINE" | awk '{print $8}')
+    sudo su "$USER" DISPLAY="$DISP_ID" DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/"$USER_ID"/bus notify-send "S.M.A.R.T Error ($SMARTD_FAILTYPE) $SMARTD_MESSAGE" --icon=dialog-warning
+done