server-scripts/qhtcp-workflow/qhtcp-workflow

#!/usr/bin/env bash
# Copyright 2024 Bryan C. Roessler
#
# Allow indirect functions
# shellcheck disable=SC2317
#
# @name Hartman Lab QHTCP Workflow
# @brief An opinionated yet flexible QHTCP analysis framework for the Hartman Lab.
# 
# @description 
#
# See the [User Input](#user-input) section for getting started.
#
# Insert a general description of Q-HTCP and the Q-HTCP process here.

shopt -s extglob # Turn on extended globbing
shopt -s nullglob # Allow null globs
DEBUG=${DEBUG:-1} # Turn debugging ON by default during development

# @description Use `--help` to print the help message.
# @internal
print_help() {
  debug "Running: ${FUNCNAME[0]}"

  install_dependencies --get-depends # Loads the dependency arrays

  cat <<-EOF
		USAGE:
		  qhtcp-workflow [[OPTION] [VALUE]]...

		  --project, --module, --wrapper, and --nomodule can be passed multiple
		   times or as comma-separated strings (see EXAMPLES below)

		OPTIONS:
		  --project, -p PROJECT[,PROJECT...]
		    PROJECT should follow the pattern ${PROJECT_PREFIX}_PROJECT
		  --module, -m MODULE[,MODULE...]
		    See MODULES section below for list of available modules
		    If no --include is specified, all modules are run
		  --wrapper, -w WRAPPER "[ARG1],[ARG2]..." (string of comma delimited arguments)
		    See WRAPPERS section below for list of available modules
		    See documentation for wrapper argument usage
		  --nomodule, -n MODULE[,MODULE...]
		    See MODULES and WRAPPERS section below for list of modules to exclude
		  --markdown
		    Generate the shdoc markdown README.md file for this program
		  --yes, -y, --auto
		    Always answer yes to questions (non-interactive mode)
		  --debug, -d
		    Print extra debugging info
		  --help, -h
		    Print this help message and exit

		MODULES:
		  ${ALL_MODULES[*]}

		WRAPPERS:
		  ${ALL_WRAPPERS[*]}

		EXAMPLES:
		  qhtcp-workflow --module ${ALL_MODULES[0]} --module ${ALL_MODULES[1]}
		  qhtcp-workflow --project ${PROJECT_PREFIX}_MY_PROJECT --module ${ALL_MODULES[0]} --module ${ALL_MODULES[1]}
		  qhtcp-workflow --project ${PROJECT_PREFIX}_MY_PROJECT --module ${ALL_MODULES[1]} --module ${ALL_MODULES[2]} --yes
		  qhtcp-workflow --module=${ALL_MODULES[0]},${ALL_MODULES[1]}
		  qhtcp-workflow --project=${PROJECT_PREFIX}_MY_PROJECT,${PROJECT_PREFIX}_MY_OTHER_PROJECT
		  qhtcp-workflow --project=${PROJECT_PREFIX}_MY_PROJECT,${PROJECT_PREFIX}_MY_OTHER_PROJECT --module=${ALL_MODULES[1]},${ALL_MODULES[2]} --yes --debug
		  qhtcp-workflow --project=${PROJECT_PREFIX}_MY_PROJECT --wrapper ${ALL_WRAPPERS[2]} \"/path/to/genefile.txt,/path/to/output/dir\" --wrapper ${ALL_WRAPPERS[3]} \"/path/to/sgofile\"
	EOF
}

		# DEPENDENCIES:
		#   deb: ${depends_deb[@]}
		#   rpm: ${depends_rpm[@]}
		#   brew: ${depends_brew[@]}
		#   perl: ${depends_perl[@]}
		#   R: ${depends_r[@]}
		#   BiocManager: ${depends_bioc[@]}


# @section Notes
# @description
#
# ### TODO
#
# * Variable scoping is horrible right now
# * I wrote this sequentially and tried to keep track the best I could
# * Local vars have a higher likelihood of being lower case, global vars are UPPER
# * See MODULE specific TODOs below
#
# ### General guidelines for writing external scripts
#
# * External scripts must be modular enough to handle input and output from multiple directories
# * Don't cd in scripts (if you must, do it in a subshell!)
# * Pass variables
# * Pass options
# * Pass arguments
# 
# ## Project layout
#
# &nbsp;&nbsp;**qhtcp-workflow/**
#
# &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;**scans/**
#
# * This directory contains raw image data and image analysis results for the entire collection of Q-HTCP experiments.
# * Subdirectories within "scans" should represent a single Q-HTCP study and be named using the following convention: yyymmdd_username_experimentDescription
# * Each subdirectory contains the Raw Image Folders for that study.
# * Each Raw Image Folder contains a series of N folders with successive integer labels 1 to N, each folder containing the time series of images for a single cell array.
# * It also contains a user-supplied subfolder, which must be named "MasterPlateFiles" and must contain two excel files, one named 'DrugMedia_experimentDescription' and the other named  'MasterPlate_experimentDescription'.
# * If the standard MasterPlate_Template file is being used, it's not needed to customize then name. 
# * If the template is modified, it is recommended to rename it and describe accordingly - a useful convention is to use the same experimentDescription for the MP files as given to the experiment
# * The 'MasterPlate_' file contain associated cell array information (culture IDs for all of the cell arrays in the experiment) while the 'DrugMedia_' file contains information about the media that the cell array is printed to. 
# * Together they encapsulate and define the experimental design. 
# * The QHTCPImageFolders and 'MasterPlateFiles' folder are the inputs for image analysis with EASY software. 
# * As further described below, EASY will automatically generate a 'Results' directory (within the ExpJobs/'ExperimentJob' folder) with a name that consists of a system-generated timestamp and an optional short description provided by the user (Fig.2). The 'Results' directory is created and entered, using the "File >> New Experiment" dropdown in EASY. Multiple 'Results' files may be created (and uniquely named) within an 'ExperimentJob' folder.
#
# &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;**apps/easy/**
#
# * This directory contains the GUI-enabled MATLAB software to accomplish image analysis and growth curve fitting. 
# * EASY analyzes Q-HTCP image data within an 'ExperimentJob'' folder (described above; each cell array has its own folder containing its entire time series of images). 
# * EASY analysis produces image quantification data and growth curve fitting results for each cell array; these results are subsequently assembled into a single file and labeled, using information contained in the 'MasterPlate_' and 'DrugMedia_' files in the 'MasterPlateFiles' subdirectory. 
# * The final files (named '!!ResultsStd_.txt' or '!!ResultsELr_.txt') are produced in a subdirectory that EASY creates within the 'ExpJob#' folder, named '/ResultsTimeStampDesc/PrintResults' (Fig. 2). 
# * The /EASY directory is simply where the latest EASY version resides (additional versions in development or legacy versions may also be stored there). 
# * The raw data inputs and result outputs for EASY are kept in the 'ExpJobs' directory. 
# * EASY also outputs a '.mat' file that is stored in the 'matResults' folder and is named with the TimeStamp and user-provided name appended to the 'Results' folder name when 'New Experiment' is executed from the 'File' Dropdown menu in the EASY console.
#
# &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;**apps/ezview/**
#
# * This directory contains the GUI-enabled MATLAB software to conveniently and efficiently mine the raw cell array image data for a Q-HTCP experiment.
# * It takes the Results.m file (created by EASY software) as an input and permits the user to navigate through the raw image data and growth curve results for the experiment.
# * The /EZview provides a place for storing the the latest EZview version (as well as other EZview versions).
# * The /EZview provides a GUI for examining the EASY results as provided in the …/matResults/… .mat file.
#   
# &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;**Master Plates**
#
# * This optional folder is a convenient place to store copies of the 'MasterPlate_' and a 'DrugMedia_' file templates, along with previously used files that may have been modified and could be reused or further modified to enable future analyses.
# * These two file types are required in the 'MasterPlateFiles' folder, which catalogs experimental information specific to individual Jobs in the ExpJobs folder, as described further below. 
#
#
#
# @description
# `--project`, `--module`, `--nomodule`, and `--wrapper` can be passed multiple times or with a comma-separated string
# @option -p<value> | --project=<value> One or more projects to analyze, can be passed multiple times or with a comma-separated string
# @option -m<value> | --module=<value> One or more modules to run (default: all), can be passed multiple times or with a comma-separated string
# @option -w<value> | --wrapper=<value> One or more wrappers and its arguments to run, can be passed multiple times or with a comma-separated string
# @option -n<value> | --nomodule=<value> One or more modules (default: none) to exclude from the analysis
# @option --markdown Generate the shdoc markdown file for this program
# @option -y | --yes | --auto Assume yes answer to all questions (non-interactive mode)
# @option -d | --debug Turn on extra debugging output
# @option -h | --help Print help message and exit (overrides other options)
# @set PROJECTS array List of projects to cycle through
# @set MODULES array List of modules to run on each project
# @set WRAPPERS array List of wrappers and their arguments to run on each project
# @set EXCLUDE_MODULES array List of modules not to run on each project
# @set DEBUG int Turn debugging on
# @set YES int Turn assume yes on
parse_input() {
  debug "Running: ${FUNCNAME[0]} $*"

  long_opts="project:,module:,wrapper:,nomodule:,markdown,yes,auto,debug,help"
  short_opts="+p:m:w:n:ydh"

  if input=$(getopt -o $short_opts -l $long_opts -- "$@"); then
    eval set -- "$input"
    while true; do
      case $1 in
        --project|-p)
            shift
            declare -ga PROJECTS
            IFS=',' read -ra PROJECTS <<< "$1"
            ;;
        --module|-m)
            shift
            declare -ga MODULES
            IFS=',' read -ra MODULES <<< "$1"
            ;;
        --wrapper|-w)
            shift
            declare -ga WRAPPERS
            IFS=',' read -ra WRAPPERS <<< "$1"
            ;;
        --nomodule|-n)
            shift
            declare -ga EXCLUDE_MODULES
            IFS=',' read -ra EXCLUDE_MODULES <<< "$1"
            ;;
        --markdown)
            generate_markdown; exit 0 # TODO disable the exit after development
            ;;
        --yes|-y|--auto)
            declare -g YES=1
            ;;
        --debug|-d)
            declare -g DEBUG=1
            ;;
        --help|-h)
            print_help
            exit 0
            ;;
        --)
            shift
            break
            ;;
      esac
      shift
    done
  else
    err "Incorrect options provided"; exit 1
  fi
}

# @section Modules
# @description 
#
# A module contains a cohesive set of tasks, including:
#
# * Filesystem operations
# * Variable setting
# * Executing other modules
# * Executing wrappers
#
# Use a module to:
#
# * Build a new type of analysis from scratch
# * Generate project directories
# * Group multiple wrappers (and modules) into a larger task
# * Dictate the ordering of multiple wrappers
# * Call wrappers with the appropriate arguments
#
# @description
module() { 
  debug "Adding $1 module"
  ALL_MODULES+=("$1")
  declare -gA "$1"
}

# @description Ask the user a yes/no question
# @arg $1 string The question to ask
# @exitcode 0 If yes
# @exitcode 1 If no
# @internal
ask() {
    declare response
    (( YES )) && return 0
    read -r -p "$* [y/N]: " response
    [[ ${response,,} =~ ^(yes|y)$ ]]
}
err() { echo "Error: $*" >&2; }

# @description Ask the user to input a name
# @arg $1 string formatted name
# @arg $2 string variable name
# @exitcode 0 If set
# @exitcode 1 If not
# @internal
ask_name() {
  debug "Running: ${FUNCNAME[0]} $*"
  declare -a to_add
  declare example_pn
  declare -g -a "$2"
  declare -n ref="$2"
  example_pn="${STUDY_PREFIX}_$(random_three_words)"
  cat <<-EOF
		${underline}Enter a new or existing $1${nounderline}
		* Suggested prefix: ${STUDY_PREFIX}_
		* You may choose any combination of words/characters following the prefix, but be sensible.
		* Make it descriptive and avoid spaces and special characters.
		* Example: $example_pn
	EOF
  trys=3 # give the user up to 3 tries to enter a valid name
  for ((i=1; i<=trys; i++)); do 
    read -r -e -p "Enter a new or existing $1: " -i "${STUDY_PREFIX}_" response
    if [[ -z $response ]]; then
      to_add+=("$example_pn")
      break
    else
      if sanitize_pn "$response"; then
        to_add+=("$response")
        echo "$response successfully added as a $1"
        i=0 # resetting trys counter in case user wants to add more than 3 names
      else
        err "Invalid project name: $response"
        echo "Retrying ($i of $trys)"
      fi
    fi
  done
  # shellcheck disable=SC2034
  ref=("${to_add[@]}")
}

# @description Sanitizer regex for prefix
sanitize_pn() {
  [[ $1 =~ ^[0-9]{8}_.+_.+$ ]]
}

# @description handle debug output globally
debug() { (( DEBUG )) && echo "Debug: $*"; }

# @description Create a random three word string
# Not super portable but nice to have
random_three_words() {
  local -a arr

  adjectives=(
    "adorable" "adventurous" "agile" "amazing" "angry" "beautiful" "bold" "brave" "bright" "calm"
    "charming" "cheerful" "courageous" "creative" "delicate" "elegant" "energetic" "exciting" "fast" "friendly"
    "gentle" "happy" "healthy" "helpful" "honest" "humble" "intelligent" "jovial" "kind" "lively"
    "lovable" "magnificent" "mellow" "modest" "noble" "outgoing" "passionate" "peaceful" "powerful" "quick"
    "radiant" "reliable" "resourceful" "respectful" "shy" "smart" "strong" "sweet" "tender" "thoughtful"
    "timid" "unique" "upbeat" "vibrant" "warm" "wise" "wonderful" "youthful" "zealous" "eager"
    "friendly" "generous" "imaginative" "independent" "inspired" "joyful" "luminous" "mysterious" "playful" "serene"
    "spontaneous" "steady" "spirited" "stylish" "tough" "understanding" "vivid" "zany" "bold" "calm"
    "dynamic" "innovative" "proud" "reliable" "sincere" "strong" "talented" "trustworthy" "vivid" "zealous"
  )

  participles=(
    "abandoning" "absorbing" "accelerating" "achieving" "acquiring" "admiring" "advising" "agreeing" 
    "allowing" "analyzing" "appearing" "applying" "arguing" "assembling" "assisting" "attracting" 
    "believing" "browsing" "calculating" "calling" "caring" "celebrating" "cleaning" "climbing" 
    "coaching" "collecting" "combining" "communicating" "competing" "confessing" "considering" 
    "cooking" "correcting" "creating" "debating" "defining" "delivering" "designing" "discussing" 
    "driving" "enjoying" "exploring" "feeling" "finishing" "fixing" "forming" "gathering" "growing" 
    "guiding" "happening" "helping" "hoping" "improving" "increasing" "influencing" "involving" 
    "learning" "leading" "looking" "managing" "measuring" "moving" "noticing" "observing" "offering" 
    "organizing" "performing" "preparing" "presenting" "producing" "protecting" "questioning" 
    "recommending" "recovering" "running" "saving" "searching" "seeing" "sharing" "solving" 
    "starting" "studying" "succeeding" "supporting" "teaching" "thinking" "understanding" "using" 
    "validating" "waiting" "working" "writing"
  )

  animals=(
    "antelope" "baboon" "badger" "bat" "bear" "beaver" "bison" "booby" "buffalo" "bull" 
    "camel" "cat" "cheetah" "chicken" "chimpanzee" "clam" "cobra" "cougar" "cow" "crab" 
    "crane" "crocodile" "crow" "deer" "dog" "dolphin" "dove" "duck" "eagle" "echidna" 
    "eel" "elephant" "emu" "falcon" "ferret" "fish" "flamingo" "fox" "frog" "gazelle" 
    "giraffe" "goat" "goose" "gorilla" "hare" "hawk" "hedgehog" "hippo" "horse" "hyena" 
    "iguana" "impala" "jaguar" "kangaroo" "koala" "lion" "llama" "lobster" "lynx" "macaw" 
    "manatee" "mole" "monkey" "moose" "mouse" "mule" "octopus" "okapi" "opossum" "ostrich" 
    "otter" "owl" "panda" "panther" "parrot" "penguin" "pig" "platypus" "porcupine" "quail" 
    "rabbit" "rat" "raven" "reindeer" "rhinoceros" "robin" "salmon" "seal" "shark" "sheep" 
    "shrimp" "skunk" "sloth" "snail" "snake" "sparrow" "spider" "squid" "squirrel" "starling" 
    "stingray" "swan" "tapir" "tiger" "toad" "toucan" "turtle" "vulture" "walrus" "wolverine" 
    "wolf" "wombat" "zebra"
  )

  arr+=(
    "$(shuf -n1 -e "${adjectives[@]}")"
    "$(shuf -n1 -e "${participles[@]}")"
    "$(shuf -n1 -e "${animals[@]}")"
  )

  printf "%s_" "${arr[@]}" | sed 's/_$//'
}

# # @description Function to set a global array variable
# # @arg $1 The name of the global array 
# # @arg $2 The values of the array
# # @internal
# set_global_array() {
#   debug "Running: ${FUNCNAME[0]} $*"
#   declare -n array="$1"
#   shift
#   # Declare the global array using the nameref
#   declare -g -a "${!arr}"
#   # Populate the array
#   arr=("$@")
# }


# @description Print an array in columns
# @arg $1 array to print
print_in_columns() {
  debug "Running: ${FUNCNAME[0]} $*"
  declare -n array="$1"
  declare num=${#array[@]}

  if [[ $num -gt 8 ]]; then
    # Calculate the number of elements in each column
    local num_columns=$(( (num + 1) / 2 ))

    # Determine the maximum width of the first column
    local max_width=0
    for ((i=0; i<num_columns; i++)); do
      # Check if the item exists
      if [[ -n "${array[i]}" ]]; then
        local item_length=${#array[i]}
        if [[ $item_length -gt $max_width ]]; then
          max_width=$item_length
        fi
      fi
    done

    # Print in two columns
    for ((i=0; i<num_columns; i++)); do
      # Print the first column
      if [[ -n "${array[i]}" ]]; then
        printf "%d. %-${max_width}s" $((i+1)) "${array[i]}"
      else
        printf "%${max_width}s" ""
      fi
      
      # Print the second column if it exists
      if [[ $((i + num_columns)) -lt $num ]]; then
        printf "\t%d. %s\n" $((i + num_columns + 1)) "${array[i + num_columns]}"
      else
        # Print a newline if no second column item
        echo
      fi
    done
  else
    # Print in a single column
    for ((i=0; i<num; i++)); do
      printf "%d. %s\n" $((i+1)) "${array[i]}"
    done
  fi
}

# @description More concise debugging
# @arg $1 array command(s) to run
# @exitcode 0 command successful
# @exitcode 1 command not successful
# @internal
execute() {
  if debug "$*"; then
    "$@"
  else
    "$@" &>/dev/null
  fi
}

is_directory_empty() {
  local dir="$1"

  # Check if the directory exists and is a directory
  if [ ! -d "$dir" ]; then
    echo "Directory does not exist or is not a directory"
    return 1
  fi

  # Iterate over the files and directories inside the specified directory
  for _ in "$dir"/*; do
    # If we find at least one entry, it's not empty
    return 1
  done

  # If the loop completes without finding any entries, the directory is empty
  return 0
}

# @description Backup one or more files to an incremented .bk file
#
# **TODO**
#
# * Make backups hidden by prepending "."?
#
# @exitcode backup iterator max 255
# @internal
backup() {
  debug "Running: ${FUNCNAME[0]} $*"
  for f in "$@"; do
    [[ -e $f ]] || continue
    declare count=1
    while [[ -e $f.bk.$count ]]; do
      ((count++))
    done
    execute mv "$f" "$f.bk.$count"
  done
}

# @description Prints a helpful message add program start
#
# @internal
interactive_header() {
  debug "Running: ${FUNCNAME[0]}"

  cat <<-EOF
		 _    _            _                           _           _     
		| |  | |          | |                         | |         | |    
		| |__| | __ _ _ __| |_ _ __ ___   __ _ _ __   | |     __ _| |__  
		|  __  |/ _  |  __| __|  _   _ \ / _  |  _ \  | |    / _  |  _ \ 
		| |  | | (_| | |  | |_| | | | | | (_| | | | | | |___| (_| | |_) |
		|_|  |_|\__,_|_|   \__|_| |_| |_|\__,_|_| |_| |______\__,_|_.__/ 
		                  ___  _   _ _____ ____ ____  
		                 / _ \| | | |_   _/ ___|  _ \ 
		                | | | | |_| | | || |   | |_) |                    
		                | |_| |  _  | | || |___|  __/ 
		                 \__\_|_| |_| |_| \____|_|          

	Scans directory: $SCANS_DIR
	Output directory: $OUT_DIR
	Change the SCANS_DIR or OUT_DIR environment variable(s) to override
	Example: SCANS_DIR=/path/to/scans OUT_DIR=/path/to/out ./qhtcp-workflow $*

	EOF

  underline=$(tput smul)
  nounderline=$(tput rmul)

  echo "${underline}Modules${nounderline}"
  print_in_columns "ALL_MODULES"
  echo ""

  # Print wrappers
  echo "${underline}Wrappers${nounderline}"
  print_in_columns "ALL_WRAPPERS"
  echo ""

  # Print projects from the scans directory
  projects=("$SCANS_DIR"/*/)
  
  if [[ ${#projects[@]} -eq 0 ]]; then
    echo "No projects found in $SCANS_DIR"
    ask_name "project" "ADD_PROJECTS" && PROJECTS+=("${ADD_PROJECTS[@]}")
  else
    echo "${underline}Projects${nounderline}"
    projects=("${projects[@]%/}") # strip comma first!
    projects=("${projects[@]##*/}")
    print_in_columns "projects"
  fi
  echo ""

  # Module selection
  if [[ ${#MODULES[@]} -eq 0 && ${#EXCLUDE_MODULES[@]} -eq 0 && ${#WRAPPERS[@]} -eq 0 ]]; then
    cat <<-EOF
			${underline}Enter modules(s) to run${nounderline}
			* <Enter> for all
			* A comma-separated list of module numbers: 2,5,12
			* 0 for none (wrappers only)
		EOF
    ((YES)) || read -r -p "(all): " response
    echo ""
    if [[ -z $response ]]; then
      MODULES=("${ALL_MODULES[@]}")
    elif [[ $response -eq 0 ]]; then
      EXCLUDE_MODULES=("${ALL_MODULES[@]}")
    else
      IFS=',' read -ra arr <<< "$response"
      for i in "${arr[@]}"; do
        if [[ $i =~ ^[0-9]+$ ]]; then
          MODULES+=("${ALL_MODULES[$((i-1))]}")
        else
          err "Module number $i is invalid, skipping"
        fi
      done
    fi
    unset response arr i
  fi

  # If we're just installing dependencies, skip the rest
  [[ ${MODULES[*]} == "install_dependencies" ]] && return 0

  # Wrapper selection
  if [[ ${#MODULES[@]} -eq 0 && ${#EXCLUDE_MODULES[@]} -eq 0 && ${#WRAPPERS[@]} -eq 0 ]]; then
    while :; do
      cat <<-EOF
				${underline}Enter wrapper(s) to run followed by its arguments in a comma-separated string${nounderline}
				* <Enter> for none (break)
				* A comma-separated list of wrappers and their arguments
				* Quote the argument string if it contains whitespace
				* Example: \""${ALL_WRAPPERS[0]},arg1,arg2,arg3...\"
			EOF
      ((YES)) || read -r -p "(none): " response
      echo ""
      [[ -z $response ]] && break
      IFS=',' read -ra arr <<< "$response"
      WRAPPERS+=("${arr[@]}")
      unset response arr i
    done
  fi

  # Project selection
  if [[ ${#PROJECTS[@]} -eq 0 ]]; then
    num=${#projects[@]}
    if [[ $num -eq 0 ]]; then
      ask_name "project" "ADD_PROJECTS" && PROJECTS+=("${ADD_PROJECTS[@]}")
    else
      for ((c=1; c<2; c++)); do # safe loop that only runs once
        cat <<-EOF
					${underline}Enter project number(s) to analyze${nounderline}
					* <Enter> for the latest project ($num)
					* A comma-separated list of project numbers: 2,5,12
					* 0 to add a new project
				EOF
        ((YES)) || read -r -p "($num): " response
        echo
        if [[ $response == 0 ]]; then 
          ask_name "project" "ADD_PROJECTS" && PROJECTS+=("${ADD_PROJECTS[@]}")
        else
          response="${response:-$num}"
          IFS=',' read -ra arr <<< "$response"
          for i in "${arr[@]}"; do
            if [[ -n ${projects[$((i-1))]} ]]; then
              PROJECTS+=("${projects[$((i-1))]}")
            else
              err "Project number $i is invalid"
            fi
          done
        fi  
        [[ ${#PROJECTS[@]} -eq 0 ]] && c=0 # repeat the loop
      done
    fi
    unset response arr i
  fi

  # Sanitize project names
  for i in "${!PROJECTS[@]}"; do
    if ! sanitize_pn "${PROJECTS[i]}"; then
      echo "Project name ${PROJECTS[i]} is invalid"
      ask_name "project" "ADD_PROJECTS" && unset "PROJECTS[i]" && PROJECTS+=("${ADD_PROJECTS[@]}")
    fi
  done
}


module install_dependencies
# @description This module will automatically install the dependencies for running QHTCP.
#
install_dependencies() {
  debug "Running: ${FUNCNAME[0]} $*"

  # Dependency arrays
  depends_rpm=(
    graphviz pandoc pdftk-java gd-devel perl-CPAN shdoc nano
    rsync coreutils libcurl-devel openssl-devel harfbuzz-devel
    fribidi-devel R-core R-core-devel java)
  depends_deb=(
    graphviz pandoc pdftk-java libgd-dev perl shdoc nano rsync
    coreutils libcurl-dev openssl-dev libharfbuzz-dev libfribidi-dev
    r-base r-base-dev default-jre)
  depends_brew=(
    graphiz pandoc gd pdftk-java shdoc nano perl rsync coreutils
    harfbuzz fribidi r java)
  depends_perl=(
    Test::Warnings Test::Fatal File::Map Sub::Uplevel ExtUtils::Config
    ExtUtils::PkgConfig IPC::Run Module::Build::Tiny GD GO::TermFinder)
  depends_r=(
    BiocManager ontologyIndex ggrepel tidyverse sos openxlsx ggplot2 
    dplyr rlang data.table unix gridExtra gplots stringr plotly ggthemes pandoc
    rmarkdown htmlwidgets gdata Hmisc future furrr)
  depends_bioc=(UCSC.utils org.Sc.sgd.db)

  [[ $1 == "--get-depends" ]] && return 0 # if we just want to read the depends vars

  # Install system-wide dependencies
  echo "Installing system dependencies"
  echo "You may be prompted for your sudo password to install packages using your system package manager"
  echo "If you do not have sudo access, you may want to use toolbox"
  case "$(uname -s)" in
    Linux*|CYGWIN*|MINGW*)
      if command -v dnf &>/dev/null; then
        ask "Detected Linux RPM platform, continue?" || return 1
        sudo dnf install "${depends_rpm[@]}"
      elif command -v apt &>/dev/null; then
        ask "Detected Linux DEB platform, continue?" || return 1
        sudo apt install "${depends_deb[@]}"
      else
        echo "Sorry, your Linux platform is not supported for automatic dependency installation"
        echo "You will need to resolve dependencies manually"
      fi
      ;;
    Darwin*)
      ask "Detected Mac platform, continue?" || return 1
      export HOMEBREW_BREW_GIT_REMOTE="https://github.com/Homebrew/brew"
      curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh|bash
      brew install "${depends_brew[@]}"
      ;;
    *)          
      echo "Your system could not be detected, please install dependencies manually"
      ;;
  esac

  # Install perl CPAN modules
  echo "Installing perl CPAN modules"
  echo "It is recommended to use the local::lib perl library if prompted"
  debug "cpan -I -i ${depends_perl[*]}"
  cpan -I -i "${depends_perl[@]}"

  # Install R packages
  echo "Installing R packages"

  # Make R library directory if it doesn't exist
  [[ -d "$R_LIBS_USER" ]] || execute mkdir -p "$R_LIBS_USER"

  depends_r_str=""
  depends_r_to_string() {
    for d in "${depends_r[@]}"; do
      depends_r_str+="$d\", \""
    done
    depends_r_str="${depends_r_str::-3}" # strip last , " (comma and quote)
  }
  depends_r_to_string
  
  # Install R packages
  for d in "${depends_r[@]}"; do
  debug "$RSCRIPT -e \"if (!require(\"$d\", quietly = TRUE)) {install.packages(\"$d\", dep=TRUE, lib=\"$R_LIBS_USER\", repos=\"https://cloud.r-project.org\")}\""
    "$RSCRIPT" -e "if (!require(\"$d\", quietly = TRUE)) {install.packages(\"$d\", dep=TRUE, lib=\"$R_LIBS_USER\", repos=\"https://cloud.r-project.org\")}"
  done

  # Install Bioc packages
  for d in "${depends_bioc[@]}"; do
    debug "$RSCRIPT -e \"BiocManager::install(\"$d\", lib=\"$R_LIBS_USER\")\""
    "$RSCRIPT" -e "BiocManager::install(\"$d\", lib=\"$R_LIBS_USER\")"
  done

  # Ask to make our custom R library the default
  if [[ $R_LIBS_USER == "$HOME/R/$SCRIPT_NAME" ]]; then
    line="export R_LIBS_USER=$R_LIBS_USER"
    if ! grep -qF "$line" "$HOME/.bashrc"; then
      echo "This script uses a local R library at $HOME/R/$SCRIPT_NAME"
      echo "You can install the R dependencies to this library using the install_dependencies module"
    fi
    if ((YES)) || ask "Would you like to make this R library the default for your user?"; then
      echo "Adding $line to your .bashrc"
      echo "If you use a different shell, update your R_LIBS_USER environment variable accordingly"
      echo "$line" >> ~/.bashrc
    fi
  else
    debug "R_LIBS_USER is set to a custom path"
  fi
  echo ""

  command -v "$MATLAB" &>/dev/null || echo "You will also need MATLAB installed for GUI modules"
}


module init_project
# @description This function creates and initializes project directories
#
# This module:
#
# * Initializes a project directory in the scans directory
#
# **TODO** 
#
# * Copy over source image directories from robot
# * MasterPlate_ file **should not be an xlsx file**, no portability
# * We can keep the existing xlsx code for old style fallback
# * But moving forward should switch to csv or something open
# * Do we need to sync a QHTCP template?
# 
# **NOTES**
#
# * Copy over the images from the robot and then DO NOT TOUCH that directory except to copy from it
# * Write-protect (read-only) if we need to
# * Copy data from scans/images directory to the project working dir and then begin analysis
# * You may think...but doesn't that 2x data?
# * No, btrfs subvolume uses reflinks, only data that is altered will be duplicated
# * Most of the data are static images that are not written to, so the data is deduplicated
#
init_project() {
  debug "Running: ${FUNCNAME[0]}"

  # Create a project scans dir
  [[ -d $PROJECT_SCANS_DIR ]] || { execute mkdir -p "$PROJECT_SCANS_DIR" || return 1; }

  # TODO eventually copy scans from robot but for now let's pause and wait for transfer
  echo "In the future we will copy scans from robot here"
  # read -r -p "Hit <Enter> to continue: "
  
  # Create the project results out directory
  [[ -d $PROJECT_RESULTS_DIR ]] || execute mkdir -p "$PROJECT_RESULTS_DIR"

  # Write skeleton files in csv
  # If we have to convert to xlsx later, so be it
  echo "TODO: in the future, offer to create the DrugMedia file here"
  # cat <<-EOF > "$DRUG_MEDIA_FILE"
		
	# EOF
  
  echo "TODO: in the future, offer to create the MasterPlate file here"  

  # cat <<-EOF > "$MASTER_PLATE_FILE"
	
	# EOF
}


module easy
# @description 
# Run the EASY matlab program
#
# INPUT
#
# * MasterPlate_.xls
# * DrugMedia_.xls
#
# OUTPUT
#
# * out/PROJECT/easy/results_std.txt
# * out/PROJECT/easy/results_elr.txt
#
# TODO
#
# * Don't create output in the scans folder, put it in an output directory
# * The !!Results output files need standardized naming
# * The input MasterPlate and DrugMedia sheets need to be converted to something standard like csv/tsv
# * This would allow them to be created programmatically as well
#
# NOTES
#
# * I've modularized EASY to fit into this workflow but there may be things broken (especially in "stand-alone" mode)
# * The scans/images and 'MasterPlateFiles' folder are the inputs for image analysis with EASY software. 
# * EASY will automatically generate a 'Results' directory (within the ExpJobs/'ExperimentJob' folder) w/ timestamp and an optional short description provided by the user (Fig.2). 
# * The 'Results' directory is created and entered, using the "File >> New Experiment" dropdown in EASY. 
# * Multiple 'Results' files may be created (and uniquely named) within an 'ExperimentJob' folder.
#
# INSTRUCTIONS
#
# * This program should handle the relevant directory and file creation and load the correct project into EASY
#
# #### Pin-tool mapping
#
# * Select at least two images from your experiment (or another experiment) to place in a 'PTmapFiles' folder. 
# * Sometimes an experiment doesn't have a complete set of quality spots for producing a pin tool map that will be used to start the spot search process.  
# * In this case the folder for Master Plate 1 (MP 1) is almost good but has a slight problem.  
# * At P13 the spot is extended. We might use this one but would like to include others that are more centered if possible.  
# * The other plates with higher drug concentrations could be used, but since a recent experiment has a better reference plate image, we will add it to the set of images to produce the pin tool map. 
#
# ![Bad Pin Map](docs/imgs/easy/1-bad-pin-map.png "See the problem in Column 1, Row 13?")
#
# * We will find a good image from another experiment
#
# ![Good Pin Map](docs/imgs/easy/2-good-pin-map.png "Nice pin map")
#
# * We now have some images to generate a composite centered map for EASY to search and find the nucleation area of each spot as it forms. 
# * Click the Run menu tab. 
# * A drop down list of options is presented.  
# * Click the first item → [Plate Map Pintool ]. 
#
# ![Open PTmaps](docs/imgs/easy/3-open-ptmaps-dir.png "Where to find PTmaps")
# 
# * Open the PTmapFiles folder. 
# * Then click on the .bmp files you wish to include to make the pin tool map.  
# * Click the Open button.
#
# ![Open BMPs](docs/imgs/easy/4-select-bmps.png "Open BMPs")
# 
# * A warning dialog box may appear.  
# * This is nothing to be concerned about. 
# * Click OK and continue.
#
# ![Direct Map](docs/imgs/easy/5-direct-map.png "Map view, very pretty")
# 
# * 'Retry' takes you back so that to can select a different  .bmp files from which to create the map from.
# * In this case the spots from the two images are well aligned and give coverage to all the spots therefore we do not have to add new images.  
# * Remember, this map is just a good guess as to where to start looking for each spot not where it will focus to capture intensities.
# * Click 'Open' again.
#
# ![Open BMPs](docs/imgs/easy/4-select-bmps.png "Open BMPs")
#
# * We can now shift these values to get a better 'hard' start for this search.
# * Maybe we can move this search point to the left a bit by decreasing the 'Initial Col Position' slightly to 120 and clicking continue.
#
# ![Open BMPs](docs/imgs/easy/6-shift-values.png "Open BMPs")
#
# * Even though the first result image using 126 may have given a good map, we will use the improve second initiation point by clicking the 'Continue/Finish' button.
#
# ![Shifted map](docs/imgs/easy/7-shifted-map.png "See the dots are redder and centered")
#
# * Note that the red “hot” spot is now well centered in each composite image spot.
# * We can now click 'Continue / Finish' to proceed.
# * The coordinates and parameters will be stored in the results folder 'PTmats'.
# * This is where the .mat files which contain localization data for use in the next section of our work. 
# * The EASY GUI will come back.
# * Now click the 'Run' → 'Image Curve ComboAnalysi'.
# * This will perform the image quantification and then generate the curve fits for each plate selected.
# * Typically we pick only one plate at a time for one or two master plates.
# * The software will present the final image of the search if only 1 master plate is selected.
# * If multiple plates are selected, no search images will be presented or stored as figures.
# * However all the position data for every spot at every time point will be stored.
# * This large data trove is used by EZview to produce the image click-on hot spot maps and the photo strips. 
#
# ![Curve analysis](docs/imgs/easy/8-curve-analysis.png "Curve analysis")
#
# * Note the 'Select Files' dialog.
# * It allow the user to select the specific .bmp files to use.
# * This can be useful if there are bad ones that need to be removed from the folder due to contamination.
# * If all are good we can select them all and click 'Open' to run the process.
# * There are other parameters that can be selected.
# * For now we will continue and come back to those later.
#
# ![Select all BMPs](docs/imgs/easy/9-select-all-dialog.png "Select all BMPs")
#
# ![Contamination view](docs/imgs/easy/10-contamination-view.png "Contamination view")
#
# * The search focus of each spot at the end of the run is presented for examination
# * Notice that these have floated and locked in to a position determined on the fly to a point where the initial growth has exceed has reach a point of maturity.
# * This prevents a jump to a late onset jump to a contamination site.
# * If we found that we need to adjust our pin tool map or make other modifications, we can do that and rerun these single runs until we are satisfied.
#
# ![Search focus](docs/imgs/easy/11-search-focus.png "Search focus")
#
# * Next we will run the entire experiment by clicking on all the other master plates from the list box.
#
# ![Run experiment](docs/imgs/easy/12-run-experiment.png "Run experiment")
#
# * Depending on the number of master plates and the number of time point images taken for each, this next step can take a while.
# * Click continue and do something else while the computer works. 
# * When the work is complete the EASY GUI will reappear without the master plate list.
# * Now look in the /Results* /PrintResults folder to check that all the plates have run and produced data.
#
# ![Experiment complete](docs/imgs/easy/13-complete-experiment.png "Complete experiment")
#
# * This is a legacy print copy of data, but is still useful to check that all the quantification was completed successfully.
#
# ![Check results](docs/imgs/easy/14-check-results.png "Check results")
#
# #### Generate Reports
#
# * Generate a MPDM.mat file from the Excel master plate and drug media sheets that the user prepared as part of the experiment preparation.
# * These sheets must conform to certain format rules.
# * It is best when creating these to use a working copy as a template and replace the data with that of the current experiment.
# * See Master Plate and Drug-Media Plate topic for details.  
# * Click on the 'GenReports' menu tab and a drop down menu is presented the the first item 'DrugMediaMP Generate .mat'.
# * This will take you to the /MasterPlateFiles folder within the experiment currently being analyzed.
# * Do as the dialog box instructs. Select the Master Plate Excel file first. 
# * Important note: These files (both for master plates and drug-medias) must be generated or converted to the Excel 95 version to be read in Linux.
# * This can be done on either a Windows or an Apple machine running Excel. 
#
#
# ![Navigate to MasterPlateFiles](docs/imgs/easy/15-generate-mpdm-mat1.png "Navigate to MasterPlateFiles")
#
# ![Create a new MPDM.mat file](docs/imgs/easy/16-generate-mpdm-mat2.png "Create a new MPDM.mat file")
#
# ![Navigate to MasterPlateFiles](docs/imgs/easy/17-generate-mpdm-mat3.png "Navigate to MasterPlateFiles")
#
# ![Click OK](docs/imgs/easy/18-generate-mpdm-mat4.png "Click OK")
#
# * A message dialog pops  up.  
# * Click 'OK'.
#
# ![Navigate to MasterPlateFiles](docs/imgs/easy/19-generate-mpdm-mat5.png "Navigate to MasterPlateFiles")
#
# * Next click on the 'GenReports' menu tab and the second item in the drop down list 'ResultsDB Generate'.
# 
# ![Generate Reports](docs/imgs/easy/20-gen-reports.png "Generate Reports")
#
# * A dialog box with options appears.
# * The default is 'Both'.
# * 'Res' produces only a standard result sheet in the current experiments /Results*/PrintResults folder.
# * 'DB' produces only a special file for convenient upload to databases.
# * This file has no blank rows separating the plates and combines the raw data for each line item into a 'blob' as this is a convenient way to store data of variant lengths in a single database field. 
# * The concatenation of data for each row take a while.  But is useful for uploading data. 
# * Typically 'Both' is the preferred option, however,  if one needs to review the results quickly, this provides that option.  
#
# * We can open the !!Results MI 16_0919 yor1-1 copy.txt text file using Libre Open Office to review the results.
#
# ![Results file](docs/imgs/easy/21-results-file.png "Results file")
#
# * We can do likewise with the !!Dbase_MI 16_0919_yor1-2 copy.txt text file. 
#
# ![Db file](docs/imgs/easy/22-dbase-file.png "Db file")
#
# * Note that there are no headers or empty rows.  
# * Since Libre may corrupt the text files, it could be advisable to only read them and refrain from any 'Save' options presented.
#
# #### Master Plate and Drug Media Spreadsheets
#
# * The Master Plate and Drug- Media Spreadsheets correlate to the collected and calculated data with the defining definitions of the cultures, drugs and media involved in producing the experimental data.
# * These spreadsheets have a very specific format which was instigated at the beginning of our work.
# * To maintain compatibility over the years, we maintain that format.  
# * To begin with, our system can be used with Windows, Linux and Apple operating systems.
# * To accommodate these OS's, the Excel version must be an older Excel 95 version which is cross compatible for Matlab versions within all three major OS's.
# * Windows is more tolerant, but to avoid problems producing results reports, ALWAYS use the Excel 95 format for your spreadsheets. 
# * Do not remove any header rows. They can be modified with exception of the triple hash (###).
# * Do not change the number or order of the columns.
# * Next place a 'space' in unused empty spreadsheet entry positions.
# * This can cause problems in general for some software utilities.
# * It is just best to make this a standard practice.
# * Avoid using special characters.
# * Depending on the OS and software utility (especially database utilities), these can be problematic.
# * Certain 'date' associated entries such as Oct1 or OCT1 will be interpreted by Excel as a date and automatically formatted as such.
# * Do not use Oct1 (which is a yeast gene name) instead use Oct1_ or it's ORF name instead.  
# * When creating a Master Plate spreadsheet, it is best to start with a working spreadsheet template and adjust it to your descriptive data.
# * Be sure that ### mark is always present in the first column of the header for each plate.
# * This is important convention as it is used to defined a new plate set of entry data.
# * Each plate is expected to have 384 rows of data correlated with the 384 wells of the source master plates.
# * These have a particular order going through all 24 columns each row before proceeding to the next row.
# * Gene names and ORF name entries should be as short as possible (4-5 character max if possible) as these are used repeatedly as part of concatenated descriptors.
# * The 'Replicate' field and the 'Specifics' fields can be used for additional information.
# * The 'Replicate' field was originally designed to allow the user to sort replicates but it can be used for other relevant information.
# * The 'Specifics' field was created to handle special cases where the liquid media in which cultures were grown on a single source plate was selectively varied.
# * This gives the researcher a way to sort by growth media as well as gene or ORF name.
# * It can also be used to sort other properties instead of modifying the gene name field.
# * Thoughtful experiment design and layout are important for the successful analysis of the resultant data.
# * It is typically a good idea to create at least one reference full plate and make that plate the first source master plate.
# * Typically we give those reference cultures the 'Gene Name' RF1.
# * Traditionally we also made a second full reference plate with its cultures labeled RF2.
# * More recently some researchers have gone to dispersing RF1 control reference cultures throughout the source master plates series in addition to the first full source master plate.
# * The EZview software has been updated accordingly to find these references and perform associated calculations.
#
# ![Master Plate file](docs/imgs/easy/23-mp-file.png "Master Plate file")
#
# * There are a number of fields on the spreadsheet which in this case were left empty.
# * This spreadsheet format was created originally with studies of whole yeast genome SGA modifications incorporated.
# * Therefore all fields may not be relevant.
# * However, when ever relevant it is strongly advised to fill in all the appropriate data. 
# * The Drug-Media spreadsheet defines the perturbation components of each type of agar plate that the source master plates are printed on.
# * Again the format adherence is essential.
# * There is a '1' in the first column- second row (A2).
# * This has as legacy going back to early use.
# * It is still necessary and should not be deleted.
# * The header row must not be deleted.
# * A triple hash(###) must be placed in the cell below the last entry in the Drug field (Column 2).
# * Again insert a 'space' in each unused or empty cell in each field.
# * Again avoid special characters which may cause problems if not in the experiment quantification in subsequent analysis utilities.
# * A utility looking for a text field may end up reading a null and respond inappropriately.
# * As with the master plate Excel sheet, it is a good idea to use a working copy of an existing Drug-Media spreadsheet and adapt it to ones needs. 
#
# ![Drug Media file](docs/imgs/easy/24-dm-file.png "Drug Media file")
#
#
#
# To analyze a new Q-HTCP experiment:
#
#   * Open the EASY Software.
#     * Open 'EstartConsole.m' with MATLAB
#     * Click the Run icon (play button)
#     * When prompted, click "Change Folder" (do not select "Add to Path").
#     * In the pop-up display, select from the 'File' dropdown: 'New Experiment'. 
#       * From the pop-up, choose where to save the new file. 
#       * Navigate to the relevant job in the ExpJobs folder, name the file accordingly, and click 'save'.
#       * The newly created .mat file in the newly created Results folder will automatically be loaded. 
#       * The file name will then be automatically appended by the code with the current date information (e.g. 'A1.mat' will become 'Results2023-07-19A1)
#         * If the experiment has already been created, it can be reloaded by clicking 'Load Experiment' instead of 'New Experiment' and selecting the relevant results 
#   * In the pop-up display, click on the 'Run' dropdown menu and select 'Image CurveFit ComboAnalysis'.
#     * In the updated pop-up, choose/highlight all desired image folders for analysis (this is generally all of the folders, since only the ones that need analysis should be there) and then click on 'continue'. 
#     * As the program is running, updates will periodically appear in the Command Window; there will be an initial pause at "Before call to NIscanIntens…..". 
#     * When the curve fitting is finished, the EASY console will pop back up.
#     * Check to see the completed analysis results in the newly created 'PrintResults' Folder, inside of the 'Results' Folder.
#     * Other folders ('CFfigs', 'figs', 'Fotos') are created for later optional use and will be empty. 
#     * **NOTE:** The image analysis is completed independent of labeling the data (strains, media type, etc. Labeling happens next with the 'GenReports' function).
#   * Click on the 'GenReports' dropdown and select 'DrugMediaMP Generate .mat'
#     * **NOTE:** The 'MasterPlate' and 'DrugMedia' files have very specific formats and should be completed from a template. 
#       * The Masterplate file must be exact (it must contain all and only the strains that were actually tested). 
#       * For example, if only part of a library is tested, the complete library file must be modified to remove irrelevant strains.
#     * You will be prompted to first select the 'MasterPlate' file. You will need to navigate away from the working directory to get to it. 
#       * It is fine for the 'MasterPlate_' file to be .xlsx (or .xls), and if you don't see it in the popup window, then change the file type from '.xls' to "all files" and then select it. 
#       * Once it is selected, a report of the number of master plates in the file will pop up; when the report appears, assuming it is correct, click on 'OK'.
#     * You will then be prompted to select the 'DrugMedia' file from the relevant job folder. You will automatically return to the correct prior directory location. 
#       * Choose it and click 'OK'. You may see a warning about column headers being modified, but that's ok.
#       * This will create an additional file in the 'MasterPlatesFiles' folder named 'MPDMmat.mat'
#   * Click on the 'GenReports' dropdown and select 'Results_Generate.' 
#     * You will first see '!!ResultsElr_.txt' generated in the 'PrintResults' folder. 
#     * Refreshing will reveal an increasing file size until you see the '!!ResultsStd_.txt' being generated. 
#     * When finished, the '!!ResultsStd_.txt' will be about the same file size and it should be used in the following StudiesQHTCP analysis.  
#  * 'NoGrowth_.txt', and 'GrowthOnly_.txt' files will be generated in the 'PrintResults' folder. 
#
#
#
#
# Issues:
#    * We need full documentation for all of the current workflow. There are different documents that need to be integrated. This will need to be updated as we make improvements to the system.
#    * MasterPlate_ file must have ydl227c in orf column, or else it Z_interaction.R will fail, because it can't calculate shift values.
#    * Make sure there are no special characters; e.g., (), “, ', ?, etc.;  dash and underscore are ok as delimiters
#    * Drug_Media_ file must have letter character to be read as 'text'.
#    * MasterPlate_ file and DrugMedia_ are .xlsx or .xls, but !!Results_ is .txt.
#    * In Z_interactions.R, does it require a zero concentration/perturbation (should we use zero for the low conc, even if it's not zero), e.g., in order to do the shift correctly.
#    * Need to enable all file types (not only .xls) as the default for GenerateResults (to select MP and DM files as .xlsx).
#    * Explore differences between the ELR and STD files - 24_0414; John R modified Z script to format ELR file for Z_interactions.R analysis.
#    * To keep time stamps when transferring with FileZilla, go to the transfer drop down and turn it on, see https://filezillapro.com/docs/v3/advanced/preserve-timestamps/ 
#    * Could we change the 'MasterPlateFiles' folder label in EASY to 'MasterPlate_DrugMedia' (since there should be only one MP and there is also a DM file required?
#    * I was also thinking of adding a 'MasterPlateFilesOnly' folder to the QHTCP directory template where one could house different MPFiles (e.g., with and without damps, with and without Refs on all MPs, etc; other custom MPFiles, updated versions, etc)
#    * Currently updated files are in '23_1011_NewUpdatedMasterPlate_Files' on Mac (yeast strains/23_0914…/)
#    * For EASY to report cell array positions (plate_row_column) to facilitate analyzing plate artifacts. The MP File in Col 3 is called 'LibraryLocation' and is reported after 'Specifics' in the !!Results.
#    * Can EASY/StudiesQ-HTCP be updated at any time by rerunning with updated MP file (new information for gene, desc, etc)- or maybe better to always start with a new template?
#    * Need to be aware of file formatting to avoid dates (e.g., with gene names like MAY24, OCT1, etc, and with plate locations 1E1, 1E2, etc)- this has been less of a problem. 
#    * In StudiesQHTCP folders, remember to annotate Exp1, Exp2, in the StudyInfo.csv file.
#    * Where are gene names called from for labeling REMc heatmaps, TSHeatmaps, Z-interaction graphs, etc? Is this file in the QHTCP 'code' folder, or is it in the the results file (and thus ultimately the MP file)?
#    * Is it ok for a MasterPlate_ file to have multiple sheets (e.g., readme tab- is only the first tab read in)?
#    * What are the rules for pulling information from the MasterPlateFile to the !!Results_ (e.g., is it the column or the Header Name, etc that is searched? Particular cells in the DrugMedia file?).
#    * Modifier, Conc are from DM sheet, and refer to the agar media arrays. OrfRep is from MasterPlate_ File. 'Specifics' (Last Column) is experiment specific and accommodate designs involving differences across the multi-well liquid arrays. 'StrainBkGrd' (now 'Library location') is in the 3rd column and reported after 'Specifics' at the last col of the '!!Results..' file.
#    * Do we have / could we make an indicator- work in progress or idle/complete with MP/DM and after gen-report. Now, we can check for the MPDMmat.mat file, or we can look in PrintResults, but would be nice to know without looking there. 
#    * File>>Load Experiment wasn't working (no popup to redirect). Check this again.

easy() {
  debug "Running: ${FUNCNAME[0]}"
  cat <<-EOF
		To analyze a new Q-HTCP experiment:
		  * Open the EASY Software.
		  * Open 'EASYconsole.m' with MATLAB
		  * Click the Run icon (play button)
		  * When prompted, click "Change Folder" (do not select "Add to Path").
		  * In the pop-up display, select from the 'File' dropdown: 'New Experiment'. 
		  * From the pop-up, choose where to save the new file. 
		  * Navigate to the relevant job in the ExpJobs folder, name the file accordingly, and click 'save'.
		  * The newly created .mat file in the newly created Results folder will automatically be loaded. 
		  * The file name will then be automatically appended by the code with the current date information (e.g. 'A1.mat' will become 'Results2023-07-19A1)
		  * If the experiment has already been created, it can be reloaded by clicking 'Load Experiment' instead of 'New Experiment' and selecting the relevant results 
		  * Next, in the pop-up display, click on the 'Run' dropdown menu and select 'Image CurveFit ComboAnalysis'.
		  * In the updated pop-up, choose/highlight all desired image folders for analysis (this is generally all of the folders, since only the ones that need analysis should be there) and then click on 'continue'. 
		  * As the program is running, updates will periodically appear in the Command Window; there will be an initial pause at "Before call to NIscanIntens…..". 
		  * When the curve fitting is finished, the EASY console will pop back up.
		  * Check to see the completed analysis results in the newly created 'PrintResults' Folder, inside of the 'Results' Folder.
		  * Other folders ('CFfigs', 'figs', 'Fotos') are created for later optional use and will be empty. 
		  * NOTE: The image analysis is completed independent of labeling the data (strains, media type, etc. Labeling happens next with the 'GenReports' function).
		  * Next, click on the 'GenReports' dropdown and select 'DrugMediaMP Generate .mat'
		  * NOTE: The 'MasterPlate' and 'DrugMedia' files have very specific formats and should be completed from a template. 
		  * The Masterplate file must be exact (it must contain all and only the strains that were actually tested). 
		  * For example, if only part of a library is tested, the complete library file must be modified to remove irrelevant strains.
		  * You will be prompted to first select the 'MasterPlate' file. You will need to navigate away from the working directory to get to it. 
		  * It is fine for the 'MasterPlate_' file to be .xlsx (or .xls), and if you don't see it in the popup window, then change the file type from '.xls' to "all files" and then select it. 
		  * Once it is selected, a report of the number of master plates in the file will pop up; when the report appears, assuming it is correct, click on 'OK'.
		  * You will then be prompted to select the 'DrugMedia' file from the relevant job folder. You will automatically return to the correct prior directory location. 
		  * Choose it and click 'OK'. You may see a warning about column headers being modified, but that's ok.
		  * This will create an additional file in the 'MasterPlatesFiles' folder named 'MPDMmat.mat'
		  * Click on the 'GenReports' dropdown and select 'Results_Generate.' 
		  * You will first see '!!ResultsElr_.txt' generated in the 'PrintResults' folder. 
		  * Refreshing will reveal an increasing file size until you see the '!!ResultsStd_.txt' being generated. 
		  * When finished, the '!!ResultsStd_.txt' will be about the same file size and it should be used in the following StudiesQHTCP analysis.  

		  'NoGrowth_.txt', and 'GrowthOnly_.txt' files will be generated in the 'PrintResults' folder. 
	EOF

  declare -gx EASY_DIR="$APPS_DIR/matlab/easy"
  script="$EASY_DIR/EASYconsole.m"

  # Prompt user for suffix
  echo "Current EASY results directory: $EASY_RESULTS_DIR"
  ((YES)) || read -r -p "Enter a custom suffix and/or hit enter to use the default directory (no suffix): " EASY_SUFFIX
  [[ -n $EASY_SUFFIX ]] && EASY_RESULTS_DIR+="_$EASY_SUFFIX"

  # This dirname is separate from the project's so multiple EASY results can be generated
  declare -gx EASY_PROJECT_NAME="${EASY_RESULTS_DIR##*/}"
  debug "EASY results project name: $EASY_PROJECT_NAME"

  # Backup and create EASY results dirs
  [[ -d $EASY_RESULTS_DIR ]] && backup "$EASY_RESULTS_DIR"
  [[ -d $EASY_RESULTS_DIR ]] || execute mkdir -p "$EASY_RESULTS_DIR"

  # Make EASY dirs
  dirs=('PrintResults' 'CFfigs' 'Fotos')
  for d in "${dirs[@]}"; do
    if [[ ! -d $EASY_RESULTS_DIR/$d ]]; then
      execute mkdir -p "$EASY_RESULTS_DIR/$d"
    fi
  done

  # Copy Templates
  declare -gx DRUG_MEDIA_FILE="$SCANS_DIR/DrugMedia_$PROJECT.xls"
  declare -gx MASTER_PLATE_FILE="$EASY_RESULTS_DIR/MasterPlate_$PROJECT.xls"
  execute rsync -a "$EASY_DIR"/{figs,PTmats} "$EASY_RESULTS_DIR"

  # Ask the user to launch EASYconsole.m in MATLAB
  # MATLAB doesn't support passing args to scripts se we have to use ENV VARS instead
  # TODO will need to play with the -sd startup option to see what works (well)
  # Skip this step altogether in auto mode since it requires graphical interaction
  if ! ((YES)) && ask "Start EASY in MATLAB? This requires a GUI."; then
    # Add EASY directory to the Matlab path
    # If this does not work we can try changing the -sd argument and if that fails then pushing/popping
    debug "Adding EASY directory to the Matlab path"
    "$MATLAB" -nodisplay -nosplash -nodesktop -nojvm -batch "addpath('$EASY_DIR')"
    # Launch matlab
    # matlab -nosplash -sd "$PROJECT_SCANS_DIR" -r "run $script"
    "$MATLAB" -nosplash -r "run $script"
  fi
}


module ezview
# @description TODO WIP
ezview() {
  debug "Running: ${FUNCNAME[0]}"

  declare -gx EZVIEW_DIR="$APPS_DIR/matlab/ezview"
  script="$EZVIEW_DIR/EZviewGui.m"

  if ! ((YES)) && ask "Start EASY in MATLAB? This requires a GUI."; then
    # Make EZview dirs

    # Start EZview
    "$MATLAB" -nosplash -r "run $script"
  fi
}


module qhtcp
# @description System for Multi-QHTCP-Experiment Gene Interaction Profiling Analysis
#
# * Functional rewrite of REMcMaster3.sh, RemcMaster2.sh, REMcJar2.sh, ExpFrontend.m, mProcess.sh, mFunction.sh, mComponent.sh
# * Added a newline character to the end of the study info file so it is a valid text file
#
# TODO 
#
# * StudiesArchive should be smarter:
#   * Create a database with as much information as possible
#   * Write a function that easily loads and parses databse into easy-to-use variables
#   * Allow users to reference those variables to write their own modules
# * Should not be using initials
#    * not unique enough and we don't have that data easily on hand
#    * usernames are unique and make more sense
#    * I don't know what all would have to be modified atm
#
# Rerunning this module uses rsync --update to only copy files that are newer in the template
# If you wish for the template to overwrite your changes, delete the file from your QHTCP project dir
#
# To create a new study (Experiment Specific Interaction Zscores generation)
#
# * StudyInfo.csv instructions:
#   * In your files directory, open the /Code folder, edit the 'StudyInfo.csv' spreadsheet, and save it as a 'csv' file to give each experiment the labels you wish to be used for the plots and specific files. 
#     * Enter the desired Experiment names- **order the names in the way you want them to appear in the REMc heatmaps; and make sure to run the front end programs (below) in the correct order (e.g., run front end in 'exp1' folder to call the !!Results file for the experiment you named as exp1 in the StudyInfo.csv file)   
#     * The GTA and pairwise, TSHeatmaps, JoinInteractions and GTF Heatmap scripts use this table to label results and heatmaps in a meaningful way for the user and others. The BackgroundSD and ZscoreJoinSD fields will be filled automatically according to user specifications, at a later step in the QHTCP study process.
#
# * MATLAB ExpFrontend.m was made for recording into a spreadsheet ('StudiesDataArchive.txt') the date and files used (i.e., directory paths to the !!Results files used as input for Z-interaction script) for each multi-experiment study.
#   Give each experiment the labels you wish to be used for the plots and specific files. 
#   Enter the desired Experiment names and order them in the way you want them to appear in the REMc heatmaps; 
#   Run the front end MATLAB programs in the correct order (e.g., run front end in 'exp1' folder to call the !!Results file for the experiment you named as exp1 in the StudyInfo.csv file)   
#   The GTA and pairwise, TSHeatmaps, JoinInteractions and GTF Heatmap scripts use this table to label results and heatmaps in a meaningful way for the user and others. 
#   The BackgroundSD and ZscoreJoinSD fields will be filled automatically according to user specifications, at a later step in the QHTCP study process.
# 
#   * Open MATLAB and in the application navigate to each specific /Exp folder, call and execute ExpFrontend.m by clicking the play icon.
#   * Use the "Open file" function from within Matlab.
#   * Do not double-click on the file from the directory. 
#   * When prompted, navigate to the ExpJobs folder and the PrintResults folder within the correct job folder. 
#   * Repeat this for every Exp# folder depending on how many experiments are being performed. 
#   * Note: Before doing this, it's a good idea to compare the ref and non-ref CPP average and median values. If they are not approximately equal, then may be helpful to standardize Ref values to the measures of central tendency of the Non-refs, because the Ref CPPs are used for the z-scores, which should be centered around zero.
#     * This script will copy the !!ResultsStd file (located in /PrintResults in the relevant job folder in /scans **rename this !!Results file before running front end; we normally use the 'STD' (not the 'ELR' file) chosen to the Exp# directory as can be seen in the “Current Folder” column in MATLAB, and it updates 'StudiesDataArchive.txt' file that resides in the /StudiesQHTCP folder. 'StudiesDataArchive.txt' is a log of file paths used for different studies, including timestamps.
#
# Do this to document the names, dates and paths of all the studies and experiment data used in each study. Note, one should only have a single '!!Results…' file for each /Exp_ to prevent ambiguity and confusion. If you decide to use a new or different '!!Results…' sheet from what was used in a previous “QHTCP Study”, remove the one not being used. NOTE:  if you copy a '!!Results…' file in by hand, it will not be recorded in the 'StudiesDataArchive.txt' file and so will not be documented for future reference. If you use the ExpFrontend.m utility it will append the new source for the raw !!Results… to the 'StudiesDataArchive.txt' file.  
# As stated above, it is advantageous to think about the comparisons one wishes to make so as to order the experiments in a rational way as it relates to the presentation of plots.  That is, which results from sheets and selected 'interaction … .R', user modified script, is used in /Exp1, Exp2, Exp3 and Exp4 as explained in the following section.
# TODO MUST CLEAN UP QHTCP TEMPLATE DIRECTORY
#
#
# As stated earlier, the user can add folders to back up temporary results, study-related notes, or other related work. 
# However, it is advised to set up and use separate STUDIES when evaluating differing data sets whether that is from experiment results files or from differing data selections in the first interaction … .R script stage.
# This reduces confusion at the time of the study and especially for those reviewing study analysis in the future.
#
# How-To Procedure: Execute a Multi-experiment Study:
#
#  * Consider the goals of the study and design a strategy of experiments to include in the study. 
#  * Consider the quality of the experiment runs using EZview to see if there are systematic problems that are readily detectable.
#    * In some cases, one may wish to design a 'pilot' study for discovery purposes. 
#    * There is no problem doing that, just take a template study, copy and rename it as XYZpilotStudy etc. 
#    * However, careful examination of the experimental results using EZview will likely save time in the long run. 
#    * One may be able to relatively quickly run the interaction Z scores (the main challenge there is the user creation of customized interaction… .R code. 
#    * I have tried to simplify this by locating the user edits near the top).
#
#
qhtcp() {
  debug "Running: ${FUNCNAME[0]}"

  [[ -d $PROJECT_RESULTS_DIR ]] || 
    err "$PROJECT_RESULTS_DIR does not exist, have you run the init_project module?"

  # # Create studies archive file if missing
  # if ! [[ -d $STUDIES_ARCHIVE_FILE ]]; then
  #   header=(StudyDate tStudyName StudyPath ExpNum ExpDate ExpPath ResultFile)
  #   printf "%s\t" "${header[@]}" > "$STUDIES_ARCHIVE_FILE"
  # fi
  # # TODO Add them all to StudiesDataArchive?
  # # Probably better to always add and remove dupes later since each invocation "counts"?
  # for f in "${EASY_RESULTS_FILES[@]}"; do
  #   for study in "${STUDIES[@]}"; do
  #     read -r num sd dir <<< "$study"
  #     # Trying to match old ExpFrontend formatting
  #     printf "%s\t" \
  #       "${DATE//_/}" "$PROJECT" "$PROJECT_RESULTS_DIR" "Exp$num" \
  #       "$PROJECT_DATE" "$PROJECT_SCANS_DIR" "$EASY_RESULTS_DIR" "${f##*/}" \
  #       >> "$STUDIES_ARCHIVE_FILE"
  #   done
  # done

  # Run R interactions script on all studies
  calculate_interaction_zscores; exit \
  && join_interaction_zscores \
  && remc \
  && gtf \
  && gta
}


module remc
# @description remc module for QHTCP
#
# TODO 
#
# * Which components can be parallelized?
#
#
# @arg $1 string study info file
remc() {
  debug "Running: ${FUNCNAME[0]}"

  # If any wrappers fail the rest will not run, this is fundamental to module design
  # Remove leading && to run regardless
  java_extract \
  && r_add_shift_values \
  && r_create_heat_maps \
  && r_heat_maps_homology
}


module gtf
# shellcheck disable=SC2120
# @description GTF module for QHTCP
# @arg $1 string output directory
# @arg $2 string gene_association.sgd
# @arg $3 string gene_ontology_edit.obo
# @arg $4 string ORF_List_Without_DAmPs.txt
gtf() {
  debug "Running: ${FUNCNAME[0]}"

  process_dir="$GTF_OUT_DIR/process"
  function_dir="$GTF_OUT_DIR/function"
  component_dir="$GTF_OUT_DIR/component"

  py_gtf_dcon \
    "$process_dir" \
    "$GTF_OUT_DIR"

  # Reproduce the function and components dirs from the process dir
  for d in "$function_dir" "$component_dir"; do
    execute rsync -a "$process_dir/" "$d/"
  done

  for d in "$process_dir" "$function_dir" "$component_dir"; do
    out_file="${d##*/}Results.txt" # Use the dirname to create each Results filename
    txts=("$d"/*.txt) # glob all txt files from each dir
    for txt in "${txts[@]}"; do
      pl_gtf_analyze "$txt"
      pl_gtf_terms2tsv "$txt"
    done
    py_gtf_concat "$GTF_OUT_DIR" "$out_file"
  done
  r_compile_gtf "$GTF_OUT_DIR"
}


module gta
# shellcheck disable=SC2120
# @description GTA module for QHTCP
#
# TODO
#
#   * 
#   * 
#
# @arg $1 string output directory
# @arg $2 string gene_association.sgd
# @arg $3 string gene_ontology_edit.obo
# @arg $4 string go_terms.tab
# @arg $5 string All_SGD_GOTerms_for_QHTCPtk.csv
gta() {
  debug "Running: ${FUNCNAME[0]}"

  # gene_association_sgd="${2:-"$APPS_DIR/r/gene_association.sgd"}"
  gene_ontology_obo="${3:-"$APPS_DIR/r/gene_ontology_edit.obo"}"
  sgd_terms_tfile="${4:-"$APPS_DIR/r/go_terms.tab"}"
  all_sgd_terms_csv="${5:-"$APPS_DIR/r/All_SGD_GOTerms_for_QHTCPtk.csv"}"  
  
 # TODO This could be wrong, it could be in main results

  [[ -d $GTA_OUT_DIR ]] && backup "$GTA_OUT_DIR"
  execute mkdir -p "$GTA_OUT_DIR"

  # Loop over the array and create pairwise arrays
  for ((i=0; i<${#EXP_NUMS[@]}; i++)); do
    for ((j=i+1; j<${#EXP_NUMS[@]}; j++)); do
      pair=("${EXP_NUMS[i]}" "${EXP_NUMS[j]}")
      echo "${pair[@]}"
    done
  done

  # Create unique parwise combinations of study nums from dir names
  exp_combos=()
  for ((i=0; i<${#EXP_NUMS[@]}; i++)); do
    # Loop through the array again
    for ((j=0; j<${#EXP_NUMS[@]}; j++)); do
      # If the indices are not the same
      if [ "$i" != "$j" ]; then
        # Print the unique combination
        exp_combos+=("${EXP_NUMS[i]},${EXP_NUMS[j]}")
      fi
    done
  done

  # The following are three types of studies

  # Individual studies
  for exp_num in "${EXP_NUMS[@]}"; do
    execute mkdir -p "$GTA_OUT_DIR/exp$exp_num"
    r_gta "exp$exp_num" "$STUDY_RESULTS_DIR/exp$exp_num/zscores/zscores_interaction.csv"
  done

  # Combination studies (for pairwise comparisons)
  for combo in "${exp_combos[@]}"; do
    # Split on comma and assign to array
    IFS=',' read -ra exps <<< "$combo"
    r_gta_pairwiselk \
      "${EXPERIMENTS[exp${exps[0]},name]}" \
      "$GTA_OUT_DIR/exp${exps[0]}/Average_GOTerms_All.csv" \
      "${EXPERIMENTS[exp${exps[1]},name]}" \
      "$GTA_OUT_DIR/exp${exps[1]}/Average_GOTerms_All.csv" \
      "$GTA_OUT_DIR"
  done

  r_gta_heatmaps \
    "$STUDY_INFO_FILE" \
    "$gene_ontology_obo" \
    "$sgd_terms_tfile" \
    "$all_sgd_terms_csv" \
    "$STUDY_RESULTS_DIR" \
    "$STUDY_RESULTS_DIR/TermSpecificHeatmaps" \
    "${EXP_NUMS[@]}"
}


# @section Wrappers
# @description
#
# Wrappers:
#
# * Allow scripts to be called by the main workflow script using input and output arguments as a translation mechanism.
# * Only run by default if called by a module.
# * Can be called directly with its arguments as a comma-separated string
#
# @description
wrapper() { 
  debug "Adding $1 wrapper"
  ALL_WRAPPERS+=("$1")
  declare -gA "$1"
}


wrapper calculate_interaction_zscores
# @description Run the R interactions analysis (deprecates Z_InteractionTemplate.R)
# shellcheck disable=SC2120
#
# SCRIPT: apps/r/calculate_interaction_zscores.R
#
# TODO
#
# * More variables can be read in from the config file to allow more configuration
#   * sd thresholds
#   * lm thresholds
#   * interaction thresholds
#   * reference gene(s)
#   * background genes
# * Add gene names, other threshold values, etc.
# * Dataframe columns and output file columns should be standardized in calculate_interactions()
# * Need to decide if conc_num_factor is numeric or factor
# * Do pdfs really need to be all different sizes?
# * We are using standard error bars using the same se values as the data now (includes Bessel's correction)
#    * Plate analysis error bars and some others will be slightly different
#    * Can be changed back but better to have plots reflect data, no?
# * Dynamically generate axis limits based on data (if desired)
# * Parallelize interaction plotting
#
# INPUT
#
# * easy/results_std.txt
#
# OUTPUT
#
# * zscores/zscores_interaction.csv
# * etc.
#
# NOTES
#
# *
#
# @arg $1 integer output directory
# @arg $2 string SGD_features.tab
# @arg $3 string easy/results_std.txt
# @arg $6 array triplets of experiment paths, names, sd threshold factor
calculate_interaction_zscores() {
  debug "Running: ${FUNCNAME[0]} $*"
  cat <<-EOF
		* Be sure to enter Background noise filter standard deviation i.e., 3 or 5 per Sean
		* Enter Standard deviation value for removing data for cultures due to high background (e.g., contaminated cultures).
		  * Generally set this very high (e.g., '20') on the first run in order NOT to remove data, e.g. '20'. Review QC data and inspect raw image data to decide if it is desirable to remove data, and then rerun analysis. 
		* Enter a Background SD threshold for EXCLUDING culture data from further analysis:
		  * This Background value removes data where there is high pixel intensity in the background regions of a spot culture (i.e., suspected contamination).
		  * 5 is a minimum recommended value, because lower values result in more data being removed, and often times this is undesirable if contamination occurs late after the carrying capacity of the yeast culture is reached.
		  * This is most often "trial and error", meaning there is a 'Frequency_Delta_Background.pdf' report in the /Exp_/ZScores/QC/ folder to evaluate whether the chosen value was suitable (and if not the analysis can simply be rerun with a more optimal choice).
		  * In general, err on the high side, with BSD of 10 or 12…. One can also use EZview to examine the raw images and individual cultures potentially included/excluded as a consequence of the selected value.
		  * Background values are reported in the results sheet and so could also be analyzed there.
	EOF

  declare script="$APPS_DIR/r/calculate_interaction_zscores.R"
  declare -a out_paths=("${1:-"$STUDY_RESULTS_DIR/zscores"}")
  for path in "${EXP_PATHS[@]}"; do
    out_paths+=("${path}/zscores")
  done

  # TODO we'll need to change this behaviour after testing
  # we can test for existence and then choose to skip or rerun later
  # possibly handle in the module?
  for out_path in "${out_paths[@]}"; do
    backup "$out_path"
    execute mkdir -p "$out_path" "$out_path/qc"
  done
    
  execute "$RSCRIPT" "$script" \
    "${1:-"$STUDY_RESULTS_DIR"}" \
    "${2:-"$APPS_DIR/r/SGD_features.tab"}" \
    "${3:-"$EASY_RESULTS_DIR/results_std.txt"}" \
    "${@:4}" \
    "${EXP_PATHS_AND_NAMES_AND_SD_FACTORS[@]}"

}


wrapper join_interaction_zscores
# shellcheck disable=SC2120
# @description JoinInteractExps3dev.R creates REMcRdy_lm_only.csv and Shift_only.csv
#
# TODO
#
# * Needs more loops to reduce verbosity
#
# INPUT
#
# * /out/PROJECT/STUDY/exp#/zscores/zscores_interaction.csv
#
# OUTPUT
#
# * combined_zscores.csv (REMcRdy_lm_only.csv)
# * combined_summary_stats (Shift_only.csv)
# * final_combined_report (parameters.csv)
#
# @arg $1 string output directory
# @arg $2 string sd value (default: 2)
# @arg $3 array pairs of experiment paths and names
join_interaction_zscores() {
  debug "Running: ${FUNCNAME[0]} $*"
  declare script="$APPS_DIR/r/join_interaction_zscores.R"
  declare -a out_files=(
    "${1:-$STUDY_RESULTS_DIR}/combined_zscores.csv"
    "${1:-$STUDY_RESULTS_DIR}/combined_summary_stats.csv" 
    "${1:-$STUDY_RESULTS_DIR}/final_combined_report.csv"
  )

  # ((DEBUG)) && declare -p # when the going gets tough
  
  execute "$RSCRIPT" "$script" \
    "${1:-$STUDY_RESULTS_DIR}" \
    "${2:-2}" \
    "${@:3:}" \
    "${EXP_PATHS_AND_NAMES[@]}"

  for f in "${out_files[@]}"; do
    [[ -f $f ]] || { echo "$f does not exist"; return 1; }
  done
}


wrapper r_gta
# @description GTAtemplate R script
#
# TODO
#
# * Is GTAtemplate.R actually a template?
# * Do we need to allow user customization?
#
# INPUT
#
# * [gene_association.sgd](https://downloads.yeastgenome.org/curation/chromosomal_feature/gene_association.sgd)
# * go_terms.tab
#
# OUTPUT
#
# * Average_GOTerms_All.csv
#
#
# @arg $1 string exp# name (required)
# @arg $2 string zscores_interaction.csv (required)
# @arg $3 string go_terms.tab file
# @arg $4 string [gene_association.sgd](https://downloads.yeastgenome.org/curation/chromosomal_feature/gene_association.sgd)
# @arg $5 string output directory
r_gta() {
  debug "Running: ${FUNCNAME[0]} $*"
  cat <<-EOF

	EOF
  script="$APPS_DIR/r/gtaTemplate.R"

  out_file="${5:-"$GTA_OUT_DIR"}/Average_GOTerms_All.csv"

  execute "$RSCRIPT" "$script" \
    "$1" \
    "$2" \
    "${3:-"$APPS_DIR/r/go_terms.tab"}" \
    "${4:-"$APPS_DIR/r/gene_association.sgd"}" \
    "${5:-"$GTA_OUT_DIR"}" \
    "${@:6}" # future arguments

  [[ -f $out_file ]] || { echo "$out_file does not exist"; return 1; }
}


wrapper r_gta_pairwiselk
# @description PairwiseLK.R R script
#
# TODO
#
# * Move directory creation from PairwiseLK.R to gta module
# * Needs better output filenames and directory organization
# * Needs more for looping to reduce verbosity
#
# INPUT
#
# * Average_GOTerms_All.csv
# * 
#
# OUTPUT
#
# *
#
# This wrapper:
#
# * Will perform both L and K comparisons for the specified experiment folders. 
# * The code uses the naming convention of PairwiseCompare_Exp’#’-Exp’#’ to standardize and keep simple the structural naming (where ‘X’ is either K or L and ‘Y’ is the number of the experiment GTA results to be found in ../GTAresult/Exp_).
# * {FYI There are also individual scripts that just do the ‘L’ or ‘K’ pairwise studies in the ../Code folder.}
#
# @arg $1 string First exp# name (required)
# @arg $2 string First exp# go terms file (required)
# @arg $3 string Second exp# name (required)
# @arg $4 string Second exp# go terms file (required)
# @arg $5 string output directory
#
r_gta_pairwiselk() {
  debug "Running: ${FUNCNAME[0]} $*"
  cat <<-EOF

	EOF

  script="$APPS_DIR/r/calculate_pairwise_lk.R"

  execute "$RSCRIPT" "$script" \
    "$1" \
    "$2" \
    "$3" \
    "$4" \
    "${5:-"$GTA_OUT_DIR"}" \
    "${@:6}" # future arguments
}


wrapper r_gta_heatmaps
# @description TSHeatmaps5dev2.R R script
#
# TODO
#
#   * Rename
#   * Refactor to automatically allow more studies
#   * Refactor with more looping to reduce verbosity
#   * Reduce cyclomatic complexity of some of the for loops
#
# Files
#
#   * 
#   * 
#
# Output
#
#   *
#
# This wrapper:
#
#   * The Term Specific Heatmaps are produced directly from the ../ExpStudy/Exp_/ZScores/ZScores_Interaction.csv file generated by the user modified interaction… .R  script. 
#   * The heatmap labeling is per the names the user wrote into the study info file
#   * Verify that the All_SGD_GOTerms_for_QHTCPtk.csv found in ../Code is what you wish to use or if you wish to use a custom modified version.  
#   * If you wish to use a custom modified version, create it and modify the TSHeatmaps template script (TSHeatmaps5dev2.R) and save it as a ‘TSH_study specific name’.
#
# @arg $1 string study info file
# @arg $2 string gene_ontology_edit.obo
# @arg $3 string go_terms.tab
# @arg $4 string All_SGD_GOTerms_for_QHTCPtk.csv
# @arg $5 string base directory
# @arg $6 string output directory
#
r_gta_heatmaps() {
  debug "Running: ${FUNCNAME[0]} $*"
  cat <<-EOF

	EOF
  script="$APPS_DIR/r/TSHeatmaps5dev2.R"

  [[ -d $7 ]] || execute mkdir -p "$7"

  execute "$RSCRIPT" "$script" \
    "${1:-$STUDY_INFO_FILE}" \
    "${2:-"$APPS_DIR/r/gene_ontology_edit.obo"}" \
    "${3:-"$APPS_DIR/r/go_terms.tab"}" \
    "${4:-"$APPS_DIR/r/All_SGD_GOTerms_for_QHTCPtk.csv"}" \
    "${5:-"$STUDY_RESULTS_DIR"}" \
    "${6:-"$STUDY_RESULTS_DIR/TermSpecificHeatmaps"}" \
    "${@:7}" # studies
}


wrapper java_extract
# shellcheck disable=SC2120
# @description Jingyu's REMc java utility
#
# TODO
#
# * Closed-source w/ hardcoded output directory, so have to pushd/popd to run (not ideal)
#
# INPUT
#
# * study_dir/combined_zscores.csv (REMcRdy_lm_only.csv)
#
# OUTPUT
#
# * study_dir/combined_zscores_final.csv (REMcRdy_lm_only.csv-finalTable.csv)
#
# @arg $1 string output directory
# @arg $2 string combined_zscores.csv
# @arg $3 string GeneByGOAttributeMatrix_nofiltering-2009Dec07.tab
# @arg $4 string ORF_List_Without_DAmPs.txt
# @exitcode 0 if expected output file exists
# @exitcode 1 if expected output file does not exist
java_extract() {
  debug "Running: ${FUNCNAME[0]}"
  classpath="$APPS_DIR/java/weka-clustering/weka-clustering.jar"

  output_file="${1:-$STUDY_RESULTS_DIR}/combined_zscores_final.csv"

  [[ -f $output_file ]] && backup "$output_file"
  
  java_cmd=(
    "$JAVA" -Xms512m -Xmx2048m -Dfile.encoding=UTF-8
    -classpath "$classpath" ExecMain
    "${2:-"$STUDY_RESULTS_DIR/combined_zscores.csv"}"
    "${3:-"$APPS_DIR/java/GeneByGOAttributeMatrix_nofiltering-2009Dec07.tab"}"
    "${4:-"$APPS_DIR/java/ORF_List_Without_DAmPs.txt"}"
    1
    true
  )

  debug "pushd && ${java_cmd[*]} && popd"
  pushd "${1:-$STUDY_RESULTS_DIR}" && "${java_cmd[@]}" && popd || return 1
  [[ -f $output_file ]]
}


wrapper r_add_shift_values
# shellcheck disable=SC2120
# @description Add shift values back to REMcRdy_lm_only.csv-finalTable.csv
# and output "REMcWithShift.csv" for use with the REMc heat maps
#
# @arg $1 string REMcRdy_lm_only.csv-finalTable.csv
# @arg $2 string Shift_only.csv
# @arg $3 string study info file
# @arg $4 string REMcWithShift.csv
r_add_shift_values() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/r/addShiftVals.R"
  
  execute "$RSCRIPT" "$script" \
    "${1:-"$STUDY_RESULTS_DIR/REMcRdy_lm_only.csv-finalTable.csv"}" \
    "${2:-"$STUDY_RESULTS_DIR/Shift_only.csv"}" \
    "${3:-$STUDY_INFO_FILE}" \
    "${4:-"$STUDY_RESULTS_DIR/REMcWithShift.csv"}" \
    "${@:5}" # future arguments

  rm -f "$STUDY_RESULTS_DIR/REMcHeatmaps/"*.pdf # TODO why?
  out_file="${4:-"$STUDY_RESULTS_DIR/REMcWithShift.csv"}"
  [[ -f $out_file ]] || { echo "$out_file does not exist"; return 1; }
}


wrapper r_create_heat_maps
# shellcheck disable=SC2120
# @description Execute createHeatMaps.R
#
# INPUT
#
# * REMcWithShift.csv
#
# OUTPUT
#
# * compiledREMcHeatmaps.pdf
#
# TODO
#
# * Needs more looping for brevity
#
#
#
# @arg $1 string The final shift table (REMcWithShift.csv)
# @arg $2 string The output directory
r_create_heat_maps() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/r/createHeatMaps.R"

  execute "$RSCRIPT" "$script" \
    "${1:-"$STUDY_RESULTS_DIR/REMcWithShift.csv"}" \
    "${2:-"$STUDY_RESULTS_DIR"}" \
    "${@:3}" # future arguments

  pdfs=(REMcHeatmaps/*.pdf)
  execute pdftk "${pdfs[@]}" output "$out_file"
  out_file="$2/compiledREMcHeatmaps.pdf"
  [[ -f $out_file ]] || { echo "$out_file does not exist"; return 1; }
}


wrapper r_heat_maps_homology
# shellcheck disable=SC2120
# @description Execute createHeatMapsHomology.R
#
# @arg $1 string output directory
# @arg $2 string REMcRdy_lm_only.csv-finalTable.csv
# @arg $3 string 170503_DAmPs_Only.txt
# @arg $4 string (Yeast_Human_Homology_Mapping_biomaRt_18_0920.csv)
r_heat_maps_homology() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/r/createHeatMapsHomology.R"

  out_file="${1:-"$STUDY_RESULTS_DIR/homology"}/compiledREMcHomologyHeatmaps.pdf"
  
  debug "Removing old pdf and csv files from ${1:-"$STUDY_RESULTS_DIR/homology"}"
  rm "${1:-"$STUDY_RESULTS_DIR/homology"}/"*.{pdf,csv}

  execute "$RSCRIPT" "$script" \
    "${1:-"$STUDY_RESULTS_DIR/homology"}" \
    "${2:-"$STUDY_RESULTS_DIR/REMcRdy_lm_only.csv-finalTable.csv"}" \
    "${3:-"$APPS_DIR/r/170503_DAmPs_Only.txt"}" \
    "${4:-"$APPS_DIR/r/Yeast_Human_Homology_Mapping_biomaRt_18_0920.csv"}" \
    "${@:5}" # future arguments

  pdfs=("${1:-"$STUDY_RESULTS_DIR/homology"}"/*.pdf)
  execute pdftk "${pdfs[@]}" output "$out_file"
  [[ -f $out_file ]] || { echo "$out_file does not exist"; return 1; }
}


wrapper py_gtf_dcon
# @description Perform python dcon portion of GTF
#
# SCRIPT: [DconJG2.py](apps/python/DconJG2.py)
#
# OUTPUT
#
# * 1-0-0-finaltable.csv
#
# @arg $1 string Directory to process
# @arg $2 string Output directory name
py_gtf_dcon() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/python/DconJG2.py"

  execute "$PYTHON" "$script" \
    "$1" \
    "$2/" \
    "${@:3}" # future arguments
  out_file="$2/1-0-0-finaltable.csv"
  [[ -f $out_file ]] || { echo "$out_file does not exist"; return 1; }
}


wrapper pl_gtf_analyze
# @description Perl analyze wrapper
#
# SCRIPT: [analyze_v2.pl](https://metacpan.org/dist/GO-TermFinder/view/examples/analyze.pl)
#
# TODO
#
# * Are we just overwriting the same data for all set2 members?
# * Why the custom version?
#
# @arg $1 string txt to analyze (required)
# @arg $2 string gene_association.sgd
# @arg $3 string gene_ontology_edit.obo
# @arg $4 string ORF_List_Without_DAmPs.txt
pl_gtf_analyze() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/perl/analyze_v2.pl"

  execute "$PERL" "$script" \
    "-an" "${2:-"$APPS_DIR/r/gene_association.sgd"}" \
    "-as" "P" \
    "-o" "${3:-"$APPS_DIR/r/gene_ontology_edit.obo"}" \
    "-b" "${4:-"$APPS_DIR/r/ORF_List_Without_DAmPs.txt"}" \
    "$1"
}


wrapper pl_gtf_terms2tsv
# @description Perl terms2tsv wrapper
#
# TODO
#
# * Probably should be translated to shell/python
#
# @arg $1 string Terms file TODO naming
pl_gtf_terms2tsv() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/perl/terms2tsv.pl"
  debug "$PERL $script $1.terms > $1.tsv"
  "$PERL" "$script" "$1.terms" > "$1.tsv"
}


wrapper py_gtf_concat
# @description Python concat wrapper for GTF
# Concat the process ontology outputs from the /REMcReady_lm_only folder
#
# TODO
#
# * Probably should be translated to bash
#
# @arg $1 string output directory name to look for txt files
# @arg $2 string output file
py_gtf_concat() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/python/concatGTFResults.py"
  execute "$PYTHON" "$script" "$1/" "$2"
  [[ -f $2 ]] || { echo "$2 does not exist"; return 1; }
}


wrapper r_compile_gtf
# @description Compile GTF in R
# @arg $1 string gtf output directory
r_compile_gtf() {
  debug "Running: ${FUNCNAME[0]} $*"
  script="$APPS_DIR/r/CompileGTF.R"
  execute "$RSCRIPT" "$script" "$1"
}


# @description Selects a results directory
# @exitcode 0 if successfully chose a results dir
# @set EASY_RESULTS_DIR string The working EASY output directory
# @arg $1 string directory containing results dirs matching the prefix regex
# @arg $2 string variable name to declare
handle_results_dir() {
  debug "Running: ${FUNCNAME[0]} $*"
  declare results_dir
  declare -a results_dirs=("$1"/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_*_*/) # glob for results directories
  results_dirs=("${results_dirs[@]%/}") # remove trailing slashes

  # Filter directory names
  declare -a filtered_dirs=()
  for dir in "${results_dirs[@]}"; do
    base_dir="${dir##*/}" # get basename
    if sanitize_pn "$base_dir"; then
      filtered_dirs+=("$dir")
    fi
  done

  # Sort filtered directories by date prefix in case the glob was mixed
  mapfile -t results_dirs < <(printf '%s\n' "${filtered_dirs[@]}" | sort)

  num=${#results_dirs[@]}

  ((DEBUG)) && print_in_columns "results_dirs"

  # Choose a results dir based on number of results dirs
  if [[ $num -eq 1 ]] && ((YES)); then
    results_dir="${results_dirs[0]}"
  elif ((YES)); then
    results_dir="${results_dirs[-1]}"
  else
    ask_name "$2" ADD_RESULTS
    results_dirs=("${ADD_RESULTS[@]}")
    if [[ $num -eq 1 ]]; then
      results_dir="${results_dirs[0]}"
    else
      # Print results dirs
      [[ ${#results_dirs[@]} -gt 0 ]] || { err "No ${2}s found"; return 1; }
      print_in_columns "results_dirs"
      # Results selection prompt
      cat <<-EOF
				${underline}Select $2 by number${nounderline}
				* <Enter> for the latest $2 ($num)
				* 0 to create a new $2
			EOF
      read -r -p "($num): " response
      echo
      response="${response:-$num}"
      if [[ $response -eq 0 ]]; then
        ask_name "$2" ADD_RESULTS
        results_dirs=("${ADD_RESULTS[@]}")
        results_dir="${results_dirs[-1]}"
      else
        results_dir="${results_dirs[$((response-1))]}"
      fi
    fi
  fi

  # Set the fallback
  [[ -z $results_dir ]] && results_dir="$1/${STUDY_PREFIX}_${PROJECT_NAME}"

  # Create directory and set global variable
  [[ -d $results_dir ]] || execute mkdir -p "$results_dir"
  declare -gx "$2"="$results_dir"
}


module generate_markdown
# @description Generates shdoc markdown from this script
# @noargs
# @internal
generate_markdown() {
  debug "Running: ${FUNCNAME[0]}"
  # Print markdown to stdout
  ((DEBUG)) && shdoc < "$SCRIPT"
  # Create markdown file
  shdoc < "$SCRIPT" > README.md
}


# @description Parses a simple "bashy" config file
# @arg $1 study config file
# @internal
handle_config() {
  debug "Running: ${FUNCNAME[0]} $*"
  
  # Determine the configuration file path
  declare config_file="${1:-${STUDY_CONFIG_FILE:-"$STUDY_RESULTS_DIR/study_config"}}"
  declare config_array_name="EXPERIMENTS"

  # Ensure the function runs only once per project (in case of multiple modules)
  (( PARSED_CONFIG )) && return 0
  declare -g PARSED_CONFIG=1

  # Default experiment settings for a study
  declare -gA default_study_config
  # shellcheck disable=SC2034
  add_experiment() {
    echo "Adding default Exp $1 to the study config file"
    default_study_config["exp$1,name"]="${2:-"Experiment $1 for ${STUDY_RESULTS_DIR##*/} study"}"
    default_study_config["exp$1,path"]="${3:-"$STUDY_RESULTS_DIR/exp$1"}"
    default_study_config["exp$1,sd"]="${4:-3}" 
    default_study_config["exp$1,sd_L"]="${5:-0}" 
    default_study_config["exp$1,sd_K"]="${6:-0}" 
    default_study_config["exp$1,sd_pair_L"]="${7:-0}" 
    default_study_config["exp$1,sd_pair_K"]="${8:-0}" 
    default_study_config["exp$1,gene_exclude"]="${9:-"YDL227C"}" 
    default_study_config["exp$1,gene_include"]="${10:-0}" 
  }

  # Check if the config file exists
  if ! [[ -f $config_file ]]; then
    declare -a exp_dirs
    [[ -n $1 ]] && dirname="${1%/*}"
    exp_dirs=("${dirname:-$STUDY_RESULTS_DIR}/exp"[0-9]*/) 
    exp_dirs=("${exp_dirs[@]%/}") 

    if [[ ${#exp_dirs[@]} -gt 0 ]]; then
      for exp_dir in "${exp_dirs[@]}"; do
        num="${exp_dir##*/}" 
        num="${num#exp}" 
        add_experiment "$num"
      done
    else
      echo "No config file found at $config_file"
      echo "No experiment directories found in $STUDY_RESULTS_DIR"
      if ask "Would you like to create a study config file at $config_file?"; then
        ((YES)) || read -r -p "Number of experiments to create (4): " response
        response="${response:-4}"
        for ((i = 1; i <= response; i++)); do
          add_experiment "$i"
        done
      fi
    fi
    write_config "$config_file" "default_study_config" "$config_array_name" &&
    echo "Configuration file generated at $config_file"
  fi

  # Source the config file for the config array
  # shellcheck disable=SC1090
  . "$config_file"
  
  # Declare a nameref to refer to the actual array using the variable name
  declare -n config_array_ref="$config_array_name"

  [[ -z ${!config_array_ref[*]} ]] && { err "No $config_array_name array found in $config_file" && return 1; }

  for key in "${!config_array_ref[@]}"; do
    IFS=',' read -r main_key sub_key <<< "$key"
    lines+=("${config_array_name}[$main_key,$sub_key]=\"${config_array_ref[$key]}\"")
  done

  mapfile -t lines < <(printf "%s\n" "${lines[@]}" | sort)

  echo "${underline}$config_file${nounderline}"
  printf "%s\n" "${lines[@]}"
  unset lines

  # Prompt user to edit study config file if not in auto mode
  if ! ((YES)) && ask "Would you like to edit the study config file?"; then
    "$EDITOR" "$config_file"
     # shellcheck disable=SC1090
    . "$config_file"
  fi

  # Create some helpful arrays
  declare -ga EXP_PATHS
  # Loop over the keys in the associative array
  for key in "${!config_array_ref[@]}"; do
    if [[ $key == *,path ]]; then
      EXP_PATHS+=("${config_array_ref[$key]}")
    fi
  done

  declare -ga EXP_NUMS=("${EXP_PATHS[@]##*[!0-9]}")

  declare -ga EXP_PATHS_AND_NAMES
  for key in "${!config_array_ref[@]}"; do
    if [[ $key == *,path ]]; then
      main_key="${key%,*}"
      name_key="${main_key},name"
      if [[ -n ${config_array_ref[$name_key]} ]]; then
        EXP_PATHS_AND_NAMES+=("${config_array_ref[$key]}" "${config_array_ref[$name_key]}")
      fi
    fi
  done

  declare -ga EXP_PATHS_AND_NAMES_AND_SD_FACTORS
  for key in "${!config_array_ref[@]}"; do
    if [[ $key == *,path ]]; then
      main_key="${key%,*}"
      name_key="${main_key},name"
      sd_key="${main_key},sd"
      if [[ -n ${config_array_ref[$name_key]} ]]; then
        EXP_PATHS_AND_NAMES_AND_SD_FACTORS+=(
          "${config_array_ref[$key]}"
          "${config_array_ref[$name_key]}"
          "${config_array_ref[$sd_key]}")
      fi
    fi
  done

  # declare -ga EXP_PATHS_AND_NAMES
  # for key in "${!config_array_ref[@]}"; do
  #   if [[ $key == *,path ]]; then
  #     EXP_PATHS_AND_NAMES+=("${config_array_ref[$key]}")
  #   fi
  #   if [[ $key == *,name ]]; then
  #     EXP_PATHS_AND_NAMES+=("${config_array_ref[$key]}")
  #   fi
  # done
  return 0
}


# @description Write an associative array to a config file
# @arg $1 file to write
# @arg $2 name of the associative array
# @arg $3 name of the config array
write_config() {
  debug "Running: ${FUNCNAME[0]} $*"
  declare file="$1"
  declare -n array="$2"
  declare array_name="${3:-"EXPERIMENTS"}"

  declare -a lines=()
  # Iterate over the associative array to populate the lines array
  for key in "${!array[@]}"; do
    IFS=',' read -r main_key sub_key <<< "$key"
    lines+=("${array_name}[$main_key,$sub_key]=\"${array[$key]}\"")
  done

  mapfile -t lines < <(printf "%s\n" "${lines[@]}" | sort) # re-sort the lines alphabetically

  # Write the contents to the file using a HEREDOC
  cat <<- EOF > "$file"
		#!/usr/bin/env bash

		# Declare the main associative array
		declare -Ax ${array_name}

		$(printf "%s\n" "${lines[@]}")

	EOF
}


# @description The main loop of qhtcp-workflow
#
# @internal
main() {
  debug "Running: ${FUNCNAME[0]} $*"

  # Libraries
  declare -g JAVA="${JAVA:-$(which java 2>/dev/null || echo java)}"
  declare -g PYTHON="${PYTHON:-$(which python3 2>/dev/null || echo python)}"
  declare -g PERL="${PERL:-$(which perl 2>/dev/null || echo perl)}"
  declare -g RSCRIPT="${RSCRIPT:-$(which Rscript 2>/dev/null || echo Rscript)}"
  declare -g MATLAB="${MATLAB:-$(which matlab 2>/dev/null || echo matlab)}"
  declare -g EDITOR="${EDITOR:-"nano"}"

  # Global vars
  SCRIPT_NAME="${BASH_SOURCE[0]##*/}"
  SCRIPT=$(realpath -s "${BASH_SOURCE[0]}")
  SCRIPT_DIR=$(dirname "$SCRIPT")
  APPS_DIR="$SCRIPT_DIR/apps"
  # TEMPLATES_DIR="$SCRIPT_DIR/templates"
  USER="${USER:-$(whoami)}"
  DATE="$(date +%Y%m%d)" # change in EASYconsole.m to match 'hardcode'

  # Find a scans directory
  # local scans_heirarchy=("./scans" "/mnt/data/scans" "/mnt/data/ExpJobs" "./scans")
  local scans_heirarchy=(
    "$SCRIPT_DIR/scans"
    "/mnt/data/scans"
    "$SCRIPT_DIR/templates/scans-demo"
  )
  
  # Find an existing scans dir if SCANS_DIR is not set
  if [[ -z $SCANS_DIR ]]; then
    for d in "${scans_heirarchy[@]}"; do
      if [[ -d $d ]]; then
        declare -gx SCANS_DIR="$d"
        break
      fi
      declare -gx SCANS_DIR="${scans_heirarchy[0]}"
    done
  fi

  if ! [[ -d $SCANS_DIR ]]; then
    # This is not something we do often, so ask
    if ask "Create the scans directory: $SCANS_DIR?"; then
      execute mkdir -p "$SCANS_DIR"
    else
      echo "No scans directory available, exiting"
      exit 1;
    fi
  fi

  # Make sure we are using the absolute path
  SCANS_DIR=$(realpath -s "$SCANS_DIR")

  # Find an output directory
  local out_heirarchy=("${SCANS_DIR%/*}/out" "$SCRIPT_DIR/out" "/mnt/data/out")
  for d in "${out_heirarchy[@]}"; do
    if [[ -d $d ]]; then
      debug "Using output directory: $d"
      declare -g OUT_DIR="$d"
      break
    fi
  done

  if [[ -z $OUT_DIR ]]; then
    echo "No output directory found"
    declare -gx OUT_DIR="$SCRIPT_DIR/out"
    # This is not something we do often, so ask
    if ask "Create $SCRIPT_DIR/out?"; then
      execute mkdir -p "$SCRIPT_DIR/out"
    else
      err "No output directory, attempting to continue..."
    fi
  fi

  # Make sure we are using the absolute path
  OUT_DIR=$(realpath -s "$OUT_DIR")

  declare -ag PROJECTS=() # this array will hold all of the projects for this run

  parse_input "$@" # parse arguments with getopt

  # ((DEBUG)) && declare -p # when the going gets rough

  interactive_header "$@"
  
  for i in "${!PROJECTS[@]}"; do
    if ! sanitize_pn "${PROJECTS[$i]}"; then
      echo "Project name ${PROJECTS[$i]} is invalid"
      echo "Enter a replacement"
      ask_name "project" "ADD_PROJECTS" && unset "PROJECTS[i]" && PROJECTS+=("${ADD_PROJECTS[@]}")
    fi
  done

  # Exclude modules from --exclude
  for i in "${!MODULES[@]}"; do
    [[ " ${EXCLUDE_MODULES[*]} " =~ [[:space:]]${MODULES[i]}[[:space:]] ]] && unset "MODULES[$i]"
  done
    
  # Sanitize MODULES
  for i in "${!MODULES[@]}"; do
    if ! [[ " ${ALL_MODULES[*]} " =~ [[:space:]]${MODULES[i]}[[:space:]] ]]; then
      echo "Module ${MODULES[$i]} is not available, removing"
      read -r -p "Enter replacement module name: " module
      ! [[ " ${ALL_MODULES[*]} " =~ [[:space:]]${module}[[:space:]] ]] || { echo "RTFM"; return 1; }
      MODULES[i]="$module"
    fi
    unset module
  done

  # Sanitize wrappers
  for i in "${!WRAPPERS[@]}"; do
    IFS=',' read -ra args <<< "${WRAPPERS[$i]}" # load the wrapper and args
    if ! [[ " ${ALL_WRAPPERS[*]} " =~ [[:space:]]${args[0]}[[:space:]] ]]; then
      echo "Wrapper ${args[0]} is not available, removing"
      unset "WRAPPERS[$i]"
    fi
  done

  # If module equals install_dependencies run install_dependencies
  declare -gx R_LIBS_USER=${R_LIBS_USER:-"$HOME/R/$SCRIPT_NAME"}
  [[ ${MODULES[*]} == "install_dependencies" ]] && install_dependencies

  # Loop over projects
  for PROJECT in "${PROJECTS[@]}"; do
    declare -gx PROJECT
    declare -gx PROJECT_SCANS_DIR="$SCANS_DIR/$PROJECT"
    declare -gx PROJECT_DATE="${PROJECT:0:8}"  # extract the first 8 characters (e.g., "20241215")
    PROJECT_NO_DATE="${PROJECT:9}" # extract the part after the date and underscore
    declare -gx PROJECT_USER="${PROJECT_NO_DATE%%_*}" # strip suffix to get user (e.g., "username")
    declare -gx PROJECT_PREFIX="${PROJECT_DATE}_${PROJECT_USER}"
    declare -gx PROJECT_NAME="${PROJECT_NO_DATE#*_}" # Remove the username and following underscore (e.g., "nameof_project")
    declare -gx STUDIES_ARCHIVE_FILE="$OUT_DIR/StudiesDataArchive.txt"
    declare -gx PROJECT_RESULTS_DIR="$OUT_DIR/$PROJECT"
    declare -gx EASY_OUT_DIR="$PROJECT_RESULTS_DIR/easy"

    # Set the automatic project directory prefix
    declare -gx STUDY_PREFIX="${DATE}_${USER}" # reversed these so easier to sort and parse by date

    # Use more advanced function to set the correct results directories
    # This area could be streamlined and improved with better logic
    [[ -z $EASY_RESULTS_DIR ]] && handle_results_dir "$EASY_OUT_DIR" "EASY_RESULTS_DIR"
    [[ -z $STUDY_RESULTS_DIR ]] && handle_results_dir "$PROJECT_RESULTS_DIR" "STUDY_RESULTS_DIR"
    # [[ -z $STUDY_RESULTS_DIR ]] && declare -gx STUDY_RESULTS_DIR="$PROJECT_RESULTS_DIR/${STUDY_PREFIX}_${PROJECT_NAME}"
  
    declare -gx GTA_OUT_DIR="$STUDY_RESULTS_DIR/gta"
    declare -gx GTF_OUT_DIR="$STUDY_RESULTS_DIR/gtf"
    declare -gx STUDY_CONFIG_FILE="$STUDY_RESULTS_DIR/study_config"
    handle_config "$STUDY_CONFIG_FILE"

    debug "Project: $PROJECT"
    debug "Active modules: ${MODULES[*]}"
    debug "Active wrappers and their args: ${WRAPPERS[*]}"

    # Run selected modules
    for m in "${MODULES[@]}"; do
      if ask "Run $m module?"; then
        "$m" || return 1
      fi
    done

    # Run selected wrappers
    for wrapper in "${WRAPPERS[@]}"; do
      IFS=',' read -ra args <<< "$wrapper" # load the wrapper and args
      if ask "Run ${args[0]} wrapper with args ${args[*]:1}?"; then
        "${args[0]}" "${args[@]:1}" || return 1
      fi
    done
  done

  [[ ${#MODULES[@]} -gt 0 ]] && echo "Successfully ran module(s): ${MODULES[*]}"
  [[ ${#WRAPPERS[@]} -gt 0 ]] && echo "Successfully ran wrapper(s): ${WRAPPERS[*]}"
  [[ ${#PROJECTS[@]} -gt 0 ]] && echo "On project(s): ${PROJECTS[*]}"

  unset MODULES WRAPPERS EXCLUDE_MODULES STUDIES PARSED_CONFIG YES
}

# During development it's better to just exit
# (Safer) main loop for automatic rerun
# if main "$@"; then
#   for ((i=1; i<2; i++)); do
#     main &&
#     i=0
#   done
# fi

main "$@"
exit $?