bryan/hartman-server
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More EASYconsole work

Browse Source
This commit is contained in:
Bryan Roessler
2024-07-30 12:29:13 -04:00
parent 50914e93c7
commit fe9b338324
6 changed files with 263 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
workflow/apps/matlab/easy/DMPexcel2mat.m
View File

@@ -196,7 +196,6 @@ if ~exist(drugMediaFile, 'file') || isempty(drugMediaFile)
sortedFiles=dmFiles{sortedIndices};
drugMediaFile=sortedFiles{1};
fprintf('Using newest DrugMedia file: %s, skipping directory selection\n', drugMediaFile);
end
catch Me
% This can be silent, not really an error

10
workflow/apps/matlab/easy/EASYconsole.m
View File

@@ -65,10 +65,10 @@ function varargout = EASYconsole(varargin)
fprintf('WARNING: Running in standalone mode without PROJECT or PROJECT_SCANS_DIR environment variables (not recommended)\n');
fprintf('Beginning parent scans directory search\n');
dirsToScan={
fullfile(parentDir, '..', '..', 'scans'),
fullfile(parentDir, '..', '..', 'ExpJobs'),
fullfile('mnt','data','scans'),
fullfile('mnt','data', 'ExpJobs'),
fullfile(parentDir, '..', '..', 'scans')
fullfile(parentDir, '..', '..', 'ExpJobs')
fullfile('mnt','data','scans')
fullfile('mnt','data', 'ExpJobs')
fullfile(parentDir, '..', '..', 'templates', 'scans-demo')
};
for i=1:length(dirsToScan)
@@ -78,7 +78,7 @@ function varargout = EASYconsole(varargin)
pattern='^\d{6}_.*_.*';
matchedDirs={dirs(regexp(subDirs.name, pattern)).name};
if ~isempty(matchedDirs)
fprintf('Found a non-empty scans directory in our list: %s\n', char(d);
fprintf('Found a non-empty scans directory in our list: %s\n', char(d));
fprintf('Setting scansDir to %s\n', char(d));
scansDir=fullfile(d);
end

2
workflow/apps/matlab/ezview/EZinterAgingDev0.m
View File

@@ -70,7 +70,7 @@ function EZinterAgingDev0
Xpos=Xln{m,j}- (Rn(j)+Rs(j));
deltaXR{m,j}=zeros(1,384);
for i=1:length(Xpos(:)),
for i=1:length(Xpos(:))
% deltaXR{m,j}(i)=Xpos(i);
if deltaXR{m,j}(i)==0
try

20
workflow/apps/matlab/ezview/EZvInitLoad.m
View File

@@ -5,6 +5,7 @@ global Expbb
global userPars
global usrExpJobsDir
global zonePB
global mpdmFile
% Initialize some variables from matlab
ezPath=which(mfilename);
@@ -22,7 +23,8 @@ end
% Set scansDir (project scans directory) intelligently
if exist('PROJECT', 'var') && ~isempty(getenv('PROJECT'))
scansDir=getenv('PROJECT');
project=getenv('PROJECT');
fprintf('Using project path: %s from environment variable PROJECT\n', scansDir);
disp('This usually indicates that we are in standalone mode');
elseif exist('PROJECT_SCANS_DIR', 'var') && ~isempty(getenv('PROJECT_SCANS_DIR'))
@@ -76,6 +78,19 @@ else
fprintf('Using EZview script directory: %s from hardcoded default\n', ezDir);
end
% Allow module to override hardcoded default EASY directory
if exist('EASY_DIR','var') && ~isempty(getenv('EASY_DIR'))
EASY_DIR=fullfile(getenv('EASY_DIR'));
if ~strcmp(easyDir, EASY_DIR) % sanity check
disp("WARNING: EASY_DIR does not match this script's hardcoded EASY location");
disp("This is probably OK but if strange beahvior arises, we'll need to fix it in code");
easyDir=EASY_DIR;
end
fprintf('Using EASY script directory: %s from environment variable EASY_DIR\n', easyDir);
else
fprintf('Using EASY script directory: %s from hardcoded default\n', easyDir);
end
userPars.BPdefault={'on','on','0.3','8','45','1000'};
userPars.boxplotFlg=1;
userPars.BPoutliers='On';
@@ -90,9 +105,6 @@ userPars.kfiltLim=str2double(userPars.BPdefault(6));
% Generate the Exp arrays
numExps=3;
for i=1:numExps

5
workflow/apps/matlab/ezview/EZviewGui.m
View File

@@ -20,11 +20,6 @@ function varargout = EZviewGui(varargin)
%
% See also: GUIDE, GUIDATA, GUIHANDLES
gui_Singleton = 0;
gui_State = struct( ...
'gui_Name', mfilename, ...

252
workflow/qhtcp-workflow
View File

@@ -586,32 +586,21 @@ easy() {
# TODO will need to play with the -sd startup option to see what works (well)
# Skip this step altogether in auto mode since it requires graphical interaction
if ! ((YES)) && ask "Start EASY in MATLAB? This requires a GUI."; then
export PROJECT_SCANS_DIR PROJECT_DATE EASY_DIR PROJECT_USER EASY_NAME EASY_SUFFIX EASY_RESULTS_DIR MASTER_PLATE_FILE DRUG_MEDIA_FILE
echo "Hit enter to use the default EASY results directory $PROJECT_SCANS_DIR/Results_${PROJECT_PREFIX}_($PROJECT_SUFFIX)"
(( YES )) || read -r -p "Or enter a custom suffix: " suffix
EASY_RESULTS_DIR="$PROJECT_SCANS_DIR/Results_${PROJECT_PREFIX}_${suffix:-$PROJECT_SUFFIX}"
while [[ -d $EASY_RESULTS_DIR ]]; do
count=1
if [[ $count -eq 1 ]] && ask "$EASY_RESULTS_DIR already exists, increment dir name (y) or overwrite (N)?"; then
EASY_SUFFIX="${suffix:-$PROJECT_SUFFIX}.$((count++))"
EASY_RESULTS_DIR="$PROJECT_SCANS_DIR/Results_${PROJECT_PREFIX}_${EASY_SUFFIX}"
else
EASY_SUFFIX=${suffix:-$PROJECT_SUFFIX}
fi
done
EASY_SUFFIX=${EASY_SUFFIX:-$suffix:-$PROJECT_SUFFIX}
EASY_NAME="${PROJECT_PREFIX}_${EASY_SUFFIX}"
EASY_RESULTS_DIR="$PROJECT_SCANS_DIR/Results_$EASY_NAME"
[[ -d $PROJECT_SCANS_DIR/MasterPlateFiles ]] || mkdir -p "$PROJECT_SCANS_DIR/MasterPlateFiles"
DRUG_MEDIA_FILE="$PROJECT_SCANS_DIR/MasterPlateFiles/DrugMedia_$PROJECT.xls"
MASTER_PLATE_FILE="$PROJECT_SCANS_DIR/MasterPlateFiles/MasterPlate_$PROJECT.xls"
export PROJECT_SCANS_DIR PROJECT_DATE PROJECT_USER PROJECT
export EASY_DIR EASY_NAME EASY_SUFFIX EASY_RESULTS_DIR
export MASTER_PLATE_FILE DRUG_MEDIA_FILE
choose_easy_results_dir
DRUG_MEDIA_FILE="$EASY_RESULTS_DIR/DrugMedia_$PROJECT.xls"
MASTER_PLATE_FILE="$EASY_RESULTS_DIR/MasterPlate_$PROJECT.xls"
# Make EASY dirs
debug "mkdir -p $EASY_RESULTS_DIR"
mkdir -p "$EASY_RESULTS_DIR"
dirs=('PrintResults' 'CFfigs' 'Fotos' 'Fotos/BkUp' 'matResults')
for d in "${dirs[@]}"; do
debug "mkdir $EASY_RESULTS_DIR/$d"
mkdir "$EASY_RESULTS_DIR/$d"
done
@@ -619,13 +608,12 @@ easy() {
rsync -a "$EASY_DIR"/{figs,Ptmats} "$EASY_RESULTS_DIR"
# Launch matlab
matlab -nosplash -r "run $script"
# matlab -nosplash -sd "$PROJECT_SCANS_DIR" -r "run $script"
matlab -nosplash -r "run $script"
fi
# Use the function return code see if we succeeded
get_easy_results "$PROJECT_SCANS_DIR" || return 1
scan_easy_results "$PROJECT_SCANS_DIR" || (err "No EASY results found in $PROJECT_SCANS_DIR"; return 1)
}
@@ -635,19 +623,17 @@ module ezview
ezview() {
debug "Running: ${FUNCNAME[0]}"
export PROJECT_SCANS_DIR PROJECT_DATE PROJECT_USER EZVIEW_DIR
export EASY_RESULTS_DIR
EZVIEW_DIR="$APPS_DIR/matlab/ezview"
script="$EZVIEW_DIR/EZviewGui.m"
# Try to load EASY output dir fromENV or easy module
# Then scan for EASY results
[[ -z $EASY_RESULTS_DIR ]] && choose_easy_results_dir
# Make EZview dirs
debug "mkdir -p $EZVIEW_DIR"
matlab -nosplash -r "run $script"
echo "$EZVIEW_DIR"
}
@@ -714,12 +700,7 @@ module qhtcp
qhtcp() {
debug "Running: ${FUNCNAME[0]}"
OUT_DIR="/mnt/data/out/qhtcp"
STUDIES_ARCHIVE_FILE="$OUT_DIR/StudiesDataArchive.txt"
QHTCP_PROJECT_DIR="$OUT_DIR/$PROJECT"
QHTCP_TEMPLATE_DIR="$TEMPLATES_DIR/qhtcp"
STUDY_TEMPLATE_DIR="$TEMPLATES_DIR/exp"
STUDY_INFO_FILE="$QHTCP_PROJECT_DIR/StudyInfo.csv"
if [[ -d $QHTCP_PROJECT_DIR ]]; then
echo "A project already exists at $QHTCP_PROJECT_DIR"
@@ -790,27 +771,8 @@ qhtcp() {
unset STUDY_DIR
# Replacing ExpFrontend.m
get_easy_results "$PROJECT_SCANS_DIR" || (err "No EASY results found in $PROJECT_SCANS_DIR"; return 1)
choose_easy_results_dir
# Set the right results directory
# TODO eventually we could run this on multiple results dirs simultaneously with some refactoring
if ((YES)) || [[ ${#EASY_RESULTS_DIRS[@]} -eq 1 ]]; then
# In automatic mode just choose the first OUT DIR in the list
EASY_RESULT_DIR="${EASY_RESULTS_DIRS[0]}" # TODO right now just choose the frist one
else
echo "Multiple EASY results dirs found in $PROJECT_SCANS_DIR"
echo "Here is a list: "
for (( i=0; i<${#EASY_RESULTS_DIRS[@]}; i++ )); do
printf "%d. %s\n" "$((i+1))" "${EASY_RESULTS_DIRS[i]}"
done
printf "%s\n" "${EASY_RESULTS_DIRS[@]}"
read -r -p "Enter the item number to select: " response
response=$(( response - 1 )) # bash arrays use zero indexing
EASY_RESULTS_DIR="${EASY_RESULTS_DIRS[$response]}"
fi
# TODO probably broken
EASY_RESULTS_FILES=("$EASY_RESULTS_DIR/"*"/PrintResults/!!"*)
# Create studies archive file if missing
if ! [[ -d $STUDIES_ARCHIVE_FILE ]]; then
header=(StudyDate tStudyName StudyPath ExpNum ExpDate ExpPath ResultFile)
@@ -823,7 +785,7 @@ qhtcp() {
# Trying to match old ExpFrontend formatting
printf "%s\t" \
"${DATE//_/}" "$PROJECT" "$QHTCP_PROJECT_DIR" "Exp$s" \
"$PROJECT_DATE" "$PROJECT_SCANS_DIR" "$EASY_RESULT_DIR" "${f##*/}" \
"$PROJECT_DATE" "$PROJECT_SCANS_DIR" "$EASY_RESULTS_DIR" "${f##*/}" \
>> "$STUDIES_ARCHIVE_FILE"
done
done
@@ -1126,43 +1088,43 @@ r_gta_heatmaps() {
submodule mat_exp_frontend
# @description Run the ExpFrontend.m program
# This submodule:
# * Pushes into the Study template directory (ExpTemplate)
# * Prompts the user to run ExpFrontend.m
# * Pops out
# NOTES:
# * ExpFrontend.m should be or is being rewritten
mat_exp_frontend() {
debug "Running: ${FUNCNAME[0]}"
cat <<-EOF
ExpFrontend.m was made for recording into a spreadsheet
('StudiesDataArchive.txt') the date and files used (i.e., directory paths to the
!!Results files used as input for Z-interaction script) for each multi-experiment study.
# submodule mat_exp_frontend
# # @description Run the ExpFrontend.m program
# # This submodule:
# # * Pushes into the Study template directory (ExpTemplate)
# # * Prompts the user to run ExpFrontend.m
# # * Pops out
# # NOTES:
# # * ExpFrontend.m should be or is being rewritten
# mat_exp_frontend() {
# debug "Running: ${FUNCNAME[0]}"
# cat <<-EOF
# ExpFrontend.m was made for recording into a spreadsheet
# ('StudiesDataArchive.txt') the date and files used (i.e., directory paths to the
# !!Results files used as input for Z-interaction script) for each multi-experiment study.
Run the front end MATLAB programs in the correct order (e.g., run front end in "exp1"
folder to call the !!Results file for the experiment you named as exp1 in the StudyInfo.csv file)
The GTA and pairwise, TSHeatmaps, JoinInteractions and GTF Heatmap scripts use this table
to label results and heatmaps in a meaningful way for the user and others.
The BackgroundSD and ZscoreJoinSD fields will be filled automatically according to user
specifications, at a later step in the QHTCP study process.
# Run the front end MATLAB programs in the correct order (e.g., run front end in "exp1"
# folder to call the !!Results file for the experiment you named as exp1 in the StudyInfo.csv file)
# The GTA and pairwise, TSHeatmaps, JoinInteractions and GTF Heatmap scripts use this table
# to label results and heatmaps in a meaningful way for the user and others.
# The BackgroundSD and ZscoreJoinSD fields will be filled automatically according to user
# specifications, at a later step in the QHTCP study process.
COpen MATLAB and in the application navigate to each specific /Exp folder,
call and execute ExpFrontend.m by clicking the play icon.
Use the "Open file" function from within Matlab.
Do not double-click on the file from the directory.
When prompted, navigate to the ExpJobs folder and the PrintResults folder within the correct job folder.
Repeat this for every Exp# folder depending on how many experiments are being performed.
The Exp# folder must correspond to the StudyInfo.csv created above.
EOF
# COpen MATLAB and in the application navigate to each specific /Exp folder,
# call and execute ExpFrontend.m by clicking the play icon.
# Use the "Open file" function from within Matlab.
# Do not double-click on the file from the directory.
# When prompted, navigate to the ExpJobs folder and the PrintResults folder within the correct job folder.
# Repeat this for every Exp# folder depending on how many experiments are being performed.
# The Exp# folder must correspond to the StudyInfo.csv created above.
# EOF
script="ExpFrontend.m"
if ! ((YES)) &&
ask "Start MATLAB to run $script? This requires a GUI."; then
matlab -nosplash -r "$script"
fi
}
# script="ExpFrontend.m"
# if ! ((YES)) &&
# ask "Start MATLAB to run $script? This requires a GUI."; then
# matlab -nosplash -r "$script"
# fi
# }
submodule r_interactions
@@ -1417,37 +1379,76 @@ get_studies() {
}
submodule get_easy_results # lol
# @description Gets info about EASY output in the project's scans directory
submodule choose_easy_results_dir #
# @description Chooses an EASY scans directory if the information is undefined
# TODO: Standardize EASY output, it's hard to understand
# @exitcode 0 if at least one results directory exists
# @exitcode 1 if no results directories exist
# @set EASY_RESULTS_DIRS array Globbed results files from PROJECT_SCANS_DIR/
# @set EASY_RESULTS_FILES array Globbed results files from PROJECT_SCANS_DIR/
# @arg $1 string Project scans
get_easy_results() {
# TODO eventually we could run this on multiple results dirs simultaneously with some refactoring
# @exitcode 0 if successfully choose an EASY results dir
# @set EASY_RESULTS_DIR string The working EASY output directory
choose_easy_results_dir() {
debug "Running: ${FUNCNAME[0]}"
# Glob EASY output dirs
shopt -s nullglob
EASY_RESULTS_DIRS=( "$1"/Results*/ ) # TODO this is probably broken
[[ ${#EASY_RESULTS_DIRS} -ge 1 ]] || (echo "Missing EASY output"; return 1)
shopt -u nullglob
declare -g EASY_RESULTS_DIR EASY_OUT_DIR
# This section is mostly to provide some example globbing
declare -a EASY_RESULTS_ARRAY=()
for EASY_RESULTS_DIR in "${EASY_RESULTS_DIRS[@]}"; do
[[ -d $EASY_RESULTS_DIR ]] && echo "Found EASY Results directory: $EASY_RESULTS_DIR"
EASY_PRINT_RESULTS_DIR="$EASY_RESULTS_DIR/PrintResults"
[[ -d $EASY_PRINT_RESULTS_DIR ]] && echo "Found EASY PrintResults directory: $EASY_PRINT_RESULTS_DIR"
EASY_PRINT_RESULTS_FILES=(
"$EASY_PRINT_RESULTS_DIR/!!"*
"$EASY_PRINT_RESULTS_DIR"/NoGrowth_*.txt
"$EASY_PRINT_RESULTS_DIR"/GrowthOnly_*.txt
)
EASY_RESULTS_ARRAY+=("$EASY_RESULTS_DIR" "$EASY_PRINT_RESULTS_DIR" "${EASY_PRINT_RESULTS_FILES[@]}")
EASY_OUT_DIR="$QHTCP_PROJECT_DIR/easy/"
# Don't override env
if [[ -n $EASY_RESULTS_DIR && ! -d $EASY_RESULTS_DIR ]]; then
mkdir -p "$EASY_RESULTS_DIR"
return 0
fi
local default_easy_results_dir="$EASY_OUT_DIR/$PROJECT"
if [[ -n $PROJECT ]]; then
if [[ ! -d $default_easy_results_dir ]]; then
EASY_RESULTS_DIR="$default_easy_results_dir"
mkdir -p "$EASY_RESULTS_DIR"
return 0
fi
fi
# Always backup existing output
# This would happen if you ran the same experiment twice in one day, for instance
if [[ -d $default_easy_results_dir ]] ; then
backup_dir="$default_easy_results_dir"
while [[ -d $default_easy_results_dir ]]; do
count=1
backup_dir="$backup_dir.$((count++))"
done
[[ ${#EASY_RESULTS_DIRS[@]} -gt 0 ]] # TODO may need to change this to something more specific
echo "Backing up existing output from $default_easy_results_dir to $backup_dir"
debug "rsync -a $default_easy_results_dir $backup_dir"
rsync -a "$default_easy_results_dir" "$backup_dir" &&
EASY_RESULTS_DIR="$default_easy_results_dir"
fi
# echo "Hit enter to use the default EASY results directory: $default_easy_results_dir"
# if ! (( YES )); then
# read -r -p "Or enter a custom directory name, example: $PROJECT" dirname
# [[ -z $dirname ]] && EASY_RESULTS_DIR="$default_easy_results_dir" && return 0
# fi
# ((YES)) && return 0
# # Let's get a little fancy
# shopt -s nullglob
# declare -la easy_results_dirs=( "$easy_out_dir/"*/ )
# shopt -u nullglob
# # Sort the dirs
# mapfile -t easy_results_dirs < <(printf '%s\n' "${easy_results_dirs[@]}" | sort)
# last_index=$(( ${#easy_results_dirs} - 1 ))
# ((YES)) && EASY_RESULTS_DIR="${easy_results_dirs[$last_index]}" && return 0
# echo "Multiple EASY results dirs found in $PROJECT_SCANS_DIR"
# echo "Here is a list: "
# for i in "${!easy_results_dirs[@]}"; do
# printf "%d. %s\n" "$((i+1))" "${easy_results_dirs[i]}"
# done
# printf "%s\n" "${easy_results_dirs[@]}"
# last_index=$(( ${#easy_results_dirs} - 1 ))
# read -r -p "Enter the item number to select EASY results directory, default ($last_index): " response
# [[ -z $response ]] && response=$last_index
# response=$(( response - 1 )) # bash arrays use zero indexing
# EASY_RESULTS_DIR="${easy_results_dirs[$response]}"
# EASY_RESULTS_FILES=("$EASY_RESULTS_DIR/"*"/PrintResults/!!"*)
# [[ ${#easy_results_dirs[@]} -gt 0 ]]
}
@@ -1482,13 +1483,20 @@ main() {
SCRIPT_DIR=$(dirname "$SCRIPT")
# Global directory variables
local out_heirarchy=("$OUT_DIR" "$SCRIPT_DIR/out" "/mnt/data/out")
for d in "${out_heirarchy[@]}"; do
if [[ -d $d ]]; then
debug "Using output directory: $d"
OUT_DIR="$d"
fi
done
TEMPLATES_DIR="$SCRIPT_DIR/templates"
APPS_DIR="$SCRIPT_DIR/apps"
DATE="$(date +%Y%m%d)" # change in EASYConsole.m to match
SCANS_HEIRARCHY=("./scans" "/mnt/data/scans" "/mnt/data/ExpJobs" "./scans")
SCANS_HEIRARCHY=("./scans" "/mnt/data/scans" "/mnt/data/ExpJobs" "./scans") # search for a scans dir in this order (and fall back to default)
# scans_heirarchy=("./scans" "/mnt/data/scans" "/mnt/data/ExpJobs" "./scans")
local scans_heirarchy=( "$SCANS_DIR" "./scans" "/mnt/data/scans" "./scans") # TODO change back for production, avoid actual scan dirs during testing
# Find a scans directory
[[ -z $SCANS_DIR ]] && for d in "${SCANS_HEIRARCHY[@]}"; do
[[ -z $SCANS_DIR ]] && for d in "${scans_heirarchy[@]}"; do
if [[ -d $d ]]; then
SCANS_DIR="$d"
fi
@@ -1556,10 +1564,16 @@ main() {
# Loop over projects
for PROJECT in "${PROJECTS[@]}"; do
declare -g PROJECT
PROJECT_SCANS_DIR="$SCANS/$PROJECT"
PROJECT_DATE="${PROJECT%"${PROJECT#????????}"}" # e.g. 20240723
PROJECT_SUFFIX="${PROJECT#????????_*_}"
PROJECT_USER="${PROJECT#????????_}"; PROJECT_USER="${PROJECT_USER%%_*}"
STUDIES_ARCHIVE_FILE="$OUT_DIR/StudiesDataArchive.txt"
QHTCP_PROJECT_DIR="$OUT_DIR/$PROJECT"
QHTCP_TEMPLATE_DIR="$TEMPLATES_DIR/qhtcp"
STUDY_TEMPLATE_DIR="$TEMPLATES_DIR/exp"
STUDY_INFO_FILE="$QHTCP_PROJECT_DIR/StudyInfo.csv"
# Run selected modules
for m in "${MODULES[@]}"; do
ask "Run $m?" && "$m"