Generalize REMc analysis
This commit is contained in:
@@ -43,7 +43,10 @@
|
||||
# TODO:
|
||||
# * Scripts should be made modular enough that they can be stored in the same dir
|
||||
# * Don't cd in scripts
|
||||
# * If you must, do it in a subshell at least!
|
||||
# * Pass variables
|
||||
# * Pass options
|
||||
# * Pass arguments
|
||||
# * Variable scoping is horrible right now
|
||||
# * I wrote this sequentially and tried to keep track the best I could
|
||||
# * Local vars have a higher likelihood of being lower case, global vars are UPPER
|
||||
@@ -250,6 +253,19 @@ random_words() {
|
||||
done
|
||||
printf "%s_" "${arr[@]}" | sed 's/_$//'
|
||||
}
|
||||
# @description Backup one or more files to an incremented .bk file
|
||||
backup() {
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
for f in "$@"; do
|
||||
[[ -f $f ]] || continue
|
||||
count=1
|
||||
while [[ -f $f.bk.$count ]]; do
|
||||
count=$((count+1))
|
||||
done
|
||||
debug "rsync -a $f $f.bk.$count"
|
||||
rsync -a "$f" "$f.bk.$count"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# @section Modules
|
||||
@@ -260,6 +276,8 @@ random_words() {
|
||||
# * Grouping multiple submodules (and modules) into a larger task
|
||||
# * Dictating the ordering of multiple submodules
|
||||
# * Modules should competently handle pushd and popd for their submodules if they do not reside in the SCANS/PROJECT_DIR
|
||||
# * Apps and submodules should avoid changing directories
|
||||
# * Pass input data from somewhere and output data somewhere
|
||||
|
||||
module install_dependencies
|
||||
# @section Install dependencies
|
||||
@@ -701,7 +719,6 @@ qhtcp() {
|
||||
(( next_study_num=next_study_num+1 ))
|
||||
done
|
||||
|
||||
# Now this is tricker than first appears
|
||||
# Use initials from project not whoami
|
||||
# Best I can do is first two letters of username
|
||||
# See TODO in markdown
|
||||
@@ -811,22 +828,33 @@ module remc
|
||||
# * Don't cd within scripts, it's confusing
|
||||
# * Use arguments to pass configuration variables
|
||||
# * This allows us to abstract the program away in script-run-workflow and treat it like a module
|
||||
# @arg $1 string studyInfo file
|
||||
remc() {
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
|
||||
# Enter REMc directory to run the scripts there
|
||||
pushd "$QHTCP_PROJECT_DIR/REMc" || return 1
|
||||
|
||||
# Run modules
|
||||
# If any modules fail the rest will not run, this is fundamental to module design
|
||||
# Remove trailing && to run regardless
|
||||
# If any submodules fail the rest will not run, this is fundamental to module design
|
||||
# Remove leading && to run regardless
|
||||
# TODO can this be
|
||||
r_join_interactions "$QHTCP_PROJECT_DIR/1-join_interactions" 2 "$1" "${@:2}" &&
|
||||
java_extract &&
|
||||
r_add_shift_values &&
|
||||
r_heat_maps_zscores &&
|
||||
r_heat_maps_homology &&
|
||||
popd || return 1
|
||||
r_join_interactions \
|
||||
"$QHTCP_PROJECT_DIR/out" # output directory
|
||||
2 \ % sd value
|
||||
"$1" # studyInfo file
|
||||
"${@:2}" \
|
||||
&& java_extract \
|
||||
"$QHTCP_PROJECT_DIR/out/" \
|
||||
&& r_add_shift_values \
|
||||
"$QHTCP_PROJECT_DIR/REMcRdy_lm_only.csv-finalTable.csv" \
|
||||
"$QHTCP_PROJECT_DIR/Shift_only.csv" \
|
||||
"$1" \
|
||||
"$QHTCP_PROJECT_DIR/REMcWithShift.csv" \
|
||||
&& r_create_heat_maps \
|
||||
"$QHTCP_PROJECT_DIR/REMcWithShift.csv" \
|
||||
"$QHTCP_PROJECT_DIR/out" \
|
||||
&& r_heat_maps_homology \
|
||||
"$QHTCP_PROJECT_DIR/REMcRdy_lm_only.csv-finalTable.csv" \
|
||||
"$APPS_DIR/r/170503_DAmPs_Only.txt" \
|
||||
"$APPS_DIR/r/Yeast_Human_Homology_Mapping_biomaRt_18_0920.csv" \
|
||||
"$QHTCP_PROJECT_DIR/out/homology"
|
||||
}
|
||||
|
||||
|
||||
@@ -835,22 +863,27 @@ module gtf
|
||||
# @description GTF module for QHTCP
|
||||
gtf() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
process_dir="GTF/Process"
|
||||
function_dir="GTF/Function"
|
||||
component_dir="GTF/Component"
|
||||
out_dir="REMcRdy_lm_only"
|
||||
process_dir="$QHTCP_PROJECT_DIR/out/gtf/process"
|
||||
function_dir="$QHTCP_PROJECT_DIR/out/gtf/function"
|
||||
component_dir="$QHTCP_PROJECT_DIR/out/gtf/component"
|
||||
out_dir="$QHTCP_PROJECT_DIR/out/gtf"
|
||||
|
||||
py_gtf_dcon "$process_dir" "$out_dir"
|
||||
py_gtf_dcon \
|
||||
"$process_dir" \
|
||||
"$out_dir"
|
||||
|
||||
# Perform operations in each directory in parallel
|
||||
# Perform operations on each directory in parallel
|
||||
for d in "$function_dir" "$component_dir"; do
|
||||
debug "rsync -a $process_dir/ $d/"
|
||||
rsync -a "$process_dir/" "$d/"
|
||||
done
|
||||
for d in "$process_dir" "$function_dir" "$component_dir"; do
|
||||
rsync -a "$process_dir/$out_dir" "$d"/
|
||||
out_file="${d##*/}Results.txt" # Use the dirname to create each Results filename
|
||||
pl_gtf "$d" "$out_dir" & # parallelize
|
||||
py_gtf_concat "$d" "$out_dir" "$out_file"
|
||||
done
|
||||
|
||||
r_compile_gtf
|
||||
r_compile_gtf "$out_dir"
|
||||
}
|
||||
|
||||
|
||||
@@ -1144,9 +1177,9 @@ submodule r_join_interactions
|
||||
r_join_interactions() {
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/r/joinInteractExps.R"
|
||||
debug "$RSCRIPT $script"
|
||||
debug "$RSCRIPT $script $*"
|
||||
"$RSCRIPT" "$script" "$@"
|
||||
local out_files=("REMcRdy_lm_only.csv" "Shift_only.csv" "parameters.csv")
|
||||
local out_files=("$1/REMcRdy_lm_only.csv" "$1/Shift_only.csv" "$1/parameters.csv")
|
||||
for f in "${out_files[@]}"; do
|
||||
[[ -f $f ]] || (echo "$f does not exist"; return 1)
|
||||
done
|
||||
@@ -1156,23 +1189,28 @@ r_join_interactions() {
|
||||
submodule java_extract
|
||||
# @description Jingyu's REMc java utility using file input file REMcRdy_lm_only.csv
|
||||
# and output REMcRdy_lm_only.csv-finalTable.csv
|
||||
# I'm not sure if the output dir is configurable so we can copy data around or push/pop
|
||||
# @arg $1 string The output directory
|
||||
java_extract() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
classpath="jingyuJava_1_7_extractLib.jar"
|
||||
out_file="REMcRdy_lm_only.csv-finalTable.csv"
|
||||
classpath="$APPS_DIR/java/javaExtract.jar"
|
||||
out_file="$1/REMcRdy_lm_only.csv-finalTable.csv"
|
||||
|
||||
# backup REMcRdy_lm_only.csv-finalTable.csv
|
||||
[[ -f $out_file ]] && mv "$out_file" "$out_file.bk"
|
||||
if ! backup "$out_file"; then
|
||||
ask "Backup of $out_file failed, continue?" || return 1
|
||||
fi
|
||||
|
||||
java_cmd=(
|
||||
"$JAVA" -Xms512m -Xmx2048m -Dfile.encoding=UTF-8 -classpath "$classpath" ExecMain
|
||||
"REMcRdy_lm_only.csv"
|
||||
"GeneByGOAttributeMatrix_nofiltering-2009Dec07.tab"
|
||||
"ORF_List_Without_DAmPs.txt" 1 true true
|
||||
"$QHTCP_PROJECT_DIR/REMcRdy_lm_only.csv"
|
||||
"$APPS_DIR/java/GeneByGOAttributeMatrix_nofiltering-2009Dec07.tab"
|
||||
"$APPS_DIR/java/ORF_List_Without_DAmPs.txt" 1 true true
|
||||
)
|
||||
|
||||
debug "${java_cmd[@]}"
|
||||
"${java_cmd[@]}"
|
||||
debug "pushd && ${java_cmd[*]} && popd"
|
||||
pushd "$1" && "${java_cmd[@]}" && popd || return 1
|
||||
|
||||
[[ -f $out_file ]] || (echo "$out_file does not exist"; return 1)
|
||||
}
|
||||
|
||||
@@ -1180,23 +1218,31 @@ java_extract() {
|
||||
submodule r_add_shift_values
|
||||
# @description Add shift values back to REMcRdy_lm_only.csv-finalTable.csv
|
||||
# and output "REMcWithShift.csv" for use with the REMc heat maps
|
||||
# @arg $1 string The output csv file REMcRdy_lm_only.csv-finalTable.csv
|
||||
# @arg $2 string Shift_only.csv
|
||||
# @arg $3 string REMcWithShift.csv
|
||||
# @arg $4 string The sd value
|
||||
r_add_shift_values() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
out_file="REMcHeatmaps/REMcWithShift.csv"
|
||||
debug "$RSCRIPT AddShiftVals2.R"
|
||||
"$RSCRIPT" AddShiftVals2.R
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/r/addShiftVals.R"
|
||||
out_file="$QHTCP_PROJECT_DIR/REMcWithShift.csv"
|
||||
debug "$RSCRIPT $script $*"
|
||||
"$RSCRIPT" "$script" "$@"
|
||||
rm -f "REMcHeatmaps/"*.pdf
|
||||
[[ -f $out_file ]] || (echo "$out_file does not exist"; return 1)
|
||||
}
|
||||
|
||||
|
||||
submodule r_heat_maps_zscores
|
||||
# @description Execute REMcHeatmaps_zscores.R
|
||||
r_heat_maps_zscores() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
out_file="REMcHeatmaps/compiledREMcHeatmaps.pdf"
|
||||
debug "$RSCRIPT REMcHeatmaps_zscores.R"
|
||||
"$RSCRIPT" REMcHeatmaps_zscores.R
|
||||
submodule r_create_heat_maps
|
||||
# @description Execute createHeatMaps.R
|
||||
# @arg $1 string The final shift table (REMcWithShift.csv)
|
||||
# @arg $2 string The output directory
|
||||
r_create_heat_maps() {
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/r/createHeatMaps.R"
|
||||
out_file="$QHTCP_PROJECT_DIR/compiledREMcHeatmaps.pdf"
|
||||
debug "$RSCRIPT $script $*"
|
||||
"$RSCRIPT" "$script" "$@"
|
||||
pdfs=(REMcHeatmaps/*.pdf)
|
||||
debug "pdftk ${pdfs[*]} output $out_file"
|
||||
pdftk "${pdfs[@]}" output "$out_file"
|
||||
@@ -1205,32 +1251,26 @@ r_heat_maps_zscores() {
|
||||
|
||||
|
||||
submodule r_heat_maps_homology
|
||||
# @description Execute REMcHeatmaps_Z_lm_wDAmPs_andHomology_221212.R
|
||||
# TODO this R script needs configurable output so we don't need to push/pop
|
||||
# @description Execute createHeatMapsAll.R
|
||||
# @arg $1 string The final shift table (REMcRdy_lm_only.csv-finalTable.csv)
|
||||
# @arg $2 string The (Shift_only.csv)
|
||||
# @arg $3 string The (Yeast_Human_Homology_Mapping_biomaRt_18_0920.csv)
|
||||
# @arg $4 string The output directory
|
||||
r_heat_maps_homology() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
work_dir="REMcHeatmapsWithHomology"
|
||||
source_file="REMcHeatmaps/REMcWithShift.csv"
|
||||
target_file="$work_dir/REMcWithShift.csv"
|
||||
out_file="$work_dir/Homology/compiledREMcHomologyHeatmaps.pdf"
|
||||
debug "rsync --archive $source_file $target_file"
|
||||
rsync --archive "$source_file" "$target_file"
|
||||
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/r/createHeatMapsHomology.R"
|
||||
out_file="$4/compiledREMcHomologyHeatmaps.pdf"
|
||||
|
||||
# Clean old output
|
||||
rm "$work_dir/Homology/"*.{pdf,csv}
|
||||
rm "$4/"*.{pdf,csv}
|
||||
|
||||
pushd "$work_dir" || return 1
|
||||
|
||||
# TODO This
|
||||
"$RSCRIPT" \
|
||||
REMcHeatmaps_Z_lm_wDAmPs_andHomology_221212.R \
|
||||
"$RSCRIPT" "$script" \
|
||||
REMcWithShift.csv \
|
||||
Homology \
|
||||
17_0503_DAmPs_Only.txt \
|
||||
"$APPS_DIR/r/170503_DAmPs_Only.txt" \
|
||||
Yeast_Human_Homology_Mapping_biomaRt_18_0920.csv
|
||||
popd || return 1
|
||||
|
||||
pdfs=("$work_dir"/Homology/*.pdf)
|
||||
pdfs=("$work_dir"/homology/*.pdf)
|
||||
pdftk "${pdfs[@]}" output "$out_file"
|
||||
|
||||
[[ -f $out_file ]] || (echo "$out_file does not exist"; return 1)
|
||||
@@ -1242,11 +1282,11 @@ submodule py_gtf_dcon
|
||||
# @arg $1 string Directory to process
|
||||
# @arg $2 string Output directory name
|
||||
py_gtf_dcon() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
in_file="REMcRdy_lm_only.csv-finalTable.csv"
|
||||
out_file="$1/$2/1-0-0-finaltable.csv"
|
||||
debug "$PYTHON DconJG2.py $in_file $1/"
|
||||
"$PYTHON" DconJG2.py "$in_file" "$1/"
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/python/DconJG2.py"
|
||||
debug "$PYTHON $SCRIPT $1 $2/"
|
||||
"$PYTHON" "$SCRIPT" "$1" "$2/"
|
||||
out_file="$2/1-0-0-finaltable.csv"
|
||||
[[ -f $out_file ]] || (echo "$out_file does not exist"; return 1)
|
||||
}
|
||||
|
||||
@@ -1257,21 +1297,16 @@ submodule pl_gtf
|
||||
# @arg $2 string Output directory name to look for txt files
|
||||
pl_gtf() {
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
|
||||
pushd "$1" || return 1
|
||||
set1="ORF_List_Without_DAmPs.txt"
|
||||
set1="$APPS_DIR/perl/ORF_List_Without_DAmPs.txt"
|
||||
shopt -s nullglob
|
||||
set2=("$2"/*.txt) # glob them all
|
||||
shopt -u nullglob
|
||||
|
||||
for s2 in "${set2[@]}"; do
|
||||
debug "pl_analyze $set1 $s2"
|
||||
debug "pl_gtf_analyze $set1 $s2"
|
||||
pl_gtf_analyze "$set1" "$s2"
|
||||
debug "pl_terms2tsv $s2"
|
||||
pl_gtf_terms2tsv "$s2"
|
||||
done
|
||||
|
||||
popd || return 1
|
||||
}
|
||||
|
||||
|
||||
@@ -1284,21 +1319,22 @@ submodule pl_gtf_analyze
|
||||
# @arg $2 string Set 2 TODO naming
|
||||
pl_gtf_analyze() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
script="analyze_v2.pl"
|
||||
an="gene_association.sgd"
|
||||
out_file="gene_ontology_edit.obo"
|
||||
debug "$PERL $script -an $an -as P -o $out_file -b $1 $2"
|
||||
"$PERL" "$script" -an "$an" -as P -o "$out_file" -b "$1" "$2"
|
||||
script="$APPS_DIR/perl/analyze_v2.pl"
|
||||
an="$APPS_DIR/perl/gene_association.sgd"
|
||||
obo="$APPS_DIR/perl/gene_ontology_edit.obo"
|
||||
debug "$PERL $script -an $an -as P -o $obo -b $1 $2"
|
||||
"$PERL" "$script" -an "$an" -as P -o "$obo" -b "$1" "$2"
|
||||
}
|
||||
|
||||
|
||||
submodule pl_gtf_terms2tsv
|
||||
# @description Perl terms2tsv submodule
|
||||
# Probably should be translated to shell/python
|
||||
#
|
||||
# @arg $1 string Terms file TODO naming
|
||||
pl_gtf_terms2tsv() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
script="terms2tsv_v4.pl"
|
||||
debug "Running: ${FUNCNAME[0]}" "$@"
|
||||
script="$APPS_DIR/perl/terms2tsv.pl"
|
||||
debug "$PERL $script $1.terms > $1.tsv"
|
||||
"$PERL" "$script" "$1.terms" > "$1.tsv"
|
||||
}
|
||||
@@ -1313,23 +1349,21 @@ submodule py_gtf_concat
|
||||
# @arg $3 string output file
|
||||
py_gtf_concat() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
pushd "$1" || return 1
|
||||
script="Concatenate_GTF_results.py"
|
||||
script="$APPS_DIR/python/concatGTFResults.py"
|
||||
debug "$PYTHON $script $2/ $3"
|
||||
"$PYTHON" "$script" "$2/" "$3"
|
||||
[[ -f $3 ]] || (echo "$3 does not exist"; return 1)
|
||||
popd || return 1
|
||||
}
|
||||
|
||||
|
||||
submodule r_compile_gtf
|
||||
# @description Compile GTF in R
|
||||
# @noargs
|
||||
# @arg $1 string gtf output directory
|
||||
r_compile_gtf() {
|
||||
debug "Running: ${FUNCNAME[0]}"
|
||||
script="CompileGTF.R"
|
||||
debug "$RSCRIPT $script"
|
||||
"$RSCRIPT" "$script"
|
||||
script="$APPS_DIR/r/CompileGTF.R"
|
||||
debug "$RSCRIPT $script $1"
|
||||
"$RSCRIPT" "$script" "$1"
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user