CI/physmon/phys_perf_mon.sh

0001 #!/bin/bash
0002
0003 set -e
0004
0005 # helper function to selectively print and run commands without a subshell
0006 function run() {
0007     set -x
0008     "$@"
0009     # save exit code
0010     { rec=$?; } 2> /dev/null
0011     { set +x;   } 2> /dev/null
0012     # restore exit code
0013     (exit $rec)
0014 }
0015
0016 export run
0017
0018
0019 shopt -s extglob
0020
0021
0022 mode=${1:-all}
0023 if ! [[ $mode = @(all|kalman|gsf|fullchains|vertexing|simulation) ]]; then
0024     echo "Usage: $0 <all|kalman|gsf|fullchains|vertexing|simulation> (outdir)"
0025     exit 1
0026 fi
0027
0028 outdir=${2:-physmon}
0029 mkdir -p $outdir
0030
0031 refdir=CI/physmon/reference
0032 refcommit=$(cat $refdir/commit)
0033 commit=$(git rev-parse --short HEAD)
0034 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}"  )" &> /dev/null && pwd  )
0035
0036 # File to accumulate the histcmp results
0037 histcmp_results=$outdir/histcmp_results.csv
0038 echo -n "" > $histcmp_results
0039
0040 memory_dir=${outdir}/memory
0041 mkdir -p "$memory_dir"
0042
0043 if [ "$(uname)" == "Darwin" ]; then
0044     function measure {
0045         label=$1
0046         slug=$2
0047         shift
0048         shift
0049         echo "Measure Darwin $label ($slug)" >&2
0050         tmp=$(mktemp)
0051         echo "+ $@" >&2
0052         /usr/bin/time -l -o "$tmp" "$@"
0053         # save exit code
0054         rec=$?
0055
0056         of="${memory_dir}/mem_${slug}.csv"
0057         {
0058             echo "# spyral-label: $label"
0059             echo "# spyral-cmd: $*"
0060             echo "time,rss,vms"
0061             # in bytes
0062             grep -E "real" "$tmp" | awk '{printf $1}'
0063             printf ","
0064             grep -E "maximum resident set size" "$tmp" | awk '{printf $1}'
0065             printf ",0\n"
0066         } > "$of"
0067         # restore exit code
0068         (exit $rec)
0069     }
0070     export measure
0071 elif [ "$(uname)" == "Linux" ]; then
0072     function measure {
0073         label=$1
0074         slug=$2
0075         shift
0076         shift
0077         echo "Measure Linux $label ($slug)" >&2
0078         tmp=$(mktemp)
0079         echo "+ $@" >&2
0080         /usr/bin/time -v -o  "$tmp" "$@"
0081         # save exit code
0082         rec=$?
0083         # in kbytes
0084         max_rss=$(grep "Maximum resident set size (kbytes):" "$tmp" | awk '{printf $(NF)}')
0085         max_rss=$(( 1000*max_rss ))
0086         wall_time=$(grep "Elapsed (wall clock)" "$tmp" | awk '{printf $(NF)}')
0087         echo $max_rss
0088         wall_time=$(python3 -c "i='${wall_time}';p=i.split(':');p = p if len(p) == 3 else ['0', *p];t=float(p[0])*60*60 + float(p[1])*60 + float(p[2]);print(t)")
0089         echo $wall_time
0090
0091         of="${memory_dir}/mem_${slug}.csv"
0092         {
0093             echo "# spyral-label: $label"
0094             echo "# spyral-cmd: $*"
0095             echo "time,rss,vms"
0096             echo "${wall_time},${max_rss},0"
0097         } > "$of"
0098         # restore exit code
0099         (exit $rec)
0100     }
0101     export measure
0102 else
0103     function measure {
0104         echo "Not measuring because unknown environment"
0105         shift
0106         shift
0107         "$@"
0108     }
0109     export measure
0110 fi
0111
0112 if [ -n "$CI" ]; then
0113     echo "CI mode, do not abort immediately on failure"
0114     set +e
0115 fi
0116 ec=0
0117
0118 source $SCRIPT_DIR/setup.sh
0119
0120 function run_physmon_gen() {
0121     title=$1
0122     slug=$2
0123
0124     script=CI/physmon/workflows/physmon_${slug}.py
0125
0126     measure "$title" "$slug" ${script} $outdir 2>&1 > $outdir/run_${slug}.log
0127
0128     this_ec=$?
0129     ec=$(($ec | $this_ec))
0130
0131     if [ $this_ec -ne 0 ]; then
0132       echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec"
0133     else
0134       echo "::notice::✅ Dataset generation succeeded: ${script}"
0135     fi
0136 }
0137
0138 echo "::group::Generate validation dataset"
0139 if [[ "$mode" == "all" || "$mode" == "kalman" ]]; then
0140     run_physmon_gen "Truth Tracking KF" "truth_tracking_kalman"
0141 fi
0142 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0143     run_physmon_gen "Truth Tracking GSF" "truth_tracking_gsf"
0144 fi
0145 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0146     run_physmon_gen "Truth Tracking GX2F" "truth_tracking_gx2f"
0147 fi
0148 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0149     run_physmon_gen "CKF Tracking" "ckf_tracking"
0150     run_physmon_gen "Track finding ttbar" "track_finding_ttbar"
0151
0152 fi
0153 if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then
0154     run_physmon_gen "Vertexing" "vertexing"
0155 fi
0156 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0157     run_physmon_gen "Simulation" "simulation"
0158 fi
0159 echo "::endgroup::"
0160
0161
0162 function run_histcmp() {
0163     a=$1
0164     b=$2
0165     title=$3
0166     slug=$4
0167     shift 4
0168
0169     echo "::group::Comparing $a vs. $b"
0170
0171     if [ ! -f "$a" ]; then
0172       echo "::error::histcmp failed: File $a does not exist"
0173       ec=1
0174     fi
0175
0176     if [ ! -f "$b" ]; then
0177       echo "::error::histcmp failed: File $b does not exist"
0178       ec=1
0179     fi
0180
0181     run histcmp $a $b \
0182         --label-reference=reference \
0183         --label-monitored=monitored \
0184         --title="$title" \
0185         -o $outdir/${slug}.html \
0186         -p $outdir/${slug}_plots \
0187         "$@"
0188
0189     this_ec=$?
0190     ec=$(($ec | $this_ec))
0191
0192     if [ $this_ec -ne 0 ]; then
0193       echo "::error::histcmp failed (${slug}): ec=$this_ec"
0194     fi
0195
0196     echo "\"${title}\",${slug},${this_ec}" >> $histcmp_results
0197
0198     echo "::endgroup::"
0199 }
0200
0201 function full_chain() {
0202     suffix=$1
0203
0204     config="CI/physmon/ckf_${suffix}.yml"
0205
0206     if [ ! -f "$config" ]; then
0207         config="CI/physmon/default.yml"
0208     fi
0209     echo $config
0210
0211     if [ $suffix != truth_smeared ]; then
0212       run_histcmp \
0213         $outdir/performance_seeding_${suffix}.root \
0214         $refdir/performance_seeding_${suffix}.root \
0215         "Seeding ${suffix}" \
0216         seeding_${suffix} \
0217         -c $config
0218     fi
0219
0220     run_histcmp \
0221         $outdir/performance_ckf_${suffix}.root \
0222         $refdir/performance_ckf_${suffix}.root \
0223         "CKF ${suffix}" \
0224         ckf_${suffix} \
0225         -c $config
0226
0227     run Examples/Scripts/generic_plotter.py \
0228         $outdir/performance_ivf_${suffix}.root \
0229         vertexing \
0230         $outdir/performance_ivf_${suffix}_hist.root \
0231         --silent \
0232         --config CI/physmon/vertexing_config.yml
0233     ec=$(($ec | $?))
0234
0235     # remove ntuple file because it's large
0236     rm $outdir/performance_ivf_${suffix}.root
0237
0238     run_histcmp \
0239         $outdir/performance_ivf_${suffix}_hist.root \
0240         $refdir/performance_ivf_${suffix}_hist.root \
0241         "IVF ${suffix}" \
0242         ivf_${suffix}
0243
0244     run Examples/Scripts/generic_plotter.py \
0245         $outdir/performance_amvf_${suffix}.root \
0246         vertexing \
0247         $outdir/performance_amvf_${suffix}_hist.root \
0248         --silent \
0249         --config CI/physmon/vertexing_config.yml
0250     ec=$(($ec | $?))
0251
0252     # remove ntuple file because it's large
0253     rm $outdir/performance_amvf_${suffix}.root
0254
0255     run_histcmp \
0256         $outdir/performance_amvf_${suffix}_hist.root \
0257         $refdir/performance_amvf_${suffix}_hist.root \
0258         "AMVF ${suffix}" \
0259         amvf_${suffix}
0260
0261     if [ $suffix == seeded ]; then
0262             run Examples/Scripts/generic_plotter.py \
0263             $outdir/performance_amvf_gridseeder_${suffix}.root \
0264             vertexing \
0265             $outdir/performance_amvf_gridseeder_${suffix}_hist.root \
0266             --silent \
0267             --config CI/physmon/vertexing_config.yml
0268         ec=$(($ec | $?))
0269
0270         # remove ntuple file because it's large
0271         rm $outdir/performance_amvf_gridseeder_${suffix}.root
0272
0273         run_histcmp \
0274             $outdir/performance_amvf_gridseeder_${suffix}_hist.root \
0275             $refdir/performance_amvf_gridseeder_${suffix}_hist.root \
0276             "AMVF (+grid seeder) ${suffix}" \
0277             amvf_gridseeder_${suffix}
0278     fi
0279
0280     run Examples/Scripts/generic_plotter.py \
0281         $outdir/tracksummary_ckf_${suffix}.root \
0282         tracksummary \
0283         $outdir/tracksummary_ckf_${suffix}_hist.root \
0284         --silent \
0285         --config CI/physmon/tracksummary_ckf_config.yml
0286     ec=$(($ec | $?))
0287
0288     # remove ntuple file because it's large
0289     rm $outdir/tracksummary_ckf_${suffix}.root
0290
0291     run_histcmp \
0292         $outdir/tracksummary_ckf_${suffix}_hist.root \
0293         $refdir/tracksummary_ckf_${suffix}_hist.root \
0294         "Track Summary CKF ${suffix}" \
0295         tracksummary_ckf_${suffix}
0296
0297 }
0298
0299 function simulation() {
0300     suffix=$1
0301
0302     config="CI/physmon/simulation_config.yml"
0303
0304     run Examples/Scripts/generic_plotter.py \
0305         $outdir/particles_${suffix}.root \
0306         particles \
0307         $outdir/particles_${suffix}_hist.root \
0308         --silent \
0309         --config $config
0310     ec=$(($ec | $?))
0311
0312     # remove ntuple file because it's large
0313     rm $outdir/particles_${suffix}.root
0314
0315     run_histcmp \
0316         $outdir/particles_${suffix}_hist.root \
0317         $refdir/particles_${suffix}_hist.root \
0318         "Particles ${suffix}" \
0319         particles_${suffix}
0320 }
0321
0322 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0323     full_chain truth_smeared
0324     full_chain truth_estimated
0325     full_chain seeded
0326     full_chain orthogonal
0327
0328     run_histcmp \
0329         $outdir/performance_ambi_seeded.root \
0330         $refdir/performance_ambi_seeded.root \
0331         "Ambisolver seeded" \
0332         ambi_seeded
0333
0334     run_histcmp \
0335         $outdir/performance_ambi_orthogonal.root \
0336         $refdir/performance_ambi_orthogonal.root \
0337         "Ambisolver orthogonal" \
0338         ambi_orthogonal
0339
0340     run_histcmp \
0341         $outdir/performance_seeding_ttbar.root \
0342         $refdir/performance_seeding_ttbar.root \
0343         "Seeding ttbar" \
0344         seeding_ttbar \
0345         -c $config
0346
0347     run_histcmp \
0348         $outdir/performance_ckf_ttbar.root \
0349         $refdir/performance_ckf_ttbar.root \
0350         "CKF ttbar" \
0351         ckf_ttbar \
0352         -c $config
0353
0354     run_histcmp \
0355         $outdir/performance_ambi_ttbar.root \
0356         $refdir/performance_ambi_ttbar.root \
0357         "Ambisolver " \
0358         ambi_ttbar
0359
0360     run Examples/Scripts/generic_plotter.py \
0361         $outdir/performance_amvf_ttbar.root \
0362         vertexing \
0363         $outdir/performance_amvf_ttbar_hist.root \
0364         --silent \
0365         --config CI/physmon/vertexing_ttbar_config.yml
0366     ec=$(($ec | $?))
0367
0368     run Examples/Scripts/generic_plotter.py \
0369         $outdir/tracksummary_ckf_ttbar.root \
0370         tracksummary \
0371         $outdir/tracksummary_ckf_ttbar_hist.root \
0372         --config CI/physmon/tracksummary_ckf_config.yml
0373     ec=$(($ec | $?))
0374
0375     # remove ntuple file because it's large
0376     rm $outdir/tracksummary_ckf_ttbar.root
0377
0378     run_histcmp \
0379         $outdir/tracksummary_ckf_ttbar_hist.root \
0380         $refdir/tracksummary_ckf_ttbar_hist.root \
0381         "Track Summary CKF ttbar" \
0382         tracksummary_ckf_ttbar
0383
0384     # remove ntuple file because it's large
0385     rm $outdir/performance_amvf_ttbar.root
0386
0387     run_histcmp \
0388         $outdir/performance_amvf_ttbar_hist.root \
0389         $refdir/performance_amvf_ttbar_hist.root \
0390         "AMVF ttbar" \
0391         amvf_ttbar
0392
0393     run Examples/Scripts/generic_plotter.py \
0394         $outdir/performance_amvf_gridseeder_ttbar.root \
0395         vertexing \
0396         $outdir/performance_amvf_gridseeder_ttbar_hist.root \
0397         --silent \
0398         --config CI/physmon/vertexing_ttbar_config.yml
0399     ec=$(($ec | $?))
0400
0401     # remove ntuple file because it's large
0402     rm $outdir/performance_amvf_gridseeder_ttbar.root
0403
0404     run_histcmp \
0405         $outdir/performance_amvf_gridseeder_ttbar_hist.root \
0406         $refdir/performance_amvf_gridseeder_ttbar_hist.root \
0407         "AMVF (+grid seeder) ttbar" \
0408         amvf_gridseeder_ttbar
0409 fi
0410
0411 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0412     run_histcmp \
0413         $outdir/performance_gsf.root \
0414         $refdir/performance_gsf.root \
0415         "Truth tracking (GSF)" \
0416         gsf \
0417         -c CI/physmon/gsf.yml
0418 fi
0419
0420 if [[ "$mode" == "all" || "$mode" == "kalman" ]]; then
0421     run_histcmp \
0422         $outdir/performance_truth_tracking.root \
0423         $refdir/performance_truth_tracking.root \
0424         "Truth tracking" \
0425         truth_tracking \
0426         -c CI/physmon/truth_tracking.yml
0427 fi
0428
0429 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0430     run_histcmp \
0431         $outdir/performance_gx2f.root \
0432         $refdir/performance_gx2f.root \
0433         "Truth tracking (GX2F)" \
0434         gx2f \
0435         -c CI/physmon/gx2f.yml
0436 fi
0437
0438 if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then
0439     run Examples/Scripts/vertex_mu_scan.py \
0440         $outdir/performance_vertexing_*mu*.root \
0441         $outdir/vertexing_mu_scan.pdf
0442
0443     rm $outdir/performance_vertexing_*mu*
0444 fi
0445
0446 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0447     simulation fatras
0448     simulation geant4
0449 fi
0450
0451 run CI/physmon/summary.py $histcmp_results \
0452   --md $outdir/summary.md \
0453   --html $outdir/summary.html
0454 ec=$(($ec | $?))
0455
0456 exit $ec