Corral/lists/make_lists.bash

0001 #!/bin/bash
0002 # Chunk a Collect production into corral job lists (run this first when a new production lands).
0003 #   Usage: lists/make_lists.bash <collect_dir> <nevt_per_list> [dataset] [runs]
0004 #   e.g.   lists/make_lists.bash /rs/rs_grp_rhi/sPHENIX/ana532 1000000 ana532
0005 #          lists/make_lists.bash /rs/rs_grp_rhi/sPHENIX/ana573 10000000 ana573_8158x 81584,81585,81586
0006 #   [runs] = comma-separated run numbers to keep (default: every run in <collect_dir>). One production split by
0007 #   run group: ana573_795xx (79514-79516), ana573_8156x (81566), ana573_8158x (81584-81586); plain ana573 = ALL runs.
0008 #   If lists/<dataset>/segment_entries.txt already exists (and no lists), it is used as is -- no recount; e.g. for
0009 #   ana573: cat the run-group segment_entries.txt files into lists/ana573/segment_entries.txt first.
0010 # Output: lists/<dataset>/  list_NN.txt, list_all.txt (all lists, in list order: the single full-statistics
0011 #         job), lists_summary.txt, manifest.txt, segment_entries.txt.  <dataset> defaults to the basename
0012 #         of <collect_dir>; it is the name used everywhere (root|pdf|log/<dataset>/, corral -d <dataset>).
0013 # Each list holds contiguous segments of one run, ~nevt_per_list events. Refuses to overwrite an
0014 # existing list set (move or rename the old folder first).
0015 if [ $# -lt 2 ] || [ $# -gt 4 ]; then
0016         echo "usage: $0 <collect_dir> <nevt_per_list> [dataset] [runs]"; exit 1
0017 fi
0018 CDIR=${1%/}
0019 NEVT=$2
0020 DS=${3:-$(basename $CDIR)}
0021 RUNS=$4
0022 HERE=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
0023 OUT=$HERE/$DS
0024 if [ ! -d "$CDIR" ]; then echo "no such directory: $CDIR"; exit 1; fi
0025 if ls $OUT/list_*.txt >/dev/null 2>&1; then
0026         echo "$OUT already has lists -- move or rename it first (nothing was changed)"; exit 1
0027 fi
0028 mkdir -p $OUT
0029 if [ -s $OUT/segment_entries.txt ]; then
0030         echo "using the existing $OUT/segment_entries.txt (no recount)"
0031 else
0032         echo "counting events in $CDIR/outputCollect_*.root ${RUNS:+(runs $RUNS)} ..."
0033         root -l -b -q "$HERE/count_entries.C(\"$CDIR\",\"$OUT/segment_entries.txt\",\"$RUNS\")" 2>&1 | grep -E "files=|rror"
0034 fi
0035 python3 $HERE/make_lists.py $OUT $NEVT $CDIR
0036 NL=$(awk '$1=="nlists"{print $2}' $OUT/manifest.txt)
0037 for ((i=0; i<NL; i++)); do cat $OUT/$(printf "list_%02d.txt" $i); done > $OUT/list_all.txt
0038 echo "lists written to $OUT"