PPG09_JetSpectrum_Analysis/macro/makeCondorJobs_jet.py

0001 import sys, os
0002 from os import environ
0003 import argparse
0004
0005 parser = argparse.ArgumentParser(description='sPHENIX MDC2 Reco Job Creator')
0006 parser.add_argument('-i', '--inputType', default="JETS", help='Input type: PYTHIA8_PP_MB, HIJING_[0-20/0-4P88], CHARM[D0], BOTTOM[D0]')
0007 parser.add_argument('-f', '--nFilesPerJob', default=30, type=int, help='Number of input files to pass to each job')
0008 parser.add_argument('-t', '--nTotEvents', default=-1, type=int, help='Total number of events to run over')
0009
0010 args = parser.parse_args()
0011
0012 with open("Run_List_Generator/dst_list/Final_RunNumbers_After_All_Cuts.txt") as file_in:
0013     lines = [line.rstrip('\n') for line in file_in]
0014 print(len(lines))
0015
0016 inputType = args.inputType.upper()
0017
0018 dstSets = ['DST_JETCALO', 'DST_JET']
0019
0020 myShell = str(environ['SHELL'])
0021 goodShells = ['/bin/bash', '/bin/tcsh']
0022 if myShell not in goodShells:
0023     print("Your shell {} was not recognised".format(myShell))
0024     sys.exit()
0025
0026 def makeCondorJob():
0027     print("Creating condor submission files for {} production".format(inputType))
0028     inputFiles = []
0029     line = []
0030     NFiles = []
0031
0032     for i in range(len(lines)):
0033         f1 = open("Run_List_Generator/dst_list/{0}_run2pp-000{1}.list".format(dstSets[0].lower(), lines[i].lower()), "r")
0034         inputFiles.append(f1)
0035         line.append(inputFiles[i].readline())
0036         with open("Run_List_Generator/dst_list/{0}_run2pp-000{1}.list".format(dstSets[0].lower(), lines[i].lower()), "r") as f:
0037            for j, _ in enumerate(f):
0038               pass
0039         NFiles.append(j+1)
0040
0041     myOutputPath = os.getcwd()
0042     condorDir = "{}/condorJob".format(myOutputPath)
0043     os.makedirs("{}/log".format(condorDir), exist_ok=True)
0044     os.makedirs("{}/fileLists".format(condorDir), exist_ok=True)
0045
0046     nJob = 0;
0047     Current_Count = 0;
0048     while line[0]:
0049         listFile = []
0050         listFileGeneric = []
0051         fileStart = "fileLists/productionFiles-{1}-{2}-".format(condorDir, inputType, dstSets[0].lower(), lines[i].lower())
0052         fileStart2 = "fileLists/productionFiles-{1}-{2}-".format(condorDir, inputType, dstSets[1].lower(), lines[i].lower())
0053         with open('listRun.txt','w') as file:
0054            for i in range(len(lines)):
0055               Current_Count = 0;
0056               while(Current_Count < NFiles[i]*1/(args.nFilesPerJob)):
0057                  file.write('{0}\n'.format(lines[i].lower()))
0058                  listFile.append("{0}/{1}{2:05d}.list".format(condorDir, fileStart, nJob))
0059                  productionFilesToUse = open(listFile[nJob], "w")
0060                  for j in range(0, args.nFilesPerJob):
0061                     splitLine = line[i].split("/")
0062                     fileName = splitLine[-1]
0063                     productionFilesToUse.write(fileName)
0064                     line[i] = inputFiles[i].readline()
0065                     if ((Current_Count + 1) > NFiles[i]*1/(args.nFilesPerJob)) and (Current_Count*args.nFilesPerJob + j == NFiles[i]):
0066                        break
0067                  nJob += 1
0068                  Current_Count += 1
0069
0070     listFileGeneric.append("$(condorDir)/{0}$INT(Process,%05d).list".format(fileStart))
0071     listFileGeneric.append("$(condorDir)/{0}$INT(Process,%05d).list".format(fileStart2))
0072     listFileGeneric.append("$INT(Process,%05d)")
0073
0074     condorFileName = "{0}/my{1}.job".format(condorDir, inputType)
0075     condorFile = open("{}".format(condorFileName), "w")
0076     condorFile.write("Universe           = vanilla\n")
0077     condorFile.write("initialDir         = {}\n".format(myOutputPath))
0078     if myShell == '/bin/bash': condorFile.write("Executable         = $(initialDir)/run_Events.sh\n")
0079     condorFile.write("PeriodicHold       = (NumJobStarts>=1 && JobStatus == 1)\n")
0080     condorFile.write("request_memory     = 6144MB\n")
0081     condorFile.write("concurrency_limits = CONCURRENCY_LIMIT_DEFAULT:1000\n")
0082     condorFile.write("Priority           = 20\n")
0083     condorFile.write("job_lease_duration = 3600\n")
0084     condorFile.write("condorDir          = $(initialDir)/condorJob\n")
0085     condorOutputInfo = "$(condorDir)/log/condor-{0}-$INT(Process,%05d)".format(inputType)
0086     condorFile.write("Output             = {0}.out\n".format(condorOutputInfo))
0087     condorFile.write("Error              = {0}.err\n".format(condorOutputInfo))
0088     condorFile.write("Log                = /tmp/condor-napple-$INT(Process,%05d).log\n")
0089     condorFile.write("Arguments          = \"{}\"\n".format(' '.join(listFileGeneric)))
0090     condorFile.write("Queue {}\n".format(nJob))
0091     print("Submission setup complete!")
0092     print("This setup will submit {} subjobs".format(nJob))
0093     print("You can submit your job with the script:\n{}".format(condorFileName))
0094
0095 makeCondorJob()