#!/usr/bin/python # -*- coding: iso-8859-1 -*- import sys import math import string import pickle import requestIdFilter import os import sys import re from optparse import OptionParser #python_db="/www-d0/WWW/docs/Run2Physics/cs/MC/RequestIdListTools/request_info.db" python_db="request_info.db" class ListOfReqIds: theG = None pXX = None def __init__( self ): print "=== Current caf versions ===" print " * p17 MC: caf p18.13.01" print " * p20 MC: caf p21.11.00" self.ConsDecay = None self.ConsProc = None self.InfoBadMC = False self.ConsBadMC = True self.print_caf = "NoPrintOut" self.Constraints = {} self.Constraints['gen_version'] = { 'optConst' : 'FilterRelGENE','constraint': None, 'type': 'string', 'constHelp' : "Filter on the generator release", 'optPrint' : 'InfoRelGENE', 'print': True, 'lenPrint' : 9, 'printHelp' : "Add generator release info [Default: True]" } self.Constraints['rec_version'] = { 'optConst' : 'FilterRelRECO','constraint': None, 'type': 'string', 'constHelp' : "Filter on the reco release", 'optPrint' : 'InfoRelRECO', 'print': False, 'lenPrint' : 9, 'printHelp' : "Add generatorreco release info [Default: False]" } self.Constraints['xsec'] = { 'optConst' : None,'constraint': None, 'type': 'float', 'constHelp' : None, 'optPrint' : 'InfoXsec', 'print' : False, 'lenPrint' : 9, 'printHelp' : "Add Xsec (alpgen) info [Default: false]" } self.Constraints['nCafCount'] = { 'optConst' : None,'constraint': None, 'type': 'int', 'constHelp' : None, 'optPrint' : 'InfoNevt', 'print': False, 'lenPrint': 6, 'printHelp' : "Add #evts(CAF) info [Default: false]" } self.Constraints['n_caf_trees'] = { 'optConst' : None,'constraint': None, 'type': 'int', 'constHelp' : None, 'optPrint' : 'InfoNcaftree', 'print': False, 'lenPrint': 6, 'printHelp' : "Add #CAF-tree info [Default: false]" } self.Constraints['n_tmb'] = { 'optConst' : None,'constraint': None, 'type': 'int', 'constHelp' : None, 'optPrint' : 'InfoNtmb', 'print': False, 'lenPrint': 6, 'printHelp' : "Add #TMB info [Default: false]" } self.Constraints['description'] = { 'optConst' : None, 'constraint': None, 'type': 'string', 'constHelp' : None, 'optPrint' : 'InfoDesc', 'print': False, 'lenPrint': 40, 'printHelp' : "Add request description [Default: false]" } self.Constraints['topmass'] = { 'optConst' : 'FilterMtop','constraint': None, 'type': 'float', 'constHelp' : "Filter on the specified top mass", 'optPrint' : 'InfoMtop', 'print': False, 'lenPrint' : 5, 'printHelp' : "Add top mass info [Default: false]" } self.Constraints['minb_dataset'] = { 'optConst' :'FilterZB' ,'constraint': None, 'type': 'string', 'constHelp' :"Filter on the specified Overlay sample", 'optPrint' : 'InfoZB' , 'print': False, 'lenPrint': 70, 'printHelp' : "Add ZB sample info [Default: false]" } self.Constraints['matching_pt'] = { 'optConst' : 'FilterPTmatching','constraint': 8, 'type': 'float', 'constHelp' : "Filter on the pT matching (Alpgen only) [Default: 8]", 'optPrint' : 'InfoPTmatching', 'print': False, 'lenPrint': 3, 'printHelp' : "Add PT matching info (Alpgen only) [Default: False]" } self.Constraints['ue_tune'] = { 'optConst' : 'FilterUEtune','constraint': 'tuneA','type': 'string', 'constHelp' : "Filter on the underlying-event tune (if parton showering pythia) [Default: tuneA]", 'optPrint' : 'InfoUEtune', 'print': False, 'lenPrint': 10, 'printHelp' : "Add underlying-event-tune info [Default: False]" } self.Constraints['fscal-val'] = { 'optConst' : 'FilterQscale','constraint': 1.0,'type': 'float', 'constHelp' : "Filter on the Q scale value [Default: 1.0]", 'optPrint' : 'InfoQscale', 'print': False, 'lenPrint': 3, 'printHelp' : "Add Q scale info [Default: False]" } self.Constraints['iq'] = { 'optConst' : 'FilterFactOpt','constraint': None,'type': 'int', 'constHelp' : "Filter on the Q scale option [Default: None], standard alpgen: 1", 'optPrint' : 'InfoFactOpt', 'print': False, 'lenPrint': 2, 'printHelp' : "Add fact scale option info [Default: False]" } def Print(self, detail = 0 ): print "List will be done for GENERATOR: %s , %s MC" %( self.theG, self.pXX ) if detail >= 1: print "" print " === Constraints used for listing: " print " * %-41s : %s" %( 'Processus', self.ConsProc ) print " * %-41s : %s" %( 'Decay', self.ConsDecay ) for cons in self.Constraints.keys(): if self.Constraints[cons].get('optConst'): print " * %-41s : %s" % ( self.Constraints[cons].get('constHelp').split('(')[0].split('[')[0][0:50], self.Constraints[cons].get('constraint') ) if detail >= 2: print "" print " === Info displayed:" if self.InfoBadMC: print " - %-41s" % ( "Include Bad MC samples in listing" ) for cons in self.Constraints.keys(): if self.Constraints[cons].get('print'): print " - %-41s" % ( self.Constraints[cons].get('printHelp').split('[')[0][0:50] ) def Get_icaf(self,request): icaf=-1 if self.pXX=='p17': icaf=1 elif self.pXX=='p20': icaf=0 return icaf def CheckRequest(self, request ): generator = request.get('generator') reco = request.get('rec_version') if self.pXX and reco.split('.')[0] != self.pXX: return False if generator == 'not_found' or generator=='unknown' or generator=='single': return False if generator != self.theG: return False icaf=self.Get_icaf(request) if icaf < 0: return False #if self.pXX=='p17': # icaf=1 #elif self.pXX=='p20': # icaf=0 #else: # return False requestStatus='unknown' if self.pXX == 'p17': requestStatus=requestIdFilter.filterReqId_p17( request ) if self.pXX == 'p20': requestStatus=requestIdFilter.filterReqId_p20( request ) if self.ConsBadMC and requestStatus == 'bad': return False n_tmb = -1 n_caf = -1 if request.has_key('n_caf_trees'): try: n_caf=request.get('n_caf_trees')[icaf] except: print " Does not work: " print " --> req:", request print " icaf=", icaf return False if request.has_key('n_tmb'): n_tmb=request.get('n_tmb') if int(n_tmb)<=0: return False proc = requestIdFilter.change_prod( request ) decay = requestIdFilter.change_decay( request ) consVal = self.ConsProc reqVal = proc if consVal: # if re.match(consVal,reqVal) is None or re.match(consVal,reqVal).end() != len(reqVal): if re.match(consVal,reqVal) is None: return False consVal = self.ConsDecay reqVal = decay if consVal: # if re.match(consVal,reqVal) is None or re.match(consVal,reqVal).end() != len(reqVal): if re.match(consVal,reqVal) is None: return False for cons in self.Constraints.keys(): consVal = self.Constraints[cons].get('constraint') if consVal == 'None' or consVal == 'none': consVal = None if consVal and request.has_key(cons): reqVal = request[cons] if self.Constraints[cons].get('type') == 'string': if re.match(consVal,reqVal) is None or re.match(consVal,reqVal).end() != len(reqVal): return False else: consVal = float(consVal) if reqVal is None: reqVal = -1 else: reqVal = float(reqVal) if reqVal != consVal: return False elif consVal and not request.has_key(cons): print "Contraint: %s = %s ... But request has no field %s" % (cons, consVal, cons) print "Request %s (%s) has no key: %s" % (request['requestid'],request['description'],cons) return True def GetListOfReqFromConstraints(self): if self.theG is None or self.theG == 'unknown': print "Generator was not written in sam metadata..." print "This is usually the case for ssingletop CompHep, special prod" print " This is not yet supported" sys.exit(0) file_all = open(python_db,'r') list_all = pickle.load(file_all) req_list=list_all.keys() list_of_p={} for reqk in req_list: request=list_all.get(reqk) if self.CheckRequest( request ): #print " Request %s: %s is selected" % ( reqk, request['description'] ) key = int( request.get('requestid') ) value = request list_of_p[key] = value pl=list_of_p.keys() pl.sort() num_evt=0 sep10 = "==========" sep20 = sep10 + sep10 sep30 = sep10 + sep20 sep80 = sep30 + sep30 + sep20 sep100 = sep30 + sep30 + sep30 + sep10 separator = "" format = "" separator += "========" format += "reqid |" separator += "=" + sep30 + sep30 + "=" format += " %-60s " % ('process__decay__pXX') for cons in self.Constraints.keys(): if self.Constraints[cons]['print']: form="=%+" + str(self.Constraints[cons]['lenPrint']) + "s==" separator += form % sep100[0:self.Constraints[cons]['lenPrint']] form=" %+" + str(self.Constraints[cons]['lenPrint']) + "s " format += form % cons[0:self.Constraints[cons]['lenPrint']] print separator print format print separator xsec = 0; nreq = 0; icaf=self.Get_icaf(request) if icaf < 0: return False for p in pl: request=list_of_p.get(p) proc = requestIdFilter.change_prod( request ) decay= requestIdFilter.change_decay( request ) reco = request.get('rec_version') gene = request.get('gen_version') proc__decay = "%s__%s__%s" % (proc,decay,self.pXX) info="%+6s | %-60s " % (p,proc__decay) #if int( request['requestid'] ) == 103999: # print request if request.has_key('nevt_caf_trees'): num_evt+=request.get('nevt_caf_trees')[icaf] for cons in self.Constraints.keys(): if self.Constraints[cons]['print']: form=" %+" + str(self.Constraints[cons]['lenPrint']) + "s " if not request.has_key(cons) or request[cons] is None: info += form % " NA" else: toprint = str(request[cons]) info += form % toprint[0:self.Constraints[cons]['lenPrint']] print info xsec += float(request['xsec']) nreq += 1 print separator file_all.close() lumi_eq = 0 if nreq > 0: xsec /= nreq lumi_eq = num_evt / (xsec*1000); print " --> Total number of events: ", num_evt print " --> Luminosity equi: %3.3f [fb-1]" % lumi_eq print " --> LL xsec: %3.3f [fb-1]" % lumi_eq if self.print_caf != 'NoPrintOut': if num_evt == 0: print "No evt for def: ", self.print_caf sys.exit(0) file_cfg=open( self.print_caf+".cfg",'w') file_cfg.write("## Requests list for csg MC\n") file_cfg.write("## sam def: %s \n" % self.print_caf) file_cfg.write("## N[evts]: %s - Lumi equi: %3.3f [fb-1]\n" % (num_evt,lumi_eq) ) file_cfg.write("\n") csg_vers = 'unknown' if pXX == 'p17': csg_vers = 'csg-p18.13.01-v4' if pXX == 'p20': csg_vers = 'csg-p21.11.00-v4' samdef = "sam create dataset definition " samdef += "--definitionName=%s --dimensions=\"version %s and global.requestid " % ( self.print_caf,csg_vers) np=1 print "config file input: " for p in pl: if int(np) == 1: file_cfg.write("requests_%s.ReqIDsInTotalMCSample: %s\n" % ( self.print_caf, p ) ) else: file_cfg.write("+requests_%s.ReqIDsInTotalMCSample: %s\n" % ( self.print_caf, p ) ) samdef += "%s" % p if int(np) != len(pl): samdef += "," np=np+1 samdef +="\"" print "CREATE SAM DEF: ", samdef print "=== Create file: %s" % ( self.print_caf + ".cfg") file_cfg.write('\n') file_cfg.close() def GetListOfConstraints(self, cons = None): if self.theG is None or self.theG == 'unknown': print "Generator was not written in sam metadata..." print "This is usually the case for ssingletop CompHep, special prod" print " This is not yet supported" sys.exit(0) file_all = open(python_db,'r') list_all = pickle.load(file_all) req_list=list_all.keys() list_of_p={} for reqk in req_list: request=list_all.get(reqk) if self.CheckRequest( request ): #print " Request %s: %s is selected" % ( reqk, request['description'] ) if not request.has_key(cons): continue key = cons value = request[cons] list_of_p[value] = cons pl=list_of_p.keys() pl.sort() for p in pl: print p file_all.close() if __name__ == '__main__': print """ Main function, deals with arguments and launch program""" usage_str="%prog [options] generator pXX (pXX is p17 or p20)" description_str = "getRequestsList gives a list of request id matching criteria in options\n" description_str += "A generator argument is requested.\n" description_str += " use -h or --help to get help.\n" listOfReq = ListOfReqIds() ### create option parser parser = OptionParser(usage=usage_str,description=description_str) # listing options parser.add_option("--generators", action="store_true",dest="generators", help="List of available generators") # refine list of request id parser.add_option("-p", action="store", type="string", dest="proc", help="specify process id, eg z+0lp") parser.add_option("-d", action="store",type="string", dest="decay", help="specify decay id, eg ee+0lp or ee") parser.add_option("--IncludeBadMCs", action="store_true", dest="IncludeBadMCs", help="Include bad MC in listing (only cerfified for p20) [Default: false]", default=False) for cons in listOfReq.Constraints.keys(): if listOfReq.Constraints[cons].get('optConst'): option = "--" + listOfReq.Constraints[cons].get('optConst') destin = listOfReq.Constraints[cons].get('optConst') parser.add_option( option, action="store",dest=destin, help = listOfReq.Constraints[cons].get('constHelp'), type = listOfReq.Constraints[cons].get('type'), default= listOfReq.Constraints[cons].get('constraint') ) # editing options for cons in listOfReq.Constraints.keys(): if listOfReq.Constraints[cons].get('optPrint'): option = "--" + listOfReq.Constraints[cons].get('optPrint') destin = listOfReq.Constraints[cons].get('optPrint') store = "store_true" if listOfReq.Constraints[cons].get('print') is True: store = "store_false" parser.add_option( option, action=store,dest=destin, help = listOfReq.Constraints[cons].get('printHelp'), default= listOfReq.Constraints[cons].get('print') ) # listing options parser.add_option("--ListProcesses", action="store_true", dest="list_processes", help="List of available constraints for opt: -p") parser.add_option("--ListDecays", action="store_true", dest="list_decays", help="List of available constraints for opt: -d") for cons in listOfReq.Constraints.keys(): if listOfReq.Constraints[cons].get('optConst'): option = "--List" + listOfReq.Constraints[cons].get('optConst') destin = "list_%s" % listOfReq.Constraints[cons].get('optConst') parser.add_option( option, action="store_true", dest=destin, help = " list available constraints for opt: " + listOfReq.Constraints[cons].get('optConst') ) # add dataset creation parser.add_option("--CAFE", action="store",type="string",dest="CAFE", help="Print out the command to get the dataset (sam def). The argument is the dataset name you want to use [Default: no print out]",default="NoPrintOut" ) # parse args (options, args) = parser.parse_args() if not options.proc is None: options.proc=options.proc.replace('+','\+') if not options.decay is None: options.decay=options.decay.replace('+','\+') print "process = ",options.proc print "decay = ",options.decay pXX=None if len(args) == 0: print "You must at least specify a generator." print "The list of available generator is: " if len(args)>=1: pXX=args[1] print "==== TIP" print description_str sys.exit(0) myG=args[0] if len(args)==1: print " --> you must specify which MCs samples: p17 or p20" sys.exit(0) elif args[1] != 'p17' and args[1] != 'p20': print "You must specify which MCs samples: p17 or p20" print " --> %s is not supported", args[1] sys.exit(0) pXX=args[1] ListOfReqIds.pXX = pXX ListOfReqIds.theG = myG if options.proc: listOfReq.ConsProc = options.proc if options.decay: listOfReq.ConsDecay = options.decay if options.IncludeBadMCs: listOfReq.ConsBadMC = False listOfReq.InfoBadMC = True listOfReq.print_caf=options.CAFE for cons in listOfReq.Constraints.keys(): if listOfReq.Constraints[cons].get('optConst'): listOfReq.Constraints[cons]['constraint'] = getattr( options, listOfReq.Constraints[cons].get('optConst') ) if listOfReq.Constraints[cons].get('type') == 'int' or \ listOfReq.Constraints[cons].get('type') == 'float': if listOfReq.Constraints[cons]['constraint'] > 99: listOfReq.Constraints[cons]['constraint'] = None if listOfReq.Constraints[cons].get('optPrint'): listOfReq.Constraints[cons]['print'] = getattr( options, listOfReq.Constraints[cons].get('optPrint') ) ### options that do not make sense for some config if not myG.find('alpgen') >= 0: listOfReq.Constraints['matching_pt']['constraint'] = None listOfReq.Constraints['fscal-val']['constraint'] = None if not myG.find('pythia') >= 0: listOfReq.Constraints['ue_tune']['constraint'] = None ### listing for cons in listOfReq.Constraints.keys(): if listOfReq.Constraints[cons].get('optConst'): list_opt = "list_%s" % listOfReq.Constraints[cons].get('optConst') if getattr( options, list_opt ): listOfReq.Constraints[cons]['constraint']=None print "Warning: only one listing option should be used at a time" print "*** Will do listing for: ", listOfReq.Constraints[cons].get('optConst') listOfReq.Print(2) listOfReq.GetListOfConstraints( cons ) sys.exit(0) listOfReq.Print(2) listOfReq.GetListOfReqFromConstraints()