#!/usr/bin/env python
#==============================================================================
# File:        uber_syserr_submitjobs.py
# Description: Script to submit single-top analysis jobs
#              for systematic errors' studies.
#              This is based on Aran's shell-script, and
#              re-written in Python
#              by Harrison B. Prosper, 2-Aug-2004)
#
# Authors:     HBP, S. Jain (harry@fnal.gov, sjain@fnal.gov)
#              11/04
# Syntax:  >   From the release area:
#          > ln -sf topd0root_singletop/bin/Systematics/uber_syserr_submitjobs.py subjob
# ./subjob -s10 --> this generates SigmaSample.txt of size 10
#                   in topd0root_singletop/bin/Systematics
# ./subjob --<SYSNAME> --<CUTSTAGE> --batch -f<HistSample_begin=say,0> -l<HistSample_end=say, 9>
# OR:
# ./subjob --<SYSNAME> --<CUTSTAGE> --batch topd0root_singletop/bin/Systematics/Sigma_zero.txt
#
#            Where: SYSNAME  = ALL_SYSTEMATICS, JES, TRIG, or TRF
#                   CUTSTAGE = slvsvt, slt, slvsvtfinal, sltfinal or finalData
#
# 
#$Revision: 1.3 $
#==============================================================================
# Imports
#==============================================================================
from sys  import exit, argv
from os   import popen, system, chmod
from stat import S_IRWXU, S_IRGRP, S_IROTH
from os.path import exists, splitext, basename
from time import time, ctime, sleep
from getopt import getopt, GetoptError
from random import normalvariate
from string import split, strip, rstrip, atoi
#==============================================================================
# Constants
#==============================================================================
VERSION = '2.0'
USAGE = '''
Usage:
   uber_syserr_submitjobs [-t | --test] [-b | --batch]
                          [-f | --firstrow=]
                          [-l | --lastrow=]
                          [-s#samples | --sample=#samples]
                          <--dirname=DIRNAME> <--sysname=SYSNAME> <--cutstage=CUTSTAGE> <--trigsig=TRIGSIG> <--mcbtag=MCBTAG> <--SVTloose=SVTLOOSE>

   where
        SYSNAME  = JES, TRF
        CUTSTAGE = pretagged, tb_EqOneTag, tb_GeqTwoTag, tqb_pretagged, tqb_EqOneTag, tqb_GeqTwoTag
'''

# Command lines switches

SHORTOPTIONS = 'thbs:f:l:'
LONGOPTIONS  = ['test','help','dirname=','sysname=','cutstage=', 'trigsig=', 'mcbtag=', 'SVTloose=',
                'batch', 'sample=', 'firstrow=', 'lastrow=']

# Various date/time formats

fulldate = ctime(time())
day, month, daynum, hour, year = split(fulldate)
date = '%2s-%3s-%4s' % (daynum,month,year)

# Complete context, that is, variables and parameters used by this script

CONTEXT = {'jobfile':'sys_err_job',
           'logfile':'',
           'loggerfile':'uber_syserr.log',
           'samplefile':'SigmaSample.txt',
           'fileprefix':'Sigma',
           'sigmafile' :'SINGLETOP_SIGMA_FILE',

           'time'   : fulldate,
           'date'   : date,
           'version': VERSION,
           'testmode': 0,
            'dataset':'Ipanema',
           'batch':    0,
           'firstrow': 0,
           'lastrow':  0,
           'sysname': '',
           'dirname': '',
           'signame': '',
           'cutstage':'',           
           'programdir': 'Analysis',
           'program': 'Analysis_x',
           'lep'    : 'e',
           'datalist':'',
           'outfile': '',
           'nn_file': '',
           'skim_file': '',
           'defcuts': '',
           'cutsfile':'',
           'sample':   '',
           'sigjes':   '',
           'trigsig':   '',
           'mcbtag':   '',
           'SVTloose':   '',
           'cputime':  '',
           'memory':   '999M',           
           'shortcpu': '8:00:00',
           'longcpu': '12:40:00',
           }

CONTEXT['samplefile'] = \
                      'topd0root_singletop/bin/%(programdir)s/%(samplefile)s'\
                      % CONTEXT
                        

CONTEXT['fileprefix'] = \
                      'topd0root_singletop/bin/%(programdir)s/%(fileprefix)s'\
                      % CONTEXT


EMQCD_Ipanema  = ['skim/datalist_1.list DATA eCC',
                  'skim/datalist_2.list DATA eCC',
                  'skim/datalist_3.list DATA eCC',
                  'skim/datalist_4.list DATA eCC',
                  'skim/datalist_5.list DATA eCC',
                  'skim/datalist_6.list DATA eCC',
                  'skim/datalist_7.list DATA eCC',
                  'skim/datalist_8.list DATA eCC',
                  'skim/datalist_9.list DATA eCC',
                  'skim/datalist_10.list DATA eCC',
                  'skim/datalist_11.list DATA eCC',
                  'skim/datalist_12.list DATA eCC',
                  'skim/datalist_13.list DATA eCC',
                  'skim/datalist_14.list DATA eCC',
                  'skim/datalist_15.list DATA eCC',
                  'skim/datalist_16.list DATA eCC',
                  'skim/datalist_17.list DATA eCC',
                  'skim/datalist_18.list DATA eCC',
                  'skim/datalist_19.list DATA eCC',
                  'skim/datalist_20.list DATA eCC',
                  'skim/datalist_21.list DATA eCC',
                  'skim/datalist_22.list DATA eCC',
                  'skim/datalist_23.list DATA eCC',
                  'skim/datalist_24.list DATA eCC',
                  'skim/datalist_25.list DATA eCC',
                  'skim/datalist_26.list DATA eCC',
                  'skim/datalist_27.list DATA eCC',
                  'skim/datalist_28.list DATA eCC',
                  'skim/datalist_29.list DATA eCC',
                  'skim/datalist_30.list DATA eCC',
                  'skim/datalist_31.list DATA eCC',
                  'skim/datalist_32.list DATA eCC',
                  'skim/datalist_33.list DATA eCC',
                  'skim/datalist_34.list DATA eCC',
                  'skim/datalist_35.list DATA eCC',
                  'skim/datalist_36.list DATA eCC',
                  'skim/datalist_37.list DATA eCC',
                  'skim/datalist_38.list DATA eCC',
                  'skim/datalist_39.list DATA eCC',
                  'skim/datalist_40.list DATA eCC']



DIEM_Ipanema  =  ['skim_diem/datalist_1.list DATA eCC',
                  'skim_diem/datalist_2.list DATA eCC',
                  'skim_diem/datalist_3.list DATA eCC',
                  'skim_diem/datalist_4.list DATA eCC',
                  'skim_diem/datalist_5.list DATA eCC',
                  'skim_diem/datalist_6.list DATA eCC']

ALLJETS_Ipanema  = ['skim_alljets/datalist_1.list DATA eCC',
                  'skim_alljets/datalist_2.list DATA eCC',
                  'skim_alljets/datalist_3.list DATA eCC',
                  'skim_alljets/datalist_4.list DATA eCC',
                  'skim_alljets/datalist_5.list DATA eCC',
                  'skim_alljets/datalist_6.list DATA eCC',
                  'skim_alljets/datalist_7.list DATA eCC',
                  'skim_alljets/datalist_8.list DATA eCC',
                  'skim_alljets/datalist_9.list DATA eCC',
                  'skim_alljets/datalist_10.list DATA eCC',
                  'skim_alljets/datalist_11.list DATA eCC',
                  'skim_alljets/datalist_12.list DATA eCC',
                  'skim_alljets/datalist_13.list DATA eCC',
                  'skim_alljets/datalist_14.list DATA eCC',
                  'skim_alljets/datalist_15.list DATA eCC',
                  'skim_alljets/datalist_16.list DATA eCC',
                  'skim_alljets/datalist_17.list DATA eCC',
                  'skim_alljets/datalist_18.list DATA eCC',
                  'skim_alljets/datalist_19.list DATA eCC',
                  'skim_alljets/datalist_21.list DATA eCC',
                  'skim_alljets/datalist_22.list DATA eCC',
                  'skim_alljets/datalist_23.list DATA eCC',
                  'skim_alljets/datalist_20.list DATA eCC']




# Jet energy scale-related jobs to be run
#Jobs below run on the whole sample of MC, qcd and Data, not the skim ones.

JES_related_JOBS  = ['datalists/ipanema_original/enbb.list tb eCC',
                     'datalists/ipanema_original/enbqb.list tqb eCC',
                     'datalists/ipanema_original/e_ttbar175dilep_TuneA.list ttbar_dilep eCC',
                     'datalists/ipanema_original/e_ttbar175lepjets_TuneA.list ttbar_lepjets eCC',
                     'datalists/ipanema_original/e_Higgs_Wbb_TuneA.list wbb_higgs eCC',
                     'datalists/ipanema_original/e_Higgs_Wjj_TuneA.list wjj_higgs eCC',
                     'datalists/ipanema_original/e_WW_lnujj.list ww_lnujj eCC',
                     'datalists/ipanema_original/e_WZ_lnujj.list wz_lnujj eCC' ]

DATA_JOBS_fs = ['datalists/ipanema_original/EMQCD.list DATA eCC']
QCD_JOBS_fs = ['datalists/ipanema_original/EMQCD.list DATA  eCC']

# Non-jet energy scale jobs to be run
#Jobs below run on the the skimmed samples of MC, qcd and Data,.

NJESJOBS_tb=   ['datalists/ipanema_skim/enbb.list tb eCC',
                'datalists/ipanema_skim/enbqb.list tqb eCC',
                'datalists/ipanema_skim/e_ttbar175dilep_TuneA.list ttbar_dilep eCC',
                'datalists/ipanema_skim/e_ttbar175lepjets_TuneA.list ttbar_lepjets eCC',
                'datalists/ipanema_skim/e_Higgs_Wbb.list wbb_higgs eCC',
                'datalists/ipanema_skim/e_Higgs_Wjj.list wjj_higgs eCC',
                'datalists/ipanema_skim/e_WW_lnujj.list ww_lnujj eCC',
                'datalists/ipanema_skim/e_WZ_lnujj.list wz_lnujj eCC' ]

DATA_JOBS = ['datalists/ipanema_skim/pretagged_EMQCD.list DATA eCC']
           
QCD_JOBS = ['datalists/ipanema_skim/pretagged_qcd.list QCD_ana_Ipanema  eCC']
#QCD_JOBS = ['datalists/ipanema_skim/pretagged_qcd.list DATA  eCC']


NJESJOBS_tqb=   ['SingleTopMC_SKIM/p14Stradivarius/Fall04_Pretagged/enbb.list tb eCC',
                 'SingleTopMC_SKIM/p14Stradivarius/Fall04_Pretagged/enbqb.list tqb eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_ttbar175dilep_TuneA.list ttbar_dilep eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_ttbar175lepjets_TuneA.list ttbar_lepjets eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_Higgs_Wbb.list wbb_higgs eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_Higgs_Wjj.list wjj_higgs eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_WW_lnujj.list ww_lnujj eCC',
                 'BackgroundMC_SKIM/p14Stradivarius/Fall04_Pretagged/e_WZ_lnujj.list wz_lnujj eCC' ]


# Cut stages and cuts

CUTSTAGE = ['Emqcd_Geq2jets_1em',
            'Diem_2_cc_em',
            'Data_pretagged',
            'Data_NoTag',
            'Data_EqOneTag',
            'Data_GeqOneTag',
            'Data_GeqTwoTag',
            'Data_tqb_GeqTwoTag',
            'QCD_pretagged',
            'QCD_NoTag',
            'QCD_EqOneTag',
            'QCD_GeqOneTag',
            'QCD_GeqTwoTag',
            'QCD_tqb_GeqTwoTag',
            'pretagged',
            'tb_NoTag',
            'tb_EqOneTag',
            'tb_GeqOneTag',
            'tb_GeqTwoTag',
            'tqb_EqOneTag',
            'tqb_GeqTwoTag']

TRIGSIG = ['zero', 'plus', 'minus']

MCBTAG = ['yes', 'no']
SVTLOOSE = ['yes', 'no']

CUTS = { 'eCC':   # Channel
        # cutstage
        {
#none of the skim_from_original cuts files have dataquality applied
    'Emqcd_Geq2jets_1em':{'cutsfile':
                      'ana_me_cft/skim/cutlist_DATA_1e_greq_2jets.txt'
                      },
    
    'Diem_2_cc_em':{'cutsfile':
                      'ana_me_cft/skim_diem/DATA.txt'
                      },
    
    'Data_pretagged':{'cutsfile':
                      'ana_me_cft/cuts/PRETAGGED/DATA.txt'
                      },
    
    'Data_NoTag':{'cutsfile':
                     'ana_me_cft/cuts/NOT_TAGGED/DATA.txt'
                     },
    
    'Data_EqOneTag':{'cutsfile':
                     'ana_me_cft/cuts/PRETAGGED_and_EQ_ONETAG/DATA.txt'
                     },
    
    'Data_GeqOneTag':{'cutsfile':
                     'ana_me_cft/cuts/PRETAGGED_and_GEQ_ONETAGS/DATA.txt'
                     },
    

    'Data_GeqTwoTag':{'cutsfile':
                      'ana_me_cft/cuts/PRETAGGED_and_GEQ_TWOTAGS/DATA.txt'
                      },
    
    'QCD_pretagged':{'cutsfile':
                     'ana_me_cft/PRETAGGED/QCD.txt'
                     },
    
    'QCD_NoTag':{'cutsfile':
                    'ana_me_cft/cuts/NOT_TAGGED/QCD.txt'
                    },
    
    'QCD_EqOneTag':{'cutsfile':
                    'ana_me_cft/cuts/PRETAGGED_and_EQ_ONETAG/QCD.txt'
                    },
    
    'QCD_GeqTwoTag':{'cutsfile':
                     'ana_me_cft/cuts/PRETAGGED_and_GEQ_TWOTAGS/QCD.txt'
                     },
    
    'QCD_GeqOneTag':{'cutsfile':
                     'ana_me_cft/cuts/PRETAGGED_and_GEQ_ONETAGS/QCD.txt'
                     },
    
    'pretagged':{'cutsfile':
                          'ana_me_cft/PRETAGGED/MC.txt'
                          },
    
    'tb_NoTag':{'cutsfile':
                          'ana_me_cft/NOT_TAGGED/MC.txt'
                          },
    
    'tb_EqOneTag':{'cutsfile':
                          'ana_me_cft/PRETAGGED_and_EQ_ONETAG/MC.txt'
                          },
    
    'tb_GeqOneTag':{'cutsfile':
                          'ana_me_cft/PRETAGGED_and_GEQ_ONETAGS/MC.txt'
                          },
    
    'Data_tqb_GeqTwoTag':{'cutsfile':
                      'ana_me_cft/cuts/PRETAGGED_and_GEQ_TWOTAGS/DATA.txt'
                      },
    
    'tb_GeqTwoTag':        {'cutsfile':
                                   'ana_me_cft/PRETAGGED_and_GEQ_TWOTAGS/MC.txt'
                                   },

    'tqb_EqOneTag':{'cutsfile':
                          'ana_me_cft/cuts/PRETAGGED_and_EQ_ONETAG/MC.txt'
                          },
    
    'tqb_GeqTwoTag':        {'cutsfile':
                                   'ana_me_cft/PRETAGGED_and_GEQ_TWOTAGS/MC.txt'
                                   }
    }
        }

# Names of systematic effects

SYSTEMATICS = ["EML1",
               "EML2",
               "EML3",
               "MUL1",
               "MUL2",
               "MUL3",
               "JETL1",
               "JETL2",
               "JETL3",
               "mcTRF", 
               "JES"]

# Default number of samples

NSAMPLES = 200

#==============================================================================
# Functions/Classes
#==============================================================================
def usage():
    print USAGE
    
def fatal(s, errorCode=1):
    s = "\n** Error ** %s\n" % s
    logger.write(s)
    print s
    exit(errorCode)
        
def warn(s):
    s = "\n** Warning ** %s\n" % s
    logger.write(s)
    print s
            
# Name without extension
            
def nameonly(name):
    return splitext(basename(name))[0]

# Logger to log all transactions

class Logger:
    def __init__(self):
        self.LOG = open(CONTEXT['loggerfile'],'w')
        
    def __def__(self):
        self.LOG.close()
            
    def write(self, record):
        self.LOG.write("%s\n" % ctime(time()))
        self.LOG.write("%s\n\n" % strip(rstrip(record)))
        
logger = Logger()
                
# Execute command in shell
                
def execute(command):
    if CONTEXT['testmode']:
        return 0
    else:
        return system(command)
                    
# Generate sample of systematic effect "sigmas"

def generateSample(sampleSize):
    if sampleSize <= 0:
        sampleSize = NSAMPLES
        
    print "\tGenerate sample of size %d" % sampleSize
    
    out = open(CONTEXT['samplefile'],'w')
    out.write('#' + 78*'-' + '\n')
    out.write('# Single Top Group Systematic Error Sample\n')
    out.write('# Created: %(time)s\n' % CONTEXT)
    out.write('#' + 78*'-' + '\n')
    
    out.write('# Names of systematic effects\n')
    out.write('NAME = [')
    for index in xrange(len(SYSTEMATICS)):
        name = SYSTEMATICS[index]
        if index == 0:
            out.write("'%s'" % name)
        else:
            out.write("        '%s'" % name)
        if index < len(SYSTEMATICS)-1:
            out.write(',\n')
    out.write(']\n')
    
    out.write('#' + 78*'-' + '\n')
    
    out.write('SAMPLE = [\n')

    # Okay, generate samples
    
    for sample in xrange(NSAMPLES):
        out.write('  [')
        count = 4
        for index in xrange(len(SYSTEMATICS)):
            x = normalvariate(0.0, 1.0)
            if count > 60:
                out.write('\n   ')
                count = 4
                
            out.write('%9.4f' % x)
            count = count + 9
            
            if index < len(SYSTEMATICS)-1:
                out.write(',')
                    
        out.write(']')
        if sample < NSAMPLES-1:
            out.write(',')
        out.write('\n')

    out.write(']\n')
    out.close()
                        
#==============================================================================
#
# Execute the job interactively (to be used only for tests...) 
#
def executeInteractive():
    record ='''
# Important: Make SINGLETOP_SIGMA_FILE point to the correct sigma-file
#            so that SigmaLoader loads the correct sigmas

setenv SINGLETOP_SIGMA_FILE %(sigmafile)s
    
%(program)s \\
-lepton          %(lep)s \\
-lepton_cut_name  good_electron  \\
-list            %(datalist)s \\
-histfile        %(outfile)s \\
-skimfile        %(skim_file)s \\
--obj-cut-file   %(defcuts)s \\
-cuts_file       %(cutsfile)s \\
-channel         %(sample)s \\
-trig_sig        %(trigsig)s \\ 
-mc_btag         %(mcbtag)s \\
-SVT_loose         %(SVTloose)s \\
-dataset          %(dataset)s\\
-event_weight     yes \\
#-nn_file         %(nn_file)s

''' % CONTEXT
    open(CONTEXT['jobfile'],'w').write(record)
    logger.write('\t%(jobfile)s\n' % CONTEXT + record)
    
    command = '''
    chmod u+x %(jobfile)s; ./%(jobfile)s
    ''' % CONTEXT
    
    logger.write(command)
    execute(command)
    
#------------------------------------------------------------------------------
#
# Create executable commands to run in batch
#
def executeBatch():
    
    CONTEXT['cputime'] = CONTEXT['shortcpu']

    # Record to be written to jobfile
    
    record = '''#!/usr/bin/env bash
. /etc/bashrc
. /usr/local/etc/pbssetup.sh

cd $PBS_O_WORKDIR
    
# Run within c-shell
    
csh <<EOF
source setup.sh
    
# Important: Make SINGLETOP_SIGMA_FILE point to the correct sigma-file
#            so that SigmaLoader loads the correct sigmas
    
setenv SINGLETOP_SIGMA_FILE %(sigmafile)s
    
%(program)s \\
-lepton          %(lep)s \\
-lepton_cut_name  good_electron  \\
-list            %(datalist)s \\
-histfile        %(outfile)s \\
-skimfile        %(skim_file)s \\
--obj-cut-file   %(defcuts)s \\
-cuts_file       %(cutsfile)s \\
-channel         %(sample)s \\
-trig_sig         %(trigsig)s \\
-mc_btag         %(mcbtag)s \\
-SVT_loose         %(SVTloose)s \\
-dataset         %(dataset)s \\
-event_weight        yes \\
#-nn_file         %(nn_file)s
EOF
''' % CONTEXT
    open(CONTEXT['jobfile'],'w').write(record)
    logger.write('\t%(jobfile)s\n' % CONTEXT + record)
    
    # Now for the command to be sent to the shell
    
    command = '''chmod u+x ./%(jobfile)s
echo " "	
echo "Submitting: " 
tail -n 2 ./%(jobfile)s
#
# Run it now!
#
cluesow \\
-j oe \\
-o %(logfile)s \\
-N %(cutstage)s_%(sysname)s \\
-l nodes=1 \\
-l cput=%(cputime)s \\
-l mem=%(memory)s \\
./%(jobfile)s

sleep 3 # this helps the clued0 batch system digest the jobs... 
''' % CONTEXT
    logger.write(command)
    
    execute(command)
    
#------------------------------------------------------------------------------
# 
# This function knows what cut files and weights to apply for each process 

def createJob(job):
    
    # Split up job string and update CONTEXT
    
    datalist, sample, leptonChannel = split(job)
    
    CONTEXT['datalist']='ana_me_cft/%s' % datalist
    CONTEXT['sample']  = sample
    CONTEXT['channel'] = leptonChannel
    if leptonChannel == 'mu':
        CONTEXT['lep'] = 'mu'
    else:
        CONTEXT['lep'] = 'e'
        
    lepton   = CONTEXT['lep']
    cutstage = CONTEXT['cutstage']
    trigsig = CONTEXT['trigsig']
    
    # Make sure leptonChannel is valid
    
    if not CUTS.has_key(leptonChannel):
	fatal('Need to specify one of these channels: mu, eCC or eEC.')

    if not CUTS[leptonChannel].has_key(cutstage):
        fatal("cutstage %s not found in map CUTS['%s']" % \
              (cutstage, leptonChannel))
            
    # Defaults
            
    CONTEXT['cutsfile']='ana_me_cft/cuts/electron_cuts/EMPTY_FILE.txt'
    CONTEXT['defcuts'] ='topd0root_singletop/cuts/default_cuts.txt'
    CONTEXT['jname'] = nameonly(datalist) # returns the last word before .list
    
    # Set the correct cut files:
    # The main difference now should be whether we apply the
    # FINAL cuts or not. 
    #
    
    CONTEXT['lep'] = lepton
    CONTEXT['cutsfile']  = CUTS[leptonChannel][cutstage]['cutsfile']
    #CONTEXT['trfweight'] = CUTS[leptonChannel][cutstage]['trfweight']

    # Create name of output root-tuple and logfile
    
    name = '%(cutstage)s'
    name = name + '_%(channel)s_%(sample)s_%(jname)s_%(sysname)s_%(runname)s'
   # name = name + '_%(channel)s_%(sample)s_%(runname)s'
    name = name % CONTEXT
    CONTEXT['name'] = name
    
    outfile = '%(dirname)s/hist/%(name)s.root'% CONTEXT
    skim_file = '%(dirname)s/skim/%(name)s_skim.root'% CONTEXT
    nn_file = '%(dirname)s/nn_skim/%(name)s_nn_skim.root'% CONTEXT
    logfile = '%(dirname)s/log/%(name)s.log'  % CONTEXT
    jobfile = '%(dirname)s/job/%(name)s.job'  % CONTEXT
    CONTEXT['outfile'] = outfile
    CONTEXT['skim_file'] = skim_file
    CONTEXT['nn_file'] = nn_file
    CONTEXT['logfile'] = logfile
    CONTEXT['jobfile'] = jobfile
    
    #
    # Check that we are not overwriting any previous root file:
    # 
    if exists(outfile):
	warn("File %(outfile)s already exists..." % CONTEXT)
	return
    #if exists(nn_file):
    #	warn("File %(nn_file)s already exists..." % CONTEXT)
    #	return

    # Locate executable and determine area containing sigma files
    
    #program = popen('which %(program)s' % CONTEXT).read()
    #if program == "":
    #    fatal("Can't locate program %(program)s" % CONTEXT)
    
    #t = split(program,'/')
    #CONTEXT['filearea'] = joinfields(t[:-3]) + \
    #                      '/topd0root_singletop/bin/%(program)s' % \
    
    if CONTEXT['batch']:
        executeBatch()
    else:
        executeInteractive()
        
    # Make jobfile executable just in case we wish to run them
    # directly from command line
    chmod(jobfile, S_IRWXU+S_IRGRP+S_IROTH)
    
#------------------------------------------------------------------------------
# 
# Run the whole show!

def run(sigmafile, jobs):
    CONTEXT['runname'] = nameonly(sigmafile)
    CONTEXT['sigmafile'] = sigmafile
    for job in jobs:
        command = 'createJob("%s")' % job
        logger.write(command)
        exec(command)
#------------------------------------------------------------------------------
# 
# Create sigma file

def createSigmaFile(row, names, sigmas):
    sigmafile = CONTEXT['fileprefix'] + '_%4.4d.txt' % row
    out = open(sigmafile,'w')
    out.write('%s\n' % (ctime(time())))
    for index in xrange(len(names)):
        out.write('%-32s\t%10.4f\n' % (names[index],sigmas[index]))
    out.close()
    return sigmafile
    
#==============================================================================
# Main Program
#==============================================================================
def main():

    # Decode command line arguments
    
    try:
        options, args = getopt(argv[1:], SHORTOPTIONS, LONGOPTIONS)
        
    except GetoptError, message:
        fatal(message)
        
    for option, value in options:
            
        if   option in ['-h','--help']:
            usage()
            exit(0)
                
        elif option in ['-t','-test']:
            CONTEXT['testmode'] = 1
                
        elif option in ['-b','--batch']:
            CONTEXT['batch'] = 1
                
        elif option in ['-s','--sample']:
            generateSample(atoi(value))
                
            exit(0)
                
        elif option in ['-f','--firstrow']:
            CONTEXT['firstrow'] = atoi(value)
                
        elif option in ['-l','--lastrow']:
            CONTEXT['lastrow'] = atoi(value)
                
        elif option == '--sysname':
            CONTEXT['sysname'] = value
                
        elif option == '--dirname':
            CONTEXT['dirname'] = value
                
        elif option == '--cutstage':
            CONTEXT['cutstage'] = value
        elif option == '--trigsig':
            CONTEXT['trigsig'] = value
        elif option == '--mcbtag':
            CONTEXT['mcbtag'] = value
        elif option == '--SVTloose':
            CONTEXT['SVTloose'] = value
                   
    # Check we have correct arguments! 
    #
    if CONTEXT['sysname'] == '':
        print USAGE
        fatal('need --sysname=<SYSNAME>')
                    
    if CONTEXT['cutstage'] == '':
        print USAGE
        fatal('need --cutstage=<CUTSTAGE>')
                        
    #
    # Check if final destination directories exists
    #
    system('mkdir -p %(dirname)s' % CONTEXT)
    system('mkdir -p %(dirname)s/hist' % CONTEXT)
    system('mkdir -p %(dirname)s/skim' % CONTEXT)
    system('mkdir -p %(dirname)s/nn_skim' % CONTEXT)    
    system('mkdir -p %(dirname)s/log' % CONTEXT)
    system('mkdir -p %(dirname)s/job' % CONTEXT)
                        
    #--------------------------------------------------------------------------
    # Here we specify which file we run on and what channels.
    # For example we can gain time if we run over skimmed (preselected) files 
    # and only apply the tagging cuts. Instead of running both preselection 
    # and tagging cuts over the raw files (many more events!). But that can
    # only be done for trigger and TRF, where we only apply a weight, not for
    # the JES, where we actually shift the number of events passing
    # preselection. So for JES we need to run from raw. So, as a rule,
    # please include all cuts (including preselection) in the (final) cut
    # files.
    #--------------------------------------------------------------------------
    
    if not CONTEXT['cutstage'] in CUTSTAGE:
        fatal('cutstage %(cutstage)s not recognized\n' \
              'cutsage should be one of:\n\t\t'\
              'Emqcd_Geq2jets_1em, Diem_2_cc_em, Data_pretagged, Data_NoTag, Data_EqOneTag, Data_GeqTwoTag, Data_tqb_GeqTwoTag, QCD_pretagged, QCD_NoTag, QCD_EqOneTag, QCD_GeqTwoTag,QCD_tqb_GeqTwoTag, pretagged, tb_NoTag, tb_EqOneTag, tb_GeqTwoTag, tqb_pretagged, tqb_EqOneTag, tqb_GeqTwoTag', 2)
    if CONTEXT['sysname'] == 'EMQCD_skim' :
        jobs = EMQCD_Ipanema
    if CONTEXT['sysname'] == 'ALLJETS_skim' :
        jobs = ALLJETS_Ipanema
    if CONTEXT['sysname'] == 'DIEM_skim' :
        jobs = DIEM_Ipanema
    if CONTEXT['sysname'] == 'JES_ana' :
        jobs = JES_related_JOBS
    elif CONTEXT['sysname'] == 'TRF_tb_ana' :
        jobs = NJESJOBS_tb
    elif CONTEXT['sysname'] == 'TRF_tqb_ana' :
        jobs = NJESJOBS_tqb
    elif CONTEXT['sysname'] == 'TRIG_tb_ana' :
        jobs = NJESJOBS_tb
    elif CONTEXT['sysname'] == 'TRIG_tqb_ana' :
        jobs = NJESJOBS_tqb
    elif CONTEXT['sysname'] == 'DATA_ana' :
        jobs = DATA_JOBS
    elif CONTEXT['sysname'] == 'DATA_ana_fs' :
        jobs = DATA_JOBS_fs
    elif CONTEXT['sysname'] == 'QCD_ana' :
        jobs = QCD_JOBS
    elif CONTEXT['sysname'] == 'QCD_ana_fs' :
        jobs = QCD_JOBS_fs

                   
    # If sigma files are specified, ignore -f and -l switches
    # and simply loop over these files
    #
    if len(args) > 0:
        for sigmafile in args:
            print "Use: ", sigmafile
            run(sigmafile, jobs)
            
    else:
            
        # Read in sigma sample. We use execfile so that the Python
        # objects within the sample file are added to the current
        # Python global namespace; that is, we make these objects
        # visible
            
        if not exists(CONTEXT['samplefile']):
            fatal("Sigma sample file %(samplefile)s NOT found" % CONTEXT)
            
        exec(open(CONTEXT['samplefile']).read())
                
        sampleSize = len(SAMPLE)
                
        # Loop over previously created samples
        # Make sure first and last row values are sensible
        firstrow = CONTEXT['firstrow']
        lastrow  = CONTEXT['lastrow']
        if lastrow < firstrow:
            fatal("Don't be an idiot lastrow should be >= firstrow")
                    
        if lastrow < 0 or firstrow < 0:
            fatal("Go boil your head! lastrow and firstrow should >= 0!")
                        
        if lastrow >= sampleSize:
            fatal("lastrow is greater than sample size (%d)" % sampleSize)
            
        row = firstrow
                            
        for sigmas in SAMPLE[firstrow:lastrow+1]:
            sigmafile = createSigmaFile(row, NAME, sigmas)
            print "Use: ", sigmafile
            run(sigmafile, jobs)
            row = row + 1
    #
    # Voila! 
    #
    print "Finished!"
                                
#==============================================================================
main()