iceprod.server

1 #! /usr/bin/env python 2 # -*- coding: utf-8 -*- 3 """ 4 copyright (c) 2005 5 the icecube collaboration 6 7 Collection of python modules for scheduling and running Grid IceTray based jobs in 8 a grid environment 9 10 @version: 0.3.8 11 @date: mar abr 6 11:37:59 CDT 2010 12 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 13 @brief: A set of python tools for submitting and managing jobs on 14 """ 15 import os,sys 16 import os.path 17 from os.path import join,expandvars 18 from ConfigParser import ConfigParser 19 from ConfigParser import SafeConfigParser 20 21 __version__ = '0.3.8' 22 23

24 -class IPConfigParser(SafeConfigParser):

25 """ 26 SafeConfig parser subclass which overrides the default case-insesitive 27 options. 28 """

29 - def optionxform(self,opt):

30 return str(opt)

31

32 -def mkcfg(basedir,path):

33 34 localhost = os.uname()[1] 35 cfg = ConfigParser() 36 37 # Read default values from template 38 cfg.read(join(basedir,'etc/template-iceprod.cfg')) 39 40 # Set some guesses 41 cfg.set('server','server',localhost) 42 cfg.set('monitoring','server',localhost) 43 cfg.set('monitoring','maxthreads','10') 44 cfg.set('soapdh','local_target','') 45 cfg.set('path','photontables','') 46 cfg.set('queue','jobstageout','True') 47 48 # photontablesdir 49 if cfg.has_option('path','photontables'): 50 if not cfg.has_option('path','photontablesdir'): 51 cfg.set('path','photontablesdir',cfg.get('path','photontables')+'/..') 52 cfg.remove_option('path','photontables') 53 54 # Import configuration from sim-prod 55 prompt = "please enter the path to an existing \niceprod.cfg or sim-prod.cfg: " 56 foundcfg = False 57 while not foundcfg: 58 try: 59 import readline 60 readline.parse_and_bind("tab: complete") 61 oldcfg = raw_input(prompt) 62 except: 63 sys.stdout.write(prompt) 64 oldcfg = sys.stdin.readline().strip() 65 foundcfg = os.path.exists(oldcfg) 66 67 try: 68 cfg.read(oldcfg) 69 newcfg = open(path,'w') 70 cfg.write(newcfg) 71 newcfg.close() 72 sys.stdout.write("A new configuration has been written to '%s'.\n" % path) 73 sys.stdout.write("Please review it before starting the server.\n") 74 except Exception,e: 75 sys.stderr.write("Unable to import config '%s'.\n" % oldcfg) 76 sys.stderr.write(e) 77 os._exit(0)

78 79

80 -def locatecfg(executable=None):

81 # See if there is a configuration file setup 82 localhost = os.uname()[1].split(".")[0] 83 basedir = os.getenv('I3PROD') 84 if not basedir: 85 if executable: 86 relpath = executable.split('/') 87 if len(relpath) == 1: 88 for dir in os.getenv('PATH').split(':'): 89 if os.path.isfile(join(dir,executable)): 90 basedir = os.path.dirname(dir) 91 break 92 else: 93 basedir = os.path.dirname(os.path.dirname(executable)) 94 if not basedir.startswith('/'): 95 basedir = os.path.abspath(basedir) 96 97 98 cfgpath_list = [ 99 # First see if there is a ~/.iceprod directory 100 join(expandvars("$HOME/.iceprod/iceprod.cfg")), 101 # Then look in $I3PROD/etc/$HOSTNAME 102 join(basedir,'etc',localhost,'iceprod.cfg'), 103 # Then look in $I3PROD/etc/ 104 join(basedir,'etc','iceprod.cfg'), 105 # Then try simprod 106 join(basedir,'etc',localhost,'simprod.cfg'), 107 join(basedir,'etc','iceprod.cfg'), 108 ] 109 for cfgpath in cfgpath_list: 110 if os.path.exists(cfgpath): 111 return basedir,cfgpath 112 mkcfg(basedir,join(basedir,'etc','iceprod.cfg')) 113 114 raise Exception, "did not find a configuration file in %s'" % ",".join(cfgpath_list)

115

116 -def getconfig(basedir,cfgpath,verbose=False):

117 118 cfg = SafeConfigParser() 119 120 # Set some default values 121 cfg.add_section('queue') 122 cfg.set('queue','maxjobs','100') 123 cfg.set('queue','jobstageout','True') 124 cfg.set('queue','sockettimeout','60') 125 cfg.set('queue','checkinterval','3') 126 cfg.set('queue','max_job_idle_time','800') 127 cfg.set('queue','max_job_submit_time','2') 128 cfg.set('queue','max_job_processing_time','500') 129 cfg.set('queue','max_job_copy_time','10') 130 cfg.set('queue','max_job_error_time','10') 131 cfg.set('queue','max_evict_time','20') 132 cfg.set('queue','days_before_clean_sdir','3') 133 cfg.set('queue','submit_jobs_per_interval','40') 134 cfg.set('queue','max_job_copy','50') 135 cfg.set('queue','ping_interval','480') 136 cfg.set('queue','fifo','True') 137 cfg.set('queue','cleanq','False') 138 cfg.set('queue','max_job_copy','10') 139 cfg.set('queue','max_failures','10') 140 cfg.set('queue','zipsafe','True') 141 cfg.set('queue','jobprefix','iceprod') 142 cfg.set('queue','stage_pymods','True') 143 144 cfg.add_section('server') 145 cfg.set('server','port','9080') 146 147 cfg.add_section('ldap') 148 cfg.set('ldap','enable','False') 149 cfg.set('ldap','url',"ldaps://ldap.icecube.wisc.edu") 150 cfg.set('ldap','path',"dc=icecube,dc=wisc,dc=edu") 151 ConfigParser.set(cfg,'ldap','cn','uid=%s,ou=People,dc=icecube,dc=wisc,dc=edu') 152 cfg.set('ldap','users','') 153 154 cfg.add_section('monitoring') 155 cfg.set('monitoring','maxthreads','10') 156 cfg.set('monitoring','port','9081') 157 158 cfg.add_section('soapdh') 159 cfg.set('soapdh','checkinterval','5') 160 161 cfg.add_section('globus') 162 cfg.set('globus','directory','$GLOBUS_LOCATION') 163 cfg.set('globus','job_globus_location','globus') 164 cfg.set('globus','libs','') 165 cfg.set('globus','proxy','/tmp/x509up_u%s' % os.getuid()) 166 cfg.set('globus','proxy-init','grid-proxy-init') 167 cfg.set('globus','proxy-info','grid-proxy-info') 168 cfg.set('globus','useproxy','True') 169 cfg.set('globus','delegate','True') 170 171 cfg.add_section('environment') 172 cfg.set('environment','scratch','$PWD') 173 cfg.set('environment','pythonhome','/usr') 174 cfg.set('environment','platform','system') 175 176 cfg.add_section('job-env') 177 cfg.add_section('server-env') 178 179 localhost = os.uname()[1].split(".")[0] 180 cfg.add_section('logging') 181 cfg.set('logging','level','INFO') 182 cfg.set('logging','format','%(module)s %(asctime)s %(levelname)s %(name)s : %(message)s') 183 cfg.set('logging','logfile', join('log',localhost,'iceprod.err')) 184 cfg.set('logging','iceprod', join('log',localhost,'iceprod.log')) 185 cfg.set('logging','soaptray', join('log',localhost,'soaptray.log')) 186 cfg.set('logging','soapmon', join('log',localhost,'soapmon.log')) 187 cfg.set('logging','soapqueue',join('log',localhost,'soapqueue.log')) 188 cfg.set('logging','soapdh', join('log',localhost,'soapdh.log')) 189 cfg.set('logging','logrecv',join('log',localhost,'logserver.log')) 190 cfg.set('logging','logserver','skua.icecube.wisc.edu:9085') 191 192 cfg.add_section('path') 193 cfg.set('path','target_url','gsiftp://data.icecube.wisc.edu/mnt/sim/data/sim/IceCube') 194 cfg.set('path','datawarehouse','/data/sim/IceCube/%(year)s/generated/%(subcategory)s/%(dataset_id)s') 195 cfg.set('path','lib_url','http://x2100.icecube.wisc.edu/downloads') 196 197 cfg.add_section('database') 198 cfg.set('database','autocommit','True') 199 cfg.set('database','server','dbs2.icecube.wisc.edu') 200 cfg.set('database','database','i3simprod') 201 cfg.set('database','port','3306') 202 cfg.set('database','non-production','/tmp/%s.iceprod.npdb'% os.getuid()) 203 cfg.set('database','maxconnections','10') 204 205 cfg.add_section('system') 206 cfg.set('system','urlcopy','1') 207 cfg.set('system','lfncopy','0') 208 cfg.set('system','localcp','0') 209 cfg.set('system','X509_CERT_DIR','$GLOBUS_LOCATION/certificates') 210 cfg.set('system','validatexml','False') 211 212 if verbose: 213 print "Reading configuration from '%s'" % cfgpath 214 if not os.path.exists(cfgpath): 215 raise Exception, "did not find a configuration file in %s'" % cfgpath 216 cfg.read(cfgpath) 217 if cfg.has_section('globus') and not cfg.has_option('globus','job_globus_location'): 218 cfg.set('globus','job_globus_location','globus') 219 cfg.set('path','basedir',basedir) 220 221 # check cache directory and set default if none 222 if cfg.has_option('system','cache') and cfg.has_option('path','cache'): 223 raise Exception, "You can't set both [system]cache and [path]cache. Pick one." 224 elif not (cfg.has_option('system','cache') or cfg.has_option('path','cache')): 225 cfg.set('system','cache','$PWD') 226 227 return cfg

228 229

230 -def zipfile(fmt= "iceprod-%(version)s"):

231 vars = {'version':__version__, 'platform':os.uname()[0],'arch': os.uname()[4]} 232 return fmt % vars

233 234

235 -def mktar(libdir,outfile):

236 curdir = os.getcwd() 237 os.chdir(os.path.join(libdir,'..')) 238 os.system("zip -q -r %s.zip %s -i \*.py" % (outfile,os.path.split(libdir)[-1])) 239 os.chdir(curdir)

240

241 -def email(cfg,msg,level='ALERT'):

242 import smtplib 243 from_addr = "From: " + cfg.get('monitoring','smtpuser') 244 if level == 'INFO': 245 to_addrs = cfg.get('monitoring','smtpinfo').split(',') 246 else: 247 to_addrs = cfg.get('monitoring','smtpnotify').split(',') 248 server = smtplib.SMTP(smtphost) 249 msg = "Subject: %s\n" % level + msg 250 server.sendmail(from_addr, to_addrs, msg) 251 server.quit()

252

Source Code for Package iceprod.server