iceprod.server

1 #! /usr/bin/env python 2 # -*- coding: utf-8 -*- 3 """ 4 copyright (c) 2005 5 the icecube collaboration 6 7 Collection of python modules for scheduling and running Grid IceTray based jobs in 8 a grid environment 9 10 @version: 0.3.4 11 @date: Wed Aug 27 18:45:46 CDT 2008 12 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 13 @brief: A set of python tools for submitting and managing jobs on 14 """ 15 import os,sys 16 import os.path 17 from os.path import join,expandvars 18 from ConfigParser import ConfigParser 19 from ConfigParser import SafeConfigParser 20 21 __version__ = '0.3.4' 22 23

24 -class IPConfigParser(SafeConfigParser):

25 """ 26 SafeConfig parser subclass which overrides the default case-insesitive 27 options. 28 """

29 - def optionxform(self,opt):

30 return str(opt)

31

32 -def mkcfg(basedir,path):

33 34 localhost = os.uname()[1] 35 cfg = ConfigParser() 36 37 # Read default values from template 38 cfg.read(join(basedir,'etc/template-iceprod.cfg')) 39 40 # Set some guesses 41 cfg.set('server','server',localhost) 42 cfg.set('monitoring','server',localhost) 43 cfg.set('monitoring','maxthreads','10') 44 cfg.set('soapdh','local_target','') 45 cfg.set('queue','jobstageout','True') 46 47 # Import configuration from sim-prod 48 prompt = "please enter the path to an existing \niceprod.cfg or sim-prod.cfg: " 49 foundcfg = False 50 while not foundcfg: 51 try: 52 import readline 53 readline.parse_and_bind("tab: complete") 54 oldcfg = raw_input(prompt) 55 except: 56 sys.stdout.write(prompt) 57 oldcfg = sys.stdin.readline().strip() 58 foundcfg = os.path.exists(oldcfg) 59 60 try: 61 cfg.read(oldcfg) 62 newcfg = open(path,'w') 63 cfg.write(newcfg) 64 newcfg.close() 65 sys.stdout.write("A new configuration has been written to '%s'.\n" % path) 66 sys.stdout.write("Please review it before starting the server.\n") 67 except Exception,e: 68 sys.stderr.write("Unable to import config '%s'.\n" % oldcfg) 69 sys.stderr.write(e) 70 os._exit(0)

71 72

73 -def locatecfg(executable=None):

74 # See if there is a configuration file setup 75 localhost = os.uname()[1].split(".")[0] 76 basedir = os.getenv('I3PROD') 77 if not basedir: 78 if executable: 79 relpath = executable.split('/') 80 if len(relpath) == 1: 81 for dir in os.getenv('PATH').split(':'): 82 if os.path.isfile(join(dir,executable)): 83 basedir = os.path.dirname(dir) 84 break 85 else: 86 basedir = os.path.dirname(os.path.dirname(executable)) 87 if not basedir.startswith('/'): 88 basedir = os.path.abspath(basedir) 89 90 91 cfgpath_list = [ 92 # First see if there is a ~/.iceprod directory 93 join(expandvars("$HOME/.iceprod/iceprod.cfg")), 94 # Then look in $I3PROD/etc/$HOSTNAME 95 join(basedir,'etc',localhost,'iceprod.cfg'), 96 # Then look in $I3PROD/etc/ 97 join(basedir,'etc','iceprod.cfg'), 98 # Then try simprod 99 join(basedir,'etc',localhost,'simprod.cfg'), 100 join(basedir,'etc','iceprod.cfg'), 101 ] 102 for cfgpath in cfgpath_list: 103 if os.path.exists(cfgpath): 104 return basedir,cfgpath 105 mkcfg(basedir,join(basedir,'etc','iceprod.cfg')) 106 107 raise Exception, "did not find a configuration file in %s'" % ",".join(cfgpath_list)

108

109 -def getconfig(basedir,cfgpath,verbose=False):

110 111 cfg = SafeConfigParser() 112 113 # Set some default values 114 cfg.add_section('queue') 115 cfg.set('queue','maxjobs','100') 116 cfg.set('queue','jobstageout','True') 117 cfg.set('queue','sockettimeout','60') 118 cfg.set('queue','checkinterval','3') 119 cfg.set('queue','max_job_idle_time','800') 120 cfg.set('queue','max_job_submit_time','2') 121 cfg.set('queue','max_job_processing_time','500') 122 cfg.set('queue','max_job_copy_time','10') 123 cfg.set('queue','max_job_error_time','10') 124 cfg.set('queue','max_evict_time','20') 125 cfg.set('queue','days_before_clean_sdir','3') 126 cfg.set('queue','submit_jobs_per_interval','40') 127 cfg.set('queue','max_job_copy','50') 128 cfg.set('queue','ping_interval','480') 129 cfg.set('queue','fifo','True') 130 cfg.set('queue','cleanq','True') 131 cfg.set('queue','max_job_copy','10') 132 cfg.set('queue','max_failures','10') 133 cfg.set('queue','zipsafe','True') 134 cfg.set('queue','jobprefix','iceprod') 135 cfg.set('queue','stage_pymods','True') 136 cfg.set('queue','jobclustering','1') 137 138 cfg.add_section('server') 139 cfg.set('server','port','9080') 140 141 cfg.add_section('security') 142 cfg.set('security','USESSL','false') 143 144 cfg.add_section('ldap') 145 cfg.set('ldap','enable','False') 146 cfg.set('ldap','url',"ldaps://ldap.icecube.wisc.edu") 147 cfg.set('ldap','path',"dc=icecube,dc=wisc,dc=edu") 148 ConfigParser.set(cfg,'ldap','cn','uid=%s,ou=People,dc=icecube,dc=wisc,dc=edu') 149 cfg.set('ldap','users','') 150 151 cfg.add_section('monitoring') 152 cfg.set('monitoring','maxthreads','10') 153 cfg.set('monitoring','port','9081') 154 155 cfg.add_section('soapdh') 156 cfg.set('soapdh','checkinterval','5') 157 158 cfg.add_section('globus') 159 cfg.set('globus','directory','$GLOBUS_LOCATION') 160 cfg.set('globus','job_globus_location','globus') 161 cfg.set('globus','libs','$GLOBUS_LIBS') 162 cfg.set('globus','proxy','/tmp/x509up_u%s' % os.getuid()) 163 cfg.set('globus','proxy-init','grid-proxy-init') 164 cfg.set('globus','proxy-info','grid-proxy-info') 165 cfg.set('globus','useproxy','True') 166 cfg.set('globus','delegate','True') 167 168 cfg.add_section('environment') 169 cfg.set('environment','scratch','$PWD') 170 cfg.set('environment','pythonhome','/usr') 171 cfg.set('environment','platform','system') 172 173 cfg.add_section('job-env') 174 cfg.add_section('server-env') 175 176 localhost = os.uname()[1].split(".")[0] 177 cfg.add_section('logging') 178 cfg.set('logging','level','INFO') 179 cfg.set('logging','format','%(module)s %(asctime)s %(levelname)s %(name)s : %(message)s') 180 cfg.set('logging','logfile', join('log',localhost,'iceprod.err')) 181 cfg.set('logging','iceprod', join('log',localhost,'iceprod.log')) 182 cfg.set('logging','soaptray', join('log',localhost,'soaptray.log')) 183 cfg.set('logging','soapmon', join('log',localhost,'soapmon.log')) 184 cfg.set('logging','soapqueue',join('log',localhost,'soapqueue.log')) 185 cfg.set('logging','soapdh', join('log',localhost,'soapdh.log')) 186 cfg.set('logging','logrecv',join('log',localhost,'logserver.log')) 187 cfg.set('logging','server','skua.icecube.wisc.edu:9085') 188 cfg.set('logging','rmtlog','False') 189 190 cfg.add_section('path') 191 cfg.set('path','target_url','gsiftp://data.icecube.wisc.edu/mnt/sim/data/sim/IceCube') 192 cfg.set('path','datawarehouse','/data/sim/IceCube/%(year)s/generated/%(subcategory)s/%(dataset_id)s') 193 cfg.set('path','lib_url','http://x2100.icecube.wisc.edu/downloads') 194 cfg.set('path','web_url','http://internal.icecube.wisc.edu/simulation') 195 196 cfg.add_section('database') 197 cfg.set('database','autocommit','False') 198 cfg.set('database','server','dbs4.icecube.wisc.edu') 199 cfg.set('database','database','i3simprod') 200 cfg.set('database','port','3306') 201 cfg.set('database','non-production','/tmp/%s.iceprod.npdb'% os.getuid()) 202 cfg.set('database','maxconnections','10') 203 204 cfg.add_section('system') 205 cfg.set('system','urlcopy','1') 206 cfg.set('system','lfncopy','0') 207 cfg.set('system','localcp','0') 208 cfg.set('system','X509_CERT_DIR','$GLOBUS_LOCATION/certificates') 209 cfg.set('system','validatexml','False') 210 cfg.set('system','jobstageout','True') 211 cfg.set('system','ignore_gftp_errors','False') 212 213 # For using a downloaded python tarball 214 cfg.set('system','external_python','False') 215 cfg.set('system','external_python_name',' ') 216 217 if verbose: 218 print "Reading configuration from '%s'" % cfgpath 219 if not os.path.exists(cfgpath): 220 raise Exception, "did not find a configuration file in %s'" % cfgpath 221 cfg.read(cfgpath) 222 if cfg.has_section('globus') and not cfg.has_option('globus','job_globus_location'): 223 cfg.set('globus','job_globus_location','globus') 224 cfg.set('path','basedir',basedir) 225 226 # check cache directory and set default if none 227 if cfg.has_option('system','cache') and cfg.has_option('path','cache'): 228 raise Exception, "You can't set both [system]cache and [path]cache. Pick one." 229 elif not (cfg.has_option('system','cache') or cfg.has_option('path','cache')): 230 cfg.set('system','cache','$PWD') 231 232 return cfg

233 234

235 -def zipfile(fmt= "iceprod-%(version)s"):

236 vars = {'version':__version__, 'platform':os.uname()[0],'arch': os.uname()[4]} 237 return fmt % vars

238 239

240 -def mktar(libdir,outfile):

241 curdir = os.getcwd() 242 os.chdir(os.path.join(libdir,'..')) 243 os.system("zip -q -r %s.zip %s -i \*.py" % (outfile,os.path.split(libdir)[-1])) 244 os.chdir(curdir)

245

246 -def email(cfg,msg,level='ALERT'):

247 import smtplib 248 from_addr = "From: " + cfg.get('monitoring','smtpuser') 249 if level == 'INFO': 250 to_addrs = cfg.get('monitoring','smtpinfo').split(',') 251 else: 252 to_addrs = cfg.get('monitoring','smtpnotify').split(',') 253 server = smtplib.SMTP(smtphost) 254 msg = "Subject: %s\n" % level + msg 255 server.sendmail(from_addr, to_addrs, msg) 256 server.quit()

257

Source Code for Package iceprod.server