Source Code for Module iceprod.core.lex

  1  #!/usr/bin/python 
  2  # @author: Juan Carlos Diaz-Velez 
  3  #/@date:   04 May 2005 
  4  # (c) IceCube collaboration. 
  5  # @brief:  Steering script for running I3Sim on Condor. 
  6  # Evaluator: Module for parsing and evaluating expressions in IceTray values 
  7  # 
  8  ###################################################################### 
  9  import os,sys,time,string,re 
 10  import signal 
 11  import random 
 12  import getpass 
 13  import os.path,getopt 
 14  import commands 
 15  import platform 
 16  from os.path import expandvars,join,exists 
 17  from dataclasses import Steering 
 18  from metadata    import DIF_Plus 
 19  from iceprod.core import ipxml 
 20  from iceprod.core.odict import OrderedDict 
 21  from random import choice 
 22  import glob 
 23   
 24   
 25  # The following modules might not be available until restart 
 26  import xmlrpclib 
 27  import socket 
 28  import cPickle 
 29  from ConfigParser import SafeConfigParser 
 30   
 31  _platform = None 
 32  _done  = False 
 33  _clean = True 
 34   
 35  EVICTED = 1 
 36  FAILED  = 2 
 37  python_version = platform.python_version()[:3] 
 38   
 39   
 40 -class XMLSummaryParser: 
 41      """ 
 42      XML Parser for files generated by the I3SummaryService in IceTray. 
 43      """ 
 44   
 45 -    def __init__(self): 
 46          self.summary_map = {} 
 47   
 48 -    def ParseFile(self,config_file_name):  
 49   
 50          # get the DOM tree 
 51          doc = None 
 52          try:  
 53              # open xml file for reading 
 54              xmlin = open(config_file_name, 'r') 
 55              doc = ipxml.Parse(xmlin) 
 56          except Exception, xe:  
 57              return  
 58           
 59          i3map = ipxml.Evaluate('boost_serialization/I3XMLSummaryService/map', doc) 
 60          if i3map: 
 61              count = int(i3map[0].getElementsByTagName('count')[0].firstChild.data) 
 62              for item in i3map[0].getElementsByTagName('item'): 
 63                  key = item.getElementsByTagName('first')[0].firstChild.data 
 64                  value = item.getElementsByTagName('second')[0].firstChild.data 
 65                  self.summary_map[key] = float(value) 
 66   
 67          return self.summary_map 
 68   
 69 -    def Write(self,fname):  
 70   
 71          dom     = ipxml.getDOMImplementation() 
 72          doctype = dom.createDocumentType("boost_serialization",None,None) 
 73          doc     = dom.createDocument( None, "boost_serialization", doctype ) 
 74   
 75          boost   = doc.documentElement 
 76          boost._attrs = OrderedDict() 
 77          boost._attrsNS = OrderedDict()  
 78          boost.setAttribute('signature', 'serialization::archive') 
 79          boost.setAttribute('version', '3') 
 80   
 81          summary = doc.createElement( 'I3XMLSummaryService' ) 
 82          summary.setAttribute('class_id','0') 
 83          summary.setAttribute('tracking_level','1') 
 84          summary.setAttribute('version','0') 
 85          summary.setAttribute('object_id','_0') 
 86          boost.appendChild(summary) 
 87   
 88          frameobj = doc.createElement( 'I3FrameObject' ) 
 89          frameobj.setAttribute("class_id","1") 
 90          frameobj.setAttribute("tracking_level","1") 
 91          frameobj.setAttribute("version","0") 
 92          frameobj.setAttribute("object_id","_1") 
 93          frameobj.appendChild(doc.createTextNode("")) 
 94          summary.appendChild(frameobj) 
 95   
 96          map = doc.createElement( 'map' ) 
 97          map.setAttribute('class_id','2') 
 98          map.setAttribute('tracking_level','0') 
 99          map.setAttribute('version','0') 
100          summary.appendChild(map) 
101   
102          count = doc.createElement( 'count' ) 
103          count.appendChild(doc.createTextNode("%u" % len(self.summary_map.items()))) 
104          map.appendChild(count) 
105   
106          for key,value in self.summary_map.items(): 
107              item   = doc.createElement( 'item' ) 
108   
109              first  = doc.createElement( 'first' ) 
110              first.appendChild(doc.createTextNode(key)) 
111              item.appendChild(first) 
112   
113              second = doc.createElement( 'second' ) 
114              second.appendChild(doc.createTextNode("%f" % value)) 
115              item.appendChild(second) 
116   
117              map.appendChild(item) 
118   
119          file_object = open(fname, "w") 
120          ipxml.myPrettyPrint(doc, file_object) 
121          file_object.close() 
122   
123  #------ end of class XMLSummaryParser ----- 
124   
125  #------ start of class ExpParser ----- 
126 -class ExpParser: 
127      """ 
128      Expression parsing class for parameter values.  
129      """ 
130      keywords = ['sprintf','args','metadata','steering','eval','file','glob','dict','system'] 
131   
132 -    def __init__(self,opts={},steering=Steering(),cache=False,noeval=False): 
133          self.opts        = opts 
134          self.steering    = steering 
135          self.steerdict   = self.steering.get_param_dict() 
136          self.sysopts     = self.steering.get_sysopt_dict() 
137          self.difplus     = DIF_Plus() 
138          if self.steering.HasDIFPlus(): 
139             self.difplus  = self.steering.GetDIFPlus() 
140   
141          self.dpdict      = self.difplus.MakeDict(int(self.opts['procnum']))  
142          self.server      = None 
143          self.noeval      = noeval 
144          self.spdict      = None 
145          self.spdictfd    = None 
146          if self.opts.has_key('url'):  
147              import xmlrpclib,socket 
148              self.server  = xmlrpclib.ServerProxy(self.opts['url']) 
149          if cache: 
150              dictpath     = 'spdict.cfg' 
151              self.spdict  = SafeConfigParser() 
152              if os.path.exists(dictpath): 
153                  self.spdict.read(dictpath) 
154              self.spdictfd = open(dictpath,'w') 
155   
156 -    def AddSysOpt(self,name,value): 
157          self.sysopts[name] = value 
158   
159 -    def AddOpt(self,name,value): 
160          self.opts[name] = value 
161   
162 -    def AddSteering(self,name,value): 
163          self.steerdict[name] = value 
164   
165 -    def sprintf(self,pstring,args): 
166        """ 
167        Format string using the arguments in args 
168        """ 
169        return_string = pstring % tuple(map(self.parse,args)) 
170   
171 -    def fetchfilename(self,key): 
172        if self.opts.has_key('url'):  
173            if self.opts.has_key('dataset'): 
174              val = self.server.getfile(key,self.opts['dataset']) 
175            else: 
176              val = self.server.getfile(key,0) 
177            return val 
178        else: return "" 
179   
180 -    def fetchval(self,key): 
181        if self.spdict:  
182          if self.spdict.has_option('dict',key):  
183              return self.spdict.get('dict',key) 
184   
185        if self.opts.has_key('url'):  
186            val = self.server.getvalue(key) 
187            if self.spdict:  
188              self.spdict.set('dict',key,val) 
189              self.spdict.write(self.spdictfd) 
190            return val 
191        else: return "$dict(%s)" % key 
192   
193 -    def safe_eval(self,pstring): 
194          """ 
195          Evalualte Python expression after verifying that it does not contain 
196          potentially harmful or exploitable expressions. 
197          @param pstring: string containing expression 
198          """ 
199          bad = re.search(r'(import)|(open)|(for)|(while)|(lambda)', pstring ) 
200          if bad: 
201              raise SyntaxError, 'Illegal operator call \'%s\'' % bad.group(0) 
202          else: 
203              return str(eval(pstring)) 
204   
205 -    def _getarg(self,pstring): 
206          """ 
207          Retrieve argument from dictionary of commandline arguments/options 
208          @param pstring: string containing expression 
209          """ 
210          pval = pstring.strip() 
211          if self.opts.has_key(pval): 
212              return str(self.opts[pval]) 
213          else: 
214              raise NameError, 'option $args(%s) is not defined try %s ' % (pval,",".join(self.opts.keys())) 
215   
216 -    def _getsteering(self,pstring): 
217          """ 
218          Retrieve value from dictionary of steering parameters 
219          @param pstring: string containing expression 
220          """ 
221          pname = pstring.strip() 
222          try: 
223                pname = self.steerdict[pname] 
224          except Exception,e: 
225                raise NameError, 'name %s is not defined' % pname 
226          return self.parse(str(pname)) 
227   
228 -    def _getdifplus(self,pstring): 
229          """ 
230          Retrieve value from dictionary of steering parameters 
231          @param pstring: string containing expression 
232          """ 
233          pstr  = pstring.strip() 
234          pval = pstr % self.dpdict 
235          return self.parse(str(pval)) 
236   
237   
238 -    def _getsysopt(self,pstring): 
239          """ 
240          Retrieve value from dictionary of system options  
241          @param pstring: string containing expression 
242          """ 
243          pname = pstring.strip() 
244          try: 
245                pname = self.sysopts[pname] 
246          except Exception,e: 
247                raise NameError, 'name %s is not defined' % pname 
248          return self.parse(str(pname)) 
249   
250 -    def parse(self,pstring): 
251          """ 
252          Parse string for valid expressions. 
253          @param pstring: string containing expression 
254          """ 
255          pstr = '' 
256          c = 0 
257          while c < len(pstring): 
258              for key in self.keywords: 
259                  substr = "$%s(" % key 
260                  if pstring[c:].startswith(substr): 
261                      args, remainder = self.match('(',pstring[c+len(substr):]) 
262                      pstr += self.call(key,args) 
263                      if len(remainder) == 0: 
264                          return  pstr 
265                      else: 
266                          pstring = remainder 
267                          c = 0 
268              pstr += pstring[c] 
269              c += 1 
270          return (str(expandvars(pstr))) 
271                   
272   
273 -    def call(self,func,args): 
274          """ 
275          Parse a function call in the form $func(args) 
276          @param func: name of function 
277          @param args: list of arguments to pass to function 
278          """ 
279          if func == 'sprintf': 
280              sp  = args.find('\"',1) 
281              strfmt = args[0:sp].strip('\"') 
282              if len(args.split('\"')) > 2: 
283                  args = map(string.strip,self.parse(args[sp+1:]).split(',')) 
284                  if self.noeval:  
285                      if len(args) > 1: 
286                          return "$sprintf(\"%s\", %s)" % (strfmt, ",".join(args[1:]) ) 
287                      else: 
288                          return strfmt  
289                  return self.format_string(strfmt,args[1:]) 
290              else: 
291                  return args[0] 
292          elif func == 'eval': 
293              #if self.noeval: return "$eval(%s)" % self.parse(args) 
294              return self.safe_eval(self.parse(args)) 
295          elif func == 'args': 
296              #if self.noeval: return "$args(%s)" % self.parse(args) 
297              return self._getarg(self.parse(args)) 
298          elif func == 'file': 
299              return self.fetchfilename(self.parse(args)) 
300          elif func == 'dict': 
301              return self.fetchval(self.parse(args)) 
302          elif func == 'steering': 
303              return self._getsteering(self.parse(args)) 
304          elif func == 'metadata': 
305              if self.noeval: return "$metadata(%s)" % self.parse(args) 
306              return self._getdifplus(self.parse(args)) 
307          elif func == 'system': 
308              if self.noeval: return "$system(%s)" % self.parse(args) 
309              return self._getsysopt(self.parse(args)) 
310          #elif func == 'glob': 
311          #    return str(glob.glob(self.parse(args))) 
312          else:  
313              return "$%s(%s)" % (func,self.parse(args)) 
314   
315   
316 -    def match(self,ochar,s): 
317          """ 
318          Parse string until finding the matching closing character for ochar 
319          (),[],{} 
320          @param ochar:  opening character  
321          @param s: string to parse 
322          """ 
323          substring = '' 
324          remainder = '' 
325          ocount = 1 
326          for c in s: 
327              if ocount > 0: 
328                  if  self.complement(ochar) == c: 
329                      ocount -= 1 
330                      if ocount == 0: continue 
331                  elif ochar == c: 
332                      ocount += 1 
333                  substring += c 
334              else: 
335                  remainder += c 
336   
337          return (substring,remainder) 
338   
339   
340 -    def complement(self,ochar): 
341          """ 
342          return complent of char or matching opening and closing symbols 
343          (),[],{} 
344          @param ochar:  character  
345          @return: complementary character for ochar 
346          """ 
347   
348          if '(': 
349              return ')' 
350          elif '[': 
351              return ']' 
352          elif '{': 
353              return '}' 
354   
355 -    def format_string(self,pstring,args): 
356          """ 
357          Format string using the arguments in args 
358          """ 
359          return_string = pstring 
360          if not args: 
361              return return_string 
362   
363          try: 
364              fstrings = re.findall(r'\%[0-9]*\.{0,1}[0-9]*[csridufegExXo]',pstring) 
365              args = map(self.cast_string,fstrings,args) 
366              return_string = pstring % tuple(args) 
367          except TypeError,e: 
368              print "%s: %s" %( sys.exc_type, str(e)) 
369              print 'Unable to format string parameter ' + pstring + ' with args ' +str(args) 
370   
371          return return_string 
372               
373 -    def cast_string(self,fstring,arg): 
374          """ 
375          cast string to value according to formatting character 
376          @param fstring: formatting string (e.g. '%006d') 
377          @param arg: value to cast 
378          @return: converted value 
379          """ 
380          if not fstring: return arg 
381          if   fstring.endswith('c'): return str(arg) 
382          elif fstring.endswith('s'): return str(arg) 
383          elif fstring.endswith('r'): return repr(arg) 
384          elif fstring.endswith('i'): return int(arg) 
385          elif fstring.endswith('d'): return int(arg) 
386          elif fstring.endswith('u'): return int(arg) 
387          elif fstring.endswith('f'): return float(arg) 
388          elif fstring.endswith('e'): return float(arg) 
389          elif fstring.endswith('g'): return float(arg) 
390          elif fstring.endswith('E'): return float(arg) 
391          elif fstring.endswith('x'): return int('0x%s' % arg, 0) 
392          elif fstring.endswith('X'): return int('0x%s' % arg, 0) 
393          elif fstring.endswith('o'): return int('0%s' % arg, 0) 
394          else: 
395              print 'Unable to cast %s using format %s' (arg,fstring) 
396              print 'assuming string' 
397              return arg 
398   
399 -class FakeExpParser(ExpParser): 
400      """ 
401      Expression parsing class for parameter values.  
402      """ 
403      steer = [] 
404   
405 -    def parse(self,pstring): 
406          """ 
407          Parse string for valid expressions. 
408          @param pstring: string containing expression 
409          """ 
410          return pstring 
411