Source Code for Module iceprod.core.lex

  1  #!/usr/bin/python 
  2  # @author: Juan Carlos Diaz-Velez 
  3  #/@date:   04 May 2005 
  4  # (c) IceCube collaboration. 
  5  # @brief:  Steering script for running I3Sim on Condor. 
  6  # Evaluator: Module for parsing and evaluating expressions in IceTray values 
  7  # 
  8  ###################################################################### 
  9  import os,sys,time,string,re 
 10  import signal 
 11  import random 
 12  import getpass 
 13  import os.path,getopt 
 14  import commands 
 15  import platform 
 16  from os.path import expandvars,join,exists 
 17  from dataclasses import Steering 
 18  from metadata    import DIF_Plus 
 19  from iceprod.core import ipxml 
 20  from iceprod.core.odict import OrderedDict 
 21  import glob 
 22   
 23   
 24  # The following modules might not be available until restart 
 25  import xmlrpclib 
 26  import socket 
 27  import cPickle 
 28  from ConfigParser import SafeConfigParser 
 29   
 30  _platform = None 
 31  _done  = False 
 32  _clean = True 
 33   
 34  EVICTED = 1 
 35  FAILED  = 2 
 36  python_version = platform.python_version()[:3] 
 37   
 38   
 39 -class XMLSummaryParser: 
 40      """ 
 41      XML Parser for files generated by the I3SummaryService in IceTray. 
 42      """ 
 43   
 44 -    def __init__(self): 
 45          self.summary_map = {} 
 46   
 47 -    def ParseFile(self,config_file_name):  
 48   
 49          # get the DOM tree 
 50          doc = None 
 51          try:  
 52              # open xml file for reading 
 53              xmlin = open(config_file_name, 'r') 
 54              doc = ipxml.Parse(xmlin) 
 55          except Exception, xe:  
 56              return  
 57           
 58          i3map = ipxml.Evaluate('boost_serialization/I3XMLSummaryService/map', doc) 
 59          if i3map: 
 60              count = int(i3map[0].getElementsByTagName('count')[0].firstChild.data) 
 61              for item in i3map[0].getElementsByTagName('item'): 
 62                  key = item.getElementsByTagName('first')[0].firstChild.data 
 63                  value = item.getElementsByTagName('second')[0].firstChild.data 
 64                  self.summary_map[key] = float(value) 
 65   
 66          return self.summary_map 
 67   
 68 -    def Write(self,fname):  
 69   
 70          dom     = ipxml.getDOMImplementation() 
 71          doctype = dom.createDocumentType("boost_serialization",None,None) 
 72          doc     = dom.createDocument( None, "boost_serialization", doctype ) 
 73   
 74          boost   = doc.documentElement 
 75          boost._attrs = OrderedDict() 
 76          boost._attrsNS = OrderedDict()  
 77          boost.setAttribute('signature', 'serialization::archive') 
 78          boost.setAttribute('version', '3') 
 79   
 80          summary = doc.createElement( 'I3XMLSummaryService' ) 
 81          summary.setAttribute('class_id','0') 
 82          summary.setAttribute('tracking_level','1') 
 83          summary.setAttribute('version','0') 
 84          summary.setAttribute('object_id','_0') 
 85          boost.appendChild(summary) 
 86   
 87          frameobj = doc.createElement( 'I3FrameObject' ) 
 88          frameobj.setAttribute("class_id","1") 
 89          frameobj.setAttribute("tracking_level","1") 
 90          frameobj.setAttribute("version","0") 
 91          frameobj.setAttribute("object_id","_1") 
 92          frameobj.appendChild(doc.createTextNode("")) 
 93          summary.appendChild(frameobj) 
 94   
 95          map = doc.createElement( 'map' ) 
 96          map.setAttribute('class_id','2') 
 97          map.setAttribute('tracking_level','0') 
 98          map.setAttribute('version','0') 
 99          summary.appendChild(map) 
100   
101          count = doc.createElement( 'count' ) 
102          count.appendChild(doc.createTextNode("%u" % len(self.summary_map.items()))) 
103          map.appendChild(count) 
104   
105          for key,value in self.summary_map.items(): 
106              item   = doc.createElement( 'item' ) 
107   
108              first  = doc.createElement( 'first' ) 
109              first.appendChild(doc.createTextNode(key)) 
110              item.appendChild(first) 
111   
112              second = doc.createElement( 'second' ) 
113              second.appendChild(doc.createTextNode("%f" % value)) 
114              item.appendChild(second) 
115   
116              map.appendChild(item) 
117   
118          file_object = open(fname, "w") 
119          ipxml.myPrettyPrint(doc, file_object) 
120          file_object.close() 
121   
122  #------ end of class XMLSummaryParser ----- 
123   
124  #------ start of class ExpParser ----- 
125 -class ExpParser: 
126      """ 
127      Expression parsing class for parameter values.  
128      """ 
129      keywords = ['sprintf','args','metadata','steering','eval','file','glob','dict','system'] 
130   
131 -    def __init__(self,opts={},steering=Steering(),cache=False,noeval=False): 
132          self.opts        = opts 
133          self.steering    = steering 
134          self.steerdict   = self.steering.get_param_dict() 
135          self.sysopts     = self.steering.get_sysopt_dict() 
136          self.difplus     = DIF_Plus() 
137          if self.steering.HasDIFPlus(): 
138             self.difplus  = self.steering.GetDIFPlus() 
139   
140          self.dpdict      = self.difplus.MakeDict(int(self.opts['procnum']))  
141          self.server      = None 
142          self.noeval      = noeval 
143          self.spdict      = None 
144          self.spdictfd    = None 
145          if self.opts.has_key('url'):  
146              import xmlrpclib,socket 
147              self.server  = xmlrpclib.ServerProxy(self.opts['url']) 
148          if cache: 
149              dictpath     = 'spdict.cfg' 
150              self.spdict  = SafeConfigParser() 
151              if os.path.exists(dictpath): 
152                  self.spdict.read(dictpath) 
153              self.spdictfd = open(dictpath,'w') 
154   
155 -    def AddSysOpt(self,name,value): 
156          self.sysopts[name] = value 
157   
158 -    def AddOpt(self,name,value): 
159          self.opts[name] = value 
160   
161 -    def AddSteering(self,name,value): 
162          self.steerdict[name] = value 
163   
164 -    def sprintf(self,pstring,args): 
165        """ 
166        Format string using the arguments in args 
167        """ 
168        return_string = pstring % tuple(map(self.parse,args)) 
169   
170 -    def fetchfilename(self,key): 
171        if self.opts.has_key('url'):  
172            if self.opts.has_key('dataset'): 
173              val = self.server.getfile(key,self.opts['dataset']) 
174            else: 
175              val = self.server.getfile(key,0) 
176            return val 
177        else: return "" 
178   
179 -    def fetchval(self,key): 
180        if self.spdict:  
181          if self.spdict.has_option('dict',key):  
182              return self.spdict.get('dict',key) 
183   
184        if self.opts.has_key('url'):  
185            val = self.server.getvalue(key) 
186            if self.spdict:  
187              self.spdict.set('dict',key,val) 
188              self.spdict.write(self.spdictfd) 
189            return val 
190        else: return "$dict(%s)" % key 
191   
192 -    def safe_eval(self,pstring): 
193          """ 
194          Evalualte Python expression after verifying that it does not contain 
195          potentially harmful or exploitable expressions. 
196          @param pstring: string containing expression 
197          """ 
198          bad = re.search(r'(import)|(open)|(for)|(while)|(lambda)', pstring ) 
199          if bad: 
200              raise SyntaxError, 'Illegal operator call \'%s\'' % bad.group(0) 
201          else: 
202              return str(eval(pstring)) 
203   
204 -    def _getarg(self,pstring): 
205          """ 
206          Retrieve argument from dictionary of commandline arguments/options 
207          @param pstring: string containing expression 
208          """ 
209          pval = pstring.strip() 
210          if self.opts.has_key(pval): 
211              return str(self.opts[pval]) 
212          else: 
213              raise NameError, 'option $args(%s) is not defined' % pval 
214   
215 -    def _getsteering(self,pstring): 
216          """ 
217          Retrieve value from dictionary of steering parameters 
218          @param pstring: string containing expression 
219          """ 
220          pname = pstring.strip() 
221          try: 
222                pname = self.steerdict[pname] 
223          except Exception,e: 
224                raise NameError, 'name %s is not defined' % pname 
225          return self.parse(str(pname)) 
226   
227 -    def _getdifplus(self,pstring): 
228          """ 
229          Retrieve value from dictionary of steering parameters 
230          @param pstring: string containing expression 
231          """ 
232          pstr  = pstring.strip() 
233          pval = pstr % self.dpdict 
234          return self.parse(str(pval)) 
235   
236   
237 -    def _getsysopt(self,pstring): 
238          """ 
239          Retrieve value from dictionary of system options  
240          @param pstring: string containing expression 
241          """ 
242          pname = pstring.strip() 
243          try: 
244                pname = self.sysopts[pname] 
245          except Exception,e: 
246                raise NameError, 'name %s is not defined' % pname 
247          return self.parse(str(pname)) 
248   
249 -    def parse(self,pstring): 
250          """ 
251          Parse string for valid expressions. 
252          @param pstring: string containing expression 
253          """ 
254          pstr = '' 
255          c = 0 
256          while c < len(pstring): 
257              for key in self.keywords: 
258                  substr = "$%s(" % key 
259                  if pstring[c:].startswith(substr): 
260                      args, remainder = self.match('(',pstring[c+len(substr):]) 
261                      pstr += self.call(key,args) 
262                      if len(remainder) == 0: 
263                          return  pstr 
264                      else: 
265                          pstring = remainder 
266                          c = 0 
267              pstr += pstring[c] 
268              c += 1 
269          return (str(expandvars(pstr))) 
270                   
271   
272 -    def call(self,func,args): 
273          """ 
274          Parse a function call in the form $func(args) 
275          @param func: name of function 
276          @param args: list of arguments to pass to function 
277          """ 
278          if func == 'sprintf': 
279              sp  = args.find('\"',1) 
280              strfmt = args[0:sp].strip('\"') 
281              if len(args.split('\"')) > 2: 
282                  args = map(string.strip,self.parse(args[sp+1:]).split(',')) 
283                  if self.noeval:  
284                      if len(args) > 2: 
285                          return strfmt + " % (" + ",".join(args[1:]) + ")" 
286                      else: 
287                          return strfmt  
288                  return self.format_string(strfmt,args[1:]) 
289              else: 
290                  return args[0] 
291          elif func == 'eval': 
292              if self.noeval: return self.parse(args) 
293              return self.safe_eval(self.parse(args)) 
294          elif func == 'args': 
295              return self._getarg(self.parse(args)) 
296          elif func == 'file': 
297              return self.fetchfilename(self.parse(args)) 
298          elif func == 'dict': 
299              return self.fetchval(self.parse(args)) 
300          elif func == 'steering': 
301              return self._getsteering(self.parse(args)) 
302          elif func == 'metadata': 
303              return self._getdifplus(self.parse(args)) 
304          elif func == 'system': 
305              return self._getsysopt(self.parse(args)) 
306          #elif func == 'glob': 
307          #    return str(glob.glob(self.parse(args))) 
308          else:  
309              return "$%s(%s)" % (func,self.parse(args)) 
310   
311   
312 -    def match(self,ochar,s): 
313          """ 
314          Parse string until finding the matching closing character for ochar 
315          (),[],{} 
316          @param ochar:  opening character  
317          @param s: string to parse 
318          """ 
319          substring = '' 
320          remainder = '' 
321          ocount = 1 
322          for c in s: 
323              if ocount > 0: 
324                  if  self.complement(ochar) == c: 
325                      ocount -= 1 
326                      if ocount == 0: continue 
327                  elif ochar == c: 
328                      ocount += 1 
329                  substring += c 
330              else: 
331                  remainder += c 
332   
333          return (substring,remainder) 
334   
335   
336 -    def complement(self,ochar): 
337          """ 
338          return complent of char or matching opening and closing symbols 
339          (),[],{} 
340          @param ochar:  character  
341          @return: complementary character for ochar 
342          """ 
343   
344          if '(': 
345              return ')' 
346          elif '[': 
347              return ']' 
348          elif '{': 
349              return '}' 
350   
351 -    def format_string(self,pstring,args): 
352          """ 
353          Format string using the arguments in args 
354          """ 
355          return_string = pstring 
356          if not args: 
357              return return_string 
358   
359          try: 
360              fstrings = re.findall(r'\%[0-9]*.{0,1}[0-9]*[csridufegExXo]',pstring) 
361              args = map(self.cast_string,fstrings,args) 
362              return_string = pstring % tuple(args) 
363          except TypeError,e: 
364              print "%s: %s" %( sys.exc_type, str(e)) 
365              print 'Unable to format string parameter ' + pstring + ' with args ' +str(args) 
366   
367          return return_string 
368               
369 -    def cast_string(self,fstring,arg): 
370          """ 
371          cast string to value according to formatting character 
372          @param fstring: formatting string (e.g. '%006d') 
373          @param arg: value to cast 
374          @return: converted value 
375          """ 
376          if not fstring: return arg 
377          if   fstring.endswith('c'): return str(arg) 
378          elif fstring.endswith('s'): return str(arg) 
379          elif fstring.endswith('r'): return repr(arg) 
380          elif fstring.endswith('i'): return int(arg) 
381          elif fstring.endswith('d'): return int(arg) 
382          elif fstring.endswith('u'): return int(arg) 
383          elif fstring.endswith('f'): return float(arg) 
384          elif fstring.endswith('e'): return float(arg) 
385          elif fstring.endswith('g'): return float(arg) 
386          elif fstring.endswith('E'): return float(arg) 
387          elif fstring.endswith('x'): return int('0x%s' % arg, 0) 
388          elif fstring.endswith('X'): return int('0x%s' % arg, 0) 
389          elif fstring.endswith('o'): return int('0%s' % arg, 0) 
390          else: 
391              print 'Unable to cast %s using format %s' (arg,fstring) 
392              print 'assuming string' 
393              return arg 
394   
395 -class FakeExpParser(ExpParser): 
396      """ 
397      Expression parsing class for parameter values.  
398      """ 
399      steer = [] 
400   
401 -    def parse(self,pstring): 
402          """ 
403          Parse string for valid expressions. 
404          @param pstring: string containing expression 
405          """ 
406          return pstring 
407