1
2
3
4 """
5 A class for parsing an IceTray XML configuration and building a Python
6 IceTrayConfig object.
7
8 copyright (c) 2005 the icecube collaboration
9
10 @version: $Revision: $
11 @date: $Date: 2005/04/06 17:32:56 $
12 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu>
13
14 """
15
16 import sys,os
17 import string,re
18 import getpass
19 import getopt
20 import types
21 from os.path import expandvars
22 from iceprod.core.dataclasses import *
23 from iceprod.core.paramdb import *
24 from xml.dom.ext.reader import Sax2
25 from xml.dom.NodeFilter import NodeFilter
26 from xml.dom.ext import Print,PrettyPrint
27 from xml.sax._exceptions import SAXParseException
28 from os.path import exists,join
29 from iceprod.core.xmlwriter import IceTrayXMLWriter
30 from modulefinder import *
31 import iceprod.modules
32 from iceprod.core.tray2xml import *
33 import logging
34
35 SupportedTypes = ['bool', 'int', 'long', 'float', 'double', 'string','OMKey']
36 VectorTypes = ['boolv', 'intv', 'longv', 'floatv', 'doublev', 'stringv','OMKeyv']
37 UnmetDependencyException = "Error: Un-met dependency:"
38
39 fileregex = r'[^/]+/private/[^/]+/[^\.]+\.cxx'
40 REGEX = r'AddParameter\s*\([^\)]*\);'
41 SERVICEREGEX = r'[a-zA-Z0-9_]*.*InstallService\(.*\)'
42 DNREGEX = r'DefaultName\s*\(\s*i\)\s*{.+}'
43 I3LIB_REGEX = re.compile(r'i3_add_library\s*\((.*\n)*')
44 DEPENDENCY_REGEX = re.compile(r'USE_PROJECTS\s*[^\)]+')
45
46 Types = ['short','int','long','double','float','bool','string','char']
47 Types += ['Int_t','Double_t','Bool_t']
48 Types += ['std::vector<std::string>']
49 Types += ['std::vector<double>']
50 Types += ['std::vector<float>']
51 Types += ['std::vector<int>']
52 Types += ['std::vector<OMKey>']
53 Types += ['OMKey']
54
55 NAMELEN = 22
56
57 default_value_table = {
58 'int':'0',
59 'long':'0',
60 'string':'-',
61 'double':'0.0',
62 'float':'0.0',
63 'NaT':'NaN',
64 'bool':'False' }
65
66 _gid_ = -1
67
69 try: float(s)
70 except: return False
71 return True
72
77
98
100 """
101 Create a default name for a class
102 @param name: of module
103 @return: string default name for module
104 """
105 if name == 'I3Module':
106 return 'genericmodule'
107 if name == 'I3ServiceFactory':
108 return 'genericservice'
109
110 name = name.lower()
111 name = re.sub('i3','',name)
112 name = re.sub('module','',name)
113 name = re.sub('factory','',name)
114 name = re.sub('service','',name)
115 name = name[:min(len(name),NAMELEN)]
116 return name
117
119 """
120 Get list of cxx files in <project>/private/<project>/
121 @param rootdir: top directory for search
122 @return: list of files
123 """
124 filelist = []
125
126 cregex = re.compile(fileregex)
127 ncregex = re.compile(r'[^/]+/private/test/')
128 if not exists(os.path.join(rootdir,project)):
129 print >> sys.stderr, "Unable to process directory %s" % rootdir
130
131 filelist = [os.path.join(d[0].replace(rootdir+"/",''),f) for d in \
132 os.walk(join(rootdir,project)) for f in d[2] ]
133
134 return [cxx for cxx in filelist if cregex.match(cxx) and not ncregex.search(cxx)]
135
136
138
139 filelist = [join(d[0],f) for d in \
140 os.walk(join(project,"public")) for f in d[2] ]
141
142 for h in filelist:
143 if re.search(r'/%s\.h' % module,h) and not re.search("\.svn",h):
144 return h
145
146
147
149 colormap = {
150 'failed':"\033[1;31m%s\033[m" % 'failed',
151 'ok':"\033[1;32m%s\033[m" % 'ok',
152 'pass':"\033[1;33m%s\033[m" % 'pass',
153 }
154
156 self.projects = {}
157 self.regex = REGEX
158 self.libdir = libdir
159 self.serviceregex = SERVICEREGEX
160 self.notcommented = r'[^\/{2}]+%s' % self.regex
161 self.db = None
162 self.topdir = path
163 self.metaproject = MetaProject()
164
165
168
170 """
171 @param dataString: string to be parsed
172 @return: True if the parsed file is actually a service
173 """
174 return re.search(self.serviceregex,dataString) \
175 or servicename.endswith('Factory')
176
178 """
179 Load projects from icetray configuration file
180 """
181 inspectElement = doc.getElementsByTagName('icetray-inspect')[0]
182 projects = doc.getElementsByTagName('icetray-inspect')
183 projects = inspectElement.getElementsByTagName('project')
184
185 for p in projects:
186 project = Container()
187
188
189 pname = p.getAttribute('name')
190 project.SetName(pname)
191 project.SetId(mkid())
192
193 pver = p.getElementsByTagName('url')
194
195 if len(pver)>0:
196 pver = pver[0].firstChild.data
197 if re.match(r'http://.+projects/%s/'%pname,pver):
198 pver = re.sub(r'http://.+projects/%s/'%pname,'',pver)
199 pver = pver.replace('/','.')
200 else:
201 pver = self.getversion(expandvars("$I3_SRC/%s" % pname))
202 project.SetVersion(pver)
203 else:
204 raise Exception,"failed to parse icetray-inspect output"
205
206
207 self.projects[project.GetName()] = self.AddServices(p,project)
208 os.chdir(topdir)
209 for file in getfilelist(topdir,project.GetName()):
210 self.searchfile(file,self.projects[project.GetName()])
211 return project
212
213
215 """
216 Load services from icetray configuration file
217 """
218 modules = {}
219
220 classes = docElement.getElementsByTagName('module')
221
222 for c in classes:
223 module = Service()
224 classname = c.getElementsByTagName('type')
225 if not len(classname) > 0: continue
226 classname = classname[0].firstChild.data
227
228
229 module.SetName(strip_name(classname))
230 module.SetClass(classname)
231 module.SetId(mkid())
232 module.AddProject(project.GetName(),project)
233
234 parameters = c.getElementsByTagName('parameter')
235
236
237 for par in parameters:
238 name = par.getElementsByTagName("name")
239 if not len(name) > 0: continue
240 parameter = Parameter()
241 name = name[0].firstChild.data
242 parameter.SetName(name)
243 parameter.SetId(mkid())
244
245 description = par.getElementsByTagName('description')
246 if len(description)>0:
247 if description[0].firstChild:
248 description = description[0].firstChild.data
249 parameter.SetDescription(description)
250
251 type = par.getElementsByTagName('type')
252 if len(type)>0:
253 if type[0].firstChild:
254 type = type[0].firstChild.data
255 parameter.SetType(self.fixType(type))
256
257 value = par.getElementsByTagName('default_value')
258 if len(value)>0:
259 if value[0].firstChild:
260 value = value[0].firstChild.data
261 parameter.SetValue(self.parse_val(parameter.GetType(),Value(value)))
262
263 module.AddParameter(parameter)
264
265 modules[module.GetClass()] = module
266 return modules
267
268
270 p = Container()
271 p.SetName(projectname)
272 p.SetId(mkid())
273 p.SetVersion(self.getversion(expandvars("$I3_SRC/%s" % projectname)))
274
275
276 if not self.metaproject.HasProject(p.GetName()):
277 p.SetPath(self.get_libpath(p.GetName()))
278 self.metaproject.AddProject(p.GetName(),p)
279 return p
280
282 modfind = ModuleFinder('iceprod.modules')
283 pobj = Container()
284 pobj.SetName('iceprod')
285 pobj.SetVersion(iceprod.__version__)
286 project,ver = pobj.GetName(),pobj.GetVersion()
287
288 ok = 'ok'
289 try:
290 for module in modfind.find_all_submodules(iceprod.modules):
291 classdict = iceprod.modules.ipinspect(module)
292 for cls in classdict.keys():
293 mpre = IceProdPre()
294 mpre.SetName(cls.split('.')[-1].lower())
295 mpre.SetClass(cls)
296 for p in classdict[cls]:
297 param = Parameter()
298 param.SetName(p[0])
299 param.SetType(gettype(p[1]))
300 param.SetValue(str(p[1]))
301 param.SetDescription(p[2])
302 mpre.AddParameter(param)
303 pobj.AddIceProdPre(mpre)
304 except Exception,e:
305 sys.excepthook(sys.exc_type,sys.exc_value,sys.exc_traceback)
306 print "Error parsing source: " + str(e)
307 ok = 'failed'
308 print "\033[1;30m%s.%s\033[m %s %s" %(project,ver,(60-(len(project)+len(ver)))*'.',self.colormap[ok])
309 self.metaproject.AddProject(pobj.GetName(),pobj)
310 return pobj
311
312
314
315
316 doc = None
317 mp = self.getmetaproject(expandvars('$I3_SRC'))
318 platform = getplatform()
319 print "platform is ", platform
320 libdir = self.libdir
321 print "lib is ", libdir
322 for project in os.listdir(i3work):
323 ver = "V??-??-??"
324 libso = os.path.join(libdir,"lib%s.so" % project)
325 dylib = os.path.join(libdir,"lib%s.dylib" % project)
326 if not (os.path.exists(libso) or os.path.exists(dylib)):
327 continue
328 ok = 'ok'
329 try:
330 stdin,stdout,stderr = os.popen3("icetray-inspect -x %s" % project)
331 doc = Sax2.Reader().fromString(stdout.read())
332 except Exception, e:
333 print "Error parsing inspect output: " + str(e)
334 sys.excepthook(sys.exc_type,sys.exc_value,sys.exc_traceback)
335 ok = 'pass'
336 try:
337 pobj = self.AddEmptyProject(project)
338 ver = pobj.GetVersion()
339 except Exception,e:
340 print e
341 ok = 'failed'
342
343
344 if ok == 'ok':
345 try:
346 pobj = self.AddProjects(i3work,doc)
347 ver = pobj.GetVersion()
348 mp.AddProject(pobj.GetName(),pobj)
349 except Exception, e:
350 print "Error parsing source: " + str(e)
351 ok = 'pass'
352 try:
353 pobj = self.AddEmptyProject(project)
354 ver = pobj.GetVersion()
355 except Exception,e:
356 print e
357 ok = 'failed'
358 print "\033[1;30m%s.%s\033[m %s %s" %(project,ver,(60-(len(project)+len(ver)))*'.',self.colormap[ok])
359
360 if ok == 'ok':
361 stdout.close()
362 stdin.close()
363 stderr.close()
364
365 if pobj.GetVersion() == Project().GetVersion():
366 print "Error:bad version read '%s'" % ver
367 os._exit(1)
368
369 for p in self.metaproject.GetProjects().values():
370 for d in self.getdependencies(p.GetName()):
371 if p.GetName() != d and self.metaproject.GetProject(d):
372 p.AddDependency(self.metaproject.GetProject(d))
373 return self.projects
374
375
377 """
378 search a file for relevant information to collect
379 @param cxxfile: name of header file to search
380 """
381 try:
382 cxxReader = open( cxxfile, 'r' )
383 except IOError:
384 print >> sys.stderr,"No such file: \'%s\'" % cxxfile
385 return
386
387
388 cxxdata = ''
389 for line in cxxReader:
390 line = re.sub(r'\/\/+[^\Z]+\Z','',line)
391 cxxdata += " " + line
392 cxxReader.close()
393
394
395 fn = re.split(r'\/|\.',cxxfile)
396 modname = fn[-2]
397 subdir = fn[-3]
398 modserv = None
399 if modname in project_modules.keys():
400 modserv = project_modules[modname]
401 else: return
402
403 project = modserv.GetProjects().values()[0]
404
405
406 pname = project.GetName()
407 hfile = getheaderfile(pname,modname)
408 hdata = cxxdata
409 if hfile:
410 try:
411 hReader = open( hfile, 'r' )
412 hdata = ''
413 for line in hReader:
414 line = re.sub(r'\/\/+[^\Z]+\Z','',line)
415 hdata += " " + line
416 hReader.close()
417 except IOError:
418 print >> sys.stderr,"could not find header file file: \'%s\'" % hfile
419 hdata = cxxdata
420 except Exception,e:
421 print >> sys.stderr,"%s - %s" % (hfile,str(e))
422 hdata = cxxdata
423
424 if not self.metaproject.HasProject(project.GetName()):
425 project.SetPath(self.get_libpath(project.GetName()))
426 self.metaproject.AddProject(project.GetName(),project)
427
428 if self.isService(modname,cxxdata):
429 if not project.HasService(modname):
430 project.AddService(modserv)
431 if project.HasModule(modname):
432 project.RemoveModule(modname)
433 else:
434 if not project.HasModule(modname):
435 project.AddModule(modserv)
436 if project.HasService(modname):
437 project.RemoveService(modname)
438
439
440
442 """
443 Extract project dependencies from the CMakeLists.txt file
444 @param projectname: name of project
445 """
446 projectmk = os.path.join(self.topdir,projectname,'CMakeLists.txt')
447 try:
448 projectmkfile = open( projectmk, 'r' )
449 cont = False
450 pdata = ""
451 except IOError:
452 print >> sys.stderr,"Could not open file: \'%s\'" % projectmk
453 return
454
455 txt =projectmkfile.read()
456 depends = I3LIB_REGEX.search(txt)
457 if depends:
458 depends = depends.group()
459 depends = DEPENDENCY_REGEX.search(depends)
460 if depends:
461 depends = depends.group()
462 depends = map(string.strip,depends.split()[1:])
463 return depends
464 return []
465
467 """
468 Extract project dependencies from the project-config.mk file
469 @param projectname: name of project
470 """
471 projectmk = os.path.join(self.topdir,projectname,'project-config.mk')
472 try:
473 projectmkfile = open( projectmk, 'r' )
474 cont = False
475 pdata = ""
476 except IOError:
477 print >> sys.stderr,"Could not open file: \'%s\'" % projectmk
478 return
479
480 for line in projectmkfile:
481 line = line.strip()
482 if line.startswith("USES_PROJECTS :="):
483 if line.endswith(r'\\'):
484 cont = True
485 pdata = line
486 elif cont:
487 if not line.endswith("\\"):
488 break
489 pdata += line
490
491 pdata = re.sub(r'USES_PROJECTS :=\s*','',pdata)
492 pdata = re.sub(r'\\','',pdata)
493 pdata = re.sub(r'\s+',' ',pdata).strip()
494 pdata = filter(None,map(string.strip,pdata.split(' ')))
495 return pdata
496
498 if ptype in VectorTypes:
499 if ptype.startswith("OMKey"):
500 valvect = re.findall(r'OMKey\(\s*[0-9]+\s*,\s*[0-9]+\s*\)',value.value)
501 valvect = map(lambda x: self.parse_val("OMKey",x),valvect)
502 return valvect
503 else:
504 val = value.value
505 val = val.strip("[").strip("]").strip().split(" ")
506 return map(Value,val)
507 elif ptype == 'OMKey' and value.value.startswith("OMKey"):
508 val = value.value.replace("OMKey",'').replace("(",'').replace(")",'').split(",")
509 return pyOMKey(val[0],val[1])
510 else: return value
511
512
514 """
515 Determine the path to the library file that corresponds to a project
516 @param projname: the name of the project
517 @return: string path to library file
518 """
519 path = os.path.join(self.topdir,getplatform())
520 path = os.path.join(path,"lib")
521 return os.path.join(self.libdir,"lib%s.so" % projname)
522
523 - def readXML(self,file=None,index=0):
524 parser = XMLParamDBParser()
525 self.metaproject = parser.ParseFile(file).values()[index]
526
527 - def toXML(self,file=None):
528 """
529 Print parameter dictionary as an XML document
530 @return: string XML doc
531 """
532 writer = XMLParamDB()
533 writer.AddMetaProject(self.metaproject)
534 if file:
535 writer.write_to_file(file)
536 else:
537 writer.write_to_file()
538
540 return [self.metaproject]
541
543 """
544 Convert root types to python types
545 and format type names to iceprod style
546 """
547 s = value.strip()
548 if s.startswith('[') and s.endswith(']'):
549 sv = s.strip('[').strip(']').split(' ')
550 return self.guessType(sv[0]) + 'v'
551
552 if s.startswith("OMKey"): return "OMKey"
553 if s.isdigit(): return 'int'
554 if isfloat(s): return 'float'
555 if 'python' in s: return 'NaT'
556 return "string"
557
558
560 """
561 Convert root types to python types
562 and format type names to iceprod style
563 """
564 if type in SupportedTypes:
565 return type
566 elif type in map(lambda x: "vector<%s>" % x,SupportedTypes):
567 type = type.replace("vector<","").replace(">","v")
568 elif type in map(lambda x: "vector<%s>" % x,SupportedTypes):
569 type = type.replace("vector<","").replace(">","v")
570
571 type = type.replace('std::vector<std::string>','stringv')
572 type = type.replace('std::vector<int>','intv')
573 type = type.replace('std::vector<double>','floatv')
574 type = type.replace('std::vector<float>','floatv')
575 type = type.replace('std::vector<bool>','boolv')
576 type = type.replace('std::vector<OMKey>','OMKeyv')
577 type = type.replace('Double_t','float')
578 type = type.replace('Int_t','int')
579 type = type.replace('Bool_t','bool')
580 type = type.replace('char','string')
581 type = type.replace('long long','long')
582 type = type.replace('unsigned','').strip()
583 type = type.replace('vector<unsigned int, allocator<unsigned int> >','intv')
584 type = type.replace('vector<signed int, allocator<signed int> >','intv')
585 type = type.replace('vector<unsigned long, allocator<unsigned long> >','longv')
586 type = type.replace('vector< int, allocator< int> >','intv')
587 type = type.replace('vector< long, allocator< long> >','longv')
588 if len(type) == 0: type = 'int'
589 return type
590
591 - def upload(self,url,username,password=''):
592 """
593 Connect to database and upload contents of dictionary
594 Requires module/class ParameterDB
595 """
596 self.db = RPCParamDB(url)
597 if not password:
598 password = getpass.getpass('Password for \'%s\': ' % username)
599 return self.db.load(self.metaproject,username,password)
600
601
602
604 """
605 Extract version tuple from project
606 @param path: path to project directory
607 @return: string corresponding to project version
608 """
609 version = None
610 version_regex = r'http://.+projects/.+'
611 svn_project = os.path.join(path,'.svn','entries')
612 try:
613 svn_projectfile = os.popen(expandvars('svn info ' + path) , 'r' )
614 svn_data = svn_projectfile.read()
615 except:
616 print >> sys.stderr, "Unable to parse: \n%s" % svn_project
617 return ''
618
619 try:
620 version_search = re.search(version_regex,svn_data)
621 version = version_search.group()
622 revision_search = re.search(r'Revis.*:.*[0-9]+',svn_data)
623 revision = revision_search.group()
624
625 pname = re.sub(r'http://.+projects/','',version)
626 pname = pname.split('/')[0]
627 version = re.sub(r'http://.+projects/%s/'%pname,'',version)
628 version = version.replace('/','.')
629 revision = revision.split(':')[-1].strip()
630
631 version = version[0:min(len(version),30)]
632 return version
633 except Exception,e:
634 print >> sys.stderr,e
635 return 'foo.trunk'
636
637
638
669
671
673 """
674 Load projects from icetray configuration file
675 """
676 inspectElement = doc.getElementsByTagName('icetray-inspect')[0]
677 projects = doc.getElementsByTagName('icetray-inspect')
678 projects = inspectElement.getElementsByTagName('project')
679
680 for p in projects:
681 project = Container()
682
683
684 pname = p.getAttribute('name')
685 project.SetName(pname)
686 project.SetId(mkid())
687
688 pver = self.getversion(expandvars("$I3_SRC/%s" % pname))
689 project.SetVersion(pver)
690
691 self.projects[project.GetName()] = self.AddServices(p,project)
692 os.chdir(topdir)
693 for file in getfilelist(topdir,project.GetName()):
694 self.searchfile(file,self.projects[project.GetName()])
695 return project
696
700
701
703 """
704 Load services from icetray configuration file
705 """
706 modules = {}
707
708 classes = docElement.getElementsByTagName('module')
709
710 for c in classes:
711 module = Service()
712 classname = c.getElementsByTagName('type')
713 if not len(classname) > 0: continue
714 classname = classname[0].firstChild.data
715
716
717 module.SetName(strip_name(classname))
718 module.SetClass(classname)
719 module.SetId(mkid())
720 module.AddProject(project.GetName(),project)
721
722 parameters = c.getElementsByTagName('parameter')
723
724
725 for par in parameters:
726 name = par.getElementsByTagName("name")
727 if not len(name) > 0: continue
728 parameter = Parameter()
729 name = name[0].firstChild.data
730 parameter.SetName(name)
731 parameter.SetId(mkid())
732
733 description = par.getElementsByTagName('description')
734 if len(description)>0:
735 if description[0].firstChild:
736 description = description[0].firstChild.data
737 parameter.SetDescription(description)
738
739
740
741
742 value = par.getElementsByTagName('default_value')
743 if len(value)>0:
744 if value[0].firstChild:
745 value = value[0].firstChild.data
746 parameter.SetType(self.guessType(value))
747 parameter.SetValue(self.parse_val(parameter.GetType(),Value(value)))
748 parameter.SetType('NaT')
749 if parameter.GetType() != 'NaT':
750 module.AddParameter(parameter)
751
752 modules[module.GetClass()] = module
753 return modules
754