iceprod.core.xmlparamdb

1 #! /usr/bin/env python 2 # 3 4 """ 5 A class for writting a Python IceTrayConfig object to XML 6 7 copyright (c) 2005 the icecube collaboration 8 9 @version: $Revision: 1.0 $ 10 @date: $Date: $ 11 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 12 13 @todo: Many of the same todo items on xmlparser.py 14 """ 15 16 import sys,os 17 import string 18 import re 19 import types 20 import logging 21 import logger 22 from dataclasses import * 23 24 from os.path import expandvars 25 from iceprod.core import ipxml 26 from xml.sax._exceptions import SAXParseException 27 28 logger = logging.getLogger('XMLParamDB') 29 30 SupportedTypes = ['bool', 'int', 'long', 'float', 'double', 'string','OMKey'] 31 VectorTypes = ['boolv', 'intv', 'longv', 'floatv', 'doublev', 'stringv','OMKeyv'] 32

33 -class XMLParamDB:

34

35 - def __init__(self,metaprojects,uri='http://x2100.icecube.wisc.edu/dtd/i3paramdb.dtd'):

36 """ 37 Initialize and build DOM tree 38 from IceTrayConfig object 39 """ 40 self.metaprojects = metaprojects 41 dom = ipxml.getDOMImplementation() 42 43 doctype = dom.createDocumentType("i3paramdb", 44 "-//W3C//DTD XML 1.0 Strict//EN", uri) 45 46 self.doc = dom.createDocument( None, "i3paramdb", doctype ) 47 config = self.doc.documentElement 48 config.setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance') 49 config.setAttribute('xsi:noNamespaceSchemaLocation','config.xsd') 50 51 self.AddMetaProjects()

52 53

54 - def AddMetaProjects(self):

55 """ 56 Load projects from IceTrayConfig and add the to DOM tree 57 """ 58 59 doc = self.doc 60 icetrayA = doc.getElementsByTagName('i3paramdb') 61 62 if icetrayA: 63 icetray = icetrayA[0] 64 else: 65 docNode = self.doc.documentElement 66 icetray = self.doc.createElement( 'i3paramdb' ) 67 docNode.appendChild(icetray) 68 69 70 for mp in self.metaprojects: 71 72 metaprojElement = self.doc.createElement( 'metaproject' ) 73 metaprojElement.setAttribute('id',str(mp.GetId())) 74 mpNameElement = self.doc.createElement( 'name' ) 75 mpNameNode = self.doc.createTextNode( mp.GetName() ) 76 mpNameElement.appendChild(mpNameNode) 77 metaprojElement.appendChild(mpNameElement) 78 79 mpVersionElement = self.doc.createElement( 'version' ) 80 ver = mp.GetVersion() 81 mpVersionNode = self.doc.createTextNode( mp.GetVersion() ) 82 mpVersionElement.appendChild(mpVersionNode) 83 metaprojElement.appendChild(mpVersionElement) 84 icetray.appendChild(metaprojElement) 85 86 projects = mp.GetProjectList() 87 for p in projects: 88 89 projElement = self.doc.createElement( 'project' ) 90 projElement.setAttribute('id',str(p.GetId())) 91 if p.GetDependencies(): 92 projElement.setAttribute('dependencies',','.join( 93 map(lambda x: type(x) is types.StringType and x or x.GetName(),p.GetDependencies()))) 94 95 nameElement = self.doc.createElement( 'name' ) 96 nameNode = self.doc.createTextNode( p.GetName() ) 97 nameElement.appendChild(nameNode) 98 projElement.appendChild(nameElement) 99 100 versionElement = self.doc.createElement( 'version' ) 101 ver = p.GetVersion() 102 versionNode = self.doc.createTextNode( p.GetVersion() ) 103 versionElement.appendChild(versionNode) 104 projElement.appendChild(versionElement) 105 metaprojElement.appendChild(projElement) 106 servicesElement = self.doc.createElement( 'services' ) 107 projElement.appendChild(servicesElement) 108 self.AddServices(servicesElement,p) 109 modulesElement = self.doc.createElement( 'modules' ) 110 projElement.appendChild(modulesElement) 111 self.AddModules(modulesElement,p) 112 if p.GetName() == 'iceprod': 113 presElement = self.doc.createElement( 'iceprodpres' ) 114 projElement.appendChild(presElement) 115 self.AddPres(presElement,p)

116 117

118 - def mkValueElement(self,ptype,pvalue,doc):

119 """ 120 Create a value element depending on the type 121 """ 122 if ptype.startswith("OMKey"): 123 pValueElement = self.doc.createElement( 'map' ) 124 pEntryElement0 = self.doc.createElement( 'cvalue' ) 125 pEntryElement0.appendChild(self.doc.createTextNode(str(pvalue.stringid))) 126 pEntryElement0.setAttribute('label', 'stringid' ) 127 pValueElement.appendChild(pEntryElement0) 128 pEntryElement1 = self.doc.createElement( 'cvalue' ) 129 pEntryElement1.appendChild(self.doc.createTextNode(str(pvalue.omid))) 130 pEntryElement1.setAttribute('label', 'omid' ) 131 pValueElement.appendChild(pEntryElement1) 132 else: 133 pValueElement = self.doc.createElement( 'value' ) 134 pValueElement.appendChild(self.doc.createTextNode(str(pvalue.value))) 135 if pvalue.unit: 136 pValueElement.setAttribute('unit', pvalue.unit ) 137 return pValueElement

138

139 - def AddServices(self,docElement,project):

140 """ 141 Load services from IceTrayConfig and add the to DOM tree 142 """ 143 144 services = project.GetServices() 145 146 for s in services: 147 148 servElement = self.doc.createElement( 'service' ) 149 servElement.setAttribute('class',s.GetClass()) 150 servElement.setAttribute('id',str(s.GetId())) 151 projects = ','.join( [p.GetName() for p in s.GetProjectList()] ) 152 servElement.setAttribute('projects',projects) 153 docElement.appendChild(servElement) 154 155 nameElement = self.doc.createElement( 'name' ) 156 nameNode = self.doc.createTextNode( s.GetName() ) 157 nameElement.appendChild(nameNode) 158 servElement.appendChild(nameElement) 159 160 paramsElement = self.doc.createElement( 'parameters' ) 161 servElement.appendChild(paramsElement) 162 163 for param in s.GetParameters(): 164 165 pNameElement = self.doc.createElement( 'name' ) 166 pNameNode = self.doc.createTextNode( param.GetName() ) 167 pNameElement.appendChild(pNameNode) 168 pTypeElement = self.doc.createElement( param.GetType() ) 169 pTypeElement.setAttribute('id',str(param.GetId())) 170 pTypeElement.appendChild(pNameElement) 171 paramsElement.appendChild(pTypeElement) 172 ptype = param.GetType() 173 174 if ptype in VectorTypes: 175 for pvalue in param.GetValue(): 176 pValueElement = self.mkValueElement(ptype,pvalue,self.doc) 177 pTypeElement.appendChild(pValueElement) 178 else: 179 pValueElement = self.mkValueElement(ptype,param.GetValue(),self.doc) 180 pTypeElement.appendChild(pValueElement)

181 182

183 - def AddPres(self,docElement,project):

184 """ 185 Load modules from IceTrayConfig and add the to DOM tree 186 """ 187 188 modules = project.GetIceProdPres() 189 190 for m in modules: 191 192 modElement = self.doc.createElement( 'iceprodpre' ) 193 modElement.setAttribute('class',m.GetClass()) 194 modElement.setAttribute('id',str(m.GetId())) 195 projects = ','.join( [p.GetName() for p in m.GetProjectList()] ) 196 modElement.setAttribute('projects',projects) 197 docElement.appendChild(modElement) 198 199 nameElement = self.doc.createElement( 'name' ) 200 nameNode = self.doc.createTextNode( m.GetName() ) 201 nameElement.appendChild(nameNode) 202 modElement.appendChild(nameElement) 203 204 paramsElement = self.doc.createElement( 'parameters' ) 205 modElement.appendChild(paramsElement) 206 207 for param in m.GetParameters(): 208 209 pNameElement = self.doc.createElement( 'name' ) 210 pNameNode = self.doc.createTextNode( param.GetName() ) 211 pNameElement.appendChild(pNameNode) 212 pTypeElement = self.doc.createElement( param.GetType() ) 213 pTypeElement.setAttribute('id',str(param.GetId())) 214 pTypeElement.appendChild(pNameElement) 215 paramsElement.appendChild(pTypeElement) 216 ptype = param.GetType() 217 218 if ptype in VectorTypes: 219 for pvalue in param.GetValue(): 220 pValueElement = self.mkValueElement(ptype,pvalue,self.doc) 221 pTypeElement.appendChild(pValueElement) 222 else: 223 pValueElement = self.mkValueElement(ptype,param.GetValue(),self.doc) 224 pTypeElement.appendChild(pValueElement)

225

226 - def AddModules(self,docElement,project):

227 """ 228 Load modules from IceTrayConfig and add the to DOM tree 229 """ 230 231 modules = project.GetModules() 232 233 for m in modules: 234 235 modElement = self.doc.createElement( 'module' ) 236 modElement.setAttribute('class',m.GetClass()) 237 modElement.setAttribute('id',str(m.GetId())) 238 projects = ','.join( [p.GetName() for p in m.GetProjectList()] ) 239 modElement.setAttribute('projects',projects) 240 docElement.appendChild(modElement) 241 242 nameElement = self.doc.createElement( 'name' ) 243 nameNode = self.doc.createTextNode( m.GetName() ) 244 nameElement.appendChild(nameNode) 245 modElement.appendChild(nameElement) 246 247 paramsElement = self.doc.createElement( 'parameters' ) 248 modElement.appendChild(paramsElement) 249 250 for param in m.GetParameters(): 251 252 pNameElement = self.doc.createElement( 'name' ) 253 pNameNode = self.doc.createTextNode( param.GetName() ) 254 pNameElement.appendChild(pNameNode) 255 pTypeElement = self.doc.createElement( param.GetType() ) 256 pTypeElement.setAttribute('id',str(param.GetId())) 257 pTypeElement.appendChild(pNameElement) 258 paramsElement.appendChild(pTypeElement) 259 ptype = param.GetType() 260 261 if ptype in VectorTypes: 262 for pvalue in param.GetValue(): 263 pValueElement = self.mkValueElement(ptype,pvalue,self.doc) 264 pTypeElement.appendChild(pValueElement) 265 else: 266 pValueElement = self.mkValueElement(ptype,param.GetValue(),self.doc) 267 pTypeElement.appendChild(pValueElement)

268 269 270

271 - def getDOM(self):

272 return self.doc

273

274 - def write_to_file(self, fname=None):

275 """ 276 Write DOM tree as human readable xml to a file 277 """ 278 if fname: 279 file_object = open(fname, "w") 280 ipxml.PrettyPrint(self.doc, file_object) 281 file_object.close() 282 else: 283 ipxml.PrettyPrint(self.doc)

284

285 -class XMLParamDBParser:

286

287 - def __init__(self):

288 self.pdb = IceTrayConfig() 289 self.metaprojects = []

290

291 - def ParseMetaProjects(self,doc):

292 """ 293 Load meatprojects from doc 294 """ 295 logger.debug('PARSE METAPROJECTS') 296 metaprojects = ipxml.Evaluate('i3paramdb/metaproject', doc) 297 298 for mp in metaprojects: 299 metaproject = MetaProject() 300 301 # fill MetaProject object: 302 mpname = ipxml.Evaluate('name', mp) 303 mpname = mpname[0].firstChild.data 304 mpver = ipxml.Evaluate('version', mp) 305 mpver = mpver[0].firstChild.data 306 mpver = tuple(mpver.split(".")) 307 id = int(mp.getAttribute('id')) 308 309 metaproject.SetName(mpname) 310 metaproject.SetVersion(mpver) 311 metaproject.SetId(id) 312 313 projects = ipxml.Evaluate('project', mp) 314 315 for p in projects: 316 sys.stdout.write('.') 317 project = Container() 318 319 # fill Project object: 320 pname = ipxml.Evaluate('name', p) 321 pname = pname[0].firstChild.data 322 pver = ipxml.Evaluate('version', p) 323 pver = pver[0].firstChild.data 324 pver = tuple(pver.split(".")) 325 id = int(p.getAttribute('id')) 326 327 project.SetName(pname) 328 project.SetVersion(pver) 329 project.SetId(id) 330 331 depends = p.getAttribute('dependencies') 332 if depends: 333 for d in depends.split(','): 334 project.AddDependency(d) 335 336 self.AddServices(project,p) 337 self.AddModules(project,p) 338 if project.GetName() == 'iceprod': 339 self.AddPres(project,p) 340 341 metaproject.AddProject(pname,project) 342 logger.debug("added project %s %d" % (project.GetName(),project.GetId())) 343 344 projects = metaproject.GetProjectList() 345 for p in projects: 346 dependencies = map(metaproject.GetProject,p.GetDependencies()) 347 p.ClearDependencies() 348 for dp in dependencies: 349 p.AddDependency(dp) 350 351 self.metaprojects.append(metaproject) 352 logger.debug("added metaproject %s" % metaproject.GetName()) 353 sys.stdout.write('done\n')

354 355

356 - def GetProjectList(self):

357 return self.projects.values()

358 359

360 - def AddServices(self,project,projectElement):

361 """ 362 Load services from icetray configuration file 363 """ 364 logger.debug('ADD SERVICES') 365 366 service = projectElement.getElementsByTagName('service') 367 368 for s in service: 369 service = Service() 370 371 sclass = s.getAttribute('class') 372 id = int(s.getAttribute('id')) 373 374 # fill Project object: prname may be a list 375 service.AddProject(project.GetName(),project) 376 377 sname = ipxml.Evaluate('name', s) 378 sname = sname[0].firstChild.data 379 380 # fill Service object 381 service.SetName(sname) 382 service.SetClass(sclass) 383 service.SetId(id) 384 project.AddService(service) 385 386 parameters = s.getElementsByTagName('parameters') 387 388 # If there are parameters, continue 389 if len(parameters) != 0: 390 for pmtr in parameters: 391 for stype in SupportedTypes + VectorTypes: 392 ps = pmtr.getElementsByTagName(stype) 393 if len(ps) != 0: 394 for p in ps: 395 sparameter = Parameter() 396 397 pname = p.getElementsByTagName('name') 398 pname = pname[0].firstChild.data 399 pvalues = p.getElementsByTagName('value') 400 pmaps = p.getElementsByTagName('map') 401 402 # fill Parameter object 403 sparameter.SetType(stype) 404 sparameter.SetName(pname) 405 406 value = [] 407 # set I3Units (if any) 408 for pvalue in pvalues: 409 val = Value(pvalue.firstChild.data) 410 if pvalue.getAttribute('unit'): 411 punit = pvalue.getAttribute('unit') 412 val.unit = punit 413 if pvalue.getAttribute('format'): 414 pformat = pvalue.getAttribute('format') 415 val.format = pformat 416 value.append( val ) 417 418 for pmap in pmaps: 419 pcvalues = pmap.getElementsByTagName('cvalue') 420 val = pyOMKey("0","0") 421 for pcval in pcvalues: 422 label = pcval.getAttribute('label') 423 if label == 'stringid': 424 val.stringid = pcval.firstChild.data 425 elif label == 'omid': 426 val.omid = pcval.firstChild.data 427 value.append( val ) 428 429 if stype in VectorTypes: sparameter.SetValue( value ) 430 else: sparameter.SetValue( value[0] ) 431 432 service.AddParameter(sparameter) 433 logger.debug("%s %s %s %s" % (sname,stype,pname,sparameter.GetValue())) 434 project.AddService(service)

435 436 437

438 - def AddModules(self,project,projectElement):

439 """ 440 Load modules from icetray configuration file 441 """ 442 443 logger.debug('ADD MODULES') 444 445 modules = projectElement.getElementsByTagName('module') 446 447 for m in modules: 448 module = Module() 449 450 mclass = m.getAttribute('class') 451 module.AddProject(project.GetName(),project) 452 453 mname = ipxml.Evaluate('name', m) 454 mname = mname[0].firstChild.data 455 id = int(m.getAttribute('id')) 456 457 # fill Module object 458 module.SetName(mname) 459 module.SetClass(mclass) 460 module.SetId(id) 461 project.AddModule(module) 462 463 parameters = m.getElementsByTagName('parameters') 464 465 # If there are parameters, continue 466 if len(parameters) != 0: 467 for pmtr in parameters: 468 # We have the parameters, now loop over the 469 # supported types and see if there are any 470 # used in the xml file (for example, if there are 471 # <int> tags used) 472 for stype in SupportedTypes + VectorTypes: 473 # This is an array of Type tags for 474 # each supported type 475 ps = pmtr.getElementsByTagName(stype) 476 # If there are any supported types used 477 # iterate over them and add the parameter 478 if len(ps) != 0: 479 for p in ps: 480 mparameter = Parameter() 481 482 pname = p.getElementsByTagName('name') 483 pname = pname[0].firstChild.data 484 pvalues = p.getElementsByTagName('value') 485 pmaps = p.getElementsByTagName('map') 486 id = int(p.getAttribute('id')) 487 # fill Parameter object 488 mparameter.SetType(stype) 489 mparameter.SetName(pname) 490 mparameter.SetId(id) 491 value = [] 492 493 # set I3Units (if any) 494 for pvalue in pvalues: 495 val = Value(pvalue.firstChild.data) 496 if pvalue.getAttribute('unit'): 497 punit = pvalue.getAttribute('unit') 498 val.unit = punit 499 if pvalue.getAttribute('format'): 500 pformat = pvalue.getAttribute('format') 501 val.format = pformat 502 value.append( val ) 503 504 for pmap in pmaps: 505 pcvalues = pmap.getElementsByTagName('cvalue') 506 val = pyOMKey("0","0") 507 for pcval in pcvalues: 508 label = pcval.getAttribute('label') 509 if label == 'stringid': 510 val.stringid = pcval.firstChild.data 511 elif label == 'omid': 512 val.omid = pcval.firstChild.data 513 value.append( val ) 514 515 if stype in VectorTypes: mparameter.SetValue( value ) 516 else: mparameter.SetValue( value[0] ) 517 518 module.AddParameter(mparameter) 519 logger.debug("%s %s %s %s" %(mname,stype,pname,mparameter.GetValue())) 520 project.AddModule(module)

521 522 523

524 - def AddPres(self,project,projectElement):

525 """ 526 Load modules from icetray configuration file 527 """ 528 529 logger.debug('ADD PRES') 530 modules = projectElement.getElementsByTagName('iceprodpre') 531 532 for m in modules: 533 module = IceProdPre() 534 535 mclass = m.getAttribute('class') 536 module.AddProject(project.GetName(),project) 537 538 mname = ipxml.Evaluate('name', m) 539 mname = mname[0].firstChild.data 540 id = int(m.getAttribute('id')) 541 542 # fill Module object 543 module.SetName(mname) 544 module.SetClass(mclass) 545 module.SetId(id) 546 project.AddPre(module) 547 548 parameters = m.getElementsByTagName('parameters') 549 550 # If there are parameters, continue 551 if len(parameters) != 0: 552 for pmtr in parameters: 553 # We have the parameters, now loop over the 554 # supported types and see if there are any 555 # used in the xml file (for example, if there are 556 # <int> tags used) 557 for stype in SupportedTypes + VectorTypes: 558 # This is an array of Type tags for 559 # each supported type 560 ps = pmtr.getElementsByTagName(stype) 561 # If there are any supported types used 562 # iterate over them and add the parameter 563 if len(ps) != 0: 564 for p in ps: 565 mparameter = Parameter() 566 567 pname = p.getElementsByTagName('name') 568 pname = pname[0].firstChild.data 569 pvalues = p.getElementsByTagName('value') 570 pmaps = p.getElementsByTagName('map') 571 id = int(p.getAttribute('id')) 572 # fill Parameter object 573 mparameter.SetType(stype) 574 mparameter.SetName(pname) 575 mparameter.SetId(id) 576 value = [] 577 578 # set I3Units (if any) 579 for pvalue in pvalues: 580 val = Value(pvalue.firstChild.data) 581 if pvalue.getAttribute('unit'): 582 punit = pvalue.getAttribute('unit') 583 val.unit = punit 584 if pvalue.getAttribute('format'): 585 pformat = pvalue.getAttribute('format') 586 val.format = pformat 587 value.append( val ) 588 589 for pmap in pmaps: 590 pcvalues = pmap.getElementsByTagName('cvalue') 591 val = pyOMKey("0","0") 592 for pcval in pcvalues: 593 label = pcval.getAttribute('label') 594 if label == 'stringid': 595 val.stringid = pcval.firstChild.data 596 elif label == 'omid': 597 val.omid = pcval.firstChild.data 598 value.append( val ) 599 600 if stype in VectorTypes: mparameter.SetValue( value ) 601 else: mparameter.SetValue( value[0] ) 602 603 module.AddParameter(mparameter) 604 logger.debug("%s %s %s %s" %(mname,stype,pname,mparameter.GetValue()))

605 606 607

608 - def ParseFile(self,config_file_name,validate=0):

609 610 # get the DOM tree 611 doc = None 612 try: # try to validate. Will fail with older configurations 613 614 # open xml file for reading 615 xmlin = open(config_file_name, 'r') 616 logger.debug("opened %s" % config_file_name) 617 618 sys.stdout.write('loading pdb ') 619 sys.stdout.write('...') 620 doc = ipxml.Parse(xmlin, validate=validate) 621 logger.debug("doc = Sax2.FromXmlStream(xmlin, validate=%s)" % validate) 622 623 except SAXParseException, xe: 624 logger.error("Exception: " + str(xe) ) 625 return 626 627 # metaprojects 628 logger.debug("self.ParseMetaProjects(doc)") 629 self.ParseMetaProjects(doc) 630 631 xmlin.close() 632 return self.metaprojects

633 634 if __name__ == '__main__': 635 636 if len(sys.argv) != 2: 637 sys.exit() 638 639 parser = XMLParamDBParser() 640 metaprojects = parser.ParseFile(sys.argv[1],validate=1) 641 642 if metaprojects: 643 writer = XMLParamDB(metaprojects) 644 writer.write_to_file() 645

Source Code for Module iceprod.core.xmlparamdb