iceprod.core.paramdb

1 """ 2 Class paramdb is an abstract interface to the parameter database 3 copyright (c) 2008 the icecube collaboration 4 5 @version: $Revision: $ 6 @date: $Date: $ 7 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 8 """ 9 10 from iceprod.core.dataclasses import * 11 import logging 12 import logger 13 import xmlrpclib 14 import copy 15 import iceprod.core.logger 16 from cPickle import loads,dumps 17 import sys,os 18 import types 19 from iceprod.core import ipxml 20 21 22

23 -class ParamDb:

24 25 logger = logging.getLogger('paramdb') 26

27 - def __init__(self):

28 self.metaprojects = {} 29 self.projects = {} 30 self.modules = {} 31 self.parameters = {} 32 self.cached = True 33 return

34 35

36 - def load(self,metaproject):

37 """ 38 load contents of metaproject tree to database 39 @param metaproject: metaproject object 40 """ 41 self.metaproject = metaproject 42 return 43

44 - def loadfile(self,filename):

45 parser = XMLParamDBParser() 46 try: 47 mplist = parser.ParseFile(filename) 48 self.loadcache(mplist) 49 except Exception,e: 50 self.logger.error('Unable to parse "%s". %s' % (filename,e))

51

52 - def loadcache(self,mpdict):

53 for mp in mpdict.values(): 54 self.metaprojects[mp.GetId()] = mp 55 for p in mp.GetProjectList(): 56 self.projects[p.GetId()] = p 57 for s in p.GetServices(): 58 self.modules[s.GetId()] = s 59 for pp in s.GetParameters(): 60 self.parameters[pp.GetId()] = pp 61 for m in p.GetModules(): 62 self.modules[m.GetId()] = m 63 for pp in m.GetParameters(): 64 self.parameters[pp.GetId()] = pp 65 self.logger.info("loaded cache")

66 67

68 - def adddependencies(self, project, mp,maxrecursion=3):

69 """ 70 Recursively add project and its dependencies to metaproject 71 """ 72 if maxrecursion < 0: return 73 for dep in self.GetProjectDependencies(project.GetId(),mp.GetId()): 74 if dep.GetName() != project.GetName(): 75 project.AddDependency(dep) 76 self.adddependencies(dep,mp,maxrecursion-1)

77

78 - def SwitchMetaProject(self, iconfig, id, name, version):

79 """ 80 Add selected metaproject to configuration. Import modules and their 81 parameters. 82 """ 83 84 # Create a new metaproject and set version 85 newmp = MetaProject() 86 newmp.SetId(id) 87 newmp.SetName(name) 88 newmp.SetVersion(version) 89 90 # Create a temporary configuration 91 new_config = IceTrayConfig() 92 new_config.AddMetaProject(newmp.GetName(),newmp) 93 94 # Get all configured modules in previous configuration 95 for oldmod in iconfig.GetModules(): 96 for project in self.GetProjectsMM(oldmod,newmp): 97 mp.AddProject(project.GetName(),project) 98 self.adddependencies(project, newmp) 99 100 # Get all configured services in previous configuration 101 for oldserv in iconfig.GetServices(): 102 for project in self.GetProjectsMM(oldserv,newmp): 103 mp.AddProject(project.GetName(),project) 104 self.adddependencies(project, newmp) 105 106 # Get missing projects and dependencies 107 if iconfig.HasMetaProject(newmp.GetName()): 108 oldmp = iconfig.GetMetaProject(newmp.GetName()) 109 for p in self.GetProjects(newmp.GetId()): 110 if oldmp.HasProject(p.GetName()) and not newmp.HasProject(p.GetName()): 111 self.logger.debug(p.GetName()) 112 newmp.AddProject(p.GetName(),p) 113 for oldmod in iconfig.GetModules(): 114 if not new_config.HasModule(oldmod.GetName()): 115 depend = self.GetProjectsMM(oldmod,newmp) 116 if not depend: 117 oldmod.SetDescription( 'class %s not found in metaproject %s %s' %( 118 oldmod.GetClass(), 119 newmp.GetName(), 120 newmp.GetVersion())) 121 else: 122 oldmod.AddProject(depend[0].GetName(),depend[0]) 123 new_config.AddModule(oldmod) 124 125 for oldserv in iconfig.GetServices(): 126 if not new_config.HasService(oldserv.GetName()): 127 depend = self.GetProjectsMM(oldserv,newmp) 128 if not depend: 129 oldserv.SetDescription('class %s not found for project %s %s' %( 130 oldserv.GetClass(), 131 p.GetName(), 132 p.GetVersion())) 133 else: 134 oldserv.AddProject(depend[0].GetName(),depend[0]) 135 new_config.AddService(oldserv) 136 137 # Overwrite metaproject 138 iconfig.AddMetaProject(newmp.GetName(),newmp) 139 140 # Overwrite modules 141 for newmod in new_config.GetModules(): 142 iconfig.AddModule(newmod) 143 144 # Overwrite services 145 for newserv in new_config.GetServices(): 146 iconfig.AddService(newserv) 147 return newmp

148 149 150

151 - def GetMetaProjects(self):

152 return self.metaprojects.values() 153

154 - def GetProjectsSM(self,module,metaproj):

155 return self.GetProjectsMM(module,metaproj,'service') 156

157 - def SetMetaProjectId(self,metaproj):

158 for m in self.metaprojects.values(): 159 if m.GetName() == metaproj.GetName() and m.GetVersion() == metaproj.GetVersion(): 160 mp = m 161 metaproj.SetId(m.GetId()) 162 return metaproj 163 raise Exception,"Could not find %s %s in metaprojects" %(metaproj.GetName(),metaproj.GetVersion()) 164

165 - def GetProjectsMM(self,module,metaproj,modtype='module'):

166 167 if metaproj.GetId() > 0: 168 mp = self.metaprojects[metaproj.GetId()] 169 else: 170 self.SetMetaProjectId(metaproj) 171 mp = self.metaprojects[metaproj.GetId()] 172 for p in mp.GetProjectList(): 173 if modtype == 'module': 174 mods = p.GetModules() 175 elif modtype == 'service': 176 mods = p.GetServices() 177 for mod in mods: 178 if mod.GetClass() == module.GetClass(): 179 deps = [] 180 for d in [p] + p.GetDependencies(): 181 newproj = Project() 182 newproj.SetName(d.GetName()) 183 newproj.SetVersion(d.GetVersion()) 184 newproj.SetId(d.GetId()) 185 deps.append(newproj) 186 return deps 187 return [] 188 189

190 - def GetProjects(self,metaproject_id):

191 plist = [] 192 if self.metaprojects.has_key(metaproject_id): 193 plist = self.metaprojects[metaproject_id].GetProjectList() 194 return plist 195 196

197 - def GetProjectDependencies(self,project_id,metaproject_id):

198 dlist = [] 199 if self.projects.has_key(project_id): 200 dlist = self.projects[project_id].GetDependencies() 201 return dlist 202 203

204 - def GetServices(self,project_id):

205 slist = [] 206 for s in self.projects[project_id].GetServices(): 207 service = Service() 208 service.SetName(s.GetName()) 209 service.SetClass(s.GetClass()) 210 service.SetId(s.GetId()) 211 slist.append(service) 212 return slist 213

214 - def GetServicesP(self,name,version):

215 slist = [] 216 for p in self.projects.values(): 217 if p.GetName() == name and p.GetVersion() == version: 218 return self.GetServices(p.GetId()) 219 return slist 220

221 - def GetModules(self,project_id):

222 mlist = [] 223 for m in self.projects[project_id].GetModules(): 224 module = Module() 225 module.SetName(m.GetName()) 226 module.SetClass(m.GetClass()) 227 module.SetId(m.GetId()) 228 mlist.append(module) 229 return mlist 230

231 - def GetModulesP(self,name,version):

232 mlist = [] 233 for p in self.projects.values(): 234 if p.GetName() == name and p.GetVersion() == version: 235 self.logger.debug("return self.GetModules(%d)" % p.GetId()) 236 return self.GetModules(p.GetId()) 237 return mlist 238

239 - def GetParameters(self,module_id):

240 plist = [] 241 for param in self.modules[module_id].GetParameters(): 242 parameter = Parameter() 243 parameter.SetName(param.GetName()) 244 parameter.SetType(param.GetType()) 245 parameter.SetValue(copy.copy(param.GetValue())) 246 plist.append(parameter) 247 return plist 248 249 250 251 252 SupportedTypes = ['bool', 'int', 'long', 'float', 'double', 'string','OMKey'] 253 VectorTypes = ['boolv', 'intv', 'longv', 'floatv', 'doublev', 'stringv','OMKeyv'] 254 255

256 -class XMLParamDB(ParamDb):

257 258 logger = logging.getLogger('XMLParamDB') 259

260 - def __init__(self,uri='http://x2100.icecube.wisc.edu/dtd/i3paramdb.dtd'):

261 """ 262 Initialize and build DOM tree 263 from IceTrayConfig object 264 """ 265 ParamDb.__init__(self) 266 self.cached = False 267 268 dom = ipxml.getDOMImplementation() 269 doctype = dom.createDocumentType("i3paramdb", "-//W3C//DTD XML 1.0 Strict//EN", uri) 270 self.doc = dom.createDocument( None, "i3paramdb", doctype ) 271 config = self.doc.documentElement 272 config.setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance') 273 config.setAttribute('xsi:noNamespaceSchemaLocation','config.xsd')

274 275

276 - def AddMetaProject(self,metaproject):

277 self.metaprojects[metaproject.GetId()] = metaproject 278 self.AddMetaProjects()

279

280 - def AddMetaProjects(self):

281 """ 282 Load projects from IceTrayConfig and add the to DOM tree 283 """ 284 285 286 doc = self.doc 287 icetrayA = doc.getElementsByTagName('i3paramdb') 288 289 if icetrayA: 290 icetray = icetrayA[0] 291 else: 292 docNode = self.doc.documentElement 293 icetray = self.doc.createElement( 'i3paramdb' ) 294 docNode.appendChild(icetray) 295 296 297 for mp in self.metaprojects.values(): 298 299 metaprojElement = self.doc.createElement( 'metaproject' ) 300 metaprojElement.setAttribute('id',str(mp.GetId())) 301 mpNameElement = self.doc.createElement( 'name' ) 302 mpNameNode = self.doc.createTextNode( mp.GetName() ) 303 mpNameElement.appendChild(mpNameNode) 304 metaprojElement.appendChild(mpNameElement) 305 306 mpVersionElement = self.doc.createElement( 'version' ) 307 ver = mp.GetVersion() 308 mpVersionNode = self.doc.createTextNode( mp.GetVersion() ) 309 mpVersionElement.appendChild(mpVersionNode) 310 metaprojElement.appendChild(mpVersionElement) 311 icetray.appendChild(metaprojElement) 312 313 projects = mp.GetProjectList() 314 for p in projects: 315 316 projElement = self.doc.createElement( 'project' ) 317 projElement.setAttribute('id',str(p.GetId())) 318 if p.GetDependencies(): 319 projElement.setAttribute('dependencies',','.join( 320 map(lambda x: type(x) is types.StringType and x or x.GetName(),p.GetDependencies()))) 321 322 nameElement = self.doc.createElement( 'name' ) 323 nameNode = self.doc.createTextNode( p.GetName() ) 324 nameElement.appendChild(nameNode) 325 projElement.appendChild(nameElement) 326 327 versionElement = self.doc.createElement( 'version' ) 328 ver = p.GetVersion() 329 versionNode = self.doc.createTextNode( p.GetVersion() ) 330 versionElement.appendChild(versionNode) 331 projElement.appendChild(versionElement) 332 metaprojElement.appendChild(projElement) 333 servicesElement = self.doc.createElement( 'services' ) 334 projElement.appendChild(servicesElement) 335 self.AddServices(servicesElement,p) 336 modulesElement = self.doc.createElement( 'modules' ) 337 projElement.appendChild(modulesElement) 338 self.AddModules(modulesElement,p)

339 340

341 - def mkValueElement(self,ptype,pvalue,doc):

342 """ 343 Create a value element depending on the type 344 """ 345 if ptype.startswith("OMKey"): 346 pValueElement = self.doc.createElement( 'map' ) 347 pEntryElement0 = self.doc.createElement( 'cvalue' ) 348 pEntryElement0.appendChild(self.doc.createTextNode(str(pvalue.stringid))) 349 pEntryElement0.setAttribute('label', 'stringid' ) 350 pValueElement.appendChild(pEntryElement0) 351 pEntryElement1 = self.doc.createElement( 'cvalue' ) 352 pEntryElement1.appendChild(self.doc.createTextNode(str(pvalue.omid))) 353 pEntryElement1.setAttribute('label', 'omid' ) 354 pValueElement.appendChild(pEntryElement1) 355 else: 356 pValueElement = self.doc.createElement( 'value' ) 357 pValueElement.appendChild(self.doc.createTextNode(str(pvalue.value))) 358 if pvalue.unit: 359 pValueElement.setAttribute('unit', pvalue.unit ) 360 return pValueElement

361

362 - def AddServices(self,docElement,project):

363 """ 364 Load services from IceTrayConfig and add the to DOM tree 365 """ 366 367 services = project.GetServices() 368 369 for s in services: 370 371 servElement = self.doc.createElement( 'service' ) 372 servElement.setAttribute('class',s.GetClass()) 373 servElement.setAttribute('id',str(s.GetId())) 374 projects = ','.join( [p.GetName() for p in s.GetProjectList()] ) 375 servElement.setAttribute('projects',projects) 376 docElement.appendChild(servElement) 377 378 nameElement = self.doc.createElement( 'name' ) 379 nameNode = self.doc.createTextNode( s.GetName() ) 380 nameElement.appendChild(nameNode) 381 servElement.appendChild(nameElement) 382 383 paramsElement = self.doc.createElement( 'parameters' ) 384 servElement.appendChild(paramsElement) 385 386 for param in s.GetParameters(): 387 388 pNameElement = self.doc.createElement( 'name' ) 389 pNameNode = self.doc.createTextNode( param.GetName() ) 390 pNameElement.appendChild(pNameNode) 391 pTypeElement = self.doc.createElement( param.GetType() ) 392 pTypeElement.setAttribute('id',str(param.GetId())) 393 pTypeElement.appendChild(pNameElement) 394 paramsElement.appendChild(pTypeElement) 395 ptype = param.GetType() 396 397 if ptype in VectorTypes: 398 for pvalue in param.GetValue(): 399 pValueElement = self.mkValueElement(ptype,pvalue,self.doc) 400 pTypeElement.appendChild(pValueElement) 401 else: 402 pValueElement = self.mkValueElement(ptype,param.GetValue(),self.doc) 403 pTypeElement.appendChild(pValueElement)

404 405 406

407 - def AddModules(self,docElement,project):

408 """ 409 Load modules from IceTrayConfig and add the to DOM tree 410 """ 411 412 modules = project.GetModules() 413 414 for m in modules: 415 416 modElement = self.doc.createElement( 'module' ) 417 modElement.setAttribute('class',m.GetClass()) 418 modElement.setAttribute('id',str(m.GetId())) 419 projects = ','.join( [p.GetName() for p in m.GetProjectList()] ) 420 modElement.setAttribute('projects',projects) 421 docElement.appendChild(modElement) 422 423 nameElement = self.doc.createElement( 'name' ) 424 nameNode = self.doc.createTextNode( m.GetName() ) 425 nameElement.appendChild(nameNode) 426 modElement.appendChild(nameElement) 427 428 paramsElement = self.doc.createElement( 'parameters' ) 429 modElement.appendChild(paramsElement) 430 431 for param in m.GetParameters(): 432 433 pNameElement = self.doc.createElement( 'name' ) 434 pNameNode = self.doc.createTextNode( param.GetName() ) 435 pNameElement.appendChild(pNameNode) 436 pTypeElement = self.doc.createElement( param.GetType() ) 437 pTypeElement.setAttribute('id',str(param.GetId())) 438 pTypeElement.appendChild(pNameElement) 439 paramsElement.appendChild(pTypeElement) 440 ptype = param.GetType() 441 442 if ptype in VectorTypes: 443 for pvalue in param.GetValue(): 444 pValueElement = self.mkValueElement(ptype,pvalue,self.doc) 445 pTypeElement.appendChild(pValueElement) 446 else: 447 pValueElement = self.mkValueElement(ptype,param.GetValue(),self.doc) 448 pTypeElement.appendChild(pValueElement)

449 450

451 - def getDOM(self):

452 return self.doc

453

454 - def write_to_file(self, fname=None):

455 """ 456 Write DOM tree as human readable xml to a file 457 """ 458 if fname: 459 file_object = open(fname, "w") 460 ipxml.PrettyPrint(self.doc, file_object) 461 file_object.close() 462 else: 463 ipxml.PrettyPrint(self.doc)

464

465 -class XMLParamDBParser:

466 467 logger = logging.getLogger('XMLParamDBParser') 468

469 - def __init__(self):

470 self.pdb = IceTrayConfig() 471 self.metaprojects = {}

472

473 - def ParseMetaProjects(self,doc):

474 """ 475 Load meatprojects from doc 476 """ 477 self.logger.debug('PARSE METAPROJECTS') 478 479 ipdb = doc.getElementsByTagName('i3paramdb') 480 if not ipdb: return 481 metaprojects = ipdb[0].getElementsByTagName('metaproject') 482 483 for mp in metaprojects: 484 metaproject = MetaProject() 485 486 # fill MetaProject object: 487 mpname = mp.getElementsByTagName('name') 488 mpname = mpname[0].firstChild.data 489 mpver = mp.getElementsByTagName('version') 490 mpver = mpver[0].firstChild.data 491 id = int(mp.getAttribute('id')) 492 493 metaproject.SetName(mpname) 494 metaproject.SetVersion(mpver) 495 metaproject.SetId(id) 496 497 projects = mp.getElementsByTagName('project') 498 499 for p in projects: 500 sys.stdout.write('.') 501 project = Container() 502 503 # fill Project object: 504 pname = p.getElementsByTagName('name') 505 pname = pname[0].firstChild.data 506 pver = p.getElementsByTagName('version') 507 pver = pver[0].firstChild.data 508 id = int(p.getAttribute('id')) 509 510 project.SetName(pname) 511 project.SetVersion(pver) 512 project.SetId(id) 513 514 depends = p.getAttribute('dependencies') 515 if depends: 516 for d in depends.split(','): 517 project.AddDependency(d) 518 519 self.AddServices(project,p) 520 self.AddModules(project,p) 521 522 metaproject.AddProject(pname,project) 523 self.logger.debug("added project %s %d" % (project.GetName(),project.GetId())) 524 525 projects = metaproject.GetProjectList() 526 for p in projects: 527 dependencies = map(metaproject.GetProject,p.GetDependencies()) 528 p.ClearDependencies() 529 for dp in dependencies: 530 p.AddDependency(dp) 531 532 self.metaprojects[metaproject.GetId()] = metaproject 533 self.logger.debug("added metaproject %s" % metaproject.GetName()) 534 sys.stdout.write('done\n')

535 536

537 - def GetProjectList(self):

538 return self.projects.values()

539 540

541 - def AddServices(self,project,projectElement):

542 """ 543 Load services from icetray configuration file 544 """ 545 self.logger.debug('ADD SERVICES') 546 547 service = projectElement.getElementsByTagName('service') 548 for s in service: 549 service = Service() 550 551 sclass = s.getAttribute('class') 552 id = int(s.getAttribute('id')) 553 554 # fill Project object: prname may be a list 555 service.AddProject(project.GetName(),project) 556 sname = s.getElementsByTagName('name') 557 sname = sname[0].firstChild.data 558 559 # fill Service object 560 service.SetName(sname) 561 service.SetClass(sclass) 562 service.SetId(id) 563 project.AddService(service) 564 565 parameters = s.getElementsByTagName('parameters') 566 # If there are parameters, continue 567 if len(parameters) != 0: 568 for pmtr in parameters: 569 for stype in SupportedTypes + VectorTypes: 570 ps = pmtr.getElementsByTagName(stype) 571 if len(ps) != 0: 572 for p in ps: 573 sparameter = Parameter() 574 575 pname = p.getElementsByTagName('name') 576 pname = pname[0].firstChild.data 577 pvalues = p.getElementsByTagName('value') 578 pmaps = p.getElementsByTagName('map') 579 580 # fill Parameter object 581 sparameter.SetType(stype) 582 sparameter.SetName(pname) 583 584 value = [] 585 # set I3Units (if any) 586 for pvalue in pvalues: 587 val = Value(pvalue.firstChild.data) 588 if pvalue.getAttribute('unit'): 589 punit = pvalue.getAttribute('unit') 590 val.unit = punit 591 if pvalue.getAttribute('format'): 592 pformat = pvalue.getAttribute('format') 593 val.format = pformat 594 value.append( val ) 595 596 for pmap in pmaps: 597 pcvalues = pmap.getElementsByTagName('cvalue') 598 val = pyOMKey("0","0") 599 for pcval in pcvalues: 600 label = pcval.getAttribute('label') 601 if label == 'stringid': 602 val.stringid = pcval.firstChild.data 603 elif label == 'omid': 604 val.omid = pcval.firstChild.data 605 value.append( val ) 606 607 if stype in VectorTypes: sparameter.SetValue( value ) 608 else: sparameter.SetValue( value[0] ) 609 610 service.AddParameter(sparameter) 611 self.logger.debug("%s %s %s %s" % (sname,stype,pname,sparameter.GetValue())) 612 project.AddService(service)

613 614 615

616 - def AddModules(self,project,projectElement):

617 """ 618 Load modules from icetray configuration file 619 """ 620 self.logger.debug('ADD MODULES') 621 622 modules = projectElement.getElementsByTagName('module') 623 624 for m in modules: 625 module = Module() 626 627 mclass = m.getAttribute('class') 628 module.AddProject(project.GetName(),project) 629 630 mname = m.getElementsByTagName('name') 631 mname = mname[0].firstChild.data 632 id = int(m.getAttribute('id')) 633 634 # fill Module object 635 module.SetName(mname) 636 module.SetClass(mclass) 637 module.SetId(id) 638 project.AddModule(module) 639 640 parameters = m.getElementsByTagName('parameters') 641 642 # If there are parameters, continue 643 if len(parameters) != 0: 644 for pmtr in parameters: 645 # We have the parameters, now loop over the 646 # supported types and see if there are any 647 # used in the xml file (for example, if there are 648 # <int> tags used) 649 for stype in SupportedTypes + VectorTypes: 650 # This is an array of Type tags for 651 # each supported type 652 ps = pmtr.getElementsByTagName(stype) 653 # If there are any supported types used 654 # iterate over them and add the parameter 655 if len(ps) != 0: 656 for p in ps: 657 mparameter = Parameter() 658 659 pname = p.getElementsByTagName('name') 660 pname = pname[0].firstChild.data 661 pvalues = p.getElementsByTagName('value') 662 pmaps = p.getElementsByTagName('map') 663 id = int(p.getAttribute('id')) 664 # fill Parameter object 665 mparameter.SetType(stype) 666 mparameter.SetName(pname) 667 mparameter.SetId(id) 668 value = [] 669 670 # set I3Units (if any) 671 for pvalue in pvalues: 672 val = Value(pvalue.firstChild.data) 673 if pvalue.getAttribute('unit'): 674 punit = pvalue.getAttribute('unit') 675 val.unit = punit 676 if pvalue.getAttribute('format'): 677 pformat = pvalue.getAttribute('format') 678 val.format = pformat 679 value.append( val ) 680 681 for pmap in pmaps: 682 pcvalues = pmap.getElementsByTagName('cvalue') 683 val = pyOMKey("0","0") 684 for pcval in pcvalues: 685 label = pcval.getAttribute('label') 686 if label == 'stringid': 687 val.stringid = pcval.firstChild.data 688 elif label == 'omid': 689 val.omid = pcval.firstChild.data 690 value.append( val ) 691 692 if stype in VectorTypes: mparameter.SetValue( value ) 693 else: mparameter.SetValue( value[0] ) 694 695 module.AddParameter(mparameter) 696 self.logger.debug("%s %s %s %s" %(mname,stype,pname,mparameter.GetValue())) 697 project.AddModule(module)

698 699

700 - def ParseFile(self,config_file_name,validate=0):

701 702 # get the DOM tree 703 doc = None 704 try: # try to validate. Will fail with older configurations 705 706 # open xml file for reading 707 xmlin = open(config_file_name, 'r') 708 self.logger.debug("opened %s" % config_file_name) 709 710 sys.stdout.write('loading pdb ') 711 sys.stdout.write('...') 712 doc = ipxml.Parse(xmlin, validate=validate) 713 self.logger.debug("doc = ipxml.Parse(xmlin, validate=%s)" % validate) 714 xmlin.close() 715 716 except SAXParseException, xe: 717 sys.excepthook(excinfo[0],excinfo[1],excinfo[2]) 718 self.logger.error("Exception: " + str(xe) ) 719 return 720 721 # metaprojects 722 self.logger.debug("self.ParseMetaProjects(doc)") 723 self.ParseMetaProjects(doc) 724 725 self.cached = True 726 return self.metaprojects

727

728 -class RPCParamDB(ParamDb):

729

730 - def __init__(self,url=None,geturl=lambda:''):

731 """ 732 Initialize and build DOM tree 733 from IceTrayConfig object 734 """ 735 ParamDb.__init__(self) 736 self.url = url 737 self.geturl = geturl 738 self.server = None 739 self.connected = False 740 self.printtext = lambda x: sys.stdout.write(x+"\n") 741 self.cached = True

742

743 - def loadfile(self,filename,username,password):

744 if not self.connect(): return None 745 parser = XMLParamDBParser() 746 try: 747 mplist = parser.ParseFile(filename) 748 for mp in mplist.values(): 749 self.load(self,mp,username,password) 750 except Exception,e: 751 self.logger.error('Unable to parse "%s". %s' % (filename,e))

752 753

754 - def setURL(self,url='http://condor.icecube.wisc.edu:9080'):

755 self.url = url

756

757 - def connect(self):

758 if not self.connected: 759 if not self.url: 760 self.url = self.geturl() 761 if self.url.startswith('http'): 762 self.server = xmlrpclib.ServerProxy(self.url) 763 self.connected = True 764 else: 765 self.printtext("Please specify a valid url: %s" % self.url) 766 return self.connected

767

768 - def SetPrinter(self,printer):

769 self.printtext = printer

770

771 - def _print(self,text):

772 if self.printtext: 773 self.printtext(text) 774 else: 775 print text

776

777 - def load(self,metaproject,username,password):

778 """ 779 load contents of metaproject tree to database 780 @param metaproject: metaproject object 781 """ 782 if not self.connect(): return None 783 return self.server.LoadParams(dumps(metaproject),username,password)

784

785 - def download(self):

786 if not self.connect(): return None 787 self.loadcache(loads(self.server.DownloadParams())) 788 789

790 - def savefile(self,filename):

791 writer = XMLParamDB(self.metaprojects.values()) 792 writer.write_to_file(filename)

793

794 - def authenticate(self,username,password):

795 if not self.connect(): return None 796 try: 797 return self.server.authenticate(username,password) 798 except Exception, e: 799 self._print("Caught exception: %s " % str(e)) 800 return False

801 802 # ParamDB wrappers

803 - def SwitchMetaProject(self,iconfig,id,name,version):

804 if not self.connect(): return None 805 return loads(self.server.SwitchMetaProject(dumps(iconfig),id,name,dumps(version))) 806

807 - def GetMetaProjects(self):

808 if not self.connect(): return None 809 return loads(self.server.GetMetaProjects()) 810

811 - def GetProjectsSM(self,module,metaproj):

812 if not self.connect(): return None 813 return loads(self.server.GetProjectsSM(dumps(module),dumps(metaproj)))

814

815 - def GetProjectsMM(self,module,metaproj):

816 if not self.connect(): return None 817 return loads(self.server.GetProjectsMM(dumps(module),dumps(metaproj)))

818

819 - def GetProjects(self,metaproject_id):

820 if not self.connect(): return None 821 return loads(self.server.GetProjects(metaproject_id))

822

823 - def GetProjectDependencies(self,project_id,metaproject_id):

824 if not self.connect(): return None 825 return loads(self.server.GetProjectDependencies(project_id,metaproject_id)) 826

827 - def GetServices(self,project_id):

828 if not self.connect(): return None 829 return loads(self.server.GetServices(project_id)) 830

831 - def GetServicesP(self,name,version):

832 if not self.connect(): return None 833 return loads(self.server.GetServicesP(name,dumps(version))) 834

835 - def GetModules(self,project_id):

836 if not self.connect(): return None 837 return loads(self.server.GetModules(project_id)) 838

839 - def GetModulesP(self,name,version):

840 if not self.connect(): return None 841 return loads(self.server.GetModulesP(name,dumps(version))) 842

843 - def GetIceProdModules(self):

844 if not self.connect(): return None 845 return loads(self.server.GetIceProdModules()) 846

847 - def GetParameters(self,module_id):

848 if not self.connect(): return None 849 return loads(self.server.GetParameters(module_id)) 850

851 - def SetMetaProjectId(self,metaproj):

852 metaproj.SetId(self.fetch_metaproject_id(metaproj.GetName(),metaproj.GetVersion())) 853

854 - def fetch_metaproject_id(self, name, version):

855 if not self.connect(): return None 856 return self.server.fetch_metaproject_id(name,dumps(version))

857

858 - def fetch_metaproject_list(self):

859 if not self.connect(): return None 860 return loads(self.server.fetch_metaproject_list()) 861

862 - def fetch_project_list(self,metaproject_id):

863 if not self.connect(): return None 864 return loads(self.server.fetch_project_list(metaproject_id)) 865

866 - def fetch_project(self,id):

867 if not self.connect(): return None 868 return loads(self.server.fetch_project(id))

869

870 - def fetch_project_id(self,pname,pversion):

871 if not self.connect(): return None 872 return loads(self.server.fetch_project_id(name,dumps(pversion)))

873

874 - def fetch_service_id(self,service,pid):

875 if not self.connect(): return None 876 return self.server.fetch_service_id(dumps(service),pid) 877

878 - def fetch_module_id(self,module,mid):

879 if not self.connect(): return None 880 return self.server.fetch_module_id(dumps(module),mid) 881

882 - def fetch_project_dependencies(self,project_id,metaproject_id):

883 if not self.connect(): return None 884 return loads(self.server.fetch_project_dependencies(project_id,metaproject_id)) 885

886 - def fetch_modules_from_project_id(self,project_id):

887 if not self.connect(): return None 888 return loads(self.server.fetch_modules_from_project_id(project_id))

889

890 - def fetch_modules_for_project(self,name,version):

891 if not self.connect(): return None 892 return loads(self.server.fetch_modules_for_project(name,dumps(version))) 893

894 - def fetch_services_for_project(self,name,version):

895 if not self.connect(): return None 896 return loads(self.server.fetch_services_for_project(name,dumps(version))) 897

898 - def fetch_module_parameters(self,module_id):

899 if not self.connect(): return None 900 return loads(self.server.fetch_module_parameters(module_id)) 901

902 - def fetch_service_parameters(self,module_id):

903 if not self.connect(): return None 904 return loads(self.server.fetch_service_parameters(module_id)) 905

906 - def show_dataset_table(self,search_string=""):

907 if not self.connect(): return None 908 try: 909 return loads(self.server.showrunlist(search_string)) 910 except Exception,e: 911 self._print("Caught exception: %s " % str(e)) 912 return [] 913 914 915 916 if __name__ == '__main__': 917 918 if len(sys.argv) != 2: 919 sys.exit() 920 921 parser = XMLParamDBParser() 922 metaprojects = parser.ParseFile(sys.argv[1],validate=1) 923 924 if metaprojects: 925 writer = XMLParamDB(metaprojects) 926 writer.write_to_file() 927

Source Code for Module iceprod.core.paramdb