1
2
3
4 """
5 A class for writting a Python IceTrayConfig object to XML
6
7 copyright (c) 2005 the icecube collaboration
8
9 @version: $Revision: 1.0 $
10 @date: $Date: $
11 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu>
12
13 @todo: Many of the same todo items on xmlparser.py
14 """
15
16 import sys,os
17 import string
18 import re
19 import types
20 import logging
21 import logger
22 from iceprod.core import ipxml
23 from dataclasses import *
24 from iceprod.core.metadata import *
25 from os.path import expandvars
26
27
28 SupportedTypes = ['bool', 'int', 'long', 'float', 'double', 'string','OMKey']
29 VectorTypes = ['boolv', 'intv', 'longv', 'floatv', 'doublev', 'stringv','OMKeyv']
30
32
33 - def __init__(self,
34 steering,uri='http://x2100.icecube.wisc.edu/dtd/icetray.v2.dtd',
35 add_descriptions=False):
36 """
37 Initialize and build DOM tree
38 from IceTrayConfig object
39 """
40 self.steering = steering
41 dom = ipxml.getDOMImplementation()
42 self.logger = logging.getLogger('XMLWriter')
43
44 doctype = dom.createDocumentType("configuration",
45 "-//W3C//DTD XML 1.0 Strict//EN", uri)
46
47 self.doc = dom.createDocument( None, "configuration", doctype )
48 config = self.doc.documentElement
49 if steering.GetParentId():
50 config.setAttribute('parentid', str(steering.GetParentId()))
51 if steering.GetVersion():
52 config.setAttribute('version', str(steering.GetVersion()))
53 config.setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance')
54 config.setAttribute('xsi:noNamespaceSchemaLocation','config.xsd')
55
56
57 if steering.HasDIFPlus():
58 mw = MetadataWriter(steering.GetDIFPlus(),False)
59 mw.AddDIFPlus(self.doc,config)
60
61 self.LoadSteering(config,steering)
62 for tray in steering.GetTrays():
63 trayElement = self.AddTray(config,tray)
64 self.AddIceProdPres(trayElement,tray)
65 self.AddServices(trayElement,tray,add_descriptions)
66 self.AddModules(trayElement,tray,add_descriptions)
67 self.AddConnections(trayElement,tray)
68 self.AddIceProdPosts(trayElement,tray)
69 self.AddMetaProjects(trayElement,tray)
70 self.AddProjects(trayElement,tray)
71
73 """
74 Read Steering parameters from IceTrayConfig tree
75 and generate the entries in DOM tree.
76 """
77
78 doc = self.doc
79
80 steeringElement = doc.createElement( 'steering' )
81 docNode.appendChild(steeringElement)
82
83 category = steering.GetCategory()
84 categoryElement = doc.createElement( 'category' )
85 if category:
86 categoryElement.appendChild(doc.createTextNode(category))
87 steeringElement.appendChild(categoryElement)
88
89 for ex in steering.GetExterns():
90 externalElement = doc.createElement( 'external' )
91 steeringElement.appendChild(externalElement)
92 nameElement = doc.createElement('name')
93 nameElement.appendChild(doc.createTextNode(ex.GetName()))
94 externalElement.appendChild(nameElement)
95
96 if ex.GetDescription():
97 descElement = doc.createElement('description')
98 descElement.appendChild(doc.createTextNode(ex.GetDescription()))
99 externalElement.appendChild(descElement)
100 if ex.GetVersion():
101 versionElement = doc.createElement('version')
102 versionElement.appendChild(doc.createTextNode(ex.GetVersion()))
103 externalElement.appendChild(versionElement)
104 execElement = doc.createElement('executable')
105 execElement.appendChild(doc.createTextNode(ex.GetExec()))
106 externalElement.appendChild(execElement)
107 if ex.GetArgs():
108 argElement = doc.createElement('args')
109 argElement.appendChild(doc.createTextNode(ex.GetArgs()))
110 externalElement.appendChild(argElement)
111 if ex.GetInFile():
112 inElement = doc.createElement('in')
113 inElement.appendChild(doc.createTextNode(ex.GetInFile()))
114 externalElement.appendChild(inElement)
115 if ex.GetOutFile():
116 outElement = doc.createElement('out')
117 outElement.appendChild(doc.createTextNode(ex.GetOutFile()))
118 externalElement.appendChild(outElement)
119 if ex.GetErrFile():
120 errElement = doc.createElement('err')
121 errElement.appendChild(doc.createTextNode(ex.GetErrFile()))
122 externalElement.appendChild(errElement)
123
124
125 for es in ex.GetSteering():
126 esteerElement = doc.createElement('extern-steering')
127
128 esNameElement = doc.createElement('name')
129 esNameElement.appendChild(doc.createTextNode(es.GetName()))
130 esteerElement.appendChild(esNameElement)
131
132 esTextElement = doc.createElement('text')
133 esTextElement.appendChild(doc.createTextNode(es.GetText()))
134 esteerElement.appendChild(esTextElement)
135
136 externalElement.appendChild(esteerElement)
137
138
139
140
141 of = steering.GetOF()
142 if of:
143 filtElement = doc.createElement( 'OF' )
144 steeringElement.appendChild(filtElement)
145 if of.GetPrefix():
146 filtElement.setAttribute('prefix',of.GetPrefix())
147 filtElement.setAttribute('search',str(of.search))
148 for path in of.GetPaths():
149 pathElement = doc.createElement( 'path' )
150 if path.GetRegex():
151 pathElement.setAttribute('regex',path.regex)
152 pathElement.appendChild(doc.createTextNode(path.path))
153 filtElement.appendChild(pathElement)
154
155
156
157 steeringParams = doc.createElement( 'parameters' )
158 steeringElement.appendChild(steeringParams)
159 for par in steering.GetParameters():
160
161 ptype = par.GetType()
162 if not ptype in SupportedTypes:
163 self.logger.error('In steering, found unsupported type: %s' % ptype)
164 sys.exit(1)
165 pTypeElement = doc.createElement( ptype )
166 steeringParams.appendChild(pTypeElement)
167
168 pname = par.GetName()
169 pNameElement = doc.createElement( 'name' )
170 pNameElement.appendChild(doc.createTextNode(pname))
171 pTypeElement.appendChild(pNameElement)
172
173 pvalue = par.GetValue()
174 pValueElement = doc.createElement( 'value' )
175 pValueElement.appendChild(doc.createTextNode(pvalue))
176 pTypeElement.appendChild(pValueElement)
177
178 systemElement = doc.createElement( 'system' )
179 steeringElement.appendChild(systemElement)
180 for opt in steering.GetBatchOpts():
181
182 batchoptElement = doc.createElement( 'batchopt' )
183
184 bname = opt.GetName()
185 bNameElement = doc.createElement( 'name' )
186 bNameElement.appendChild(doc.createTextNode(bname))
187 batchoptElement.appendChild(bNameElement)
188
189 btype = opt.GetType()
190 bTypeElement = doc.createElement( 'type' )
191 bTypeElement.appendChild(doc.createTextNode(btype))
192 batchoptElement.appendChild(bTypeElement)
193
194 bvalue = opt.GetValue()
195 bValueElement = doc.createElement( 'value' )
196 bValueElement.appendChild(doc.createTextNode(bvalue))
197 batchoptElement.appendChild(bValueElement)
198
199 systemElement.appendChild(batchoptElement)
200
201 for opt in steering.GetSysOpts():
202 sysoptElement = doc.createElement( 'sysopt' )
203
204 sname = opt.GetName()
205 sNameElement = doc.createElement( 'name' )
206 sNameElement.appendChild(doc.createTextNode(sname))
207 sysoptElement.appendChild(sNameElement)
208
209 svalue = opt.GetValue()
210 sValueElement = doc.createElement( 'value' )
211 sValueElement.appendChild(doc.createTextNode(str(svalue)))
212 sysoptElement.appendChild(sValueElement)
213 systemElement.appendChild(sysoptElement)
214
215
216 for depend in steering.GetDependencies():
217 dependencyElement = doc.createElement( 'dependency' )
218 dependencyElement.appendChild(doc.createTextNode(depend))
219 steeringElement.appendChild(dependencyElement)
220
221 desc = steering.GetDescription()
222 if desc:
223 descElement = doc.createElement( 'description' )
224 descElement.appendChild(doc.createTextNode(desc))
225 steeringElement.appendChild(descElement)
226
227
228 tasks = steering.GetTaskDefinitions()
229 if tasks:
230 tasksElement = doc.createElement( 'tasks' )
231
232
233 for name,task in tasks.items():
234 logger.debug("found task %s" % name)
235 taskElement = doc.createElement( 'task' )
236 taskElement.setAttribute('id', name)
237 if task.ParallelExecutionEnabled():
238 taskElement.setAttribute('parallel', 'true')
239
240 trays = task.GetTrays().items()
241 for idx,tray in task.GetTrays().items():
242 trayElement = doc.createElement( 'taskTray' )
243 trayElement.setAttribute('tray', str(idx))
244 iters = ",".join(map(str, tray.GetIters()))
245 trayElement.setAttribute('iters', iters)
246 msg = "task %s has tray %u, iters %s"
247 logger.debug(msg % (name,idx,tray.GetIters()))
248 if tray.RunExternsEnabled():
249 trayElement.setAttribute('externs', 'true')
250 taskElement.appendChild(trayElement)
251
252 if task.GetRequirements():
253 reqsElement = doc.createElement( 'taskReqs' )
254 reqsElement.appendChild(doc.createTextNode(task.GetRequirements()))
255 taskElement.appendChild(reqsElement)
256 tasksElement.appendChild(taskElement)
257
258
259 for name,task in tasks.items():
260 for parent in task.GetParents():
261 logger.debug("task %s has parent %s" % (name, parent))
262 relElement = doc.createElement( 'taskRel' )
263 parentElement = doc.createElement( 'taskParent' )
264 parentElement.setAttribute('taskId', parent)
265 childElement = doc.createElement( 'taskChild' )
266 childElement.setAttribute('taskId', name)
267
268 relElement.appendChild(parentElement)
269 relElement.appendChild(childElement)
270 tasksElement.appendChild(relElement)
271
272 logger.debug("finished processing all tasks")
273 steeringElement.appendChild(tasksElement)
274
275 - def AddTray(self,docElement,tray):
276 """
277 Adds icetray project libraries from IceTrayConfig to DOM tree
278 and sets them as attributes to <icetray> tag.
279 """
280 doc = self.doc
281 icetrayElement = doc.createElement( 'icetray' )
282 icetrayElement.setAttribute('events',str(tray.GetEvents()))
283 icetrayElement.setAttribute('iter',str(tray.GetIterations()))
284
285 if tray.GetName():
286 icetrayNameElement = doc.createElement( 'name' )
287 icetrayNameNode = doc.createTextNode( tray.GetName() )
288 icetrayNameElement.appendChild( icetrayNameNode )
289 icetrayElement.appendChild( icetrayNameElement )
290
291 file_types = ('input', 'output')
292 icetrayFilesElement = doc.createElement( 'files' )
293 for file_type in file_types:
294 if file_type == 'input':
295 func = tray.GetInputFiles
296 else:
297 func = tray.GetOutputFiles
298 iofElement = doc.createElement( file_type )
299 for file in func():
300 fileElement = doc.createElement( 'file' )
301 if file.IsPhotonicsTable():
302 fileElement.setAttribute( 'photonics', 'true' )
303 fileNode = doc.createTextNode( file.GetName() )
304 fileElement.appendChild( fileNode )
305 iofElement.appendChild( fileElement )
306 logger.debug("Added %s as an %s file" % (file.GetName(), file_type))
307 if func():
308 icetrayFilesElement.appendChild(iofElement)
309
310 icetrayElement.appendChild(icetrayFilesElement)
311 docElement.appendChild(icetrayElement)
312 return icetrayElement
313
354
356 """
357 Load projects from IceTrayConfig and add the to DOM tree
358 """
359
360 doc = self.doc
361 projects = i3config.GetProjectList()
362
363 for p in projects:
364
365 projElement = doc.createElement( 'project' )
366 if p.GetDependencies():
367 projElement.setAttribute('dependencies',','.join(p.GetDependencies()))
368 icetray.appendChild(projElement)
369
370 nameElement = doc.createElement( 'name' )
371 nameNode = doc.createTextNode( p.GetName() )
372 nameElement.appendChild(nameNode)
373 projElement.appendChild(nameElement)
374
375 versionElement = doc.createElement( 'version' )
376 versionNode = doc.createTextNode( p.GetVersion() )
377 versionElement.appendChild(versionNode)
378 projElement.appendChild(versionElement)
379
381 """
382 Create a value element depending on the type
383 """
384 if ptype.startswith("OMKey"):
385 if default_value:
386 pValueElement = doc.createElement( 'defaultmap' )
387 else:
388 pValueElement = doc.createElement( 'map' )
389 pEntryElement0 = doc.createElement( 'cvalue' )
390 pEntryElement0.appendChild(doc.createTextNode(str(pvalue.stringid)))
391 pEntryElement0.setAttribute('label', 'stringid' )
392 pValueElement.appendChild(pEntryElement0)
393 pEntryElement1 = doc.createElement( 'cvalue' )
394 pEntryElement1.appendChild(doc.createTextNode(str(pvalue.omid)))
395 pEntryElement1.setAttribute('label', 'omid' )
396 pValueElement.appendChild(pEntryElement1)
397 else:
398 if default_value:
399 pValueElement = doc.createElement( 'default' )
400 else:
401 pValueElement = doc.createElement( 'value' )
402 pValueElement.appendChild(doc.createTextNode(str(pvalue.value)))
403 if pvalue.unit:
404 pValueElement.setAttribute('unit', pvalue.unit )
405 return pValueElement
406
407 - def AddServices(self,icetray,i3config,add_descriptions=False):
408 """
409 Load services from IceTrayConfig and add the to DOM tree
410 """
411
412 doc = self.doc
413 services = i3config.GetServices()
414
415 for s in services:
416
417 servElement = doc.createElement( 'service' )
418 servElement.setAttribute('class',s.GetClass())
419 projects = ','.join( [p.GetName() for p in s.GetProjectList()] )
420 projects = projects.strip()
421 if projects:
422 servElement.setAttribute('projects',projects)
423 icetray.appendChild(servElement)
424
425 nameElement = doc.createElement( 'name' )
426 nameNode = doc.createTextNode( s.GetName() )
427 nameElement.appendChild(nameNode)
428 servElement.appendChild(nameElement)
429 if s.GetDescription():
430 servElement.appendChild(doc.createComment(s.GetDescription()))
431
432 paramsElement = doc.createElement( 'parameters' )
433 servElement.appendChild(paramsElement)
434
435 for param in s.GetParameters():
436
437 ptype = param.GetType()
438 pTypeElement = doc.createElement( param.GetType() )
439 paramsElement.appendChild(pTypeElement)
440
441 pNameElement = doc.createElement( 'name' )
442 pNameNode = doc.createTextNode( param.GetName() )
443 pNameElement.appendChild(pNameNode)
444 pTypeElement.appendChild(pNameElement)
445
446 if add_descriptions:
447 pDescriptionElement = doc.createElement( 'description' )
448 pDescriptionNode = doc.createTextNode( param.GetDescription() )
449 pDescriptionElement.appendChild(pDescriptionNode)
450 pTypeElement.appendChild(pDescriptionElement)
451
452
453 if ptype in VectorTypes:
454 for pvalue in param.GetValue():
455 pValueElement = self.mkValueElement(ptype,pvalue,doc)
456 pTypeElement.appendChild(pValueElement)
457 else:
458 pValueElement = self.mkValueElement(ptype,param.GetValue(),doc)
459 pTypeElement.appendChild(pValueElement)
460 if param.GetDefault():
461 if ptype in VectorTypes:
462
463
464
465 pass
466 else:
467 pValueElement = self.mkValueElement(ptype,param.GetDefault(),doc,True)
468 pTypeElement.appendChild(pValueElement)
469
470
471
472 - def AddModules(self,icetray,i3config,add_descriptions=False):
473 """
474 Load modules from IceTrayConfig and add the to DOM tree
475 """
476
477 doc = self.doc
478 modules = i3config.GetModules()
479
480 for m in modules:
481
482 modElement = doc.createElement( 'module' )
483 modElement.setAttribute('class',m.GetClass())
484 projects = ','.join( [p.GetName() for p in m.GetProjectList()] )
485 if projects:
486 modElement.setAttribute('projects',projects)
487 icetray.appendChild(modElement)
488
489 nameElement = doc.createElement( 'name' )
490 nameNode = doc.createTextNode( m.GetName() )
491 nameElement.appendChild(nameNode)
492 modElement.appendChild(nameElement)
493 if m.GetDescription():
494 modElement.appendChild(doc.createComment(m.GetDescription()))
495
496 paramsElement = doc.createElement( 'parameters' )
497 modElement.appendChild(paramsElement)
498
499 for param in m.GetParameters():
500
501 ptype = param.GetType()
502 pTypeElement = doc.createElement( param.GetType() )
503
504 pNameElement = doc.createElement( 'name' )
505 pNameNode = doc.createTextNode( param.GetName() )
506 pNameElement.appendChild(pNameNode)
507 pTypeElement.appendChild(pNameElement)
508 paramsElement.appendChild(pTypeElement)
509
510 if add_descriptions:
511 pDescriptionElement = doc.createElement( 'description' )
512 pDescriptionNode = doc.createTextNode( param.GetDescription() )
513 pDescriptionElement.appendChild(pDescriptionNode)
514 pTypeElement.appendChild(pDescriptionElement)
515
516
517 if ptype in VectorTypes:
518 for pvalue in param.GetValue():
519 pValueElement = self.mkValueElement(ptype,pvalue,doc)
520 pTypeElement.appendChild(pValueElement)
521 else:
522 pValueElement = self.mkValueElement(ptype,param.GetValue(),doc)
523 pTypeElement.appendChild(pValueElement)
524
525 if param.GetDefault():
526 if ptype in VectorTypes:
527
528
529
530 pass
531 else:
532 pValueElement = self.mkValueElement(ptype,param.GetDefault(),doc,True)
533 pTypeElement.appendChild(pValueElement)
534
536 """
537 Load modules from IceTrayConfig and add the to DOM tree
538 """
539
540 doc = self.doc
541 pres = i3config.GetIceProdPres()
542
543 for m in pres:
544
545 modElement = doc.createElement( 'IceProdPre' )
546 modElement.setAttribute('class',m.GetClass())
547 modElement.setAttribute('projects','iceprod.modules')
548 icetray.appendChild(modElement)
549
550 nameElement = doc.createElement( 'name' )
551 nameNode = doc.createTextNode( m.GetName() )
552 nameElement.appendChild(nameNode)
553 modElement.appendChild(nameElement)
554 if m.GetDescription():
555 modElement.appendChild(doc.createComment(m.GetDescription()))
556
557 paramsElement = doc.createElement( 'parameters' )
558 modElement.appendChild(paramsElement)
559
560 for param in m.GetParameters():
561
562 pNameElement = doc.createElement( 'name' )
563 pNameNode = doc.createTextNode( param.GetName() )
564 pNameElement.appendChild(pNameNode)
565 pTypeElement = doc.createElement( param.GetType() )
566 pTypeElement.appendChild(pNameElement)
567 paramsElement.appendChild(pTypeElement)
568 ptype = param.GetType()
569
570 if ptype in VectorTypes:
571 for pvalue in param.GetValue():
572 pValueElement = self.mkValueElement(ptype,pvalue,doc)
573 pTypeElement.appendChild(pValueElement)
574 else:
575 pValueElement = self.mkValueElement(ptype,param.GetValue(),doc)
576 pTypeElement.appendChild(pValueElement)
577
578 if param.GetDefault() != None:
579 if ptype in VectorTypes:
580 for pvalue in param.GetDefault():
581 pValueElement = self.mkValueElement(ptype,pvalue,doc,True)
582 pTypeElement.appendChild(pValueElement)
583 else:
584 pValueElement = self.mkValueElement(ptype,param.GetDefault(),doc,True)
585 pTypeElement.appendChild(pValueElement)
586
587
588 - def AddIceProdPosts(self,icetray,i3config):
589 """
590 Load modules from IceTrayConfig and add the to DOM tree
591 """
592
593 doc = self.doc
594 posts = i3config.GetIceProdPosts()
595
596 for m in posts:
597
598 modElement = doc.createElement( 'IceProdPost' )
599 modElement.setAttribute('class',m.GetClass())
600 modElement.setAttribute('projects','iceprod.modules')
601 icetray.appendChild(modElement)
602
603 nameElement = doc.createElement( 'name' )
604 nameNode = doc.createTextNode( m.GetName() )
605 nameElement.appendChild(nameNode)
606 modElement.appendChild(nameElement)
607 if m.GetDescription():
608 modElement.appendChild(doc.createComment(m.GetDescription()))
609
610 paramsElement = doc.createElement( 'parameters' )
611 modElement.appendChild(paramsElement)
612
613 for param in m.GetParameters():
614
615 pNameElement = doc.createElement( 'name' )
616 pNameNode = doc.createTextNode( param.GetName() )
617 pNameElement.appendChild(pNameNode)
618 pTypeElement = doc.createElement( param.GetType() )
619 pTypeElement.appendChild(pNameElement)
620 paramsElement.appendChild(pTypeElement)
621 ptype = param.GetType()
622
623 if ptype in VectorTypes:
624 for pvalue in param.GetValue():
625 pValueElement = self.mkValueElement(ptype,pvalue,doc)
626 pTypeElement.appendChild(pValueElement)
627 else:
628 pValueElement = self.mkValueElement(ptype,param.GetValue(),doc)
629 pTypeElement.appendChild(pValueElement)
630
631 if param.GetDefault() != None:
632 if ptype in VectorTypes:
633 for pvalue in param.GetDefault():
634 pValueElement = self.mkValueElement(ptype,pvalue,doc,True)
635 pTypeElement.appendChild(pValueElement)
636 else:
637 pValueElement = self.mkValueElement(ptype,param.GetDefault(),doc,True)
638 pTypeElement.appendChild(pValueElement)
639
640
641
643 """
644 Read connections from IceTrayConfig and load them to DOM tree
645 """
646
647 doc = self.doc
648 connections = i3config.GetConnections()
649
650
651
652
653
654 for c in connections:
655
656 inbox = c.GetInbox()
657 outbox = c.GetOutbox()
658
659 connectElement = doc.createElement( 'connection' )
660 icetray.appendChild(connectElement)
661
662 outModuleElement = doc.createElement( 'module' )
663 outModuleNode = doc.createTextNode( outbox.GetModule() )
664 outModuleElement.appendChild(outModuleNode)
665
666 outboxElement = doc.createElement( 'outbox' )
667 outboxNode = doc.createTextNode( outbox.GetBoxName() )
668 outboxElement.appendChild(outboxNode)
669
670 fromElement = doc.createElement( 'from' )
671 fromElement.appendChild(outModuleElement)
672 fromElement.appendChild(outboxElement)
673 connectElement.appendChild(fromElement)
674
675 inModuleElement = doc.createElement( 'module' )
676 inModuleNode = doc.createTextNode( inbox.GetModule() )
677 inModuleElement.appendChild(inModuleNode)
678
679 inboxElement = doc.createElement( 'inbox' )
680 inboxNode = doc.createTextNode( inbox.GetBoxName() )
681 inboxElement.appendChild(inboxNode)
682
683 toElement = doc.createElement( 'to' )
684 toElement.appendChild(inModuleElement)
685 toElement.appendChild(inboxElement)
686 connectElement.appendChild(toElement)
687
689 return self.doc
690
691 - def toXML(self, fname=None):
692 return self.doc.toxml()
693
694
696 """
697 Write DOM tree as human readable xml to a file
698 """
699 if fname:
700 file_object = open(fname, "w")
701 ipxml.PrettyPrint(self.doc, file_object)
702 file_object.close()
703 else:
704 ipxml.PrettyPrint(self.doc)
705