iceprod.core.dataclasses

1 #!/usr/bin/env python 2 # 3 """ 4 A set of classes that describe configuration elements in IceTray. 5 6 copyright (c) 2005 the icecube collaboration 7 8 @version: $Revision: $ 9 @date: $Date: $ 10 @author: T. McCauley <tpmccauley@lbl.gov> 11 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 12 13 @todo: Add support for more project content (the "optional" option) 14 @todo: Be a bit more clever about handling of type tags 15 @todo: Units 16 @todo: Commas and whitespace handling in projects attribute 17 @todo: What to do if libraries are included more than once? 18 @todo: Add some overall elegance 19 @todo: Fix the "inbox" problem 20 """ 21 tver='$Revision: 1.4 $' 22 __version__= 2 23 24 import re,types 25 import logging 26 from odict import OrderedDict 27 from iceprod.core.constants import * 28 29 UnmetDependencyException = "Error: Un-met dependency:" 30 UndefinedTypeException = "Error: invalid type:" 31 32 SupportedTypes = [ 33 'bool', 'int', 'long', 'float', 34 'double', 'string','NaT','OMKey'] 35 36 VectorTypes = [ 37 'boolv', 'intv', 'longv', 'floatv', 38 'doublev', 'stringv','OMKeyv'] 39 40 SimulationCategories = [ 41 'CORSIKA-ice-top', 42 'CORSIKA-in-ice', 43 'neutrino-generator', 44 'JULIeT', 'simple-generator', 45 'WIMP','SuperNova', 46 'TWRFilter', 'OfflineFilter', 47 'L1', 'L2', 'L3' 48 ] 49 50 DatasetTypes = [ 51 'TEST', 52 'BENCHMARK', 53 'PHYSICS', 54 'TEMPLATE', 55 ] 56 57

58 -class pyOMKey:

59 - def __init__(self,stringid=0,omid=0):

60 self.stringid= stringid 61 self.omid= omid

62

63 - def SetOMId(self,omid):

64 self.omid= omid

65 - def GetOMId(self):

66 return self.omid

67 - def SetStringId(self,stringid):

68 self.stringid= stringid

69 - def GetStringId(self):

70 return self.stringid

71

72 -class Value:

73 - def __init__(self,value,unit=None,format=None):

74 self.value = value 75 self.unit = unit 76 self.format = format # Applies only to string types

77 - def SetValue(self,value):

78 self.value = value

79 - def GetValue(self):

80 return self.value

81 - def SetUnit(self,unit):

82 self.unit = unit

83 - def GetUnit(self):

84 return self.unit

85

86 -class Parameter:

87 - def __init__(self,ptype='NaT',pname='',value=''):

88 self.ptype = ptype 89 self.pname = pname 90 self.pvalue = value 91 self.description = 'No description' 92 self.id = -1 93 self.default = None

94

95 - def SetId(self, id):

96 self.id = id

97 - def GetId(self):

98 return self.id

99

100 - def SetDescription(self, description):

101 self.description = description

102

103 - def GetDescription(self):

104 return self.description

105

106 - def SetType(self, ptype):

107 if ptype in SupportedTypes or ptype in VectorTypes: 108 self.ptype = ptype 109 else: 110 raise UndefinedTypeException, ptype

111

112 - def GetType(self):

113 return self.ptype

114

115 - def SetName(self, pname):

116 self.pname = pname

117 - def GetName(self):

118 return self.pname

119

120 - def SetValue(self, pvalue):

121 self.pvalue = pvalue

122 - def GetValue(self):

123 return self.pvalue

124

125 - def SetUnit(self, punit):

126 self.punit = punit

127 - def GetUnit(self):

128 return self.punit

129

130 - def SetFormat(self, pformat):

131 self.pformat = pformat

132 - def GetFormat(self):

133 return self.pformat

134

135 - def SetDefault(self,value):

136 self.default = value

137 - def GetDefault(self):

138 return self.default

139

140 -class TaskDefinition:

141 - def __init__(self, name, id=None):

142 self.SetName(name) 143 self.trays = OrderedDict() 144 self.parents = [] 145 self.children = [] 146 self.requirements = '' 147 self.parallel = False 148 self.id = id 149 self.photonics = False

150

151 - def SetId(self, id):

152 self.id = id

153 - def GetId(self):

154 return self.id

155

156 - def AddTray(self, idx, iters, externs=False):

157 self.trays[int(idx)] = TaskTrayDefinition(idx, iters, externs)

158 - def GetTrays(self):

159 return self.trays

160 - def GetTray(self, idx):

161 return self.GetTrays()[int(idx)]

162

163 - def SetName(self, name):

164 self.name = name

165 - def GetName(self):

166 return self.name

167

168 - def SetRequirements(self, reqs):

169 self.requirements = reqs.strip().replace('\n', ' ')

170 - def GetRequirements(self):

171 return self.requirements

172

173 - def AddParent(self, parent):

174 self.parents.append(parent)

175 - def GetParents(self):

176 return self.parents

177

178 - def AddChild(self, child):

179 self.children.append(child)

180 - def GetChildren(self):

181 return self.children

182

183 - def SetParallelExecution(self, parallel):

184 self.parallel = bool(parallel)

185 - def ParallelExecutionEnabled(self):

186 return self.parallel

187

188 - def SetUsesPhotonics(self, photonics):

189 self.photonics = bool(photonics)

190 - def UsesPhotonics(self):

191 return self.photonics

192

193 - def HasTray(self, idx):

194 return self.GetTrays().has_key(idx)

195

196 - def HasIter(self, tray, iter):

197 return self.HasTray(tray) and self.GetTray(tray).HasIteration(iter)

198

199 - def HasInput(self):

200 return not self.IsCleanup() and self.GetParents()

201 - def HasOutput(self):

202 children = self.GetChildren() 203 return children and not (len(children) == 1 and TASK_TRASHCAN_NAME in children)

204

205 - def IsCleanup(self):

206 return self.GetName() == TASK_TRASHCAN_NAME

207

208 - def __str__(self):

209 return "%s: %s" % (self.GetName(),self.GetTrays().items())

210

211 -class TaskTrayDefinition:

212 - def __init__(self, idx, iters, externs=False):

213 self.iters = [] 214 self.externs = False 215 self.idx = 0 216 217 self.SetIndex(idx) 218 self.SetRunExterns(externs) 219 self.SetIters(iters)

220

221 - def SetIndex(self, idx):

222 self.idx = int(idx)

223 - def GetIndex(self):

224 return self.idx

225

226 - def SetIters(self, iters):

227 if not self.RunExternsEnabled(): 228 real_iters = [] 229 else: 230 real_iters = [TASK_EXTERN_ITER] 231 iters = str(iters).split(",") 232 for iter in iters: 233 if iter.count("-") > 0 and not iter == str(TASK_EXTERN_ITER): 234 endpoints = map(int, iter.split("-")) 235 for val in range(endpoints[0], endpoints[1]+1): 236 if val not in real_iters: 237 real_iters.append(val) 238 else: 239 iter = int(iter) 240 if iter not in real_iters: 241 real_iters.append(iter) 242 self.iters = map(int, real_iters)

243 - def GetIters(self):

244 return self.iters

245

246 - def HasIter(self, iter):

247 return self.GetIters().has_key(iter)

248

249 - def SetRunExterns(self, externs):

250 self.externs = bool(externs)

251 - def RunExternsEnabled(self):

252 return self.externs or TASK_EXTERN_ITER in self.GetIters()

253

254 - def __str__(self):

255 return "tray %d, iters %s" % (self.GetIndex(),self.GetIters())

256

257 -class Steering:

258 - def __init__(self):

259 self.version = __version__ 260 self.parent_id = 0 261 self.trays = [] 262 self.difplus = None 263 self.parameters = OrderedDict() # list of Parameter objects 264 self.batchopts = OrderedDict() # list of batch options 265 self.sysopts = OrderedDict() # list of system-specific options 266 self.dependencies = OrderedDict() # list of file dendencies at runtime 267 self.description = "Run was not given a description" 268 self.category = None 269 self.externs = [] 270 self.extras = OrderedDict() 271 self.filter = None 272 self.datasettype = "PHYSICS" 273 self.task_defs = OrderedDict() 274 275 maxjobs = Parameter() 276 maxjobs.SetName("MAXJOBS") 277 maxjobs.SetType("int") 278 maxjobs.SetValue("1") 279 self.AddParameter(maxjobs)

280

281 - def AddDIFPlus(self,difplus):

282 self.difplus = difplus

283

284 - def GetDIFPlus(self):

285 return self.difplus

286

287 - def HasDIFPlus(self):

288 return self.difplus

289

290 - def SetVersion(self,version):

291 self.version = version

292 - def GetVersion(self):

293 return self.version

294

295 - def SetParentId(self,id):

296 self.parent_id = id

297 - def GetParentId(self):

298 return self.parent_id

299

300 - def CheckConfig(self):

301 for i in range(len(self.trays)): 302 check,str = self.trays[i].CheckConfig() 303 if not check: 304 return check, "tray %d: %s" % (i,str) 305 return True,""

306

307 - def SetOF(self,filter):

308 self.filter = filter

309

310 - def GetOF(self):

311 return self.filter

312

313 - def SetDescription(self, desc):

314 self.description = desc

315

316 - def GetDescription(self):

317 return self.description

318

319 - def SetDatasetType(self, dtype):

320 self.datasettype = dtype

321

322 - def GetDatasetType(self):

323 return self.datasettype

324

325 - def SetCategory(self, cat):

326 self.category = cat

327

328 - def GetCategory(self):

329 return self.category

330

331 - def AddParameter(self, *args):

332 """ 333 usage: 334 AddParameter(p) 335 where p is a Parameter object as of iceprod 1.2: you can also call 336 337 AddParameter("type","name","value") 338 """ 339 if len(args) < 1: raise Exception, "AddParameter requires at least 1 argument, 0 given" 340 if isinstance(args[0],Parameter): 341 parameter = args[0] 342 else: 343 parameter = Parameter(*args) 344 self.parameters[parameter.GetName()] = parameter

345

346 - def RemoveParameter(self, parametername):

347 if self.parameters.has_key(parametername): 348 del self.parameters[parametername]

349

350 - def GetParameters(self):

351 return self.parameters.values()

352

353 - def GetParameter(self,pname):

354 if self.parameters.has_key(pname): 355 return self.parameters[pname]

356

357 - def AddDependency(self, depend):

358 self.dependencies[depend] = depend

359

360 - def GetDependencies(self):

361 return self.dependencies.values()

362

363 - def GetDependency(self,dname):

364 if self.dependencies.has_key(dname): 365 return self.dependencies[dname]

366

367 - def RemoveDependency(self,dname):

368 if self.dependencies.has_key(dname): 369 del self.dependencies[dname]

370

371 - def AddBatchOpt(self, opt):

372 self.batchopts[(opt.GetType(),opt.GetName())] = opt

373

374 - def GetBatchOpts(self):

375 return self.batchopts.values()

376

377 - def GetBatchOpt(self,bname):

378 if self.batchopts.has_key(bname): 379 return self.batchopts[bname]

380

381 - def RemoveBatchOpt(self, bname):

382 if self.batchopts.has_key(bname): 383 del self.batchopts[bname]

384

385 - def AddSysOpt(self, opt):

386 self.sysopts[opt.GetName()] = opt

387

388 - def GetSysOpts(self):

389 return self.sysopts.values()

390

391 - def GetSysOpt(self,bname):

392 if self.sysopts.has_key(bname): 393 return self.sysopts[bname]

394

395 - def RemoveSysOpt(self, bname):

396 if self.sysopts.has_key(bname): 397 del self.sysopts[bname]

398

399 - def AddExtern(self, extern):

400 self.externs.append(extern)

401

402 - def GetExterns(self):

403 return self.externs

404

405 - def AddTaskDefinition(self, task_def):

406 self.task_defs[task_def.GetName()] = task_def

407

408 - def GetTaskDefinition(self, name):

409 return self.task_defs[name]

410

411 - def GetTaskDefinitions(self):

412 return self.task_defs

413

414 - def get_param_dict(self):

415 d = {} 416 for param in self.parameters.values(): 417 d[param.GetName()] = param.GetValue() 418 return d

419

420 - def get_sysopt_dict(self):

421 d = {} 422 for param in self.sysopts.values(): 423 d[param.GetName()] = param.GetValue() 424 return d

425

426 - def AddTray(self, config):

427 self.trays.append(config)

428

429 - def GetTrays(self):

430 return self.trays

431

432 - def GetTray(self,t):

433 return self.trays[t]

434

435 - def RemoveTray(self,t):

436 tray = self.trays[t] 437 del self.trays[t] 438 return tray

439

440 - def HasExtra(self, name):

441 return self.extras.has_key(name)

442

443 - def AddExtra(self, name, extra):

444 self.extras[name] = extra

445

446 - def GetExtra(self,name):

447 if self.HasExtra(name): 448 return self.extras[name]

449

450 - def ClearExtras(self):

451 self.extras = OrderedDict()

452

453 - def GetExtras(self):

454 return self.extras

455

456 - def GetExtraList(self):

457 return self.extras.values()

458

459 - def RemoveExtra(self,name):

460 if self.HasExtra(name): 461 del self.extras[name]

462

463 -class BatchOpt(Parameter):

464

465 - def __init__(self):

466 Parameter.__init__(self) 467 self.ptype = 'condor'

468

469 - def SetType(self,ptype):

470 self.ptype = ptype

471

472 -class SysOpt(Parameter):

473

474 - def __init__(self,name='',value=None,type='string'):

475 Parameter.__init__(self) 476 self.pname = name 477 self.pvalue = Value(value) 478 self.ptype = type

479

480 - def GetValue(self):

481 return self.pvalue.GetValue()

482 483 484

485 -class Module:

486 - def __init__(self):

487 self.name = '' 488 self.description = '' 489 self.parameters = OrderedDict() # list of Parameter objects 490 self.mclass = '' 491 self.id = -1 492 self.projects = OrderedDict() 493 self.inboxes = [] 494 self.outboxes = []

495 496

497 - def CheckParams(self):

498 for p in self.GetParameters(): 499 if p.GetType() == 'NaT': 500 return False, "%s (%s) : %s %s" % ( 501 self.GetClass(),self.GetName(), 502 p.GetType(),p.GetName()) 503 return True,""

504

505 - def SetName(self, name):

506 self.name = name

507 - def GetName(self):

508 return self.name

509

510 - def SetDescription(self, description):

511 self.description = description

512

513 - def GetDescription(self):

514 return self.description

515

516 - def SetClass(self, mclass):

517 self.mclass = mclass

518

519 - def GetClass(self):

520 return self.mclass

521

522 - def AddParameter(self, parameter):

523 if not self.parameters.has_key(parameter.GetName()): 524 self.parameters[parameter.GetName()] = parameter

525

526 - def RemoveParameter(self, parametername):

527 if self.parameters.has_key(parametername): 528 del self.parameters[parametername]

529

530 - def GetParameters(self):

531 return [self.parameters[k] for k in self.parameters.keys()]

532

533 - def GetParameter(self,pname):

534 if self.parameters.has_key(pname): 535 return self.parameters[pname]

536

537 - def AddProject(self, name, project):

538 if not self.projects.has_key(name): 539 self.projects[name] = project

540

541 - def RemoveProject(self, projectname):

542 if self.projects.has_key(projectname): 543 del self.projects[projectname]

544

545 - def ClearProjects(self):

546 self.projects = OrderedDict()

547

548 - def GetProjects(self):

549 return self.projects

550

551 - def GetProject(self,pname):

552 if self.projects.has_key(projectname): 553 return self.projects[pname]

554

555 - def GetProjectList(self):

556 return [self.projects[k] for k in self.projects.keys()]

557

558 - def SetId(self, id):

559 self.id = id

560

561 - def GetId(self):

562 return self.id

563

564 -class Service(Module):

565 - def __init__(self):

566 Module.__init__(self)

567

568 -class IceProdPre(Module):

569 """ 570 This class provides an interface for preprocessing files in iceprod 571 """

572 - def __init__(self):

573 Module.__init__(self)

574

575 -class IceProdPost(IceProdPre):

576 """ 577 This class provides an interface for postprocessing files in iceprod 578 """

579 - def __init__(self):

580 IceProdPre.__init__(self)

581

582 -class Project:

583 - def __init__(self):

584 self.name = '' 585 self.version = '' 586 self.revision = 0 587 self.pid = -1 588 self.path = '.' 589 self.type = 'c++' 590 self.dependencies = {}

591

592 - def SetName(self, name):

593 self.name = name

594 - def GetName(self):

595 return self.name

596

597 - def SetVersion(self, version):

598 self.version = version

599

600 - def GetVersion(self):

601 return self.version

602

603 - def GetVersionTxt(self,fmt="%s.%s.%s"):

604 tok = self.version.split('.')[-1].replace('V','').split('-') 605 versionStr = fmt % ( tok[0], tok[1], tok[2]) 606 return versionStr

607

608 - def SetRevision(self, rev):

609 self.revision = rev

610 - def GetRevision(self):

611 return self.revision

612

613 - def SetType(self, type):

614 self.type = type

615 - def GetType(self):

616 return self.type

617 618

619 - def SetHost(self, version):

620 self.host = host

621 - def GetHost(self):

622 return self.host

623

624 - def SetDateTime(self, dateTime):

625 self.dateTime = dateTime

626 - def GetDateTime(self):

627 return self.dateTime

628

629 - def SetOS(self, os):

630 self.os = os

631 - def GetOS(self):

632 return self.os

633

634 - def SetCompiler(self, compiler):

635 self.compiler = compiler

636

637 - def GetCompiler(self):

638 return self.compiler

639

640 - def SetId(self, pid):

641 self.pid = pid

642

643 - def GetId(self):

644 return self.pid

645

646 - def SetPath(self, path):

647 self.path = path

648

649 - def GetPath(self):

650 return self.path

651

652 - def AddDependency(self, depend):

653 if type(depend) in types.StringTypes: 654 self.dependencies[depend] = depend 655 else: 656 self.dependencies[depend.GetName()]= depend

657

658 - def GetDependencies(self):

659 return self.dependencies.values()

660

661 - def GetDependencyKeys(self):

662 return self.dependencies.keys()

663

664 - def ClearDependencies(self):

665 self.dependencies = {}

666

667 -class Container(Project):

668 - def __init__(self):

669 Project.__init__(self) 670 self.modules = OrderedDict() 671 self.services = OrderedDict() 672 self.iceprodpre = OrderedDict() 673 self.iceprodpost = OrderedDict()

674

675 - def AddIceProdPre(self, pre):

676 sname = pre.GetName() 677 self.iceprodpre[sname] = pre

678

679 - def GetIceProdPres(self):

680 return self.iceprodpre.values()

681

682 - def AddIceProdPost(self, post):

683 sname = post.GetName() 684 self.iceprodpost[sname] = post

685

686 - def GetIceProdPosts(self):

687 return self.iceprodpost.values()

688

689 - def HasModule(self, name):

690 return self.modules.has_key(name)

691 - def AddModule(self, module):

692 self.modules[module.GetName()] = module

693 - def GetModules(self):

694 return self.modules.values()

695

696 - def RemoveModule(self, modulename):

697 return self.modules.pop(modulename)

698

699 - def HasService(self, name):

700 return self.services.has_key(name)

701 - def AddService(self, service):

702 self.services[service.GetName()] = service

703 - def GetServices(self):

704 return self.services.values()

705 - def RemoveService(self, name):

706 return self.services.pop(name)

707

708 -class MetaProject(Project):

709 """ 710 Class inherits from Project 711 Represents an svn metaproject 712 """

713 - def __init__(self):

714 Project.__init__(self) 715 self.projects = OrderedDict()

716

717 - def HasProject(self, pname):

718 return self.projects.has_key(pname)

719

720 - def AddProject(self, name, project):

721 self.projects[name] = project

722

723 - def GetProject(self,pname):

724 if self.projects.has_key(pname): 725 return self.projects[pname]

726

727 - def ClearProjects(self):

728 self.projects = OrderedDict()

729

730 - def GetProjects(self):

731 return self.projects

732

733 - def GetProjectList(self):

734 return self.projects.values()

735

736 - def RemoveProject(self, projectname):

737 if self.projects.has_key(projectname): 738 del self.projects[projectname]

739

740 - def AddProjectAttr(self, name, project):

741 if not self.ice_project_attr.has_key(name): 742 self.ice_project_attr[name] = project

743 744

745 -class Connection:

746 - def __init__(self):

747 self.outbox = None 748 self.inbox = Box('')

749

750 - def From(self, from_module, outbox):

751 self.outbox = Box(from_module,outbox)

752

753 - def To(self, to_module, inbox):

754 self.inbox = Box(to_module,inbox)

755

756 - def To(self, to_module, inbox=None):

757 if inbox: 758 self.inbox = Box(to_module,inbox) 759 else: 760 self.inbox = Box(to_module)

761

762 - def GetInbox(self):

763 return self.inbox

764

765 - def GetOutbox(self):

766 return self.outbox

767

768 -class Box:

769 - def __init__(self,module,boxname='default'):

770 self.module = module 771 self.boxname = boxname

772

773 - def GetModule(self):

774 return self.module

775

776 - def SetModule(self,module):

777 self.module = module

778

779 - def GetBoxName(self):

780 return self.boxname

781

782 - def SetBoxName(self,name):

783 self.boxname = name

784

785 -class IceTrayConfig:

786 - def __init__(self):

787 self.steering = Steering() 788 self.name = None 789 self.events = '0' 790 self.iterations = '1' 791 self.iceprodpre = OrderedDict() 792 self.modules = [] 793 self.services = OrderedDict() 794 self.projects = OrderedDict() 795 self.metaprojects = OrderedDict() 796 self.ice_project_attr = OrderedDict() 797 self.connections = [] 798 self.iceprodpost = OrderedDict() 799 self.input_files = [] 800 self.output_files = []

801

802 - def SetName(self,name):

803 self.name = name

804

805 - def GetName(self):

806 return self.name

807 808

809 - def CheckConfig(self):

810 for m in self.GetModules(): 811 check,str = m.CheckParams() 812 if not check: 813 return check,"in module \'%s\' needs to be configured or removed" %str 814 for s in self.GetServices(): 815 check,str = s.CheckParams() 816 if not check: 817 return check,"service %s needs to be configured or removed" %str 818 return True,""

819

820 - def SetEvents(self,events):

821 self.events = events

822 - def GetEvents(self):

823 return self.events

824

825 - def SetIterations(self,iter):

826 self.iterations = iter

827 - def GetIterations(self):

828 return self.iterations

829

830 - def GetParameter(self,pname):

831 if self.parameters.has_key(pname): 832 return self.parameters[pname] 833 else: 834 return None

835

836 - def SetSteering(self, steering):

837 self.steering=steering

838

839 - def GetSteering(self):

840 return self.steering

841

842 - def HasModule(self, mname):

843 for m in self.modules: 844 if m.GetName() == mname: 845 return True 846 return False

847

848 - def AddModule(self, module):

849 mname = module.GetName() 850 for p in module.GetProjectList(): 851 if not self.HasProject(p.GetName()): 852 print ",".join(map(Project.GetName,self.GetProjectList())) 853 raise UnmetDependencyException, \ 854 "module %s requires project %s" % (mname,p.GetName()) 855 break # we only care about the first project 856 857 for i in range(len(self.modules)): 858 if self.modules[i].GetName() == module.GetName(): 859 self.modules.pop(i) 860 self.modules.insert(i,module) 861 return 862 self.modules.append(module)

863

864 - def InsertModule(self, index,module):

865 for p in module.GetProjectList(): 866 if self.GetProject(p.GetName()) == None: 867 raise UnmetDependencyException, \ 868 "module %s requires project %s" % (mname,p.GetName()) 869 if index < 0: 870 self.modules.insert(0,module) 871 elif index >= len(self.modules): 872 self.modules.append(module) 873 else: 874 self.modules.insert(index,module)

875

876 - def RemoveModule(self, modulename):

877 index = 0 878 for m in self.modules: 879 if m.GetName() == modulename: 880 return self.modules.pop(index) 881 index += 1 882 return False

883

884 - def PopModule(self, index):

885 return self.modules.pop(index)

886

887 - def GetModules(self):

888 return self.modules

889

890 - def GetModule(self,modulename):

891 for m in self.modules: 892 if m.GetName() == modulename: 893 return m 894 return False

895

896 - def ClearModules(self):

897 self.modules = []

898

899 - def HasMetaProject(self, mpname):

900 return self.metaprojects.has_key(mpname)

901

902 - def AddMetaProject(self, name, metaproject):

903 self.metaprojects[name] = metaproject

904

905 - def GetMetaProject(self,mpname):

906 if self.metaprojects.has_key(mpname): 907 return self.metaprojects[mpname]

908

909 - def GetMetaProjectById(self,mpid):

910 for mp in self.metaprojects.values(): 911 if mp.Get() == mpid: 912 return mp 913 mp = filter((lambda mp: mp.GetId() == mpid), self.metaprojects.values()) 914 if len(mp) > 0: 915 return mp[0]

916

917 - def ClearMetaProjects(self):

918 self.metaprojects = OrderedDict()

919

920 - def GetMetaProjects(self):

921 return self.metaprojects

922

923 - def GetMetaProjectList(self):

924 return [self.metaprojects[k] for k in self.metaprojects.keys()]

925

926 - def RemoveMetaProject(self, metaprojectname):

927 if self.metaprojects.has_key(metaprojectname): 928 del self.metaprojects[metaprojectname]

929 930

931 - def HasProject(self, pname):

932 return self.GetProject(pname)

933 934

935 - def AddProject(self, name, project):

936 try: 937 metaproject = self.GetMetaProjectById(project.GetId()) 938 metaproject.AddProject(name,project) 939 except: 940 self.projects[name] = project

941

942 - def GetProject(self,pname):

943 for metaproject in self.GetMetaProjectList(): 944 if metaproject.HasProject(pname): 945 return metaproject.GetProject(pname) 946 947 # If not found in metaprojects look at top level 948 if self.projects.has_key(pname): 949 return self.projects[pname]

950 951

952 - def ClearProjects(self):

953 self.projects = OrderedDict()

954

955 - def GetProjects(self):

956 return self.projects

957

958 - def GetProjectList(self):

959 return self.projects.values()

960

961 - def RemoveProject(self, projectname):

962 if self.projects.has_key(projectname): 963 del self.projects[projectname]

964

965 - def AddProjectAttr(self, name, project):

966 if not self.ice_project_attr.has_key(name): 967 self.ice_project_attr[name] = project

968

969 - def GetProjectAttr(self):

970 return self.ice_project_attr

971

972 - def GetProjectAttrList(self):

973 return self.ice_project_attr.values()

974

975 - def RemoveProjectAttr(self, projectname):

976 if self.ice_project_attr.has_key(projectname): 977 del self.ice_project_attr[projectname]

978

979 - def HasService(self, sname):

980 return self.services.has_key(sname)

981

982 - def AddService(self, service):

983 sname = service.GetName() 984 if self.services.has_key(sname): 985 return 986 for p in service.GetProjectList(): 987 if not self.HasProject( p.GetName() ): 988 plist = [] 989 for metaproject in self.GetMetaProjectList(): 990 plist += map(Project.GetName,metaproject.GetProjectList()) 991 plist = ",".join(plist) 992 raise UnmetDependencyException, \ 993 "service %s requires project %s : %s " % \ 994 (sname,p.GetName(),plist) 995 continue # we only care about the first project 996 self.services[sname] = service

997

998 - def InsertService(self, index,service):

999 sname = service.GetName() 1000 for p in service.GetProjectList(): 1001 if self.GetProject(p.GetName()) == None: 1002 raise UnmetDependencyException, \ 1003 "service %s requires project %s" % (service.GetClass(),p.GetName()) 1004 if index < 0: 1005 self.services.insert(0,sname,service) 1006 elif index >= len(self.modules): 1007 self.services[sname] = service 1008 else: 1009 self.services.insert(index,sname,service)

1010

1011 - def RemoveService(self, servicename):

1012 return self.services.pop(servicename)

1013

1014 - def GetServices(self):

1015 return self.services.values()

1016

1017 - def GetService(self,s):

1018 if self.services.has_key(s): 1019 return self.services[s]

1020

1021 - def ClearServices(self):

1022 self.services = OrderedDict()

1023 1024 #####

1025 - def HasIceProdPre(self, sname):

1026 return self.iceprodpre.has_key(sname)

1027

1028 - def AddIceProdPre(self, pre):

1029 sname = pre.GetName() 1030 self.iceprodpre[sname] = pre

1031

1032 - def InsertIceProdPre(self, index,pre):

1033 sname = pre.GetName() 1034 if index < 0: 1035 self.iceprodpre.insert(0,sname,pre) 1036 elif index >= len(self.modules): 1037 self.iceprodpre[sname] = pre 1038 else: 1039 self.iceprodpre.insert(index,sname,pre)

1040

1041 - def RemoveIceProdPre(self, prename):

1042 return self.iceprodpre.pop(prename)

1043

1044 - def GetIceProdPres(self):

1045 return self.iceprodpre.values()

1046

1047 - def GetIceProdPre(self,s):

1048 if self.iceprodpre.has_key(s): 1049 return self.iceprodpre[s]

1050

1051 - def ClearIceProdPre(self):

1052 self.iceprodpre = OrderedDict()

1053 ###

1054 - def HasIceProdPost(self, sname):

1055 return self.iceprodpost.has_key(sname)

1056

1057 - def AddIceProdPost(self, post):

1058 sname = post.GetName() 1059 self.iceprodpost[sname] = post

1060

1061 - def InsertIceProdPost(self, index,post):

1062 sname = post.GetName() 1063 if index < 0: 1064 self.iceprodpost.insert(0,sname,post) 1065 elif index >= len(self.modules): 1066 self.iceprodpost[sname] = post 1067 else: 1068 self.iceprodpost.insert(index,sname,post)

1069

1070 - def RemoveIceProdPost(self, postname):

1071 return self.iceprodpost.pop(postname)

1072

1073 - def GetIceProdPosts(self):

1074 return self.iceprodpost.values()

1075

1076 - def GetIceProdPost(self,s):

1077 if self.iceprodpost.has_key(s): 1078 return self.iceprodpost[s]

1079

1080 - def ClearIceProdPosts(self):

1081 self.iceprodpost = OrderedDict()

1082 ##### 1083

1084 - def AddConnection(self, connection):

1085 self.connections.append(connection)

1086

1087 - def InsertConnection(self,pos, connection):

1088 self.connections.insert(pos,connection)

1089

1090 - def GetConnections(self):

1091 return self.connections

1092

1093 - def RemoveConnection(self,index):

1094 con = self.connections[index] 1095 del self.connections[index] 1096 return con

1097

1098 - def ClearConnections(self):

1099 self.connections = []

1100

1101 - def AddInputFile(self,file):

1102 self.input_files.append(file)

1103 - def GetInputFiles(self):

1104 return self.input_files

1105 - def AddOutputFile(self,file):

1106 self.output_files.append(file)

1107 - def GetOutputFiles(self):

1108 return self.output_files

1109

1110 -class IceTrayFile:

1111 - def __init__(self, name, photonics=False):

1112 self.name = name 1113 self.photonics = bool(photonics)

1114 - def GetName(self):

1115 return self.name

1116 - def IsPhotonicsTable(self):

1117 return self.photonics

1118

1119 -class Block(IceTrayConfig):

1120 """ 1121 Container of externals and IceTrays 1122 """

1123 - def __init__(self):

1124 IceTrayConfig.__init__(self) 1125 self.trays = [] 1126 self.externs = []

1127

1128 - def AddTray(self, config):

1129 self.trays.append(config)

1130

1131 - def GetTrays(self):

1132 return self.trays

1133

1134 - def GetTray(self,t):

1135 return self.trays[t]

1136

1137 - def RemoveTray(self,t):

1138 tray = self.trays[t] 1139 del self.trays[t] 1140 return tray

1141

1142 - def AddExtern(self, extern):

1143 self.externs.append(extern)

1144

1145 - def GetExterns(self):

1146 return self.externs

1147 1148

1149 -class Dependency:

1150 - def __init__(self,path=None,format=None):

1151 self.path = path 1152 self.format = None

1153

1154 - def SetPath(self,path):

1155 self.path = path

1156

1157 - def GetPath(self,path):

1158 return self.path

1159

1160 - def SetFormat(self,format):

1161 self.format = format

1162

1163 - def GetFormat(self):

1164 return self.format

1165

1166 -class ExternSteering:

1167 - def __init__(self):

1168 self.name = None 1169 self.text = None

1170

1171 - def SetName(self,name):

1172 self.name = name

1173 - def GetName(self):

1174 return self.name

1175

1176 - def SetText(self,text):

1177 self.text = text

1178 - def GetText(self):

1179 return self.text

1180

1181 - def Write(self):

1182 fd = open(self.name,'w') 1183 print >> fd,self.text 1184 fd.close()

1185 1186

1187 -class Extern:

1188 - def __init__(self):

1189 self.name = None 1190 self.version = None 1191 self.description = '' 1192 self.arguments = '' 1193 self.infile = None 1194 self.pformat = "%(name)s %(value)s %(description)s" 1195 self.outfile = None 1196 self.errfile = None 1197 self.command = None 1198 self.parameters = OrderedDict() # list of Parameter objects 1199 self.steering = []

1200

1201 - def SetName(self, name):

1202 self.name = name

1203 - def GetName(self):

1204 return self.name

1205

1206 - def SetVersion(self, version):

1207 self.version = version

1208 - def GetVersion(self):

1209 return self.version

1210

1211 - def SetDescription(self, description):

1212 self.description = description

1213 - def GetDescription(self):

1214 return self.description

1215

1216 - def SetArgs(self, args):

1217 self.arguments= args

1218 - def GetArgs(self):

1219 return self.arguments

1220

1221 - def AddParameter(self, parameter):

1222 self.parameters[parameter.GetName()] = parameter

1223

1224 - def RemoveParameter(self, parametername):

1225 if self.parameters.has_key(parametername): 1226 del self.parameters[parametername]

1227

1228 - def GetParameters(self):

1229 return self.parameters.values()

1230

1231 - def SetInputFileFormat(self, formatstring):

1232 self.pformat = formatstring

1233

1234 - def GetInputFileFormat(self):

1235 return self.pformat

1236

1237 - def WriteInputFile(self, filename):

1238 fd = open(filename,'w') 1239 for p in self.parameters.values(): 1240 pd = { 1241 "type":p.GetType(), 1242 "name":p.GetName(), 1243 "value":p.GetValue(), 1244 "description":p.GetDescription() 1245 } 1246 print >> fd,self.pformat % pd 1247 fd.close()

1248 1249

1250 - def GetParameter(self,pname):

1251 if self.parameters.has_key(pname): 1252 return self.parameters[pname]

1253

1254 - def SetOutFile(self, filename):

1255 self.outfile = filename

1256 - def GetOutFile(self):

1257 return self.outfile

1258

1259 - def SetErrFile(self, filename):

1260 self.errfile = filename

1261 - def GetErrFile(self):

1262 return self.errfile

1263

1264 - def SetInFile(self, filename):

1265 self.infile = filename

1266 - def GetInFile(self):

1267 return self.infile

1268

1269 - def SetExec(self, command):

1270 self.command = command

1271 - def GetExec(self):

1272 return self.command

1273

1274 - def AddSteering(self, steering):

1275 self.steering.append(steering)

1276 - def GetSteering(self):

1277 return self.steering

1278

1279 -class OfflineFilter:

1280 - def __init__(self, prefix='/data/sim', search=True, filedict={}, filter=''):

1281 self.prefix = prefix 1282 self.search = search 1283 self.filedict = filedict 1284 self.paths = [] 1285 self.tokens = [] 1286 self.filter = filter

1287

1288 - def AddPath(self,path):

1289 self.paths.append(path)

1290

1291 - def GetPaths(self):

1292 return self.paths

1293

1294 - def SetTokens(self,tok):

1295 self.tokens = tok

1296

1297 - def GetTokens(self,prefix):

1298 return self.tokens

1299

1300 - def SetPrefix(self,prefix):

1301 self.prefix = prefix

1302

1303 - def GetPrefix(self):

1304 return self.prefix

1305

1306 - def SetFileDict(self,filedict):

1307 self.filedict = filedict

1308

1309 - def GetFileDict(self):

1310 return self.filedict

1311

1312 - def Search(self):

1313 """ 1314 Get list of files in rootdir and add them to database dictonary table 1315 """ 1316 import re,os, os.path 1317 from os.path import join 1318 filecount = 0 1319 for path in self.paths: 1320 if path.regex: cregex = re.compile(path.regex) 1321 for p, dirs, files in os.walk(join(self.prefix,path.path)): 1322 for file in files: 1323 filepath = os.path.join(p,file) 1324 match = cregex.match(filepath) 1325 if match: 1326 year = match.group(0) 1327 day = match.group(1) 1328 run = match.group(2) 1329 truncated_path = p.replace(self.prefix,'') 1330 filename = os.path.join(truncated_path,file) 1331 subdir = os.path.basename(p) 1332 truncated_path = truncated_path[:-len(subdir)] 1333 key = filecount 1334 self.filedict[key] = [truncated_path,subdir,file] 1335 filecount += 1 1336 return len(self.filedict)

1337 1338

1339 -class Path:

1340 - def __init__(self,path, regex=r'.*.i3(.gz){0,1}'):

1341 self.path = path 1342 self.regex = regex

1343

1344 - def SetPath(self,path):

1345 self.path = path

1346 - def GetPath(self):

1347 return self.path

1348 - def SetRegex(self,regex):

1349 self.regex= regex

1350 - def GetRegex(self):

1351 return self.regex

1352

1353 -class I3PreTray:

1354 """ 1355 This class provides an interface for preprocessing files in iceprod 1356 """ 1357

1358 - def __init__(self):

1359 self.modules = OrderedDict() 1360 self.status = 0 1361 self.stats = {} 1362 self.last_added = None 1363 1364 # Aggregate CPU times 1365 self.realtime = 0.0 1366 self.usertime = 0.0 1367 self.systime = 0.0 1368 self.logger = logging.getLogger('I3PreTray')

1369

1370 - def AddModule(self,type,name):

1371 """ 1372 Add entry for module 1373 """ 1374 import iceprod.modules 1375 self.modules[name] = iceprod.modules.get_plugin(type)() 1376 self.last_added = name 1377 self.logger.info("added ipmodule %s %s" %(type,name)) 1378 return self

1379

1380 - def SetParameter(self, module, param, value):

1381 """ 1382 Set parameter value for plugin 1383 """ 1384 self.modules[module].SetParameter(param, value) 1385 return self

1386

1387 - def SetParser(self, module, parser):

1388 """ 1389 Set ExpParser object for plugin 1390 """ 1391 self.modules[module].SetParser(parser) 1392 return self

1393

1394 - def __call__(self, *args):

1395 for pair in args: 1396 print self.last_added +': ', pair[0], '=', pair[1] 1397 self.modules[self.last_added].SetParameter(pair[0], pair[1]) 1398 return self

1399

1400 - def Execute(self):

1401 for key in self.modules.keys(): 1402 self.logger.info("executing PreModule %s"%key) 1403 #retval = self.modules.pop(key).Execute(self.stats) 1404 retval = self.modules[key].Execute(self.stats) 1405 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1406

1407 - def Finish(self):

1408 for key in self.modules.keys(): 1409 retval = self.modules[key].Finish(self.stats) 1410 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1411 1412

1413 -class I3PostTray(I3PreTray):

1414 """ 1415 This class provides an interface for post-processing files in iceprod 1416 """

1417 - def __init__(self):

1418 I3PreTray.__init__(self) 1419 self.logger = logging.getLogger('I3PostTray')

1420

1421 - def Execute(self):

1422 for key in self.modules.keys(): 1423 self.logger.info("executing Post %s"%key) 1424 retval = self.modules.pop(key).Execute(self.stats) 1425 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1426 1427

1428 -class I3Tarball:

1429 """ 1430 Holder for metaproject tarball information 1431 """

1432 - def __init__(self):

1433 self.name = None 1434 self.version = None 1435 self.path = None 1436 self.url = None 1437 self.md5sum = None 1438 self.filebase = None 1439 self.suffix = None 1440 self.platform = None

1441

1442 -class I3Cookie:

1443 """ 1444 Cookie for tracking datasets on client 1445 """

1446 - def __init__(self):

1447 self.plugin = None 1448 self.batchsys = 'condor.Condor' 1449 self.dataset_id = 0 1450 self.queue_id = 0 1451 self.job_ids = [] 1452 self.url = "https://condor.icecube.wisc.edu:9080"

1453

1454 - def GetJobIds(self):

1455 return self.job_ids

1456

1457 - def AddJobId(self,id):

1458 self.job_ids.append(id)

1459

1460 - def GetClusterId(self):

1461 if self.dataset_id: 1462 return self.dataset_id 1463 elif self.job_ids: 1464 return self.job_ids[0] 1465 else: 1466 return 0

1467

1468 - def CheckQ(self,db):

1469 """ 1470 Querie status of cluster or job on condor queue 1471 """ 1472 from iceprod.server import dbqueue,plugins 1473 if self.dataset_id: 1474 cluster = dbqueue.I3DBQueue() 1475 else: 1476 cluster = plugins.get_plugin(self.batchsys)() 1477 cluster.dataset_id = self.dataset_id 1478 for job in self.job_ids: 1479 cluster.job_ids.append(job) 1480 return cluster.CheckQ(db)

1481

1482 - def QRemove(self,db):

1483 """ 1484 Remove cluster or job from condor queue 1485 """ 1486 from iceprod.server import dbqueue,plugins 1487 if self.dataset_id: 1488 cluster = dbqueue.I3DBQueue() 1489 else: 1490 cluster = plugins.get_plugin(self.batchsys)() 1491 cluster.dataset_id = self.dataset_id 1492 for job in self.job_ids: 1493 cluster.job_ids.append(job) 1494 return cluster.QRemove(db)

1495

Source Code for Module iceprod.core.dataclasses