iceprod.core.dataclasses

1 #!/usr/bin/env python 2 # 3 """ 4 A set of classes that describe configuration elements in IceTray. 5 6 copyright (c) 2005 the icecube collaboration 7 8 @version: $Revision: $ 9 @date: $Date: $ 10 @author: T. McCauley <tpmccauley@lbl.gov> 11 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 12 13 @todo: Add support for more project content (the "optional" option) 14 @todo: Be a bit more clever about handling of type tags 15 @todo: Units 16 @todo: Commas and whitespace handling in projects attribute 17 @todo: What to do if libraries are included more than once? 18 @todo: Add some overall elegance 19 @todo: Fix the "inbox" problem 20 """ 21 tver='$Revision: 1.4 $' 22 __version__= 2 23 24 import re,types 25 import logging 26 from odict import OrderedDict 27 from iceprod.core.constants import * 28 29 UnmetDependencyException = "Error: Un-met dependency:" 30 UndefinedTypeException = "Error: invalid type:" 31 32 SupportedTypes = [ 33 'bool', 'int', 'long', 'float', 34 'double', 'string','NaT','OMKey'] 35 36 VectorTypes = [ 37 'boolv', 'intv', 'longv', 'floatv', 38 'doublev', 'stringv','OMKeyv'] 39 40 SimulationCategories = [ 41 'CORSIKA-ice-top', 42 'CORSIKA-in-ice', 43 'neutrino-generator', 44 'JULIeT', 'simple-generator', 45 'WIMP','SuperNova', 46 'TWRFilter', 'OfflineFilter', 47 'L1', 'L2', 'L3' 48 ] 49 50 DatasetTypes = [ 51 'PHYSICS', 52 'SYSTEMATICS', 53 'BENCHMARK', 54 'TEST', 55 'TEMPLATE', 56 'SYSCHK', 57 ] 58 59

60 -class pyOMKey:

61 - def __init__(self,stringid=0,omid=0):

62 self.stringid= stringid 63 self.omid= omid

64

65 - def SetOMId(self,omid):

66 self.omid= omid

67 - def GetOMId(self):

68 return self.omid

69 - def SetStringId(self,stringid):

70 self.stringid= stringid

71 - def GetStringId(self):

72 return self.stringid

73

74 -class Value:

75 - def __init__(self,value,unit=None,format=None):

76 self.value = value 77 self.unit = unit 78 self.format = format # Applies only to string types

79 - def SetValue(self,value):

80 self.value = value

81 - def GetValue(self):

82 return self.value

83 - def SetUnit(self,unit):

84 self.unit = unit

85 - def GetUnit(self):

86 return self.unit

87

88 -class Parameter:

89 - def __init__(self,ptype='NaT',pname='',value=''):

90 self.ptype = ptype 91 self.pname = pname 92 self.pvalue = value 93 self.description = 'No description' 94 self.id = -1 95 self.default = None

96

97 - def SetId(self, id):

98 self.id = id

99 - def GetId(self):

100 return self.id

101

102 - def SetDescription(self, description):

103 self.description = description

104

105 - def GetDescription(self):

106 return self.description

107

108 - def SetType(self, ptype):

109 if ptype in SupportedTypes or ptype in VectorTypes: 110 self.ptype = ptype 111 else: 112 raise UndefinedTypeException, ptype

113

114 - def GetType(self):

115 return self.ptype

116

117 - def SetName(self, pname):

118 self.pname = pname

119 - def GetName(self):

120 return self.pname

121

122 - def SetValue(self, pvalue):

123 self.pvalue = pvalue

124 - def GetValue(self):

125 return self.pvalue

126

127 - def SetUnit(self, punit):

128 self.punit = punit

129 - def GetUnit(self):

130 return self.punit

131

132 - def SetFormat(self, pformat):

133 self.pformat = pformat

134 - def GetFormat(self):

135 return self.pformat

136

137 - def SetDefault(self,value):

138 self.default = value

139 - def GetDefault(self):

140 return self.default

141

142 -class TaskDefinition:

143 - def __init__(self, name, id=None):

144 self.SetName(name) 145 self.trays = OrderedDict() 146 self.parents = [] 147 self.children = [] 148 self.requirements = '' 149 self.batchoptions = '' 150 self.grids = [] 151 self.parallel = False 152 self.id = id 153 self.photonics = False

154

155 - def SetId(self, id):

156 self.id = id

157 - def GetId(self):

158 return self.id

159

160 - def AddTray(self, idx, iters, externs=False):

161 self.trays[int(idx)] = TaskTrayDefinition(idx, iters, externs)

162 - def GetTrays(self):

163 return self.trays

164 - def GetTray(self, idx):

165 return self.GetTrays()[int(idx)]

166

167 - def SetName(self, name):

168 self.name = name

169 - def GetName(self):

170 return self.name

171

172 - def AddGrid(self, grid):

173 self.grids.append(grid)

174 - def GetGrids(self):

175 return self.grids

176

177 - def SetRequirements(self, reqs):

178 self.requirements = reqs.strip().replace('\n', ' ')

179 - def GetRequirements(self):

180 return self.requirements

181

182 - def SetBatchOpts(self, opts):

183 self.batchoptions = opts.strip().replace('\n', ' ')

184 - def GetBatchOpts(self):

185 return self.batchoptions

186

187 - def AddParent(self, parent):

188 self.parents.append(parent)

189 - def GetParents(self):

190 return self.parents

191

192 - def AddChild(self, child):

193 self.children.append(child)

194 - def GetChildren(self):

195 return self.children

196

197 - def SetParallelExecution(self, parallel):

198 self.parallel = bool(parallel)

199 - def ParallelExecutionEnabled(self):

200 return self.parallel

201

202 - def SetUsesPhotonics(self, photonics):

203 self.photonics = bool(photonics)

204 - def UsesPhotonics(self):

205 return self.photonics

206

207 - def HasTray(self, idx):

208 return self.GetTrays().has_key(idx)

209

210 - def HasIter(self, tray, iter):

211 return self.HasTray(tray) and self.GetTray(tray).HasIteration(iter)

212

213 - def HasInput(self):

214 return not self.IsCleanup() and self.GetParents()

215 - def HasOutput(self):

216 children = self.GetChildren() 217 return children and not (len(children) == 1 and TASK_TRASHCAN_NAME in children)

218

219 - def IsCleanup(self):

220 return self.GetName() == TASK_TRASHCAN_NAME

221

222 - def __str__(self):

223 return "%s: %s" % (self.GetName(),self.GetTrays().items())

224

225 -class TaskTrayDefinition:

226 - def __init__(self, idx, iters, externs=False):

227 self.iters = [] 228 self.externs = False 229 self.idx = 0 230 231 self.SetIndex(idx) 232 self.SetRunExterns(externs) 233 self.SetIters(iters)

234

235 - def SetIndex(self, idx):

236 self.idx = int(idx)

237 - def GetIndex(self):

238 return self.idx

239

240 - def SetIters(self, iters):

241 if not self.RunExternsEnabled(): 242 real_iters = [] 243 else: 244 real_iters = [TASK_EXTERN_ITER] 245 iters = str(iters).split(",") 246 for iter in iters: 247 if iter.count("-") > 0 and not iter == str(TASK_EXTERN_ITER): 248 endpoints = map(int, iter.split("-")) 249 for val in range(endpoints[0], endpoints[1]+1): 250 if val not in real_iters: 251 real_iters.append(val) 252 else: 253 iter = int(iter) 254 if iter not in real_iters: 255 real_iters.append(iter) 256 self.iters = map(int, real_iters)

257 - def GetIters(self):

258 return self.iters

259

260 - def HasIter(self, iter):

261 return self.GetIters().has_key(iter)

262

263 - def SetRunExterns(self, externs):

264 self.externs = bool(externs)

265 - def RunExternsEnabled(self):

266 return self.externs or TASK_EXTERN_ITER in self.GetIters()

267

268 - def __str__(self):

269 return "tray %d, iters %s" % (self.GetIndex(),self.GetIters())

270

271 -class JobDependency:

272 - def __init__(self,dataset=0,job="0"):

273 self.dataset = dataset 274 self.job = job

275 276

277 -class Steering:

278 - def __init__(self):

279 self.version = __version__ 280 self.parent_id = 0 281 self.trays = [] 282 self.difplus = None 283 self.parameters = OrderedDict() # list of Parameter objects 284 self.batchopts = OrderedDict() # list of batch options 285 self.sysopts = OrderedDict() # list of system-specific options 286 self.dependencies = OrderedDict() # list of file dendencies at runtime 287 self.jobdeps = [] # list of input dataset rules 288 self.description = "Run was not given a description" 289 self.category = None 290 self.externs = [] 291 self.extras = OrderedDict() 292 self.filter = None 293 self.datasettype = "PHYSICS" 294 self.task_defs = OrderedDict() 295 self.statistics = OrderedDict() 296 297 self.AddStatistic('i3exec runtime') 298 self.AddStatistic('sys_time') 299 self.AddStatistic('user_time') 300 self.AddStatistic('real_time')

301

302 - def AddStatistic(self,stat):

303 if not hasattr(self, 'statistics'): 304 self.statistics = OrderedDict() 305 self.statistics[stat] = stat

306

307 - def GetStatistics(self):

308 if not hasattr(self, 'statistics'): 309 self.statistics = OrderedDict() 310 return self.statistics.values()

311

312 - def AddDIFPlus(self,difplus):

313 self.difplus = difplus

314

315 - def GetDIFPlus(self):

316 return self.difplus

317

318 - def HasDIFPlus(self):

319 return self.difplus

320

321 - def SetVersion(self,version):

322 self.version = version

323 - def GetVersion(self):

324 return self.version

325

326 - def SetParentId(self,id):

327 self.parent_id = id

328 - def GetParentId(self):

329 return self.parent_id

330

331 - def CheckConfig(self):

332 for i in range(len(self.trays)): 333 check,str = self.trays[i].CheckConfig() 334 if not check: 335 return check, "tray %d: %s" % (i,str) 336 return True,""

337

338 - def SetOF(self,filter):

339 self.filter = filter

340

341 - def GetOF(self):

342 return self.filter

343

344 - def AddJobDependency(self, dataset,job):

345 if not hasattr(self,'jobdeps'): 346 self.jobdeps = [] 347 self.jobdeps.append(JobDependency(dataset,job))

348

349 - def GetJobDependencies(self):

350 if not hasattr(self,'jobdeps'): 351 self.jobdeps = [] 352 return self.jobdeps

353

354 - def SetDescription(self, desc):

355 self.description = desc

356

357 - def GetDescription(self):

358 return self.description

359

360 - def SetDatasetType(self, dtype):

361 self.datasettype = dtype

362

363 - def GetDatasetType(self):

364 return self.datasettype

365

366 - def SetCategory(self, cat):

367 self.category = cat

368

369 - def GetCategory(self):

370 return self.category

371

372 - def AddParameter(self, *args):

373 """ 374 usage: 375 AddParameter(p) 376 where p is a Parameter object as of iceprod 1.2: you can also call 377 378 AddParameter("type","name","value") 379 """ 380 if len(args) < 1: raise Exception, "AddParameter requires at least 1 argument, 0 given" 381 if isinstance(args[0],Parameter): 382 parameter = args[0] 383 else: 384 parameter = Parameter(*args) 385 self.parameters[parameter.GetName()] = parameter

386

387 - def RemoveParameter(self, parametername):

388 if self.parameters.has_key(parametername): 389 del self.parameters[parametername]

390

391 - def GetParameters(self):

392 return self.parameters.values()

393

394 - def GetParameter(self,pname):

395 if self.parameters.has_key(pname): 396 return self.parameters[pname]

397

398 - def AddDependency(self, depend):

399 if type(depend) in types.StringTypes: 400 self.dependencies[depend] = depend 401 else: 402 self.dependencies[depend.GetName()]= depend

403

404 - def GetDependencies(self):

405 return self.dependencies.values()

406

407 - def GetDependency(self,dname):

408 if self.dependencies.has_key(dname): 409 return self.dependencies[dname]

410

411 - def RemoveDependency(self,dname):

412 if self.dependencies.has_key(dname): 413 del self.dependencies[dname]

414

415 - def AddBatchOpt(self, opt):

416 self.batchopts[(opt.GetType(),opt.GetName())] = opt

417

418 - def GetBatchOpts(self):

419 return self.batchopts.values()

420

421 - def GetBatchOpt(self,bname):

422 if self.batchopts.has_key(bname): 423 return self.batchopts[bname]

424

425 - def RemoveBatchOpt(self, bname):

426 if self.batchopts.has_key(bname): 427 del self.batchopts[bname]

428

429 - def AddSysOpt(self, opt):

430 self.sysopts[opt.GetName()] = opt

431

432 - def GetSysOpts(self):

433 return self.sysopts.values()

434

435 - def GetSysOpt(self,bname):

436 if self.sysopts.has_key(bname): 437 return self.sysopts[bname]

438

439 - def RemoveSysOpt(self, bname):

440 if self.sysopts.has_key(bname): 441 del self.sysopts[bname]

442

443 - def AddExtern(self, extern):

444 self.externs.append(extern)

445

446 - def GetExterns(self):

447 return self.externs

448

449 - def AddTaskDefinition(self, task_def):

450 self.task_defs[task_def.GetName()] = task_def

451

452 - def GetTaskDefinition(self, name):

453 return self.task_defs[name]

454

455 - def GetTaskDefinitions(self):

456 return self.task_defs

457

458 - def get_param_dict(self):

459 d = {} 460 for param in self.parameters.values(): 461 d[param.GetName()] = param.GetValue() 462 return d

463

464 - def get_sysopt_dict(self):

465 d = {} 466 for param in self.sysopts.values(): 467 d[param.GetName()] = param.GetValue() 468 return d

469

470 - def AddTray(self, config):

471 self.trays.append(config)

472

473 - def GetTrays(self):

474 return self.trays

475

476 - def GetTray(self,t):

477 return self.trays[t]

478

479 - def RemoveTray(self,t):

480 tray = self.trays[t] 481 del self.trays[t] 482 return tray

483

484 - def HasExtra(self, name):

485 return self.extras.has_key(name)

486

487 - def AddExtra(self, name, extra):

488 self.extras[name] = extra

489

490 - def GetExtra(self,name):

491 if self.HasExtra(name): 492 return self.extras[name]

493

494 - def ClearExtras(self):

495 self.extras = OrderedDict()

496

497 - def GetExtras(self):

498 return self.extras

499

500 - def GetExtraList(self):

501 return self.extras.values()

502

503 - def RemoveExtra(self,name):

504 if self.HasExtra(name): 505 del self.extras[name]

506 507

508 -class BatchOpt(Parameter):

509

510 - def __init__(self):

511 Parameter.__init__(self) 512 self.ptype = 'condor'

513

514 - def SetType(self,ptype):

515 self.ptype = ptype

516

517 -class SysOpt(Parameter):

518

519 - def __init__(self,name='',value=None,type='string'):

520 Parameter.__init__(self) 521 self.pname = name 522 self.pvalue = Value(value) 523 self.ptype = type

524

525 - def GetValue(self):

526 return self.pvalue.GetValue()

527 528 529

530 -class Module:

531 - def __init__(self):

532 self.name = '' 533 self.description = '' 534 self.parameters = OrderedDict() # list of Parameter objects 535 self.mclass = '' 536 self.id = -1 537 self.projects = OrderedDict() 538 self.inboxes = [] 539 self.outboxes = []

540 541

542 - def CheckParams(self):

543 for p in self.GetParameters(): 544 if p.GetType() == 'NaT': 545 return False, "%s (%s) : %s %s" % ( 546 self.GetClass(),self.GetName(), 547 p.GetType(),p.GetName()) 548 return True,""

549

550 - def SetName(self, name):

551 self.name = name

552 - def GetName(self):

553 return self.name

554

555 - def SetDescription(self, description):

556 self.description = description

557

558 - def GetDescription(self):

559 return self.description

560

561 - def SetClass(self, mclass):

562 self.mclass = mclass

563

564 - def GetClass(self):

565 return self.mclass

566

567 - def AddParameter(self, parameter):

568 if not self.parameters.has_key(parameter.GetName()): 569 self.parameters[parameter.GetName()] = parameter

570

571 - def RemoveParameter(self, parametername):

572 if self.parameters.has_key(parametername): 573 del self.parameters[parametername]

574

575 - def GetParameters(self):

576 return [self.parameters[k] for k in self.parameters.keys()]

577

578 - def GetParameter(self,pname):

579 if self.parameters.has_key(pname): 580 return self.parameters[pname]

581

582 - def AddProject(self, name, project):

583 if not self.projects.has_key(name): 584 self.projects[name] = project

585

586 - def RemoveProject(self, projectname):

587 if self.projects.has_key(projectname): 588 del self.projects[projectname]

589

590 - def ClearProjects(self):

591 self.projects = OrderedDict()

592

593 - def GetProjects(self):

594 return self.projects

595

596 - def GetProject(self,pname):

597 if self.projects.has_key(projectname): 598 return self.projects[pname]

599

600 - def GetProjectList(self):

601 return [self.projects[k] for k in self.projects.keys()]

602

603 - def SetId(self, id):

604 self.id = id

605

606 - def GetId(self):

607 return self.id

608

609 -class Service(Module):

610 - def __init__(self):

611 Module.__init__(self)

612

613 -class Segement(Module):

614 - def __init__(self):

615 Module.__init__(self)

616

617 -class IceProdPre(Module):

618 """ 619 This class provides an interface for preprocessing files in iceprod 620 """

621 - def __init__(self):

622 Module.__init__(self)

623

624 -class IceProdPost(IceProdPre):

625 """ 626 This class provides an interface for postprocessing files in iceprod 627 """

628 - def __init__(self):

629 IceProdPre.__init__(self)

630

631 -class Project:

632 - def __init__(self):

633 self.name = '' 634 self.version = '' 635 self.revision = 0 636 self.pid = -1 637 self.path = '.' 638 self.type = 'c++' 639 self.dependencies = {}

640

641 - def SetName(self, name):

642 self.name = name

643 - def GetName(self):

644 return self.name

645

646 - def SetVersion(self, version):

647 self.version = version

648

649 - def GetVersion(self):

650 return self.version

651

652 - def GetVersionTxt(self,fmt="%s.%s.%s"):

653 tok = self.version.split('.')[-1].replace('V','').split('-') 654 versionStr = fmt % ( tok[0], tok[1], tok[2]) 655 return versionStr

656

657 - def SetRevision(self, rev):

658 self.revision = rev

659 - def GetRevision(self):

660 return self.revision

661

662 - def SetType(self, type):

663 self.type = type

664 - def GetType(self):

665 return self.type

666 667

668 - def SetHost(self, version):

669 self.host = host

670 - def GetHost(self):

671 return self.host

672

673 - def SetDateTime(self, dateTime):

674 self.dateTime = dateTime

675 - def GetDateTime(self):

676 return self.dateTime

677

678 - def SetOS(self, os):

679 self.os = os

680 - def GetOS(self):

681 return self.os

682

683 - def SetCompiler(self, compiler):

684 self.compiler = compiler

685

686 - def GetCompiler(self):

687 return self.compiler

688

689 - def SetId(self, pid):

690 self.pid = pid

691

692 - def GetId(self):

693 return self.pid

694

695 - def SetPath(self, path):

696 self.path = path

697

698 - def GetPath(self):

699 return self.path

700

701 - def AddDependency(self, depend):

702 if type(depend) in types.StringTypes: 703 self.dependencies[depend] = depend 704 else: 705 self.dependencies[depend.GetName()]= depend

706

707 - def GetDependencies(self):

708 return self.dependencies.values()

709

710 - def GetDependencyKeys(self):

711 return self.dependencies.keys()

712

713 - def ClearDependencies(self):

714 self.dependencies = {}

715

716 -class Container(Project):

717 - def __init__(self):

718 Project.__init__(self) 719 self.modules = OrderedDict() 720 self.services = OrderedDict() 721 self.iceprodpre = OrderedDict() 722 self.iceprodpost = OrderedDict()

723

724 - def AddIceProdPre(self, pre):

725 sname = pre.GetName() 726 self.iceprodpre[sname] = pre

727

728 - def GetIceProdPres(self):

729 return self.iceprodpre.values()

730

731 - def AddIceProdPost(self, post):

732 sname = post.GetName() 733 self.iceprodpost[sname] = post

734

735 - def GetIceProdPosts(self):

736 return self.iceprodpost.values()

737

738 - def HasModule(self, name):

739 return self.modules.has_key(name)

740 - def AddModule(self, module):

741 self.modules[module.GetName()] = module

742 - def GetModules(self):

743 return self.modules.values()

744

745 - def RemoveModule(self, modulename):

746 return self.modules.pop(modulename)

747

748 - def HasService(self, name):

749 return self.services.has_key(name)

750 - def AddService(self, service):

751 self.services[service.GetName()] = service

752 - def GetServices(self):

753 return self.services.values()

754 - def RemoveService(self, name):

755 return self.services.pop(name)

756

757 -class MetaProject(Project):

758 """ 759 Class inherits from Project 760 Represents an svn metaproject 761 """

762 - def __init__(self):

763 Project.__init__(self) 764 self.projects = OrderedDict()

765

766 - def HasProject(self, pname):

767 return self.projects.has_key(pname)

768

769 - def AddProject(self, name, project):

770 self.projects[name] = project

771

772 - def GetProject(self,pname):

773 if self.projects.has_key(pname): 774 return self.projects[pname]

775

776 - def ClearProjects(self):

777 self.projects = OrderedDict()

778

779 - def GetProjects(self):

780 return self.projects

781

782 - def GetProjectList(self):

783 return self.projects.values()

784

785 - def RemoveProject(self, projectname):

786 if self.projects.has_key(projectname): 787 del self.projects[projectname]

788

789 - def AddProjectAttr(self, name, project):

790 if not self.ice_project_attr.has_key(name): 791 self.ice_project_attr[name] = project

792 793

794 -class Connection:

795 - def __init__(self):

796 self.outbox = None 797 self.inbox = Box('')

798

799 - def From(self, from_module, outbox):

800 self.outbox = Box(from_module,outbox)

801

802 - def To(self, to_module, inbox):

803 self.inbox = Box(to_module,inbox)

804

805 - def To(self, to_module, inbox=None):

806 if inbox: 807 self.inbox = Box(to_module,inbox) 808 else: 809 self.inbox = Box(to_module)

810

811 - def GetInbox(self):

812 return self.inbox

813

814 - def GetOutbox(self):

815 return self.outbox

816

817 -class Box:

818 - def __init__(self,module,boxname='default'):

819 self.module = module 820 self.boxname = boxname

821

822 - def GetModule(self):

823 return self.module

824

825 - def SetModule(self,module):

826 self.module = module

827

828 - def GetBoxName(self):

829 return self.boxname

830

831 - def SetBoxName(self,name):

832 self.boxname = name

833

834 -class IceTrayConfig:

835 - def __init__(self):

836 self.steering = Steering() 837 self.name = None 838 self.events = '0' 839 self.iterations = '1' 840 self.iceprodpre = OrderedDict() 841 self.modules = [] 842 self.services = OrderedDict() 843 self.projects = OrderedDict() 844 self.metaprojects = OrderedDict() 845 self.ice_project_attr = OrderedDict() 846 self.connections = [] 847 self.iceprodpost = OrderedDict() 848 self.input_files = [] 849 self.output_files = [] 850 self.python = None

851

852 - def SetName(self,name):

853 self.name = name

854

855 - def GetName(self):

856 return self.name

857

858 - def SetPython(self,python):

859 self.python = python

860

861 - def GetPython(self):

862 if not hasattr(self, 'python'): 863 self.python = None 864 return self.python

865

866 - def CheckConfig(self):

867 for m in self.GetModules(): 868 check,str = m.CheckParams() 869 if not check: 870 return check,"in module \'%s\' needs to be configured or removed" %str 871 for s in self.GetServices(): 872 check,str = s.CheckParams() 873 if not check: 874 return check,"service %s needs to be configured or removed" %str 875 return True,""

876

877 - def SetEvents(self,events):

878 self.events = events

879 - def GetEvents(self):

880 return self.events

881

882 - def SetIterations(self,iter):

883 self.iterations = iter

884 - def GetIterations(self):

885 return self.iterations

886

887 - def GetParameter(self,pname):

888 if self.parameters.has_key(pname): 889 return self.parameters[pname] 890 else: 891 return None

892

893 - def SetSteering(self, steering):

894 self.steering=steering

895

896 - def GetSteering(self):

897 return self.steering

898

899 - def HasModule(self, mname):

900 for m in self.modules: 901 if m.GetName() == mname: 902 return True 903 return False

904

905 - def AddModule(self, module):

906 mname = module.GetName() 907 for p in module.GetProjectList(): 908 if not self.HasProject(p.GetName()): 909 print ",".join(map(Project.GetName,self.GetProjectList())) 910 raise UnmetDependencyException, \ 911 "module %s requires project %s" % (mname,p.GetName()) 912 break # we only care about the first project 913 914 for i in range(len(self.modules)): 915 if self.modules[i].GetName() == module.GetName(): 916 self.modules.pop(i) 917 self.modules.insert(i,module) 918 return 919 self.modules.append(module)

920

921 - def InsertModule(self, index,module):

922 for p in module.GetProjectList(): 923 if self.GetProject(p.GetName()) == None: 924 raise UnmetDependencyException, \ 925 "module %s requires project %s" % (mname,p.GetName()) 926 if index < 0: 927 self.modules.insert(0,module) 928 elif index >= len(self.modules): 929 self.modules.append(module) 930 else: 931 self.modules.insert(index,module)

932

933 - def RemoveModule(self, modulename):

934 index = 0 935 for m in self.modules: 936 if m.GetName() == modulename: 937 return self.modules.pop(index) 938 index += 1 939 return False

940

941 - def PopModule(self, index):

942 return self.modules.pop(index)

943

944 - def GetModules(self):

945 return self.modules

946

947 - def GetModule(self,modulename):

948 for m in self.modules: 949 if m.GetName() == modulename: 950 return m 951 return False

952

953 - def ClearModules(self):

954 self.modules = []

955

956 - def HasMetaProject(self, mpname):

957 return self.metaprojects.has_key(mpname)

958

959 - def AddMetaProject(self, name, metaproject):

960 self.metaprojects[name] = metaproject

961

962 - def GetMetaProject(self,mpname):

963 if self.metaprojects.has_key(mpname): 964 return self.metaprojects[mpname]

965

966 - def GetMetaProjectById(self,mpid):

967 for mp in self.metaprojects.values(): 968 if mp.Get() == mpid: 969 return mp 970 mp = filter((lambda mp: mp.GetId() == mpid), self.metaprojects.values()) 971 if len(mp) > 0: 972 return mp[0]

973

974 - def ClearMetaProjects(self):

975 self.metaprojects = OrderedDict()

976

977 - def GetMetaProjects(self):

978 return self.metaprojects

979

980 - def GetMetaProjectList(self):

981 return [self.metaprojects[k] for k in self.metaprojects.keys()]

982

983 - def RemoveMetaProject(self, metaprojectname):

984 if self.metaprojects.has_key(metaprojectname): 985 del self.metaprojects[metaprojectname]

986 987

988 - def HasProject(self, pname):

989 return self.GetProject(pname)

990 991

992 - def AddProject(self, name, project):

993 try: 994 metaproject = self.GetMetaProjectById(project.GetId()) 995 metaproject.AddProject(name,project) 996 except: 997 self.projects[name] = project

998

999 - def GetProject(self,pname):

1000 for metaproject in self.GetMetaProjectList(): 1001 if metaproject.HasProject(pname): 1002 return metaproject.GetProject(pname) 1003 1004 # If not found in metaprojects look at top level 1005 if self.projects.has_key(pname): 1006 return self.projects[pname]

1007 1008

1009 - def ClearProjects(self):

1010 self.projects = OrderedDict()

1011

1012 - def GetProjects(self):

1013 return self.projects

1014

1015 - def GetProjectList(self):

1016 return self.projects.values()

1017

1018 - def RemoveProject(self, projectname):

1019 if self.projects.has_key(projectname): 1020 del self.projects[projectname]

1021

1022 - def AddProjectAttr(self, name, project):

1023 if not self.ice_project_attr.has_key(name): 1024 self.ice_project_attr[name] = project

1025

1026 - def GetProjectAttr(self):

1027 return self.ice_project_attr

1028

1029 - def GetProjectAttrList(self):

1030 return self.ice_project_attr.values()

1031

1032 - def RemoveProjectAttr(self, projectname):

1033 if self.ice_project_attr.has_key(projectname): 1034 del self.ice_project_attr[projectname]

1035

1036 - def HasService(self, sname):

1037 return self.services.has_key(sname)

1038

1039 - def AddService(self, service):

1040 sname = service.GetName() 1041 if self.services.has_key(sname): 1042 return 1043 for p in service.GetProjectList(): 1044 if not self.HasProject( p.GetName() ): 1045 plist = [] 1046 for metaproject in self.GetMetaProjectList(): 1047 plist += map(Project.GetName,metaproject.GetProjectList()) 1048 plist = ",".join(plist) 1049 raise UnmetDependencyException, \ 1050 "service %s requires project %s : %s " % \ 1051 (sname,p.GetName(),plist) 1052 continue # we only care about the first project 1053 self.services[sname] = service

1054

1055 - def InsertService(self, index,service):

1056 sname = service.GetName() 1057 for p in service.GetProjectList(): 1058 if self.GetProject(p.GetName()) == None: 1059 raise UnmetDependencyException, \ 1060 "service %s requires project %s" % (service.GetClass(),p.GetName()) 1061 if index < 0: 1062 self.services.insert(0,sname,service) 1063 elif index >= len(self.modules): 1064 self.services[sname] = service 1065 else: 1066 self.services.insert(index,sname,service)

1067

1068 - def RemoveService(self, servicename):

1069 return self.services.pop(servicename)

1070

1071 - def GetServices(self):

1072 return self.services.values()

1073

1074 - def GetService(self,s):

1075 if self.services.has_key(s): 1076 return self.services[s]

1077

1078 - def ClearServices(self):

1079 self.services = OrderedDict()

1080 1081 #####

1082 - def HasIceProdPre(self, sname):

1083 return self.iceprodpre.has_key(sname)

1084

1085 - def AddIceProdPre(self, pre):

1086 sname = pre.GetName() 1087 self.iceprodpre[sname] = pre

1088

1089 - def InsertIceProdPre(self, index,pre):

1090 sname = pre.GetName() 1091 if index < 0: 1092 self.iceprodpre.insert(0,sname,pre) 1093 elif index >= len(self.modules): 1094 self.iceprodpre[sname] = pre 1095 else: 1096 self.iceprodpre.insert(index,sname,pre)

1097

1098 - def RemoveIceProdPre(self, prename):

1099 return self.iceprodpre.pop(prename)

1100

1101 - def GetIceProdPres(self):

1102 return self.iceprodpre.values()

1103

1104 - def GetIceProdPre(self,s):

1105 if self.iceprodpre.has_key(s): 1106 return self.iceprodpre[s]

1107

1108 - def ClearIceProdPre(self):

1109 self.iceprodpre = OrderedDict()

1110 ###

1111 - def HasIceProdPost(self, sname):

1112 return self.iceprodpost.has_key(sname)

1113

1114 - def AddIceProdPost(self, post):

1115 sname = post.GetName() 1116 self.iceprodpost[sname] = post

1117

1118 - def InsertIceProdPost(self, index,post):

1119 sname = post.GetName() 1120 if index < 0: 1121 self.iceprodpost.insert(0,sname,post) 1122 elif index >= len(self.modules): 1123 self.iceprodpost[sname] = post 1124 else: 1125 self.iceprodpost.insert(index,sname,post)

1126

1127 - def RemoveIceProdPost(self, postname):

1128 return self.iceprodpost.pop(postname)

1129

1130 - def GetIceProdPosts(self):

1131 return self.iceprodpost.values()

1132

1133 - def GetIceProdPost(self,s):

1134 if self.iceprodpost.has_key(s): 1135 return self.iceprodpost[s]

1136

1137 - def ClearIceProdPosts(self):

1138 self.iceprodpost = OrderedDict()

1139 ##### 1140

1141 - def AddConnection(self, connection):

1142 self.connections.append(connection)

1143

1144 - def InsertConnection(self,pos, connection):

1145 self.connections.insert(pos,connection)

1146

1147 - def GetConnections(self):

1148 return self.connections

1149

1150 - def RemoveConnection(self,index):

1151 con = self.connections[index] 1152 del self.connections[index] 1153 return con

1154

1155 - def ClearConnections(self):

1156 self.connections = []

1157

1158 - def AddInputFile(self,file):

1159 self.input_files.append(file)

1160 - def GetInputFiles(self):

1161 return self.input_files

1162 - def AddOutputFile(self,file):

1163 self.output_files.append(file)

1164 - def GetOutputFiles(self):

1165 return self.output_files

1166

1167 -class IceTrayFile:

1168 - def __init__(self, name, photonics=False):

1169 self.name = name 1170 self.photonics = bool(photonics) 1171 self.unpack = False

1172

1173 - def GetName(self):

1174 return self.name

1175 - def IsPhotonicsTable(self):

1176 return self.photonics

1177

1178 -class Block(IceTrayConfig):

1179 """ 1180 Container of externals and IceTrays 1181 """

1182 - def __init__(self):

1183 IceTrayConfig.__init__(self) 1184 self.trays = [] 1185 self.externs = []

1186

1187 - def AddTray(self, config):

1188 self.trays.append(config)

1189

1190 - def GetTrays(self):

1191 return self.trays

1192

1193 - def GetTray(self,t):

1194 return self.trays[t]

1195

1196 - def RemoveTray(self,t):

1197 tray = self.trays[t] 1198 del self.trays[t] 1199 return tray

1200

1201 - def AddExtern(self, extern):

1202 self.externs.append(extern)

1203

1204 - def GetExterns(self):

1205 return self.externs

1206 1207

1208 -class Dependency:

1209 - def __init__(self,path=None,format=None):

1210 self.path = path 1211 self.format = None 1212 self.cache = False

1213

1214 - def SetPath(self,path):

1215 self.path = path

1216

1217 - def GetPath(self,path):

1218 return self.path

1219

1220 - def SetFormat(self,format):

1221 self.format = format

1222

1223 - def GetFormat(self):

1224 return self.format

1225

1226 -class ExternSteering:

1227 - def __init__(self):

1228 self.name = None 1229 self.text = None

1230

1231 - def SetName(self,name):

1232 self.name = name

1233 - def GetName(self):

1234 return self.name

1235

1236 - def SetText(self,text):

1237 self.text = text

1238 - def GetText(self):

1239 return self.text

1240

1241 - def Write(self):

1242 fd = open(self.name,'w') 1243 print >> fd,self.text 1244 fd.close()

1245 1246

1247 -class Extern:

1248 - def __init__(self):

1249 self.name = None 1250 self.version = None 1251 self.description = '' 1252 self.arguments = '' 1253 self.infile = None 1254 self.pformat = "%(name)s %(value)s %(description)s" 1255 self.outfile = None 1256 self.errfile = None 1257 self.command = None 1258 self.parameters = OrderedDict() # list of Parameter objects 1259 self.steering = []

1260

1261 - def SetName(self, name):

1262 self.name = name

1263 - def GetName(self):

1264 return self.name

1265

1266 - def SetVersion(self, version):

1267 self.version = version

1268 - def GetVersion(self):

1269 return self.version

1270

1271 - def SetDescription(self, description):

1272 self.description = description

1273 - def GetDescription(self):

1274 return self.description

1275

1276 - def SetArgs(self, args):

1277 self.arguments= args

1278 - def GetArgs(self):

1279 return self.arguments

1280

1281 - def AddParameter(self, parameter):

1282 self.parameters[parameter.GetName()] = parameter

1283

1284 - def RemoveParameter(self, parametername):

1285 if self.parameters.has_key(parametername): 1286 del self.parameters[parametername]

1287

1288 - def GetParameters(self):

1289 return self.parameters.values()

1290

1291 - def SetInputFileFormat(self, formatstring):

1292 self.pformat = formatstring

1293

1294 - def GetInputFileFormat(self):

1295 return self.pformat

1296

1297 - def WriteInputFile(self, filename):

1298 fd = open(filename,'w') 1299 for p in self.parameters.values(): 1300 pd = { 1301 "type":p.GetType(), 1302 "name":p.GetName(), 1303 "value":p.GetValue(), 1304 "description":p.GetDescription() 1305 } 1306 print >> fd,self.pformat % pd 1307 fd.close()

1308 1309

1310 - def GetParameter(self,pname):

1311 if self.parameters.has_key(pname): 1312 return self.parameters[pname]

1313

1314 - def SetOutFile(self, filename):

1315 self.outfile = filename

1316 - def GetOutFile(self):

1317 return self.outfile

1318

1319 - def SetErrFile(self, filename):

1320 self.errfile = filename

1321 - def GetErrFile(self):

1322 return self.errfile

1323

1324 - def SetInFile(self, filename):

1325 self.infile = filename

1326 - def GetInFile(self):

1327 return self.infile

1328

1329 - def SetExec(self, command):

1330 self.command = command

1331 - def GetExec(self):

1332 return self.command

1333

1334 - def AddSteering(self, steering):

1335 self.steering.append(steering)

1336 - def GetSteering(self):

1337 return self.steering

1338

1339 -class OfflineFilter:

1340 - def __init__(self, prefix='/data/sim', search=True, filedict={}, filter=''):

1341 self.prefix = prefix 1342 self.search = search 1343 self.filedict = filedict 1344 self.paths = [] 1345 self.tokens = [] 1346 self.filter = filter

1347

1348 - def AddPath(self,path):

1349 self.paths.append(path)

1350

1351 - def GetPaths(self):

1352 return self.paths

1353

1354 - def SetTokens(self,tok):

1355 self.tokens = tok

1356

1357 - def GetTokens(self,prefix):

1358 return self.tokens

1359

1360 - def SetPrefix(self,prefix):

1361 self.prefix = prefix

1362

1363 - def GetPrefix(self):

1364 return self.prefix

1365

1366 - def SetFileDict(self,filedict):

1367 self.filedict = filedict

1368

1369 - def GetFileDict(self):

1370 return self.filedict

1371

1372 - def Search(self):

1373 """ 1374 Get list of files in rootdir and add them to database dictonary table 1375 """ 1376 import re,os, os.path 1377 from os.path import join 1378 filecount = 0 1379 for path in self.paths: 1380 if path.regex: cregex = re.compile(path.regex) 1381 for p, dirs, files in os.walk(join(self.prefix,path.path)): 1382 for file in files: 1383 filepath = os.path.join(p,file) 1384 match = cregex.match(filepath) 1385 if match: 1386 year = match.group(0) 1387 day = match.group(1) 1388 run = match.group(2) 1389 truncated_path = p.replace(self.prefix,'') 1390 filename = os.path.join(truncated_path,file) 1391 subdir = os.path.basename(p) 1392 truncated_path = truncated_path[:-len(subdir)] 1393 key = filecount 1394 self.filedict[key] = [truncated_path,subdir,file] 1395 filecount += 1 1396 return len(self.filedict)

1397 1398

1399 -class Path:

1400 - def __init__(self,path, regex=r'.*.i3(.gz){0,1}'):

1401 self.path = path 1402 self.regex = regex

1403

1404 - def SetPath(self,path):

1405 self.path = path

1406 - def GetPath(self):

1407 return self.path

1408 - def SetRegex(self,regex):

1409 self.regex= regex

1410 - def GetRegex(self):

1411 return self.regex

1412

1413 -class I3PreTray:

1414 """ 1415 This class provides an interface for preprocessing files in iceprod 1416 """ 1417

1418 - def __init__(self):

1419 self.modules = OrderedDict() 1420 self.status = 0 1421 self.stats = {} 1422 self.last_added = None 1423 1424 # Aggregate CPU times 1425 self.realtime = 0.0 1426 self.usertime = 0.0 1427 self.systime = 0.0 1428 self.logger = logging.getLogger('I3PreTray')

1429

1430 - def AddModule(self,type,name):

1431 """ 1432 Add entry for module 1433 """ 1434 import iceprod.modules 1435 self.modules[name] = iceprod.modules.get_plugin(type)() 1436 self.last_added = name 1437 self.logger.info("added ipmodule %s %s" %(type,name)) 1438 return self

1439

1440 - def SetParameter(self, module, param, value):

1441 """ 1442 Set parameter value for plugin 1443 """ 1444 self.modules[module].SetParameter(param, value) 1445 return self

1446

1447 - def SetParser(self, module, parser):

1448 """ 1449 Set ExpParser object for plugin 1450 """ 1451 self.modules[module].SetParser(parser) 1452 return self

1453

1454 - def SetParser(self, *args):

1455 """ 1456 Set ExpParser object for plugin 1457 """ 1458 if len(args) == 1: 1459 for mod in self.modules.values(): 1460 mod.SetParser(args[0]) 1461 elif len(args) == 2: 1462 module = args[0] 1463 parser = args[1] 1464 self.modules[module].SetParser(parser) 1465 else: 1466 raise Exception, "Wrong number of arguments to SetParser"

1467 1468

1469 - def __call__(self, *args):

1470 for pair in args: 1471 print self.last_added +': ', pair[0], '=', pair[1] 1472 self.modules[self.last_added].SetParameter(pair[0], pair[1]) 1473 return self

1474

1475 - def Execute(self):

1476 for key in self.modules.keys(): 1477 self.logger.info("executing PreModule %s"%key) 1478 #retval = self.modules.pop(key).Execute(self.stats) 1479 retval = self.modules[key].Execute(self.stats) 1480 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1481

1482 - def Finish(self):

1483 for key in self.modules.keys(): 1484 retval = self.modules[key].Finish(self.stats) 1485 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1486 1487

1488 -class I3PostTray(I3PreTray):

1489 """ 1490 This class provides an interface for post-processing files in iceprod 1491 """

1492 - def __init__(self):

1493 I3PreTray.__init__(self) 1494 self.logger = logging.getLogger('I3PostTray')

1495

1496 - def Execute(self):

1497 for key in self.modules.keys(): 1498 self.logger.info("executing Post %s"%key) 1499 retval = self.modules.pop(key).Execute(self.stats) 1500 if not retval == 0: raise Exception, "%s returned %d" % (key,retval)

1501 1502

1503 -class I3Tarball:

1504 """ 1505 Holder for metaproject tarball information 1506 """

1507 - def __init__(self):

1508 self.name = None 1509 self.version = None 1510 self.path = None 1511 self.url = None 1512 self.md5sum = None 1513 self.filebase = None 1514 self.suffix = None 1515 self.platform = None

1516

1517 -class I3Cookie:

1518 """ 1519 Cookie for tracking datasets on client 1520 """

1521 - def __init__(self):

1522 self.plugin = None 1523 self.batchsys = 'condor.Condor' 1524 self.dataset_id = 0 1525 self.queue_id = 0 1526 self.job_ids = [] 1527 self.url = "https://condor.icecube.wisc.edu:9080"

1528

1529 - def GetJobIds(self):

1530 return self.job_ids

1531

1532 - def AddJobId(self,id):

1533 self.job_ids.append(id)

1534

1535 - def GetClusterId(self):

1536 if self.dataset_id: 1537 return self.dataset_id 1538 elif self.job_ids: 1539 return self.job_ids[0] 1540 else: 1541 return 0

1542

1543 - def CheckQ(self,db):

1544 """ 1545 Querie status of cluster or job on condor queue 1546 """ 1547 from iceprod.server import dbqueue,plugins 1548 if self.dataset_id: 1549 cluster = dbqueue.I3DBQueue() 1550 else: 1551 cluster = plugins.get_plugin(self.batchsys)() 1552 cluster.dataset_id = self.dataset_id 1553 for job in self.job_ids: 1554 cluster.job_ids.append(job) 1555 return cluster.CheckQ(db)

1556

1557 - def QRemove(self,db):

1558 """ 1559 Remove cluster or job from condor queue 1560 """ 1561 from iceprod.server import dbqueue,plugins 1562 if self.dataset_id: 1563 cluster = dbqueue.I3DBQueue() 1564 else: 1565 cluster = plugins.get_plugin(self.batchsys)() 1566 cluster.dataset_id = self.dataset_id 1567 for job in self.job_ids: 1568 cluster.job_ids.append(job) 1569 return cluster.QRemove(db)

1570

1571 -class HostInfo:

1572 """ 1573 Information about host 1574 """

1575 - def __init__(self):

1576 self.id = '' 1577 self.hostname = '' 1578 self.ip = '' 1579 self.grid = '' 1580 self.platform = ''

1581

Source Code for Module iceprod.core.dataclasses