iceprod.core.tray2xml

1 #! /usr/bin/env python 2 # 3 import os,sys,os.path 4 import types 5 from dataclasses import * 6 from xmlwriter import * 7 from os.path import expandvars 8 # this needs to be in the client 9 from iceprod.core.paramdb import * 10 from iceprod.core.inspectv2 import * 11 from iceprod.core import functions 12 import dataclasses 13 import logger 14 import iceprod.core.logger 15 16 libs = [] 17 _outfile = sys.argv[-1] 18 sys.argv = sys.argv[:-1] # remove last entry 19 20 _paramdb = XMLParamDB() 21 _pdbcache = dict() 22 _metaproject_name = None 23 _metaproject_version = None 24 25 logger = logging.getLogger('tray2xml') 26

27 -def setmetaproject(meta):

28 global _metaproject_name 29 global _metaproject_version 30 meta = meta.split(".") 31 _metaproject_name = meta[0] 32 _metaproject_version = meta

33

34 -def getmetaproject():

35 """ 36 Extract metaproject name from svn 37 @return: name 38 """ 39 global _metaproject_name 40 if _metaproject_name: 41 return _metaproject_name 42 43 revision = "0" 44 handle = os.popen("svn info " + expandvars("$I3_SRC")) 45 for line in handle.readlines(): 46 if line.strip().startswith("URL"): 47 url = line.split("/") 48 for i in range(len(url)): 49 if url[i].startswith("meta-projects"): 50 _metaproject_name = url[i+1] 51 return _metaproject_name

52

53 -def getversion(project,metaproject=False):

54 """ 55 Extract version tuple from project 56 @param project: name of project 57 @return: integer tuple corresponding to project version 58 """ 59 global _metaproject_version 60 if _metaproject_version and not project: 61 return _metaproject_version 62 version_regex = r'http://.+projects/.+' 63 64 if metaproject: 65 svn_projectfile = os.popen('svn info ' + expandvars('$I3_SRC') , 'r' ) 66 else: 67 svn_projectfile = os.popen('svn info ' + expandvars('$I3_SRC/') + project, 'r' ) 68 svn_data = svn_projectfile.read() 69 70 version = "V00-00-00" 71 revision = "Revision: 0" 72 version_search = re.search(version_regex,svn_data) 73 try: 74 version = version_search.group() 75 except: pass 76 revision_search = re.search(r'Revis.*:.*[0-9]+',svn_data) 77 try: 78 revision = revision_search.group() 79 except: pass 80 81 version = re.sub(r'http://.+projects/%s/' % project,'',version) 82 version = version.replace('/','.') 83 revision = revision.split(':')[-1].strip() 84 return version

85 86

87 -def mkversiontuple(versiontxt,revision):

88 """ 89 Convert text version number into an integer tuple 90 @param versiontxt: text containing version information 91 @return: integer tuple 92 """ 93 94 vtuple = ('0',)*3 95 if versiontxt.startswith('trunk'): 96 return ('-1','0',str(revision)) 97 98 pver = re.sub(r'[A-Z]','',versiontxt) 99 pver = pver.split("-")[0:3] 100 if len(pver) < 3: return vtuple 101 try: 102 pver = tuple(pver) 103 except: 104 os._exit(1) 105 print >> sys.stderr, pver 106 return pver

107 108 109

110 -def loadpdb(url=None):

111 global _paramdb 112 _cache = expandvars('$I3_BUILD/inspect_cache.xml') 113 if url : # use the server instead of cache 114 _paramdb = RPCParamDB(url) 115 logger.info("pdb access through %s" % url) 116 return 117 elif not os.path.exists(_cache): 118 _paramdb = XMLParamDB() 119 srcpath = expandvars("$I3_SRC") 120 libdir = os.path.join(expandvars("$I3_BUILD"),"lib") 121 i3inspect = IceTrayInspectV2(srcpath,libdir) 122 logger.info("inspect",srcpath) 123 i3inspect.InspectProjects(expandvars('$I3_SRC')) 124 logger.info("writing",_cache) 125 i3inspect.toXML(_cache) 126 logger.info("loading",_cache) 127 _paramdb.loadfile(_cache)

128 129

130 -def load(library):

131 libs.append(library[3:])

132

133 -def gettype(object):

134 135 if isinstance(object,T2XType): 136 return object.type 137 elif type(object) is types.IntType: 138 return "int" 139 elif type(object) is types.FloatType: 140 return "float" 141 elif type(object) is types.StringType: 142 return "string" 143 elif type(object) is types.BooleanType: 144 return "bool" 145 elif isinstance(object,OMKey): 146 return "OMKey" 147 elif isinstance(object,list) and len(object) > 0: 148 return gettype(object[0])+'v' 149 else: 150 return 'NaT'

151

152 -def getval(object):

153 if isinstance(object,T2XType): 154 return object.value 155 else: return object

156 157

158 -def parse_parameter(value):

159 if not repr(value) == 'OMKey': 160 return Value(value) 161 elif repr(value) == 'OMKey': 162 return pyOMKey(getval(value.stringid),getval(value.omid)) 163 else: 164 print >> sys.stderr,repr(value) 165 os._exit(1) 166 167 return None

168 169

170 -def make_parameter(name,value):

171 vect = False 172 p = Parameter() 173 p.SetName(name) 174 try: 175 p.SetType(gettype(value)) 176 except UndefinedTypeException,e: 177 sys.excepthook(sys.exc_type,sys.exc_value,sys.exc_traceback) 178 logger.error(e) 179 print >> sys.stderr, name,type(value),e 180 181 if p.GetType() in VectorTypes: 182 p.SetValue(map(parse_parameter,map(getval,value))) 183 else: 184 p.SetValue(parse_parameter(getval(value))) 185 return p

186

187 -class T2XType:

188

189 - def __init__(self,type,value):

190 self.type = type 191 self.value = value

192

193 - def __mul__(self,u):

194 return T2XType(self.type,self.value+'*'+repr(u))

195 196

197 -class OMKey(pyOMKey):

198 - def __init__(self,stringid=0,omid=0):

199 self.stringid=stringid 200 self.omid=omid

201 - def __repr__(self): return 'OMKey'

202 - def __str__(self): return 'OMKey(%d,%d)' % (self.stringid,self.omid)

203

204 -class I3Tray:

205 206

207 - def __init__(self):

208 global _outfile 209 if not _paramdb.cached: loadpdb() 210 self.steering = Steering() 211 self.tray = IceTrayConfig() 212 self.connect_boxes = True 213 mname = getmetaproject() 214 m = MetaProject() 215 m.SetName(mname) 216 mver = getversion(mname,True) 217 m.SetVersion(mver) 218 _paramdb.SetMetaProjectId(m) 219 self.tray.AddMetaProject(mname, m) 220 221 for lib in libs: 222 for p in _paramdb.GetProjects(m.GetId()): 223 if p.GetName() == lib: 224 m.AddProject(lib, p)

225

226 - def AddIceProdPre(self,pre):

227 self.tray.AddIceProdPre(pre)

228

229 - def AddIceProdPost(self,post):

230 self.tray.AddIceProdPost(post)

231

232 - def MissingProject(self,module,metaproject):

233 if _pdbcache.has_key(module.GetClass()): 234 project = _pdbcache[module.GetClass()] 235 module.AddProject(project.GetName(),project) 236 return 237 print "missing project for class '%s'" % module.GetClass() 238 projectlist = map(Project.GetName,metaproject.GetProjectList()) 239 projectlist.append('[Specify a missing project]') 240 projectname = functions.get_choice(projectlist,"choose the project for '%s':" % module.GetClass()) 241 if projectname == '[Specify a missing project]': 242 sys.stdout.write('type the name of the project: ') 243 projectname = sys.stdin.readline().strip() 244 if metaproject.HasProject(projectname): 245 project = metaproject.GetProject(projectname) 246 module.AddProject(project.GetName(),project) 247 else: 248 for project in _paramdb.GetProjects(metaproject.GetId()): 249 if project.GetName() == projectname: 250 metaproject.AddProject(project.GetName(),project) 251 module.AddProject(project.GetName(),project) 252 break 253 _pdbcache[module.GetClass()] = project

254

255 - def AddService(self,classname,objname):

256 global _paramdb 257 s = Service() 258 s.SetClass(classname) 259 s.SetName(objname) 260 mp = self.tray.GetMetaProjects().values()[0] 261 for project in _paramdb.GetProjectsSM(s,mp): 262 for d in _paramdb.GetProjectDependencies(project.GetId(),mp.GetId()): 263 if not mp.HasProject(d.GetName()): 264 mp.AddProject(d.GetName(),d) 265 project.AddDependency(d.GetName()) 266 s.AddProject(project.GetName(),project) 267 if not mp.HasProject(project.GetName()): 268 mp.AddProject(project.GetName(),project) 269 break 270 if not s.GetProjectList(): 271 self.MissingProject(s,mp) 272 self.tray.AddService(s) 273 setpar = lambda (param,val): s.AddParameter(make_parameter(param,val)) 274 return lambda *pair: map(setpar,pair)

275

276 - def AddModule(self,classname,objname):

277 global _paramdb 278 m = Module() 279 m.SetClass(classname) 280 m.SetName(objname) 281 mp = self.tray.GetMetaProjects().values()[0] 282 for project in _paramdb.GetProjectsMM(m,mp): 283 for d in _paramdb.GetProjectDependencies(project.GetId(),mp.GetId()): 284 if not mp.HasProject(d.GetName()): 285 mp.AddProject(d.GetName(),d) 286 project.AddDependency(d.GetName()) 287 m.AddProject(project.GetName(),project) 288 if not mp.HasProject(project.GetName()): 289 mp.AddProject(project.GetName(),project) 290 break 291 if not m.GetProjectList(): 292 self.MissingProject(m,mp) 293 self.tray.AddModule(m) 294 setpar = lambda (param,val): m.AddParameter(make_parameter(param,val)) 295 return lambda *pair: map(setpar,pair)

296

297 - def SetParameter(self,modname,param,value):

298 299 if self.tray.HasService(modname): 300 self.tray.GetService(modname).AddParameter(make_parameter(param,value)) 301 elif self.tray.HasModule(modname): 302 self.tray.GetModule(modname).AddParameter(make_parameter(param,value))

303

304 - def ConnectBoxes(self,outmod,outbox,inmod,inbox="default"):

305 self.connect_boxes = False 306 conn = Connection() 307 conn.From(outmod,outbox) 308 conn.To(inmod,inbox) 309 self.tray.AddConnection(conn)

310

311 - def Execute(self,frames=0):

312 outmod = None 313 # if self.connect_boxes: 314 # for module in self.tray.GetModules(): 315 # if outmod: 316 # inmod = module.GetName() 317 # conn = Connection() 318 # conn.From(outmod,"OutBox") 319 # conn.To(inmod,"default") 320 # self.tray.AddConnection(conn) 321 # outmod = module.GetName() 322 323 for mp in self.tray.GetMetaProjects().values(): 324 for project in mp.GetProjects().values(): 325 logger.debug("dependencies for %d, %s:" % (project.GetId(),project.GetName())) 326 for d in _paramdb.GetProjectDependencies(project.GetId(),mp.GetId()): 327 logger.debug("\t%s" % d.GetName()) 328 if not mp.HasProject(d.GetName()): 329 mp.AddProject(d.GetName(),d) 330 project.AddDependency(d.GetName()) 331 if frames: 332 self.tray.SetEvents(frames) 333 return self.tray

334

335 - def Finish(self):

336 self.steering.AddTray(self.tray) 337 writer = IceTrayXMLWriter(self.steering) 338 if self.output == '-': 339 writer.write_to_file() 340 else: 341 writer.write_to_file(self.output)

342

343 - def SetSteering(self,steering):

344 self.steering = steering

345

346 -class I3PreTray(dataclasses.I3PreTray):

347

348 - def __init__(self,tray):

349 dataclasses.I3PreTray.__init__(self) 350 self.tray = tray

351

352 - def AddModule(self,type,name):

353 """ 354 Add entry for module 355 """ 356 pre = IceProdPre() 357 pre.SetName(name) 358 pre.SetClass(type) 359 self.tray.AddIceProdPre(pre) 360 self.modules[name] = pre 361 self.last_added = name 362 return self

363

364 - def SetParameter(self, module, param, value):

365 """ 366 Set parameter value for plugin 367 """ 368 self.modules[module].AddParameter(make_parameter(param,value)) 369 return self

370

371 - def __call__(self, *args):

372 for pair in args: 373 self.modules[self.last_added].AddParameter(make_parameter(pair[0],pair[1])) 374 return self

375

376 - def Execute(self): return None

377

378 -class I3PostTray(dataclasses.I3PostTray):

379

380 - def __init__(self,tray):

381 dataclasses.I3PostTray.__init__(self) 382 self.tray = tray

383

384 - def AddModule(self,type,name):

385 """ 386 Add entry for module 387 """ 388 post = IceProdPost() 389 post.SetName(name) 390 post.SetClass(type) 391 self.tray.AddIceProdPost(post) 392 self.modules[name] = post 393 self.last_added = name 394 return self

395

396 - def SetParameter(self, module, param, value):

397 """ 398 Set parameter value for plugin 399 """ 400 self.modules[module].AddParameter(make_parameter(param,value)) 401 return self

402

403 - def __call__(self, *args):

404 for pair in args: 405 self.modules[self.last_added].AddParameter(make_parameter(pair[0],pair[1])) 406 return self

407

408 - def Execute(self): return None

409

Source Code for Module iceprod.core.tray2xml