iceprod.modules.lfn

1 #!/bin/env python 2 # 3 """ 4 Interface for configuring pre/post icetray scripts 5 6 copyright (c) 2009 the icecube collaboration 7 8 @version: $Revision: $ 9 @date: $Date: $ 10 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 11 @author: Fabian Clevermann <fabian.clevermann@udo.edu> 12 """ 13 14 import os,sys 15 import time 16 import string 17 import glob 18 import commands 19 from os.path import expandvars,basename 20 from ipmodule import IPBaseClass 21 from iceprod.core.inventory import FileInventory 22 import xmlrpclib 23 import logging 24 25

26 -class LFN_CR_Copy(IPBaseClass):

27 """ 28 This class provides an interface for preprocessing files in iceprod 29 """

30 - def __init__(self):

31 IPBaseClass.__init__(self) 32 self.executable = 'lcg-cr' 33 self.AddParameter('destinationPATH','Path on the SE where the files go.','') # lfn:/grid/icecube/organizing/folders/output.file 34 self.AddParameter('source','source URL to copy','') # file:$RUNDIR/output.file 35 self.AddParameter('destination','On this SE will the files be copied. Leave empty or you get problems if this SE is down/full.','') 36 self.AddParameter('ldpath','library path to lfn','') 37 self.AddParameter('path','path to lfn bin directory','') 38 self.AddParameter('opts','lcg-cr options',['--vo icecube']) 39 self.AddParameter('executable','name of lcg-cr executable','lcg-cr') 40 self.AddParameter('inventory','File with source dest mappings','$I3_TOPDIR/inventory.xml') 41 self.AddParameter('emulate',"Don't actually transfer files. Just write inventory",False) 42 self.logger = logging.getLogger('iceprod::lcg-cr_Copy')

43 44

45 - def Execute(self,stats):

46 if not IPBaseClass.Execute(self,stats): return 0 47 src = self.GetParameter('source') 48 destinationSE = self.GetParameter('destination') 49 destinationPATH = self.GetParameter('destinationPATH') 50 opts = self.GetParameter('opts') 51 ldpath = self.GetParameter('ldpath') 52 path = self.GetParameter('path') 53 inventory= self.GetParameter('inventory') 54 emulate = self.GetParameter('emulate') 55 exe = self.GetParameter('executable') 56 exe = os.path.join(path,exe) 57 58 oi = FileInventory() 59 inventory = expandvars(inventory) 60 if os.path.exists(inventory): 61 oi.Read(inventory) 62 63 64 os.putenv('LD_LIBRARY_PATH',expandvars("%s:$LD_LIBRARY_PATH" % ldpath)) 65 os.putenv('PATH',expandvars("%s:$PATH" % path)) 66 67 # lcg-cr --vo icecube [-d $SE] -l lfn:/grid/icecube/$NAME/here/the/new/file.txt file:/here/the/old/local/file.txt 68 cmd = [] 69 cmd.append(exe) 70 cmd.extend(opts) 71 72 if destinationSE: 73 cmd.append("-d %s"% destinationSE) 74 cmd.append("-l %s"% destinationPATH) 75 cmd.append(src) 76 cmd = " ".join(cmd) 77 78 SE_Path = os.path.split(destinationPATH)[0][4:] # The Path on the SE without lfn: and without the filename 79 os.system("export LFC_HOST=`lcg-infosites --vo icecube lfc`") # needed for lfc-mkdir 80 os.system("export LCG_GFAL_INFOSYS=grid-bdii.desy.de:2170") # also needed 81 82 if not emulate: 83 status, output = commands.getstatusoutput("lfc-mkdir -p " + SE_Path) 84 if status: 85 self.logger.error("Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output)) 86 raise Exception, "Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output) 87 else: 88 status, output = commands.getstatusoutput(cmd) 89 if status: 90 self.logger.error("Failed to execute command '%s',%s" % (cmd,output)) 91 raise Exception, "Failed to execute command '%s',%s" % (cmd,output) 92 93 oi.AddFile(src,dest) 94 oi.Write(inventory) 95 self.logger.info(output) 96 return 0

97 98

99 -class LFN_CR_GlobCopy(LFN_CR_Copy):

100 """ 101 This class provides an interface for preprocessing files in iceprod 102 """ 103

104 - def __init__(self):

105 LFN_CR_Copy.__init__(self) 106 self.logger = logging.getLogger('iceprod::URLGlobCopy')

107 108

109 - def Execute(self,stats):

110 if not IPBaseClass.Execute(self,stats): return 0 111 src = self.GetParameter('source') 112 destinationSE = self.GetParameter('destination') 113 destinationPATH = self.GetParameter('destinationPATH') 114 opts = self.GetParameter('opts') 115 ldpath = self.GetParameter('ldpath') 116 path = self.GetParameter('path') 117 inventory= self.GetParameter('inventory') 118 emulate = self.GetParameter('emulate') 119 exe = self.GetParameter('executable') 120 exe = os.path.join(path,exe) 121 122 123 os.putenv('LD_LIBRARY_PATH',os.path.expandvars("%s:$LD_LIBRARY_PATH" % ldpath)) 124 os.putenv('PATH',os.path.expandvars("%s:$PATH" % path)) 125 126 oi = FileInventory() 127 inventory = expandvars(inventory) 128 if os.path.exists(inventory): 129 oi.Read(inventory) 130 131 retval = 0 132 133 for file in glob.glob(expandvars(src.replace('file:',''))): 134 135 oi.AddFile(file,dest) 136 cmd = [] 137 cmd.append(exe) 138 cmd.extend(opts) 139 if destinationSE: 140 cmd.append("-d %s"% destinationSE) 141 cmd.append("-l %s"% destinationPATH) 142 cmd.append('file:'+os.path.abspath(os.path.normpath(file))) 143 144 cmd = " ".join(cmd) 145 SE_Path = os.path.split(destinationPATH)[0][4:] # The Path on the SE without lfn: and without the filename 146 os.system("export LFC_HOST=`lcg-infosites --vo icecube lfc`") # needed for lfc-mkdir 147 os.system("export LCG_GFAL_INFOSYS=grid-bdii.desy.de:2170") # also needed 148 if not emulate: 149 self.logger.info(cmd) 150 status, output = commands.getstatusoutput("lfc-mkdir -p " + SE_Path) 151 if status: 152 self.logger.error("Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output)) 153 raise Exception, "Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output) 154 else: 155 status, output = commands.getstatusoutput(cmd) 156 self.logger.info(output) 157 if status: 158 self.logger.error("Failed to execute command '%s',%s" % (cmd,output)) 159 raise Exception, "Failed to execute command '%s',%s" % (cmd,output) 160 161 oi.Write(inventory) 162 return 0

163 164

165 -class LFN_CR_MultiCopy(LFN_CR_Copy):

166 """ 167 This class provides an interface for preprocessing files in iceprod 168 """ 169

170 - def __init__(self):

171 LFN_CR_Copy.__init__(self) 172 self.logger = logging.getLogger('iceprod::URLMultiCopy') 173 self.AddParameter('sourcelist','list of source URLs (files) to copy','')

174 175

176 - def Execute(self,stats):

177 if not IPBaseClass.Execute(self,stats): return 0 178 src = self.GetParameter('sourcelist') 179 destinationSE = self.GetParameter('destination') 180 destinationPATH = self.GetParameter('destinationPATH') 181 opts = self.GetParameter('opts') 182 ldpath = self.GetParameter('ldpath') 183 path = self.GetParameter('path') 184 inventory= self.GetParameter('inventory') 185 emulate = self.GetParameter('emulate') 186 exe = self.GetParameter('executable') 187 exe = os.path.join(path,exe) 188 189 190 os.putenv('LD_LIBRARY_PATH',os.path.expandvars("%s:$LD_LIBRARY_PATH" % ldpath)) 191 os.putenv('PATH',os.path.expandvars("%s:$PATH" % path)) 192 193 oi = FileInventory() 194 inventory = expandvars(inventory) 195 if os.path.exists(inventory): 196 oi.Read(inventory) 197 198 retval = 0 199 for file in src: 200 oi.AddFile(file,dest) 201 cmd = [] 202 cmd.append(exe) 203 cmd.extend(opts) 204 if destinationSE: 205 cmd.append("-d %s"% destinationSE) 206 cmd.append("-l %s"% destinationPATH) 207 cmd.append('file:'+os.path.abspath(os.path.normpath(file))) 208 209 cmd = " ".join(cmd) 210 SE_Path = os.path.split(destinationPATH)[0][4:] # The Path on the SE without lfn: and without the filename 211 os.system("export LFC_HOST=`lcg-infosites --vo icecube lfc`") # needed for lfc-mkdir 212 os.system("export LCG_GFAL_INFOSYS=grid-bdii.desy.de:2170") # also needed 213 if not emulate: 214 self.logger.info(cmd) 215 status, output = commands.getstatusoutput("lfc-mkdir -p " + SE_Path) 216 if status: 217 self.logger.error("Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output)) 218 raise Exception, "Failed to execute command 'lfc-mkdir -p %s',%s" % (SE_Path,output) 219 else: 220 status, output = commands.getstatusoutput(cmd) 221 self.logger.info(output) 222 if status: 223 self.logger.error("Failed to execute command '%s',%s" % (cmd,output)) 224 raise Exception, "Failed to execute command '%s',%s" % (cmd,output) 225 226 oi.Write(inventory) 227 return 0

228

229 -class LFN_CP_Copy(IPBaseClass):

230 """ 231 This class provides an interface for preprocessing files in iceprod 232 """

233 - def __init__(self):

234 IPBaseClass.__init__(self) 235 self.executable = 'lcg-cp' 236 self.AddParameter('destinationPATH','Path where the files go.','') # /here/the/new/local/output.file 237 self.AddParameter('source','source on the SE to copy','') # lfn:/grid/icecube/$name/output.file 238 # self.AddParameter('destination','destination URL to copy to','') 239 self.AddParameter('ldpath','library path to lfn','') 240 self.AddParameter('path','path to lfn bin directory','') 241 self.AddParameter('opts','lcg-cr options',['-v','--vo icecube']) 242 self.AddParameter('executable','name of lcg-cr executable','lcg-cp') 243 self.AddParameter('inventory','File with source dest mappings','$I3_TOPDIR/inventory.xml') 244 self.AddParameter('emulate',"Don't actually transfer files. Just write inventory",False) 245 self.logger = logging.getLogger('iceprod::lcg-cp_Copy')

246 247

248 - def Execute(self,stats):

249 if not IPBaseClass.Execute(self,stats): return 0 250 src = self.GetParameter('source') 251 # destinationSE = self.GetParameter('destination') 252 destinationPATH = self.GetParameter('destinationPATH') 253 opts = self.GetParameter('opts') 254 ldpath = self.GetParameter('ldpath') 255 path = self.GetParameter('path') 256 inventory= self.GetParameter('inventory') 257 emulate = self.GetParameter('emulate') 258 exe = self.GetParameter('executable') 259 exe = os.path.join(path,exe) 260 261 oi = FileInventory() 262 inventory = expandvars(inventory) 263 if os.path.exists(inventory): 264 oi.Read(inventory) 265 266 267 os.putenv('LD_LIBRARY_PATH',expandvars("%s:$LD_LIBRARY_PATH" % ldpath)) 268 os.putenv('PATH',expandvars("%s:$PATH" % path)) 269 270 # lcg-cp --vo icecube -v lfn:/grid/icecube/$NAME/here/the/old/output.file file:/here/the/new/local/output.file 271 cmd = [] 272 cmd.append(exe) 273 cmd.extend(opts) 274 # cmd.append("-d %s"% destinationSE) 275 cmd.append(src) 276 cmd.append("file:%s"% destinationPATH) 277 278 cmd = " ".join(cmd) 279 280 destinationPATH = os.path.split(destinationPATH)[0] # remove filename for mkdir 281 if not emulate: 282 status, output = commands.getstatusoutput("mkdir -p %s"% destinationPATH) 283 if status: 284 self.logger.error("Failed to execute command 'mkdir -p %s',%s" % (destinationPATH,output)) 285 raise Exception, "Failed to execute command 'mkdir -p %s',%s" % (destinationPATH,output) 286 else: 287 status, output = commands.getstatusoutput(cmd) 288 if status: 289 self.logger.error("Failed to execute command '%s',%s" % (cmd,output)) 290 raise Exception, "Failed to execute command '%s',%s" % (cmd,output) 291 292 oi.AddFile(src,dest) 293 oi.Write(inventory) 294 self.logger.info(output) 295 return 0

296

Source Code for Module iceprod.modules.lfn