Source Code for Module iceprod.modules.fileutils

  1   
  2  #!/bin/env python 
  3  # 
  4  """ 
  5   Interface for configuring pre/post icetray scripts 
  6   
  7   copyright  (c) 2005 the icecube collaboration 
  8   
  9   @version: $Revision: $ 
 10   @date: $Date: $ 
 11   @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu> 
 12  """ 
 13   
 14  import os 
 15  import re 
 16  import sys 
 17  import math 
 18  import dircache 
 19  import time 
 20  import string 
 21  import shutil 
 22  import cPickle 
 23  from ipmodule import IPBaseClass 
 24  import logging 
 25   
 26 -class RenameFile(IPBaseClass): 
 27      """ 
 28      This class provides an interface for preprocessing files in iceprod 
 29      """ 
 30   
 31 -    def __init__(self): 
 32          IPBaseClass.__init__(self) 
 33          self.AddParameter('infile','Name of file you want to rename','') 
 34          self.AddParameter('outfile','What you want to rename the file to','') 
 35          self.logger = logging.getLogger('iceprod::RenameFile') 
 36   
 37   
 38 -    def Execute(self,stats): 
 39          if not IPBaseClass.Execute(self,stats): return 0 
 40          infile = self.GetParameter('infile') 
 41          outfile = self.GetParameter('outfile') 
 42          cmd = "mv %s %s" % (infile,outfile) 
 43          retval = os.system(cmd) 
 44          if retval == 0: 
 45              return retval 
 46          else: 
 47              self.logger.error("Failed to execute command '%s'" % cmd) 
 48              raise Exception, "Failed to execute command '%s'" % cmd 
 49   
 50   
 51 -class RenameMultipleFiles(IPBaseClass): 
 52      """ 
 53      This class provides functionality for renaming multiple files using Bash 
 54      substitutions. 
 55      """ 
 56   
 57 -    def __init__(self): 
 58          IPBaseClass.__init__(self) 
 59          self.AddParameter('inpattern','Pattern to match against file names when checking for files to rename','') 
 60          self.AddParameter('outpattern','Bash substitution expression to be applied to filenames when output','') 
 61          self.logger = logging.getLogger('iceprod::RenameMultipleFiles') 
 62   
 63   
 64 -    def Execute(self,stats): 
 65          if not IPBaseClass.Execute(self,stats): return 0 
 66          inpattern  = self.GetParameter('inpattern') 
 67          outpattern = self.GetParameter('outpattern') 
 68          cmd = "for i in %s; do mv $i ${i/%s}; done" % (inpattern, outpattern) 
 69          retval = os.system(cmd) 
 70          if retval == 0: 
 71              return retval 
 72          else: 
 73              msg = "Failed to execute command '%s'" % cmd 
 74              self.logger.error(msg) 
 75              raise Exception, msg 
 76   
 77   
 78 -class RemoveFiles(IPBaseClass): 
 79      """ 
 80      This class provides an interface for preprocessing files in iceprod 
 81      """ 
 82   
 83 -    def __init__(self): 
 84          IPBaseClass.__init__(self) 
 85          self.AddParameter('filelist','The names of the files you want to delete',[]) 
 86          self.AddParameter('IgnoreError','ignore error from cmd',False) 
 87          self.logger = logging.getLogger('iceprod::RemoveFiles') 
 88   
 89   
 90 -    def Execute(self,stats): 
 91          if not IPBaseClass.Execute(self,stats): return 0 
 92          filelist = self.GetParameter('filelist') 
 93          retval = 0 
 94          cmd = '' 
 95          for file in filelist: 
 96              cmd = "rm -f %s" % file 
 97              retval += os.system(cmd) 
 98          if retval == 0: 
 99              return retval 
100          else: 
101              self.logger.error("Failed to execute command '%s'" % cmd) 
102              raise Exception, "Failed to execute command '%s'" % cmd 
103   
104   
105 -class ChmodFile(IPBaseClass): 
106      """ 
107      This class provides an interface for changing permissions of a file in iceprod 
108      """ 
109   
110 -    def __init__(self): 
111          IPBaseClass.__init__(self) 
112          self.AddParameter('file','Name of file you want to chmod','') 
113          self.AddParameter('flags','Flags to pass to chmod','g+w') 
114          self.logger = logging.getLogger('iceprod::ChmodFile') 
115   
116   
117 -    def Execute(self,stats): 
118          if not IPBaseClass.Execute(self,stats): return 0 
119          file = self.GetParameter('file') 
120          flags = self.GetParameter('flags') 
121          cmd = "chmod %s %s" % (flags,file) 
122          retval = os.system(cmd) 
123          if retval == 0: 
124              return retval 
125          else: 
126              self.logger.error("Failed to execute command '%s'" % cmd) 
127              raise Exception, "Failed to execute command '%s'" % cmd 
128   
129   
130 -class CompressFile(IPBaseClass): 
131      """ 
132      This class provides an interface for compressing files in iceprod 
133      """ 
134   
135 -    def __init__(self): 
136          IPBaseClass.__init__(self) 
137          self.AddParameter('infile','Name of file you want to compress','') 
138          self.AddParameter('outfile','Name of output file','') 
139          self.logger = logging.getLogger('iceprod::CompressFile') 
140   
141   
142 -    def Execute(self,stats): 
143          if not IPBaseClass.Execute(self,stats): return 0 
144          infile = self.GetParameter('infile') 
145          outfile = self.GetParameter('outfile') 
146          cmd = "gzip -f -q -6 -c %s > %s" % (infile,outfile) 
147          retval = os.system(cmd) 
148          if retval == 0: 
149              return retval 
150          else: 
151              self.logger.error("Failed to execute command '%s'" % cmd) 
152              raise Exception, "Failed to execute command '%s'" % cmd 
153   
154 -class Tarball(IPBaseClass): 
155      """ 
156      This class provides an interface for preprocessing files in iceprod 
157      """ 
158   
159 -    def __init__(self): 
160          IPBaseClass.__init__(self) 
161          self.AddParameter('filelist','The names of the files you want to delete',[]) 
162          self.AddParameter('outputfile','Name of output tarball',"Tarball.tar") 
163          self.logger = logging.getLogger('iceprod::Tarball') 
164   
165   
166 -    def Execute(self,stats): 
167          if not IPBaseClass.Execute(self,stats): return 0 
168          filelist = self.GetParameter('filelist') 
169          tar  = self.GetParameter('outputfile') 
170          retval = 0 
171          cmd = '' 
172          if isinstance(filelist,str): 
173              # not a list, just a single file 
174              cmd = "tar" 
175              if not os.path.exists(tar): 
176                 cmd += " -cf" 
177              else: 
178                 cmd += " -uf" 
179              cmd += " %s %s" %(tar,filelist) 
180              self.logger.debug(cmd) 
181              retval += os.system(cmd) 
182          else: 
183              # iterate over list 
184              for file in filelist: 
185                  dir = os.path.dirname(file) 
186                  cmd = "tar" 
187                  if os.path.isdir(dir): 
188                     cmd += " -C%s " % dir 
189                  if not os.path.exists(tar): 
190                     cmd += " -cf" 
191                  else: 
192                     cmd += " -uf" 
193                  cmd += " %s %s" %(tar,os.path.basename(file)) 
194                  self.logger.debug(cmd) 
195                  retval += os.system(cmd) 
196          if retval == 0: 
197              return retval 
198          else: 
199              self.logger.error("Failed to execute command '%s'" % cmd) 
200              raise Exception, "Failed to execute command '%s'" % cmd 
201   
202   
203 -class SummaryMerger(IPBaseClass): 
204      """ 
205      This class provides an interface for preprocessing files in iceprod 
206      """ 
207   
208 -    def __init__(self): 
209          IPBaseClass.__init__(self) 
210          self.AddParameter('inputfilelist','The names of the files you want to merge',[]) 
211          self.AddParameter('outputfile','Name of output tarball',"summary.xml") 
212          self.logger = logging.getLogger('iceprod::SummaryMerger') 
213   
214   
215 -    def Execute(self,stats): 
216          if not IPBaseClass.Execute(self,stats): return 0 
217          from iceprod.core.lex import XMLSummaryParser 
218   
219          filelist = self.GetParameter('inputfilelist') 
220          outfile  = self.GetParameter('outputfile') 
221          retval = 0 
222          cmd = '' 
223          summary = XMLSummaryParser() 
224          summary.summary_map = stats 
225          for file in filelist: 
226              stats = XMLSummaryParser().ParseFile(file) 
227              for key,value in stats.items(): 
228                  if summary.summary_map.has_key(key): 
229                      summary.summary_map[key] += value 
230                  else: 
231                      summary.summary_map[key]  = value 
232          summary.Write(outfile) 
233          return 0 
234