Source Code for Module iceprod.modules.i3_data

  1  #!/bin/env python 
  2  # 
  3   
  4   
  5  import os,sys 
  6  #sys.path.insert(0,'usr/lib/python2.3/site-packages/') 
  7  #import MySQLdb 
  8  import time 
  9  import string 
 10  import glob 
 11  import cPickle 
 12  import commands 
 13  from os.path import expandvars,basename 
 14  from iceprod.modules import ipmodule 
 15  from ipmodule import IPBaseClass 
 16  from iceprod.core.inventory import FileInventory 
 17  from iceprod.core import functions 
 18  import xmlrpclib 
 19  import logging 
 20  from cPickle import dumps,loads 
 21  #from MySQLdb import OperationalError 
 22  #from Queue import Queue 
 23   
 24   
 25   
 26 -class TrackURLCopy: 
 27          """ 
 28          This class provides an interface for preprocessing files in iceprod 
 29          It also tracks the destination of files in the monitoring database 
 30          throught the soapmon server. 
 31          """ 
 32           
 33   
 34 -        def __init__(self): 
 35                  URLCopy.__init__(self) 
 36                  self.logger = logging.getLogger('iceprod::TrackURLCopy') 
 37                  print "try" 
 38                  return 1 
 39               
 40   
 41   
 42  #class URLCopy(IPBaseClass): 
 43  #       """ 
 44  #       This class provides an interface for preprocessing files in iceprod 
 45  #       """ 
 46  # 
 47  #def __init__(self): 
 48  #           IPBaseClass.__init__(self) 
 49  #           self.executable = 'globus-url-copy' 
 50  #           self.AddParameter('source','source URL to copy','') 
 51  #           self.AddParameter('destination','detination URL to copy to','') 
 52  #           self.AddParameter('certdir','certificate directory','') 
 53  #           self.AddParameter('proxyfile','File path to globus proxy','$X509_USER_PROXY') 
 54  #           self.AddParameter('ldpath','library path to globus','globus/lib') 
 55  #           self.AddParameter('path','path to globus bin directory','globus/bin') 
 56  #           self.AddParameter('opts','globus-url-copy options', 
 57  #                       ['-rst','-cd','-r','-nodcau','-rst-retries 5','-rst-interval 60'] 
 58  #                      ) 
 59  #           self.AddParameter('executable','name of gridFTP executable','globus-url-copy') 
 60  #           self.AddParameter('StorageElement','LFN SE','') 
 61  #           self.AddParameter('lfn-opts','LFN Options','') 
 62  #           self.AddParameter('inventory','File with source dest mappings','$I3_TOPDIR/inventory.xml') 
 63  #           self.AddParameter('emulate',"Don't actually transfer files. Just write inventory",False) 
 64  #           self.logger = logging.getLogger('iceprod::URLCopy') 
 65  # 
 66  # 
 67  #       def Execute(self,stats): 
 68  #           if not IPBaseClass.Execute(self,stats): return 0 
 69  #           src      = self.GetParameter('source') 
 70  #           dest     = self.GetParameter('destination') 
 71  #           proxy    = self.GetParameter('proxyfile') 
 72  #           opts     = self.GetParameter('opts') 
 73  #           ldpath   = self.GetParameter('ldpath') 
 74  #           path     = self.GetParameter('path') 
 75  #           inventory= self.GetParameter('inventory') 
 76  #           emulate  = self.GetParameter('emulate') 
 77  #           exe      = self.GetParameter('executable') 
 78  #           se       = self.GetParameter('StorageElement') 
 79  #           lfnopts  = self.GetParameter('lfn-opts') 
 80  #           #exe      = os.path.join(path,exe) 
 81  # 
 82  #           if src.startswith('lfn:') or dest.startswith('lfn:'): 
 83  #              self.logger.info("detected LFN URL. Hading control to lfn module") 
 84  #              from lfn import LFN_CR_Copy 
 85  #              ipmod = LFN_CR_Copy() 
 86  #              ipmod.SetParameter('source',src) 
 87  #              ipmod.SetParameter('destinationPATH',dest) 
 88  #              if se: 
 89  #                 ipmod.SetParameter('destination',se) 
 90  #              if lfnopts: 
 91  #                 ipmod.SetParameter('opts',lfnopts) 
 92  #              return ipmod.Execute(stats) 
 93  #              
 94  # 
 95  #           certdir  = self.GetParameter('certdir') 
 96  #           if certdir and os.path.exists(expandvars(certdir)): 
 97  #              os.putenv('X509_CERT_DIR',expandvars(certdir)) 
 98  # 
 99  #           inventory = expandvars(inventory) 
100  #           oi = FileInventory() 
101  #           if os.path.exists(inventory): 
102  #              oi.Read(inventory) 
103  # 
104  #           os.putenv('X509_USER_PROXY',expandvars(proxy)) 
105  #           os.chmod(expandvars(proxy),0600) 
106  #           os.putenv('LD_LIBRARY_PATH',expandvars("%s:$LD_LIBRARY_PATH" % ldpath)) 
107  #           os.putenv('PATH',expandvars("%s:$PATH" % path)) 
108  # 
109  #           cmd = [] 
110  #           cmd.append(exe) 
111  #           cmd.extend(opts) 
112  #           cmd.append(src) 
113  #           cmd.append(dest) 
114  #           cmd = " ".join(cmd) 
115  # 
116  #           if not emulate: 
117  #              status, output = commands.getstatusoutput(cmd) 
118  #              if  status: 
119  #                  self.logger.error("Failed to execute command '%s',%s" % (cmd,output)) 
120  #                  raise Exception, "Failed to execute command '%s',%s" % (cmd,output) 
121  #              self.logger.info(output) 
122  #           else: 
123  #              oi.AddFile(src,dest) 
124  #              oi.Write(inventory) 
125  #           return 0 
126               
127               
128  # 
129  #class TrackURLCopy(URLCopy): 
130  #       """ 
131  #       This class provides an interface for preprocessing files in iceprod 
132  #       It also tracks the destination of files in the monitoring database 
133  #       throught the soapmon server. 
134  #       """ 
135  #       print "try" 
136  #       return 1 
137  # 
138  #def __init__(self): 
139  #           URLCopy.__init__(self) 
140  #           self.logger = logging.getLogger('iceprod::TrackURLCopy') 
141  #           self.AddParameter( 
142  #              'monitorURL', 
143  #              'soapmon url', 
144  #              'http://x2100.icecube.wisc.edu/cgi-bin/simulation/mon/soapmon-cgi') 
145  #           self.AddParameter('dataset','dataset ID',0) 
146  #           self.AddParameter('job','job ID',0) 
147  #           self.AddParameter('key','Temporary password for soapmon','') 
148  # 
149  #       def Execute(self,stats): 
150  #           url       = self.GetParameter('monitorURL') 
151  #           src       = self.GetParameter('source') 
152  #           dest      = self.GetParameter('destination') 
153  #           datasetid = int(self.GetParameter('dataset')) 
154  #           jobid     = int(self.GetParameter('job')) 
155  #           passcode  = self.GetParameter('key') 
156  #           starttime = time.time() 
157  #           if not URLCopy.Execute(self,stats): 
158  #              md5sum       = '' 
159  #              filesize     = 0. 
160  #              transfertime = time.time() - starttime 
161  # 
162  #              if src.startswith('file:'): 
163  #                 md5sum = functions.md5sum(expandvars(src.replace('file:',''))) 
164  #                 filesize = float(os.path.getsize(expandvars(src.replace('file:','')))) 
165  #              server = xmlrpclib.ServerProxy(url) 
166  #              if dest.endswith('/'): 
167  #                 dest += basename(src) 
168  #              self.logger.info('%s %s' % (dest,md5sum)) 
169  #              if not server.AddFileURL(datasetid,jobid,dest,md5sum,filesize,transfertime,passcode): 
170  #                 raise Exception, "Failed to set URL for for %s -> %s" % (src,dest) 
171  #              return 0 
172  #           return 1 
173