1
2
3
4 """
5 Module containing common functions
6
7 copyright (c) 2009 the icecube collaboration
8
9 @version: $Revision: $
10 @date: $Date: $
11 @author: Juan Carlos Diaz Velez <juancarlos@icecube.wisc.edu>
12
13 """
14
15 import sys,os,re
16 import logging
17 import os.path
18 import socket
19
20 logger = logging.getLogger('functions')
21
22
24 os.system('rm -rf %s' %path)
25
26
27 -def find(pathlist,pattern,filetype='file'):
28 """
29 Return a list of files that match a given pattern
30 @param pathlist: base directory to start search from
31 @param pattern: pattern to match
32 """
33
34 if filetype not in ['dir','path','file']:
35 logger.error('find: invalid filetype %s'%filetype)
36 matches = list()
37 regex = re.compile(r'%s'%pattern)
38
39 for root in os.path.expandvars(pathlist).split(os.pathsep):
40 if not root: continue
41 for path, dirs, files in os.walk(root):
42 if filetype == 'file':
43 for f in files:
44 if regex.match(f):
45 matches.append(os.path.join(path,f))
46 elif filetype.startswith('dir'):
47 for d in dirs:
48 if regex.match(d):
49 matches.append(os.path.join(path,f))
50 elif filetype.startswith('path'):
51 if regex.search(path):
52 matches.append(os.path.abspath(path))
53 return matches
54
55
57 return socket.gethostbyaddr(socket.gethostbyname(socket.gethostname()))[0]
58
60 """
61 Get MAC of firt network interface.
62 Todo: Mac OS (Windows???)
63 """
64 eth0 = 'NULL'
65 try:
66 ifconfig = os.popen('/sbin/ifconfig eth0')
67 regex = re.compile(r'HWaddr\s+([0-9A-Z]{2}:){5}[0-9A-Z]{2}')
68 eth = regex.search(ifconfig.read())
69 eth0 = eth.group(0).split()[1]
70 ifconfig.close()
71 except Exception,e:
72 logger.error(e)
73 return eth0
74
75 -def findjava(searchpath='/usr/java:/usr/local/java'):
76 platform = os.uname()[4].replace('i686','i386').replace('x86_64','amd64')
77 javaregex = re.compile(r'jre/lib/%s/server'%platform)
78 java = None
79 for path in find(searchpath,'libjvm.so'):
80 if javaregex.search(path):
81 javahome = path.split('/jre/lib')[0]
82 if not java or os.path.basename(javahome) > os.path.basename(java):
83 java = javahome
84
85 if not java: return java
86
87 ldpath = os.path.join(java,'jre/lib/%s/server'%platform)
88 ldpath += ":"+os.path.join(java,'jre/lib/%s'%platform)
89 return java,ldpath
90
91
92 -def md5sum(filename,buffersize=4096):
93 """
94 Return md5 digest of file
95 """
96 filed = open(filename)
97 try:
98 import hashlib
99 except ImportError:
100 import md5
101 digest = md5.new()
102 else:
103 digest = hashlib.md5()
104
105 buffer = filed.read(buffersize)
106 while buffer:
107 digest.update(buffer)
108 buffer = filed.read(buffersize)
109 filed.close()
110 return digest.hexdigest()
111
113 print prompt
114 for i in range(len(choices)):
115 print "(%d): %s" % (i,choices[i])
116 sys.stdout.write('choice: ')
117 line = sys.stdin.readline().strip()
118 try:
119 item = int(line)
120 print 'You selected (%d): %s.' % (item,choices[item])
121 sys.stdout.write('is this correct? (Y/N): ')
122 if sys.stdin.readline().strip().upper() == 'Y':
123 return choices[item]
124 else:
125 return get_choice(choices,prompt)
126 except Exception,e:
127 print 'Invalid choice: %s' % str(e)
128 return get_choice(choices,prompt)
129
131 """
132 function to set environment variables
133 """
134 if value == None: print name,value
135 else: os.environ[name]=value
136
138 """
139 function to copy a local file
140 """
141 return not os.system('cp %s %s' % (src,dest))
142
143
144 -def wget(url,dest='./',cache=False,options=[]):
145 """
146 wrapper for downloading from multiple protocols
147 """
148
149 dest = os.path.expandvars(dest)
150 url = os.path.expandvars(url)
151 if os.path.isdir(dest.replace('file:','')):
152 dest = os.path.join(dest, os.path.basename(url))
153 if not dest.startswith("file:"):
154 dest = "file:" + dest
155
156 if cache and os.path.exists(dest.replace('file:','')):
157 logger.info('file "%s" exists. skipping download ' % dest)
158 return 0
159
160 if url.startswith("http://") or url.startswith("ftp://"):
161 dest = dest.replace("file:","")
162 cmd = 'wget -nv --tries=4 --output-document=%s %s'%(dest,url)
163 elif url.startswith("lfn:"):
164 cmd = "lcg-cp --vo icecube -v %s %s" % (url,dest)
165 elif url.startswith("file:"):
166 src = url.replace("file:","")
167 dest = dest.replace("file:","")
168 cmd = 'cp %s %s'% (src,dest)
169 elif url.startswith("gsiftp://"):
170 cmd = 'globus-url-copy -cd -r -nodcau %s %s'% (url,dest)
171 else:
172 raise Exception, "unsupported protocol %s" % url
173
174 for o in options:
175 cmd += ' --'+o+' '
176 print cmd
177 retval = os.system(cmd)
178 if retval:
179 os.system('rm -f %s' % dest.replace("file:",""))
180 return retval
181
182
184
185 """
186 Determine if this is a supported protocol
187 """
188 prefixes = ('http:','ftp:','gsiftp:','srm:','lfn:')
189 bools = map( lambda x: url.startswith(x) , prefixes)
190 return reduce(lambda a,b: a or b, bools)
191
193
194 stats = {
195 'VmLib:' : 0.0,
196 'VmData:': 0.0,
197 'VmExe:' : 0.0,
198 'VmRSS:' : 0.0,
199 'VmSize:': 0.0,
200 'VmLck:' : 0.0,
201 'VmStk:' : 0.0,
202 }
203 if os.uname()[0] == 'Linux':
204 usage = open("/proc/self/status")
205 for line in usage.readlines():
206 if line.startswith('Vm'):
207 name,value,unit = line.split()
208 value = float(value.strip())
209 if unit == "MB":
210 stats[name] = value*1024
211 else:
212 stats[name] = value
213 else:
214 self.logger.warn("getmemusage: Not a Linux machine")
215 return stats
216
217
219 """
220 function to caculate CORSIKA fluxsum
221 FLUXSUM is the integral in energy of the primary cosmic ray between
222 the minimum and the maximum set energy.
223
224 The cosmic ray energy spectrum is from Hoerandel polygonato model [Astrop.
225 Phys. Vol 19, Issue 2, Pages 193-312 (2003)]. The maximum energy is assumed
226 to be much higher than the minimum energy (the maximum energy actually is not explicitly
227 used in this calculation). Note : DSLOPE = 0 for unweighted CORSIKA sample and = -1 for weighted
228 CORSIKA sample.
229 """
230 integral = 0.;
231 nmax = 26;
232 norm = [
233 0.0873, 0.0571, 0.00208, 0.000474,0.000895,
234 0.0106, 0.00235, 0.0157, 0.000328,
235 0.0046, 0.000754, 0.00801, 0.00115,
236 0.00796, 0.00027, 0.00229, 0.000294,
237 0.000836, 0.000536, 0.00147, 0.000304,
238 0.00113, 0.000631, 0.00136, 0.00135, 0.0204 ];
239
240 gamma = [
241 2.71, 2.64, 2.54, 2.75, 2.95, 2.66, 2.72, 2.68, 2.69, 2.64,
242 2.66, 2.64, 2.66, 2.75, 2.69, 2.55, 2.68, 2.64, 2.65, 2.7,
243 2.64, 2.61, 2.63, 2.67, 2.46, 2.59 ];
244
245 crs = [
246 1.00797, 4.0026, 6.939, 9.0122, 10.811, 12.0112, 14.0067,
247 15.9994, 18.9984, 20.183, 22.9898, 24.312, 26.9815, 28.086,
248 30.984, 32.064, 35.453, 39.948, 39.102, 40.08, 44.956, 47.9,
249 50.942, 51.996, 54.938, 55.847];
250
251 for i in range(nmax):
252 prwght = round(crs[i]);
253 gamma[i] += dslope;
254 integral += norm[i] * pow( (emin*prwght), (1-gamma[i]) ) / (gamma[i]-1);
255
256 return integral;
257
258
260
261 smtpuser = "%s@%s" % (os.getlogin(),os.uname()[1].split(".")[0])
262 proto = 'sendmail'
263 if cfg.has_option('monitoring','smtphost'):
264 try:
265 import smtplib
266 except ImportError, e:
267 logger.error(e)
268 else:
269 proto = 'smtp'
270
271 if proto == 'stmp':
272 smtphost = cfg.get('monitoring','smtphost')
273 smtpuser = cfg.get('monitoring','smtpuser')
274
275 from_addr = "From: " + smtpuser
276 to_addrs = cfg.get('monitoring','smtpnotify').split(',')
277 subject = "Subject: %s\n" % subject
278
279 server = smtplib.SMTP(smtphost)
280 server.sendmail(from_addr, to_addrs, subject + msg)
281 server.quit()
282
283 else:
284 sendmail_location = "/usr/sbin/sendmail"
285 p = os.popen("%s -t" % sendmail_location, "w")
286 p.write("From: %s\n" % smtpuser)
287 p.write("To: %s\n" % smtpuser)
288 p.write("Subject: %s\n" % subject)
289 p.write("\n")
290 p.write(msg)
291 status = p.close()
292 if status != 0:
293 raise Exception, "Sendmail exited with status %u" % status
294
295 -def tail(logpath,chars=75):
296 """
297 Read log4cplus to write output logging
298 @param logpath: path to logfile
299 @param chars: number of characters end of file
300 @return: last n characters in file
301 """
302 if not os.path.exists(logpath):
303 return "no log output found %s" % logpath
304 logfile = open(logpath,'r')
305
306
307 st_results = os.stat(logpath)
308 st_size = st_results[6]
309 logfile.seek(max(0,st_size-chars))
310 header = '<br>----%s----:<br>' % os.path.basename(logpath)
311 tailtxt = "<br>".join(logfile.readlines())
312 logfile.close()
313 return header + tailtxt
314