1 """
2 Command library for iceprodsh
3
4 copyright (c) 2011 the icecube collaboration
5
6 @version: $Revision: $
7 @date: $Date: $
8 @author: David Schultz <dschultz@icecube.wisc.edu>
9 """
10
11
12
13
14
15
16
17 import sys
18 import iceprod.client
19
20
22 """Command: prototype command"""
23 shortdoc = None
26
27 numArgs = 0
29
30 if len(args) < self.numArgs:
31 return "Please specify all arguments"
32 self.args = args
33 return None
34
36 """Convert arguments to dataset,job pairs"""
37 dataset_job = []
38 for a in self.args:
39 dj = a.split('.')
40 datasets = []
41 jobs = []
42 if dj[0].find('-') != -1:
43
44 d = dj[0].split('-')
45 if len(d) < 2 or int(d[0]) > int(d[1]):
46 print "Invalid range for dataset"
47 return []
48 datasets = range(int(d[0]),int(d[1])+1)
49 elif dj[0].find(',') != -1:
50
51 datasets = map(int,dj[0].split(','))
52 else:
53
54 datasets.append(int(dj[0]))
55 if len(dj) > 1:
56
57 if dj[1].find('-') != -1:
58
59 d = dj[1].split('-')
60 if len(d) < 2 or int(d[0]) > int(d[1]):
61 print "Invalid range for job"
62 return []
63 jobs = range(int(d[0]),int(d[1])+1)
64 elif dj[1].find(',') != -1:
65
66 jobs = map(int,dj[1].split(','))
67 else:
68
69 jobs.append(int(dj[1]))
70 else:
71 jobs.append(-1)
72 for d in datasets:
73 for j in jobs:
74 dataset_job.append((d,j))
75 return dataset_job
76
78 """Convert arguments to dataset,job,status tuples"""
79 args2 = []
80 name = None
81 for a in self.args:
82 if name is None:
83 name = a
84 else:
85 args2.append((name,a))
86 name = None
87 dataset_job = []
88 for a,status in args2:
89 dj = a.split('.')
90 datasets = []
91 jobs = []
92 if dj[0].find('-') != -1:
93
94 d = dj[0].split('-')
95 if len(d) < 2 or int(d[0]) > int(d[1]):
96 print "Invalid range for dataset"
97 return []
98 datasets = range(int(d[0]),int(d[1])+1)
99 elif dj[0].find(',') != -1:
100
101 datasets = map(int,dj[0].split(','))
102 else:
103
104 datasets.append(int(dj[0]))
105 if len(dj) > 1:
106
107 if dj[1].find('-') != -1:
108
109 d = dj[1].split('-')
110 if len(d) < 2 or int(d[0]) > int(d[1]):
111 print "Invalid range for job"
112 return []
113 jobs = range(int(d[0]),int(d[1])+1)
114 elif dj[1].find(',') != -1:
115
116 jobs = map(int,dj[1].split(','))
117 else:
118
119 jobs.append(int(dj[1]))
120 else:
121 jobs.append(-1)
122 for d in datasets:
123 for j in jobs:
124 dataset_job.append((d,j,status))
125 return dataset_job
126
128 """Convert arguments to grid,daemon pairs"""
129 grid_daemon = []
130 for a in self.args:
131 gd = a.rsplit('.',1)
132 if gd[0].isdigit():
133 gd[0] = int(gd[0])
134 if len(gd) > 1:
135 grid_daemon.append((gd[0],gd[1]))
136 else:
137 grid_daemon.append((gd[0],'all'))
138 return grid_daemon
139
141 """Convert arguments to grid,dataset pairs"""
142 args2 = []
143 grid = None
144 for a in self.args:
145 if grid is None:
146 grid = a
147 else:
148 args2.append((grid,a))
149 grid = None
150 grid_dataset = []
151 for grid,dataset in args2:
152 grids = []
153 datasets = []
154 if grid.find(',') != -1:
155
156 grids = grid.split(',')
157 else:
158
159 grids.append(grid)
160
161 if dataset.find('-') != -1:
162
163 d = dataset.split('-')
164 if len(d) < 2 or int(d[0]) > int(d[1]):
165 print "Invalid range for dataset"
166 return []
167 datasets = range(int(d[0]),int(d[1])+1)
168 elif dataset.find(',') != -1:
169
170 datasets = map(int,dataset.split(','))
171 else:
172
173 datasets.append(int(dataset))
174 for g in grids:
175 if g.isdigit():
176 g = int(g)
177 for d in datasets:
178 grid_dataset.append((g,d))
179 return grid_dataset
180
182 """Command: set <variable> <value> [<variable> <value>]
183
184 Set local variable(s) in iceprodsh.
185 Can set multiple variables at once.
186
187 Arguments:
188 <variable> The variable to set
189 (username, url, production, test, editor)
190 <value> The value to set the variable to
191
192 Returns:
193 Echo variables and new values out to shell.
194 Print error on unknown variable.
195 """
196 numArgs = 2
197 shortdoc = "set <variable> <value> : Set a local variable"
199
200 options = {}
201 name = None
202 for a in self.args:
203 if name is None:
204 name = a
205 else:
206 options[name] = a
207 name = None
208
209 ret = False
210 for opt in options.iterkeys():
211 if opt == 'username':
212 shell.username = options[opt]
213 shell.password = None
214 shell.cfg.set('iceprodsh','username',options[opt])
215 elif opt == 'url':
216 if not options[opt].startswith('http'):
217 print 'url must start with http or https'
218 continue
219 from iceprod.client.soaptrayclient import i3SOAPClient
220 shell.url = options[opt]
221 shell.cfg.set('iceprodsh','url',options[opt])
222 print "now connecting to %s" % shell.url
223 shell.client = i3SOAPClient(url=shell.url)
224 elif opt == 'production':
225 shell.production = int(options[opt])
226 elif opt == 'test':
227 shell.test = int(options[opt])
228 elif opt == 'template':
229 shell.template = int(options[opt])
230 elif opt == 'editor':
231 shell.editor = options[opt]
232 elif opt == 'prefix':
233 shell.prefix = options[opt]
234 else:
235 print "unknown option '%s'" % opt
236 continue
237 print opt,'set to',options[opt]
238 ret = True
239 return ret
240
241
243
244 if len(args)%2 == 1:
245 return "Invalid arguments. Must be name value pairs."
246 return super(set,self).CheckArgs(args)
247
249 """Command: get <variable> [<variable>]
250
251 Get local variable(s) in iceprodsh.
252 Can get multiple variables at once.
253
254 Arguments:
255 <variable> The variable to get
256 (username, url, production, test, editor)
257
258 Returns:
259 Print variables and values out to shell.
260 Print error on unknown variables.
261 """
262 shortdoc = "get <variable> : Get a local variable"
263 numArgs = 1
265
266 options = {}
267 for a in self.args:
268 options[a] = None
269
270 for opt in options.iterkeys():
271 if opt == 'username':
272 options[opt] = shell.username
273 elif opt == 'url':
274 options[opt] = shell.url
275 elif opt == 'production':
276 options[opt] = str(shell.production)
277 elif opt == 'test':
278 options[opt] = str(shell.test)
279 elif opt == 'template':
280 options[opt] = str(shell.template)
281 elif opt == 'editor':
282 options[opt] = shell.editor
283 else:
284 print "unknown option '%s'" % opt
285 continue
286 print opt,'=',options[opt]
287 return options
288
289
291 """Command: suspend <dataset_id>[.<job>]
292
293 Suspend a whole dataset or a specific job from a dataset.
294 Can suspend multiple jobs or datasets at once.
295
296 Arguments:
297 <dataset_id> Specify the dataset to suspend.
298 [.<job>] (Optional) Specify the job within the dataset to suspend.
299
300 Returns:
301 Returns the result of the mysql query (success or failure).
302 Warning that if the dataset or job id is wrong it will likely
303 print success because there was no mysql error.
304
305 Examples:
306 Suspend whole dataset (dataset 1234, all jobs)
307 suspend 1234
308
309 Suspend individual jobs from different datasets
310 (dataset 1234, job 10 and dataset 4321, job 20)
311 suspend 1234.10 4321.20
312
313 Suspend multiple datasets and jobs using commas
314 (datasets 1234, 1243 and jobs 1, 3, and 5)
315 suspend 1234,1243.1,3,5
316
317 Suspend multile datasets and jobs using ranges
318 (datasets 1234 - 1235, jobs 1 - 5)
319 suspend 1234-1235.1-5
320 """
321 shortdoc = "suspend <dataset_id>[.<job>] : Suspend jobs."
322 numArgs = 1
324
325 dataset_job = self._getDatasetJob()
326
327
328 ret = True
329 for dataset,job in dataset_job:
330 self._suspend(shell,dataset,job)
331 return ret
332
337
339 """Command: resume <dataset_id>[.<job>]
340
341 Resume a dataset or a specific job from a dataset.
342 Can resume multiple jobs or datasets at once.
343
344 Arguments:
345 <dataset_id> Specify the dataset to resume.
346 [.<job>] (Optional) Specify the job within the dataset to resume.
347
348 Returns:
349 Returns the result of the mysql query (success or failure).
350 Warning that if the dataset or job id is wrong it will likely
351 print success because there was no mysql error.
352
353 Examples:
354 Resume whole dataset (dataset 1234, all jobs)
355 resume 1234
356
357 Resume individual jobs from different datasets
358 (dataset 1234, job 10 and dataset 4321, job 20)
359 resume 1234.10 4321.20
360
361 Resume multiple datasets and jobs using commas
362 (datasets 1234, 1243 and jobs 1, 3, and 5)
363 resume 1234,1243.1,3,5
364
365 Resume multile datasets and jobs using ranges
366 (datasets 1234 - 1235, jobs 1 - 5)
367 resume 1234-1235.1-5
368 """
369 shortdoc = "resume <dataset_id>[.<job>] : Resume jobs."
370 numArgs = 1
372
373 dataset_job = self._getDatasetJob()
374
375
376 ret = True
377 for dataset,job in dataset_job:
378 self._resume(shell,dataset,job)
379 return ret
380
381 - def _resume(self,shell,dataset,job):
385
387 """Command: reset <dataset_id>[.<job>]
388
389 Reset a dataset or a specific job from a dataset.
390 Can reset multiple jobs or datasets at once.
391
392 Arguments:
393 <dataset_id> Specify the dataset to reset.
394 [.<job>] (Optional) Specify the job within the dataset to reset.
395
396 Returns:
397 Returns the result of the mysql query (success or failure).
398 Warning that if the dataset or job id is wrong it will likely
399 print success because there was no mysql error.
400
401 Examples:
402 Reset whole dataset (dataset 1234, all jobs)
403 reset 1234
404
405 Reset individual jobs from different datasets
406 (dataset 1234, job 10 and dataset 4321, job 20)
407 reset 1234.10 4321.20
408
409 Reset multiple datasets and jobs using commas
410 (datasets 1234, 1243 and jobs 1, 3, and 5)
411 reset 1234,1243.1,3,5
412
413 Reset multile datasets and jobs using ranges
414 (datasets 1234 - 1235, jobs 1 - 5)
415 reset 1234-1235.1-5
416 """
417 shortdoc = "reset <dataset_id>[.<job>] : Reset jobs."
418 numArgs = 1
420
421 dataset_job = self._getDatasetJob()
422
423
424 ret = True
425 for dataset,job in dataset_job:
426 self._reset(shell,dataset,job)
427 return ret
428
429 - def _reset(self,shell,dataset,job):
433
435 """Command: status <dataset_id>[.<job>]
436 or
437 getstatus <dataset_id>[.<job>]
438
439 Get the status of all jobs or a specific job from a dataset.
440 Can get the status of multiple jobs over multiple datasets at once.
441
442 Arguments:
443 <dataset_id> Specify the dataset.
444 [.<job>] (Optional) Specify the job within the dataset.
445
446 Returns:
447 Returns the result of the mysql query (success or failure).
448 Warning that if the dataset or job id is wrong it will likely
449 print success because there was no mysql error.
450
451 Examples:
452 Get status of whole dataset (dataset 1234, all jobs)
453 status 1234
454
455 Get status of individual jobs from different datasets
456 (dataset 1234, job 10 and dataset 4321, job 20)
457 status 1234.10 4321.20
458
459 Get status of multiple datasets and jobs using commas
460 (datasets 1234, 1243 and jobs 1, 3, and 5)
461 status 1234,1243.1,3,5
462
463 Get status of multile datasets and jobs using ranges
464 (datasets 1234 - 1235, jobs 1 - 5)
465 status 1234-1235.1-5
466 """
467 shortdoc = "status <dataset_id>[.<job>] : Get status of jobs. AKA: getstatus"
468 numArgs = 1
478
479 - def _status(self,shell,dataset,job):
482
486
488 """Command: datasetstatus <dataset_id>
489 or
490 getdatasetstatus <dataset_id>
491
492 Get the status of the dataset.
493 Can get the status of multiple datasets at once.
494
495 Arguments:
496 <dataset_id> Specify the dataset.
497
498 Returns:
499 Returns the result of the mysql query (success or failure).
500 Warning that if the dataset is wrong it will likely
501 print success because there was no mysql error.
502
503 Examples:
504 Get status of single dataset (dataset 1234)
505 datasetstatus 1234
506
507 Get status of multiple datasets using spaces
508 (dataset 1234 and dataset 4321)
509 datasetstatus 1234 4321
510
511 Get status of multiple datasets using commas
512 (datasets 1234, 1243)
513 datasetstatus 1234,1243
514
515 Get status of multile datasets using ranges
516 (datasets 1234 - 1235)
517 datasetstatus 1234-1235
518 """
519 shortdoc = "datasetstatus <dataset_id> : Get status of dataset. AKA: getdatasetstatus"
520 numArgs = 1
530
534
538
540 """Command: setstatus <dataset_id>[.<job>] <status>
541
542 Set the status of all jobs or a specific job from a dataset.
543 Can set the status of multiple jobs over multiple datasets at once.
544
545 Arguments:
546 <dataset_id> Specify the dataset.
547 [.<job>] (Optional) Specify the job within the dataset.
548 <status> Specify the status
549 (WAITING, QUEUEING, QUEUED, PROCESSING, OK, ERROR,
550 READYTOCOPY, COPYING, SUSPENDED, RESET, FAILED,
551 COPIED, EVICTED, CLEANING, IDLE)
552
553 Returns:
554 Returns the result of the mysql query (success or failure).
555 Warning that if the dataset or job id is wrong it will likely
556 print success because there was no mysql error.
557
558 Examples:
559 Set status of all jobs in dataset to RESET (dataset 1234, all jobs)
560 status 1234 RESET
561
562 Set status of individual jobs from different datasets
563 (dataset 1234, job 10 to waiting and dataset 4321, job 20 to RESET)
564 status 1234.10 WAITING 4321.20 RESET
565
566 Set status of multiple datasets and jobs using commas
567 (jobs 1, 3, and 5 from datasets 1234 and 1243 to OK)
568 status 1234,1243.1,3,5 OK
569
570 Set status of multile datasets and jobs using ranges
571 (datasets 1234 - 1235, jobs 1 - 5 to SUSPENDED)
572 status 1234-1235.1-5 SUSPENDED
573 """
574 shortdoc = "setstatus <dataset_id>[.<job>] <status> : Set status of jobs."
575 numArgs = 2
585
586 - def _status(self,shell,dataset,job,status):
590
592
593 if len(args)%2 == 1:
594 return "Invalid arguments. Must be grid dataset pairs."
595 return super(setstatus,self).CheckArgs(args)
596
598 """Command: setstatus <dataset_id> <status>
599
600 Set the status of a dataset.
601 Can set the status of multiple datasets at once.
602
603 Arguments:
604 <dataset_id> Specify the dataset.
605 <status> Specify the status
606 (PROCESSING, COMPLETE, ERRORS, READYTOPUBLISH,
607 MOVING, OBSOLETE, READYTODELETE, DELETED, TEMPLATE)
608
609 Returns:
610 Returns the result of the mysql query (success or failure).
611 Warning that if the dataset is wrong it will likely
612 print success because there was no mysql error.
613
614 Examples:
615 Set status of dataset to PROCESSING
616 status 1234 PROCESSING
617
618 Set status of multiple datasets to different states
619 (dataset 1234 to PROCESSING and dataset 4321 to ERRORS)
620 status 1234 PROCESSING 4321 ERRORS
621
622 Set status of multiple datasets using commas
623 (datasets 1234, 1243 to COMPLETE)
624 status 1234,1243 COMPLETE
625
626 Set status of multile datasets using ranges
627 (datasets 1234 - 1235 to READYTOPUBLISH)
628 status 1234-1235 READYTOPUBLISH
629 """
630 shortdoc = "setdatasetstatus <dataset_id> <status> : Set status of dataset."
631 numArgs = 2
641
642 - def _status(self,shell,dataset,status):
646
648
649 if len(args)%2 == 1:
650 return "Invalid arguments. Must be grid dataset pairs."
651 return super(setdatasetstatus,self).CheckArgs(args)
652
654 """Command: clean <dataset_id>
655
656 Clean a dataset.
657 Can clean multiple datasets at once.
658
659 Arguments:
660 <dataset_id> Specify the dataset to clean.
661
662 Returns:
663 Returns the result of the mysql query (success or failure).
664 Warning that if the dataset is wrong it will likely
665 print success because there was no mysql error.
666
667 Examples:
668 Clean single dataset (dataset 1234)
669 clean 1234
670
671 Clean multiple datasets using commas
672 (datasets 1234, 1243)
673 clean 1234,1243
674
675 Clean multile datasets using ranges
676 (datasets 1234 - 1235)
677 clean 1234-1235
678 """
679 shortdoc = "clean <dataset_id> : Clean a dataset."
680 numArgs = 1
682
683 dataset_job = self._getDatasetJob()
684
685
686 ret = True
687 for dataset,job in dataset_job:
688 self._clean(shell,dataset)
689 return ret
690
691 - def _clean(self,shell,dataset):
695
697 """Command: finish <dataset_id>
698
699 Finish a dataset.
700 Can finish multiple datasets at once.
701
702 Arguments:
703 <dataset_id> Specify the dataset to finish.
704
705 Returns:
706 Returns the result of the mysql query (success or failure).
707 Warning that if the dataset is wrong it will likely
708 print success because there was no mysql error.
709
710 Examples:
711 Finish single dataset (dataset 1234)
712 finish 1234
713
714 Finish multiple datasets using commas
715 (datasets 1234, 1243 and jobs 1, 3, and 5)
716 finish 1234,1243
717
718 Finish multile datasets using ranges
719 (datasets 1234 - 1235, jobs 1 - 5)
720 finish 1234-1235
721 """
722 shortdoc = "finish <dataset_id> : Finish a dataset."
723 numArgs = 1
725
726 dataset_job = self._getDatasetJob()
727
728
729 ret = True
730 for dataset,job in dataset_job:
731 self._finish(shell,dataset)
732 return ret
733
738
740 """Command: retire <dataset_id>
741
742 Retire a dataset. Send it to the archives.
743 Can retire multiple datasets at once.
744
745 Arguments:
746 <dataset_id> Specify the dataset to retire.
747
748 Returns:
749 Returns the result of the mysql query (success or failure).
750 Warning that if the dataset is wrong it will likely
751 print success because there was no mysql error.
752
753 Examples:
754 Retire single dataset (dataset 1234)
755 retire 1234
756
757 Retire multiple datasets using commas
758 (datasets 1234, 1243 and jobs 1, 3, and 5)
759 retire 1234,1243
760
761 Retire multile datasets using ranges
762 (datasets 1234 - 1235, jobs 1 - 5)
763 retire 1234-1235
764 """
765 shortdoc = "retire <dataset_id> : Retire a dataset."
766 numArgs = 1
768
769 dataset_job = self._getDatasetJob()
770
771
772 ret = True
773 for dataset,job in dataset_job:
774 self._retire(shell,dataset)
775 return ret
776
781
782 -class nuke(Command):
783 """Command: nuke <dataset_id>
784
785 Nuke (delete) a dataset. Similar to 'hide'+'clean', but more
786 destructive. Can nuke multiple datasets at once.
787
788 Arguments:
789 <dataset_id> Specify the dataset to nuke.
790
791 Returns:
792 Returns the result of the mysql query (success or failure).
793 Warning that if the dataset is wrong it will likely
794 print success because there was no mysql error.
795
796 Examples:
797 Nuke single dataset (dataset 1234)
798 nuke 1234
799
800 Nuke multiple datasets using commas
801 (datasets 1234, 1243 and jobs 1, 3, and 5)
802 nuke 1234,1243
803
804 Nuke multile datasets using ranges
805 (datasets 1234 - 1235, jobs 1 - 5)
806 nuke 1234-1235
807 """
808 shortdoc = "nuke <dataset_id> : Nuke (delete) a dataset."
809 numArgs = 1
811
812 dataset_job = self._getDatasetJob()
813
814
815 ret = True
816 for dataset,job in dataset_job:
817 self._nuke(shell,dataset)
818 return ret
819
820 - def _nuke(self,shell,dataset):
824
826 """Command: valid <dataset_id>
827 or
828 show <dataset_id>
829
830 Validate a dataset and let it show up in iceprod.
831 Can validate multiple datasets at once.
832
833 Arguments:
834 <dataset_id> Specify the dataset to validate.
835
836 Returns:
837 Returns the result of the mysql query (success or failure).
838 Warning that if the dataset id is wrong it will likely
839 print success because there was no mysql error.
840
841 Examples:
842 Validate single dataset (dataset 1234)
843 valid 1234
844
845 Validate multiple datasets using commas
846 (datasets 1234, 1243 and jobs 1, 3, and 5)
847 valid 1234,1243
848
849 Validate multile datasets using ranges
850 (datasets 1234 - 1235, jobs 1 - 5)
851 valid 1234-1235
852 """
853 shortdoc = "valid <dataset_id> : Validate a dataset. AKA: show"
854 numArgs = 1
856
857 dataset_job = self._getDatasetJob()
858
859
860 ret = True
861 for dataset,job in dataset_job:
862 self._valid(shell,dataset,True)
863 return ret
864
865 - def _valid(self,shell,dataset,state):
869
873
875 """Command: invalid <dataset_id>
876 or
877 nvalid <dataset_id>
878 or
879 hide <dataset_id>
880
881 Invalidate a dataset and hide it from iceprod.
882 Can invalidate multiple datasets at once.
883
884 Arguments:
885 <dataset_id> Specify the dataset to invalidate.
886
887 Returns:
888 Returns the result of the mysql query (success or failure).
889 Warning that if the dataset id is wrong it will likely
890 print success because there was no mysql error.
891
892 Examples:
893 Invalidate single dataset (dataset 1234)
894 invalid 1234
895
896 Invalidate multiple datasets using commas
897 (datasets 1234, 1243 and jobs 1, 3, and 5)
898 invalid 1234,1243
899
900 Invalidate multile datasets using ranges
901 (datasets 1234 - 1235, jobs 1 - 5)
902 invalid 1234-1235
903 """
904 shortdoc = "invalid <dataset_id> : Invalidate a dataset. AKA: nvalid,hide"
905 numArgs = 1
907
908 dataset_job = self._getDatasetJob()
909
910
911 ret = True
912 for dataset,job in dataset_job:
913 self._valid(shell,dataset,False)
914 return ret
915
919
920 -class hide(invalid):
923
925 """Command: startserver <grid>[.<daemon>]
926
927 Start a server or a specific daemon on the server.
928 Can start multiple servers and daemons at once.
929
930 Only works if main iceprodd daemon is running on the server,
931 otherwise the change is only in the database.
932
933 Arguments:
934 <grid> Specify the grid to start, either by name or id
935 [.<daemon>] (Optional) Specify the daemon to start.
936 (all,soapdh,soapqueue,soaphist,soapmon)
937
938 Returns:
939 Returns the result of the mysql query (success or failure).
940 Warning that if the grid or daemon is wrong it will likely
941 print success because there was no mysql error.
942
943 Examples:
944 Start GLOW
945 startserver GLOW
946 or
947 startserver 1
948
949 Start soapqueue on GLOW and soapdh on glow-test
950 startserver GLOW.soapqueue glow-test.soapdh
951
952 Start EGEE.Madison, or others with a dot in the name
953 (must specify daemon to get grid name correct)
954 startserver EGEE.Madison.all
955 """
956 shortdoc = "startserver <grid>[.<daemon>] : Start an iceprod server."
957 numArgs = 1
959
960 grid_daemon = self._getGridDaemon()
961
962
963 ret = True
964 for grid,daemon in grid_daemon:
965 self._start(shell,grid,daemon)
966 return ret
967
968 - def _start(self,shell,grid,daemon):
972
974 """Command: stopserver <grid>[.<daemon>]
975
976 Stop a server or a specific daemon on the server.
977 Can stop multiple servers and daemons at once.
978
979 Only works if main iceprodd daemon is running on the server,
980 otherwise the change is only in the database.
981
982 Arguments:
983 <grid> Specify the grid to stop, either by name or id
984 [.<daemon>] (Optional) Specify the daemon to stop.
985 (soapdh,soapqueue,soaphist,soapmon)
986
987 Returns:
988 Returns the result of the mysql query (success or failure).
989 Warning that if the grid or daemon is wrong it will likely
990 print success because there was no mysql error.
991
992 Examples:
993 Stop GLOW
994 stopserver GLOW
995 or
996 stopserver 1
997
998 Stop soapqueue on GLOW and soapdh on glow-test
999 stopserver GLOW.soapqueue glow-test.soapdh
1000 """
1001 shortdoc = "stopserver <grid>[.<daemon>] : Stop an iceprod server."
1002 numArgs = 1
1004
1005 grid_daemon = self._getGridDaemon()
1006
1007
1008 ret = True
1009 for grid,daemon in grid_daemon:
1010 self._stop(shell,grid,daemon)
1011 return ret
1012
1013 - def _stop(self,shell,grid,daemon):
1017
1018
1020 """Command: includegrid <grid> <dataset_id>
1021
1022 Resume a dataset on a grid.
1023 Can resume multiple datasets and grids at once.
1024
1025 Arguments:
1026 <grid> Specify the grid, either by name or id
1027 <dataset_id> Specify dataset to act on
1028
1029 Returns:
1030 Returns the result of the mysql query (success or failure).
1031 Warning that if the grid or dataset is wrong it will likely
1032 print success because there was no mysql error.
1033
1034 Examples:
1035 Resume 1234 on GLOW
1036 includegrid GLOW 1234
1037 or
1038 includegrid 1 1234
1039
1040 Resume 1234 and 1243 on GLOW and glow-test
1041 includegrid GLOW,glow-test 1234,1243
1042 """
1043 shortdoc = "includegrid <grid> <dataset_id> : Resume a dataset on a grid."
1044 numArgs = 2
1046
1047 grid_dataset = self._getGridDataset()
1048
1049
1050 ret = True
1051 for grid,dataset in grid_dataset:
1052 self._include(shell,grid,dataset)
1053 return ret
1054
1055 - def _include(self,shell,grid,dataset):
1059
1061
1062 if len(args)%2 == 1:
1063 return "Invalid arguments. Must be grid dataset pairs."
1064 return super(includegrid,self).CheckArgs(args)
1065
1067 """Command: excludegrid <grid> <dataset_id>
1068
1069 Suspend a dataset on a grid
1070 Can suspend multiple datasets and grids at once.
1071
1072 Arguments:
1073 <grid> Specify the grid, either by name or id
1074 <dataset_id> Specify dataset to act on
1075
1076 Returns:
1077 Returns the result of the mysql query (success or failure).
1078 Warning that if the grid or dataset is wrong it will likely
1079 print success because there was no mysql error.
1080
1081 Examples:
1082 Suspend 1234 on GLOW
1083 excludegrid GLOW 1234
1084 or
1085 excludegrid 1 1234
1086
1087 Suspend 1234 and 1243 on GLOW and glow-test
1088 excludegrid GLOW,glow-test 1234,1243
1089 """
1090 shortdoc = "excludegrid <grid> <dataset_id> : Suspend a dataset on a grid."
1091 numArgs = 2
1093
1094 grid_dataset = self._getGridDataset()
1095
1096
1097 ret = True
1098 for grid,dataset in grid_dataset:
1099 self._exclude(shell,grid,dataset)
1100 return ret
1101
1102 - def _exclude(self,shell,grid,dataset):
1106
1108
1109 if len(args)%2 == 1:
1110 return "Invalid arguments. Must be grid dataset pairs."
1111 return super(excludegrid,self).CheckArgs(args)
1112
1114 """Command: addgrid <grid> <dataset_id>
1115
1116 Add a grid to a dataset
1117 Can add multiple grids to multiple datasets at once.
1118
1119 Arguments:
1120 <grid> Specify the grid, either by name or id
1121 <dataset_id> Specify dataset to act on
1122
1123 Returns:
1124 Returns the result of the mysql query (success or failure).
1125 Warning that if the grid or dataset is wrong it will likely
1126 print success because there was no mysql error.
1127
1128 Examples:
1129 Add GLOW to 1234
1130 addgrid GLOW 1234
1131 or
1132 addgrid 1 1234
1133
1134 Add GLOW to 1234 and glow-test to 4321
1135 addgrid GLOW 1234 glow-test 4321
1136
1137 Add GLOW and glow-test to 1234 and 4321
1138 addgrid GLOW,glow-test 1234,4321
1139 """
1140 shortdoc = "addgrid <grid> <dataset_id> : Add a grid to a dataset."
1141 numArgs = 2
1143
1144 grid_dataset = self._getGridDataset()
1145
1146
1147 ret = True
1148 for grid,dataset in grid_dataset:
1149 self._add(shell,grid,dataset)
1150 return ret
1151
1152 - def _add(self,shell,grid,dataset):
1156
1158
1159 if len(args)%2 == 1:
1160 return "Invalid arguments. Must be grid dataset pairs."
1161 return super(addgrid,self).CheckArgs(args)
1162
1163 -class open(Command):
1187
1188 -class edit(Command):
1222
1224 if len(args) > 0:
1225 print "Ignoring arguments"
1226 return None
1227
1229 """Command: close
1230
1231 Close the currently open steering file.
1232 (Does nothing if no file is open)
1233
1234 Arguments: none
1235
1236 Returns: none
1237 """
1238 shortdoc = "close : Close the currently open steering file."
1240 shell.steering = None
1241
1243 if len(args) > 0:
1244 print "Ignoring arguments"
1245 return None
1246
1247 -class save(Command):
1248 """Command: save [<filename>]
1249
1250 Save the currently open steering file.
1251 (Must have previously opened or downloaded a steering file)
1252
1253 Arguments:
1254 <filename> (Optional) Name of file to save to.
1255 If not specified, filename is either the
1256 file it was opened from, the file it was
1257 last saved to, or the dataset id if it
1258 was downloaded.
1259
1260 Returns:
1261 Saves steering to file. No visible return.
1262 """
1263 shortdoc = "save [<filename>] : Save the currently open steering file."
1265 from iceprod.core.xmlwriter import IceTrayXMLWriter
1266 if not shell.steering:
1267 print "No configuration loaded"
1268 return
1269 if len(self.args) > 0:
1270 shell.filename = self.args[0]
1271 writer = IceTrayXMLWriter(shell.steering)
1272 writer.write_to_file(shell.filename)
1273
1275 """Command: download <dataset>
1276
1277 Download a dataset (replaces currently open steering file)
1278
1279 Arguments:
1280 <dataset> Specify dataset id to download.
1281
1282 Returns:
1283 Downloads steering from database and opens it.
1284 """
1285 shortdoc = "download <dataset> : Download a dataset."
1286 numArgs = 1
1288 dataset = int(self.args[0])
1289 shell.steering = shell.client.download_config(dataset)
1290 shell.filename = 'dataset_%u.xml' % dataset
1291
1293
1294 s = super(download,self).CheckArgs(args)
1295 if s is not None:
1296 return s
1297 if not self.args[0].isdigit():
1298 return "Invalid argument. Must be integer."
1299 if len(args) > 1:
1300 print "Ignoring additional arguments. Using only ",arg[0]
1301 return None
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1339 """Command: loadfiles <rootdir> [<rootdir>] [regex=<regex>] [dataset=<dataset_id>]
1340
1341 Get list of files in rootdir(s) and add them to database
1342 dictionary table if they match the regex.
1343
1344 Arguments:
1345 <rootdir> Directory of files
1346 regex=<regex> (Optional) A regex to match against
1347 regex=<regex> (Optional) A regex to match against
1348
1349 Returns:
1350 Error message on failure.
1351 """
1352 shortdoc = "loadfiles <rootdir> [<rootdir>] [regex=<regex>] [dataset=<dataset_id>]"
1353 numArgs = 1
1355 import os, re
1356 from iceprod.core import odict
1357 odict = OrderedDict();
1358 if self.regex: cregex = re.compile(self.regex)
1359 filecount = 0
1360 for root in self.args:
1361 for path, dirs, files in os.walk(root):
1362 for file in files:
1363 if not self.regex or cregex.match(file):
1364 truncated_path = path.replace(shell.prefix,'')
1365 filename = os.path.join(truncated_path,file)
1366 key = filecount
1367 odict[key] = filename
1368 print key, odict[key]
1369 filecount += 1
1370 shell.client.loaddict(odict, shell.username, shell.password,self.dataset)
1371 return len(odict)
1372
1374 s = super(summary,self).CheckArgs(args)
1375 if s is not None:
1376 return s
1377
1378 self.regex = None
1379 self.dataset = 0
1380 for a in args:
1381 if a.startswith('regex='):
1382 self.regex = a.split('=',1)[1]
1383 self.args.remove(a)
1384 elif a.startswith('dataset='):
1385 self.dataset = a.split('=',1)[1]
1386 self.args.remove(a)
1387 return None
1388
1389 -class exit(Command):
1390 """Command: exit
1391 or quit
1392
1393 Exit iceprodsh.
1394 Also saves the default configuration settings
1395 and history to the home directory.
1396 """
1397 shortdoc = "exit : Exit iceprodsh. AKA: quit"
1399 import os,readline,__builtin__
1400 cfgfile = __builtin__.open(os.path.join(os.getenv('HOME'),".iceprodshrc"),'w')
1401 shell.cfg.write(cfgfile)
1402 cfgfile.close()
1403 print "adios."
1404 try:
1405 readline.write_history_file(os.path.expandvars('$HOME/.iceprodsh_history'))
1406 except:pass
1407 os._exit(0)
1408
1412
1414 """Command: usage
1415 or help
1416
1417 Print usage information.
1418 This is the basic help system.
1419 """
1420 shortdoc = "usage : Print usage information. AKA: help"
1423
1424 usage = """Usage: iceprodsh [option] <url> <command> <args>
1425
1426 options:
1427
1428 -i,
1429 --interactive : Interactive shell (default).
1430
1431 -h,
1432 --help : This screen.
1433
1434 -r <url>,
1435 --url=<url> : Specify a url of soaptray server.
1436
1437 -u <username>,
1438 --username=<username> : Specify a username.
1439
1440 --production : Authenticate, update production database
1441 and create metadata. For production by
1442 authorized users.
1443 --test : Specify submissions as tests.
1444
1445 --meta=<file> : Use .ini style meta-data file instead of
1446 interactive form.
1447 -m <key:value> : Key value pair to override metadata file.
1448 Can use multiple times.
1449
1450 --prefix=<value> : Specify prefix for files.
1451
1452 -v <value>,
1453 --validate=<value> : Turn validation on or off.
1454 """
1456 command_list = self._listcommands()
1457 print self.usage
1458 print " "
1459 print "commands:"
1460 for cmd in command_list:
1461 print " "
1462 if cmd.shortdoc is None:
1463 print " "+cmd.__doc__.split("\n",1)[0].split(':',1)[1]
1464 else:
1465 print " "+cmd.shortdoc
1466 print " "
1467
1469 import inspect
1470 command_list = []
1471 for name, obj in inspect.getmembers(inspect.getmodule(Command)):
1472 if inspect.isclass(obj) and name != "Command":
1473 command_list.append(obj)
1474 return command_list
1475
1477 """Command: help [<command>]
1478
1479 Lists the help, either for general usage or for specific commands.
1480
1481 Arguments:
1482 <command> (Optional) If omitted, prints usage.
1483 If specified, prints help for specific command.
1484 """
1485 shortdoc = "help [<command>] : Print usage or detailed help"
1487 if len(self.args) < 1:
1488 self._usage()
1489 return
1490
1491 item = self.args[0]
1492 command_list = self._listcommands()
1493 modules = {'iceprod':'iceprod',
1494 'iceprod.core':'iceprod.core',
1495 'iceprod.server':'iceprod.server',
1496 'iceprod.client':'iceprod.client',
1497 'iceprod.modules':'iceprod.modules',
1498 'core':'iceprod.core',
1499 'server':'iceprod.server',
1500 'client':'iceprod.client',
1501 'modules':'iceprod.modules'}
1502 modules.update(self._listmodules('iceprod'))
1503 modules.update(self._listmodules('iceprod.core'))
1504 modules.update(self._listmodules('iceprod.server'))
1505 modules.update(self._listmodules('iceprod.client'))
1506 modules.update(self._listmodules('iceprod.modules'))
1507 for m in modules:
1508 if item == m:
1509 self._pythonhelp(modules[item])
1510 return
1511 elif item.startswith(m):
1512 self._pythonhelp(item)
1513 return
1514 for cmd in command_list:
1515 if cmd.__name__ == item:
1516 print cmd.__doc__
1517 return
1518
1519 print "Item not found in help"
1520
1522 import os,imp
1523 package_name_os = package_name.replace('.','/')
1524 file, pathname, description = imp.find_module(package_name_os)
1525 if file:
1526
1527 return {}
1528
1529 ret = {}
1530 for module in os.listdir(pathname):
1531 if module.endswith('.py') and module != '__init__.py':
1532 tmp = os.path.splitext(module)[0]
1533 ret[tmp] = package_name+'.'+tmp
1534 return ret
1535
1544
1546 """Command: submit [<filename>]
1547
1548 Submit a new dataset. If in production mode, submits to iceprod.
1549 Otherwise, the dataset is run locally.
1550
1551 Arguments:
1552 <filename> (Optional) Specify the filename to submit.
1553 Defaults to the currently open file.
1554
1555 Returns:
1556 Prints success or failure, as well as other details.
1557 """
1558 shortdoc = "submit [<filename>] : Submit a new dataset."
1560 try:
1561 import signal
1562 from iceprod.core.xmlparser import IceTrayXMLParser
1563 from iceprod.client.soaptrayclient import i3SOAPClient
1564 from iceprod.core.dataclasses import Steering
1565
1566 default_handler = {
1567 signal.SIGQUIT: signal.getsignal(signal.SIGQUIT),
1568 signal.SIGINT: signal.getsignal(signal.SIGINT) ,
1569 signal.SIGCHLD: signal.getsignal(signal.SIGCHLD),
1570 }
1571
1572
1573 def handler(signum,frame):
1574 shell.logger.warn("caught signal %s" % signum)
1575 raise Exception, "Operation cancelled"
1576
1577 signal.signal(signal.SIGQUIT, handler)
1578 signal.signal(signal.SIGINT, handler)
1579
1580 print "production flag is set to %s" % str(shell.production)
1581 print "test flag is set to %s" % str(shell.test)
1582 print "template flag is set to %s" % str(shell.template)
1583
1584 if len(self.args) >= 1:
1585 shell.steering = Steering()
1586 myparser = IceTrayXMLParser(shell.steering)
1587 if not myparser.ParseFile(self.args[0],validate=shell.xmlvalidate):
1588 raise "unable to parse configuration file"
1589
1590 if not shell.steering:
1591 shell.logger.fatal("no steering object loaded:Cannot submit")
1592 return False
1593
1594
1595 shell.descmap['geometry'] = '%(geometry)s'
1596 shell.descmap['simcat'] = '%(simcat)s'
1597 shell.descmap['composition'] = '%(composition)s'
1598 shell.descmap['weighted'] = '%(weighted)s'
1599 shell.descmap['spectrum'] = '%(spectrum)s'
1600 shell.descmap['icemodel'] = '%(icemodel)s'
1601 shell.descmap['photonpropagator'] = '%(photonpropagator)s'
1602 shell.descmap['angularrange'] = '%(angularrange)s'
1603 shell.descmap['energyrange'] = '%(energyrange)s'
1604
1605
1606 if shell.meta:
1607 self._meta(shell)
1608 return
1609 elif shell.production:
1610 steering = self._production(shell)
1611 else:
1612 steering = shell.steering
1613
1614 shell.auth()
1615
1616 i3q = shell.client.submit( steering, shell.username, shell.password, shell.production)
1617 if i3q:
1618 shell.client.check_q(i3q, shell.username, shell.password)
1619
1620
1621 signal.signal(signal.SIGQUIT, default_handler[signal.SIGQUIT])
1622 signal.signal(signal.SIGINT, default_handler[signal.SIGINT])
1623 except Exception,e:
1624 apply(sys.excepthook,sys.exc_info())
1625 print e
1626
1717
1719 import time
1720 from iceprod.core import metadata, lex
1721 from iceprod.core.dataclasses import SimulationCategories, DatasetTypes
1722
1723 steering = shell.steering
1724
1725 if shell.template:
1726 grids = 'Hut'
1727 else:
1728
1729 msg = "Grid: enter the name(s) of grids to run this dataset on:"
1730 grids = self._get_text(msg)
1731 steering.AddExtra("Grid", grids)
1732
1733 if shell.template:
1734 maxjobs = 10
1735 else:
1736
1737 msg = "Maxjobs: enter the number of jobs in this dataset:"
1738 maxjobs = self._get_int(msg)
1739 steering.AddExtra("Maxjobs", maxjobs)
1740
1741
1742 difplus = metadata.DIF_Plus()
1743 dif = difplus.GetDIF()
1744 plus = difplus.GetPlus()
1745
1746
1747 if shell.test:
1748 dtext = 'Test Dataset'
1749 elif shell.template:
1750 dtext = 'Template'
1751 else:
1752 dtext = 'Production Dataset'
1753
1754 dif.SetEntryTitle(dtext)
1755
1756
1757 ticket = 0
1758 if not shell.test and not shell.template:
1759 ticket = self._get_ticket()
1760 steering.AddExtra("Ticket",ticket)
1761
1762
1763 msg = "Simulation Category: Enter a generator from the choices below:"
1764 if shell.test:
1765 cat = 'Test'
1766 elif shell.template:
1767 cat = 'Template'
1768 else:
1769 cat = self._get_choice(SimulationCategories, msg)
1770 steering.SetCategory(cat)
1771
1772
1773 msg = "Category: enter a number from the choices below:"
1774 if shell.test or shell.template:
1775 dtext = 'unclassified'
1776 else:
1777 dtext = self._get_choice(plus.GetValidCategories(),msg)
1778 plus.SetCategory(dtext)
1779
1780 plus.SetSubCategory(self._get_subcat())
1781
1782
1783
1784
1785
1786 startdatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime())
1787 plus.SetStartDatetime(startdatetime)
1788
1789
1790
1791
1792
1793 enddatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime())
1794 plus.SetEndDatetime(enddatetime)
1795
1796 for iconfig in steering.GetTrays():
1797
1798 for metaproject in iconfig.GetMetaProjectList():
1799 for project in metaproject.GetProjectList():
1800 plus.AddProject(project)
1801
1802
1803 for project in iconfig.GetProjectList():
1804 plus.AddProject(project)
1805
1806 msg = "Parameters: enter a number from the choices below:"
1807
1808
1809
1810
1811 dtext = dif.GetValidParameters()[0]
1812 dif.SetParameters(dtext)
1813
1814
1815 msg = "Dataset type: enter a number from the choices below:"
1816 if shell.test:
1817 dtext = 'TEST'
1818 elif shell.template:
1819 dtext = 'TEMPLATE'
1820 else:
1821 dtext = self._get_choice(DatasetTypes, msg)
1822 steering.SetDatasetType(dtext)
1823
1824
1825 expparser = lex.ExpParser( {
1826 'extern':0,
1827 'procnum':0,
1828 'tray':0,
1829 'iter':0,
1830 'dataset':0,
1831 'nproc': int(steering.GetParameter('MAXJOBS').GetValue()),
1832 },
1833 steering)
1834
1835
1836 for p in steering.GetParameters():
1837 try:
1838 shell.descmap[p.GetName()] = expparser.parse(p.GetValue())
1839 except Exception,e: pass
1840 shell.descmap['simcat'] = steering.GetCategory()
1841
1842 descriptionstr = iceprod.client.descriptionstr
1843 what = steering.GetParameter('description')
1844 if what:
1845 descriptionstr = what.GetValue()
1846 else:
1847 steering.AddParameter("string","description",descriptionstr)
1848
1849
1850 description = descriptionstr % shell.descmap
1851 dtext = self._get_description(shell.completer,description)
1852
1853 if shell.test:
1854 dtext = 'Test: ' + dtext
1855 steering.SetDescription(dtext)
1856 dif.SetSummary(dtext)
1857
1858 steering.AddExtra("Metadata",difplus)
1859 return steering
1860
1862 """
1863 Prompt for and read a description for the simulation run.
1864 @return: the text entered by the user as a single string
1865 """
1866 import sys
1867
1868 print 'Please type a brief description of this run.'
1869 print 'You may write multiple lines. To finish just enter a blank line:'
1870
1871 text = []
1872 try:
1873 import readline
1874 import rlcompleter
1875 except:
1876 print 'e.g. "%s"' % initial
1877 line = sys.stdin.readline().strip()
1878 while line:
1879 text.append(line)
1880 line = sys.stdin.readline().strip()
1881 else:
1882 readline.parse_and_bind("tab: complete")
1883 readline.set_completer(completer)
1884
1885 readline.add_history(initial)
1886 readline.set_startup_hook(None)
1887 while True:
1888 try:
1889 line = raw_input('> ')
1890 except EOFError:
1891 break
1892 if not line: break
1893 text.append(line)
1894
1895 return ' '.join(text)
1896
1897 - def _get_text(self,prompt):
1898 """
1899 Prompt for input
1900 @return: the text entered by the user
1901 """
1902 import sys
1903 print prompt
1904 ret = sys.stdin.readline().strip()
1905 if not ret:
1906 return self._get_text(prompt)
1907 else:
1908 return ret
1909
1912
1914 """
1915 Prompt for and read a title for the simulation run.
1916 @return: the text entered by the user
1917 """
1918 import sys
1919 print 'Please type title for this run:'
1920 ret = sys.stdin.readline().strip()
1921 if not ret:
1922 return self._get_title()
1923 else:
1924 return ret
1925
1927 """
1928 Prompt for and read a ticket number associated with dataset.
1929 @return: the number entered
1930 """
1931 import sys
1932 print "Is there a ticket number associated with this dataset?"
1933 print "If so, enter it. Otherwise enter 0."
1934 val = sys.stdin.readline().strip()
1935 if val.isdigit():
1936 return int(val)
1937 else:
1938 print " Not a number."
1939 print " "
1940 return self._get_ticket()
1941
1943 """
1944 Prompt for and read a subcategrory for the DIF_Plus
1945 @return: the text entered by the user
1946 """
1947 print "Sub-category will be automatically filled by server"
1948 return "subcat"
1949
1951 """
1952 Prompt for and read a title for the simulation run.
1953 @return: the text entered by the user
1954 """
1955 import sys, time, re
1956 date_regex = r'^[0-9]{4,4}(-[0-9]{2,2}){2,2}T([0-9]{2,2}:){2,2}[0-9]{2,2}'
1957 print 'Format: (yyyy-mm-ddThh:mm:ss)'
1958 print 'e.g. current local datetime: %s ' % time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime())
1959 print prompt
1960 ret = sys.stdin.readline().strip()
1961 if not ret or not re.search(date_regex,ret):
1962 return self._get_date(prompt)
1963 else:
1964 return ret
1965
1967 import sys
1968 print prompt
1969 for i in range(len(choices)):
1970 print "(%d): %s" % (i,choices[i])
1971 sys.stdout.write('choice: ')
1972 line = sys.stdin.readline().strip()
1973 try:
1974 item = int(line)
1975 print 'You selected (%d): %s.' % (item,choices[item])
1976 sys.stdout.write('is this correct? (Y/N): ')
1977 val = sys.stdin.readline().strip().upper()
1978 if val == 'Y' or val == 'YES':
1979 return choices[item]
1980 else:
1981 return self._get_choice(choices,prompt)
1982 except Exception,e:
1983 print 'Invalid choice: %s' % str(e)
1984 return self._get_choice(choices,prompt)
1985
1987
1988 s = super(submit,self).CheckArgs(args)
1989 if s is not None:
1990 return s
1991 if len(args) > 1:
1992 print "Ignoring additional arguments. Using only ",arg[0]
1993 return None
1994