Package iceprod :: Package client :: Module commands
[hide private]
[frames] | no frames]

Source Code for Module iceprod.client.commands

   1  """ 
   2   Command library for iceprodsh 
   3    
   4   copyright  (c) 2011 the icecube collaboration 
   5    
   6   @version: $Revision: $ 
   7   @date: $Date: $ 
   8   @author: David Schultz <dschultz@icecube.wisc.edu> 
   9  """ 
  10  #from os.path import expandvars 
  11  #from ConfigParser import ConfigParser,SafeConfigParser 
  12   
  13  #from iceprod.core.xmlparser import IceTrayXMLParser 
  14  #from iceprod.core.xmlwriter import IceTrayXMLWriter 
  15  #from iceprod.core.dataclasses import Steering 
  16  #from iceprod.client.soaptrayclient import i3SOAPClient 
  17  import sys 
  18  import iceprod.client 
  19   
  20   
21 -class Command(object):
22 """Command: prototype command""" 23 shortdoc = None
24 - def Execute(self,shell):
25 pass
26 27 numArgs = 0
28 - def CheckArgs(self,args):
29 # fail if numArgs not present 30 if len(args) < self.numArgs: 31 return "Please specify all arguments" 32 self.args = args 33 return None
34
35 - def _getDatasetJob(self):
36 """Convert arguments to dataset,job pairs""" 37 dataset_job = [] 38 for a in self.args: 39 dj = a.split('.') 40 datasets = [] 41 jobs = [] 42 if dj[0].find('-') != -1: 43 # dataset range 44 d = dj[0].split('-') 45 if len(d) < 2 or int(d[0]) > int(d[1]): 46 print "Invalid range for dataset" 47 return [] 48 datasets = range(int(d[0]),int(d[1])+1) 49 elif dj[0].find(',') != -1: 50 # dataset list 51 datasets = map(int,dj[0].split(',')) 52 else: 53 # single dataset 54 datasets.append(int(dj[0])) 55 if len(dj) > 1: 56 # get job ids 57 if dj[1].find('-') != -1: 58 # job range 59 d = dj[1].split('-') 60 if len(d) < 2 or int(d[0]) > int(d[1]): 61 print "Invalid range for job" 62 return [] 63 jobs = range(int(d[0]),int(d[1])+1) 64 elif dj[1].find(',') != -1: 65 # job list 66 jobs = map(int,dj[1].split(',')) 67 else: 68 # single job 69 jobs.append(int(dj[1])) 70 else: 71 jobs.append(-1) 72 for d in datasets: 73 for j in jobs: 74 dataset_job.append((d,j)) 75 return dataset_job
76
77 - def _getDatasetJobStatus(self):
78 """Convert arguments to dataset,job,status tuples""" 79 args2 = [] 80 name = None 81 for a in self.args: 82 if name is None: 83 name = a 84 else: 85 args2.append((name,a)) 86 name = None 87 dataset_job = [] 88 for a,status in args2: 89 dj = a.split('.') 90 datasets = [] 91 jobs = [] 92 if dj[0].find('-') != -1: 93 # dataset range 94 d = dj[0].split('-') 95 if len(d) < 2 or int(d[0]) > int(d[1]): 96 print "Invalid range for dataset" 97 return [] 98 datasets = range(int(d[0]),int(d[1])+1) 99 elif dj[0].find(',') != -1: 100 # dataset list 101 datasets = map(int,dj[0].split(',')) 102 else: 103 # single dataset 104 datasets.append(int(dj[0])) 105 if len(dj) > 1: 106 # get job ids 107 if dj[1].find('-') != -1: 108 # job range 109 d = dj[1].split('-') 110 if len(d) < 2 or int(d[0]) > int(d[1]): 111 print "Invalid range for job" 112 return [] 113 jobs = range(int(d[0]),int(d[1])+1) 114 elif dj[1].find(',') != -1: 115 # job list 116 jobs = map(int,dj[1].split(',')) 117 else: 118 # single job 119 jobs.append(int(dj[1])) 120 else: 121 jobs.append(-1) 122 for d in datasets: 123 for j in jobs: 124 dataset_job.append((d,j,status)) 125 return dataset_job
126
127 - def _getGridDaemon(self):
128 """Convert arguments to grid,daemon pairs""" 129 grid_daemon = [] 130 for a in self.args: 131 gd = a.rsplit('.',1) 132 if gd[0].isdigit(): 133 gd[0] = int(gd[0]) 134 if len(gd) > 1: 135 grid_daemon.append((gd[0],gd[1])) 136 else: 137 grid_daemon.append((gd[0],'all')) 138 return grid_daemon
139
140 - def _getGridDataset(self):
141 """Convert arguments to grid,dataset pairs""" 142 args2 = [] 143 grid = None 144 for a in self.args: 145 if grid is None: 146 grid = a 147 else: 148 args2.append((grid,a)) 149 grid = None 150 grid_dataset = [] 151 for grid,dataset in args2: 152 grids = [] 153 datasets = [] 154 if grid.find(',') != -1: 155 # grid list 156 grids = grid.split(',') 157 else: 158 # single grid 159 grids.append(grid) 160 # get dataset ids 161 if dataset.find('-') != -1: 162 # dataset range 163 d = dataset.split('-') 164 if len(d) < 2 or int(d[0]) > int(d[1]): 165 print "Invalid range for dataset" 166 return [] 167 datasets = range(int(d[0]),int(d[1])+1) 168 elif dataset.find(',') != -1: 169 # dataset list 170 datasets = map(int,dataset.split(',')) 171 else: 172 # single dataset 173 datasets.append(int(dataset)) 174 for g in grids: 175 if g.isdigit(): 176 g = int(g) 177 for d in datasets: 178 grid_dataset.append((g,d)) 179 return grid_dataset
180
181 -class set(Command):
182 """Command: set <variable> <value> [<variable> <value>] 183 184 Set local variable(s) in iceprodsh. 185 Can set multiple variables at once. 186 187 Arguments: 188 <variable> The variable to set 189 (username, url, production, test, editor) 190 <value> The value to set the variable to 191 192 Returns: 193 Echo variables and new values out to shell. 194 Print error on unknown variable. 195 """ 196 numArgs = 2 197 shortdoc = "set <variable> <value> : Set a local variable"
198 - def Execute(self,shell):
199 # format options from self.args list 200 options = {} 201 name = None 202 for a in self.args: 203 if name is None: 204 name = a 205 else: 206 options[name] = a 207 name = None 208 # evaluate options 209 ret = False 210 for opt in options.iterkeys(): 211 if opt == 'username': 212 shell.username = options[opt] 213 shell.password = None 214 shell.cfg.set('iceprodsh','username',options[opt]) 215 elif opt == 'url': 216 if not options[opt].startswith('http'): 217 print 'url must start with http or https' 218 continue 219 from iceprod.client.soaptrayclient import i3SOAPClient 220 shell.url = options[opt] 221 shell.cfg.set('iceprodsh','url',options[opt]) 222 print "now connecting to %s" % shell.url 223 shell.client = i3SOAPClient(url=shell.url) 224 elif opt == 'production': 225 shell.production = int(options[opt]) 226 elif opt == 'test': 227 shell.test = int(options[opt]) 228 elif opt == 'template': 229 shell.template = int(options[opt]) 230 elif opt == 'editor': 231 shell.editor = options[opt] 232 elif opt == 'prefix': 233 shell.prefix = options[opt] 234 else: 235 print "unknown option '%s'" % opt 236 continue 237 print opt,'set to',options[opt] 238 ret = True 239 return ret
240 # return True on success, False on failure 241
242 - def CheckArgs(self,args):
243 # must have pairs of arguments 244 if len(args)%2 == 1: 245 return "Invalid arguments. Must be name value pairs." 246 return super(set,self).CheckArgs(args)
247
248 -class get(Command):
249 """Command: get <variable> [<variable>] 250 251 Get local variable(s) in iceprodsh. 252 Can get multiple variables at once. 253 254 Arguments: 255 <variable> The variable to get 256 (username, url, production, test, editor) 257 258 Returns: 259 Print variables and values out to shell. 260 Print error on unknown variables. 261 """ 262 shortdoc = "get <variable> : Get a local variable" 263 numArgs = 1
264 - def Execute(self,shell):
265 # format options from args list 266 options = {} 267 for a in self.args: 268 options[a] = None 269 # evaluate options 270 for opt in options.iterkeys(): 271 if opt == 'username': 272 options[opt] = shell.username 273 elif opt == 'url': 274 options[opt] = shell.url 275 elif opt == 'production': 276 options[opt] = str(shell.production) 277 elif opt == 'test': 278 options[opt] = str(shell.test) 279 elif opt == 'template': 280 options[opt] = str(shell.template) 281 elif opt == 'editor': 282 options[opt] = shell.editor 283 else: 284 print "unknown option '%s'" % opt 285 continue 286 print opt,'=',options[opt] 287 return options
288 # return dict of <variable>:<value> on success, empty dict on failure 289
290 -class suspend(Command):
291 """Command: suspend <dataset_id>[.<job>] 292 293 Suspend a whole dataset or a specific job from a dataset. 294 Can suspend multiple jobs or datasets at once. 295 296 Arguments: 297 <dataset_id> Specify the dataset to suspend. 298 [.<job>] (Optional) Specify the job within the dataset to suspend. 299 300 Returns: 301 Returns the result of the mysql query (success or failure). 302 Warning that if the dataset or job id is wrong it will likely 303 print success because there was no mysql error. 304 305 Examples: 306 Suspend whole dataset (dataset 1234, all jobs) 307 suspend 1234 308 309 Suspend individual jobs from different datasets 310 (dataset 1234, job 10 and dataset 4321, job 20) 311 suspend 1234.10 4321.20 312 313 Suspend multiple datasets and jobs using commas 314 (datasets 1234, 1243 and jobs 1, 3, and 5) 315 suspend 1234,1243.1,3,5 316 317 Suspend multile datasets and jobs using ranges 318 (datasets 1234 - 1235, jobs 1 - 5) 319 suspend 1234-1235.1-5 320 """ 321 shortdoc = "suspend <dataset_id>[.<job>] : Suspend jobs." 322 numArgs = 1
323 - def Execute(self,shell):
324 # get dataset job pairs 325 dataset_job = self._getDatasetJob() 326 327 # for each dataset and job, suspend 328 ret = True 329 for dataset,job in dataset_job: 330 self._suspend(shell,dataset,job) 331 return ret
332
333 - def _suspend(self,shell,dataset,job):
334 #print "suspend dataset",dataset,"job",job 335 shell.auth() 336 shell.client.q_suspend(shell.username, shell.password, dataset,job)
337
338 -class resume(Command):
339 """Command: resume <dataset_id>[.<job>] 340 341 Resume a dataset or a specific job from a dataset. 342 Can resume multiple jobs or datasets at once. 343 344 Arguments: 345 <dataset_id> Specify the dataset to resume. 346 [.<job>] (Optional) Specify the job within the dataset to resume. 347 348 Returns: 349 Returns the result of the mysql query (success or failure). 350 Warning that if the dataset or job id is wrong it will likely 351 print success because there was no mysql error. 352 353 Examples: 354 Resume whole dataset (dataset 1234, all jobs) 355 resume 1234 356 357 Resume individual jobs from different datasets 358 (dataset 1234, job 10 and dataset 4321, job 20) 359 resume 1234.10 4321.20 360 361 Resume multiple datasets and jobs using commas 362 (datasets 1234, 1243 and jobs 1, 3, and 5) 363 resume 1234,1243.1,3,5 364 365 Resume multile datasets and jobs using ranges 366 (datasets 1234 - 1235, jobs 1 - 5) 367 resume 1234-1235.1-5 368 """ 369 shortdoc = "resume <dataset_id>[.<job>] : Resume jobs." 370 numArgs = 1
371 - def Execute(self,shell):
372 # get dataset job pairs 373 dataset_job = self._getDatasetJob() 374 375 # for each dataset and job, resume 376 ret = True 377 for dataset,job in dataset_job: 378 self._resume(shell,dataset,job) 379 return ret
380
381 - def _resume(self,shell,dataset,job):
382 #print "resume dataset",dataset,"job",job 383 shell.auth() 384 shell.client.q_resume(shell.username, shell.password, dataset,job)
385
386 -class reset(Command):
387 """Command: reset <dataset_id>[.<job>] 388 389 Reset a dataset or a specific job from a dataset. 390 Can reset multiple jobs or datasets at once. 391 392 Arguments: 393 <dataset_id> Specify the dataset to reset. 394 [.<job>] (Optional) Specify the job within the dataset to reset. 395 396 Returns: 397 Returns the result of the mysql query (success or failure). 398 Warning that if the dataset or job id is wrong it will likely 399 print success because there was no mysql error. 400 401 Examples: 402 Reset whole dataset (dataset 1234, all jobs) 403 reset 1234 404 405 Reset individual jobs from different datasets 406 (dataset 1234, job 10 and dataset 4321, job 20) 407 reset 1234.10 4321.20 408 409 Reset multiple datasets and jobs using commas 410 (datasets 1234, 1243 and jobs 1, 3, and 5) 411 reset 1234,1243.1,3,5 412 413 Reset multile datasets and jobs using ranges 414 (datasets 1234 - 1235, jobs 1 - 5) 415 reset 1234-1235.1-5 416 """ 417 shortdoc = "reset <dataset_id>[.<job>] : Reset jobs." 418 numArgs = 1
419 - def Execute(self,shell):
420 # get dataset job pairs 421 dataset_job = self._getDatasetJob() 422 423 # for each dataset and job, reset 424 ret = True 425 for dataset,job in dataset_job: 426 self._reset(shell,dataset,job) 427 return ret
428
429 - def _reset(self,shell,dataset,job):
430 #print "reset dataset",dataset,"job",job 431 shell.auth() 432 shell.client.q_reset(shell.username, shell.password, dataset,job)
433
434 -class status(Command):
435 """Command: status <dataset_id>[.<job>] 436 or 437 getstatus <dataset_id>[.<job>] 438 439 Get the status of all jobs or a specific job from a dataset. 440 Can get the status of multiple jobs over multiple datasets at once. 441 442 Arguments: 443 <dataset_id> Specify the dataset. 444 [.<job>] (Optional) Specify the job within the dataset. 445 446 Returns: 447 Returns the result of the mysql query (success or failure). 448 Warning that if the dataset or job id is wrong it will likely 449 print success because there was no mysql error. 450 451 Examples: 452 Get status of whole dataset (dataset 1234, all jobs) 453 status 1234 454 455 Get status of individual jobs from different datasets 456 (dataset 1234, job 10 and dataset 4321, job 20) 457 status 1234.10 4321.20 458 459 Get status of multiple datasets and jobs using commas 460 (datasets 1234, 1243 and jobs 1, 3, and 5) 461 status 1234,1243.1,3,5 462 463 Get status of multile datasets and jobs using ranges 464 (datasets 1234 - 1235, jobs 1 - 5) 465 status 1234-1235.1-5 466 """ 467 shortdoc = "status <dataset_id>[.<job>] : Get status of jobs. AKA: getstatus" 468 numArgs = 1
469 - def Execute(self,shell):
470 # get dataset job pairs 471 dataset_job = self._getDatasetJob() 472 473 # for each dataset and job, get status 474 ret = True 475 for dataset,job in dataset_job: 476 self._status(shell,dataset,job) 477 return ret
478
479 - def _status(self,shell,dataset,job):
480 #print "status dataset",dataset,"job",job 481 shell.client.q_status(dataset,job)
482
483 -class getstatus(status):
484 __doc__ = status.__doc__ 485 shortdoc = "getstatus <dataset_id>[.<job>] : Get status of jobs."
486
487 -class datasetstatus(Command):
488 """Command: datasetstatus <dataset_id> 489 or 490 getdatasetstatus <dataset_id> 491 492 Get the status of the dataset. 493 Can get the status of multiple datasets at once. 494 495 Arguments: 496 <dataset_id> Specify the dataset. 497 498 Returns: 499 Returns the result of the mysql query (success or failure). 500 Warning that if the dataset is wrong it will likely 501 print success because there was no mysql error. 502 503 Examples: 504 Get status of single dataset (dataset 1234) 505 datasetstatus 1234 506 507 Get status of multiple datasets using spaces 508 (dataset 1234 and dataset 4321) 509 datasetstatus 1234 4321 510 511 Get status of multiple datasets using commas 512 (datasets 1234, 1243) 513 datasetstatus 1234,1243 514 515 Get status of multile datasets using ranges 516 (datasets 1234 - 1235) 517 datasetstatus 1234-1235 518 """ 519 shortdoc = "datasetstatus <dataset_id> : Get status of dataset. AKA: getdatasetstatus" 520 numArgs = 1
521 - def Execute(self,shell):
522 # get dataset job pairs 523 dataset_job = self._getDatasetJob() 524 525 # for each dataset and job, get status 526 ret = True 527 for dataset,job in dataset_job: 528 self._datasetstatus(shell,dataset) 529 return ret
530
531 - def _datasetstatus(self,shell,dataset):
532 #print "status dataset",dataset 533 shell.client.q_dataset_getstatus(dataset)
534
535 -class getdatasetstatus(datasetstatus):
536 __doc__ = datasetstatus.__doc__ 537 shortdoc = "getdatasetstatus <dataset_id> : Get status of dataset."
538
539 -class setstatus(Command):
540 """Command: setstatus <dataset_id>[.<job>] <status> 541 542 Set the status of all jobs or a specific job from a dataset. 543 Can set the status of multiple jobs over multiple datasets at once. 544 545 Arguments: 546 <dataset_id> Specify the dataset. 547 [.<job>] (Optional) Specify the job within the dataset. 548 <status> Specify the status 549 (WAITING, QUEUEING, QUEUED, PROCESSING, OK, ERROR, 550 READYTOCOPY, COPYING, SUSPENDED, RESET, FAILED, 551 COPIED, EVICTED, CLEANING, IDLE) 552 553 Returns: 554 Returns the result of the mysql query (success or failure). 555 Warning that if the dataset or job id is wrong it will likely 556 print success because there was no mysql error. 557 558 Examples: 559 Set status of all jobs in dataset to RESET (dataset 1234, all jobs) 560 status 1234 RESET 561 562 Set status of individual jobs from different datasets 563 (dataset 1234, job 10 to waiting and dataset 4321, job 20 to RESET) 564 status 1234.10 WAITING 4321.20 RESET 565 566 Set status of multiple datasets and jobs using commas 567 (jobs 1, 3, and 5 from datasets 1234 and 1243 to OK) 568 status 1234,1243.1,3,5 OK 569 570 Set status of multile datasets and jobs using ranges 571 (datasets 1234 - 1235, jobs 1 - 5 to SUSPENDED) 572 status 1234-1235.1-5 SUSPENDED 573 """ 574 shortdoc = "setstatus <dataset_id>[.<job>] <status> : Set status of jobs." 575 numArgs = 2
576 - def Execute(self,shell):
577 # get dataset job pairs 578 dataset_job = self._getDatasetJobStatus() 579 580 # for each dataset and job, set status 581 ret = True 582 for dataset,job,status in dataset_job: 583 self._status(shell,dataset,job,status) 584 return ret
585
586 - def _status(self,shell,dataset,job,status):
587 #print "status dataset",dataset,"job",job,"status",status 588 shell.auth() 589 shell.client.q_setstatus(shell.username,shell.password,dataset,job,status)
590
591 - def CheckArgs(self,args):
592 # must have pairs of arguments 593 if len(args)%2 == 1: 594 return "Invalid arguments. Must be grid dataset pairs." 595 return super(setstatus,self).CheckArgs(args)
596
597 -class setdatasetstatus(Command):
598 """Command: setstatus <dataset_id> <status> 599 600 Set the status of a dataset. 601 Can set the status of multiple datasets at once. 602 603 Arguments: 604 <dataset_id> Specify the dataset. 605 <status> Specify the status 606 (PROCESSING, COMPLETE, ERRORS, READYTOPUBLISH, 607 MOVING, OBSOLETE, READYTODELETE, DELETED, TEMPLATE) 608 609 Returns: 610 Returns the result of the mysql query (success or failure). 611 Warning that if the dataset is wrong it will likely 612 print success because there was no mysql error. 613 614 Examples: 615 Set status of dataset to PROCESSING 616 status 1234 PROCESSING 617 618 Set status of multiple datasets to different states 619 (dataset 1234 to PROCESSING and dataset 4321 to ERRORS) 620 status 1234 PROCESSING 4321 ERRORS 621 622 Set status of multiple datasets using commas 623 (datasets 1234, 1243 to COMPLETE) 624 status 1234,1243 COMPLETE 625 626 Set status of multile datasets using ranges 627 (datasets 1234 - 1235 to READYTOPUBLISH) 628 status 1234-1235 READYTOPUBLISH 629 """ 630 shortdoc = "setdatasetstatus <dataset_id> <status> : Set status of dataset." 631 numArgs = 2
632 - def Execute(self,shell):
633 # get dataset status pairs 634 dataset_job = self._getDatasetJobStatus() 635 636 # for each dataset, set status 637 ret = True 638 for dataset,job,status in dataset_job: 639 self._status(shell,dataset,status) 640 return ret
641
642 - def _status(self,shell,dataset,status):
643 #print "status dataset",dataset,"status",status 644 shell.auth() 645 shell.client.q_dataset_setstatus(shell.username,shell.password,dataset,status)
646
647 - def CheckArgs(self,args):
648 # must have pairs of arguments 649 if len(args)%2 == 1: 650 return "Invalid arguments. Must be grid dataset pairs." 651 return super(setdatasetstatus,self).CheckArgs(args)
652
653 -class clean(Command):
654 """Command: clean <dataset_id> 655 656 Clean a dataset. 657 Can clean multiple datasets at once. 658 659 Arguments: 660 <dataset_id> Specify the dataset to clean. 661 662 Returns: 663 Returns the result of the mysql query (success or failure). 664 Warning that if the dataset is wrong it will likely 665 print success because there was no mysql error. 666 667 Examples: 668 Clean single dataset (dataset 1234) 669 clean 1234 670 671 Clean multiple datasets using commas 672 (datasets 1234, 1243) 673 clean 1234,1243 674 675 Clean multile datasets using ranges 676 (datasets 1234 - 1235) 677 clean 1234-1235 678 """ 679 shortdoc = "clean <dataset_id> : Clean a dataset." 680 numArgs = 1
681 - def Execute(self,shell):
682 # get datasets 683 dataset_job = self._getDatasetJob() 684 685 # for each dataset, clean 686 ret = True 687 for dataset,job in dataset_job: 688 self._clean(shell,dataset) 689 return ret
690
691 - def _clean(self,shell,dataset):
692 #print "clean dataset",dataset 693 shell.auth() 694 shell.client.q_clean(shell.username, shell.password, dataset)
695
696 -class finish(Command):
697 """Command: finish <dataset_id> 698 699 Finish a dataset. 700 Can finish multiple datasets at once. 701 702 Arguments: 703 <dataset_id> Specify the dataset to finish. 704 705 Returns: 706 Returns the result of the mysql query (success or failure). 707 Warning that if the dataset is wrong it will likely 708 print success because there was no mysql error. 709 710 Examples: 711 Finish single dataset (dataset 1234) 712 finish 1234 713 714 Finish multiple datasets using commas 715 (datasets 1234, 1243 and jobs 1, 3, and 5) 716 finish 1234,1243 717 718 Finish multile datasets using ranges 719 (datasets 1234 - 1235, jobs 1 - 5) 720 finish 1234-1235 721 """ 722 shortdoc = "finish <dataset_id> : Finish a dataset." 723 numArgs = 1
724 - def Execute(self,shell):
725 # get datasets 726 dataset_job = self._getDatasetJob() 727 728 # for each dataset, finish 729 ret = True 730 for dataset,job in dataset_job: 731 self._finish(shell,dataset) 732 return ret
733
734 - def _finish(self,shell,dataset):
735 #print "finish dataset",dataset 736 shell.auth() 737 shell.client.q_finish(shell.username, shell.password, dataset)
738
739 -class retire(Command):
740 """Command: retire <dataset_id> 741 742 Retire a dataset. Send it to the archives. 743 Can retire multiple datasets at once. 744 745 Arguments: 746 <dataset_id> Specify the dataset to retire. 747 748 Returns: 749 Returns the result of the mysql query (success or failure). 750 Warning that if the dataset is wrong it will likely 751 print success because there was no mysql error. 752 753 Examples: 754 Retire single dataset (dataset 1234) 755 retire 1234 756 757 Retire multiple datasets using commas 758 (datasets 1234, 1243 and jobs 1, 3, and 5) 759 retire 1234,1243 760 761 Retire multile datasets using ranges 762 (datasets 1234 - 1235, jobs 1 - 5) 763 retire 1234-1235 764 """ 765 shortdoc = "retire <dataset_id> : Retire a dataset." 766 numArgs = 1
767 - def Execute(self,shell):
768 # get datasets 769 dataset_job = self._getDatasetJob() 770 771 # for each dataset, retire 772 ret = True 773 for dataset,job in dataset_job: 774 self._retire(shell,dataset) 775 return ret
776
777 - def _retire(self,shell,dataset):
778 #print "retire dataset",dataset 779 shell.auth() 780 shell.client.q_retire(shell.username, shell.password, dataset)
781
782 -class nuke(Command):
783 """Command: nuke <dataset_id> 784 785 Nuke (delete) a dataset. Similar to 'hide'+'clean', but more 786 destructive. Can nuke multiple datasets at once. 787 788 Arguments: 789 <dataset_id> Specify the dataset to nuke. 790 791 Returns: 792 Returns the result of the mysql query (success or failure). 793 Warning that if the dataset is wrong it will likely 794 print success because there was no mysql error. 795 796 Examples: 797 Nuke single dataset (dataset 1234) 798 nuke 1234 799 800 Nuke multiple datasets using commas 801 (datasets 1234, 1243 and jobs 1, 3, and 5) 802 nuke 1234,1243 803 804 Nuke multile datasets using ranges 805 (datasets 1234 - 1235, jobs 1 - 5) 806 nuke 1234-1235 807 """ 808 shortdoc = "nuke <dataset_id> : Nuke (delete) a dataset." 809 numArgs = 1
810 - def Execute(self,shell):
811 # get datasets 812 dataset_job = self._getDatasetJob() 813 814 # for each dataset, nuke 815 ret = True 816 for dataset,job in dataset_job: 817 self._nuke(shell,dataset) 818 return ret
819
820 - def _nuke(self,shell,dataset):
821 #print "nuke dataset",dataset 822 shell.auth() 823 shell.client.q_delete(shell.username, shell.password, dataset)
824
825 -class valid(Command):
826 """Command: valid <dataset_id> 827 or 828 show <dataset_id> 829 830 Validate a dataset and let it show up in iceprod. 831 Can validate multiple datasets at once. 832 833 Arguments: 834 <dataset_id> Specify the dataset to validate. 835 836 Returns: 837 Returns the result of the mysql query (success or failure). 838 Warning that if the dataset id is wrong it will likely 839 print success because there was no mysql error. 840 841 Examples: 842 Validate single dataset (dataset 1234) 843 valid 1234 844 845 Validate multiple datasets using commas 846 (datasets 1234, 1243 and jobs 1, 3, and 5) 847 valid 1234,1243 848 849 Validate multile datasets using ranges 850 (datasets 1234 - 1235, jobs 1 - 5) 851 valid 1234-1235 852 """ 853 shortdoc = "valid <dataset_id> : Validate a dataset. AKA: show" 854 numArgs = 1
855 - def Execute(self,shell):
856 # get datasets 857 dataset_job = self._getDatasetJob() 858 859 # for each dataset, validate 860 ret = True 861 for dataset,job in dataset_job: 862 self._valid(shell,dataset,True) 863 return ret
864
865 - def _valid(self,shell,dataset,state):
866 #print "validate dataset",dataset,"state",state 867 shell.auth() 868 shell.client.q_validate(shell.username, shell.password, dataset,state)
869
870 -class show(valid):
871 __doc__ = valid.__doc__ 872 shortdoc = "show <dataset_id> : Validate a dataset. AKA: validate"
873
874 -class invalid(valid):
875 """Command: invalid <dataset_id> 876 or 877 nvalid <dataset_id> 878 or 879 hide <dataset_id> 880 881 Invalidate a dataset and hide it from iceprod. 882 Can invalidate multiple datasets at once. 883 884 Arguments: 885 <dataset_id> Specify the dataset to invalidate. 886 887 Returns: 888 Returns the result of the mysql query (success or failure). 889 Warning that if the dataset id is wrong it will likely 890 print success because there was no mysql error. 891 892 Examples: 893 Invalidate single dataset (dataset 1234) 894 invalid 1234 895 896 Invalidate multiple datasets using commas 897 (datasets 1234, 1243 and jobs 1, 3, and 5) 898 invalid 1234,1243 899 900 Invalidate multile datasets using ranges 901 (datasets 1234 - 1235, jobs 1 - 5) 902 invalid 1234-1235 903 """ 904 shortdoc = "invalid <dataset_id> : Invalidate a dataset. AKA: nvalid,hide" 905 numArgs = 1
906 - def Execute(self,shell):
907 # get datasets 908 dataset_job = self._getDatasetJob() 909 910 # for each dataset, invalidate 911 ret = True 912 for dataset,job in dataset_job: 913 self._valid(shell,dataset,False) 914 return ret
915
916 -class nvalid(invalid):
917 __doc__ = invalid.__doc__ 918 shortdoc = "nvalid <dataset_id> : Invalidate a dataset. AKA: invalid,hide"
919
920 -class hide(invalid):
921 __doc__ = invalid.__doc__ 922 shortdoc = "hide <dataset_id> : Invalidate a dataset. AKA: invalid,nvalid"
923
924 -class startserver(Command):
925 """Command: startserver <grid>[.<daemon>] 926 927 Start a server or a specific daemon on the server. 928 Can start multiple servers and daemons at once. 929 930 Only works if main iceprodd daemon is running on the server, 931 otherwise the change is only in the database. 932 933 Arguments: 934 <grid> Specify the grid to start, either by name or id 935 [.<daemon>] (Optional) Specify the daemon to start. 936 (all,soapdh,soapqueue,soaphist,soapmon) 937 938 Returns: 939 Returns the result of the mysql query (success or failure). 940 Warning that if the grid or daemon is wrong it will likely 941 print success because there was no mysql error. 942 943 Examples: 944 Start GLOW 945 startserver GLOW 946 or 947 startserver 1 948 949 Start soapqueue on GLOW and soapdh on glow-test 950 startserver GLOW.soapqueue glow-test.soapdh 951 952 Start EGEE.Madison, or others with a dot in the name 953 (must specify daemon to get grid name correct) 954 startserver EGEE.Madison.all 955 """ 956 shortdoc = "startserver <grid>[.<daemon>] : Start an iceprod server." 957 numArgs = 1
958 - def Execute(self,shell):
959 # get grid, daemon pairs 960 grid_daemon = self._getGridDaemon() 961 962 # for each grid and daemon, start 963 ret = True 964 for grid,daemon in grid_daemon: 965 self._start(shell,grid,daemon) 966 return ret
967
968 - def _start(self,shell,grid,daemon):
969 print "start grid",grid,"daemon",daemon 970 shell.auth() 971 shell.client.q_daemon_resume(shell.username, shell.password, grid, daemon)
972
973 -class stopserver(Command):
974 """Command: stopserver <grid>[.<daemon>] 975 976 Stop a server or a specific daemon on the server. 977 Can stop multiple servers and daemons at once. 978 979 Only works if main iceprodd daemon is running on the server, 980 otherwise the change is only in the database. 981 982 Arguments: 983 <grid> Specify the grid to stop, either by name or id 984 [.<daemon>] (Optional) Specify the daemon to stop. 985 (soapdh,soapqueue,soaphist,soapmon) 986 987 Returns: 988 Returns the result of the mysql query (success or failure). 989 Warning that if the grid or daemon is wrong it will likely 990 print success because there was no mysql error. 991 992 Examples: 993 Stop GLOW 994 stopserver GLOW 995 or 996 stopserver 1 997 998 Stop soapqueue on GLOW and soapdh on glow-test 999 stopserver GLOW.soapqueue glow-test.soapdh 1000 """ 1001 shortdoc = "stopserver <grid>[.<daemon>] : Stop an iceprod server." 1002 numArgs = 1
1003 - def Execute(self,shell):
1004 # get grid, daemon pairs 1005 grid_daemon = self._getGridDaemon() 1006 1007 # for each grid and daemon, stop 1008 ret = True 1009 for grid,daemon in grid_daemon: 1010 self._stop(shell,grid,daemon) 1011 return ret
1012
1013 - def _stop(self,shell,grid,daemon):
1014 print "stop grid",grid,"daemon",daemon 1015 shell.auth() 1016 shell.client.q_daemon_suspend(shell.username, shell.password, grid, daemon)
1017 1018
1019 -class includegrid(Command):
1020 """Command: includegrid <grid> <dataset_id> 1021 1022 Resume a dataset on a grid. 1023 Can resume multiple datasets and grids at once. 1024 1025 Arguments: 1026 <grid> Specify the grid, either by name or id 1027 <dataset_id> Specify dataset to act on 1028 1029 Returns: 1030 Returns the result of the mysql query (success or failure). 1031 Warning that if the grid or dataset is wrong it will likely 1032 print success because there was no mysql error. 1033 1034 Examples: 1035 Resume 1234 on GLOW 1036 includegrid GLOW 1234 1037 or 1038 includegrid 1 1234 1039 1040 Resume 1234 and 1243 on GLOW and glow-test 1041 includegrid GLOW,glow-test 1234,1243 1042 """ 1043 shortdoc = "includegrid <grid> <dataset_id> : Resume a dataset on a grid." 1044 numArgs = 2
1045 - def Execute(self,shell):
1046 # get grid, dataset pairs 1047 grid_dataset = self._getGridDataset() 1048 1049 # for each grid and dataset, resume 1050 ret = True 1051 for grid,dataset in grid_dataset: 1052 self._include(shell,grid,dataset) 1053 return ret
1054
1055 - def _include(self,shell,grid,dataset):
1056 print "include grid",grid,"dataset",dataset 1057 shell.auth() 1058 shell.client.q_grid_resume_dataset(shell.username, shell.password, grid, dataset)
1059
1060 - def CheckArgs(self,args):
1061 # must have pairs of arguments 1062 if len(args)%2 == 1: 1063 return "Invalid arguments. Must be grid dataset pairs." 1064 return super(includegrid,self).CheckArgs(args)
1065
1066 -class excludegrid(Command):
1067 """Command: excludegrid <grid> <dataset_id> 1068 1069 Suspend a dataset on a grid 1070 Can suspend multiple datasets and grids at once. 1071 1072 Arguments: 1073 <grid> Specify the grid, either by name or id 1074 <dataset_id> Specify dataset to act on 1075 1076 Returns: 1077 Returns the result of the mysql query (success or failure). 1078 Warning that if the grid or dataset is wrong it will likely 1079 print success because there was no mysql error. 1080 1081 Examples: 1082 Suspend 1234 on GLOW 1083 excludegrid GLOW 1234 1084 or 1085 excludegrid 1 1234 1086 1087 Suspend 1234 and 1243 on GLOW and glow-test 1088 excludegrid GLOW,glow-test 1234,1243 1089 """ 1090 shortdoc = "excludegrid <grid> <dataset_id> : Suspend a dataset on a grid." 1091 numArgs = 2
1092 - def Execute(self,shell):
1093 # get grid, dataset pairs 1094 grid_dataset = self._getGridDataset() 1095 1096 # for each grid and dataset, suspend 1097 ret = True 1098 for grid,dataset in grid_dataset: 1099 self._exclude(shell,grid,dataset) 1100 return ret
1101
1102 - def _exclude(self,shell,grid,dataset):
1103 print "exclude grid",grid,"dataset",dataset 1104 shell.auth() 1105 shell.client.q_grid_suspend_dataset(shell.username, shell.password, grid, dataset)
1106
1107 - def CheckArgs(self,args):
1108 # must have pairs of arguments 1109 if len(args)%2 == 1: 1110 return "Invalid arguments. Must be grid dataset pairs." 1111 return super(excludegrid,self).CheckArgs(args)
1112
1113 -class addgrid(Command):
1114 """Command: addgrid <grid> <dataset_id> 1115 1116 Add a grid to a dataset 1117 Can add multiple grids to multiple datasets at once. 1118 1119 Arguments: 1120 <grid> Specify the grid, either by name or id 1121 <dataset_id> Specify dataset to act on 1122 1123 Returns: 1124 Returns the result of the mysql query (success or failure). 1125 Warning that if the grid or dataset is wrong it will likely 1126 print success because there was no mysql error. 1127 1128 Examples: 1129 Add GLOW to 1234 1130 addgrid GLOW 1234 1131 or 1132 addgrid 1 1234 1133 1134 Add GLOW to 1234 and glow-test to 4321 1135 addgrid GLOW 1234 glow-test 4321 1136 1137 Add GLOW and glow-test to 1234 and 4321 1138 addgrid GLOW,glow-test 1234,4321 1139 """ 1140 shortdoc = "addgrid <grid> <dataset_id> : Add a grid to a dataset." 1141 numArgs = 2
1142 - def Execute(self,shell):
1143 # get grid, dataset pairs 1144 grid_dataset = self._getGridDataset() 1145 1146 # for each grid and dataset, add 1147 ret = True 1148 for grid,dataset in grid_dataset: 1149 self._add(shell,grid,dataset) 1150 return ret
1151
1152 - def _add(self,shell,grid,dataset):
1153 print "add grid",grid,"dataset",dataset 1154 shell.auth() 1155 shell.client.q_grid_add_dataset(shell.username, shell.password, grid, dataset)
1156
1157 - def CheckArgs(self,args):
1158 # must have pairs of arguments 1159 if len(args)%2 == 1: 1160 return "Invalid arguments. Must be grid dataset pairs." 1161 return super(addgrid,self).CheckArgs(args)
1162
1163 -class open(Command):
1164 """Command: open <filename> 1165 1166 Open a steering file. 1167 1168 Arguments: 1169 <filename> File name to open 1170 1171 Returns: 1172 No return if successful, error on failure. 1173 1174 Examples: 1175 Open file dataset_1234.xml 1176 open dataset_1234.xml 1177 """ 1178 shortdoc = "open <filename> : Open a steering file." 1179 numArgs = 1
1180 - def Execute(self,shell):
1181 from iceprod.core.xmlparser import IceTrayXMLParser 1182 from iceprod.core.dataclasses import Steering 1183 shell.steering = Steering() 1184 shell.filename = self.args[0] 1185 myparser = IceTrayXMLParser(shell.steering) 1186 myparser.ParseFile(shell.filename,validate=shell.xmlvalidate)
1187
1188 -class edit(Command):
1189 """Command: edit 1190 1191 Edit the currently open steering file. 1192 (Must have previously opened or downloaded a steering file) 1193 1194 Arguments: none 1195 1196 Returns: 1197 Opens the steering file in the editor set by the 1198 environment variable EDITOR. Default is vi. 1199 """ 1200 shortdoc = "edit : Edit the currently open steering file."
1201 - def Execute(self,shell):
1202 import sys,os,time 1203 from iceprod.core.xmlparser import IceTrayXMLParser 1204 from iceprod.core.xmlwriter import IceTrayXMLWriter 1205 from iceprod.core.dataclasses import Steering 1206 if not shell.steering: 1207 print "No configuration loaded" 1208 return 1209 writer = IceTrayXMLWriter(shell.steering) 1210 tmpfile = ".iceprod.%u.tmp.xml" % time.time() 1211 writer.write_to_file(tmpfile) 1212 newsteering = None 1213 while not newsteering: 1214 os.system("%s %s" % (shell.editor,tmpfile)) 1215 myparser = IceTrayXMLParser(Steering()) 1216 newsteering = myparser.ParseFile(tmpfile,validate=shell.xmlvalidate) 1217 if not newsteering: 1218 print "Hit any key to continue." 1219 sys.stdin.readline() 1220 shell.steering = newsteering 1221 os.remove(tmpfile)
1222
1223 - def CheckArgs(self,args):
1224 if len(args) > 0: 1225 print "Ignoring arguments" 1226 return None
1227
1228 -class close(Command):
1229 """Command: close 1230 1231 Close the currently open steering file. 1232 (Does nothing if no file is open) 1233 1234 Arguments: none 1235 1236 Returns: none 1237 """ 1238 shortdoc = "close : Close the currently open steering file."
1239 - def Execute(self,shell):
1240 shell.steering = None
1241
1242 - def CheckArgs(self,args):
1243 if len(args) > 0: 1244 print "Ignoring arguments" 1245 return None
1246
1247 -class save(Command):
1248 """Command: save [<filename>] 1249 1250 Save the currently open steering file. 1251 (Must have previously opened or downloaded a steering file) 1252 1253 Arguments: 1254 <filename> (Optional) Name of file to save to. 1255 If not specified, filename is either the 1256 file it was opened from, the file it was 1257 last saved to, or the dataset id if it 1258 was downloaded. 1259 1260 Returns: 1261 Saves steering to file. No visible return. 1262 """ 1263 shortdoc = "save [<filename>] : Save the currently open steering file."
1264 - def Execute(self,shell):
1265 from iceprod.core.xmlwriter import IceTrayXMLWriter 1266 if not shell.steering: 1267 print "No configuration loaded" 1268 return 1269 if len(self.args) > 0: 1270 shell.filename = self.args[0] 1271 writer = IceTrayXMLWriter(shell.steering) 1272 writer.write_to_file(shell.filename)
1273
1274 -class download(Command):
1275 """Command: download <dataset> 1276 1277 Download a dataset (replaces currently open steering file) 1278 1279 Arguments: 1280 <dataset> Specify dataset id to download. 1281 1282 Returns: 1283 Downloads steering from database and opens it. 1284 """ 1285 shortdoc = "download <dataset> : Download a dataset." 1286 numArgs = 1
1287 - def Execute(self,shell):
1288 dataset = int(self.args[0]) 1289 shell.steering = shell.client.download_config(dataset) 1290 shell.filename = 'dataset_%u.xml' % dataset
1291
1292 - def CheckArgs(self,args):
1293 # must have single integer argument 1294 s = super(download,self).CheckArgs(args) 1295 if s is not None: 1296 return s 1297 if not self.args[0].isdigit(): 1298 return "Invalid argument. Must be integer." 1299 if len(args) > 1: 1300 print "Ignoring additional arguments. Using only ",arg[0] 1301 return None
1302 1303 # DEPRICATED 1304 #class summary(Command): 1305 # """Command: summary [<days>] 1306 # 1307 # Print a summary of the production sites for the last few days. 1308 # 1309 # Arguments: 1310 # <days> (Optional) Specify the number of days to look back. 1311 # Defaults to 7 days. 1312 # 1313 # Returns: 1314 # Prints summary to terminal. 1315 # """ 1316 # shortdoc = "summary [<days>] : Print a summary of the production sites." 1317 # def Execute(self,shell): 1318 # if len(self.args) < 1: 1319 # days = 7 1320 # else: 1321 # days = int(self.args[0]) 1322 # try: 1323 # print shell.client.printsummary(days) 1324 # except Exception,e: 1325 # print e 1326 # 1327 # def CheckArgs(self,args): 1328 # # can have single integer argument 1329 # s = super(summary,self).CheckArgs(args) 1330 # if s is not None: 1331 # return s 1332 # if len(self.args) > 0 and not self.args[0].isdigit(): 1333 # return "Invalid argument. Must be integer." 1334 # if len(args) > 1: 1335 # print "Ignoring additional arguments. Using only ",arg[0] 1336 # return None 1337
1338 -class loadfiles(Command):
1339 """Command: loadfiles <rootdir> [<rootdir>] [regex=<regex>] [dataset=<dataset_id>] 1340 1341 Get list of files in rootdir(s) and add them to database 1342 dictionary table if they match the regex. 1343 1344 Arguments: 1345 <rootdir> Directory of files 1346 regex=<regex> (Optional) A regex to match against 1347 regex=<regex> (Optional) A regex to match against 1348 1349 Returns: 1350 Error message on failure. 1351 """ 1352 shortdoc = "loadfiles <rootdir> [<rootdir>] [regex=<regex>] [dataset=<dataset_id>]" 1353 numArgs = 1 # must have at least one argument
1354 - def Execute(self,shell):
1355 import os, re 1356 from iceprod.core import odict 1357 odict = OrderedDict(); 1358 if self.regex: cregex = re.compile(self.regex) 1359 filecount = 0 1360 for root in self.args: 1361 for path, dirs, files in os.walk(root): 1362 for file in files: 1363 if not self.regex or cregex.match(file): 1364 truncated_path = path.replace(shell.prefix,'') 1365 filename = os.path.join(truncated_path,file) 1366 key = filecount 1367 odict[key] = filename 1368 print key, odict[key] 1369 filecount += 1 1370 shell.client.loaddict(odict, shell.username, shell.password,self.dataset) 1371 return len(odict)
1372
1373 - def CheckArgs(self,args):
1374 s = super(summary,self).CheckArgs(args) 1375 if s is not None: 1376 return s 1377 # find a regex or dataset in the args 1378 self.regex = None 1379 self.dataset = 0 1380 for a in args: 1381 if a.startswith('regex='): 1382 self.regex = a.split('=',1)[1] 1383 self.args.remove(a) 1384 elif a.startswith('dataset='): 1385 self.dataset = a.split('=',1)[1] 1386 self.args.remove(a) 1387 return None
1388
1389 -class exit(Command):
1390 """Command: exit 1391 or quit 1392 1393 Exit iceprodsh. 1394 Also saves the default configuration settings 1395 and history to the home directory. 1396 """ 1397 shortdoc = "exit : Exit iceprodsh. AKA: quit"
1398 - def Execute(self,shell):
1399 import os,readline,__builtin__ 1400 cfgfile = __builtin__.open(os.path.join(os.getenv('HOME'),".iceprodshrc"),'w') 1401 shell.cfg.write(cfgfile) 1402 cfgfile.close() 1403 print "adios." 1404 try: 1405 readline.write_history_file(os.path.expandvars('$HOME/.iceprodsh_history')) 1406 except:pass 1407 os._exit(0)
1408
1409 -class quit(exit):
1410 __doc__ = exit.__doc__ 1411 shortdoc = "quit : Exit iceprodsh. AKA: exit"
1412
1413 -class usage(Command):
1414 """Command: usage 1415 or help 1416 1417 Print usage information. 1418 This is the basic help system. 1419 """ 1420 shortdoc = "usage : Print usage information. AKA: help"
1421 - def Execute(self,shell):
1422 self._usage()
1423 1424 usage = """Usage: iceprodsh [option] <url> <command> <args> 1425 1426 options: 1427 1428 -i, 1429 --interactive : Interactive shell (default). 1430 1431 -h, 1432 --help : This screen. 1433 1434 -r <url>, 1435 --url=<url> : Specify a url of soaptray server. 1436 1437 -u <username>, 1438 --username=<username> : Specify a username. 1439 1440 --production : Authenticate, update production database 1441 and create metadata. For production by 1442 authorized users. 1443 --test : Specify submissions as tests. 1444 1445 --meta=<file> : Use .ini style meta-data file instead of 1446 interactive form. 1447 -m <key:value> : Key value pair to override metadata file. 1448 Can use multiple times. 1449 1450 --prefix=<value> : Specify prefix for files. 1451 1452 -v <value>, 1453 --validate=<value> : Turn validation on or off. 1454 """
1455 - def _usage(self):
1456 command_list = self._listcommands() 1457 print self.usage 1458 print " " 1459 print "commands:" 1460 for cmd in command_list: 1461 print " " 1462 if cmd.shortdoc is None: 1463 print " "+cmd.__doc__.split("\n",1)[0].split(':',1)[1] 1464 else: 1465 print " "+cmd.shortdoc 1466 print " "
1467
1468 - def _listcommands(self):
1469 import inspect 1470 command_list = [] 1471 for name, obj in inspect.getmembers(inspect.getmodule(Command)): 1472 if inspect.isclass(obj) and name != "Command": 1473 command_list.append(obj) 1474 return command_list
1475
1476 -class help(usage):
1477 """Command: help [<command>] 1478 1479 Lists the help, either for general usage or for specific commands. 1480 1481 Arguments: 1482 <command> (Optional) If omitted, prints usage. 1483 If specified, prints help for specific command. 1484 """ 1485 shortdoc = "help [<command>] : Print usage or detailed help"
1486 - def Execute(self,shell):
1487 if len(self.args) < 1: 1488 self._usage() 1489 return 1490 1491 item = self.args[0] 1492 command_list = self._listcommands() 1493 modules = {'iceprod':'iceprod', 1494 'iceprod.core':'iceprod.core', 1495 'iceprod.server':'iceprod.server', 1496 'iceprod.client':'iceprod.client', 1497 'iceprod.modules':'iceprod.modules', 1498 'core':'iceprod.core', 1499 'server':'iceprod.server', 1500 'client':'iceprod.client', 1501 'modules':'iceprod.modules'} 1502 modules.update(self._listmodules('iceprod')) 1503 modules.update(self._listmodules('iceprod.core')) 1504 modules.update(self._listmodules('iceprod.server')) 1505 modules.update(self._listmodules('iceprod.client')) 1506 modules.update(self._listmodules('iceprod.modules')) 1507 for m in modules: 1508 if item == m: 1509 self._pythonhelp(modules[item]) 1510 return 1511 elif item.startswith(m): 1512 self._pythonhelp(item) 1513 return 1514 for cmd in command_list: 1515 if cmd.__name__ == item: 1516 print cmd.__doc__ 1517 return 1518 1519 print "Item not found in help"
1520
1521 - def _listmodules(self,package_name=''):
1522 import os,imp 1523 package_name_os = package_name.replace('.','/') 1524 file, pathname, description = imp.find_module(package_name_os) 1525 if file: 1526 # Not a package 1527 return {} 1528 # Use a set because some may be both source and compiled. 1529 ret = {} 1530 for module in os.listdir(pathname): 1531 if module.endswith('.py') and module != '__init__.py': 1532 tmp = os.path.splitext(module)[0] 1533 ret[tmp] = package_name+'.'+tmp 1534 return ret
1535
1536 - def _pythonhelp(self,item):
1537 import iceprod 1538 import iceprod.core 1539 import iceprod.server 1540 import iceprod.client 1541 import iceprod.modules 1542 import __builtin__ 1543 __builtin__.help(item)
1544
1545 -class submit(Command):
1546 """Command: submit [<filename>] 1547 1548 Submit a new dataset. If in production mode, submits to iceprod. 1549 Otherwise, the dataset is run locally. 1550 1551 Arguments: 1552 <filename> (Optional) Specify the filename to submit. 1553 Defaults to the currently open file. 1554 1555 Returns: 1556 Prints success or failure, as well as other details. 1557 """ 1558 shortdoc = "submit [<filename>] : Submit a new dataset."
1559 - def Execute(self,shell):
1560 try: 1561 import signal 1562 from iceprod.core.xmlparser import IceTrayXMLParser 1563 from iceprod.client.soaptrayclient import i3SOAPClient 1564 from iceprod.core.dataclasses import Steering 1565 1566 default_handler = { 1567 signal.SIGQUIT: signal.getsignal(signal.SIGQUIT), 1568 signal.SIGINT: signal.getsignal(signal.SIGINT) , 1569 signal.SIGCHLD: signal.getsignal(signal.SIGCHLD), 1570 } 1571 1572 1573 def handler(signum,frame): 1574 shell.logger.warn("caught signal %s" % signum) 1575 raise Exception, "Operation cancelled"
1576 1577 signal.signal(signal.SIGQUIT, handler) 1578 signal.signal(signal.SIGINT, handler) 1579 1580 print "production flag is set to %s" % str(shell.production) 1581 print "test flag is set to %s" % str(shell.test) 1582 print "template flag is set to %s" % str(shell.template) 1583 1584 if len(self.args) >= 1: 1585 shell.steering = Steering() 1586 myparser = IceTrayXMLParser(shell.steering) 1587 if not myparser.ParseFile(self.args[0],validate=shell.xmlvalidate): 1588 raise "unable to parse configuration file" 1589 1590 if not shell.steering: 1591 shell.logger.fatal("no steering object loaded:Cannot submit") 1592 return False 1593 1594 # Prompt for description and add to steering object 1595 shell.descmap['geometry'] = '%(geometry)s' 1596 shell.descmap['simcat'] = '%(simcat)s' 1597 shell.descmap['composition'] = '%(composition)s' 1598 shell.descmap['weighted'] = '%(weighted)s' 1599 shell.descmap['spectrum'] = '%(spectrum)s' 1600 shell.descmap['icemodel'] = '%(icemodel)s' 1601 shell.descmap['photonpropagator'] = '%(photonpropagator)s' 1602 shell.descmap['angularrange'] = '%(angularrange)s' 1603 shell.descmap['energyrange'] = '%(energyrange)s' 1604 1605 # do submit based on type 1606 if shell.meta: 1607 self._meta(shell) 1608 return 1609 elif shell.production: 1610 steering = self._production(shell) 1611 else: 1612 steering = shell.steering 1613 1614 shell.auth() 1615 # Instantiate SOAP client and submit job to cluster. 1616 i3q = shell.client.submit( steering, shell.username, shell.password, shell.production) 1617 if i3q: 1618 shell.client.check_q(i3q, shell.username, shell.password) 1619 1620 # Reset signal handers back to original 1621 signal.signal(signal.SIGQUIT, default_handler[signal.SIGQUIT]) 1622 signal.signal(signal.SIGINT, default_handler[signal.SIGINT]) 1623 except Exception,e: 1624 apply(sys.excepthook,sys.exc_info()) 1625 print e 1626
1627 - def _meta(self,shell):
1628 import time 1629 from ConfigParser import ConfigParser,SafeConfigParser 1630 from iceprod.core import metadata, lex 1631 # read meta-data from file 1632 metafile = ConfigParser() 1633 steering = shell.steering 1634 print "reading parameters from", shell.meta 1635 try: 1636 metafile.read(shell.meta) 1637 except Exception,e: 1638 shell.logger.fatal("%s:Cannot read %s" % (e,shell.meta)) 1639 raise 1640 for key,val in shell.metadict.items(): 1641 try: 1642 metafile.set('iceprod-meta',key,val) 1643 except Exception,e: 1644 shell.logger.fatal(key + ":"+ e) 1645 1646 grids = metafile.get('iceprod-meta','grid') 1647 steering.AddExtra("Grid", grids) 1648 1649 maxjobs = metafile.getint('iceprod-meta','maxjobs') 1650 steering.AddExtra("Maxjobs", maxjobs) 1651 1652 ticket = metafile.getint('iceprod-meta','ticket') 1653 steering.AddExtra("Ticket",ticket) 1654 1655 simcat = metafile.get('iceprod-meta','sim-category') 1656 steering.SetCategory(simcat) 1657 1658 dtype = metafile.get('iceprod-meta','dataset-type') 1659 steering.SetDatasetType(dtype) 1660 1661 # Initialize metafile 1662 difplus = metadata.DIF_Plus() 1663 dif = difplus.GetDIF() 1664 plus = difplus.GetPlus() 1665 1666 dtext = metafile.get('iceprod-meta','title') 1667 dif.SetEntryTitle(dtext) 1668 1669 cat = metafile.get('iceprod-meta','category') 1670 plus.SetCategory(cat) 1671 plus.SetSubCategory(self._get_subcat()) 1672 datetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1673 plus.SetStartDatetime(datetime) 1674 plus.SetEndDatetime(datetime) 1675 1676 dtext = dif.GetValidParameters()[0] 1677 dif.SetParameters(dtext) 1678 1679 steering.AddExtra("Metadata",difplus) 1680 1681 dtext = metafile.get('iceprod-meta','description',raw=True) 1682 1683 # iterate over steering parameters and store them parsed in dict 1684 shell.descmap['simcat'] = steering.GetCategory() 1685 shell.descmap['category'] = plus.GetCategory() 1686 shell.descmap['subcategory'] = plus.GetSubCategory() 1687 1688 #instantiate a parser to evaluate expressions 1689 expparser = lex.ExpParser( { 1690 'extern':0, 1691 'procnum':0, 1692 'tray':0, 1693 'iter':0, 1694 'dataset':0, 1695 'nproc': int(steering.GetParameter('MAXJOBS').GetValue()), 1696 }, 1697 steering) 1698 1699 # iterate over steering parameters and store them parsed in dict 1700 for p in steering.GetParameters(): 1701 try: 1702 shell.descmap[p.GetName()] = expparser.parse(p.GetValue()) 1703 except Exception,e: pass 1704 1705 try: 1706 dtext = dtext % shell.descmap 1707 except Exception,e: 1708 shell.logger.fatal(e) 1709 steering.SetDescription(dtext) 1710 dif.SetSummary(dtext) 1711 1712 username = metafile.get('iceprod-meta','username') 1713 passwd = metafile.get('iceprod-meta','passwd') 1714 i3q = shell.client.submit( steering, username, passwd, 1) 1715 if i3q: 1716 shell.client.check_q(i3q, username, passwd)
1717
1718 - def _production(self,shell):
1719 import time 1720 from iceprod.core import metadata, lex 1721 from iceprod.core.dataclasses import SimulationCategories, DatasetTypes 1722 # submit to iceprod 1723 steering = shell.steering 1724 1725 if shell.template: 1726 grids = 'Hut' 1727 else: 1728 # Prompt for grid 1729 msg = "Grid: enter the name(s) of grids to run this dataset on:" 1730 grids = self._get_text(msg) 1731 steering.AddExtra("Grid", grids) 1732 1733 if shell.template: 1734 maxjobs = 10 1735 else: 1736 # Prompt for number of jobs 1737 msg = "Maxjobs: enter the number of jobs in this dataset:" 1738 maxjobs = self._get_int(msg) 1739 steering.AddExtra("Maxjobs", maxjobs) 1740 1741 # Initialize metadata 1742 difplus = metadata.DIF_Plus() 1743 dif = difplus.GetDIF() 1744 plus = difplus.GetPlus() 1745 1746 # Prompt for title description and add to metadata object 1747 if shell.test: 1748 dtext = 'Test Dataset' 1749 elif shell.template: 1750 dtext = 'Template' 1751 else: 1752 dtext = 'Production Dataset' 1753 #dtext = self._get_title() 1754 dif.SetEntryTitle(dtext) 1755 1756 # Prompt for ticket number 1757 ticket = 0 1758 if not shell.test and not shell.template: 1759 ticket = self._get_ticket() 1760 steering.AddExtra("Ticket",ticket) 1761 1762 # Prompt for simulation category choice 1763 msg = "Simulation Category: Enter a generator from the choices below:" 1764 if shell.test: 1765 cat = 'Test' 1766 elif shell.template: 1767 cat = 'Template' 1768 else: 1769 cat = self._get_choice(SimulationCategories, msg) 1770 steering.SetCategory(cat) 1771 1772 # Prompt for category choice 1773 msg = "Category: enter a number from the choices below:" 1774 if shell.test or shell.template: 1775 dtext = 'unclassified' 1776 else: 1777 dtext = self._get_choice(plus.GetValidCategories(),msg) 1778 plus.SetCategory(dtext) 1779 1780 plus.SetSubCategory(self._get_subcat()) 1781 1782 #if self.test: 1783 # startdatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1784 #else: 1785 # startdatetime = get_date('Please enter a start validity datetime: ') 1786 startdatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1787 plus.SetStartDatetime(startdatetime) 1788 1789 #if self.test: 1790 # enddatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1791 #else: 1792 # enddatetime = get_date('Please enter an end validity datetime: ') 1793 enddatetime = time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1794 plus.SetEndDatetime(enddatetime) 1795 1796 for iconfig in steering.GetTrays(): 1797 # Add projects from any included metaprojects to DIFPlus 1798 for metaproject in iconfig.GetMetaProjectList(): 1799 for project in metaproject.GetProjectList(): 1800 plus.AddProject(project) 1801 1802 # Add projects not included in any metaprojects to DIFPlus 1803 for project in iconfig.GetProjectList(): 1804 plus.AddProject(project) 1805 1806 msg = "Parameters: enter a number from the choices below:" 1807 #if self.test: 1808 # dtext = dif.GetValidParameters()[0] 1809 #else: 1810 # dtext = get_choice(dif.GetValidParameters(),msg) 1811 dtext = dif.GetValidParameters()[0] 1812 dif.SetParameters(dtext) 1813 1814 # Prompt for dataset category 1815 msg = "Dataset type: enter a number from the choices below:" 1816 if shell.test: 1817 dtext = 'TEST' 1818 elif shell.template: 1819 dtext = 'TEMPLATE' 1820 else: 1821 dtext = self._get_choice(DatasetTypes, msg) 1822 steering.SetDatasetType(dtext) 1823 1824 #instantiate a parser to evaluate expressions 1825 expparser = lex.ExpParser( { 1826 'extern':0, 1827 'procnum':0, 1828 'tray':0, 1829 'iter':0, 1830 'dataset':0, 1831 'nproc': int(steering.GetParameter('MAXJOBS').GetValue()), 1832 }, 1833 steering) 1834 1835 # iterate over steering parameters and store them parsed in dict 1836 for p in steering.GetParameters(): 1837 try: 1838 shell.descmap[p.GetName()] = expparser.parse(p.GetValue()) 1839 except Exception,e: pass 1840 shell.descmap['simcat'] = steering.GetCategory() 1841 1842 descriptionstr = iceprod.client.descriptionstr 1843 what = steering.GetParameter('description') 1844 if what: 1845 descriptionstr = what.GetValue() 1846 else: 1847 steering.AddParameter("string","description",descriptionstr) 1848 1849 #initialize descripton 1850 description = descriptionstr % shell.descmap 1851 dtext = self._get_description(shell.completer,description) 1852 1853 if shell.test: 1854 dtext = 'Test: ' + dtext 1855 steering.SetDescription(dtext) 1856 dif.SetSummary(dtext) 1857 1858 steering.AddExtra("Metadata",difplus) 1859 return steering
1860
1861 - def _get_description(self,completer,initial=''):
1862 """ 1863 Prompt for and read a description for the simulation run. 1864 @return: the text entered by the user as a single string 1865 """ 1866 import sys 1867 1868 print 'Please type a brief description of this run.' 1869 print 'You may write multiple lines. To finish just enter a blank line:' 1870 1871 text = [] 1872 try: 1873 import readline 1874 import rlcompleter 1875 except: 1876 print 'e.g. "%s"' % initial 1877 line = sys.stdin.readline().strip() 1878 while line: 1879 text.append(line) 1880 line = sys.stdin.readline().strip() 1881 else: 1882 readline.parse_and_bind("tab: complete") 1883 readline.set_completer(completer) 1884 #readline.insert_text(initial) 1885 readline.add_history(initial) 1886 readline.set_startup_hook(None) 1887 while True: 1888 try: 1889 line = raw_input('> ') 1890 except EOFError: 1891 break 1892 if not line: break 1893 text.append(line) 1894 1895 return ' '.join(text)
1896
1897 - def _get_text(self,prompt):
1898 """ 1899 Prompt for input 1900 @return: the text entered by the user 1901 """ 1902 import sys 1903 print prompt 1904 ret = sys.stdin.readline().strip() 1905 if not ret: 1906 return self._get_text(prompt) 1907 else: 1908 return ret
1909
1910 - def _get_int(self,prompt):
1911 return int(self._get_text(prompt))
1912
1913 - def _get_title(self):
1914 """ 1915 Prompt for and read a title for the simulation run. 1916 @return: the text entered by the user 1917 """ 1918 import sys 1919 print 'Please type title for this run:' 1920 ret = sys.stdin.readline().strip() 1921 if not ret: 1922 return self._get_title() 1923 else: 1924 return ret
1925
1926 - def _get_ticket(self):
1927 """ 1928 Prompt for and read a ticket number associated with dataset. 1929 @return: the number entered 1930 """ 1931 import sys 1932 print "Is there a ticket number associated with this dataset?" 1933 print "If so, enter it. Otherwise enter 0." 1934 val = sys.stdin.readline().strip() 1935 if val.isdigit(): 1936 return int(val) 1937 else: 1938 print " Not a number." 1939 print " " 1940 return self._get_ticket()
1941
1942 - def _get_subcat(self):
1943 """ 1944 Prompt for and read a subcategrory for the DIF_Plus 1945 @return: the text entered by the user 1946 """ 1947 print "Sub-category will be automatically filled by server" 1948 return "subcat"
1949
1950 - def _get_date(self,prompt):
1951 """ 1952 Prompt for and read a title for the simulation run. 1953 @return: the text entered by the user 1954 """ 1955 import sys, time, re 1956 date_regex = r'^[0-9]{4,4}(-[0-9]{2,2}){2,2}T([0-9]{2,2}:){2,2}[0-9]{2,2}' 1957 print 'Format: (yyyy-mm-ddThh:mm:ss)' 1958 print 'e.g. current local datetime: %s ' % time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime()) 1959 print prompt 1960 ret = sys.stdin.readline().strip() 1961 if not ret or not re.search(date_regex,ret): 1962 return self._get_date(prompt) 1963 else: 1964 return ret
1965
1966 - def _get_choice(self,choices,prompt=''):
1967 import sys 1968 print prompt 1969 for i in range(len(choices)): 1970 print "(%d): %s" % (i,choices[i]) 1971 sys.stdout.write('choice: ') 1972 line = sys.stdin.readline().strip() 1973 try: 1974 item = int(line) 1975 print 'You selected (%d): %s.' % (item,choices[item]) 1976 sys.stdout.write('is this correct? (Y/N): ') 1977 val = sys.stdin.readline().strip().upper() 1978 if val == 'Y' or val == 'YES': 1979 return choices[item] 1980 else: 1981 return self._get_choice(choices,prompt) 1982 except Exception,e: 1983 print 'Invalid choice: %s' % str(e) 1984 return self._get_choice(choices,prompt)
1985
1986 - def CheckArgs(self,args):
1987 # can have single string argument 1988 s = super(submit,self).CheckArgs(args) 1989 if s is not None: 1990 return s 1991 if len(args) > 1: 1992 print "Ignoring additional arguments. Using only ",arg[0] 1993 return None
1994