#!/bin/bash
#
# automate TestDAQ, and other scripts...
# Mark Krasberg, May, 2004
#
# your classpath must be previously defined!
#
# Modified to take a second argument, either -a or -t:
#   -a: process steering files alphabetically
#   -t: process steering files in reverse time order (i.e., FIFO)
#
# Doug Cowen, Jan 2006
#
# stuff to do at init:
#
echo STARTING AUTOMATE at `date`

rm -f stf.log
rm testdaq.log
#rm domhubapp.log

killall rmiregistry
#killall testdaq
#killall python

sleep 2

#
# start rmi_registry
#
$JAVA_HOME/bin/rmiregistry &
#/usr/local/bin/startpulser &
#/home/testdaq/bin/startpulser2 &
#/usr/local/bin/startfilterwheel &
#/usr/local/bin/startmonochromator &
sleep 5

#
#
# now loop through the tests we want...
#
#

#steering_dir=$1/run_5e6_dn_fullreadout

#steering_dir=$1/special_steering_files/darknoise
###steering_dir=$1/interim_run_plan
#steering_dir=/mnt/data/testdaq/special_steering_files/flasherboard/part3/
#steering_dir=/mnt/data/testdaq/junk2/
#steering_dir=$1/dec2005checkout/ 
#steering_dir=$1/interim_run_plan2

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21flasher
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/icetop/
#steering_dir=/mnt/data/testdaq/special_steering_files/iniceonly/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29/
#steering_dir=$1/interim_run_plan3

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str29flasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/icetopflasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str29lc/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29str39/flasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str39/flasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29str39/flasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29str39/flasher/HoleIce/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/triboluminescence/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str29flasher/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str39/HoleIce/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29str39/flasher/fbout_ver_4dom/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21flasher/spencer-run/

#steering_dir=/mnt/data/testdaq/special_steering_files/muon-tagger/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str30str21str29/flasher/HoleIce/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/4strings/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str30/SanityCheckFlasher/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str21str29str30str39/muons/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str39/HoleIce/pass3b/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/6strings-316DOMs/muons/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/5strings-268DOMs/muons/
#steering_dir=/mnt/data/testdaq/special_steering_files/standardcandle/
#steering_dir=/mnt/data/testdaq/special_steering_files/muon-tagger/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str30/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/test2/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str38/

#steering_dir=/mnt/data/testdaq/special_steering_files/standardcandle/string39/
#steering_dir=/mnt/data/testdaq/special_steering_files/standardcandle/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/triboluminescence/
#steering_dir=/mnt/data/testdaq/special_steering_files/amanda/
#steering_dir=/mnt/data/testdaq/special_steering_files/standardcandle/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str40/InitialVerification2/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str39str40/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/8hubs/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/7hubs/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str38/InitialVerification/


#steering_dir=/mnt/data/testdaq/special_steering_files/icetop/standard-steering-file/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str50/


#steering_dir=/mnt/data/testdaq/special_steering_files/muon-tagger/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/10hubs/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/11hubs/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/str59/InitialVerification/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/10hubs/

#steering_dir=/mnt/data/testdaq/special_steering_files/icetop/standard-steering-file/newstations/post_lc/

#steering_dir=/net/data/serap/xmls/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/lcruns/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/W2F/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/icetop/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/PMTLIN/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/icetop/

#steering_dir=/mnt/data/testdaq/special_steering_files/brokenLC/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/INITVER_FBOUT/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/icetop/

#steering_dir=/mnt/data/testdaq/special_steering_files/lctests/

#steering_dir=/mnt/data/testdaq/special_steering_files/Fusilli/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/BULK_AND_PMTLIN_AND_FBOUT/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/lcruns/

#steering_dir=/mnt/data/testdaq/special_steering_files/high-rate-flasher-runs

#steering_dir=/mnt/data/testdaq/special_steering_files/amanda/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/darknoiseruns/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/icetop/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/darknoiseruns/

#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/lcruns-hit/




#
# first set of runs - DO THESE RUNS EVERY TIME!
#

#steering_dir=/mnt/data/testdaq/special_steering_files/special-darknoise-run/
#steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/icetop/
#steering_dir=/mnt/data/testdaq/special_steering_files/do-these-runs-every-time/
#steering_dir=/mnt/data/testdaq/special_steering_files/dawn_weekly/
#steering_dir=/mnt/data/testdaq/special_steering_files/dark-noise-only/

#steering_dir=/mnt/data/testdaq/special_steering_files/weekly_runs/
#steering_dir=/mnt/data/testdaq/special_steering_files/dn-10sec/

steering_dir=/mnt/data/testdaq/special_steering_files/do-these-runs-every-time/
#steering_dir=/mnt/data/testdaq/special_steering_files/special-darknoise-run/

#steering_dir=/mnt/data/testdaq/special_steering_files/verification/BULK_AND_PMTLIN_AND_FBOUT/
#steering_dir=/mnt/data/testdaq/special_steering_files/verification/BULK_AND_PMTLIN_AND_FBOUT/


if [[ "$2" == "-t" ]]; then
   SteeringFiles=`ls -1rt $steering_dir/*.xml`
fi

if [[ "$2" == "-a" ]]; then
   SteeringFiles=`ls -1 $steering_dir/*.xml`
fi

if [[ "$2" == "" ]]; then
   echo "automate requires either a -t or -a argument. Exiting."
   exit
fi

#  for steering in $steering_dir/*.xml; do

#while [ 1 -gt 0 ]; do

  daqid=`cat /usr/local/etc/.domhub_name`  
#   daqid=$FAT_HUB
  runno=`cat /usr/local/etc/.run_number`     
  checklogs.pl init ${daqid}_run${runno} $*


  for steering in $SteeringFiles; do
    increment_run_number.pl
    sleep 1
    stbase=`basename $steering`
    runtype=${stbase%.xml}
    sync
    daqid=`cat /usr/local/etc/.domhub_name`
    runno=`cat /usr/local/etc/.run_number`
    mkdir $HOME/output${runno}

    tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
    while [ ${tooquick} -gt 1 ]; do
      echo " ERROR prevent_too_many_bad_runs already running - we will wait one minute before we start a new TestDAQ run"
      tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
      sleep 60
    done
    nohup prevent_too_many_bad_runs &

    testdaq $steering $HOME/output${runno} ${daqid}_run${runno}_$runtype \
	-d sps-ichub21 -d sps-ichub29 -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50  -d sps-ichub59 -d sps-amanda -d sps-ithub01 -d sps-ithub02
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda -d sps-ithub01 -d sps-ithub02
#     -d sps-amanda
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda
#     -d sps-ithub01 -d sps-ithub02
#	-d sps-ichub21 -d sps-ichub29 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50 -d sps-ichub30 -d sps-ichub59    -d sps-ithub01 -d sps-ithub02 -d sps-amanda
#        -d spts-ichub-cont01 -p localhost
#        -d domhub2 -d domhub3 -p localhost -m localhost -f localhost
#        -d spts-ichub-cont01 -p localhost
#        -p localhost -m localhost -f localhost
    bkgs=`ps -ef | grep background_it.pl -c`
    maxproc=2
    while [ ${bkgs} -gt ${maxproc} ]; do
       echo "sleeping for 60 seconds because there are $bkgs background_it.pl"
       sleep 60
       bkgs=`ps -ef | grep background_it.pl -c`
    done
#    sleep 20
  done
#done

#exit

#
# 2nd set of runs - the user specified runs
#

steering_dir=/mnt/data/testdaq/special_steering_files/verification/BULK_AND_PMTLIN_AND_FBOUT/
steering_dir=/mnt/data/testdaq/special_steering_files/standard-lc-run/

if [[ "$2" == "-t" ]]; then
   SteeringFiles=`ls -1rt $steering_dir/*.xml`
fi

if [[ "$2" == "-a" ]]; then
   SteeringFiles=`ls -1 $steering_dir/*.xml`
fi

if [[ "$2" == "" ]]; then
   echo "automate requires either a -t or -a argument. Exiting."
   exit
fi

#  for steering in $steering_dir/*.xml; do


#while [ 1 -gt 0 ]; do

  for steering in $SteeringFiles; do
    increment_run_number.pl
    sleep 1
    stbase=`basename $steering`
    runtype=${stbase%.xml}
    sync
    daqid=`cat /usr/local/etc/.domhub_name`
    runno=`cat /usr/local/etc/.run_number`
    mkdir $HOME/output${runno}

    tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
    while [ ${tooquick} -gt 1 ]; do
      echo " ERROR prevent_too_many_bad_runs already running - we will wait one minute before we start a new TestDAQ run"
      tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
      sleep 60
    done
    nohup prevent_too_many_bad_runs &

    testdaq $steering $HOME/output${runno} ${daqid}_run${runno}_$runtype \
	-d sps-ichub21 -d sps-ichub29 -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50  -d sps-ichub59 -d sps-ithub01 -d sps-ithub02 -d sps-amanda
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda -d sps-ithub01 -d sps-ithub02
#     -d sps-amanda
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda
#     -d sps-ithub01 -d sps-ithub02
#	-d sps-ichub21 -d sps-ichub29 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50 -d sps-ichub30 -d sps-ichub59    -d sps-ithub01 -d sps-ithub02 -d sps-amanda
#        -d spts-ichub-cont01 -p localhost
#        -d domhub2 -d domhub3 -p localhost -m localhost -f localhost
#        -d spts-ichub-cont01 -p localhost
#        -p localhost -m localhost -f localhost
    bkgs=`ps -ef | grep background_it.pl -c`
    maxproc=2
    while [ ${bkgs} -gt ${maxproc} ]; do
       echo "sleeping for 60 seconds because there are $bkgs background_it.pl"
       sleep 60
       bkgs=`ps -ef | grep background_it.pl -c`
    done
#    sleep 20
  done
#done



#
# if we are out of runs then do the muon runs in a continuous loop at the end...
#

steering_dir=/mnt/data/testdaq/special_steering_files/frozenver/lcruns/
steering_dir=/mnt/data/testdaq/special_steering_files/standard-lc-run/

if [[ "$2" == "-t" ]]; then
   SteeringFiles=`ls -1rt $steering_dir/*.xml`
fi

if [[ "$2" == "-a" ]]; then
   SteeringFiles=`ls -1 $steering_dir/*.xml`
fi

if [[ "$2" == "" ]]; then
   echo "automate requires either a -t or -a argument. Exiting."
   exit
fi


while [ 1 -gt 0 ]; do

  for steering in $SteeringFiles; do
    increment_run_number.pl
    sleep 1
    stbase=`basename $steering`
    runtype=${stbase%.xml}
    sync
    daqid=`cat /usr/local/etc/.domhub_name`
    runno=`cat /usr/local/etc/.run_number`
    mkdir $HOME/output${runno}

    tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
    while [ ${tooquick} -gt 1 ]; do
      echo " ERROR prevent_too_many_bad_runs already running - we will wait one minute before we start a new TestDAQ run"
      tooquick=`ps -ef | grep prevent_too_many_bad_runs -c`
      sleep 60
    done
    nohup prevent_too_many_bad_runs &

    testdaq $steering $HOME/output${runno} ${daqid}_run${runno}_$runtype \
	-d sps-ichub21 -d sps-ichub29 -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50  -d sps-ichub59    -d sps-ithub01 -d sps-ithub02 -d sps-amanda
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda -d sps-ithub01 -d sps-ithub02
#     -d sps-amanda
#     -d sps-ichub30 -d sps-ichub38 -d sps-ichub39 -d sps-ichub21 -d sps-ichub29 -d sps-amanda
#     -d sps-ithub01 -d sps-ithub02
#	-d sps-ichub21 -d sps-ichub29 -d sps-ichub38 -d sps-ichub39 -d sps-ichub40 -d sps-ichub49 -d sps-ichub50 -d sps-ichub30 -d sps-ichub59    -d sps-ithub01 -d sps-ithub02 -d sps-amanda
#        -d spts-ichub-cont01 -p localhost
#        -d domhub2 -d domhub3 -p localhost -m localhost -f localhost
#        -d spts-ichub-cont01 -p localhost
#        -p localhost -m localhost -f localhost
    bkgs=`ps -ef | grep background_it.pl -c`
    maxproc=2
    while [ ${bkgs} -gt ${maxproc} ]; do
       echo "sleeping for 60 seconds because there are $bkgs background_it.pl"
       sleep 60
       bkgs=`ps -ef | grep background_it.pl -c`
    done
#    sleep 20
  done
done
checklogs.pl fini ${daqid}_run${runno} $*
