#!/usr/bin/perl
#print
#"Note: this program is in development; functionality is subject change without warning.\n\n";


# domhub.pl
# the basic domhub untility!
#
# Author: Victor Bittorf
# Author: Mark Krasberg
#
#
# This program is used to run commands on DOMHubs
# remotely. The result of the command is returned
# to the user on the remote string-proc.
#

use threads;
use threads::shared;
use strict;

my $VERSION = 3.62;
my $starttime = time();
use vars qw ($starttime);

#print "$starttime\n";

#	see (almost) EOF for the sub 'chooseDomhubs' if you
#	want to change the key-words for different DOMHub
#	Lists

# The # of characters allowed per hub...
my $CHARACTERS_PER_HUB = 3;
my @warnings           = ();
# charactesr allowed per label (i.e. crd0 or Hubs: )
my $CHARACTERS_PER_LABEL = 6;
my $HUBS_PER_ROW         = 20;
# The number of hubs to display per table...
my $HUBS_PER_TABLE = ( 75 - $CHARACTERS_PER_LABEL ) / $CHARACTERS_PER_HUB;
my $HOME = $ENV{HOME};
my $command;
my @domhubs;
my $hubs;
my %Dash_Map = ();


# search for -seq in args...

# make ARGV the temp arr (an array w/o -seq, if present)

my @temp_arr = ();

foreach (@ARGV) {
	if (/^\-\w+$/) {
		$Dash_Map{$_} = 1;
		print "Running with `dash command' : $_\n";
#		return 0;
	} else {
#		return 1;
 push (@temp_arr, $_);
 
	}
} 

@ARGV = @temp_arr;
         

if ( defined $ARGV[0] ) { $hubs = $ARGV[0]; }
if ( defined $ARGV[1] ) { $command = $ARGV[1]; }
else {
	if ( $hubs eq "table" || $hubs eq "status" ) {
		$hubs    = "all";
		$command = "table";
	}
}

if ( defined $ARGV[2] ) {
  print " \n\n";
  print " too many arguments!!!! \n";
  print " \n put command line arguments inside quotes\n";
  print " eg: domhub all \"versions all\"\n\n";
  exit(1);
}
          
my %Completed_Threads : shared = ();

# map commands to sub references....
my %commandHooks = (
	"checkGPS"          => \&checkGPS,
	"getstffails"       => \&getFails,
	"getcurrent"        => \&getCurrent,
	"getvarlogmessages" => \&getVarLogMsg,
	"getdomcal"         => \&getDomcal,
        "table"             => \&table,
	"status"             => \&table,
	"monitorDomhubApp"  => \&monitorApp,
	"pDown"             => \&pDown,
	"pUp"               => \&pUp,
	"pCycle"            => \&pCycle,
	"ready-iceboot"     => \&readyIceboot,
	"quickstatus"       => \&quickstatus,
	"quicktable"        => \&quickstatus,
        "checkVersions"     => \&checkVersions,
);

# choose the domhubs to use!
print "DOMHub (v$VERSION)   using command: $command\n";
        
chooseDomhubs();
@domhubs = sort { ( $a =~ /(\d+)/ )[0] <=> ( $b =~ /(\d+)/ )[0] } @domhubs;
if ( $commandHooks{$command} ) {

	# in this case, we know that the command is in our hash
#	print "DOMHub (v$VERSION)   using command: $command\n";
	my $ref = $commandHooks{$command};
	&$ref(@domhubs);
	exit(0);
}
elsif ( defined($command) ) {

	# we know we have a command,
	# but we don't understand it.
	# print "Threading $command onto @domhubs...\n";
#	print "DOMHub (v$VERSION)   using command: $command\n";
	print "Threading $command onto @domhubs...\n";
	my %threadMap = dispatchApps2( "$command", @domhubs );
	my @keys      = keys(%threadMap);
	my @infoArr   = joinThreads(%threadMap);
	my $str       = join( "\n", @infoArr );
	print "$str", "\n";
	exit(0);
}

# prints the usage for this program.
print "usage:\tdomhub [domhubs] [command]\n\n";
print "version: $VERSION\n";
my @keys = keys %commandHooks;
my $str = join( ", ", @keys );
print "supported commands: $str\n\n";

# # # # # # # # # #
#
#
exit(0);    #
            #

#
# # # # # # # # # #

# does the monitor Domhub App thing...
# Not exactly sure what that does yet - but here it is!
sub monitorApp {
	my @domhubs = @_;
	while (1) {
		my %threadMap   = dispatchApps( "monitorDomhubApp", @domhubs );
		my @infoArr     = joinThreads(%threadMap);
		my @singleLines = ();
		foreach my $block (@infoArr) {
			@singleLines = ( @singleLines, split( "\n", $block ) );
		}
		monitorHACK(@singleLines);
		sleep(1);
	}
}


# takes a map of threads and joins the threads, returning
# each thread result in an array...
# the order of the array is based on the order that
# the keys() function returns the keys for the hash
sub joinThreads {
	my %threadMap = @_;
	my $len       = keys %threadMap;
	my @returnArr = ();
	print "Waiting for $len DOMHubs to finish...\n";
	my @mapKeys =
	  sort { ( $a =~ /(\d+)/ )[0] <=> ( $b =~ /(\d+)/ )[0] } keys(%threadMap);
	my %reverseMap = ();
	foreach (@mapKeys) {
		$reverseMap{ $threadMap{$_}->tid() } = $_;
	}

#        my $endtime = time();
#        my $duration = $endtime - $starttime;
	foreach my $key (@mapKeys) {
		my $thr   = $threadMap{$key};
		my $done  = 0;
		my $count = 0;
		while ( not $done ) {
			$count++;
			my @threads_to_join = ();
			{
				lock %Completed_Threads;
				$done = $Completed_Threads{ $thr->tid() };
			}
			if ( not $done ) {
				my @initialarr = threads->list();
                                my @arr = ();
				my $open_thread_count = 0;
				foreach (@initialarr) {
				        my $initial = $_;
					if (!$Completed_Threads{$_->tid()}) {
					  $open_thread_count++;
					  push( @arr, $initial);
					}
				}
     				my $num = $#arr + 1;
#				print "open thread count = $open_thread_count\n";
				if ( $#arr eq 0 ) {
					my $first = $reverseMap{ $arr[0]->tid() };
					print "Waiting on $num DOMHUBs... [ $first ] \n";
				}
				elsif ( $#arr eq 1 ) {
					my $first = $reverseMap{ $arr[0]->tid() };
					my $sec   = $reverseMap{ $arr[1]->tid() };
					print "Waiting on $num DOMHUBs... [ $first, $sec ] \n";
                                }
				elsif ( $#arr eq 2 ) {
					my $first = $reverseMap{ $arr[0]->tid() };
					my $sec   = $reverseMap{ $arr[1]->tid() };
					my $third = $reverseMap{ $arr[2]->tid() };
					print "Waiting on $num DOMHUBs... [ $first, $sec, $third ] \n";
                                }
				elsif ( $#arr eq 3 ) {
					my $first  = $reverseMap{ $arr[0]->tid() };
					my $sec    = $reverseMap{ $arr[1]->tid() };
					my $third  = $reverseMap{ $arr[2]->tid() };
					my $fourth = $reverseMap{ $arr[3]->tid() };
					print "Waiting on $num DOMHUBs... [ $first, $sec, $third, $fourth ] \n";
				} else {
#					my $open_thread_count = 0;
#					foreach (@arr) {
#						if (!$Completed_Threads{$_->tid()}) {
#							$open_thread_count++;
#						}
#					}
					print "Waiting on $num DOMHUBs... \n";
				}
			}
			if ( $count % 5 eq 0 ) {
                                my $endtime = time();
                                my $duration = $endtime - $starttime;
				print "$key is taking a long time...              ($duration seconds so far)\n";
			}
			sleep 4 unless $done;
		}

		my $stuff = $thr->join();
		if ( $stuff eq "" ) {
			push( @warnings, ">>> Hub did not respond: $key." );
		}
		else {
			push( @returnArr, $stuff );
		}
	}
        my $endtime = time();
        my $duration = $endtime - $starttime;
	print "All DOMHubs have finished.                      (took $duration seconds)\n\n\n";
	return @returnArr;
}

# a simple function that takes a command line
# and threads the command, and returns the thread.
# if the thread is joined, the thread returns
# what ever the command returns.
sub threadCommand {
	my ( $commandLine, $flag ) = @_;
	my $sub;
	if ($Dash_Map{"-seq"}) {
		print "Running in seq.\n";
		print "::[domhub] Execute: `$commandLine' ... ";
		my $txt = `$commandLine`;
		print "done\n";
		print $txt;
		$sub = sub {
			{
				lock %Completed_Threads;
				# add this thread ID to the completed thread hashmap.
				$Completed_Threads{ threads->self()->tid() } = 1;
			}
			return $txt;
		}
	} else {
		$sub = sub {
			my $cmd = shift;
			my $toReturn;
			if ($flag) {
				system($cmd);
                                print "Done with `$cmd'\n";
      				$toReturn = "Done with `$cmd'\n";
			}
			else {
				$toReturn = `$cmd`;
			}

			{
				lock %Completed_Threads;
				# add this thread ID to the completed thread hashmap.
				$Completed_Threads{ threads->self()->tid() } = 1;
			}
			return $toReturn;
		};
	}
	my $thr = threads->new( $sub, $commandLine );
	return $thr;
}


# takes a command and a lsit of domhubs,
# then threads the command onto each domhub.
# returns a hash in the form of:
# 	{ DOMHUB => THREAD }
sub dispatchApps {
	my ( $cmd, @domhubs ) = @_;
	my %threadMap = ();
	foreach my $domhub (@domhubs) {
		my $thr = threadCommand("ssh $domhub $cmd");
		$threadMap{$domhub} = $thr;
	}
	return %threadMap;
}

# takes a command and a lsit of domhubs,
# then threads the command onto each domhub.
# returns a hash in the form of:
# 	{ DOMHUB => THREAD }
# There is one key difference between this
# one and the other dispatch apps....
# this one will allow the commands to print
# to the terminal where the one one will collect
# the command's out put. This one should only be
# used if you don't care about the output
# from the commands and you want the user to see that output.
sub dispatchApps2 {
	my ( $cmd, @domhubs ) = @_;
	my %threadMap = ();
	foreach my $domhub (@domhubs) {
		my $thr = threadCommand( "ssh $domhub $cmd", 1 );
		$threadMap{$domhub} = $thr;
	}
	return %threadMap;
}

# Does the `get current' thing...
sub getCurrent {
	my @domhubs = @_;
	system("mkdir -p /mnt/data/testdaq/Results/Current/");
#	system("rm -rf /tmp/current/current.*");
	foreach my $domhub (@domhubs) {
		system("mkdir -p /mnt/data/testdaq/Results/Current/current.$domhub/");
		print "getting current files for $domhub \n";
		system(
"scp -p testdaq\@$domhub:~/Results/Current/* /mnt/data/testdaq/Results/Current/current.$domhub/"
		);
                system("zip current.$domhub.zip current.$domhub/*");
		print "\n";
	}
}

# gets Fails...
# not sure what fails are - but it gets them!
sub getFails {
	my @domhubs = @_;
	system("mkdir -p /tmp/stffails/");
	system("rm -rf /tmp/stffails/stffails.*");
	foreach my $domhub (@domhubs) {
		system("mkdir -p /tmp/stffails/stffails.$domhub/");
		print "getting STFfails files for $domhub \n";
		system(
"scp -p testdaq\@$domhub:stf_fail.txt* /tmp/stffails/stf_fail.$domhub.txt"
		);
		system(
"scp -p testdaq\@$domhub:stf_results.txt* /tmp/stffails/stf_results.$domhub.txt"
		);
		print "\n";
	}
}

# from original code
# sorry for lack of documentation here...
sub monitorHACK {
	my @page = @_;
	my @hub;
	my $thubs = 0;
	my $hubs  = 0;

	my $line1 = "HUBs                                    = ";
	my $line2 = "Waiting for RMI method calls            = ";
	my $line3 = "Getting drivers                         = ";
	my $line4 = "SOFTbooting                             = ";
	my $line5 = "Discovery phase                         = ";
	my $line6 = "DOMStatusList - waiting for other hubs  = ";
	my $line7 = "reserveDOM - hub should be taking data  = ";
	my $line8 = "TruncatedRead                           = ";
	my $line0 = "java DOMHubApp                          = ";
	my @domhubappstatus;
	my $truncateflag;
	my ( $WRM, $GDR, $SOF, $DIS, $DSL, $RDO, $TRN ) =
	  ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 );
	my $JAVA_DOMHUB_APP = 1;
	my $junk            = "";

	foreach (@page) {
		my $line = $_;
		chop $line;

		my $truncateflag = 0;
		if (/SUMMARY/) {
			$hubs = $hubs + 1;
			( $hub[$hubs] ) = split( " domhubApp", $line );
			( my $junk, $hub[$hubs] ) = split( "HUB", $hub[$hubs] );
			if (/AMANDA/) { $hub[$hubs] = "AM" }
			if (/SCUBE/) { $hub[$hubs] = "SC" }
			$line1 = $line1 . " $hub[$hubs]";
			$thubs = $thubs + 1;

			if ( $WRM < 1 ) { $line2 = $line2 . "  0"; }
			if ( $GDR < 1 ) { $line3 = $line3 . "  0"; }
			if ( $SOF < 1 ) { $line4 = $line4 . "  0"; }
			if ( $DIS < 1 ) { $line5 = $line5 . "  0"; }
			if ( $DSL < 1 ) { $line6 = $line6 . "   "; }
			if ( $RDO < 1 ) { $line7 = $line7 . "  0"; }
			if ( $TRN < 1 ) { $line8 = $line8 . "  0"; }

			$WRM             = 0;
			$GDR             = 0;
			$SOF             = 0;
			$DIS             = 0;
			$DSL             = 0;
			$RDO             = 0;
			$TRN             = 0;
			$JAVA_DOMHUB_APP = 0;
		}
		if (/software:\s*?(\d+)\s*domhubapps/) {
			$line0           = $line0 . "  $1";
			$JAVA_DOMHUB_APP = 1;
		}
		if (/Waiting/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line2 = $line2 . " $domhubappstatus[$hubs]";
			$WRM   = 1;
		}

		if (/drivers/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line3 = $line3 . " $domhubappstatus[$hubs]";
			$GDR   = 1;
		}

		if (/Softboot/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line4 = $line4 . " $domhubappstatus[$hubs]";
			$SOF   = 1;
		}

		if (/Discovery/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line5 = $line5 . " $domhubappstatus[$hubs]";
			$DIS   = 1;
		}

		if (/DOMStatus/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );
			if ( int( $domhubappstatus[$hubs] ) ) {
				$line6 = $line6 . "  *";
				$DSL   = 1;
			}
		}

		if (/reserve/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line7 = $line7 . " $domhubappstatus[$hubs]";
			$RDO   = 1;
		}

		if (/Oh/) {
			( my $junk, $domhubappstatus[$hubs] ) = split( "=", $line );

			$line8        = $line8 . " $domhubappstatus[$hubs]";
			&TRN          = 1;
			$truncateflag = 1;
		}

	}

	if ( $WRM < 1 )             { $line2 = $line2 . "  0"; }
	if ( $GDR < 1 )             { $line3 = $line3 . "  0"; }
	if ( $SOF < 1 )             { $line4 = $line4 . "  0"; }
	if ( $DIS < 1 )             { $line5 = $line5 . "  0"; }
	if ( $DSL < 1 )             { $line6 = $line6 . "   "; }
	if ( $RDO < 1 )             { $line7 = $line7 . "  0"; }
	if ( $TRN < 1 )             { $line8 = $line8 . "  0"; }
	if ( $JAVA_DOMHUB_APP < 1 ) { $line0 = $line0 . "  0"; }
	printHoriz();
	print "SUMMARY\n";
	print "$line1\n";
	print "$line0\n";
	print "$line2\n";
	print "$line3\n";
	print "$line4\n";
	print "$line5\n";
	print "$line6\n";
	print "$line7\n";

	if ( $truncateflag == 1 ) {
		print "$line8\n";
	}
	printHoriz();
	print "waiting to restart momentarily... (hit ^C to stop the program) \n";
	sleep 5;
}

# # # # # # # # # # # # # # #
#							#
#							#
#			TABLE			#
#							#
#							#
# # # # # # # # # # # # # # #
sub table {
	my @domhubs   = @_;
	my %threadMap = dispatchApps( "status short", @domhubs );
	my @infoArr   = joinThreads(%threadMap);

	# we have @infoArr now... but in order
	# to make it work with the old table code,
	# which reads a file, we have to break
	# @infoArr into an arary of lines...
	# to make it look like it came from a file.
	my @page = ();
	foreach (@infoArr) {
		my @arr = split( "\n", $_ );
		@page = ( @page, @arr );
	}
	printTableTable(@page);
}

# Prints the Table command's table.
sub printTableTable {
	my @page      = @_;
	my $hubs      = 0;
	my @hub_line  = ();
	my @comm_line = ();
	my @conf_line = ();
	my @iceb_line = ();
	my @busy_line = ();
	my @stf_line  = ();
	my $thubs     = 0;
	my $tcomm     = 0;
	my $tconf     = 0;
	my $ticeb     = 0;
	my $tbusy     = 0;
	my $tstf      = 0;
	my @hub;
	my $hub_number;
	my @commdoms;
	my @busydoms;
	my @stfservdoms;
	my @icebootdoms;
	my @configbootdoms;

	my $linewarning = "";
	my $junk        = "";

	# # # # # # # # # # #
	#
	# # # # # # # # # # #
	#
	# BIG FOR LOOP!!!

	foreach (@page) {
		my $line = $_;
		if (/SUMMARY/) {
			$hubs++;
			my ($full_hub_name) = split( " SUMMARY", $line );
			my ( $junk, $hub_number ) = split( "HUB", $full_hub_name );
			if (/AMANDA/) { $hub_number = "AM" }
			if (/SCUBE/) { $hub_number = "SC" }
			push( @hub_line, "$hub_number" );
			$hub[$hubs] = $full_hub_name;
			$thubs = $thubs + 1;
		}
		elsif (/communicating (\d+) DOMs/) {
			( $junk, $commdoms[$hubs] ) = split( "communicating ", $line );
			( $commdoms[$hubs], $junk ) = split( " DOMs", $commdoms[$hubs] );
			if ( $commdoms[$hubs] < 10 ) {
				$commdoms[$hubs] = " " . $commdoms[$hubs];
			}
			push( @comm_line, "$commdoms[$hubs]" );
			$tcomm = $tcomm + $commdoms[$hubs];

			if (/configboot/) {
				( $junk, $configbootdoms[$hubs] ) =
				  split( "configboot ", $line );
				( $configbootdoms[$hubs], $junk ) =
				  split( " DOMs", $configbootdoms[$hubs] );
				if ( $configbootdoms[$hubs] < 10 ) {
					$configbootdoms[$hubs] = " " . $configbootdoms[$hubs];
				}
				push( @conf_line, "$configbootdoms[$hubs]" );
				$tconf = $tconf + $configbootdoms[$hubs];
			}
			else {
				push( @conf_line, "  " );
			}
			if (/iceboot/) {
				( $junk, $icebootdoms[$hubs] ) = split( "iceboot ", $line );
				( $icebootdoms[$hubs], $junk ) =
				  split( " DOMs", $icebootdoms[$hubs] );
				if ( $icebootdoms[$hubs] < 10 ) {
					$icebootdoms[$hubs] = " " . $icebootdoms[$hubs];
				}
				push( @iceb_line, "$icebootdoms[$hubs]" );
				$ticeb = $ticeb + $icebootdoms[$hubs];
			}
			else {
				push( @iceb_line, "  " );
			}
			if (/busy/) {
				( $junk, $busydoms[$hubs] ) = split( "busy ", $line );
				( $busydoms[$hubs], $junk ) =
				  split( " DOMs", $busydoms[$hubs] );
				if ( $busydoms[$hubs] < 10 ) {
					$busydoms[$hubs] = " " . $busydoms[$hubs];
				}
				push( @busy_line, "$busydoms[$hubs]" );
				$tbusy = $tbusy + $busydoms[$hubs];
			}
			else {
				push( @busy_line, "  " );
			}
			if (/stfserv/) {
				( $junk, $stfservdoms[$hubs] ) = split( "stfserv ", $line );
				( $stfservdoms[$hubs], $junk ) =
				  split( " DOMs", $stfservdoms[$hubs] );
				if ( $stfservdoms[$hubs] < 10 ) {
					$stfservdoms[$hubs] = " " . $stfservdoms[$hubs];
				}
				push( @stf_line, "$stfservdoms[$hubs]" );
				$tstf = $tstf + $stfservdoms[$hubs];
			}
			else {
				push( @stf_line, "  " );
			}
		}

		if (/(>>>.+)/) {
			push( @warnings, $1 );
		}

	}
	my @lbls = ("HUB:  ", "COMM: ", "CONF: ", "ICEB: ", "BUSY: ", "STF:  ");
	print "SUMMARY\n";
	displayTable( \@lbls,
		(
			\@hub_line, \@comm_line, \@conf_line,
			\@iceb_line, \@busy_line, \@stf_line,
		)
	);
	print "HUBS= $thubs; COMM= $tcomm; CONF= $tconf; ICEB= $ticeb; BUSY= $tbusy; STF= $tstf\n";
	if ( ($tbusy>1) && ($tbusy<100) && ($ticeb>500)) {
           print "\n";
           print ">>> Number of busy DOMs and number of Iceboot DOMs is strange!\n";
           print ">>> Number of busy DOMs and number of Iceboot DOMs is strange!\n";
           print ">>> Number of busy DOMs and number of Iceboot DOMs is strange!\n";
           print ">>> (it is not possible to say there is a definite problem though) \n";
#           print "\n";
	}
	printWarnings();
}

sub getSubRow {
	my ( $arr_ref, $init, $end ) = @_;
	my @arr = @$arr_ref;
	return @arr[ $init .. $end ];
}


# confirm
# ask the user to confirm checkGPS.
# let them know there may be iterruptions to other people.
sub confirm {
	print "\n\n\t\tCheck GPS\nChecking GPS may conflict with processes\n";
	print "currently running on the DOMs.\n";
	print "Do you with to continue? (yes/no) ";
	my $answer = <STDIN>;
	if ( $answer =~ /yes/ ) {
		print "Check GPS: confirmed.\n";
	}
	else {
		print "Check GPS exiting.\n";
		exit(0);
	}
}

=getCardList @domhubs
 slightly miss-named.
 takes an arr of domhubs (i.e. strings like sps-ithub01)
 and returns an array of threads (one for each domhub)
 which are currently processing a checkGPS on a domhub
 the list of threads is returned in an order such that:
 the first thread is doing a checkGPS for the first domhub
 in the argument list for this sub.
 The threads, if joined, will return REFERENCE to a list of strings,
 where the list is that status of all the cards (idecating
 missing cards with a " ") and the first of the list is
 the domhub name (in full: ex. sps-ithub01)
=cut

sub getCardList {
	my @domhubs    = @_;
	my @hubColumns = ();

	# foreach domhub...
	foreach my $domhub (@domhubs) {

		# make a new Thread!!!!
		my $thread = threads->new(
			sub {
				my $dh     = shift;
				my $warned = "";

				# call checkGPS with args 'noconf' and 'nodisp'
				my $stuff = `ssh $dh checkGPS noconf nodisp`;
				my ( $stats, @warnings ) = split( "\n", $stuff );

				foreach (@warnings) {
					if (/>>>/) {
						$warned = $warned . "$_\n";
					}
				}

				# split the results into a list
				my @cardsStats = split( "-", $stats );
				@cardsStats = ( $warned, $dh, @cardsStats );
				return \@cardsStats;
			},
			$domhub    # domhub is the argument for the `anon' sub we just made.
		);
		push( @hubColumns, $thread );
	}
	return (@hubColumns);
}

=checkGPS DOMHub
takes a list of DOMHubs (ex. sps-ithub01 ...) and reports
to the terminal the status of all the cards on all the given
domhubs. This processes is threaded.
Note: this sub will request a confirmation form the user.
=cut

sub checkGPS {
	my @domhubs = @_;
	confirm();                             # ask the user to confirm.
	print "Getting card listings...\n";    # Let them know what we're doing.
	    # get a list of threads working on the DOMHubs
	my @thrColumns = getCardList(@domhubs);

	# announe that we're joining the threads!
	print "Joining threads... please wait... \n";
	my @hubColumns = ();    # <<<< this is our array of arrays of strings.
	foreach my $thr (@thrColumns) {

		# For each thraed...
		my $id = $thr->tid();        # get ID
		my ($ref) = $thr->join();    # Join the thread; store result in $ref
		     # since $ref is a reference to an Array of strings;
		my ( $warning, @hubCol ) = @$ref;
		push( @warnings,   $warning );
		push( @hubColumns, \@hubCol );
	}

	# generate the report for the user.
	printKey();
	my $cur_tbl = 0;
	my $tbl = 0;    # the current table we're on.
	while ( $cur_tbl * $HUBS_PER_ROW <= $#hubColumns ) {
		my $init = $cur_tbl * $HUBS_PER_ROW;
		my $end  = ( $cur_tbl + 1 ) * $HUBS_PER_ROW - 1;
		my @subTable = @hubColumns[ $init .. $end ];
		generateReport(@subTable);
		$cur_tbl++;
	}
	printWarnings();
}

=printKey
 prints the key to the table...
 does nothing else...
=cut

sub printKey {

# big format thing!!! Go perl!
# basic overview of `format' (just incase)
# format's are built on 2 line pairs: a picture and a text.
# the @ just means text... a | means 'center', and < means pad left, and > means right
# the list of strings is put into the picture....
	format STDOUT =
@|||||||
" "
@|||||||
" "
@||||||||||||||||||||||||||||||||||||||||||||||
"Check GPS report"
@<<<<<<<<<<<
"Key codes:"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
".", "External & GPS; all looks good"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"BAD", "External & No GPS"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"INT", "Internal & GPS"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"DOR", "Internal & No GPS"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"\" \"", "No DOR card detected"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"dt", "Delta-time not 2,000,000"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"*?*", "Unknown card type (clksel file?)"
@>>>>>>>    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
"#?", 	   "Unexpect directory structure on hub"
@|||||||
" "
.
	write;
}

# generateReport
# this just takes a 2D data structure and makes a table
# and outputs it for the user.
# very messy.
sub generateReport {
	my @domhubs = @_;

	#print2d(@domhubs);
	my $report =
	    "format STDOUT =\n"
	  . "@|||\n\" \"\n"
	  . "@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
	  . "\"-----------------------------------------------\"\n";
	my ( $pic, $nums ) = formatRow( getRow( 0, @domhubs ) );

	$report =
	    $report . "@"
	  . ( "<" x ( $CHARACTERS_PER_LABEL - 1 ) )
	  . $pic . "\n"
	  . "\"Hubs:\", "
	  . $nums . "\n";
	for ( 1 .. 8 ) {
		my ( $pic, $nums ) = formatRow( getRow( $_, @domhubs ) );
		$report =
		    $report . "@"
		  . ( "<" x ( $CHARACTERS_PER_LABEL - 1 ) )
		  . $pic . "\n" . "\"crd"
		  . ( $_ - 1 ) . "\", "
		  . $nums . "\n";
	}
	$report = $report
	  . "@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
	  . "\"-----------------------------------------------\"\n" . ".";
	eval($report);
	write;
}

=getRow index (ARRAYREF ARRAYREF ARRAYREF ...)
takes a 2D data structure (an array of columns) and
returns 1 row of that arary, the row is the given index.
any element in the row that fits /hub(\d\d)/ or /-(..)/
will be condenced to the ( ) part.
=cut

sub getRow {
	my ( $i, @arr ) = @_;
	my @retrn = ();
	foreach (@arr) {
		my $temp = $_->[$i];
		if ( $temp =~ /hub(\d{2})/ || $temp =~ /-(..)/ ) {
			$temp = $1;
		}
		push( @retrn, $temp );
	}
	return @retrn;
}

=formatRow
takes a row of data, and returns an array of the form:
 ( "@|| @|| @|| ...",
 	"$_[0], $_[1], $_[2] ..." )
note that all elements in the array that is given are double quoted
prior to processing. See doubleString.
=cut

sub formatRow {
	my @rowData = doubleString(@_);
	my $pic     =
	  ( "@" . ( "|" x ( $CHARACTERS_PER_HUB - 1 ) ) ) x ( $#rowData + 1 );
	my $dat = join( ", ", @rowData );
	return $pic, $dat;
}

=doubleString (String String ...)
 takes an array of strings, returns
 those strings in the same order
 except they're double quoted,
 ex. "Hello World" => ""Hello World""
=cut

sub doubleString {
	my @arr       = @_;
	my @returnArr = ();
	foreach (@arr) {
		push( @returnArr, "\"$_\"" );
	}
	return @returnArr;
}

# chooses the DOMHubs and returns them.
# the choice is based on what the global variable
# $hubs is equal to...
sub chooseDomhubs {
	if ( $hubs =~ /\./ ) {
		print "Using $hubs file for domhubs...\n";
		my $page = `cat $hubs`;
		my @lines = split( "\n", $page );
		foreach (@lines) {
			my @parts = split( " ", $_ );
			foreach (@parts) {
				if (/([\w\d]+\-[\w\d]+)/) {
					push( @domhubs, $_ );
				}
			}
		}
		print "found domhubs: @domhubs\n";
	}
	if ( $hubs =~ /([\,\;\:])/ ) {
		@domhubs = split( "$1", $hubs );
	}
	else {
		my $cat = `cat ~/domhubConfig.dat`;
		if ( $cat =~ /"$hubs"(.*?)"/s ) {
			print "Using entry for $hubs in ~/domhubConfig.dat.\n";
			@domhubs = split( " ", $1 );
			print( "Using hubs: " . join( ", ", @domhubs ) . "\n" );
		}
		else {
			@domhubs = $hubs;
		}
	}
}

=printHoriz 
	print a horizontal line, i.e.
	--------------------------------------------------
=cut

sub printHoriz {
	print "-" x 66, "\n";
}

=pDown
	Runs the command pDown on the given DOMHubs.
	Also calls parseInfo to extract meaningful info
	from the power down.
=cut

sub pDown {
	my @domhubs = @_;
	print "Powering DOWN DOMHubs...\n";
	my $str = join( ", ", @domhubs );
	print "DOMHubs: $str\n";
	my %threadMap = dispatchApps( "pDown", @domhubs );
	my @keys      = keys(%threadMap);
	my @infoArr   = joinThreads(%threadMap);
	parseInfo(@infoArr);
	print "DOMHubs powered DOWN.\n\n";
	printWarnings();
}

=pUp
	Runs the command pUp on the given DOMHubs.
	Also calls parseInfo to extract meaningful info
	from the power up.
=cut

sub pUp {
	my @domhubs = @_;
	print "Powering UP DOMHubs...\n";
	my $str = join( ", ", @domhubs );
	print "DOMHubs: $str\n";
	my %threadMap = dispatchApps( "pUp", @domhubs );
	my @keys      = keys(%threadMap);
	my @infoArr   = joinThreads(%threadMap);
	parseInfo(@infoArr);
	print "DOMHubs powered UP.\n\n";
	printWarnings();
}

=pCycle
	Runs the command pCycle on the given DOMHubs.
	Also calls parseInfo to extract meaningful info
	from the power cycle.
=cut

sub pCycle {
	my @domhubs = @_;
	print "Power Cycling DOMHubs...\n";
	my $str = join( ", ", @domhubs );
	print "DOMHbus: $str\n";
	my %threadMap = dispatchApps( "pCycle", @domhubs );
	my @keys      = keys(%threadMap);
	my @infoArr   = joinThreads(%threadMap);
	parseInfo(@infoArr);
	print "DOMHubs power-cycled.\n\n";
	printWarnings();
}

sub checkVersions {
	print "\n\ncheckVersions softboots DOMs and may conflict\n";
	print "with already running processes.\n";
	print "Do you wish to continue? (yes/no) ";
	my $answer = <STDIN>;
	if ( $answer =~ /yes/ ) {
		print "checkVersions: confirmed.\n\n";
	}
	else {
		print "checkVersions exiting.\n\n";
		exit(0);
	}



	my @domhubs = @_;
	print "Checking mainboard build versions...\n";
	my $str = join( ", ", @domhubs );
	print "DOMHubs: $str\n";
	my %threadMap = dispatchApps( "checkVersions all", @domhubs );
	my @keys      = keys(%threadMap);
        my @infoArr   = joinThreads(%threadMap);
	parseInfo(@infoArr);
        printWarnings();
}


=parseInfo
	Takes an array of strings and sorts through it
	to find line warnings and build versions. This
	is only for use in pUp, pDown, and pCycle.
	Line warnings are anything that start with >>> after a new line.
=cut

sub parseInfo {
	my @infoArr   = @_;
	my %build_map = ();
	my @hubs      = ();
	my @turned_on = ();
	my @moved_ib = ();
	foreach (@infoArr) {
#            print $_;
		my @lines = split( "\n", $_ );    
		my @warns = grep { />>>/ } @lines;
		foreach (@warns) {
			if (/(>>>.+)/) {
				push( @warnings, "$1" );
			}
		}
		my @line = grep { /hub:/ } @lines;
		$line[0] =~ /hub: (\S+)/;
		my $name = $1;
		push( @hubs, $name );
		foreach (@lines) {
			if (/Moved (\d+) DOMs into iceboot build version ([\d\w]+)/) {
				if ( defined( $build_map{$2} ) ) {
					my $ref = $build_map{$2};
					if ( defined( $ref->{$name} ) ) {
						$ref->{$name} = $ref->{$name} + 1;
					}
					else {
						$ref->{$name} = $1;
					}
				}
				else {
					$build_map{$2} = { $name => $1 };
				}
			}
			if (/There are (\d+) DOMs in iceboot/) {
				my $str = $1;
				if ($str < 10) {
					$str = " " . $str;
				}
				push (@moved_ib, $str);
			}
		}
	}
	print "\n";
	my @hubNums = ();
	foreach (@hubs) {
#                print "hub = *$_*\n";
		if (/hub(.+)/) {
			push (@hubNums, $1);
		} elsif (/AMANDA/i) {
			push (@hubNums, "AM");
                } elsif (/SCUBE/i) {
                        push (@hubNums, "SC");
		} else {
		        push (@hubNums, "UN");
		}
	}
	displayTable(["HUB:  ", "ICEB: "], (\@hubNums, \@moved_ib));
	my $hubstr = join (" ", @hubNums);
	my @builds = keys(%build_map);
	my $format = "format STDOUT =\n"
	. "@<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
	. "\"HUB\", \"$hubstr\"\n";
        $format = $format."-" x 66;
        $format = $format."\n";
	foreach (@builds) {
		my $mapref = $build_map{$_};
		my @nums = ();
		foreach (@hubs) {
			if (defined ($mapref->{$_})) {
				my $num = $mapref->{$_};
				if ($num < 10) {
					push (@nums, " $num");
				} else {
					push (@nums, "$num");
				}
			} else {
				push (@nums, "  ");
			}
		}
		my $str = join (" ", @nums);
		$format = $format
		. "@<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
		. "\"$_\", \"$str\"\n";
	}
        my $totalsum = 0;
	foreach (@builds) {
		my $map_ref = $build_map{$_};
		my $sum = 0;
		foreach (keys (%$map_ref)) {
			$sum += $$map_ref{$_};
		}
		printf ("There are %4s DOMs that have build%6s.\n",$sum,$_);
                $totalsum =$totalsum + $sum;
	}
	print "\n";
	$format = $format . ".";
	if ( $#builds > 0 ) {
		push( @warnings,
			">>> Not all $totalsum DOMs have the same iceboot build version." );	
		eval ($format);
		write;
	}
}


#
# Displays a table; arguments are as follows:
# Label_Array_Ref Row_Array_Ref Row_Array_Ref ...
#
#
sub displayTable {
	my $labels = shift @_;
	my @rows = @_;
	my $cur_tbl = 0;
	while ( $cur_tbl * $HUBS_PER_ROW <= $#{$rows[0]} ) {
		printHoriz();
		my $init = $cur_tbl * $HUBS_PER_ROW;
		my $end  = ( $cur_tbl + 1 ) * $HUBS_PER_ROW - 1;
		my $num = 0;
		foreach (@$labels) {
			print "$_" . join( " ",  getSubRow( $rows[$num], $init, $end )) . "\n";
			$num++;
		}
		printHoriz();
		$cur_tbl++;
	}
}

sub readyIceboot {
	my @domhubs   = @_;
	my %threadMap = dispatchApps( "ready-iceboot", @domhubs );
	my @values    = keys %threadMap;
	foreach (@values) {
		$threadMap{$_}->detach();
		print "$_ => ready-iceboot\n";
	}
	sleep 5;
	print "DOMHubs are `ready-iceboot'\n";
	printWarnings();
}


#
# This was put together quickly -- sorry
# Its not very well mashed together....
# The idea of this sub is to have something like
# domhub all table
# execpt its domhub all quicktable (status vs quickstatus)
sub quickstatus {
	my @domhubs   = @_;
	my %threadMap = dispatchApps( "quickstatus", @domhubs );
	my $len       = keys %threadMap;
	my @page      = ();
	## START JOINING
	print "Waiting for $len DOMHubs to finish...\n";
	my @mapKeys =
	  sort { ( $a =~ /(\d+)/ )[0] <=> ( $b =~ /(\d+)/ )[0] } keys(%threadMap);
	my %reverseMap = ();
	foreach (@mapKeys) {
		$reverseMap{ $threadMap{$_}->tid() } = $_;
	}

	my $linewarning = "";
	foreach my $key (@mapKeys) {
		my $thr   = $threadMap{$key};
		my $done  = 0;
		my $count = 0;
		while ( not $done ) {
			$count++;
			{
				lock %Completed_Threads;
				$done = $Completed_Threads{ $thr->tid() };
			}
			if ( not $done ) {
				my @arr = threads->list();
				my $num = $#arr + 1;
				if ( $#arr eq 0 ) {
					my $first = $reverseMap{ $arr[0]->tid() };
					print "Waiting on $num DOMHUBs... [ $first ]\n";
				}
				elsif ( $#arr eq 1 ) {
					my $first = $reverseMap{ $arr[0]->tid() };
					my $sec   = $reverseMap{ $arr[1]->tid() };
					print "Waiting on $num DOMHUBs... [ $first, $sec ]\n";
				}
				else {
					print "Waiting on $num DOMHUBs...\n";
				}
			}
			if ( $count % 5 eq 0 ) {
				print "$key is taking a long time...\n";
			}
			sleep 1 unless $done;
		}

		my $stuff = $thr->join();
		if ( $stuff eq "" ) {
			push( @warnings, ">>> Hub did not respond: $key." );
		}
		else {
			push( @page, $stuff );
		}
	}
	print "All DOMHubs have finished.\n\n\n";

	### END THIS PART!

	my $hubs = 0;

	my $line1     = "HUB:  ";
	my $line2     = "COMM: ";
	my @hub_line  = ();
	my @comm_line = ();
	my $thubs     = 0;
	my $tcomm     = 0;
	my @hub;
	my $hub_number;
	my @commdoms;

	my $junk = "";

	foreach (@page) {
		my $line = $_;
		if (/(.+) SUMMARY/) {
			$hubs++;
			my ($full_hub_name) = $1;
			my ( $junk, $hub_number ) = split( "HUB", $full_hub_name );
			if (/AMANDA/) { $hub_number = "AM" }
                        if (/SCUBE/) { $hub_number = "SC" }
			push( @hub_line, " $hub_number" );
			$hub[$hubs] = $full_hub_name;
			$thubs = $thubs + 1;
		}
		if (/communicating (\d+) DOMs/) {
			my $stuff = $1;
			if ( int($stuff) < 10 ) {
				$stuff = " " . $stuff;
			}
			push( @comm_line, " $stuff" );
			$tcomm = $tcomm + int($stuff);
		}
		foreach (split ("\n", $_)) {
			if (/(>>>.+)/) {
				push( @warnings, $1 );
			}
		}
		

	}
	print "SUMMARY\n";
	my $cur_tbl = 0;
	while ( $cur_tbl * $HUBS_PER_ROW <= $#hub_line ) {
		my $init = $cur_tbl * $HUBS_PER_ROW;
		my $end  = ( $cur_tbl + 1 ) * $HUBS_PER_ROW - 1;
		printHoriz();
		print "HUB:  " . join( "", @hub_line[ $init .. $end ] ) . "\n";
		print "COMM: " . join( "", @comm_line[ $init .. $end ] ) . "\n";
		printHoriz();
		$cur_tbl++;
	}
	print "\n";
	print "HUBS= $thubs; COMM= $tcomm\n";
	printWarnings();
}

#
# Prints any warnings that were found...
# i.e. prints the contents of the array @warnings
# that array is global, any sub can add to it; this
# just prints it all out...
#
sub printWarnings {
        my $endtime = time();
	my $duration = $endtime - $starttime;
                
	if ( $#warnings > -1 ) {
		print "\n";
		foreach (@warnings) {
			my $warn = $_;
			print "$warn\007\n";
		}
		print "\n";             
#		print "domhub completed in $duration seconds\n";
	} else {
  		print "\n***  NO PROBLEMS FOUND  ***\n\n";
  	}
}
