/*
   p-run - run programs on remote hosts in parallel
   Copyright (C) 2003 Frank Sorenson (frank@byu.net)

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

#include <stdio.h>
#include <termios.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <libgen.h>
#include <expect.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <math.h>
#include <signal.h>

#define MAXCOMMANDS 400
#define STRINGLENGTH 400
#define MAXSTRINGLENGTH 400
#define BUFFERLENGTH 1000
#define MAXHOSTS 600
#define DEFAULTTIMEOUT 1800
#define INTSTOKILL 2

char Prompt1[] = "]$ ";
char Prompt2[] = "]# ";
char WinPrompt[] = "$ ";
int IntSignalCaught = 0;

struct Globals;

struct ThreadInfo
{
   pid_t Thread; /* thread PID */
   int Valid; /* set to 1 when startng a thread, and set back to 0 after it exits */
   int HostID;
   double StartTime;
};

struct PerHostInfo
{
   int HostNum;
   int Active; /* set 1 when host processing starts */
   int Completed; /* set to 1 as it completes processing */
   int Output; /* whether the file has been output yet */
   int Success; /* whether the process exited successfully */
   char *HostName;
   char *OutputFileName; /* name of the output file */
   int FD; /* file descriptor for host */
   struct Globals *GlobalData;
   pid_t Thread;
};

struct Globals
{
   char *ProgramName;
   char User[STRINGLENGTH];
   char Password[STRINGLENGTH];
   double StartTime;

   int NumCommands;
   char *CommandList[MAXCOMMANDS];

   int NumThreads;
   int PromptForPassword;

   int Timeout; /*Timeout in seconds */

   struct PerHostInfo HostInfo[MAXHOSTS];
   int NumHosts; /* the number of hosts */
   struct ThreadInfo *Threads;
};

FILE *RedirStdout(char *NewFile)
{
   FILE *TempStdout;

   TempStdout = freopen(NewFile, "a+", stdout);
   if (TempStdout == NULL)
   {
      printf("Cannot redirect stdout\n");
      return((FILE *)(-1));
   }
   return(TempStdout);
}

double GetCurrentTime()
{
   struct timeval tp;
   gettimeofday(&tp, NULL);
   return ((double) tp.tv_sec + (double) tp.tv_usec * 1e-6);
}

//signal(SIGALRM, TimerHandler);
void TermHandler(int sig)
{
   printf("Caught TERM signal\n\r");
}

void IntHandler(int sig)
{
   IntSignalCaught+=1;
}

void AlarmHandler(int sig)
{
  IntSignalCaught = INTSTOKILL;
}

char *PrettyPrintTime(double Time)
{
   int Hours, Minutes, Seconds, Hundreths;
   static char ReturnString[STRINGLENGTH];
   unsigned long TotalSeconds;

   Hundreths = (Time * 100.0);
   Hundreths = Hundreths % 100;

   TotalSeconds = (unsigned long)(Time);
   Seconds = TotalSeconds % 60;
   Minutes = (TotalSeconds / 60) % 60;
   Hours = (TotalSeconds / 3600) % 3600;
   sprintf(ReturnString, "%02d:%02d:%02d.%02d", Hours, Minutes, Seconds, Hundreths);
   return(ReturnString);
}

int CheckError(const char *Message)
{
   if (errno == 0) return(0);

   perror(Message);

   errno = 0;
   return(1);
}

int WaitPrompt(int Fd, FILE *OutputFile)
{
   if (-1 == exp_expectl(Fd,
            exp_glob, Prompt1, 0,
            exp_glob, Prompt2, 1,
            exp_glob, WinPrompt,2,
            exp_end)) return(0);
/***   fwrite(exp_buffer, sizeof(exp_buffer[0]), strlen(exp_buffer), OutputFile); */
   return(1);
}

int EchoOff(void)
{
   struct termios TermIO;

   tcgetattr(fileno(stdin), &TermIO);
   TermIO.c_lflag &= (~ECHO);
   tcsetattr(fileno(stdin), TCSANOW, &TermIO);
   return(0);
}

int EchoOn(void)
{
   struct termios TermIO;

   tcgetattr(fileno(stdin), &TermIO);
   TermIO.c_lflag |= ECHO;
   tcsetattr(fileno(stdin), TCSANOW, &TermIO);
   return(0);
}

char *GetPassword()
{
   int Result;
   static char Password[MAXSTRINGLENGTH];

   printf("Enter password: ");
   fflush(stdout);
   EchoOff();
   Result = read(fileno(stdin), Password, MAXSTRINGLENGTH);
   EchoOn();
#if DEBUG
   printf("\nGot a password: \n");
#endif
   return(Password);
}

int Login(struct PerHostInfo *MyInfo)
{
   int LoggedIn;
   int Result;

   LoggedIn = 0;

   while (!LoggedIn)
   {
      Result = exp_expectl(MyInfo->FD,
         exp_glob, Prompt1, 0,
         exp_glob, Prompt2, 0,
         exp_glob, WinPrompt, 0,
         exp_glob, "'s password: ", 1,
         exp_glob, "Permission denied, please try again.", 2,
         exp_regexp, "(The authenticity of host)(.)*(Are you sure you want to continue connecting (yes/no)?)", 3,
         exp_end);
      switch (Result)
      {
         case EXP_TIMEOUT:
#if DEBUG
            printf("Got a timeout\n");
#endif
            break;
         case EXP_EOF:
#if DEBUG
            printf("Got an EOF\n");
#endif
            return(1);
            break;
         case 3:
#if DEBUG
            printf("host key checking\n");
#endif
            write(MyInfo->FD, "yes\n", 4);
            break;
         case 2:
            printf("Bad Password.  Try again.\n");
            strncpy(MyInfo->GlobalData->Password, GetPassword(), MAXSTRINGLENGTH);
            break;
         case 1:
#if DEBUG
            printf("Got a login prompt\n");
#endif
            write(MyInfo->FD, MyInfo->GlobalData->Password, strlen(MyInfo->GlobalData->Password));
            break;
         default:
#if DEBUG
            printf("Got %d.  That should be a prompt.\n", Result);
#endif
            LoggedIn = 1;
            break;
      } /* end switch */
   } /* end while */
#if DEBUG
   printf("Should be logged in now\n");
#endif
   return(0);
}

int StartConnection(struct PerHostInfo *MyInfo)
{
   char LoginCommand[MAXSTRINGLENGTH];
   int Result;

   sprintf(LoginCommand, "%s@%s", MyInfo->GlobalData->User, MyInfo->HostName);
   CheckError("sprintf");
   Result = exp_spawnl("ssh", "ssh", LoginCommand, (char *)0);
   CheckError("spawning ssh");
   MyInfo->FD = Result;
   if (-1 == Result)
   {
      perror("couldn't start ssh");
      return(2);
   }
   return(0);
}

int PrintUsage(int argc, char *argv[])
{
   printf("Usage: %s <arguments>\n", argv[0]);
   printf("\t[-user <username>] - use this user to log in\n");
   printf("\t[-host <hostname>] - add <hostname> to the list of hosts\n");
   printf("\t[-hosts <filename>] - add hosts from <filename> to the list of hosts\n");
   printf("\t[-cmd <command>] - add <command> to the list of commands\n");
   printf("\t[-script <filename>] - add commands from <filename>\n");
   printf("\t[-threads #] - execute in # threads\n");
   printf("\t[-nopass] - turn off password prompt\n");
   printf("\t[-timeout #] - total time %s is allowed to run (in seconds)\n", argv[0]);
   printf("\t[-version] - print version information\n");
   exit(-1);
}

char *Strip(char *InputString)
{
   while ((InputString[strlen(InputString) - 1] == '\n') ||
         (InputString[strlen(InputString) - 1] == '\t') ||
         (InputString[strlen(InputString) - 1] == ' '))
   {
      InputString[strlen(InputString) - 1] = '\0';
   }
   return(InputString);
}

int AddHost(struct Globals *GlobalData, char *HostName)
{
   struct PerHostInfo TempHost;

   TempHost = GlobalData->HostInfo[GlobalData->NumHosts];
   TempHost.HostName = (char *)(calloc(strlen(HostName) + 1, sizeof(HostName[0])));
   strncpy(TempHost.HostName, HostName, MAXSTRINGLENGTH);
   TempHost.HostNum = GlobalData->NumHosts;
   TempHost.GlobalData = GlobalData;
   TempHost.Active = 0;
   TempHost.Completed = 0;
   TempHost.Output = 0;

   GlobalData->HostInfo[GlobalData->NumHosts] = TempHost;
   GlobalData->NumHosts ++;
   return(0);
}

int CommandLine(int argc, char *argv[], struct Globals *GlobalData)
{
   int Counter;
   char InputFileName[MAXSTRINGLENGTH];
   FILE *InputFile;
   char TempString[MAXSTRINGLENGTH];
   int TempInt;

   for (Counter = 1 ; Counter < argc ; Counter ++)
   {
      if (!strncmp(argv[Counter], "-user", 6))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         Counter ++;
         strncpy(GlobalData->User, argv[Counter], MAXSTRINGLENGTH);
         continue;
      } /* end if -user */
      if (!strncmp(argv[Counter], "-host", 6))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         Counter ++;

         if (GlobalData->NumHosts >= MAXHOSTS)
         {
            printf("Too many hosts\n");
            exit(-1);
         }
         AddHost(GlobalData, argv[Counter]);
         continue;
      } /* end if -host */
      if (!strncmp(argv[Counter], "-hosts", 7))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         while (1)
         {
            Counter ++;
            strncpy(InputFileName, argv[Counter], MAXSTRINGLENGTH);
            InputFile = fopen(InputFileName, "rt");
            if (!InputFile)
            {
               printf("Could not open hosts file %s\n", InputFileName);
               exit(-1);
            }
            while (!feof(InputFile))
            {
               fgets(TempString, MAXSTRINGLENGTH, InputFile);
               if (feof(InputFile)) break;
               if (GlobalData->NumHosts >= MAXHOSTS)
               {
                  printf("Too many hosts\n");
                  exit(-1);
               }
               Strip(TempString);
//               printf("Adding host %s\n", TempString);
               AddHost(GlobalData, TempString);
            } /* end while !feof */
//            fclose(InputFile);
            if (((Counter + 1) >= argc) || (argv[Counter + 1][0] == '-')) break;
         } /* end while (1) */
         continue;
      } /* end if -hosts */
      if (!strncmp(argv[Counter], "-cmd", 5))
      {
         char *CmdTempString;

         if (Counter == argc) PrintUsage(argc, argv);
         if (GlobalData->NumCommands >= MAXCOMMANDS)
         {
            printf("Too many commands\n");
            exit(-1);
         }
         Counter ++;
         CmdTempString = (char *)(calloc(sizeof(char), strlen(argv[Counter]) + 1));
         strncpy(CmdTempString, argv[Counter], strlen(argv[Counter]));
         Strip(CmdTempString);
         GlobalData->CommandList[GlobalData->NumCommands] = (char *)(calloc(strlen(CmdTempString) + 1, sizeof(CmdTempString[0])));
         strncpy(GlobalData->CommandList[GlobalData->NumCommands++], CmdTempString, strlen(CmdTempString));
      } /* end if -cmd */
      if (!strncmp(argv[Counter], "-script", 8))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         Counter ++;
         strncpy(InputFileName, argv[Counter], MAXSTRINGLENGTH);
         InputFile = fopen(InputFileName, "rt");
         if (!InputFile)
         {
            printf("Could not open commands file %s\n", InputFileName);
            exit(-1);
         }
         while (!feof(InputFile))
         {
            fgets(TempString, MAXSTRINGLENGTH, InputFile);
            if (feof(InputFile)) break;
            if (GlobalData->NumCommands >= MAXCOMMANDS)
            {
               printf("Too many commands\n");
               exit(-1);
            }
            Strip(TempString);
            GlobalData->CommandList[GlobalData->NumCommands] = (char *)(calloc(strlen(TempString) + 1, sizeof(TempString[0])));
            strncpy(GlobalData->CommandList[GlobalData->NumCommands++], TempString, strlen(TempString) + 1);
         } /* end while */
      } /* end if -script */
      if (!strncmp(argv[Counter], "-threads", 9))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         Counter ++;
         TempInt = atoi(argv[Counter]);
         if (TempInt <= 0)
         {
            printf("Error:  must run with at least one thread.  Setting to 1.\n");
            GlobalData->NumThreads = 1;
         }
         else GlobalData->NumThreads = TempInt;
      } /* end if -threads */
      if (!strncmp(argv[Counter], "-timeout", 9))
      {
         if (Counter == argc) PrintUsage(argc, argv);
         Counter ++;
         TempInt = atoi(argv[Counter]);
         if (TempInt <= 0)
         {
            printf("Error:  Timeout must be greater than 0.  Setting to %d.\n", DEFAULTTIMEOUT);
            GlobalData->Timeout = DEFAULTTIMEOUT;
         }
         else GlobalData->Timeout = TempInt;
      } /* end if -timeout */
      if (!strncmp(argv[Counter], "-nopass", 8))
      {
         GlobalData->PromptForPassword = 0;
      } /* end if -nopass */
      if (!strncmp(argv[Counter], "-version", 9))
      {
         printf("p-run by Frank Sorenson\n\rcompiled %s - %s\n\r", __DATE__, __TIME__);
         exit(0);
      } /* end if -version */
   } /* end for */
   return(0);
}

int OutputHost(struct Globals *GlobalData, int HostNum)
{
   FILE *DataFile;
   char Buffer[BUFFERLENGTH];
   int DataLength;

//   printf("Preparing to dump output for %s (from %s)\n", GlobalData->HostInfo[HostNum].HostName, GlobalData->HostInfo[HostNum].OutputFileName);

   DataFile = fopen(GlobalData->HostInfo[HostNum].OutputFileName, "r");
   if (!DataFile)
   {
      printf("Couldn't fopen %s to dump output\n", GlobalData->HostInfo[HostNum].OutputFileName);
      exit(-1);
   }
//   fseek(DataFile, 0, SEEK_END);
//   if (!ftell(DataFile))
//   {
//      fclose(DataFile);
//      return(0);
//   }
//   printf("%ld bytes in file\n", ftell(DataFile));
   fseek(DataFile, 0, SEEK_SET);
   printf("*** - %s - ***\n", GlobalData->HostInfo[HostNum].HostName);
   while (!feof(DataFile))
   {
      DataLength = fread(Buffer, sizeof(Buffer[0]), BUFFERLENGTH, DataFile);
//      printf("Got %d bytes\n", DataLength);
      fwrite(Buffer, sizeof(Buffer[0]), DataLength, stdout);
   }
   fclose(DataFile);
   fflush(stdout);
   GlobalData->HostInfo[HostNum].Output = 1;
   unlink(GlobalData->HostInfo[HostNum].OutputFileName);
   return(0);
}

int ProcessHost(struct PerHostInfo *MyInfo)
{
   int CommandCounter;
   FILE *MyStdout;
   int Result;

/***
   MyStdout = fopen(MyInfo->OutputFileName, "a+");
*/
   MyStdout = RedirStdout(MyInfo->OutputFileName);
   Result = StartConnection(MyInfo);
   if (Result) return(Result);
   CheckError("starting connection");
   Result = Login(MyInfo);
   if (Result) return(Result);
   CheckError("while logging in");
   for (CommandCounter = 0 ; CommandCounter < MyInfo->GlobalData->NumCommands ; CommandCounter ++)
   {
      write(MyInfo->FD, MyInfo->GlobalData->CommandList[CommandCounter], strlen(MyInfo->GlobalData->CommandList[CommandCounter]));
      CheckError("sending command");
      write(MyInfo->FD, "\n", 2);
      CheckError("sending return");
/***      fwrite(exp_buffer, sizeof(exp_buffer[0]), strlen(exp_buffer), MyStdout); */
      WaitPrompt(MyInfo->FD, MyStdout);
   }
   close(MyInfo->FD);
   CheckError("closing remote fd");
   fprintf(MyStdout, "\n");
   fflush(MyStdout);
   fclose(MyStdout);
   return(0);
}

int ProcessHosts(struct Globals *GlobalData)
{
   int HostCounter;
   int ThreadCounter;
   int HostsActive, HostsCompleted, HostsOutput;
   int ThreadsInUse;
   int TotalHosts;
   int Result;
   int Status;
//   FILE *DebugFile;
   int TempFile;
   char TempFileName[STRINGLENGTH];
   int StartedCount;
   int FailedHosts;
   double ClockTime;
   double TotalTime;

   StartedCount = 0;

   HostsActive = 0;
   HostsCompleted = 0;
   HostsOutput = 0;
   ThreadsInUse = 0;
   FailedHosts = 0;
   ClockTime = 0.0;
   TotalTime = 0.0;

   TotalHosts = GlobalData->NumHosts;

//   DebugFile = fopen("/tmp/p-run.debug", "w");

   GlobalData->Threads = (struct ThreadInfo *)(calloc(GlobalData->NumThreads, sizeof(struct ThreadInfo)));

//   for (ThreadCounter = 0 ; ThreadCounter < GlobalData->NumThreads ; ThreadCounter ++)
//   {
//      GlobalData->Threads[ThreadCounter].Valid = 0;
//   }

   GlobalData->StartTime = GetCurrentTime();

   while ((HostsActive != TotalHosts) || (HostsCompleted != TotalHosts) || (HostsOutput != TotalHosts))
   {
      HostsActive = 0;
      HostsCompleted = 0;
      HostsOutput = 0;
      ThreadsInUse = 0;
      for (HostCounter = 0 ; HostCounter < GlobalData->NumHosts ; HostCounter ++)
      {
         /* check if host can be output */
//         if (GlobalData->HostInfo[HostCounter].Active && GlobalData->HostInfo[HostCounter].Completed && (! GlobalData->HostInfo[HostCounter].Output))
//         {
//            OutputHost(GlobalData, HostCounter);
//         }
//         else if (GlobalData->HostInfo[HostCounter].Active != 1)
         if (GlobalData->HostInfo[HostCounter].Active != 1)
         {
            /* if we have free threads, we can start one processing (search for one) */
            for (ThreadCounter = 0 ; ThreadCounter < GlobalData->NumThreads ; ThreadCounter ++)
            {
               if (GlobalData->Threads[ThreadCounter].Valid == 0)
               {
//printf("%d: Preparing to spawn something on %s (thread %d)\n", HostCounter, GlobalData->HostInfo[HostCounter].HostName, ThreadCounter);
                  sprintf(TempFileName, "/tmp/%s.%s-XXXXXX", GlobalData->ProgramName, GlobalData->HostInfo[HostCounter].HostName);
                  TempFile = mkstemp(TempFileName);
                  close(TempFile); /* close it so we can reopen it */
                  TempFileName[strlen(TempFileName)] = '\0';
                  GlobalData->HostInfo[HostCounter].OutputFileName = (char *)(calloc(strlen(TempFileName) + 1, sizeof(TempFileName[0])));
                  strncpy(GlobalData->HostInfo[HostCounter].OutputFileName, TempFileName, strlen(TempFileName));
                  GlobalData->Threads[ThreadCounter].StartTime = GetCurrentTime();

//                  printf("Starting %d time\n", ++StartedCount);
//                  printf("%d: Preparing to spawn something on %s (thread %d and file %s)\n", HostCounter, GlobalData->HostInfo[HostCounter].HostName, ThreadCounter, GlobalData->HostInfo[HostCounter].OutputFileName);

                  fflush(stdout);
                  fflush(stderr);
                  GlobalData->HostInfo[HostCounter].Thread = fork();
                  if (GlobalData->HostInfo[HostCounter].Thread == -1)
                  {
                     fprintf(stderr, "fork failed: %s\n", strerror(errno));
                     perror("Forking");
                     exit(-1);
                  }
                  else if (GlobalData->HostInfo[HostCounter].Thread == 0) /* we're the child */
                  {
                     Result = ProcessHost(&GlobalData->HostInfo[HostCounter]);
//                     printf("Child exiting\n");
                     exit(Result);
                  }
                  else
                  {
//                     printf("Spawned a process\n");
                     GlobalData->HostInfo[HostCounter].Active = 1; /* mark this one as processing */
                     GlobalData->Threads[ThreadCounter].HostID = HostCounter;
                     GlobalData->Threads[ThreadCounter].Valid = 1;
                     GlobalData->Threads[ThreadCounter].Thread = GlobalData->HostInfo[HostCounter].Thread;
                     ThreadCounter = GlobalData->NumThreads;
                  }
               } /* Found an inactive thread */
            } /* finding a free thread */
         } /* if !Active */
         if (GlobalData->HostInfo[HostCounter].Active) HostsActive ++;
         if (GlobalData->HostInfo[HostCounter].Completed) HostsCompleted ++;
         if (GlobalData->HostInfo[HostCounter].Output) HostsOutput ++;
         if (IntSignalCaught >= INTSTOKILL) break;
      } /* for each host */

      if (IntSignalCaught >= INTSTOKILL)
      {
         printf("INT Caught.  Cleaning up.\n\r");
         for (ThreadCounter = 0 ; ThreadCounter < GlobalData->NumThreads ; ThreadCounter ++)
         {
            if (GlobalData->Threads[ThreadCounter].Valid)
            {
#ifdef DEBUG
               printf("Killing a thread...\n");
#endif
               Result = kill(GlobalData->Threads[ThreadCounter].Thread, SIGKILL);
               if (Result) printf("Error while sending kill signal\n\r");
#ifdef DEBUG
               printf("Waiting for thread (%d) to receive the SIGKILL\n", GlobalData->Threads[ThreadCounter].Thread);
#endif
               Result = waitpid(GlobalData->Threads[ThreadCounter].Thread, &Status, 0);
               OutputHost(GlobalData, GlobalData->Threads[ThreadCounter].HostID);
               GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].Output = 1;
               FailedHosts ++;
               unlink(GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].OutputFileName);
               GlobalData->Threads[ThreadCounter].Valid = 0;
            }
         }
         for (HostCounter = 0 ; HostCounter < GlobalData->NumHosts ; HostCounter ++)
         {
            if (GlobalData->HostInfo[HostCounter].Active && GlobalData->HostInfo[HostCounter].Completed && (! GlobalData->HostInfo[HostCounter].Output))
               OutputHost(GlobalData, HostCounter);
            if (GlobalData->HostInfo[HostCounter].Active != 1) /* haven't started this one yet */
            {
               GlobalData->HostInfo[HostCounter].Output = 1;
               FailedHosts ++;
            }
         }
         HostsActive = TotalHosts;
         HostsCompleted = TotalHosts;
         HostsOutput = TotalHosts;
//         printf("Done.\n\r");
//         exit(-1);
      }

      /* check our threads for any that have completed */
      for (ThreadCounter = 0 ; ThreadCounter < GlobalData->NumThreads ; ThreadCounter ++)
      {
         if (GlobalData->Threads[ThreadCounter].Valid)
         {
            Result = waitpid(GlobalData->Threads[ThreadCounter].Thread, &Status, WNOHANG);
//            if (Result != 0) printf("PID: %d, Result: %d\n", GlobalData->Threads[ThreadCounter].Thread, Result);
            if (Result < 0)
            {
               fprintf(stderr, "waitpid failed: %s\n", strerror(errno));
               perror("waitpid");
            }
            else if (Result) /* it's exited */
            {
               if (WIFEXITED(Status)) Result = WEXITSTATUS(Status);
               else Result = 3;
//               printf("%s exited with status %d\n", GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].HostName, Result);
               if (Result)
               {
                  GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].Output = 1;
                  FailedHosts ++;
                  unlink(GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].OutputFileName);
               }
               else
               {
                  GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].Success = 1;
                  OutputHost(GlobalData, GlobalData->Threads[ThreadCounter].HostID);
               }
               fflush(stdout);
               GlobalData->Threads[ThreadCounter].Valid = 0;
               GlobalData->HostInfo[GlobalData->Threads[ThreadCounter].HostID].Completed = 1;
               TotalTime += GetCurrentTime() - GlobalData->Threads[ThreadCounter].StartTime;
            }
            else
            { /* now, what do we do?  Result was 0 */

            }
         }
         if (GlobalData->Threads[ThreadCounter].Valid) ThreadsInUse ++;
      } /* for ThreadCounter */
//      if (ThreadsInUse == GlobalData->NumThreads) usleep(100);
      usleep(10);
//      fprintf(DebugFile, "%d active, %d completed, %d output, %d threads in use\n", HostsActive, HostsCompleted, HostsOutput, ThreadsInUse);
   } /* while loop */

   ClockTime = GetCurrentTime() - GlobalData->StartTime;
   printf("Total Time: %s, ", PrettyPrintTime(TotalTime));
   printf("clock time: %s, ", PrettyPrintTime(ClockTime));
   printf("savings: %s, ", PrettyPrintTime(TotalTime - ClockTime));
   printf("speedup: %.2lf\n", TotalTime/ClockTime);
   if (FailedHosts)
   {
      printf("Failed hosts: ");
      for (HostCounter = 0 ; HostCounter < TotalHosts ; HostCounter ++)
      {
         if (! GlobalData->HostInfo[HostCounter].Success) printf("%s ", GlobalData->HostInfo[HostCounter].HostName);
      }
      printf("\n");
   }
   return(0);
} /* end of ProcessHosts */

int main(int argc, char *argv[])
{
   struct Globals GlobalData[1];
   int TempInt;

   GlobalData->NumHosts = 0;
   GlobalData->NumCommands = 0;
   GlobalData->NumThreads = 1;
   GlobalData->PromptForPassword = 1;
   GlobalData->Timeout = DEFAULTTIMEOUT;

   exp_loguser = 1; /* turn on/off expect verbose output */
   exp_timeout = 3600;
   exp_match_max = 10000; /* set the expect buffer size */

   TempInt = strlen(basename(argv[0]));
   GlobalData->ProgramName = (char *)(calloc(TempInt + 1, sizeof(char)));

   strncpy(GlobalData->ProgramName, basename(argv[0]), TempInt + 1);
   strncpy(GlobalData->User, "root", MAXSTRINGLENGTH);

   CommandLine(argc, argv, GlobalData);

   printf("%d hosts, %d commands, %d threads\nTimeout: %d seconds\n", GlobalData->NumHosts, GlobalData->NumCommands, GlobalData->NumThreads, GlobalData->Timeout);

   if (GlobalData->NumHosts <= 0)
   {
      printf("There are no hosts in the host list\n");
      PrintUsage(argc, argv);
   }
   if (GlobalData->NumCommands <= 0)
   {
      printf("There are no commands in the command list\n");
      PrintUsage(argc, argv);
   }

   if (GlobalData->PromptForPassword)
   {
      strncpy(GlobalData->Password, GetPassword(), MAXSTRINGLENGTH);
      printf("\n");
   }


   signal(SIGTERM, TermHandler);
   signal(SIGINT, IntHandler);
   signal(SIGALRM, AlarmHandler);
   alarm(GlobalData->Timeout);
   ProcessHosts(GlobalData);
   return(0);
}