/*******************************************************************/
/* 25/09/2004 - Gerard Torrent - gerard@fobos.generacio.com        */
/*              initial release                                    */
/* 04/10/2004 - Gerard Torrent - gerard@fobos.generacio.com        */
/*              simplified struct chunk (added status/numops)      */
/*              send/receive type MPI_LONG's, not structs          */
/*              added compile directive -O3 (optimize speed)       */
/*                                                                 */
/* Description: Perform cluster megaflops measurement              */
/*                                                                 */
/* Note1: Master node don't work (no FOPS). If you need compute    */
/*        the real megaflops of cluster, add 1 cpu to master       */
/*        machine in lam-bhost.def file.                           */
/*                                                                 */
/* Note2: compile using -> mpicc -O3 -o mflops mflops.c            */
/*        run it using  -> mpirun C mflops                         */
/*                                                                 */
/*******************************************************************/

/* =============================================================== */
/* headers                                                         */
/* =============================================================== */
#include <stdio.h>
#include <unistd.h>
#include <mpi.h>
#include <sys/time.h>
#include <time.h>

/* =============================================================== */
/* defines                                                         */
/* =============================================================== */
#define WORKTAG              1
#define DIETAG               2
#define MAX_TASKS        10000L
#define DEFAULT_NUMOPS 1000000L

/* =============================================================== */
/* structs definitions                                             */
/* =============================================================== */
struct chunk {
  int status;            /* WORKTAG=working, DIETAG=finish */
  long numchunks;        /* number of chuncks procesed     */
  double t_numops;       /* total numops realized          */
  double t_microseconds; /* total microseconds elapsed     */
};

/* =============================================================== */
/* function definitions                                            */
/* =============================================================== */
static int master(void);
static int slave(void);
static long getElapsedTime(struct timeval *tv0, struct timeval *tv1);
static struct chunk * initializeTasks(int n);
static int saveChunk(long nummicroseconds, struct chunk *chunklist, int lastpos) ;
static int isTasksCompleted(struct chunk *chunklist, int lastpos);
static void showReport(struct chunk *chunklist, int size);

/* =============================================================== */
/* main function                                                   */
/* =============================================================== */
int main(int argc, char* argv[])
{
  int ret, size, rank, child;
  
  /* Initialize MPI */
  MPI_Init(&argc, &argv);

  /* Determine rank process */
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  
  /* bifurcation flow */
  if (rank == 0) {
    ret = master();
  } else {
    ret = slave();
  }
  
  /* finalize and exit */
  MPI_Finalize();
  return(ret);
}


/* =============================================================== */
/* master function                                                 */
/* =============================================================== */
static int master(void)
{
  int i, size;
  MPI_Status status;
  struct chunk *tasks;
  int numtasks=0;
  int completed = 0;
  long numops=DEFAULT_NUMOPS, nummicroseconds;

  /* Determine the number of processes */  
  MPI_Comm_size(MPI_COMM_WORLD, &size);

  /* tasks initialization */
  tasks = initializeTasks(size);
  
  /* initial work dispatching */
  for (i=1;i<size;i++) {
    MPI_Send(&numops, 1, MPI_LONG, i, WORKTAG, MPI_COMM_WORLD);
    numtasks++;
  }

  /* we need more than one rank to run (see notes) */
  if (size == 1) {
    completed = 1;
  }  
    
  /* retrieving results and redispatching */
  while (completed == 0) {
  
    /* receiving chunk procesed */
    MPI_Recv(&nummicroseconds, 1, MPI_LONG, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
      
    /* procesing task received */
    saveChunk(nummicroseconds, tasks, status.MPI_SOURCE);
      
    /* asigning new task */
    if (numtasks >= MAX_TASKS) {
      tasks[status.MPI_SOURCE].status = DIETAG;
      MPI_Send(0, 0, MPI_INT, status.MPI_SOURCE, DIETAG, MPI_COMM_WORLD);
      if (isTasksCompleted(tasks, size)) {
        completed = 1;
      }
    } else {
      MPI_Send(&numops, 1, MPI_LONG, status.MPI_SOURCE, WORKTAG, MPI_COMM_WORLD);
      numtasks++;      
    }
  }

  /* megaflops report */
  showReport(tasks, size);
  
  /* memory management */
  free(tasks);
}


/* =============================================================== */
/* slave function                                                  */
/* =============================================================== */
static int slave(void)
{
  MPI_Status status;
  struct timeval tv0, tv1;
  struct timezone tz0, tz1;
  long numops, nummicroseconds, counter;
  double val=2.0, M_PI=3.14159265358979323846;
    
  /* awaiting begining order */
  MPI_Recv(&numops, 1, MPI_LONG, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &status);

  /* slave loop. Make FOPS and send results to master  */
  while(status.MPI_TAG == WORKTAG) {

    /* setting start time */
    if (gettimeofday(&tv0, &tz0) != 0) {
      MPI_Finalize();
      return(1);
    }

    /* numops FLoating OPerationS computation */
    for (counter=0; counter<numops; counter++) {
      val = val*M_PI;
    }

    /* setting finish time */
    if (gettimeofday(&tv1, &tz1) != 0) {
      MPI_Finalize();
      return(1);
    }
    
    /* computing elapsed time in microseconds */
    nummicroseconds = getElapsedTime(&tv0, &tv1);

    /* sending results to master */
    MPI_Send(&nummicroseconds, 1, MPI_LONG, 0, WORKTAG, MPI_COMM_WORLD);

    /* awaiting new task reception */
    MPI_Recv(&numops, 1, MPI_LONG, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
  }
}


/* =============================================================== */
/* getElapsedTime.                                                 */
/* Returns microseconds diference between 2 timevals as long       */
/* =============================================================== */
long getElapsedTime(struct timeval *tv0, struct timeval *tv1) 
{
  long ret;
  
  ret = (tv1->tv_sec - tv0->tv_sec) * 1000000L;
  ret += tv1->tv_usec - tv0->tv_usec;  
  
  return ret;
}    


/* =============================================================== */
/* initializeTasks                                                 */
/* =============================================================== */
struct chunk * initializeTasks(int n) 
{
  struct chunk *tasks;
  int i;

  tasks = (struct chunk *) calloc(n, sizeof(struct chunk)); 
      
  for (i=0;i<n;i++) {
    tasks[i].status = (i==0?DIETAG:WORKTAG);
    tasks[i].numchunks = 0L;
    tasks[i].t_numops = 0.0;
    tasks[i].t_microseconds = 0.0;
  }
  
  return tasks;
}


/* =============================================================== */
/* saveChunk                                                       */
/* =============================================================== */
int saveChunk(long nummicroseconds, struct chunk *chunklist, int pos) 
{
  chunklist[pos].numchunks += 1;
  chunklist[pos].t_numops += (double) DEFAULT_NUMOPS;
  chunklist[pos].t_microseconds += (double) nummicroseconds;
  
  return 1;
}


/* =============================================================== */
/* isTasksCompleted                                                */
/* =============================================================== */
int isTasksCompleted(struct chunk *chunklist, int lastpos)
{
  long i;
  
  for (i=0;i<lastpos;i++) {
    if (chunklist[i].status != DIETAG) {
      return 0;
    }
  }
  
  return 1;
}


/* =============================================================== */
/* showReport                                                      */
/* =============================================================== */
void showReport(struct chunk *chunklist, int n)
{
  int i;
  double mflops, seconds;
  double sum1=0L, sum2=0L;
  long sum3=0L;
  
  /* print header */
  printf("RANK    CHUNKS         NUMOPS     TIME (sec)      MEGAFLOPS\n");
  printf("------ ------- -------------- -------------- --------------\n");
  
  /* print content */
  for(i=0;i<n;i++) {
    sum1 += chunklist[i].t_numops;
    sum2 += chunklist[i].t_microseconds;
    sum3 += chunklist[i].numchunks;
    seconds = chunklist[i].t_microseconds/1000000.0;
    mflops = (chunklist[i].t_microseconds==0L?0:chunklist[i].t_numops/seconds)/1000000.0;
    printf("%-6d  %6ld %14.0lf %14lf %14lf\n", i, chunklist[i].numchunks, chunklist[i].t_numops, seconds, mflops);
  }
  
  /* print grand total */
  printf("------ ------- -------------- -------------- --------------\n");
  seconds = (double)(sum2)/1000000;
  mflops = (sum2==0L?0:(double)(sum1)/seconds)/1000000;
  printf("TOTAL   %6ld %14.0lf %14lf %14lf\n", sum3, sum1, seconds, mflops);
  printf("\n");
}