test-aoat.c

/*
 * test-aoat.c - Checks the correctness and performance of all of the
 *     student's aoat functions and records the results for their
 *     official submitted version as well.
 */
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <getopt.h>
#include <sys/types.h>
#include "cachelab.h"
#include <sys/wait.h> // fir WEXITSTATUS
#include <limits.h> // for INT_MAX

/* Maximum array dimension */
#define MAXN 256

/* The description string for the aoat_submit() function that the
 student submits for credit */
#define SUBMIT_DESCRIPTION "Aoat submission"

/* External function defined in aoat.c */
extern void registerAoatFunctions();

/* External variables defined in cachelab-tools.c */
extern aoat_func_t func_list_aoat[MAX_AOAT_FUNCS];
extern int func_counter_aoat;

/* Globals set on the command line */
static int N = 0;

/* The correctness and performance for the submitted aoat function */
struct results {
    int funcid;
    int correct;
    int misses;

    int bestfuncid;
    int bestmisses;
};
static struct results results = {-1, 0, INT_MAX, -1, INT_MAX};

/*
 * eval_perf - Evaluate the performance of the registered aoat functions
 */
void eval_perf (unsigned int s, unsigned int E, unsigned int b) {
  int i, flag;
  unsigned int len, hits, misses, evictions;
  unsigned long long int marker_start, marker_end, addr;
  char buf[1000], cmd[255];
  char filename[128];

  registerAoatFunctions();

  /* Open the complete trace file */
  FILE *full_trace_fp;
  FILE *part_trace_fp;

  /* Evaluate the performance of each registered aoat function */

  for (i = 0; i < func_counter_aoat; i++) {
    if (strcmp (func_list_aoat[i].description, SUBMIT_DESCRIPTION) == 0)
      results.funcid = i; /* remember which function is the submission */


    printf ("\nFunction %d (%d total)\nStep 1: Validating and generating memory traces\n", i, func_counter_aoat);
    /* Use valgrind to generate the trace */

    sprintf (cmd, "valgrind --tool=lackey --trace-mem=yes --log-fd=1 -v ./tracegenaoat -N %d -F %d  > trace.tmp", N, i);
    flag = WEXITSTATUS (system (cmd));

    if (0 != flag) {
      printf ("Validation error at function %d! Run ./tracegenaoat -N %d -F %d for details.\nSkipping performance evaluation for this function.\n", flag - 1, N, i);
      continue;
    }

    /* Get the start and end marker addresses */
    FILE *marker_fp = fopen (".marker", "r");
    assert (marker_fp);
    fscanf (marker_fp, "%llx %llx", &marker_start, &marker_end);
    fclose (marker_fp);


    func_list_aoat[i].correct = 1;

    /* Save the correctness of the aoat submission */
    if (results.funcid == i) {
      results.correct = 1;
    }

    full_trace_fp = fopen ("trace.tmp", "r");
    assert (full_trace_fp);


    /* Filtered trace for each aoat function goes in a separate file */
    sprintf (filename, "trace.f%d", i);
    part_trace_fp = fopen (filename, "w");
    assert (part_trace_fp);

    /* Locate trace corresponding to the aoat function */
    flag = 0;

    while (fgets (buf, 1000, full_trace_fp) != NULL) {

      /* We are only interested in memory access instructions */
      if (buf[0] == ' ' && buf[2] == ' ' &&
          (buf[1] == 'S' || buf[1] == 'M' || buf[1] == 'L')) {
        sscanf (buf + 3, "%llx,%u", &addr, &len);

        /* If start marker found, set flag */
        if (addr == marker_start)
          flag = 1;

        /* Valgrind creates many spurious accesses to the
         stack that have nothing to do with the students
         code. At the moment, we are ignoring all stack
         accesses by using the simple filter of recording
         accesses to only the low 32-bit portion of the
         address space. At some point it would be nice to
         try to do more informed filtering so that would
         eliminate the valgrind stack references while
         include the student stack references. */
        if (flag && addr < 0xffffffff) {
          fputs (buf, part_trace_fp);
        }

        /* if end marker found, close trace file */
        if (addr == marker_end) {
          flag = 0;
          fclose (part_trace_fp);
          break;
        }
      }
    }

    fclose (full_trace_fp);

    /* Run the reference simulator */
    printf ("Step 2: Evaluating performance (s=%d, E=%d, b=%d)\n", s, E, b);
    char cmd[255];
    sprintf (cmd, "./csim-ref -s %u -E %u -b %u -t trace.f%d > /dev/null",
             s, E, b, i);
    system (cmd);

    /* Collect results from the reference simulator */
    FILE *in_fp = fopen (".csim_results", "r");
    assert (in_fp);
    fscanf (in_fp, "%u %u %u", &hits, &misses, &evictions);
    fclose (in_fp);
    func_list_aoat[i].num_hits = hits;
    func_list_aoat[i].num_misses = misses;
    func_list_aoat[i].num_evictions = evictions;
    printf ("func %u (%s): hits:%u, misses:%u, evictions:%u\n",
            i, func_list_aoat[i].description, hits, misses, evictions);

    /* Beat the best. */
    if (results.bestmisses > misses) {
      results.bestfuncid = i;
      results.bestmisses = misses;
    }

    /* If it is aoat_submit(), record number of misses */
    if (results.funcid == i) {
      results.misses = misses;
    }
  }

}

/*
 * usage - Print usage info
 */
void usage (char *argv[]) {
  printf ("Usage: %s [-h] -N <size>\n", argv[0]);
  printf ("Options:\n");
  printf ("  -h          Print this help message.\n");
  printf ("  -N <rows>   Number of matrix rows/columns (max %d)\n", MAXN);
  printf ("Example: %s -N 8\n", argv[0]);
}

/*
 * sigsegv_handler - SIGSEGV handler
 */
void sigsegv_handler (int signum) {
  printf ("Error: Segmentation Fault.\n");
  printf ("TEST_AOAT_RESULTS=0:0\n");
  fflush (stdout);
  exit (1);
}

/*
 * sigalrm_handler - SIGALRM handler
 */
void sigalrm_handler (int signum) {
  printf ("Error: Program timed out.\n");
  printf ("TEST_AOAT_RESULTS=0:0\n");
  fflush (stdout);
  exit (1);
}

/*
 * main - Main routine
 */
int main (int argc, char *argv[]) {
  char c;

  while ( (c = getopt (argc, argv, "N:h")) != -1) {
    switch (c) {
      case 'N':
        N = atoi (optarg);
        break;

      case 'h':
        usage (argv);
        exit (0);

      default:
        usage (argv);
        exit (1);
    }
  }

  if (N == 0) {
    printf ("Error: Missing required argument\n");
    usage (argv);
    exit (1);
  }

  if (N > MAXN) {
    printf ("Error: N exceeds %d\n", MAXN);
    usage (argv);
    exit (1);
  }

  /* Install SIGSEGV and SIGALRM handlers */
  if (signal (SIGSEGV, sigsegv_handler) == SIG_ERR) {
    fprintf (stderr, "Unable to install SIGALRM handler\n");
    exit (1);
  }

  if (signal (SIGALRM, sigalrm_handler) == SIG_ERR) {
    fprintf (stderr, "Unable to install SIGALRM handler\n");
    exit (1);
  }

  /* Time out and give up after a while */
  alarm (10 * 60);

  /* Check the performance of the student's aoat function */
  eval_perf (5, 1, 5);

  if (results.bestfuncid != -1)
    printf ("\nBest correct function: Function %d%s, with %d misses.",
            results.bestfuncid, results.bestfuncid == results.funcid ? "" : " (NOT SUBMITTED VERSION)",
            results.bestmisses);
  /* Emit the results for this particular test */
  if (results.funcid == -1) {
    printf ("\nError: We could not find your aoat_submit() function\n");
    printf ("Error: Please ensure that description field is exactly \"%s\"\n",
            SUBMIT_DESCRIPTION);
    printf ("\nTEST_AOAT_RESULTS=0:0\n");
  } else {
    printf ("\nSummary for official submission (func %d): correctness=%d misses=%d\n",
            results.funcid, results.correct, results.misses);
    printf ("\nTEST_AOAT_RESULTS=%d:%d\n", results.correct, results.misses);
  }

  return 0;
}