experiment

#!/usr/bin/env python2.7
import subprocess
import argparse
import csv
import sys
import os

def main(args):
  groundtruth_name = "%s-%s" % (args.name, "groundtruth.json")
  afl_commands_name = "%s-commands" % args.name 
  itrace_commands_name = "%s-itrace-commands" % args.name
  afl_vectors_name = "%s-vectors" % args.name
  itrace_vectors_name = "%s-itrace-vectors" % args.name
  experiments = []
  experiments.append(("KMeans clustering with AFL", 
                      "kmeans", 
                      args.clusters, 
                      "%s-kmeans.json" % args.name, 
                      "kmeans", 
                      args.variance, 
                      afl_vectors_name,
                      None))
  #experiments.append(("KMeans clustering with PIN", 
  #                    "kmeans", 
  #                    args.clusters, 
  #                    "%s-kmeans-itrace.json" % args.name, 
  #                    "kmeans-itrace", 
  #                    args.variance, 
  #                    itrace_vectors_name, 
  #                    None))
  #experiments.append(("KMeans(PCA) clustering with PIN", 
  #                    "kmeans", 
  #                    args.clusters, 
  #                    "%s-kmeans-pca-itrace.json" % args.name, 
  #                    "kmeans-pca-itrace", 
  #                    args.variance, 
  #                    itrace_vectors_name, 
  #                    0.95))
  experiments.append(("KMeans(PCA) clustering with AFL", 
                      "kmeans", 
                      args.clusters, 
                      "%s-kmeans-pca.json" % args.name, 
                      "kmeans-pca", 
                      args.variance, 
                      afl_vectors_name, 
                      0.95))
  experiments.append(("Spectral clustering with AFL", 
                      "spectral", 
                      args.clusters, 
                      "%s-spectral.json" % args.name, 
                      "spectral", 
                      args.variance, 
                      afl_vectors_name,
                      None))
  """
  experiments.append(("Spectral clustering with PIN", 
                      "spectral", 
                      args.clusters, 
                      "%s-spectral-itrace.json" % args.name, 
                      "spectral-itrace", 
                      args.variance, 
                      itrace_vectors_name, 
                      None))
  experiments.append(("Spectral(PCA) clustering with PIN", 
                      "spectral", 
                      args.clusters, 
                      "%s-spectral-pca-itrace.json" % args.name, 
                      "spectral-pca-itrace", 
                      args.variance, 
                      itrace_vectors_name, 
                      0.95))
  """
  experiments.append(("Spectral(PCA) clustering with AFL", 
                      "spectral", 
                      args.clusters, 
                      "%s-spectral-pca.json" % args.name, 
                      "spectral-pca", 
                      args.variance, 
                      afl_vectors_name, 
                      0.95))
  """
  experiments.append(("MeanShift(PCA) clustering with PIN", 
                      "meanshift", 
                      None, 
                      "%s-meanshift-pca-itrace.json" % args.name, 
                      "meanshift-pca-itrace", 
                      args.variance, 
                      itrace_vectors_name,
                      0.95))
  """
  experiments.append(("MeanShift(PCA) clustering with AFL", 
                      "meanshift", 
                      None, 
                      "%s-meanshift-pca.json" % args.name, 
                      "meanshift-pca", 
                      args.variance, 
                      afl_vectors_name, 
                      0.95))
  """
  experiments.append(("MeanShift clustering with PIN", 
                      "meanshift", 
                      None, 
                      "%s-meanshift-itrace.json" % args.name, 
                      "meanshift-itrace", 
                      args.variance, 
                      itrace_vectors_name,
                      None))
  """
  experiments.append(("MeanShift clustering with AFL", 
                      "meanshift", 
                      None, 
                      "%s-meanshift.json" % args.name, 
                      "meanshift", 
                      args.variance, 
                      afl_vectors_name,
                      None))
  experiments.append(("Aggregate clustering with PIN", 
                      "agg", 
                      args.clusters, 
                      "%s-agg-itrace.json" % args.name, 
                      "agg-itrace", 
                      args.variance, 
                      itrace_vectors_name,
                      None))
  """
  experiments.append(("Aggregate clustering with AFL", 
                      "agg", 
                      args.clusters, 
                      "%s-agg.json" % args.name, 
                      "agg", 
                      args.variance, 
                      afl_vectors_name, 
                      0.95))
  experiments.append(("DBSCAN clustering with PIN", 
                      "dbscan", 
                      None, 
                      "%s-dbscan-itrace.json" % args.name, 
                      "dbscan-itrace", 
                      args.variance, 
                      itrace_vectors_name, 
                      None))
  """
  experiments.append(("DBSCAN clustering with AFL", 
                      "dbscan", 
                      None, 
                      "%s-dbscan.json" % args.name, 
                      "dbscan", 
                      args.variance, 
                      afl_vectors_name, 
                      None))

  print "Generating ground truth data"
  format = "--data-source=%s" % args.data_source
  subprocess.check_call(["./analyze_groundtruth", format, args.name, args.csv, groundtruth_name, args.datapath])

  #print "Generating vector extraction commands for AFL"
  #of = open(afl_commands_name, 'w')
  #subprocess.check_call(["./mk_command", "./get_afl_vec \"%s\"" % args.program, groundtruth_name, afl_vectors_name], stdout=of)
  #of.close()

  #print "Generating vector extraction commands for PIN"
  #of = open(itrace_commands_name, 'w')
  #subprocess.check_call(["./mk_command", "./get_itrace_vec \"%s\"" % args.program, groundtruth_name, itrace_vectors_name], stdout=of)
  #of.close()

  #try:
  #  os.mkdir(afl_vectors_name)
  #except:
  #  pass

  #try:
  #  os.mkdir(itrace_vectors_name)
  #except:
  #  pass

  #print "Extracting vectors for AFL"
  #inf = open(afl_commands_name, 'r')
  #subprocess.check_call(["parallel"], stdin=inf)
  #inf.close() 

  #print "Extracting vectors for PIN"
  #inf = open(itrace_commands_name, 'r')
  #subprocess.check_call(["parallel"], stdin=inf)
  #inf.close()

# Do experiments.
  for (ename,cmeth,clusters,outfile,elabel,cutoff,tracedir,pca) in experiments:
    print "Running %s" % ename
    cmd = ["./cluster"]
    cmd.append("--cluster-method=%s" % cmeth)
    if clusters:
      cmd.append("--clusters=%d" % clusters)
    cmd.append("--outfile=%s" % outfile)
    cmd.append("--name=%s" % elabel)
    if cutoff:
      cmd.append("--variance-threshold=%.2f" % cutoff)
    if pca:
      cmd.append("--pca=%.2f" % pca)
    cmd.append(tracedir)
    subprocess.check_call(cmd)

  results = [["method","groundtruth","experiment","score"]]
  measurements = ["f", "fmi", "ari", "ami"]
# Measure results.
  for method in measurements:
    print "Comparing clusters with %s" % method
    for (ename,cmeth,clusters,outfile,elabel,cutoff,tracedir,pca) in experiments:
      cmd = ["./analyze_clusters"]
      cmd.append("--method=%s" % method)
      cmd.append(groundtruth_name)
      cmd.append(outfile)
      res = subprocess.check_output(cmd)
      res = res.strip()
      print res
      u = [method]
      u.extend(res.split(","))
      results.append(u)

  resultsout = open(args.results, 'w')
  csvw = csv.writer(resultsout)
  for r in results:
    csvw.writerow(r)
  return 0

if __name__ == '__main__':
  parser = argparse.ArgumentParser('experiment')
  parser.add_argument('csv', type=str)
  parser.add_argument('name', type=str)
  parser.add_argument('datapath', type=str)
  parser.add_argument('program', type=str)
  parser.add_argument('--results', type=str, default="results.csv")
  parser.add_argument('--data-source', type=str, default='ed', choices=['ed', 'benji', 'andrew'])
  parser.add_argument('--variance', type=float, default=None)
  parser.add_argument('--clusters', type=int, default=3)
  args = parser.parse_args()
  sys.exit(main(args))