Add table data generator

Basic functionality.  Create a .ddf (data definition file) that
specifies the number of rows and columns to generate data for.
Currently only creates sequences.

Author: Mark Wong

.gitignore

@@ -17,9 +17,15 @@ Testing
 
 *.patch
 
+config.h
+
+src/bin/*
+!src/bin/CMakeLists.txt
+
 src/test/*
 !/src/test/CMakeLists.txt
 !/src/test/*.c
 !/src/test/*.expected
 !/src/test/*.sh
 !/src/test/*.sql
+!/src/test/*.ddf

CMakeLists.txt

@@ -13,4 +13,10 @@ ENABLE_TESTING()
 
 SET(CMAKE_C_FLAGS "-g -Wall")
 
+IF(ENABLE_CASSERT EQUAL 1)
+	SET(ENABLE_CASSERT 1)
+ELSE()
+	UNSET(ENABLE_CASSERT)
+ENDIF(ENABLE_CASSERT EQUAL 1)
+
 ADD_SUBDIRECTORY(src)

INSTALL

@@ -5,6 +5,11 @@ Installing
     make
     make install
 
+Debugging
+=========
+
+Use `-DENABLE_CASSERT=1` when running `cmake` for additional debugging.
+
 Uninstalling
 ============
 

README

@@ -0,0 +1 @@
+Touchstone is a library that can be used for random data generation.

doc/table.txt

@@ -0,0 +1,16 @@
+Table definition files instruct the data generator how to generate data.  Files
+should be named after the name of the table to generate with a .ddf extension.
+The resulting data files will have a .data extension.  For example,
+tablename.ddf will generate a tablename.data file.
+
+The format of the file will contain the total number of row to generate on the
+first line.  Each subsequent line in the file will represent the columns to
+generate.
+
+For example:
+
+    10
+    s1
+
+This will generate a data file with 10 rows and 1 column, where that column is
+a sequence of numbers starting from 1.

src/CMakeLists.txt

@@ -4,3 +4,10 @@ ADD_SUBDIRECTORY(test)
 INSTALL(FILES touchstone.h DESTINATION "${INSTALLDEST}/include")
 INSTALL(PROGRAMS libtouchstone.so DESTINATION "${INSTALLDEST}/lib")
 SET_TARGET_PROPERTIES(touchstone PROPERTIES LINK_FLAGS "-lm")
+
+CONFIGURE_FILE(
+    ${CMAKE_CURRENT_SOURCE_DIR}/config.h.in
+    ${CMAKE_CURRENT_BINARY_DIR}/config.h
+)
+
+ADD_SUBDIRECTORY(bin)

src/bin/CMakeLists.txt

@@ -0,0 +1,4 @@
+ADD_EXECUTABLE(${PROJECT_NAME}-generate-table-data ../generate-table-data.c)
+
+INSTALL(PROGRAMS ${PROJECT_NAME}-generate-table-data
+		DESTINATION "${INSTALLDEST}/bin")

src/config.h.in

@@ -0,0 +1 @@
+#cmakedefine ENABLE_CASSERT @ENABLE_CASSERT@

src/generate-table-data.c

@@ -0,0 +1,288 @@
+/*
+ * Copyright 2019 PostgreSQL Global Development Group
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <errno.h>
+#include <limits.h>
+
+#include "config.h"
+
+#define MAX_BUFFER_LEN 1024
+#define MAX_COLS 255
+
+struct sequence_t
+{
+	long long arg1;
+};
+
+union arguments_t
+{
+	struct sequence_t sequence;
+};
+
+struct column_t
+{
+	char type;
+	union arguments_t arguments;
+};
+
+struct table_definition_t
+{
+	long long rows;
+	int columns;
+	struct column_t column[MAX_COLS];
+};
+
+void sequence(char *, long long);
+
+void usage(char *filename)
+{
+	printf("usage: %s [options]\n", filename);
+	printf("  options:\n");
+	printf("    -d <char> - column delimiter, default <tab>\n");
+	printf("    -f <filename> - data definition file\n");
+	printf("    -o <dir> - location to create data file else use stdout\n");
+}
+
+int generate_data(FILE *stream, struct table_definition_t *table,
+		char delimiter)
+{
+	char str[MAX_BUFFER_LEN];
+	int end = table->columns - 1;
+
+	for (long long row = 0; row < table->rows; row++) {
+		for (long long col = 0; col < table->columns; col++) {
+			switch (table->column[col].type) {
+			case 's':
+				sequence(str, row +
+						((struct sequence_t *)
+								&table->column[col].arguments)->arg1);
+				fprintf(stream, "%s", str);
+				break;
+			default:
+				fprintf(stderr, "ERROR: unhandled column definition: %c\n",
+						table->column[col].type);
+				return 7;
+			}
+			if (col < end)
+				fprintf(stream, "%c", delimiter);
+		}
+		fprintf(stream, "\n");
+		fflush(stream);
+	}
+
+	return 0;
+}
+
+int read_data_definition_file(struct table_definition_t *table, char *filename)
+{
+	FILE *f;
+	char *line = NULL;
+	size_t len = 0;
+	ssize_t nread;
+	int *column;
+
+	fprintf(stderr, "reading %s\n", filename);
+
+	f = fopen(filename, "r");
+	if (f == NULL) {
+		fprintf(stderr, "ERROR: cannot open data definition file [%d]: %s\n",
+				errno, filename);
+		return 1;
+	}
+
+	/* Read just the first line for the table cardinality. */
+
+	nread = getline(&line, &len, f);
+	if (nread == -1) {
+		fprintf(stderr, "ERROR: no data on first line of ddf?: %d",
+				errno);
+		return 2;
+	}
+	errno = 0;
+	table->rows = strtoll(line, NULL, 10);
+	if (errno != 0 || table->rows == 0) {
+		fprintf(stderr, "ERROR: number of rows invalid [errno %d]: %s\n",
+				errno, line);
+		free(line);
+		fclose(f);
+		return 5;
+	}
+
+	fprintf(stderr, "%lld row(s)\n", table->rows);
+
+	/* Read the column definition from rest of the file. */
+
+	fprintf(stderr, "column definitions:\n");
+	while ((nread = getline(&line, &len, f)) != -1) {
+		column = &table->columns;
+
+		if (nread == 0) {
+			fprintf(stderr, "ERROR: empty line in data definition file file, "
+					"aborting\n");
+			free(line);
+			fclose(f);
+			return 4;
+		}
+
+		fprintf(stderr, "[%d] %s", table->columns + 1, line);
+
+		table->column[table->columns].type = line[0];
+		switch (line[0]) {
+		case 's':
+			errno = 0;
+			((struct sequence_t *) &table->column[*column].arguments)->arg1 =
+					strtoll(line + 1, NULL, 10);
+			if (errno != 0 || table->rows == 0) {
+				fprintf(stderr,
+						"ERROR: invalid argument to sequence [errno %d]: %s\n",
+						errno, line + 1);
+				free(line);
+				fclose(f);
+				return 7;
+			}
+			break;
+		default:
+			fprintf(stderr, "ERROR: unrecognized column definition: %s\n",
+					line);
+			free(line);
+			fclose(f);
+			return 6;
+		}
+
+		++(*column);
+	}
+
+	free(line);
+	fclose(f);
+
+	if (table->columns == 0) {
+		fprintf(stderr, "ERROR: no columns defined\n");
+		return 3;
+	}
+
+	if (table->columns > MAX_COLS) {
+		fprintf(stderr, "ERROR: more than %d columns defined\n", MAX_COLS);
+		return 3;
+	}
+
+	fprintf(stderr, "%d column(s)\n", *column);
+
+	return 0;
+}
+
+void sequence(char *result, long long value)
+{
+	int rc;
+
+	rc = snprintf(result, MAX_BUFFER_LEN - 1, "%lld", value);
+#ifdef ENABLE_CASSERT
+	if (rc < 0 || rc == MAX_BUFFER_LEN - 1)
+		fprintf(stderr, "WARNING: sequence issue converting %lld to string\n",
+				value);
+#endif /* ENABLE_CASSERT */
+}
+
+int main(int argc, char *argv[])
+{
+	int c;
+	struct table_definition_t table;
+	char datafile[FILENAME_MAX] = "";
+
+	char tmp[FILENAME_MAX];
+	char *p;
+
+	FILE *stream = stdout;
+	char delimiter = '\t';
+	char data_definition_file[FILENAME_MAX] = "";
+	char outdir[FILENAME_MAX] = "";
+
+	memset(&table, 0, sizeof(struct table_definition_t));
+
+	if (argc == 1) {
+		usage(argv[0]);
+		return 1;
+	}
+
+	while (1) {
+		int option_index = 1;
+		static struct option long_options[] = {
+			{0, 0, 0, 0,}
+		};
+
+		c = getopt_long(argc, argv, "d:f:ho:", long_options, &option_index);
+		if (c == -1)
+			break;
+
+		switch (c) {
+		case 0:
+			break;
+		case 'd':
+			delimiter = optarg[0];
+			break;
+		case 'f':
+			strncpy(data_definition_file, optarg, FILENAME_MAX - 1);
+			break;
+		case 'h':
+			usage(argv[0]);
+			return 0;
+		case 'o':
+			strncpy(outdir, optarg, FILENAME_MAX - 1);
+			break;
+		default:
+			printf("?? getopt returned character code 0%o ??\n", c);
+			return 2;
+		}
+	}
+
+	if (data_definition_file[0] == '\0') {
+		fprintf(stderr, "ERROR: use -f to specify data definition file\n");
+		return 3;
+	}
+
+	if (outdir[0] != '\0') {
+		/* Naively remove any extension to the data definition file. */
+		strncpy(tmp, data_definition_file, FILENAME_MAX -1);
+		p = strstr(tmp, ".");
+		*p = '\0';
+
+		/* Make sure the new filename doesn't exceed FILENAME_MAX. */
+		c = FILENAME_MAX - (strlen(outdir) + strlen(tmp) + 7);
+		if (c < 0) {
+			fprintf(stderr, "ERROR: resulting datafile path and name is too "
+					"long: %s/%s.data\n", outdir, tmp);
+			return 6;
+		}
+
+		strcat(datafile, outdir);
+		strcat(datafile, "/");
+		strcat(datafile, tmp);
+		strcat(datafile, ".data");
+
+		stream = fopen(datafile, "w");
+		if (stream == NULL) {
+			fprintf(stderr, "ERROR: cannot open datafile [%d]: %s\n",
+					errno, datafile);
+			return 7;
+		}
+
+		fprintf(stderr, "datafile: %s\n", datafile);
+	}
+
+	c = read_data_definition_file(&table, data_definition_file);
+	if (c != 0)
+		return 4;
+
+	c = generate_data(stream, &table, delimiter);
+	if (c != 0)
+		return 5;
+	if (outdir[0] != '\0') {
+		fclose(stream);
+	}
+
+	return 0;
+}

src/test/CMakeLists.txt

@@ -14,6 +14,7 @@ ADD_EXECUTABLE(tget_days tget_days.c)
 SET_TARGET_PROPERTIES(${BFILES} PROPERTIES LINK_FLAGS "-L.. -ltouchstone")
 
 ADD_TEST(alpha sh tget_alpha.sh)
+ADD_TEST(data_generator sh tdata_generator.sh)
 ADD_TEST(days sh tget_days.sh)
 ADD_TEST(generate_analyze sh tgenerate_analyze.sh)
 ADD_TEST(generate_plan sh tgenerate_plan.sh)

src/test/empty.ddf

@@ -0,0 +1,2 @@
+10
+0

src/test/invalid-column-definition.ddf

@@ -0,0 +1,2 @@
+this should be a number
+1

src/test/invalid-row-line.ddf

@@ -0,0 +1 @@
+1