* Further simplification of the config structures.

* Avoid long parameter lists in function calls by passing config object(s)
* Java API handled - configs are still passed as temp JSON files i.e.
Protobuf objects are serialized to JSON and then read by the C++
modules. However, no more mixing of JSON and Protobuf structures in the
JNI functions. Eventually, I would like to add a set of Protobuf JNI
functions, but this is low priority.
* Modifications to the Protobuf structures to allow editing of vid

Author: kgururaj

CMakeLists.txt

@@ -37,7 +37,7 @@ set(LIBDBI_DIR "" CACHE PATH "Path to libdbi install directory")
 set(MAVEN_QUIET False CACHE BOOL "Do not print mvn messages")
 set(GENOMICSDB_SPARK_PROFILE "" CACHE STRING "Profile to choose Apache Spark version") # used in Maven builds
 set(IPPROOT "" CACHE PATH "Path to IPP libraries - used when optimized zlib is used")
-set(GENERATE_PROTOBUF_FILES_IN_BUILD_DIR False CACHE BOOL "Generate all protobuf files in build directory")
+set(GENERATE_PROTOBUF_FILES_IN_BUILD_DIR True CACHE BOOL "Generate all protobuf files in build directory")
 if (NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release)
 endif()

example/java/TestBufferStreamGenomicsDBImporter.java

@@ -147,7 +147,7 @@ public static void main(final String[] args)
       Boolean.parseBoolean(args[argsLoaderFileIdx + 6]);
     //<loader.json> first arg
     String loaderJSONFile = args[argsLoaderFileIdx];
-    GenomicsDBImporter loader = new GenomicsDBImporter(loaderJSONFile, rank, lbRowIdx, ubRowIdx);
+    GenomicsDBImporter loader = new GenomicsDBImporter(loaderJSONFile, rank);
     //<stream_name_to_file.json> - useful for the driver only
     //JSON file that contains "stream_name": "vcf_file_path" entries
     FileReader mappingReader = new FileReader(args[argsLoaderFileIdx+1]);

example/java/TestGenomicsDB.java

@@ -157,15 +157,13 @@ public enum ArgsIdxEnum
     ARGS_IDX_DO_LOAD(1001),
     ARGS_IDX_REFERENCE_GENOME(1002),
     ARGS_IDX_TEMPLATE_VCF_HEADER(1003),
-    ARGS_IDX_LB_ROW_IDX(1004),
-    ARGS_IDX_UB_ROW_IDX(1005),
-    ARGS_IDX_CHROMOSOME(1006),
-    ARGS_IDX_BEGIN(1007),
-    ARGS_IDX_END(1008),
-    ARGS_IDX_COUNT_ONLY(1009),
-    ARGS_IDX_PASS_AS_VCF(1010),
-    ARGS_IDX_PASS_THROUGH_QUERY_JSON(1011),
-    ARGS_IDX_AFTER_LAST_ARG_IDX(1012);
+    ARGS_IDX_CHROMOSOME(1004),
+    ARGS_IDX_BEGIN(1005),
+    ARGS_IDX_END(1006),
+    ARGS_IDX_COUNT_ONLY(1007),
+    ARGS_IDX_PASS_AS_VCF(1008),
+    ARGS_IDX_PASS_THROUGH_QUERY_JSON(1009),
+    ARGS_IDX_AFTER_LAST_ARG_IDX(1010);
 
     private final int mArgsIdx;
     ArgsIdxEnum(final int idx)
@@ -182,27 +180,22 @@ int idx()
   public static void main(final String[] args) throws IOException
   {
     int firstEnumIdx = ArgsIdxEnum.ARGS_IDX_DO_QUERY.idx();
-    LongOpt[] longopts = new LongOpt[16];
+    LongOpt[] longopts = new LongOpt[14];
     longopts[0] = new LongOpt("query", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_DO_QUERY.idx());
     longopts[1] = new LongOpt("load", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_DO_LOAD.idx());
     //Specify rank (or partition idx) of this process
     longopts[2] = new LongOpt("rank", LongOpt.REQUIRED_ARGUMENT, null, 'r');
-    longopts[5] = new LongOpt("workspace", LongOpt.REQUIRED_ARGUMENT, null, 'w');
-    longopts[6] = new LongOpt("array", LongOpt.REQUIRED_ARGUMENT, null, 'A');
     longopts[3] = new LongOpt("reference_genome", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_REFERENCE_GENOME.idx());
     longopts[4] = new LongOpt("template_vcf_header", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_TEMPLATE_VCF_HEADER.idx());
-    //Specify smallest row idx from which to start loading - useful for
-    //incremental loading into existing array
-    longopts[7] = new LongOpt("lb_row_idx", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_LB_ROW_IDX.idx());
-    //Specify largest row idx up to which loading should be performed - for completeness
-    longopts[8] = new LongOpt("ub_row_idx", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_UB_ROW_IDX.idx());
-    longopts[9] = new LongOpt("chromosome", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_CHROMOSOME.idx());
-    longopts[10] = new LongOpt("begin", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_BEGIN.idx());
-    longopts[11] = new LongOpt("end", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_END.idx());
-    longopts[12] = new LongOpt("loader_json_file", LongOpt.REQUIRED_ARGUMENT, null, 'l');
-    longopts[13] = new LongOpt("count_only", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_COUNT_ONLY.idx());
-    longopts[14] = new LongOpt("pass_as_vcf", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_PASS_AS_VCF.idx());
-    longopts[15] = new LongOpt("pass_through_query_json", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_PASS_THROUGH_QUERY_JSON.idx());
+    longopts[5] = new LongOpt("workspace", LongOpt.REQUIRED_ARGUMENT, null, 'w');
+    longopts[6] = new LongOpt("array", LongOpt.REQUIRED_ARGUMENT, null, 'A');
+    longopts[7] = new LongOpt("chromosome", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_CHROMOSOME.idx());
+    longopts[8] = new LongOpt("begin", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_BEGIN.idx());
+    longopts[9] = new LongOpt("end", LongOpt.REQUIRED_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_END.idx());
+    longopts[10] = new LongOpt("loader_json_file", LongOpt.REQUIRED_ARGUMENT, null, 'l');
+    longopts[11] = new LongOpt("count_only", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_COUNT_ONLY.idx());
+    longopts[12] = new LongOpt("pass_as_vcf", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_PASS_AS_VCF.idx());
+    longopts[13] = new LongOpt("pass_through_query_json", LongOpt.NO_ARGUMENT, null, ArgsIdxEnum.ARGS_IDX_PASS_THROUGH_QUERY_JSON.idx());
     if(args.length < 2)
     {
       System.err.println("Usage:\n\tFor querying: --query <loader.json> [<query.json> |"
@@ -220,8 +213,6 @@ public static void main(final String[] args) throws IOException
     String array = "";
     String referenceGenome = "";
     String templateVCFHeader = "";
-    long lbRowIdx = 0;
-    long ubRowIdx = Long.MAX_VALUE-1;
     String chromosome = "";
     int chrBegin = 1;
     int chrEnd = Integer.MAX_VALUE-1;
@@ -270,12 +261,6 @@ public static void main(final String[] args) throws IOException
               case ARGS_IDX_TEMPLATE_VCF_HEADER:
                 templateVCFHeader = g.getOptarg();
                 break;
-              case ARGS_IDX_LB_ROW_IDX:
-                lbRowIdx = Long.parseLong(g.getOptarg());
-                break;
-              case ARGS_IDX_UB_ROW_IDX:
-                ubRowIdx = Long.parseLong(g.getOptarg());
-                break;
               case ARGS_IDX_CHROMOSOME:
                 chromosome = g.getOptarg();
                 break;
@@ -338,7 +323,7 @@ public static void main(final String[] args) throws IOException
         loaderJSONFile = args[g.getOptind()];
       //<loader.json>
       GenomicsDBImporter loader = new GenomicsDBImporter(loaderJSONFile);
-      loader.write(rank, lbRowIdx, ubRowIdx);
+      loader.write(rank);
     }
   }
 }

src/main/cpp/include/config/genomicsdb_config_base.h

@@ -79,6 +79,7 @@ class GenomicsDBConfigBase
     void set_segment_size(const size_t v) { m_segment_size = v; }
     inline unsigned get_determine_sites_with_max_alleles() const { return m_determine_sites_with_max_alleles; }
     inline unsigned get_max_diploid_alt_alleles_that_can_be_genotyped() const { return m_max_diploid_alt_alleles_that_can_be_genotyped; }
+    void set_combined_vcf_records_buffer_size_limit(const size_t val) { m_combined_vcf_records_buffer_size_limit = val; }
     inline size_t get_combined_vcf_records_buffer_size_limit() const { return m_combined_vcf_records_buffer_size_limit; }
     void set_vcf_header_filename(const std::string& vcf_header_filename);
     const std::string& get_vcf_header_filename() const { return m_vcf_header_filename; }

src/main/cpp/include/config/json_config.h

@@ -53,4 +53,6 @@ class JSONConfigBase : public GenomicsDBConfigBase
     rapidjson::Document m_json;
 };
 
+rapidjson::Document parse_json_file(const std::string& s);
+
 #endif

src/main/cpp/include/utils/vid_mapper.h

@@ -803,6 +803,16 @@ class VidMapper
         const int num_partition_callset_mapping_files,
         const int rank) const;
 
+    //Buffer streams for import
+    void set_buffer_stream_info(
+        const std::vector<BufferStreamInfo>& buffer_stream_info_vec);
+    //Read callsets information from json_doc structure
+    void read_callsets_info(const rapidjson::Value& json_doc, const int rank=0);
+    void parse_callsets_json(
+        const std::string& filename,
+        const bool is_file);
+    void parse_callsets_json(
+        const rapidjson::Value& callsets_container); 
 
   protected:
     void add_mandatory_fields();
@@ -876,78 +886,16 @@ class FileBasedVidMapper : public VidMapper
     {
     }
 
-    FileBasedVidMapper(
-        const std::string& filename,
-        const std::string& callset_mapping_file="",
-        const bool is_callset_mapping_required=true) : VidMapper() {
-      std::vector<BufferStreamInfo> empty_vec;
-      common_constructor_initialization(
-          filename,
-          empty_vec,
-          callset_mapping_file,
-          "",
-          is_callset_mapping_required);
-    }
-
-    FileBasedVidMapper(
-        const rapidjson::Document& json_doc,
-        const std::string& callset_mapping_file="",
-        const bool is_callset_mapping_required=true) : VidMapper() {
-      std::vector<BufferStreamInfo> empty_vec;
-      common_constructor_initialization(
-          json_doc,
-          empty_vec,
-          callset_mapping_file,
-          "",
-          is_callset_mapping_required);
-    }
-
-    FileBasedVidMapper(
-        const rapidjson::Document& json_doc,
-        const std::vector<BufferStreamInfo>& buffer_stream_info_vec,
-        const std::string& callset_mapping_file="",
-        const std::string& buffer_stream_callset_mapping_json_string="",
-        const bool is_callset_mapping_required=true) : VidMapper() {
-      common_constructor_initialization(
-          json_doc,
-          buffer_stream_info_vec,
-          callset_mapping_file,
-          buffer_stream_callset_mapping_json_string,
-          is_callset_mapping_required);
-    }
-
-    FileBasedVidMapper(
-        const std::string& filename,
-        const std::vector<BufferStreamInfo>& buffer_stream_info_vec,
-        const std::string& callset_mapping_file="",
-        const std::string& buffer_stream_callset_mapping_json_string="",
-        const bool is_callset_mapping_required=true) : VidMapper() {
-      common_constructor_initialization(
-          filename,
-          buffer_stream_info_vec,
-          callset_mapping_file,
-          buffer_stream_callset_mapping_json_string,
-          is_callset_mapping_required);
+    FileBasedVidMapper(const std::string& filename);
+    FileBasedVidMapper(const rapidjson::Value& json_doc)
+      : VidMapper()
+    {
+      common_constructor_initialization(json_doc);
     }
   private:
     void common_constructor_initialization(
-        const std::string& filename,
-        const std::vector<BufferStreamInfo>& buffer_stream_info_vec,
-        const std::string& callset_mapping_file="",
-        const std::string& buffer_stream_callset_mapping_json_string="",
-        const bool is_callset_mapping_required=true);
-    
-    void common_constructor_initialization(
-        const rapidjson::Document& json_doc,
-        const std::vector<BufferStreamInfo>& buffer_stream_info_vec,
-        const std::string& callset_mapping_file="",
-        const std::string& buffer_stream_callset_mapping_json_string="",
-        const bool is_callset_mapping_required=true);
+        const rapidjson::Value& json_doc);
   private:
-    void parse_callsets_json(
-        const std::string& filename,
-        const std::vector<BufferStreamInfo>& buffer_stream_info_vec,
-        const bool is_file);
     void parse_length_descriptor(const char* field_name,
         const rapidjson::Value& length_json_value,
         FieldLengthDescriptor& length_descriptor, const size_t length_dim_idx);

src/main/cpp/include/vcf/vcf_adapter.h

@@ -95,7 +95,7 @@ class VCFAdapter
     VCFAdapter(bool open_output=true);
     virtual ~VCFAdapter();
     void clear();
-    void initialize(const GenomicsDBConfigBase& config_base);
+    virtual void initialize(const GenomicsDBConfigBase& config_base);
     //Allocates header
     bcf_hdr_t* initialize_default_header();
     bcf_hdr_t* get_vcf_header() { return m_template_vcf_hdr; }
@@ -156,7 +156,7 @@ class BufferedVCFAdapter : public VCFAdapter, public CircularBufferController
 class VCFSerializedBufferAdapter: public VCFAdapter
 {
   public:
-    VCFSerializedBufferAdapter(bool keep_idx_fields_in_bcf_header=true)
+    VCFSerializedBufferAdapter(bool keep_idx_fields_in_bcf_header=true, bool do_output=false)
       : VCFAdapter(false)
     {
       m_keep_idx_fields_in_bcf_header = keep_idx_fields_in_bcf_header;
@@ -166,17 +166,23 @@ class VCFSerializedBufferAdapter: public VCFAdapter
       m_hts_string.m = 4096u;
       m_hts_string.s = (char*)malloc(m_hts_string.m);
       assert(m_hts_string.s);
+      m_write_fptr = 0;
+      m_do_output = do_output;
     }
     ~VCFSerializedBufferAdapter()
     {
       if(m_hts_string.s && m_hts_string.m > 0)
         free(m_hts_string.s);
       m_hts_string.s = 0;
       m_hts_string.m = 0;
+      if(m_write_fptr && m_write_fptr != stdout && m_write_fptr != stderr)
+        fclose(m_write_fptr);
+      m_write_fptr = 0;
     }
     //Delete copy and move constructors
     VCFSerializedBufferAdapter(const VCFSerializedBufferAdapter& other) = delete;
     VCFSerializedBufferAdapter(VCFSerializedBufferAdapter&& other) = delete;
+    void initialize(const GenomicsDBConfigBase& config_base);
     void set_buffer(RWBuffer& buffer) { m_rw_buffer = &buffer; }
     void print_header();
     void handoff_output_bcf_line(bcf1_t*& line, const size_t bcf_record_size);
@@ -188,12 +194,17 @@ class VCFSerializedBufferAdapter: public VCFAdapter
     }
     void do_output()
     {
-      //FIXME: used only for debugging - write out to stdout
+      assert(m_write_fptr);
+      assert(m_rw_buffer);
+      auto write_size = fwrite(&(m_rw_buffer->m_buffer[0]), 1u,  m_rw_buffer->m_num_valid_bytes, m_write_fptr);
+      assert(write_size == m_rw_buffer->m_num_valid_bytes);
     }
   private:
     bool m_keep_idx_fields_in_bcf_header;
     RWBuffer* m_rw_buffer;
     kstring_t m_hts_string;
+    bool m_do_output;
+    FILE* m_write_fptr;
 };
 
 #endif  //ifdef HTSDIR