Flag to allow SQL on incoherent nodes

cdb2api/cdb2api.c

@@ -3162,7 +3162,7 @@ static int cdb2_send_query(cdb2_hndl_tp *hndl, cdb2_hndl_tp *event_hndl,
     }
 
     int n_features = 0;
-    int features[10]; // Max 10 client features??
+    int features[20]; // Max 20 client features??
     CDB2QUERY query = CDB2__QUERY__INIT;
     CDB2SQLQUERY sqlquery = CDB2__SQLQUERY__INIT;
     CDB2SQLQUERY__Snapshotinfo snapshotinfo;
@@ -3272,6 +3272,9 @@ static int cdb2_send_query(cdb2_hndl_tp *hndl, cdb2_hndl_tp *event_hndl,
     if (hndl && (hndl->flags & CDB2_REQUIRE_FASTSQL) != 0) {
         features[n_features++] = CDB2_CLIENT_FEATURES__REQUIRE_FASTSQL;
     }
+    if (hndl && (hndl->flags & CDB2_ALLOW_INCOHERENT) != 0) {
+        features[n_features++] = CDB2_CLIENT_FEATURES__ALLOW_INCOHERENT;
+    }
 
     if (n_features) {
         sqlquery.n_features = n_features;

cdb2api/cdb2api.h

@@ -39,6 +39,7 @@ enum cdb2_hndl_alloc_flags {
     CDB2_TYPE_IS_FD = 256,
     CDB2_REQUIRE_FASTSQL = 512,
     CDB2_MASTER = 1024,
+    CDB2_ALLOW_INCOHERENT = 2048
 };
 
 enum cdb2_request_type {

plugins/newsql/newsql.c

@@ -2260,6 +2260,7 @@ newsql_loop_result newsql_loop(struct sqlclntstate *clnt, CDB2SQLQUERY *sql_quer
     clnt->sql = sql_query->sql_query;
     Pthread_mutex_unlock(&clnt->sql_lk);
     clnt->added_to_hist = 0;
+    int allow_incoherent = 0;
 
     free_original_normalized_sql(clnt);
 
@@ -2278,6 +2279,10 @@ newsql_loop_result newsql_loop(struct sqlclntstate *clnt, CDB2SQLQUERY *sql_quer
     if (clnt->tzname[0] == 0 && sql_query->tzname) {
         strncpy0(clnt->tzname, sql_query->tzname, sizeof(clnt->tzname));
     }
+    for (size_t i = 0; i < sql_query->n_features; i++) {
+        if (sql_query->features[i] == CDB2_CLIENT_FEATURES__ALLOW_INCOHERENT)
+            allow_incoherent = 1;
+    }
     if (sql_query->dbname && thedb->envname && strcasecmp(sql_query->dbname, thedb->envname)) {
         CDB2SQLQUERY__Cinfo *info = sql_query->client_info;
         const char *query = sql_query->sql_query;
@@ -2339,7 +2344,7 @@ newsql_loop_result newsql_loop(struct sqlclntstate *clnt, CDB2SQLQUERY *sql_quer
     if (clnt->plugin.has_ssl(clnt)) ATOMIC_ADD64(gbl_nnewsql_ssl, 1);
 
     /* coherent  _or_ in middle of transaction */
-    if (!incoh_reject(clnt->admin, thedb->bdb_env) || clnt->ctrl_sqlengine != SQLENG_NORMAL_PROCESS) {
+    if ((!incoh_reject(clnt->admin, thedb->bdb_env) || allow_incoherent) || clnt->ctrl_sqlengine != SQLENG_NORMAL_PROCESS) {
         return NEWSQL_SUCCESS;
     }
     if (gbl_incoherent_clnt_wait > 0) {

protobuf/sqlquery.proto

@@ -31,6 +31,8 @@ enum CDB2ClientFeatures {
     REQUIRE_FASTSQL        = 10;
     /* To tell the server that the client can redirect an fdb query */
     CAN_REDIRECT_FDB       = 11;
+    /* Useful for utilities - allow queries on incoherent nodes. */
+    ALLOW_INCOHERENT       = 12;
 }
 
 message CDB2_FLAG {

tests/incoherent_sql.test/Makefile

@@ -0,0 +1,8 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+ifeq ($(TEST_TIMEOUT),)
+	export TEST_TIMEOUT=1m
+endif

tests/incoherent_sql.test/README

@@ -0,0 +1,4 @@
+Tests cdb2api mode to allow SQL on incoherent nodes
+
+Generally a bad idea, but useful for tools that need to
+get the status of all the nodes, etc.

tests/incoherent_sql.test/lrl.options

@@ -0,0 +1 @@
+table t t.csc2

tests/incoherent_sql.test/runit

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+bash -n "$0" | exit 1
+
+set -x
+db=$1
+export CDB2_LOG_CALLS=1
+
+host=$(cdb2sql --tabs ${CDB2_OPTIONS} $db default "select host from comdb2_cluster where is_master='N' limit 1")
+leader=$(cdb2sql --tabs ${CDB2_OPTIONS} $db default "select host from comdb2_cluster where is_master='Y'")
+cdb2sql --host ${host} --tabs ${CDB2_OPTIONS} $db "put tunable rep_debug_delay 11000"
+for i in $(seq 1 100); do
+    cdb2sql ${CDB2_OPTIONS} --host ${leader} $db  "insert into t values(1)" >/dev/null &
+done
+wait
+state=$(cdb2sql --tabs --host ${leader} ${CDB2_OPTIONS} $db "select coherent_state from comdb2_cluster where host='${host}'")
+if [[ !"$state" =~ ^INCOHERENT* ]]; then
+    echo "Expected host to be incoherent!"
+    exit 1
+fi
+checkhost=$(cdb2sql --tabs --allow-incoherent --host ${host} ${CDB2_OPTIONS} $db 'select comdb2_host()')
+if [[ "$checkhost" != "$host" ]]; then
+    echo "Couldn't run query on incoherent host with --allow-incoherent option"
+    exit 1
+fi
+
+checkerr=$(cdb2sql --host ${host} ${CDB2_OPTIONS} $db 'select comdb2_host()' 2>&1)
+if [[ ! "$checkerr" =~ .*Cannot\ connect\ to\ db.* ]]; then
+    echo "Able to run query on incoherent node?"
+    exit 1
+fi
+
+# remove debug delay so unsetup can successfully query all the nodes
+cdb2sql --tabs --allow-incoherent --host ${host} ${CDB2_OPTIONS} $db 'put tunable rep_debug_delay 0'
+cdb2sql ${CDB2_OPTIONS} --host ${leader} $db  "insert into t values(1)" >/dev/null
+cdb2sql ${CDB2_OPTIONS} --host ${leader} $db  "insert into t values(1)" >/dev/null
+
+sleep 20
+
+
+echo "Testcase passed."
+exit 0

tests/incoherent_sql.test/t.csc2

@@ -0,0 +1,3 @@
+schema {
+    int a
+}

tools/cdb2sql/cdb2sql.cpp

@@ -110,6 +110,7 @@ static char *gensql_tbl = NULL;
 static char *prompt = main_prompt;
 static int connect_to_master = 0;
 static int cdb2_master = 0;
+static int allow_incoherent = 0;
 
 static int now_ms(void)
 {
@@ -170,25 +171,25 @@ static const char *usage_text =
     "Usage: cdb2sql [options] dbname [sql [type1 [type2 ...]]]\n"
     "\n"
     "Options:\n"
-    " -c, --cdb2cfg FL    Set the config file to FL\n"
-    "     --coltype       Prefix column output with associated type\n"
-    "     --cost          Log the cost of query in db trace files\n"
-    "     --debugtrace    Set debug trace flag on api handle\n"
-    " -d, --delim str     Set string used to separate two sql statements read "
+    " -c, --cdb2cfg FL        Set the config file to FL\n"
+    "     --coltype           Prefix column output with associated type\n"
+    "     --cost              Log the cost of query in db trace files\n"
+    "     --debugtrace        Set debug trace flag on api handle\n"
+    " -d, --delim str         Set string used to separate two sql statements read "
     "from a file or input stream\n"
-    " -f, --file FL       Read queries from the specified file FL\n"
-    " -h, --help          Help on usage \n"
-    " -n, --host HOST     Host to connect to and run query.\n"
-    " -p, --precision #   Set precision for floation point outputs\n"
-    " -s, --script        Script mode (less verbose output)\n"
-    "     --showeffects   Show the effects of query at the end\n"
-    "     --strblobs      Display blobs as strings\n"
-    "     --tabs          Set column separator to tabs rather than commas\n"
-    "     --tabular       Display result in tabular format\n"
-    " -t, --type TYPE     Type of database or tier ('dev' or 'prod',"
+    " -f, --file FL           Read queries from the specified file FL\n"
+    " -h, --help              Help on usage \n"
+    " -n, --host HOST         Host to connect to and run query.\n"
+    " -p, --precision #       Set precision for floation point outputs\n"
+    " -s, --script            Script mode (less verbose output)\n"
+    "     --showeffects       Show the effects of query at the end\n"
+    "     --strblobs          Display blobs as strings\n"
+    "     --tabs              Set column separator to tabs rather than commas\n"
+    "     --tabular           Display result in tabular format\n"
+    " -t, --type TYPE         Type of database or tier ('dev' or 'prod',"
     " default 'local')\n"
-    " -v, --verbose       Verbose debug output, implies --debugtrace"
-    "\n"
+    " -v, --verbose           Verbose debug output, implies --debugtrace\n"
+    " -i, --allow-incoherent  Allow SQL to run on an incoherent node\n"
     "Examples: \n"
     " * Querying db with name mydb on local server \n"
     "     cdb2sql mydb 'select 1'\n"
@@ -334,6 +335,9 @@ static char *db_generator(const int state, const char *sql)
         if (isadmin)
             flags |= CDB2_ADMIN;
 
+        if (allow_incoherent)
+            flags |= CDB2_ALLOW_INCOHERENT;
+
         rc = cdb2_open(&cdb2h_2, dbname, type, flags);
         if (rc) {
             if (debug_trace)
@@ -1504,6 +1508,9 @@ static int run_statement(const char *sql, int ntypes, int *types,
         if (isadmin)
             flags |= CDB2_ADMIN;
 
+        if (allow_incoherent)
+            flags |= CDB2_ALLOW_INCOHERENT;
+
         rc = cdb2_open(&cdb2h, dbname, type, flags);
 
         cdb2_push_context(cdb2h, "cdb2sql");
@@ -2046,6 +2053,7 @@ int main(int argc, char *argv[])
         {"host", required_argument, NULL, 'n'},
         {"minretries", required_argument, NULL, 'R'},
         {"connect-to-master", no_argument, NULL, 'm'},
+        {"allow-incoherent", no_argument, NULL, 'i'},
         {0, 0, 0, 0}};
 
     while ((c = bb_getopt_long(argc, argv, (char *)"hsvr:p:d:c:f:g:t:n:R:mM", long_options, &opt_indx)) != -1) {
@@ -2098,6 +2106,9 @@ int main(int argc, char *argv[])
         case 'M':
             cdb2_master = 1;
             break;
+        case 'i':
+            allow_incoherent = 1;
+            break;
         case '?':
             cdb2sql_usage(EXIT_FAILURE);
             break;