Merge pull request #1091 from clohfink/deadlock_fix

Fail cass admin requests when jmx isnt available early
Netflix · May 15, 2024 · 187143d · 187143d
2 parents 839077c + ae7be8c
commit 187143d
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java
@@ -103,7 +103,7 @@ public static <T> T getRemoteBean(
      *
      * @return
      */
-    private static boolean testConnection() {
+    public static boolean testConnection() {
         // connecting first time hence return false.
         if (tool == null) return false;
 

diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java
@@ -111,6 +111,9 @@ public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces)
     @GET
     @Path("/info")
     public Response cassInfo() throws JSONException {
+        if (!JMXNodeTool.testConnection()) {
+            return Response.status(503).entity("JMXConnectionException").build();
+        }
         JMXNodeTool nodeTool;
         try {
             nodeTool = JMXNodeTool.instance(config);
@@ -137,6 +140,9 @@ public Response cassPartitioner() {
     @GET
     @Path("/ring/{id}")
     public Response cassRing(@PathParam("id") String keyspace) throws JSONException {
+        if (!JMXNodeTool.testConnection()) {
+            return Response.status(503).entity("JMXConnectionException").build();
+        }
         JMXNodeTool nodeTool;
         try {
             nodeTool = JMXNodeTool.instance(config);
@@ -150,6 +156,9 @@ public Response cassRing(@PathParam("id") String keyspace) throws JSONException
     @GET
     @Path("/status")
     public Response statusInfo() throws JSONException {
+        if (!JMXNodeTool.testConnection()) {
+            return Response.status(503).entity("JMXConnectionException").build();
+        }
         if (!instanceState.isGossipActive() && !instanceState.isNativeTransportActive()) {
             // if native is active but gossip isn't we still want to return to show that
             // there is a problem. but if both gossip and native are down the instance is
@@ -228,6 +237,9 @@ public Response cassRepair(
             @QueryParam("localDC") boolean localDCOnly,
             @DefaultValue("false") @QueryParam("primaryRange") boolean primaryRange)
             throws IOException, ExecutionException, InterruptedException {
+        if (!JMXNodeTool.testConnection()) {
+            return Response.status(503).entity("JMXConnectionException").build();
+        }
         JMXNodeTool nodeTool;
         try {
             nodeTool = JMXNodeTool.instance(config);
@@ -242,6 +254,9 @@ public Response cassRepair(
     @GET
     @Path("/version")
     public Response version() {
+        if (!JMXNodeTool.testConnection()) {
+            return Response.status(503).entity("JMXConnectionException").build();
+        }
         JMXNodeTool nodeTool;
         try {
             nodeTool = JMXNodeTool.instance(config);