Add a new check to the /readiness, verify that the node has bootstrapped before indicating we are ready to serve requests

Author: burmanm

management-api-server/src/main/java/com/datastax/mgmtapi/resources/K8OperatorResources.java

@@ -42,6 +42,8 @@ public class K8OperatorResources extends BaseResources {
 
   private static final ObjectMapper jsonMapper = new ObjectMapper();
 
+  private boolean bootStrapped = false;
+
   public K8OperatorResources(ManagementApplication application) {
     super(application);
   }
@@ -93,11 +95,22 @@ public Response checkLiveness() {
   public Response checkReadiness() {
     return handle(
         () -> {
+
+          if(!bootStrapped) {
+            ResultSet bootstrapped = app.cqlService.executeCql(app.dbUnixSocketFile, "SELECT bootstrapped FROM system.local WHERE key = 'local'");
+            Row resultBootstrap = bootstrapped.one();
+            if(resultBootstrap != null) {
+              if ("COMPLETED".equalsIgnoreCase(resultBootstrap.getString("bootstrapped"))) {
+                bootStrapped = true;
+              }
+            }
+          }
+
           ResultSet resultSet =
               app.cqlService.executeCql(app.dbUnixSocketFile, "SELECT * from system.local");
           Row result = resultSet.one();
 
-          if (result != null) {
+          if (result != null && bootStrapped) {
             return Response.ok("OK").build();
           } else {
             Response.ResponseBuilder rb = Response.status(Response.Status.INTERNAL_SERVER_ERROR);

management-api-server/src/test/java/com/datastax/mgmtapi/K8OperatorResourcesTest.java

@@ -133,6 +133,93 @@ public void testLiveness() throws Exception {
     Assert.assertTrue(response.getContentAsString().contains("OK"));
   }
 
+  @Test
+  public void testReadinessFailsWhenNotBootstrapped() throws Exception {
+    Context context = setup();
+
+    ResultSet bootstrapResultSet = mock(ResultSet.class);
+    Row bootstrapRow = mock(Row.class);
+    ResultSet localResultSet = mock(ResultSet.class);
+    Row localRow = mock(Row.class);
+
+    when(context.cqlService.executeCql(
+            any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'")))
+        .thenReturn(bootstrapResultSet);
+    when(bootstrapResultSet.one()).thenReturn(bootstrapRow);
+    when(bootstrapRow.getString("bootstrapped")).thenReturn("IN_PROGRESS");
+
+    when(context.cqlService.executeCql(any(), eq("SELECT * from system.local")))
+        .thenReturn(localResultSet);
+    when(localResultSet.one()).thenReturn(localRow);
+
+    MockHttpRequest request = MockHttpRequest.get(ROOT_PATH + "/probes/readiness");
+    MockHttpResponse response = context.invoke(request);
+
+    Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, response.getStatus());
+
+    verify(context.cqlService)
+        .executeCql(any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'"));
+    verify(context.cqlService).executeCql(any(), eq("SELECT * from system.local"));
+  }
+
+  @Test
+  public void testReadinessFailsWhenSystemLocalRowMissing() throws Exception {
+    Context context = setup();
+
+    ResultSet bootstrapResultSet = mock(ResultSet.class);
+    Row bootstrapRow = mock(Row.class);
+    ResultSet localResultSet = mock(ResultSet.class);
+
+    when(context.cqlService.executeCql(
+            any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'")))
+        .thenReturn(bootstrapResultSet);
+    when(bootstrapResultSet.one()).thenReturn(bootstrapRow);
+    when(bootstrapRow.getString("bootstrapped")).thenReturn("COMPLETED");
+
+    when(context.cqlService.executeCql(any(), eq("SELECT * from system.local")))
+        .thenReturn(localResultSet);
+    when(localResultSet.one()).thenReturn(null);
+
+    MockHttpRequest request = MockHttpRequest.get(ROOT_PATH + "/probes/readiness");
+    MockHttpResponse response = context.invoke(request);
+
+    Assert.assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, response.getStatus());
+
+    verify(context.cqlService)
+        .executeCql(any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'"));
+    verify(context.cqlService).executeCql(any(), eq("SELECT * from system.local"));
+  }
+
+  @Test
+  public void testReadinessSucceedsWhenBootstrappedAndSystemLocalPresent() throws Exception {
+    Context context = setup();
+
+    ResultSet bootstrapResultSet = mock(ResultSet.class);
+    Row bootstrapRow = mock(Row.class);
+    ResultSet localResultSet = mock(ResultSet.class);
+    Row localRow = mock(Row.class);
+
+    when(context.cqlService.executeCql(
+            any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'")))
+        .thenReturn(bootstrapResultSet);
+    when(bootstrapResultSet.one()).thenReturn(bootstrapRow);
+    when(bootstrapRow.getString("bootstrapped")).thenReturn("COMPLETED");
+
+    when(context.cqlService.executeCql(any(), eq("SELECT * from system.local")))
+        .thenReturn(localResultSet);
+    when(localResultSet.one()).thenReturn(localRow);
+
+    MockHttpRequest request = MockHttpRequest.get(ROOT_PATH + "/probes/readiness");
+    MockHttpResponse response = context.invoke(request);
+
+    Assert.assertEquals(HttpStatus.SC_OK, response.getStatus());
+    Assert.assertEquals("OK", response.getContentAsString());
+
+    verify(context.cqlService)
+        .executeCql(any(), eq("SELECT bootstrapped FROM system.local WHERE key = 'local'"));
+    verify(context.cqlService).executeCql(any(), eq("SELECT * from system.local"));
+  }
+
   @Test
   public void testSeedReload() throws Exception {
     Context context = setup();