HBASE-22964 Fix flaky TestClusterRestartFailover and TestClusterRestartFailoverSplitWithoutZk

Author: infraio

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClusterRestartFailover.java

@@ -17,8 +17,14 @@
  */
 package org.apache.hadoop.hbase.master;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
 import java.util.List;
 import java.util.stream.Collectors;
+
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
@@ -29,7 +35,6 @@
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.junit.Assert;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -61,54 +66,51 @@ public void test() throws Exception {
     UTIL.waitFor(60000, () -> UTIL.getMiniHBaseCluster().getMaster().isInitialized());
     // wait for all SCPs finished
     UTIL.waitFor(60000, () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-      .noneMatch(p -> p instanceof ServerCrashProcedure));
+        .noneMatch(p -> p instanceof ServerCrashProcedure));
     TableName tableName = TABLES[0];
     ServerName testServer = UTIL.getHBaseCluster().getRegionServer(0).getServerName();
     UTIL.waitFor(30000, () -> getServerStateNode(testServer) != null);
     ServerStateNode serverNode = getServerStateNode(testServer);
-    Assert.assertNotNull(serverNode);
-    Assert.assertTrue("serverNode should be ONLINE when cluster runs normally",
-      serverNode.isInState(ServerState.ONLINE));
+    assertNotNull(serverNode);
+    assertTrue("serverNode should be ONLINE when cluster runs normally",
+        serverNode.isInState(ServerState.ONLINE));
     UTIL.createMultiRegionTable(tableName, FAMILY);
     UTIL.waitTableEnabled(tableName);
     Table table = UTIL.getConnection().getTable(tableName);
     for (int i = 0; i < 100; i++) {
       UTIL.loadTable(table, FAMILY);
     }
     List<Integer> ports =
-      UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServersList().stream()
-        .map(serverName -> serverName.getPort()).collect(Collectors.toList());
+        UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServersList().stream()
+            .map(serverName -> serverName.getPort()).collect(Collectors.toList());
     LOG.info("Shutting down cluster");
     UTIL.getHBaseCluster().killAll();
     UTIL.getHBaseCluster().waitUntilShutDown();
     LOG.info("Starting cluster the second time");
     UTIL.restartHBaseCluster(3, ports);
-    UTIL.waitFor(30000, () -> UTIL.getHBaseCluster().getMaster().isInitialized());
+    UTIL.waitFor(60000, () -> UTIL.getHBaseCluster().getMaster().isInitialized());
     serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
-      .getServerNode(testServer);
-    Assert.assertNotNull("serverNode should not be null when restart whole cluster", serverNode);
-    Assert.assertFalse(serverNode.isInState(ServerState.ONLINE));
-    LOG.info("start to find the procedure of SCP for the severName we choose");
-    UTIL.waitFor(60000,
-      () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-        .anyMatch(procedure -> (procedure instanceof ServerCrashProcedure) &&
-          ((ServerCrashProcedure) procedure).getServerName().equals(testServer)));
-    Assert.assertFalse("serverNode should not be ONLINE during SCP processing",
-      serverNode.isInState(ServerState.ONLINE));
-    LOG.info("start to submit the SCP for the same serverName {} which should fail", testServer);
-    Assert
-      .assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
-    Procedure<?> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream()
-      .filter(p -> (p instanceof ServerCrashProcedure) &&
-        ((ServerCrashProcedure) p).getServerName().equals(testServer))
-      .findAny().get();
-    UTIL.waitFor(60000, () -> procedure.isFinished());
+        .getServerNode(testServer);
+    if (serverNode != null) {
+      assertFalse(serverNode.isInState(ServerState.ONLINE));
+      LOG.info("start to find the procedure of SCP for the severName we choose");
+      Procedure<?> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream().filter(
+          p -> (p instanceof ServerCrashProcedure) &&
+              ((ServerCrashProcedure) p).getServerName().equals(testServer)).findAny().get();
+      if (procedure != null) {
+        assertFalse("serverNode should not be ONLINE during SCP processing",
+            serverNode.isInState(ServerState.ONLINE));
+        LOG.info("start to submit the SCP for the same serverName {} which should fail",
+            testServer);
+        assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
+        UTIL.waitFor(60000, () -> procedure.isFinished());
+      }
+    }
     LOG.info("even when the SCP is finished, the duplicate SCP should not be scheduled for {}",
-      testServer);
-    Assert
-      .assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
+        testServer);
+    assertFalse(UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
     serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
-      .getServerNode(testServer);
-    Assert.assertNull("serverNode should be deleted after SCP finished", serverNode);
+        .getServerNode(testServer);
+    assertNull("serverNode should be deleted after SCP finished", serverNode);
   }
 }