test: update ITDatastoreTest test to attempt multiple times if a test fails (#104)

Datastore (when not Firestore in Datastore mode) is eventually
consistent. Sometimes this means queries may not show the expected
results within the short timespan of a test. This change adds a new
JUnit Rule - MultipleAttemptsRule - which allows tests to attempt to
succeed up to three times. If a test is unable to pass within the 3
allowed attempts, the errors from each attempt will be returned as an
overall failure.

Fixes #97
Fixes #101

Author: BenWhitehead

google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java

@@ -154,6 +154,8 @@ public class ITDatastoreTest {
 
   @Rule public Timeout globalTimeout = Timeout.seconds(100);
 
+  @Rule public MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(3);
+
   @AfterClass
   public static void afterClass() {
     HELPER.deleteNamespace();

google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/MultipleAttemptsRule.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.datastore.it;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+
+/**
+ * A JUnit rule that allows us to allow multiple attempts of a test execution before it is
+ * ultimately failed. When it fails, all failures will be propagated as the result of the test.
+ */
+public final class MultipleAttemptsRule implements TestRule {
+  private final long initialBackoffMillis;
+  private final int attemptCount;
+
+  MultipleAttemptsRule(int attemptCount) {
+    this(attemptCount, 1000L);
+  }
+
+  MultipleAttemptsRule(int attemptCount, long initialBackoffMillis) {
+    checkState(attemptCount > 0, "attemptCount must be > 0");
+    checkState(initialBackoffMillis > 0, "initialBackoffMillis must be > 0");
+    this.initialBackoffMillis = initialBackoffMillis;
+    this.attemptCount = attemptCount;
+  }
+
+  @Override
+  public Statement apply(final Statement base, Description description) {
+    return new Statement() {
+      @Override
+      public void evaluate() throws Throwable {
+        List<Throwable> failures = new ArrayList<>();
+
+        long retryIntervalMillis = initialBackoffMillis;
+
+        for (int i = 1; i <= attemptCount; i++) {
+          try {
+            base.evaluate();
+            return;
+          } catch (Throwable t) {
+            failures.add(t);
+            Thread.sleep(retryIntervalMillis);
+            retryIntervalMillis *= 1.5f;
+          }
+        }
+
+        MultipleFailureException.assertEmpty(failures);
+      }
+    };
+  }
+}

google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/MultipleAttemptsRuleTest.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.datastore.it;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Rule;
+import org.junit.Test;
+
+public final class MultipleAttemptsRuleTest {
+
+  private static final int NUMBER_OF_ATTEMPTS = 5;
+
+  @Rule public MultipleAttemptsRule rr = new MultipleAttemptsRule(NUMBER_OF_ATTEMPTS, 10);
+
+  private int numberAttempted = 0;
+
+  @Test
+  public void wontPassUntil5() {
+    numberAttempted += 1;
+    assertEquals(NUMBER_OF_ATTEMPTS, numberAttempted);
+  }
+}