perf: speed up when startup meet timeout (#64)

* perf: speed up startup when meet timeout of meta service and config service

lazy init

* test: mockMetaSeverWithDelay simulate timeout

* Update ConfigServiceLocatorTest.java

* test: move some function to parent

* Create ConfigServiceTimeoutIntegrationTest.java

* Update CHANGES.md

* Update ConfigServiceTimeoutIntegrationTest.java

* Revert "perf: speed up startup when meet timeout of meta service and config service"

This reverts commit a8e2070.

* Update BaseIntegrationTest.java

* Update MockedConfigService.java

* Update ConfigServiceLocatorTest.java

* feat: submit tash tryUpdateConfigServices

* test: fix setup will trigger context load

* Update ConfigIntegrationTest.java

* feat: speed up http request when http get to meta server

* feat: add rate limiter and custom timeout for discovery.

* Update RemoteConfigRepositoryTest.java

* Apply suggestions from code review

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

---------

Co-authored-by: Jason Song <nobodyiam@gmail.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: 3 people

CHANGES.md

@@ -9,6 +9,7 @@ Apollo Java 2.3.0
 * [Implement parsing time based on pattern for @ApolloJsonValue](https://github.com/apolloconfig/apollo-java/pull/53)
 * [Enhance to load mocked properties from apollo.cache-dir](https://github.com/apolloconfig/apollo-java/pull/58)
 * [perf: speed up the first loading of namespace when startup meet 404](https://github.com/apolloconfig/apollo-java/pull/61)
+* [perf: speed up when startup meets timeout](https://github.com/apolloconfig/apollo-java/pull/64)
 
 ------------------
 All issues and pull requests are [here](https://github.com/apolloconfig/apollo-java/milestone/3?closed=1)

apollo-client/src/main/java/com/ctrip/framework/apollo/internals/ConfigServiceLocator.java

@@ -21,15 +21,16 @@
 import com.ctrip.framework.apollo.core.utils.DeferredLoggerFactory;
 import com.ctrip.framework.apollo.core.utils.DeprecatedPropertyNotifyUtil;
 import com.ctrip.framework.foundation.Foundation;
+import com.google.common.util.concurrent.RateLimiter;
 import java.lang.reflect.Type;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.ctrip.framework.apollo.build.ApolloInjector;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
@@ -57,6 +58,13 @@ public class ConfigServiceLocator {
   private AtomicReference<List<ServiceDTO>> m_configServices;
   private Type m_responseType;
   private ScheduledExecutorService m_executorService;
+  /**
+   * forbid submit multiple task to {@link #m_executorService}，
+   * <p>
+   * so use this AtomicBoolean as a signal
+   */
+  protected AtomicBoolean discoveryTaskQueueMark;
+  private RateLimiter m_discoveryRateLimiter;
   private static final Joiner.MapJoiner MAP_JOINER = Joiner.on("&").withKeyValueSeparator("=");
   private static final Escaper queryParamEscaper = UrlEscapers.urlFormParameterEscaper();
 
@@ -72,6 +80,8 @@ public ConfigServiceLocator() {
     m_configUtil = ApolloInjector.getInstance(ConfigUtil.class);
     this.m_executorService = Executors.newScheduledThreadPool(1,
         ApolloThreadFactory.create("ConfigServiceLocator", true));
+    this.discoveryTaskQueueMark = new AtomicBoolean(false);
+    this.m_discoveryRateLimiter = RateLimiter.create(m_configUtil.getDiscoveryQPS());
     initConfigServices();
   }
 
@@ -154,14 +164,39 @@ private String getDeprecatedCustomizedConfigService() {
     return configServices;
   }
 
+  void doSubmitUpdateTask() {
+    m_executorService.submit(() -> {
+      boolean needUpdate = this.discoveryTaskQueueMark.getAndSet(false);
+      if (needUpdate) {
+        this.tryUpdateConfigServices();
+      }
+    });
+  }
+
+  void trySubmitUpdateTask() {
+    // forbid multiple submit in a short period
+    boolean needForceAlready = this.discoveryTaskQueueMark.getAndSet(true);
+    if (needForceAlready) {
+      // do not submit because submit already, task running
+    } else {
+      doSubmitUpdateTask();
+    }
+  }
+
   /**
    * Get the config service info from remote meta server.
    *
    * @return the services dto
    */
   public List<ServiceDTO> getConfigServices() {
     if (m_configServices.get().isEmpty()) {
-      updateConfigServices();
+      trySubmitUpdateTask();
+      // quick fail
+      throw new ApolloConfigException(
+          "No available config service, "
+              + "server side maybe crash or network cannot connect to server from this ip, "
+              + "one of meta service url is " + assembleMetaServiceUrl()
+      );
     }
 
     return m_configServices.get();
@@ -190,10 +225,22 @@ public void run() {
         m_configUtil.getRefreshIntervalTimeUnit());
   }
 
+  synchronized boolean tryAcquireForUpdate() {
+    return this.m_discoveryRateLimiter.tryAcquire();
+  }
+
   private synchronized void updateConfigServices() {
+    if (!tryAcquireForUpdate()) {
+      // quick return without wait
+      return;
+    }
     String url = assembleMetaServiceUrl();
 
     HttpRequest request = new HttpRequest(url);
+
+    request.setConnectTimeout(m_configUtil.getDiscoveryConnectTimeout());
+    request.setReadTimeout(m_configUtil.getDiscoveryReadTimeout());
+
     int maxRetries = 2;
     Throwable exception = null;
 

apollo-client/src/main/java/com/ctrip/framework/apollo/util/ConfigUtil.java

@@ -38,6 +38,18 @@
 public class ConfigUtil {
 
   private static final Logger logger = LoggerFactory.getLogger(ConfigUtil.class);
+
+  /**
+   * qps limit: discovery config service from meta
+   * <p>
+   * 1 times per second
+   */
+  private int discoveryQPS = 1;
+  /** 1 second */
+  private int discoveryConnectTimeout = 1000;
+  /** 1 second */
+  private int discoveryReadTimeout = 1000;
+
   private int refreshInterval = 5;
   private TimeUnit refreshIntervalTimeUnit = TimeUnit.MINUTES;
   private int connectTimeout = 1000; //1 second
@@ -210,24 +222,63 @@ public TimeUnit getRefreshIntervalTimeUnit() {
     return refreshIntervalTimeUnit;
   }
 
-  private void initQPS() {
-    String customizedLoadConfigQPS = System.getProperty("apollo.loadConfigQPS");
-    if (!Strings.isNullOrEmpty(customizedLoadConfigQPS)) {
+  static Integer getCustomizedIntegerValue(String systemKey) {
+    String customizedValue = System.getProperty(systemKey);
+    if (!Strings.isNullOrEmpty(customizedValue)) {
       try {
-        loadConfigQPS = Integer.parseInt(customizedLoadConfigQPS);
+        return Integer.parseInt(customizedValue);
       } catch (Throwable ex) {
-        logger.error("Config for apollo.loadConfigQPS is invalid: {}", customizedLoadConfigQPS);
+        logger.error("Config for {} is invalid: {}", systemKey, customizedValue);
       }
     }
+    return null;
+  }
 
-    String customizedLongPollQPS = System.getProperty("apollo.longPollQPS");
-    if (!Strings.isNullOrEmpty(customizedLongPollQPS)) {
-      try {
-        longPollQPS = Integer.parseInt(customizedLongPollQPS);
-      } catch (Throwable ex) {
-        logger.error("Config for apollo.longPollQPS is invalid: {}", customizedLongPollQPS);
+  private void initQPS() {
+    {
+      Integer value = getCustomizedIntegerValue("apollo.discoveryConnectTimeout");
+      if (null != value) {
+        discoveryConnectTimeout = value;
+      }
+    }
+    {
+      Integer value = getCustomizedIntegerValue("apollo.discoveryReadTimeout");
+      if (null != value) {
+        discoveryReadTimeout = value;
+      }
+    }
+    {
+      Integer value = getCustomizedIntegerValue("apollo.discoveryQPS");
+      if (null != value) {
+        discoveryQPS = value;
       }
     }
+
+    {
+      Integer value = getCustomizedIntegerValue("apollo.loadConfigQPS");
+      if (null != value) {
+        loadConfigQPS = value;
+      }
+    }
+
+    {
+      Integer value = getCustomizedIntegerValue("apollo.longPollQPS");
+      if (null != value) {
+        longPollQPS = value;
+      }
+    }
+  }
+
+  public int getDiscoveryQPS() {
+    return discoveryQPS;
+  }
+
+  public int getDiscoveryConnectTimeout() {
+    return discoveryConnectTimeout;
+  }
+
+  public int getDiscoveryReadTimeout() {
+    return discoveryReadTimeout;
   }
 
   public int getLoadConfigQPS() {

apollo-client/src/test/java/com/ctrip/framework/apollo/BaseIntegrationTest.java

@@ -16,12 +16,20 @@
  */
 package com.ctrip.framework.apollo;
 
+import com.ctrip.framework.apollo.build.ApolloInjector;
 import com.ctrip.framework.apollo.core.ConfigConsts;
 import com.ctrip.framework.apollo.core.MetaDomainConsts;
 import com.ctrip.framework.apollo.core.dto.ApolloConfig;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
+import com.ctrip.framework.apollo.internals.RemoteConfigLongPollService;
+import com.google.common.base.Joiner;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.ServerSocket;
+import java.nio.file.Files;
+import java.util.Map;
+import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import com.ctrip.framework.apollo.build.MockInjector;
@@ -39,7 +47,7 @@
  * @author Jason Song(song_s@ctrip.com)
  */
 public abstract class BaseIntegrationTest {
-  private static final Logger logger = LoggerFactory.getLogger(BaseIntegrationTest.class);
+  protected final Logger log = LoggerFactory.getLogger(this.getClass());
 
   private static final String someAppName = "someAppName";
   private static final String someInstanceId = "someInstanceId";
@@ -53,7 +61,11 @@ public abstract class BaseIntegrationTest {
   protected static TimeUnit refreshTimeUnit;
   protected static boolean propertiesOrderEnabled;
   private MockedConfigService mockedConfigService;
-  protected Gson gson = new Gson();
+
+  protected final String defaultNamespace = ConfigConsts.NAMESPACE_APPLICATION;
+
+  private File configDir;
+
 
   protected MockedConfigService newMockedConfigService() {
     this.mockedConfigService = new MockedConfigService(port);
@@ -108,10 +120,22 @@ public void setUp() throws Exception {
     ReflectionTestUtils.invokeMethod(MetaDomainConsts.class, "reset");
 
     MockInjector.setInstance(ConfigUtil.class, new MockConfigUtil());
+
+    configDir = new File(ClassLoaderUtil.getClassPath() + "config-cache");
+    if (configDir.exists()) {
+      configDir.delete();
+    }
+    configDir.mkdirs();
   }
 
   @AfterEach
   public void tearDown() throws Exception {
+    // get the instance will trigger long poll task execute, so move it from setup to tearDown
+    RemoteConfigLongPollService remoteConfigLongPollService
+        = ApolloInjector.getInstance(RemoteConfigLongPollService.class);
+    ReflectionTestUtils.invokeMethod(remoteConfigLongPollService, "stopLongPollingRefresh");
+    recursiveDelete(configDir);
+
     //as ConfigService is singleton, so we must manually clear its container
     ConfigService.reset();
     MockInjector.reset();
@@ -124,6 +148,23 @@ public void tearDown() throws Exception {
     }
   }
 
+  private void recursiveDelete(File file) {
+    if (!file.exists()) {
+      return;
+    }
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        recursiveDelete(f);
+      }
+    }
+    try {
+      Files.deleteIfExists(file.toPath());
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+  }
+
   protected void setRefreshInterval(int refreshInterval) {
     BaseIntegrationTest.refreshInterval = refreshInterval;
   }
@@ -136,6 +177,38 @@ protected void setPropertiesOrderEnabled(boolean propertiesOrderEnabled) {
     BaseIntegrationTest.propertiesOrderEnabled = propertiesOrderEnabled;
   }
 
+  protected void createLocalCachePropertyFile(Properties properties) {
+    createLocalCachePropertyFile(defaultNamespace, properties);
+  }
+
+  protected void createLocalCachePropertyFile(String namespace, Properties properties) {
+    String filename = assembleLocalCacheFileName(namespace);
+    File file = new File(configDir, filename);
+    try (FileOutputStream in = new FileOutputStream(file)) {
+      properties.store(in, "Persisted by " + this.getClass().getSimpleName());
+    } catch (IOException e) {
+      throw new IllegalStateException("fail to save " + namespace + " to file", e);
+    }
+  }
+
+  private String assembleLocalCacheFileName(String namespace) {
+    return String.format("%s.properties", Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR)
+        .join(someAppId, someClusterName, namespace));
+  }
+
+  protected ApolloConfig assembleApolloConfig(
+      String namespace,
+      String releaseKey,
+      Map<String, String> configurations
+  ) {
+    ApolloConfig apolloConfig =
+        new ApolloConfig(someAppId, someClusterName, namespace, releaseKey);
+
+    apolloConfig.setConfigurations(configurations);
+
+    return apolloConfig;
+  }
+
   public static class MockConfigUtil extends ConfigUtil {
 
     @Override

apollo-client/src/test/java/com/ctrip/framework/apollo/MockedConfigService.java

@@ -38,6 +38,8 @@
  */
 public class MockedConfigService implements AutoCloseable {
 
+  private static final String META_SERVER_PATH = "/services/config?.*";
+
   private final Logger log = LoggerFactory.getLogger(this.getClass());
 
   private final Gson gson = new Gson();
@@ -95,8 +97,8 @@ public void mockMetaServer(ServiceDTO ... serviceDTOList) {
    * @param serviceDTOList apollo meta server's response
    */
   public void mockMetaServer(boolean failedAtFirstTime, ServiceDTO ... serviceDTOList) {
-    final String path = "/services/config";
-    RequestDefinition requestDefinition = HttpRequest.request("GET").withPath(path);
+    RequestDefinition requestDefinition = HttpRequest.request("GET")
+        .withPath(META_SERVER_PATH);
 
     // need clear
     server.clear(requestDefinition);
@@ -118,6 +120,26 @@ public void mockMetaServer(boolean failedAtFirstTime, ServiceDTO ... serviceDTOL
         );
   }
 
+  /**
+   * simulate timeout
+   */
+  public void mockMetaSeverWithDelay(long milliseconds, ServiceDTO ... serviceDTOList) {
+    RequestDefinition requestDefinition = HttpRequest.request("GET")
+        .withPath(META_SERVER_PATH);
+
+    // need clear
+    server.clear(requestDefinition);
+
+    String body = gson.toJson(Lists.newArrayList(serviceDTOList));
+    server.when(requestDefinition)
+        .respond(HttpResponse.response()
+            .withDelay(TimeUnit.MILLISECONDS, milliseconds)
+            .withStatusCode(HttpServletResponse.SC_OK)
+            .withContentType(MediaType.JSON_UTF_8)
+            .withBody(body)
+        );
+  }
+
   public void mockConfigs(
       int mockedStatusCode,
       ApolloConfig apolloConfig