Skip to content

Commit 2ad5042

Browse files
Apache9Apache9
committed
HBASE-26172 Deprecated MasterRegistry and allow getBootstrapNodes to return master address instead of region server
1 parent d4aed4d commit 2ad5042

5 files changed

Lines changed: 152 additions & 37 deletions

File tree

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterRegistry.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,9 @@
4949
* {@link #MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY} to a value greater than {@code 1} will enable
5050
* it(the default value is {@link AbstractRpcBasedConnectionRegistry#HEDGED_REQS_FANOUT_DEFAULT}).
5151
* <p/>
52-
* TODO: Handle changes to the configuration dynamically without having to restart the client.
52+
* @deprecated Since 2.5.0, will be removed in 4.0.0. Use {@link RpcConnectionRegistry} instead.
5353
*/
54+
@Deprecated
5455
@InterfaceAudience.Private
5556
public class MasterRegistry extends AbstractRpcBasedConnectionRegistry {
5657

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,6 @@
374374
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.ClientMetaService;
375375
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetActiveMasterRequest;
376376
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetActiveMasterResponse;
377-
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetBootstrapNodesRequest;
378-
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetBootstrapNodesResponse;
379377
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetClusterIdRequest;
380378
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetClusterIdResponse;
381379
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegistryProtos.GetMastersRequest;
@@ -3046,13 +3044,18 @@ public GetMastersResponse getMasters(RpcController rpcController, GetMastersRequ
30463044
}
30473045

30483046
@Override
3049-
public GetBootstrapNodesResponse getBootstrapNodes(RpcController controller,
3050-
GetBootstrapNodesRequest request) throws ServiceException {
3051-
GetBootstrapNodesResponse.Builder builder = GetBootstrapNodesResponse.newBuilder();
3052-
for (ServerName sn : master.getServerManager().getOnlineServers().keySet()) {
3053-
builder.addServerName(ProtobufUtil.toServerName(sn));
3047+
protected List<ServerName> getRegionServers() {
3048+
return master.getServerManager().getOnlineServersList();
3049+
}
3050+
3051+
@Override
3052+
protected List<ServerName> getMasters() {
3053+
List<ServerName> list = new ArrayList<>();
3054+
master.getActiveMaster().ifPresent(list::add);
3055+
for (ServerName backupMaster : master.getBackupMasters()) {
3056+
list.add(backupMaster);
30543057
}
3055-
return builder.build();
3058+
return list;
30563059
}
30573060

30583061
@Override

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.util.concurrent.atomic.AtomicBoolean;
4646
import java.util.concurrent.atomic.AtomicLong;
4747
import java.util.concurrent.atomic.LongAdder;
48+
import java.util.stream.Stream;
4849
import org.apache.commons.lang3.mutable.MutableObject;
4950
import org.apache.hadoop.conf.Configuration;
5051
import org.apache.hadoop.fs.FileSystem;
@@ -153,6 +154,7 @@
153154
import org.apache.hadoop.hbase.wal.WALKey;
154155
import org.apache.hadoop.hbase.wal.WALSplitUtil;
155156
import org.apache.hadoop.hbase.wal.WALSplitUtil.MutationReplay;
157+
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
156158
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
157159
import org.apache.yetus.audience.InterfaceAudience;
158160
import org.apache.zookeeper.KeeperException;
@@ -308,18 +310,35 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
308310
*/
309311
private static final long DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10;
310312

311-
/*
313+
/**
312314
* Whether to reject rows with size > threshold defined by
313315
* {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME}
314316
*/
315317
private static final String REJECT_BATCH_ROWS_OVER_THRESHOLD =
316318
"hbase.rpc.rows.size.threshold.reject";
317319

318-
/*
320+
/**
319321
* Default value of config {@link RSRpcServices#REJECT_BATCH_ROWS_OVER_THRESHOLD}
320322
*/
321323
private static final boolean DEFAULT_REJECT_BATCH_ROWS_OVER_THRESHOLD = false;
322324

325+
/**
326+
* Determine the bootstrap nodes we want to return to the client connection registry.
327+
* <ul>
328+
* <li>{@link #MASTER}: return masters as bootstrap nodes.</li>
329+
* <li>{@link #REGIONSERVER}: return region servers as bootstrap nodes.</li>
330+
* <li>{@link #ALL}: return both masters and region servers as bootstrap nodes.</li>
331+
* </ul>
332+
*/
333+
public enum ConnectionRegistryBootstrapNodeType {
334+
MASTER, REGIONSERVER, ALL
335+
}
336+
337+
public static final String CLIENT_BOOTSTRAP_NODE_TYPE = "hbase.client.bootstrap.node_type";
338+
339+
public static final ConnectionRegistryBootstrapNodeType DEFAULT_CLIENT_BOOTSTRAP_NODE_TYPE =
340+
ConnectionRegistryBootstrapNodeType.REGIONSERVER;
341+
323342
// Request counter. (Includes requests that are not serviced by regions.)
324343
// Count only once for requests with multiple actions like multi/caching-scan/replayBatch
325344
final LongAdder requestCount = new LongAdder();
@@ -4102,21 +4121,17 @@ public GetActiveMasterResponse getActiveMaster(RpcController controller,
41024121
@Override
41034122
public GetMastersResponse getMasters(RpcController controller, GetMastersRequest request)
41044123
throws ServiceException {
4105-
try {
4106-
GetMastersResponse.Builder builder = GetMastersResponse.newBuilder();
4107-
ServerName activeMaster = regionServer.getMasterAddressTracker().getMasterAddress();
4108-
if (activeMaster != null) {
4109-
builder.addMasterServers(GetMastersResponseEntry.newBuilder()
4110-
.setServerName(ProtobufUtil.toServerName(activeMaster)).setIsActive(true));
4111-
}
4112-
for (ServerName backupMaster : regionServer.getMasterAddressTracker().getBackupMasters()) {
4113-
builder.addMasterServers(GetMastersResponseEntry.newBuilder()
4114-
.setServerName(ProtobufUtil.toServerName(backupMaster)).setIsActive(false));
4115-
}
4116-
return builder.build();
4117-
} catch (IOException e) {
4118-
throw new ServiceException(e);
4124+
GetMastersResponse.Builder builder = GetMastersResponse.newBuilder();
4125+
ServerName activeMaster = regionServer.getMasterAddressTracker().getMasterAddress();
4126+
if (activeMaster != null) {
4127+
builder.addMasterServers(GetMastersResponseEntry.newBuilder()
4128+
.setServerName(ProtobufUtil.toServerName(activeMaster)).setIsActive(true));
41194129
}
4130+
for (ServerName backupMaster : regionServer.getMasterAddressTracker().getBackupMasters()) {
4131+
builder.addMasterServers(GetMastersResponseEntry.newBuilder()
4132+
.setServerName(ProtobufUtil.toServerName(backupMaster)).setIsActive(false));
4133+
}
4134+
return builder.build();
41204135
}
41214136

41224137
@Override
@@ -4130,12 +4145,45 @@ public GetMetaRegionLocationsResponse getMetaRegionLocations(RpcController contr
41304145
return builder.build();
41314146
}
41324147

4148+
protected List<ServerName> getRegionServers() {
4149+
return regionServer.getRegionServerAddressTracker().getRegionServers();
4150+
}
4151+
4152+
protected List<ServerName> getMasters() {
4153+
List<ServerName> list = new ArrayList<>();
4154+
MasterAddressTracker tracker = regionServer.getMasterAddressTracker();
4155+
ServerName activeMaster = tracker.getMasterAddress();
4156+
if (activeMaster != null) {
4157+
list.add(activeMaster);
4158+
}
4159+
for (ServerName backupMaster : tracker.getBackupMasters()) {
4160+
list.add(backupMaster);
4161+
}
4162+
return list;
4163+
}
4164+
41334165
@Override
4134-
public GetBootstrapNodesResponse getBootstrapNodes(RpcController controller,
4166+
public final GetBootstrapNodesResponse getBootstrapNodes(RpcController controller,
41354167
GetBootstrapNodesRequest request) throws ServiceException {
4168+
ConnectionRegistryBootstrapNodeType type =
4169+
ConnectionRegistryBootstrapNodeType.valueOf(regionServer.getConfiguration()
4170+
.get(CLIENT_BOOTSTRAP_NODE_TYPE, DEFAULT_CLIENT_BOOTSTRAP_NODE_TYPE.name()));
4171+
Stream<ServerName> bootstrapNodes;
4172+
switch (type) {
4173+
case MASTER:
4174+
bootstrapNodes = getMasters().stream();
4175+
break;
4176+
case REGIONSERVER:
4177+
bootstrapNodes = getRegionServers().stream();
4178+
break;
4179+
case ALL:
4180+
bootstrapNodes = Stream.concat(getMasters().stream(), getRegionServers().stream());
4181+
break;
4182+
default:
4183+
throw new IllegalArgumentException("Unknown bootstrap node type:" + type);
4184+
}
41364185
GetBootstrapNodesResponse.Builder builder = GetBootstrapNodesResponse.newBuilder();
4137-
regionServer.getRegionServerAddressTracker().getRegionServers().stream()
4138-
.map(ProtobufUtil::toServerName).forEach(builder::addServerName);
4186+
bootstrapNodes.map(ProtobufUtil::toServerName).forEach(builder::addServerName);
41394187
return builder.build();
41404188
}
41414189
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRpcConnectionRegistry.java‎

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,22 @@
2020
import static org.hamcrest.CoreMatchers.hasItems;
2121
import static org.hamcrest.MatcherAssert.assertThat;
2222
import static org.junit.Assert.assertEquals;
23+
import static org.junit.Assert.fail;
2324

2425
import java.io.IOException;
26+
import java.util.ArrayList;
2527
import java.util.Arrays;
2628
import java.util.Collections;
2729
import java.util.List;
2830
import org.apache.hadoop.hbase.HBaseClassTestRule;
2931
import org.apache.hadoop.hbase.HBaseTestingUtil;
32+
import org.apache.hadoop.hbase.HConstants;
3033
import org.apache.hadoop.hbase.HRegionLocation;
3134
import org.apache.hadoop.hbase.ServerName;
35+
import org.apache.hadoop.hbase.StartTestingClusterOption;
3236
import org.apache.hadoop.hbase.TableName;
3337
import org.apache.hadoop.hbase.master.HMaster;
38+
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
3439
import org.apache.hadoop.hbase.testclassification.ClientTests;
3540
import org.apache.hadoop.hbase.testclassification.MediumTests;
3641
import org.junit.After;
@@ -40,9 +45,14 @@
4045
import org.junit.ClassRule;
4146
import org.junit.Test;
4247
import org.junit.experimental.categories.Category;
48+
import org.junit.runner.RunWith;
49+
import org.junit.runners.Parameterized;
50+
import org.junit.runners.Parameterized.Parameter;
51+
import org.junit.runners.Parameterized.Parameters;
4352

4453
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
4554

55+
@RunWith(Parameterized.class)
4656
@Category({ MediumTests.class, ClientTests.class })
4757
public class TestRpcConnectionRegistry {
4858

@@ -54,13 +64,38 @@ public class TestRpcConnectionRegistry {
5464

5565
private RpcConnectionRegistry registry;
5666

67+
@Parameter
68+
public RSRpcServices.ConnectionRegistryBootstrapNodeType bootstrapNodeType;
69+
70+
@Parameters(name = "{index}: bootstrapNodeType={0}")
71+
public static List<Object[]> params() {
72+
List<Object[]> params = new ArrayList<>();
73+
for (RSRpcServices.ConnectionRegistryBootstrapNodeType type : RSRpcServices.ConnectionRegistryBootstrapNodeType
74+
.values()) {
75+
params.add(new Object[] { type });
76+
}
77+
return params;
78+
}
79+
5780
@BeforeClass
5881
public static void setUpBeforeClass() throws Exception {
5982
// allow refresh immediately so we will switch to use region servers soon.
6083
UTIL.getConfiguration().setLong(RpcConnectionRegistry.PERIODIC_REFRESH_INTERVAL_SECS, 1);
6184
UTIL.getConfiguration().setLong(RpcConnectionRegistry.MIN_SECS_BETWEEN_REFRESHES, 0);
62-
UTIL.startMiniCluster(3);
85+
UTIL.startMiniCluster(StartTestingClusterOption.builder().numMasters(1)
86+
.numAlwaysStandByMasters(1).numRegionServers(3).build());
6387
HBaseTestingUtil.setReplicas(UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
88+
try (ConnectionRegistry registry =
89+
ConnectionRegistryFactory.getRegistry(UTIL.getConfiguration())) {
90+
// Add wait on all replicas being assigned before proceeding w/ test. Failed on occasion
91+
// because not all replicas had made it up before test started.
92+
RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry);
93+
}
94+
// set only the active master as bootstrap node, so it is easy to make sure we have refreshed
95+
// the bootstrap nodes by check the number of bootstrap nodes.
96+
ServerName activeMasterAddr = UTIL.getMiniHBaseCluster().getMaster().getServerName();
97+
UTIL.getConfiguration().set(HConstants.MASTER_ADDRS_KEY,
98+
activeMasterAddr.getHostname() + ":" + activeMasterAddr.getPort());
6499
}
65100

66101
@AfterClass
@@ -70,6 +105,12 @@ public static void tearDownAfterClass() throws Exception {
70105

71106
@Before
72107
public void setUp() throws IOException {
108+
UTIL.getMiniHBaseCluster().getMasterThreads().stream()
109+
.map(t -> t.getMaster().getConfiguration()).forEach(
110+
conf -> conf.set(RSRpcServices.CLIENT_BOOTSTRAP_NODE_TYPE, bootstrapNodeType.name()));
111+
UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
112+
.map(t -> t.getRegionServer().getConfiguration()).forEach(
113+
conf -> conf.set(RSRpcServices.CLIENT_BOOTSTRAP_NODE_TYPE, bootstrapNodeType.name()));
73114
registry = new RpcConnectionRegistry(UTIL.getConfiguration());
74115
}
75116

@@ -78,17 +119,39 @@ public void tearDown() throws IOException {
78119
Closeables.close(registry, true);
79120
}
80121

122+
private void assertHasMasters() {
123+
assertThat(registry.getParsedServers(), hasItems(UTIL.getMiniHBaseCluster().getMasterThreads()
124+
.stream().map(t -> t.getMaster().getServerName()).toArray(ServerName[]::new)));
125+
}
126+
127+
private void assertHasRegionServers() {
128+
assertThat(registry.getParsedServers(),
129+
hasItems(UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
130+
.map(t -> t.getRegionServer().getServerName()).toArray(ServerName[]::new)));
131+
}
132+
81133
@Test
82134
public void testRegistryRPCs() throws Exception {
83135
HMaster activeMaster = UTIL.getHBaseCluster().getMaster();
84-
// wait until we switch to use region servers
85-
UTIL.waitFor(10000, () -> registry.getParsedServers().size() == 3);
86-
assertThat(registry.getParsedServers(),
87-
hasItems(activeMaster.getServerManager().getOnlineServersList().toArray(new ServerName[0])));
136+
// wait until we switch to use refresh bootstrap nodes
137+
switch (bootstrapNodeType) {
138+
case MASTER:
139+
UTIL.waitFor(10000, () -> registry.getParsedServers().size() == 2);
140+
assertHasMasters();
141+
break;
142+
case REGIONSERVER:
143+
UTIL.waitFor(10000, () -> registry.getParsedServers().size() == 3);
144+
assertHasRegionServers();
145+
break;
146+
case ALL:
147+
UTIL.waitFor(10000, () -> registry.getParsedServers().size() == 5);
148+
assertHasMasters();
149+
assertHasRegionServers();
150+
break;
151+
default:
152+
fail("Unknown bootstrap node type: " + bootstrapNodeType);
153+
}
88154

89-
// Add wait on all replicas being assigned before proceeding w/ test. Failed on occasion
90-
// because not all replicas had made it up before test started.
91-
RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry);
92155

93156
assertEquals(registry.getClusterId().get(), activeMaster.getClusterId());
94157
assertEquals(registry.getActiveMaster().get(), activeMaster.getServerName());

‎hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ public static boolean deleteIfEquals(ZKWatcher zkw, final String content) {
307307
return false;
308308
}
309309

310-
public List<ServerName> getBackupMasters() throws InterruptedIOException {
310+
public List<ServerName> getBackupMasters() {
311311
return backupMasters;
312312
}
313313

0 commit comments

Comments
 (0)