Skip to content

Commit b90a833

Browse files
Apache9Wellington Chevreuil
Apache9
authored and
Wellington Chevreuil
committed
HBASE-26640 Reimplement master local region initialization to better work with SFT (apache#4111)
Signed-off-by: Josh Elser <elserj@apache.org> Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org> Change-Id: Iafad7ae046fbbb5ae91a78191aace693030f1909
1 parent 0be09c2 commit b90a833

9 files changed

Lines changed: 377 additions & 35 deletions

File tree

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3844,6 +3844,12 @@ public MetaLocationSyncer getMetaLocationSyncer() {
38443844
return metaLocationSyncer;
38453845
}
38463846

3847+
@RestrictedApi(explanation = "Should only be called in tests", link = "",
3848+
allowedOnPath = ".*/src/test/.*")
3849+
public MasterRegion getMasterRegion() {
3850+
return masterRegion;
3851+
}
3852+
38473853
@RestrictedApi(explanation = "Should only be called in tests", link = "",
38483854
allowedOnPath = ".*/src/test/.*")
38493855
void setLoadBalancer(LoadBalancer loadBalancer) {

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java‎

Lines changed: 117 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,33 @@
2020
import static org.apache.hadoop.hbase.HConstants.HREGION_LOGDIR_NAME;
2121

2222
import java.io.IOException;
23+
import java.util.List;
2324
import org.apache.hadoop.conf.Configuration;
2425
import org.apache.hadoop.fs.FileStatus;
2526
import org.apache.hadoop.fs.FileSystem;
2627
import org.apache.hadoop.fs.Path;
2728
import org.apache.hadoop.hbase.HBaseIOException;
2829
import org.apache.hadoop.hbase.Server;
2930
import org.apache.hadoop.hbase.TableName;
31+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
3032
import org.apache.hadoop.hbase.client.Get;
3133
import org.apache.hadoop.hbase.client.RegionInfo;
3234
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
3335
import org.apache.hadoop.hbase.client.Result;
3436
import org.apache.hadoop.hbase.client.Scan;
3537
import org.apache.hadoop.hbase.client.TableDescriptor;
38+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3639
import org.apache.hadoop.hbase.regionserver.HRegion;
3740
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
3841
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
3942
import org.apache.hadoop.hbase.regionserver.RegionScanner;
43+
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
44+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
45+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
4046
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
4147
import org.apache.hadoop.hbase.util.Bytes;
4248
import org.apache.hadoop.hbase.util.CommonFSUtils;
49+
import org.apache.hadoop.hbase.util.FSTableDescriptors;
4350
import org.apache.hadoop.hbase.util.FSUtils;
4451
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
4552
import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
@@ -91,6 +98,10 @@ public final class MasterRegion {
9198

9299
private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
93100

101+
static final String INITIALIZING_FLAG = ".initializing";
102+
103+
static final String INITIALIZED_FLAG = ".initialized";
104+
94105
private static final int REGION_ID = 1;
95106

96107
private final WALFactory walFactory;
@@ -191,32 +202,39 @@ private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRol
191202

192203
private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
193204
Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
194-
MasterRegionWALRoller walRoller, String serverName) throws IOException {
205+
MasterRegionWALRoller walRoller, String serverName, boolean touchInitializingFlag)
206+
throws IOException {
195207
TableName tn = td.getTableName();
196208
RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
197-
Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
198-
TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
199-
if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
200-
throw new IOException("Can not delete partial created proc region " + tmpTableDir);
201-
}
202-
HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
203209
Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
204-
if (!fs.rename(tmpTableDir, tableDir)) {
205-
throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
210+
// persist table descriptor
211+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, td, true);
212+
HRegion.createHRegion(conf, regionInfo, fs, tableDir, td).close();
213+
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
214+
if (!fs.mkdirs(initializedFlag)) {
215+
throw new IOException("Can not touch initialized flag: " + initializedFlag);
216+
}
217+
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
218+
if (!fs.delete(initializingFlag, true)) {
219+
LOG.warn("failed to clean up initializing flag: " + initializingFlag);
206220
}
207221
WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
208222
return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
209223
}
210224

211-
private static HRegion open(Configuration conf, TableDescriptor td, FileSystem fs, Path rootDir,
212-
FileSystem walFs, Path walRootDir, WALFactory walFactory, MasterRegionWALRoller walRoller,
213-
String serverName) throws IOException {
214-
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
215-
Path regionDir =
216-
fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
217-
.getPath();
218-
RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
225+
private static RegionInfo loadRegionInfo(FileSystem fs, Path tableDir) throws IOException {
226+
// on branch-2, the RegionInfo.isEncodedRegionName will returns true for .initializing and
227+
// .initialized, see HBASE-25368. Since RegionInfo is IA.Public, changing the implementation may
228+
// raise compatibility concerns, so here we just skip them by our own.
229+
Path regionDir = fs.listStatus(tableDir, p -> !p.getName().startsWith(".")
230+
&& RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0].getPath();
231+
return HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
232+
}
219233

234+
private static HRegion open(Configuration conf, TableDescriptor td, RegionInfo regionInfo,
235+
FileSystem fs, Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
236+
MasterRegionWALRoller walRoller, String serverName) throws IOException {
237+
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
220238
Path walRegionDir = FSUtils.getRegionDirFromRootDir(walRootDir, regionInfo);
221239
Path replayEditsDir = new Path(walRegionDir, REPLAY_EDITS_DIR);
222240
if (!walFs.exists(replayEditsDir) && !walFs.mkdirs(replayEditsDir)) {
@@ -274,6 +292,39 @@ private static HRegion open(Configuration conf, TableDescriptor td, FileSystem f
274292
return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
275293
}
276294

295+
private static void tryMigrate(Configuration conf, FileSystem fs, Path tableDir,
296+
RegionInfo regionInfo, TableDescriptor oldTd, TableDescriptor newTd) throws IOException {
297+
Class<? extends StoreFileTracker> oldSft =
298+
StoreFileTrackerFactory.getTrackerClass(oldTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
299+
Class<? extends StoreFileTracker> newSft =
300+
StoreFileTrackerFactory.getTrackerClass(newTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
301+
if (oldSft.equals(newSft)) {
302+
LOG.debug("old store file tracker {} is the same with new store file tracker, skip migration",
303+
StoreFileTrackerFactory.getStoreFileTrackerName(oldSft));
304+
if (!oldTd.equals(newTd)) {
305+
// we may change other things such as adding a new family, so here we still need to persist
306+
// the new table descriptor
307+
LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
308+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
309+
}
310+
return;
311+
}
312+
LOG.info("Migrate store file tracker from {} to {}", oldSft.getSimpleName(),
313+
newSft.getSimpleName());
314+
HRegionFileSystem hfs =
315+
HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false);
316+
for (ColumnFamilyDescriptor oldCfd : oldTd.getColumnFamilies()) {
317+
StoreFileTracker oldTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
318+
StoreFileTracker newTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
319+
List<StoreFileInfo> files = oldTracker.load();
320+
LOG.debug("Store file list for {}: {}", oldCfd.getNameAsString(), files);
321+
newTracker.set(oldTracker.load());
322+
}
323+
// persist the new table descriptor after migration
324+
LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
325+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
326+
}
327+
277328
public static MasterRegion create(MasterRegionParams params) throws IOException {
278329
TableDescriptor td = params.tableDescriptor();
279330
LOG.info("Create or load local region for table " + td);
@@ -308,16 +359,58 @@ public static MasterRegion create(MasterRegionParams params) throws IOException
308359

309360
WALFactory walFactory = new WALFactory(conf, server.getServerName().toString());
310361
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
362+
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
363+
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
311364
HRegion region;
312-
if (fs.exists(tableDir)) {
313-
// load the existing region.
314-
region = open(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
315-
server.getServerName().toString());
316-
} else {
317-
// bootstrapping...
365+
if (!fs.exists(tableDir)) {
366+
// bootstrap, no doubt
367+
if (!fs.mkdirs(initializedFlag)) {
368+
throw new IOException("Can not touch initialized flag");
369+
}
318370
region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
319-
server.getServerName().toString());
371+
server.getServerName().toString(), true);
372+
} else {
373+
if (!fs.exists(initializedFlag)) {
374+
if (!fs.exists(initializingFlag)) {
375+
// should be old style, where we do not have the initializing or initialized file, persist
376+
// the table descriptor, touch the initialized flag and then open the region.
377+
// the store file tracker must be DEFAULT
378+
LOG.info("No {} or {} file, try upgrading", INITIALIZING_FLAG, INITIALIZED_FLAG);
379+
TableDescriptor oldTd =
380+
TableDescriptorBuilder.newBuilder(td).setValue(StoreFileTrackerFactory.TRACKER_IMPL,
381+
StoreFileTrackerFactory.Trackers.DEFAULT.name()).build();
382+
FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, oldTd, true);
383+
if (!fs.mkdirs(initializedFlag)) {
384+
throw new IOException("Can not touch initialized flag: " + initializedFlag);
385+
}
386+
RegionInfo regionInfo = loadRegionInfo(fs, tableDir);
387+
tryMigrate(conf, fs, tableDir, regionInfo, oldTd, td);
388+
region = open(conf, td, regionInfo, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
389+
server.getServerName().toString());
390+
} else {
391+
// delete all contents besides the initializing flag, here we can make sure tableDir
392+
// exists(unless someone delete it manually...), so we do not do null check here.
393+
for (FileStatus status : fs.listStatus(tableDir)) {
394+
if (!status.getPath().getName().equals(INITIALIZING_FLAG)) {
395+
fs.delete(status.getPath(), true);
396+
}
397+
}
398+
region = bootstrap(conf, td, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
399+
server.getServerName().toString(), false);
400+
}
401+
} else {
402+
if (fs.exists(initializingFlag) && !fs.delete(initializingFlag, true)) {
403+
LOG.warn("failed to clean up initializing flag: " + initializingFlag);
404+
}
405+
// open it, make sure to load the table descriptor from fs
406+
TableDescriptor oldTd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
407+
RegionInfo regionInfo = loadRegionInfo(fs, tableDir);
408+
tryMigrate(conf, fs, tableDir, regionInfo, oldTd, td);
409+
region = open(conf, td, regionInfo, fs, rootDir, walFs, walRootDir, walFactory, walRoller,
410+
server.getServerName().toString());
411+
}
320412
}
413+
321414
Path globalArchiveDir = HFileArchiveUtil.getArchivePath(baseConf);
322415
MasterRegionFlusherAndCompactor flusherAndCompactor = new MasterRegionFlusherAndCompactor(conf,
323416
server, region, params.flushSize(), params.flushPerChanges(), params.flushIntervalMs(),

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@
2525
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
2626
import org.apache.hadoop.hbase.client.TableDescriptor;
2727
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
28+
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
29+
import org.apache.hadoop.hbase.regionserver.BloomType;
30+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
31+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
2832
import org.apache.hadoop.hbase.util.Bytes;
33+
import org.apache.hadoop.hbase.util.ReflectionUtils;
2934
import org.apache.yetus.audience.InterfaceAudience;
3035

3136
/**
@@ -75,17 +80,32 @@ public final class MasterRegionFactory {
7580

7681
private static final int DEFAULT_RING_BUFFER_SLOT_COUNT = 128;
7782

83+
public static final String TRACKER_IMPL = "hbase.master.store.region.file-tracker.impl";
84+
7885
public static final TableName TABLE_NAME = TableName.valueOf("master:store");
7986

8087
public static final byte[] PROC_FAMILY = Bytes.toBytes("proc");
8188

8289
private static final TableDescriptor TABLE_DESC = TableDescriptorBuilder.newBuilder(TABLE_NAME)
8390
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(PROC_FAMILY)).build();
8491

92+
private static TableDescriptor withTrackerConfigs(Configuration conf) {
93+
String trackerImpl = conf.get(TRACKER_IMPL, conf.get(StoreFileTrackerFactory.TRACKER_IMPL,
94+
StoreFileTrackerFactory.Trackers.DEFAULT.name()));
95+
Class<? extends StoreFileTracker> trackerClass =
96+
StoreFileTrackerFactory.getTrackerClass(trackerImpl);
97+
if (StoreFileTrackerFactory.isMigration(trackerClass)) {
98+
throw new IllegalArgumentException("Should not set store file tracker to " +
99+
StoreFileTrackerFactory.Trackers.MIGRATION.name() + " for master local region");
100+
}
101+
StoreFileTracker tracker = ReflectionUtils.newInstance(trackerClass, conf, true, null);
102+
return tracker.updateWithTrackerConfigs(TableDescriptorBuilder.newBuilder(TABLE_DESC)).build();
103+
}
104+
85105
public static MasterRegion create(Server server) throws IOException {
86-
MasterRegionParams params = new MasterRegionParams().server(server)
87-
.regionDirName(MASTER_STORE_DIR).tableDescriptor(TABLE_DESC);
88106
Configuration conf = server.getConfiguration();
107+
MasterRegionParams params = new MasterRegionParams().server(server)
108+
.regionDirName(MASTER_STORE_DIR).tableDescriptor(withTrackerConfigs(conf));
89109
long flushSize = conf.getLong(FLUSH_SIZE_KEY, DEFAULT_FLUSH_SIZE);
90110
long flushPerChanges = conf.getLong(FLUSH_PER_CHANGES_KEY, DEFAULT_FLUSH_PER_CHANGES);
91111
long flushIntervalMs = conf.getLong(FLUSH_INTERVAL_MS_KEY, DEFAULT_FLUSH_INTERVAL_MS);

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public static String getStoreFileTrackerName(Configuration conf) {
8989
return conf.get(TRACKER_IMPL, Trackers.DEFAULT.name());
9090
}
9191

92-
static String getStoreFileTrackerName(Class<? extends StoreFileTracker> clazz) {
92+
public static String getStoreFileTrackerName(Class<? extends StoreFileTracker> clazz) {
9393
Trackers name = CLASS_TO_ENUM.get(clazz);
9494
return name != null ? name.name() : clazz.getName();
9595
}
@@ -176,6 +176,10 @@ public static TableDescriptor updateWithTrackerConfigs(Configuration conf,
176176
return descriptor;
177177
}
178178

179+
public static boolean isMigration(Class<?> clazz) {
180+
return MigrationStoreFileTracker.class.isAssignableFrom(clazz);
181+
}
182+
179183
// should not use MigrationStoreFileTracker for new family
180184
private static void checkForNewFamily(Configuration conf, TableDescriptor table,
181185
ColumnFamilyDescriptor family) throws IOException {

‎hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,13 @@ private static Optional<Pair<FileStatus, TableDescriptor>> getTableDescriptorFro
511511
return td != null ? Optional.of(Pair.newPair(descFile, td)) : Optional.empty();
512512
}
513513

514+
@RestrictedApi(explanation = "Should only be called in tests", link = "",
515+
allowedOnPath = ".*/src/test/.*")
516+
public static void deleteTableDescriptors(FileSystem fs, Path tableDir) throws IOException {
517+
Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
518+
deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE);
519+
}
520+
514521
/**
515522
* Deletes files matching the table info file pattern within the given directory whose sequenceId
516523
* is at most the given max sequenceId.

‎hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/MasterRegionTestBase.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3535
import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
3636
import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
37+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3738
import org.apache.hadoop.hbase.util.Bytes;
3839
import org.apache.hadoop.hbase.util.CommonFSUtils;
3940
import org.junit.After;
@@ -84,19 +85,24 @@ public void setUp() throws IOException {
8485
/**
8586
* Creates a new MasterRegion using an existing {@code htu} on this class.
8687
*/
87-
protected void createMasterRegion() throws IOException {
88-
configure(htu.getConfiguration());
88+
protected final void createMasterRegion() throws IOException {
89+
Configuration conf = htu.getConfiguration();
90+
configure(conf);
8991
choreService = new ChoreService(getClass().getSimpleName());
9092
cleanerPool = new DirScanPool(htu.getConfiguration());
9193
Server server = mock(Server.class);
92-
when(server.getConfiguration()).thenReturn(htu.getConfiguration());
94+
when(server.getConfiguration()).thenReturn(conf);
9395
when(server.getServerName())
9496
.thenReturn(ServerName.valueOf("localhost", 12345, System.currentTimeMillis()));
9597
when(server.getChoreService()).thenReturn(choreService);
9698
Path testDir = htu.getDataTestDir();
97-
CommonFSUtils.setRootDir(htu.getConfiguration(), testDir);
99+
CommonFSUtils.setRootDir(conf, testDir);
98100
MasterRegionParams params = new MasterRegionParams();
99-
params.server(server).regionDirName(REGION_DIR_NAME).tableDescriptor(TD)
101+
TableDescriptor td = TableDescriptorBuilder
102+
.newBuilder(TD).setValue(StoreFileTrackerFactory.TRACKER_IMPL, conf
103+
.get(StoreFileTrackerFactory.TRACKER_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()))
104+
.build();
105+
params.server(server).regionDirName(REGION_DIR_NAME).tableDescriptor(td)
100106
.flushSize(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE).flushPerChanges(1_000_000)
101107
.flushIntervalMs(TimeUnit.MINUTES.toMillis(15)).compactMin(4).maxWals(32).useHsync(false)
102108
.ringBufferSlotCount(16).rollPeriodMs(TimeUnit.MINUTES.toMillis(15))

0 commit comments

Comments
 (0)