Skip to content

HBASE-24822 Add a command to support archive the earliest log file ma… #2202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bsglz wants to merge 4 commits into
base: master
Choose a base branch
from
Open

HBASE-24822 Add a command to support archive the earliest log file ma… #2202

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7d28d95
HBASE-24822 Add a command to support archive the earliest log file ma…
bsglz Aug 5, 2020
65c5310
fix review issue
bsglz Aug 11, 2020
a12868d
fix typo
bsglz Aug 11, 2020
72f97a5
trigger QA
bsglz Aug 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1322,6 +1322,14 @@ Future<Boolean> abortProcedureAsync(long procId, boolean mayInterruptIfRunning)
*/
void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException;

/**
* Archive the earliest log file.
*
* @param serverName The servername of the regionserver.
* @throws IOException if a remote or network exception occurs
*/
void archiveEarliestWAL(ServerName serverName) throws IOException;

/**
* Helper that delegates to getClusterMetrics().getMasterCoprocessorNames().
* @return an array of master coprocessors
Expand Down
5 changes: 5 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,11 @@ public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCl
get(admin.rollWALWriter(serverName));
}

@Override
public void archiveEarliestWAL(ServerName serverName) throws IOException {
get(admin.archiveEarliestWAL(serverName));
}

@Override
public CompactionState getCompactionState(TableName tableName) throws IOException {
return get(admin.getCompactionState(tableName));
Expand Down
7 changes: 7 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1127,6 +1127,13 @@ default CompletableFuture<Integer> getMasterInfoPort() {
*/
CompletableFuture<Void> rollWALWriter(ServerName serverName);

/**
* Archive the earliest log file.
*
* @param serverName The servername of the region server.
*/
CompletableFuture<Void> archiveEarliestWAL(ServerName serverName);

/**
* Clear compacting queues on a region server.
* @param serverName
Expand Down
5 changes: 5 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,11 @@ public CompletableFuture<Void> rollWALWriter(ServerName serverName) {
return wrap(rawAdmin.rollWALWriter(serverName));
}

@Override
public CompletableFuture<Void> archiveEarliestWAL(ServerName serverName) {
return wrap(rawAdmin.archiveEarliestWAL(serverName));
}

@Override
public CompletableFuture<Void> clearCompactionQueues(ServerName serverName, Set<String> queues) {
return wrap(rawAdmin.clearCompactionQueues(serverName, queues));
Expand Down
13 changes: 13 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@
import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos.RevokeRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos.RevokeResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
Expand Down Expand Up @@ -2913,6 +2915,17 @@ public CompletableFuture<Void> rollWALWriter(ServerName serverName) {
.serverName(serverName).call();
}

@Override
public CompletableFuture<Void> archiveEarliestWAL(ServerName serverName) {
return this
.<Void> newAdminCaller()
.action(
(controller, stub) -> this.<ArchiveWALRequest, ArchiveWALResponse, Void> adminCall(
controller, stub, RequestConverter.buildArchiveWALRequest(),
(s, c, req, done) -> s.archiveEarliestWAL(controller, req, done), resp -> null))
.serverName(serverName).call();
}

@Override
public CompletableFuture<Void> clearCompactionQueues(ServerName serverName, Set<String> queues) {
return this
Expand Down
15 changes: 15 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;

import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearSlowLogResponseRequest;
Expand Down Expand Up @@ -883,6 +884,20 @@ public static RollWALWriterRequest buildRollWALWriterRequest() {
return ROLL_WAL_WRITER_REQUEST;
}

/**
* @see #buildArchiveWALRequest()
*/
private static ArchiveWALRequest ARCHIVE_WAL_REQUEST = ArchiveWALRequest.newBuilder()
.build();

/**
* Get the static ArchiveWALRequest instance
* @return a ArchiveWALRequest
*/
public static ArchiveWALRequest buildArchiveWALRequest() {
return ARCHIVE_WAL_REQUEST;
}

/**
* @see #buildGetServerInfoRequest()
*/
Expand Down
9 changes: 9 additions & 0 deletions hbase-protocol-shaded/src/main/protobuf/server/region/Admin.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,12 @@ message RollWALWriterResponse {
repeated bytes region_to_flush = 1;
}

message ArchiveWALRequest {
Copy link
Copy Markdown
Contributor

@tedyu tedyu Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be renamed to go with the updated method name ?

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will fix later. Thanks.

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thought again, it should be a common name if we want to improve it in future, such as archive a specify log file, archive all log files, etc.
WDYT? Thanks.

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it make sense, i will remove the "earliest" from method name also.

Copy link
Copy Markdown
Contributor

@tedyu tedyu Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think ArchiveWALResponse should contain the status of archival (including error).

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think ArchiveWALResponse should contain the status of archival (including error).

Make sense.

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 11, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Checked the existing api in Admin.java, such as flushRegion, compactRegion, seems currently we dont use status, if there is something wrong at server side, we notice client by throw ServiceException.
Though the FlushRegionResponse has a flushed flag, but does not be used at client.
Maybe we should keep consistent?
Thanks for comment. @tedyu

}

message ArchiveWALResponse {
}

message StopServerRequest {
required string reason = 1;
}
Expand Down Expand Up @@ -351,6 +357,9 @@ service AdminService {
rpc RollWALWriter(RollWALWriterRequest)
returns(RollWALWriterResponse);

rpc archiveEarliestWAL(ArchiveWALRequest)
returns(ArchiveWALResponse);

rpc GetServerInfo(GetServerInfoRequest)
returns(GetServerInfoResponse);

Expand Down
18 changes: 18 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,24 @@ default void postRollWALWriterRequest(
final ObserverContext<RegionServerCoprocessorEnvironment> ctx)
throws IOException {}

/**
* This will be called before executing user request to archive earliest log
* file of a region server.
* @param ctx the environment to interact with the framework and region server.
*/
default void preArchiveWALRequest(
final ObserverContext<RegionServerCoprocessorEnvironment> ctx)
throws IOException {}

/**
* This will be called after executing user request to archive earliest log
* file of a region server.
* @param ctx the environment to interact with the framework and region server.
*/
default void postArchiveWALRequest(
final ObserverContext<RegionServerCoprocessorEnvironment> ctx)
throws IOException {}

/**
* This will be called after the replication endpoint is instantiated.
* @param ctx the environment to interact with the framework and region server.
Expand Down
23 changes: 23 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ArchiveWALResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
Expand Down Expand Up @@ -2394,6 +2396,27 @@ public RollWALWriterResponse rollWALWriter(final RpcController controller,
}
}

/**
* Archive the earliest log file of the region server.
* @param controller the RPC controller
* @param request the request
* @throws ServiceException
*/
@Override
public ArchiveWALResponse archiveEarliestWAL(final RpcController controller,
final ArchiveWALRequest request) throws ServiceException {
try {
checkOpen();
requestCount.increment();
regionServer.getRegionServerCoprocessorHost().preArchiveWALRequest();
regionServer.getWalRoller().requestArchive();
regionServer.getRegionServerCoprocessorHost().postArchiveWALRequest();
ArchiveWALResponse.Builder builder = ArchiveWALResponse.newBuilder();
return builder.build();
} catch (IOException ie) {
throw new ServiceException(ie);
}
}

/**
* Stop the region server.
Expand Down
18 changes: 18 additions & 0 deletions ...erver/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,24 @@ public void call(RegionServerObserver observer) throws IOException {
});
}

public void preArchiveWALRequest() throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() {
@Override
public void call(RegionServerObserver observer) throws IOException {
observer.preArchiveWALRequest(this);
}
});
}

public void postArchiveWALRequest() throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() {
@Override
public void call(RegionServerObserver observer) throws IOException {
observer.postArchiveWALRequest(this);
}
});
}

public void preReplicateLogEntries()
throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() {
Expand Down
33 changes: 31 additions & 2 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,10 @@
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerCall;
import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.trace.TraceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
Expand Down Expand Up @@ -718,7 +720,7 @@ private void cleanOldLogs() throws IOException {
if (logsToArchive != null) {
for (Pair<Path, Long> logAndSize : logsToArchive) {
this.totalLogSize.addAndGet(-logAndSize.getSecond());
archiveLogFile(logAndSize.getFirst());
moveLogFileToArchiveDir(logAndSize.getFirst());
this.walFile2Props.remove(logAndSize.getFirst());
}
}
Expand All @@ -732,7 +734,7 @@ public static Path getWALArchivePath(Path archiveDir, Path p) {
return new Path(archiveDir, p.getName());
}

private void archiveLogFile(final Path p) throws IOException {
private void moveLogFileToArchiveDir(final Path p) throws IOException {
Path newPath = getWALArchivePath(this.walArchiveDir, p);
// Tell our listeners that a log is going to be archived.
if (!this.listeners.isEmpty()) {
Expand Down Expand Up @@ -872,6 +874,33 @@ public Map<byte[], List<byte[]>> rollWriter(boolean force) throws IOException {
}
}

public void archive(RegionServerServices services) throws IOException{
if (getNumRolledLogFiles() < 1) {
return;
}
// get the earliest log of this WAL instance
Map.Entry<Path, WalProps> firstWALEntry = this.walFile2Props.firstEntry();
// flush reigons if necessary
Map<byte[], List<byte[]>> regions =
this.sequenceIdAccounting.findLower(firstWALEntry.getValue().encodedName2HighestSequenceId);
if (regions != null) {
for (Map.Entry<byte[], List<byte[]>> entry : regions.entrySet()) {
String encodedRegionName = Bytes.toString(entry.getKey());
HRegion r = (HRegion) services.getRegion(encodedRegionName);
if (r == null) {
LOG.warn("Failed to flush of {} when archive manually, because it is not online on us",
encodedRegionName);
return;
Copy link
Copy Markdown
Contributor

@tedyu tedyu Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems the return value should not be void - otherwise the caller wouldn't know this scenario (region not found).

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think of just continue, since the region is not online(maybe closed or moved), anyway we will not lost data.

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When region close, the onRegionClose of SequenceIdAccounting should be called, so its regions will consist with onlineRegions in HRegionServer, if not, there must be something wrong, then the onlineRegions shall prevail.

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 11, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or throw an Exception to notice client?

}
r.flushcache(entry.getValue(), false, FlushLifeCycleTracker.DUMMY);
}
}
// move the log file to archive dir
this.totalLogSize.addAndGet(-firstWALEntry.getValue().logSize);
moveLogFileToArchiveDir(firstWALEntry.getKey());
Copy link
Copy Markdown
Contributor

@tedyu tedyu Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if IOException is thrown from this call ?
Would totalLogSize be correct ?

Copy link
Copy Markdown
Contributor Author

@bsglz bsglz Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, the move action should be called first.
BTW, this part copied from existing code, maybe need a follow-on jira to fix it too.
Thanks.

this.walFile2Props.remove(firstWALEntry.getKey());
}

// public only until class moves to o.a.h.h.wal
/** @return the size of log files in use */
public long getLogFileSize() {
Expand Down
9 changes: 9 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractWALRoller.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
Expand Down Expand Up @@ -100,6 +101,14 @@ public void requestRollAll() {
}
}

public void requestArchive() throws IOException {
synchronized (this) {
for (WAL wal : wals.keySet()) {
wal.archive((RegionServerServices) this.abortable);
}
}
}

protected AbstractWALRoller(String name, Configuration conf, T abortable) {
super(name);
this.abortable = abortable;
Expand Down
5 changes: 5 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
import org.apache.hadoop.hbase.util.CommonFSUtils;
Expand Down Expand Up @@ -143,6 +144,10 @@ public Map<byte[], List<byte[]>> rollWriter(boolean force) {
return rollWriter();
}

@Override
public void archive(RegionServerServices services) {
}

@Override
public void shutdown() {
if(closed.compareAndSet(false, true)) {
Expand Down
6 changes: 6 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Set;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
Expand Down Expand Up @@ -83,6 +84,11 @@ public interface WAL extends Closeable, WALFileLengthProvider {
*/
Map<byte[], List<byte[]>> rollWriter(boolean force) throws IOException;

/**
* Archive the earliest log file.
*/
void archive(RegionServerServices services) throws IOException;

/**
* Stop accepting new writes. If we have unsynced writes still in buffer, sync them.
* Extant edits are left in place in backing storage to be replayed later.
Expand Down
Loading