HBASE-26170 handleTooBigRequest in NettyRpcServer didn't skip enough bytes (#3564)

Signed-off-by: stack <stack@apache.com>

Author: sunhelly

hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcFrameDecoder.java

@@ -87,7 +87,6 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out)
       NettyRpcServer.LOG.warn(requestTooBigMessage);
 
       if (connection.connectionHeaderRead) {
-        in.skipBytes(FRAME_LENGTH_FIELD_LENGTH);
         handleTooBigRequest(in);
         return;
       }
@@ -107,6 +106,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out)
   }
 
   private void handleTooBigRequest(ByteBuf in) throws IOException {
+    in.skipBytes(FRAME_LENGTH_FIELD_LENGTH);
     in.markReaderIndex();
     int preIndex = in.readerIndex();
     int headerSize = readRawVarint32(in);
@@ -118,6 +118,7 @@ private void handleTooBigRequest(ByteBuf in) throws IOException {
     }
 
     if (in.readableBytes() < headerSize) {
+      NettyRpcServer.LOG.debug("headerSize is larger than readableBytes");
       in.resetReaderIndex();
       return;
     }

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRequestTooBigException.java

@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import static org.apache.hadoop.hbase.ipc.RpcServer.MAX_REQUEST_SIZE;
 import static org.junit.Assert.assertTrue;
-
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
@@ -48,6 +48,7 @@ public class TestRequestTooBigException {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt(MAX_REQUEST_SIZE, 10000);
     TEST_UTIL.startMiniCluster();
   }
 
@@ -64,17 +65,18 @@ public void testHbasePutDeleteCell() throws Exception {
     TEST_UTIL.waitTableAvailable(tableName.getName(), 5000);
     try {
       byte[] value = new byte[2 * 2014 * 1024];
-
-      Put p = new Put(Bytes.toBytes("bigrow"));
-      // big request = 400*2 M
-      for (int i = 0; i < 400; i++) {
-        p.addColumn(family, Bytes.toBytes("someQualifier" + i), value);
-      }
-      try {
-        table.put(p);
-        assertTrue("expected RequestTooBigException", false);
-      } catch (RequestTooBigException e) {
-        assertTrue("expected RequestTooBigException", true);
+      for (int m = 0; m < 10000; m++) {
+        Put p = new Put(Bytes.toBytes("bigrow"));
+        // big request = 400*2 M
+        for (int i = 0; i < 400; i++) {
+          p.addColumn(family, Bytes.toBytes("someQualifier" + i), value);
+        }
+        try {
+          table.put(p);
+          assertTrue("expected RequestTooBigException", false);
+        } catch (RequestTooBigException e) {
+          assertTrue("expected RequestTooBigException", true);
+        }
       }
     } finally {
       table.close();