HBASE-29310 Handle Bulk Load Operations in Continuous Backup (apache#7150)

Signed-off-by: Tak Lon (Stephen) Wu <[email protected]>
Reviewed by: Kevin Geiszler <[email protected]>

Author: ankitsol

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/AbstractPitrRestoreHandler.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupType;
 import org.apache.hadoop.hbase.backup.PointInTimeRestoreRequest;
 import org.apache.hadoop.hbase.backup.RestoreRequest;
 import org.apache.hadoop.hbase.backup.util.BackupUtils;
@@ -248,6 +249,8 @@ private PitrBackupMetadata getValidBackup(TableName sTableName, TableName tTable
 
         try {
           if (backupAdmin.validateRequest(restoreRequest)) {
+            // check if any bulkload entry exists post this backup time and before "endtime"
+            checkBulkLoadAfterBackup(conn, sTableName, backup, endTime);
             return backup;
           }
         } catch (IOException e) {
@@ -259,6 +262,31 @@ private PitrBackupMetadata getValidBackup(TableName sTableName, TableName tTable
     return null;
   }
 
+  /**
+   * Checks if any bulk load operation occurred for the specified table post last successful backup
+   * and before restore time.
+   * @param conn       Active HBase connection
+   * @param sTableName Table for which to check bulk load history
+   * @param backup     Last successful backup before the target recovery time
+   * @param endTime    Target recovery time
+   * @throws IOException if a bulkload entry is found in between backup time and endtime
+   */
+  private void checkBulkLoadAfterBackup(Connection conn, TableName sTableName,
+    PitrBackupMetadata backup, long endTime) throws IOException {
+    try (BackupSystemTable backupSystemTable = new BackupSystemTable(conn)) {
+      List<BulkLoad> bulkLoads = backupSystemTable.readBulkloadRows(List.of(sTableName));
+      for (BulkLoad load : bulkLoads) {
+        long lastBackupTs = (backup.getType() == BackupType.FULL)
+          ? backup.getStartTs()
+          : backup.getIncrCommittedWalTs();
+        if (lastBackupTs < load.getTimestamp() && load.getTimestamp() < endTime) {
+          throw new IOException("Bulk load operation detected after last successful backup for "
+            + "table: " + sTableName);
+        }
+      }
+    }
+  }
+
   /**
    * Determines if the given backup is valid for PITR.
    * <p>

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupImageAdapter.java

@@ -19,6 +19,7 @@
 
 import java.util.List;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupType;
 import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
 import org.apache.yetus.audience.InterfaceAudience;
 
@@ -57,4 +58,14 @@ public String getBackupId() {
   public String getRootDir() {
     return image.getRootDir();
   }
+
+  @Override
+  public BackupType getType() {
+    return image.getType();
+  }
+
+  @Override
+  public long getIncrCommittedWalTs() {
+    return image.getIncrCommittedWalTs();
+  }
 }

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupInfoAdapter.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupType;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -57,4 +58,14 @@ public String getBackupId() {
   public String getRootDir() {
     return info.getBackupRootDir();
   }
+
+  @Override
+  public BackupType getType() {
+    return info.getType();
+  }
+
+  @Override
+  public long getIncrCommittedWalTs() {
+    return info.getIncrCommittedWalTs();
+  }
 }

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java

@@ -101,6 +101,11 @@ Builder withCompleteTime(long completeTime) {
         return this;
       }
 
+      Builder withIncrCommittedWalTs(long incrCommittedWalTs) {
+        image.setIncrCommittedWalTs(incrCommittedWalTs);
+        return this;
+      }
+
       BackupImage build() {
         return image;
       }
@@ -115,6 +120,7 @@ BackupImage build() {
     private long completeTs;
     private ArrayList<BackupImage> ancestors;
     private Map<TableName, Map<String, Long>> incrTimeRanges;
+    private long incrCommittedWalTs;
 
     static Builder newBuilder() {
       return new Builder();
@@ -125,20 +131,22 @@ public BackupImage() {
     }
 
     private BackupImage(String backupId, BackupType type, String rootDir, List<TableName> tableList,
-      long startTs, long completeTs) {
+      long startTs, long completeTs, long incrCommittedWalTs) {
       this.backupId = backupId;
       this.type = type;
       this.rootDir = rootDir;
       this.tableList = tableList;
       this.startTs = startTs;
       this.completeTs = completeTs;
+      this.incrCommittedWalTs = incrCommittedWalTs;
     }
 
     static BackupImage fromProto(BackupProtos.BackupImage im) {
       String backupId = im.getBackupId();
       String rootDir = im.getBackupRootDir();
       long startTs = im.getStartTs();
       long completeTs = im.getCompleteTs();
+      long incrCommittedWalTs = im.getIncrCommittedWalTs();
       List<HBaseProtos.TableName> tableListList = im.getTableListList();
       List<TableName> tableList = new ArrayList<>();
       for (HBaseProtos.TableName tn : tableListList) {
@@ -151,7 +159,8 @@ static BackupImage fromProto(BackupProtos.BackupImage im) {
         ? BackupType.FULL
         : BackupType.INCREMENTAL;
 
-      BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
+      BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs,
+        incrCommittedWalTs);
       for (BackupProtos.BackupImage img : ancestorList) {
         image.addAncestor(fromProto(img));
       }
@@ -170,6 +179,7 @@ BackupProtos.BackupImage toProto() {
       builder.setBackupId(backupId);
       builder.setCompleteTs(completeTs);
       builder.setStartTs(startTs);
+      builder.setIncrCommittedWalTs(incrCommittedWalTs);
       if (type == BackupType.FULL) {
         builder.setBackupType(BackupProtos.BackupType.FULL);
       } else {
@@ -287,6 +297,14 @@ public long getCompleteTs() {
       return completeTs;
     }
 
+    public long getIncrCommittedWalTs() {
+      return incrCommittedWalTs;
+    }
+
+    public void setIncrCommittedWalTs(long incrCommittedWalTs) {
+      this.incrCommittedWalTs = incrCommittedWalTs;
+    }
+
     private void setCompleteTs(long completeTs) {
       this.completeTs = completeTs;
     }

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/PitrBackupMetadata.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupType;
 import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
 import org.apache.yetus.audience.InterfaceAudience;
 
@@ -47,4 +48,10 @@ public interface PitrBackupMetadata {
 
   /** Returns Root directory where the backup is stored */
   String getRootDir();
+
+  /** Returns backup type */
+  BackupType getType();
+
+  /** Returns incrCommittedWalTs */
+  long getIncrCommittedWalTs();
 }

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithContinuous.java

@@ -21,6 +21,7 @@
 import static org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -33,16 +34,13 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
 import org.apache.hadoop.hbase.backup.impl.BackupManifest;
 import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
 import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
@@ -68,8 +66,6 @@ public class TestIncrementalBackupWithContinuous extends TestContinuousBackup {
   private static final Logger LOG =
     LoggerFactory.getLogger(TestIncrementalBackupWithContinuous.class);
 
-  private byte[] ROW = Bytes.toBytes("row1");
-  private final byte[] COLUMN = Bytes.toBytes("col");
   private static final int ROWS_IN_BULK_LOAD = 100;
 
   @Test
@@ -186,11 +182,55 @@ public void testIncrementalBackupCopyingBulkloadTillIncrCommittedWalTs() throws
     }
   }
 
-  private void verifyTable(Table t1) throws IOException {
-    Get g = new Get(ROW);
-    Result r = t1.get(g);
-    assertEquals(1, r.size());
-    assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN));
+  @Test
+  public void testPitrFailureDueToMissingBackupPostBulkload() throws Exception {
+    conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, true);
+    String methodName = Thread.currentThread().getStackTrace()[1].getMethodName();
+    TableName tableName1 = TableName.valueOf("table_" + methodName);
+    TEST_UTIL.createTable(tableName1, famName);
+    try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
+
+      // The test starts with no data, and no bulk loaded rows.
+      int expectedRowCount = 0;
+      assertEquals(expectedRowCount, TEST_UTIL.countRows(tableName1));
+      assertTrue(systemTable.readBulkloadRows(List.of(tableName1)).isEmpty());
+
+      // Create continuous backup, bulk loads are now being tracked
+      String backup1 = backupTables(BackupType.FULL, List.of(tableName1), BACKUP_ROOT_DIR, true);
+      assertTrue(checkSucceeded(backup1));
+
+      loadTable(TEST_UTIL.getConnection().getTable(tableName1));
+      expectedRowCount = expectedRowCount + NB_ROWS_IN_BATCH;
+      performBulkLoad("bulkPreIncr", methodName, tableName1);
+      expectedRowCount += ROWS_IN_BULK_LOAD;
+      assertEquals(expectedRowCount, TEST_UTIL.countRows(tableName1));
+      assertEquals(1, systemTable.readBulkloadRows(List.of(tableName1)).size());
+
+      loadTable(TEST_UTIL.getConnection().getTable(tableName1));
+      Thread.sleep(5000);
+
+      // Incremental backup
+      String backup2 =
+        backupTables(BackupType.INCREMENTAL, List.of(tableName1), BACKUP_ROOT_DIR, true);
+      assertTrue(checkSucceeded(backup2));
+      assertEquals(0, systemTable.readBulkloadRows(List.of(tableName1)).size());
+
+      performBulkLoad("bulkPostIncr", methodName, tableName1);
+      assertEquals(1, systemTable.readBulkloadRows(List.of(tableName1)).size());
+
+      loadTable(TEST_UTIL.getConnection().getTable(tableName1));
+      Thread.sleep(10000);
+      long restoreTs = BackupUtils.getReplicationCheckpoint(TEST_UTIL.getConnection());
+
+      // expect restore failure due to no backup post bulkPostIncr bulkload
+      TableName restoredTable = TableName.valueOf("restoredTable");
+      String[] args = PITRTestUtil.buildPITRArgs(new TableName[] { tableName1 },
+        new TableName[] { restoredTable }, restoreTs, null);
+      int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
+      assertNotEquals("Restore should fail since there is one bulkload without any backup", 0, ret);
+    } finally {
+      conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, REPLICATION_MARKER_ENABLED_DEFAULT);
+    }
   }
 
   private void performBulkLoad(String keyPrefix, String testDir, TableName tableName)

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestPointInTimeRestore.java

@@ -67,8 +67,8 @@ private static void setUpBackups() throws Exception {
     // Simulate a backup taken 20 days ago
     EnvironmentEdgeManager
       .injectEdge(() -> System.currentTimeMillis() - 20 * ONE_DAY_IN_MILLISECONDS);
-    PITRTestUtil.loadRandomData(TEST_UTIL, table1, famName, 1000); // Insert initial data into
-                                                                   // table1
+    // Insert initial data into table1
+    PITRTestUtil.loadRandomData(TEST_UTIL, table1, famName, 1000);
 
     // Perform a full backup for table1 with continuous backup enabled
     String[] args =

hbase-protocol-shaded/src/main/protobuf/Backup.proto

@@ -65,6 +65,7 @@ message BackupImage {
   optional uint64 complete_ts = 6;
   repeated BackupImage ancestors = 7;
   repeated TableServerTimestamp tst_map = 8;
+  optional uint64 incr_committed_wal_ts = 9;
 
 }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java

@@ -1195,6 +1195,11 @@ public int run(String[] args) throws Exception {
   public static void main(String[] args) throws Exception {
     Configuration conf = HBaseConfiguration.create();
     int ret = ToolRunner.run(conf, new BulkLoadHFilesTool(conf), args);
+    if (ret == 0) {
+      System.out.println("Bulk load completed successfully.");
+      System.out.println("IMPORTANT: Please take a backup of the table immediately if this table "
+        + "is part of continuous backup");
+    }
     System.exit(ret);
   }