001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot;
021import static org.apache.hadoop.hbase.backup.BackupInfo.withState;
022import static org.apache.hadoop.hbase.backup.BackupInfo.withType;
023
024import edu.umd.cs.findbugs.annotations.Nullable;
025import java.io.Closeable;
026import java.io.IOException;
027import java.io.InterruptedIOException;
028import java.nio.charset.StandardCharsets;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collection;
032import java.util.Collections;
033import java.util.HashMap;
034import java.util.HashSet;
035import java.util.Iterator;
036import java.util.List;
037import java.util.Map;
038import java.util.Map.Entry;
039import java.util.Objects;
040import java.util.Set;
041import java.util.TreeMap;
042import java.util.TreeSet;
043import java.util.function.Predicate;
044import java.util.stream.Collectors;
045import java.util.stream.Stream;
046import org.apache.commons.lang3.ArrayUtils;
047import org.apache.commons.lang3.StringUtils;
048import org.apache.hadoop.conf.Configuration;
049import org.apache.hadoop.fs.Path;
050import org.apache.hadoop.hbase.Cell;
051import org.apache.hadoop.hbase.CellUtil;
052import org.apache.hadoop.hbase.HBaseConfiguration;
053import org.apache.hadoop.hbase.NamespaceDescriptor;
054import org.apache.hadoop.hbase.NamespaceExistException;
055import org.apache.hadoop.hbase.ServerName;
056import org.apache.hadoop.hbase.TableExistsException;
057import org.apache.hadoop.hbase.TableName;
058import org.apache.hadoop.hbase.TableNotDisabledException;
059import org.apache.hadoop.hbase.backup.BackupInfo;
060import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
061import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
062import org.apache.hadoop.hbase.backup.BackupType;
063import org.apache.hadoop.hbase.backup.util.BackupUtils;
064import org.apache.hadoop.hbase.client.Admin;
065import org.apache.hadoop.hbase.client.BufferedMutator;
066import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
067import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
068import org.apache.hadoop.hbase.client.Connection;
069import org.apache.hadoop.hbase.client.Delete;
070import org.apache.hadoop.hbase.client.Get;
071import org.apache.hadoop.hbase.client.Put;
072import org.apache.hadoop.hbase.client.Result;
073import org.apache.hadoop.hbase.client.ResultScanner;
074import org.apache.hadoop.hbase.client.Scan;
075import org.apache.hadoop.hbase.client.SnapshotDescription;
076import org.apache.hadoop.hbase.client.Table;
077import org.apache.hadoop.hbase.client.TableDescriptor;
078import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
079import org.apache.hadoop.hbase.util.Bytes;
080import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
081import org.apache.yetus.audience.InterfaceAudience;
082import org.slf4j.Logger;
083import org.slf4j.LoggerFactory;
084
085import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
086import org.apache.hbase.thirdparty.com.google.common.base.Splitter;
087import org.apache.hbase.thirdparty.com.google.common.collect.Iterators;
088
089import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
090import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
091
092/**
093 * This class provides API to access backup system table<br>
094 * Backup system table schema:<br>
095 * <p>
096 * <ul>
097 * <li>1. Backup sessions rowkey= "session:"+backupId; value =serialized BackupInfo</li>
098 * <li>2. Backup start code rowkey = "startcode:"+backupRoot; value = startcode</li>
099 * <li>3. Incremental backup set rowkey="incrbackupset:"+backupRoot; table="meta:"+tablename of
100 * include table; value=empty</li>
101 * <li>4. Table-RS-timestamp map rowkey="trslm:"+backupRoot+table_name; value = map[RS-> last WAL
102 * timestamp]</li>
103 * <li>5. RS - WAL ts map rowkey="rslogts:"+backupRoot +server; value = last WAL timestamp</li>
104 * <li>6. WALs recorded rowkey="wals:"+WAL unique file name; value = backupId and full WAL file
105 * name</li>
106 * </ul>
107 * </p>
108 */
109@InterfaceAudience.Private
110public final class BackupSystemTable implements Closeable {
111
112  private static final Logger LOG = LoggerFactory.getLogger(BackupSystemTable.class);
113
114  static class WALItem {
115    String backupId;
116    String walFile;
117    String backupRoot;
118
119    WALItem(String backupId, String walFile, String backupRoot) {
120      this.backupId = backupId;
121      this.walFile = walFile;
122      this.backupRoot = backupRoot;
123    }
124
125    public String getBackupId() {
126      return backupId;
127    }
128
129    public String getWalFile() {
130      return walFile;
131    }
132
133    public String getBackupRoot() {
134      return backupRoot;
135    }
136
137    @Override
138    public String toString() {
139      return Path.SEPARATOR + backupRoot + Path.SEPARATOR + backupId + Path.SEPARATOR + walFile;
140    }
141  }
142
143  /**
144   * Backup system table (main) name
145   */
146  private TableName tableName;
147
148  /**
149   * Backup System table name for bulk loaded files. We keep all bulk loaded file references in a
150   * separate table because we have to isolate general backup operations: create, merge etc from
151   * activity of RegionObserver, which controls process of a bulk loading
152   * {@link org.apache.hadoop.hbase.backup.BackupObserver}
153   */
154  private TableName bulkLoadTableName;
155
156  /**
157   * Stores backup sessions (contexts)
158   */
159  final static byte[] SESSIONS_FAMILY = Bytes.toBytes("session");
160  /**
161   * Stores other meta
162   */
163  final static byte[] META_FAMILY = Bytes.toBytes("meta");
164  final static byte[] BULK_LOAD_FAMILY = Bytes.toBytes("bulk");
165  /**
166   * Connection to HBase cluster, shared among all instances
167   */
168  private final Connection connection;
169
170  private final static String BACKUP_INFO_PREFIX = "session:";
171  private final static String START_CODE_ROW = "startcode:";
172  private final static byte[] ACTIVE_SESSION_ROW = Bytes.toBytes("activesession:");
173  private final static byte[] ACTIVE_SESSION_COL = Bytes.toBytes("c");
174
175  private final static byte[] ACTIVE_SESSION_YES = Bytes.toBytes("yes");
176  private final static byte[] ACTIVE_SESSION_NO = Bytes.toBytes("no");
177
178  private final static String INCR_BACKUP_SET = "incrbackupset:";
179  private final static String CONTINUOUS_BACKUP_SET = "continuousbackupset";
180  /**
181   * Row key identifier for storing the last replicated WAL timestamp in the backup system table for
182   * continuous backup.
183   */
184  private static final String CONTINUOUS_BACKUP_REPLICATION_TIMESTAMP_ROW =
185    "continuous_backup_last_replicated";
186  private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm:";
187  private final static String RS_LOG_TS_PREFIX = "rslogts:";
188
189  private final static String BULK_LOAD_PREFIX = "bulk:";
190  private final static byte[] BULK_LOAD_PREFIX_BYTES = Bytes.toBytes(BULK_LOAD_PREFIX);
191  private final static byte[] DELETE_OP_ROW = Bytes.toBytes("delete_op_row");
192  private final static byte[] MERGE_OP_ROW = Bytes.toBytes("merge_op_row");
193
194  final static byte[] TBL_COL = Bytes.toBytes("tbl");
195  final static byte[] FAM_COL = Bytes.toBytes("fam");
196  final static byte[] PATH_COL = Bytes.toBytes("path");
197
198  private final static String SET_KEY_PREFIX = "backupset:";
199
200  // separator between BULK_LOAD_PREFIX and ordinals
201  private final static String BLK_LD_DELIM = ":";
202  private final static byte[] EMPTY_VALUE = new byte[] {};
203
204  // Safe delimiter in a string
205  private final static String NULL = "\u0000";
206
207  public BackupSystemTable(Connection conn) throws IOException {
208    this.connection = conn;
209    Configuration conf = this.connection.getConfiguration();
210    tableName = BackupSystemTable.getTableName(conf);
211    bulkLoadTableName = BackupSystemTable.getTableNameForBulkLoadedData(conf);
212    checkSystemTable();
213  }
214
215  private void checkSystemTable() throws IOException {
216    try (Admin admin = connection.getAdmin()) {
217      verifyNamespaceExists(admin);
218      Configuration conf = connection.getConfiguration();
219      if (!admin.tableExists(tableName)) {
220        TableDescriptor backupHTD = BackupSystemTable.getSystemTableDescriptor(conf);
221        createSystemTable(admin, backupHTD);
222      }
223      ensureTableEnabled(admin, tableName);
224      if (!admin.tableExists(bulkLoadTableName)) {
225        TableDescriptor blHTD = BackupSystemTable.getSystemTableForBulkLoadedDataDescriptor(conf);
226        createSystemTable(admin, blHTD);
227      }
228      ensureTableEnabled(admin, bulkLoadTableName);
229      waitForSystemTable(admin, tableName);
230      waitForSystemTable(admin, bulkLoadTableName);
231    }
232  }
233
234  private void createSystemTable(Admin admin, TableDescriptor descriptor) throws IOException {
235    try {
236      admin.createTable(descriptor);
237    } catch (TableExistsException e) {
238      // swallow because this class is initialized in concurrent environments (i.e. bulkloads),
239      // so may be subject to race conditions where one caller succeeds in creating the
240      // table and others fail because it now exists
241      LOG.debug("Table {} already exists, ignoring", descriptor.getTableName(), e);
242    }
243  }
244
245  private void verifyNamespaceExists(Admin admin) throws IOException {
246    String namespaceName = tableName.getNamespaceAsString();
247    NamespaceDescriptor ns = NamespaceDescriptor.create(namespaceName).build();
248    NamespaceDescriptor[] list = admin.listNamespaceDescriptors();
249    boolean exists = false;
250    for (NamespaceDescriptor nsd : list) {
251      if (nsd.getName().equals(ns.getName())) {
252        exists = true;
253        break;
254      }
255    }
256    if (!exists) {
257      try {
258        admin.createNamespace(ns);
259      } catch (NamespaceExistException e) {
260        // swallow because this class is initialized in concurrent environments (i.e. bulkloads),
261        // so may be subject to race conditions where one caller succeeds in creating the
262        // namespace and others fail because it now exists
263        LOG.debug("Namespace {} already exists, ignoring", ns.getName(), e);
264      }
265    }
266  }
267
268  private void waitForSystemTable(Admin admin, TableName tableName) throws IOException {
269    // Return fast if the table is available and avoid a log message
270    if (admin.tableExists(tableName) && admin.isTableAvailable(tableName)) {
271      return;
272    }
273    long TIMEOUT = 60000;
274    long startTime = EnvironmentEdgeManager.currentTime();
275    LOG.debug("Backup table {} is not present and available, waiting for it to become so",
276      tableName);
277    while (!admin.tableExists(tableName) || !admin.isTableAvailable(tableName)) {
278      try {
279        Thread.sleep(100);
280      } catch (InterruptedException e) {
281        throw (IOException) new InterruptedIOException().initCause(e);
282      }
283      if (EnvironmentEdgeManager.currentTime() - startTime > TIMEOUT) {
284        throw new IOException(
285          "Failed to create backup system table " + tableName + " after " + TIMEOUT + "ms");
286      }
287    }
288    LOG.debug("Backup table {} exists and available", tableName);
289  }
290
291  @Override
292  public void close() {
293    // do nothing
294  }
295
296  /**
297   * Updates status (state) of a backup session in backup system table table
298   * @param info backup info
299   * @throws IOException exception
300   */
301  public void updateBackupInfo(BackupInfo info) throws IOException {
302    if (LOG.isTraceEnabled()) {
303      LOG.trace("update backup status in backup system table for: " + info.getBackupId()
304        + " set status=" + info.getState());
305    }
306    try (Table table = connection.getTable(tableName)) {
307      Put put = createPutForBackupInfo(info);
308      table.put(put);
309    }
310  }
311
312  /*
313   * @param backupId the backup Id
314   * @return Map of rows to path of bulk loaded hfile
315   */
316  Map<byte[], String> readBulkLoadedFiles(String backupId) throws IOException {
317    Scan scan = BackupSystemTable.createScanForBulkLoadedFiles(backupId);
318    try (Table table = connection.getTable(bulkLoadTableName);
319      ResultScanner scanner = table.getScanner(scan)) {
320      Result res = null;
321      Map<byte[], String> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
322      while ((res = scanner.next()) != null) {
323        res.advance();
324        byte[] row = CellUtil.cloneRow(res.listCells().get(0));
325        for (Cell cell : res.listCells()) {
326          if (
327            CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0,
328              BackupSystemTable.PATH_COL.length) == 0
329          ) {
330            map.put(row, Bytes.toString(CellUtil.cloneValue(cell)));
331          }
332        }
333      }
334      return map;
335    }
336  }
337
338  /**
339   * Deletes backup status from backup system table table
340   * @param backupId backup id
341   * @throws IOException exception
342   */
343  public void deleteBackupInfo(String backupId) throws IOException {
344    if (LOG.isTraceEnabled()) {
345      LOG.trace("delete backup status in backup system table for " + backupId);
346    }
347    try (Table table = connection.getTable(tableName)) {
348      Delete del = createDeleteForBackupInfo(backupId);
349      table.delete(del);
350    }
351  }
352
353  /**
354   * Registers a bulk load.
355   * @param tableName     table name
356   * @param region        the region receiving hfile
357   * @param cfToHfilePath column family and associated hfiles
358   */
359  public void registerBulkLoad(TableName tableName, byte[] region,
360    Map<byte[], List<Path>> cfToHfilePath) throws IOException {
361    if (LOG.isDebugEnabled()) {
362      LOG.debug("Writing bulk load descriptor to backup {} with {} entries", tableName,
363        cfToHfilePath.size());
364    }
365    try (BufferedMutator bufferedMutator = connection.getBufferedMutator(bulkLoadTableName)) {
366      List<Put> puts = BackupSystemTable.createPutForBulkLoad(tableName, region, cfToHfilePath);
367      bufferedMutator.mutate(puts);
368      LOG.debug("Written {} rows for bulk load of table {}", puts.size(), tableName);
369    }
370  }
371
372  /**
373   * Removes entries from the table that tracks all bulk loaded hfiles.
374   * @param rows the row keys of the entries to be deleted
375   */
376  public void deleteBulkLoadedRows(List<byte[]> rows) throws IOException {
377    try (BufferedMutator bufferedMutator = connection.getBufferedMutator(bulkLoadTableName)) {
378      List<Delete> deletes = new ArrayList<>();
379      for (byte[] row : rows) {
380        Delete del = new Delete(row);
381        deletes.add(del);
382        LOG.debug("Deleting bulk load entry with key: {}", Bytes.toString(row));
383      }
384      bufferedMutator.mutate(deletes);
385      LOG.debug("Deleted {} bulk load entries.", rows.size());
386    }
387  }
388
389  /**
390   * Reads the rows from backup table recording bulk loaded hfiles
391   */
392  public List<BulkLoad> readBulkloadRows() throws IOException {
393    Scan scan = BackupSystemTable.createScanForOrigBulkLoadedFiles(null);
394    return processBulkLoadRowScan(scan, Long.MAX_VALUE);
395  }
396
397  /**
398   * Reads the rows from backup table recording bulk loaded hfiles
399   * @param tableList list of table names
400   */
401  public List<BulkLoad> readBulkloadRows(Collection<TableName> tableList) throws IOException {
402    return readBulkloadRows(tableList, Long.MAX_VALUE);
403  }
404
405  /**
406   * Reads the rows from backup table recording bulk loaded hfiles
407   * @param tableList    list of table names
408   * @param endTimestamp upper bound timestamp for bulkload entries retrieval
409   */
410  public List<BulkLoad> readBulkloadRows(Collection<TableName> tableList, long endTimestamp)
411    throws IOException {
412    List<BulkLoad> result = new ArrayList<>();
413    for (TableName table : tableList) {
414      Scan scan = BackupSystemTable.createScanForOrigBulkLoadedFiles(table);
415      result.addAll(processBulkLoadRowScan(scan, endTimestamp));
416    }
417    return result;
418  }
419
420  private List<BulkLoad> processBulkLoadRowScan(Scan scan, long endTimestamp) throws IOException {
421    List<BulkLoad> result = new ArrayList<>();
422    try (Table bulkLoadTable = connection.getTable(bulkLoadTableName);
423      ResultScanner scanner = bulkLoadTable.getScanner(scan)) {
424      Result res;
425      while ((res = scanner.next()) != null) {
426        res.advance();
427        TableName table = null;
428        String fam = null;
429        String path = null;
430        String region = null;
431        byte[] row = null;
432        long timestamp = 0L;
433        for (Cell cell : res.listCells()) {
434          row = CellUtil.cloneRow(cell);
435          timestamp = cell.getTimestamp();
436          String rowStr = Bytes.toString(row);
437          region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr);
438          if (
439            CellUtil.compareQualifiers(cell, BackupSystemTable.TBL_COL, 0,
440              BackupSystemTable.TBL_COL.length) == 0
441          ) {
442            table = TableName.valueOf(CellUtil.cloneValue(cell));
443          } else if (
444            CellUtil.compareQualifiers(cell, BackupSystemTable.FAM_COL, 0,
445              BackupSystemTable.FAM_COL.length) == 0
446          ) {
447            fam = Bytes.toString(CellUtil.cloneValue(cell));
448          } else if (
449            CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0,
450              BackupSystemTable.PATH_COL.length) == 0
451          ) {
452            path = Bytes.toString(CellUtil.cloneValue(cell));
453          }
454        }
455        LOG.debug("Found orig path {} for family {} of table {} and region {} with timestamp {}",
456          path, fam, table, region, timestamp);
457        if (timestamp <= endTimestamp) {
458          result.add(new BulkLoad(table, region, fam, path, row, timestamp));
459        }
460      }
461    }
462    return result;
463  }
464
465  /**
466   * Reads backup status object (instance of backup info) from backup system table table
467   * @param backupId backup id
468   * @return Current status of backup session or null
469   */
470  public BackupInfo readBackupInfo(String backupId) throws IOException {
471    if (LOG.isTraceEnabled()) {
472      LOG.trace("read backup status from backup system table for: " + backupId);
473    }
474
475    try (Table table = connection.getTable(tableName)) {
476      Get get = createGetForBackupInfo(backupId);
477      Result res = table.get(get);
478      if (res.isEmpty()) {
479        return null;
480      }
481      return resultToBackupInfo(res);
482    }
483  }
484
485  /**
486   * Exclusive operations are: create, delete, merge
487   * @throws IOException if a table operation fails or an active backup exclusive operation is
488   *                     already underway
489   */
490  public void startBackupExclusiveOperation() throws IOException {
491    LOG.debug("Start new backup exclusive operation");
492
493    try (Table table = connection.getTable(tableName)) {
494      Put put = createPutForStartBackupSession();
495      // First try to put if row does not exist
496      if (
497        !table.checkAndMutate(ACTIVE_SESSION_ROW, SESSIONS_FAMILY).qualifier(ACTIVE_SESSION_COL)
498          .ifNotExists().thenPut(put)
499      ) {
500        // Row exists, try to put if value == ACTIVE_SESSION_NO
501        if (
502          !table.checkAndMutate(ACTIVE_SESSION_ROW, SESSIONS_FAMILY).qualifier(ACTIVE_SESSION_COL)
503            .ifEquals(ACTIVE_SESSION_NO).thenPut(put)
504        ) {
505          throw new ExclusiveOperationException();
506        }
507      }
508    }
509  }
510
511  private Put createPutForStartBackupSession() {
512    Put put = new Put(ACTIVE_SESSION_ROW);
513    put.addColumn(SESSIONS_FAMILY, ACTIVE_SESSION_COL, ACTIVE_SESSION_YES);
514    return put;
515  }
516
517  public void finishBackupExclusiveOperation() throws IOException {
518    LOG.debug("Finish backup exclusive operation");
519
520    try (Table table = connection.getTable(tableName)) {
521      Put put = createPutForStopBackupSession();
522      if (
523        !table.checkAndMutate(ACTIVE_SESSION_ROW, SESSIONS_FAMILY).qualifier(ACTIVE_SESSION_COL)
524          .ifEquals(ACTIVE_SESSION_YES).thenPut(put)
525      ) {
526        throw new IOException("There is no active backup exclusive operation");
527      }
528    }
529  }
530
531  private Put createPutForStopBackupSession() {
532    Put put = new Put(ACTIVE_SESSION_ROW);
533    put.addColumn(SESSIONS_FAMILY, ACTIVE_SESSION_COL, ACTIVE_SESSION_NO);
534    return put;
535  }
536
537  /**
538   * Get the Region Servers log information after the last log roll from backup system table.
539   * @param backupRoot root directory path to backup
540   * @return RS log info
541   * @throws IOException exception
542   */
543  public HashMap<String, Long> readRegionServerLastLogRollResult(String backupRoot)
544    throws IOException {
545    LOG.trace("read region server last roll log result to backup system table");
546
547    Scan scan = createScanForReadRegionServerLastLogRollResult(backupRoot);
548
549    try (Table table = connection.getTable(tableName);
550      ResultScanner scanner = table.getScanner(scan)) {
551      Result res;
552      HashMap<String, Long> rsTimestampMap = new HashMap<>();
553      while ((res = scanner.next()) != null) {
554        res.advance();
555        Cell cell = res.current();
556        byte[] row = CellUtil.cloneRow(cell);
557        String server = getServerNameForReadRegionServerLastLogRollResult(row);
558        byte[] data = CellUtil.cloneValue(cell);
559        // Older versions wrote mixed-case keys, so a host can have a row under each spelling. Both
560        // rows map to the same key here. Roll results only increase, so the larger value is the
561        // newest.
562        rsTimestampMap.merge(BackupUtils.toHostKey(server), Bytes.toLong(data), Math::max);
563      }
564      return rsTimestampMap;
565    }
566  }
567
568  /**
569   * Writes Region Server last roll log result (timestamp) to backup system table table
570   * @param server     Region Server name
571   * @param ts         last log timestamp
572   * @param backupRoot root directory path to backup
573   * @throws IOException exception
574   */
575  public void writeRegionServerLastLogRollResult(String server, Long ts, String backupRoot)
576    throws IOException {
577    LOG.trace("write region server last roll log result to backup system table");
578
579    try (Table table = connection.getTable(tableName)) {
580      Put put = createPutForRegionServerLastLogRollResult(server, ts, backupRoot);
581      table.put(put);
582    }
583  }
584
585  /**
586   * Retrieve all table names that are part of any known completed backup
587   */
588  public Set<TableName> getTablesIncludedInBackups() throws IOException {
589    // Incremental backups have the same tables as the preceding full backups
590    List<BackupInfo> infos =
591      getBackupHistory(withState(BackupState.COMPLETE), withType(BackupType.FULL));
592    return infos.stream().flatMap(info -> info.getTableNames().stream())
593      .collect(Collectors.toSet());
594  }
595
596  /**
597   * Goes through all backup history corresponding to the provided root folder, and collects all
598   * backup info mentioning each of the provided tables.
599   * @param set        the tables for which to collect the {@code BackupInfo}
600   * @param backupRoot backup destination path to retrieve backup history for
601   * @return a map containing (a subset of) the provided {@code TableName}s, mapped to a list of at
602   *         least one {@code BackupInfo}
603   * @throws IOException if getting the backup history fails
604   */
605  public Map<TableName, List<BackupInfo>> getBackupHistoryForTableSet(Set<TableName> set,
606    String backupRoot) throws IOException {
607    List<BackupInfo> history = getBackupHistory(withRoot(backupRoot));
608    Map<TableName, List<BackupInfo>> tableHistoryMap = new HashMap<>();
609    for (BackupInfo info : history) {
610      List<TableName> tables = info.getTableNames();
611      for (TableName tableName : tables) {
612        if (set.contains(tableName)) {
613          List<BackupInfo> list =
614            tableHistoryMap.computeIfAbsent(tableName, k -> new ArrayList<>());
615          list.add(info);
616        }
617      }
618    }
619    return tableHistoryMap;
620  }
621
622  /**
623   * Get all backup information passing the given filters, ordered by descending backupId. I.e. from
624   * newest to oldest.
625   */
626  public List<BackupInfo> getBackupHistory(BackupInfo.Filter... toInclude) throws IOException {
627    return getBackupHistory(Order.NEW_TO_OLD, Integer.MAX_VALUE, toInclude);
628  }
629
630  /**
631   * Retrieves the first n entries of the sorted, filtered list of backup infos.
632   * @param order desired ordering of the results.
633   * @param n     number of entries to return
634   */
635  public List<BackupInfo> getBackupHistory(Order order, int n, BackupInfo.Filter... toInclude)
636    throws IOException {
637    Preconditions.checkArgument(n >= 0, "n should be >= 0");
638    LOG.trace("get backup infos from backup system table");
639
640    if (n == 0) {
641      return Collections.emptyList();
642    }
643
644    Predicate<BackupInfo> combinedPredicate = Stream.of(toInclude)
645      .map(filter -> (Predicate<BackupInfo>) filter).reduce(Predicate::and).orElse(x -> true);
646
647    Scan scan = createScanForBackupHistory(order);
648    List<BackupInfo> list = new ArrayList<>();
649
650    try (Table table = connection.getTable(tableName);
651      ResultScanner scanner = table.getScanner(scan)) {
652      Result res;
653      while ((res = scanner.next()) != null) {
654        res.advance();
655        BackupInfo context = cellToBackupInfo(res.current());
656        if (combinedPredicate.test(context)) {
657          list.add(context);
658          if (list.size() == n) {
659            break;
660          }
661        }
662      }
663      return list;
664    }
665  }
666
667  /**
668   * Write the current timestamps for each regionserver to backup system table after a successful
669   * full or incremental backup. For a region server that took part in the backup's log roll, the
670   * saved timestamp is the result of that roll. For a region server that did not (offline or
671   * decommissioned), it is the creation time of the newest of its log files that was backed up.
672   * @param tables        tables
673   * @param newTimestamps timestamps
674   * @param backupRoot    root directory path to backup
675   * @throws IOException exception
676   */
677  public void writeRegionServerLogTimestamp(Set<TableName> tables, Map<String, Long> newTimestamps,
678    String backupRoot) throws IOException {
679    if (LOG.isTraceEnabled()) {
680      LOG.trace("write RS log time stamps to backup system table for tables ["
681        + StringUtils.join(tables, ",") + "]");
682    }
683    List<Put> puts = new ArrayList<>();
684    for (TableName table : tables) {
685      byte[] smapData = toTableServerTimestampProto(table, newTimestamps).toByteArray();
686      Put put = createPutForWriteRegionServerLogTimestamp(table, smapData, backupRoot);
687      puts.add(put);
688    }
689    try (BufferedMutator bufferedMutator = connection.getBufferedMutator(tableName)) {
690      bufferedMutator.mutate(puts);
691    }
692  }
693
694  /**
695   * Read the timestamp for each region server log after the last successful backup. Each table has
696   * its own set of the timestamps. The info is stored for each table as a concatenated string of
697   * rs->timestapmp
698   * @param backupRoot root directory path to backup
699   * @return the timestamp for each region server. key: tableName value:
700   *         RegionServer,PreviousTimeStamp
701   * @throws IOException exception
702   */
703  public Map<TableName, Map<String, Long>> readLogTimestampMap(String backupRoot)
704    throws IOException {
705    if (LOG.isTraceEnabled()) {
706      LOG.trace("read RS log ts from backup system table for root=" + backupRoot);
707    }
708
709    Map<TableName, Map<String, Long>> tableTimestampMap = new HashMap<>();
710
711    Scan scan = createScanForReadLogTimestampMap(backupRoot);
712    try (Table table = connection.getTable(tableName);
713      ResultScanner scanner = table.getScanner(scan)) {
714      Result res;
715      while ((res = scanner.next()) != null) {
716        res.advance();
717        Cell cell = res.current();
718        byte[] row = CellUtil.cloneRow(cell);
719        String tabName = getTableNameForReadLogTimestampMap(row);
720        TableName tn = TableName.valueOf(tabName);
721        byte[] data = CellUtil.cloneValue(cell);
722        if (data == null) {
723          throw new IOException("Data of last backup data from backup system table "
724            + "is empty. Create a backup first.");
725        }
726        if (data != null && data.length > 0) {
727          HashMap<String, Long> lastBackup =
728            fromTableServerTimestampProto(BackupProtos.TableServerTimestamp.parseFrom(data));
729          tableTimestampMap.put(tn, lastBackup);
730        }
731      }
732      return tableTimestampMap;
733    }
734  }
735
736  private BackupProtos.TableServerTimestamp toTableServerTimestampProto(TableName table,
737    Map<String, Long> map) {
738    BackupProtos.TableServerTimestamp.Builder tstBuilder =
739      BackupProtos.TableServerTimestamp.newBuilder();
740    tstBuilder
741      .setTableName(org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toProtoTableName(table));
742
743    for (Entry<String, Long> entry : map.entrySet()) {
744      BackupProtos.ServerTimestamp.Builder builder = BackupProtos.ServerTimestamp.newBuilder();
745      HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
746      ServerName sn = ServerName.parseServerName(entry.getKey());
747      snBuilder.setHostName(sn.getHostname());
748      snBuilder.setPort(sn.getPort());
749      builder.setServerName(snBuilder.build());
750      builder.setTimestamp(entry.getValue());
751      tstBuilder.addServerTimestamp(builder.build());
752    }
753
754    return tstBuilder.build();
755  }
756
757  private HashMap<String, Long>
758    fromTableServerTimestampProto(BackupProtos.TableServerTimestamp proto) {
759
760    HashMap<String, Long> map = new HashMap<>();
761    List<BackupProtos.ServerTimestamp> list = proto.getServerTimestampList();
762    for (BackupProtos.ServerTimestamp st : list) {
763      ServerName sn =
764        org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toServerName(st.getServerName());
765      map.put(BackupUtils.toHostKey(sn.getHostname() + ":" + sn.getPort()), st.getTimestamp());
766    }
767    return map;
768  }
769
770  /**
771   * Return the current tables covered by incremental backup.
772   * @param backupRoot root directory path to backup
773   * @return set of tableNames
774   * @throws IOException exception
775   */
776  public Set<TableName> getIncrementalBackupTableSet(String backupRoot) throws IOException {
777    LOG.trace("get incremental backup table set from backup system table");
778
779    TreeSet<TableName> set = new TreeSet<>();
780
781    try (Table table = connection.getTable(tableName)) {
782      Get get = createGetForIncrBackupTableSet(backupRoot);
783      Result res = table.get(get);
784      if (res.isEmpty()) {
785        return set;
786      }
787      List<Cell> cells = res.listCells();
788      for (Cell cell : cells) {
789        // qualifier = table name - we use table names as qualifiers
790        set.add(TableName.valueOf(CellUtil.cloneQualifier(cell)));
791      }
792      return set;
793    }
794  }
795
796  /**
797   * Retrieves the current set of tables covered by continuous backup along with the timestamp
798   * indicating when continuous backup started for each table.
799   * @return a map where the key is the table name and the value is the timestamp representing the
800   *         start time of continuous backup for that table.
801   * @throws IOException if an I/O error occurs while accessing the backup system table.
802   */
803  public Map<TableName, Long> getContinuousBackupTableSet() throws IOException {
804    LOG.trace("Retrieving continuous backup table set from the backup system table.");
805    Map<TableName, Long> tableMap = new TreeMap<>();
806
807    try (Table systemTable = connection.getTable(tableName)) {
808      Get getOperation = createGetForContinuousBackupTableSet();
809      Result result = systemTable.get(getOperation);
810
811      if (result.isEmpty()) {
812        return tableMap;
813      }
814
815      // Extract table names and timestamps from the result cells
816      List<Cell> cells = result.listCells();
817      for (Cell cell : cells) {
818        TableName tableName = TableName.valueOf(CellUtil.cloneQualifier(cell));
819        long timestamp = Bytes.toLong(CellUtil.cloneValue(cell));
820        tableMap.put(tableName, timestamp);
821      }
822    }
823
824    return tableMap;
825  }
826
827  /**
828   * Add tables to global incremental backup set
829   * @param tables     set of tables
830   * @param backupRoot root directory path to backup
831   * @throws IOException exception
832   */
833  public void addIncrementalBackupTableSet(Set<TableName> tables, String backupRoot)
834    throws IOException {
835    if (LOG.isTraceEnabled()) {
836      LOG.trace("Add incremental backup table set to backup system table. ROOT=" + backupRoot
837        + " tables [" + StringUtils.join(tables, " ") + "]");
838    }
839    if (LOG.isDebugEnabled()) {
840      tables.forEach(table -> LOG.debug(Objects.toString(table)));
841    }
842    try (Table table = connection.getTable(tableName)) {
843      Put put = createPutForIncrBackupTableSet(tables, backupRoot);
844      table.put(put);
845    }
846  }
847
848  /**
849   * Add tables to the global continuous backup set. Only updates tables that are not already in the
850   * continuous backup set.
851   * @param tables         set of tables to add
852   * @param startTimestamp timestamp indicating when continuous backup started
853   * @throws IOException if an error occurs while updating the backup system table
854   */
855  public void addContinuousBackupTableSet(Set<TableName> tables, long startTimestamp)
856    throws IOException {
857    if (LOG.isTraceEnabled()) {
858      LOG.trace("Add continuous backup table set to backup system table. tables ["
859        + StringUtils.join(tables, " ") + "]");
860    }
861    if (LOG.isDebugEnabled()) {
862      tables.forEach(table -> LOG.debug(Objects.toString(table)));
863    }
864
865    // Get existing continuous backup tables
866    Map<TableName, Long> existingTables = getContinuousBackupTableSet();
867
868    try (Table table = connection.getTable(tableName)) {
869      Put put = createPutForContinuousBackupTableSet(tables, existingTables, startTimestamp);
870      if (!put.isEmpty()) {
871        table.put(put);
872      }
873    }
874  }
875
876  /**
877   * Updates the system table with the new start timestamps for continuous backup tables.
878   * @param tablesToUpdate    The set of tables that need their start timestamps updated.
879   * @param newStartTimestamp The new start timestamp to be set.
880   */
881  public void updateContinuousBackupTableSet(Set<TableName> tablesToUpdate, long newStartTimestamp)
882    throws IOException {
883    if (tablesToUpdate == null || tablesToUpdate.isEmpty()) {
884      LOG.warn("No tables provided for updating start timestamps.");
885      return;
886    }
887
888    try (Table table = connection.getTable(tableName)) {
889      Put put = new Put(rowkey(CONTINUOUS_BACKUP_SET));
890
891      for (TableName tableName : tablesToUpdate) {
892        put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(tableName.getNameAsString()),
893          Bytes.toBytes(newStartTimestamp));
894      }
895
896      table.put(put);
897      LOG.info("Successfully updated start timestamps for {} tables in the backup system table.",
898        tablesToUpdate.size());
899    }
900  }
901
902  /**
903   * Removes tables from the global continuous backup set. Only removes entries that currently exist
904   * in the backup system table.
905   * @param tables set of tables to remove
906   * @throws IOException if an error occurs while updating the backup system table
907   */
908  public void removeContinuousBackupTableSet(Set<TableName> tables) throws IOException {
909    if (LOG.isTraceEnabled()) {
910      LOG.trace("Remove continuous backup table set from backup system table. tables ["
911        + StringUtils.join(tables, " ") + "]");
912    }
913    if (LOG.isDebugEnabled()) {
914      tables.forEach(table -> LOG.debug("Removing: " + table));
915    }
916
917    Map<TableName, Long> existingTables = getContinuousBackupTableSet();
918    Set<TableName> toRemove =
919      tables.stream().filter(existingTables::containsKey).collect(Collectors.toSet());
920
921    if (toRemove.isEmpty()) {
922      LOG.debug("No matching tables found to remove from continuous backup set.");
923      return;
924    }
925
926    try (Table table = connection.getTable(tableName)) {
927      Delete delete = createDeleteForContinuousBackupTableSet(toRemove);
928      table.delete(delete);
929    }
930  }
931
932  /**
933   * Updates the latest replicated WAL timestamp for a region server in the backup system table.
934   * This is used to track the replication checkpoint for continuous backup and PITR (Point-in-Time
935   * Restore).
936   * @param serverName the server for which the latest WAL timestamp is being recorded
937   * @param timestamp  the timestamp (in milliseconds) of the last WAL entry replicated
938   * @throws IOException if an error occurs while writing to the backup system table
939   */
940  public void updateBackupCheckpointTimestamp(ServerName serverName, long timestamp)
941    throws IOException {
942
943    HBaseProtos.ServerName.Builder serverProto =
944      HBaseProtos.ServerName.newBuilder().setHostName(serverName.getHostname())
945        .setPort(serverName.getPort()).setStartCode(serverName.getStartCode());
946
947    try (Table table = connection.getTable(tableName)) {
948      Put put = createPutForBackupCheckpoint(serverProto.build().toByteArray(), timestamp);
949      if (!put.isEmpty()) {
950        table.put(put);
951      }
952    }
953  }
954
955  /**
956   * Retrieves the latest replicated WAL timestamps for all region servers from the backup system
957   * table. This is used to track the replication checkpoint state for continuous backup and PITR
958   * (Point-in-Time Restore).
959   * @return a map where the key is {@link ServerName} and the value is the latest replicated WAL
960   *         timestamp in milliseconds
961   * @throws IOException if an error occurs while reading from the backup system table
962   */
963  public Map<ServerName, Long> getBackupCheckpointTimestamps() throws IOException {
964    LOG.trace("Fetching latest backup checkpoint timestamps for all region servers.");
965
966    Map<ServerName, Long> checkpointMap = new HashMap<>();
967
968    byte[] rowKey = rowkey(CONTINUOUS_BACKUP_REPLICATION_TIMESTAMP_ROW);
969    Get get = new Get(rowKey);
970    get.addFamily(BackupSystemTable.META_FAMILY);
971
972    try (Table table = connection.getTable(tableName)) {
973      Result result = table.get(get);
974
975      if (result.isEmpty()) {
976        LOG.debug("No checkpoint timestamps found in backup system table.");
977        return checkpointMap;
978      }
979
980      List<Cell> cells = result.listCells();
981      for (Cell cell : cells) {
982        try {
983          HBaseProtos.ServerName protoServer =
984            HBaseProtos.ServerName.parseFrom(CellUtil.cloneQualifier(cell));
985          ServerName serverName = ServerName.valueOf(protoServer.getHostName(),
986            protoServer.getPort(), protoServer.getStartCode());
987
988          long timestamp = Bytes.toLong(CellUtil.cloneValue(cell));
989          checkpointMap.put(serverName, timestamp);
990        } catch (IllegalArgumentException e) {
991          LOG.warn("Failed to parse server name or timestamp from cell: {}", cell, e);
992        }
993      }
994    }
995
996    return checkpointMap;
997  }
998
999  /**
1000   * Constructs a {@link Put} operation to update the last replicated WAL timestamp for a given
1001   * server in the backup system table.
1002   * @param serverNameBytes the serialized server name as bytes
1003   * @param timestamp       the WAL entry timestamp to store
1004   * @return a {@link Put} object ready to be written to the system table
1005   */
1006  private Put createPutForBackupCheckpoint(byte[] serverNameBytes, long timestamp) {
1007    Put put = new Put(rowkey(CONTINUOUS_BACKUP_REPLICATION_TIMESTAMP_ROW));
1008    put.addColumn(BackupSystemTable.META_FAMILY, serverNameBytes, Bytes.toBytes(timestamp));
1009    return put;
1010  }
1011
1012  /**
1013   * Deletes incremental backup set for a backup destination
1014   * @param backupRoot backup root
1015   */
1016  public void deleteIncrementalBackupTableSet(String backupRoot) throws IOException {
1017    if (LOG.isTraceEnabled()) {
1018      LOG.trace("Delete incremental backup table set to backup system table. ROOT=" + backupRoot);
1019    }
1020    try (Table table = connection.getTable(tableName)) {
1021      Delete delete = createDeleteForIncrBackupTableSet(backupRoot);
1022      table.delete(delete);
1023    }
1024  }
1025
1026  /**
1027   * Checks if we have at least one backup session in backup system table This API is used by
1028   * BackupLogCleaner
1029   * @return true, if at least one session exists in backup system table
1030   * @throws IOException exception
1031   */
1032  public boolean hasBackupSessions() throws IOException {
1033    LOG.trace("Has backup sessions from backup system table");
1034
1035    Scan scan = createScanForBackupHistory(Order.OLD_TO_NEW);
1036    scan.setCaching(1);
1037    try (Table table = connection.getTable(tableName);
1038      ResultScanner scanner = table.getScanner(scan)) {
1039      return scanner.next() != null;
1040    }
1041  }
1042
1043  /**
1044   * BACKUP SETS
1045   */
1046
1047  /**
1048   * Get backup set list
1049   * @return backup set list
1050   * @throws IOException if a table or scanner operation fails
1051   */
1052  public List<String> listBackupSets() throws IOException {
1053    LOG.trace("Backup set list");
1054
1055    List<String> list = new ArrayList<>();
1056    try (Table table = connection.getTable(tableName)) {
1057      Scan scan = createScanForBackupSetList();
1058      scan.readVersions(1);
1059      try (ResultScanner scanner = table.getScanner(scan)) {
1060        Result res;
1061        while ((res = scanner.next()) != null) {
1062          res.advance();
1063          list.add(cellKeyToBackupSetName(res.current()));
1064        }
1065        return list;
1066      }
1067    }
1068  }
1069
1070  /**
1071   * Get backup set description (list of tables)
1072   * @param name set's name
1073   * @return list of tables in a backup set
1074   * @throws IOException if a table operation fails
1075   */
1076  public List<TableName> describeBackupSet(String name) throws IOException {
1077    if (LOG.isTraceEnabled()) {
1078      LOG.trace(" Backup set describe: " + name);
1079    }
1080    try (Table table = connection.getTable(tableName)) {
1081      Get get = createGetForBackupSet(name);
1082      Result res = table.get(get);
1083      if (res.isEmpty()) {
1084        return null;
1085      }
1086      res.advance();
1087      String[] tables = cellValueToBackupSet(res.current());
1088      return Arrays.asList(tables).stream().map(item -> TableName.valueOf(item))
1089        .collect(Collectors.toList());
1090    }
1091  }
1092
1093  /**
1094   * Add backup set (list of tables)
1095   * @param name      set name
1096   * @param newTables list of tables, comma-separated
1097   * @throws IOException if a table operation fails
1098   */
1099  public void addToBackupSet(String name, String[] newTables) throws IOException {
1100    if (LOG.isTraceEnabled()) {
1101      LOG.trace("Backup set add: " + name + " tables [" + StringUtils.join(newTables, " ") + "]");
1102    }
1103    String[] union = null;
1104    try (Table table = connection.getTable(tableName)) {
1105      Get get = createGetForBackupSet(name);
1106      Result res = table.get(get);
1107      if (res.isEmpty()) {
1108        union = newTables;
1109      } else {
1110        res.advance();
1111        String[] tables = cellValueToBackupSet(res.current());
1112        union = merge(tables, newTables);
1113      }
1114      Put put = createPutForBackupSet(name, union);
1115      table.put(put);
1116    }
1117  }
1118
1119  /**
1120   * Remove tables from backup set (list of tables)
1121   * @param name     set name
1122   * @param toRemove list of tables
1123   * @throws IOException if a table operation or deleting the backup set fails
1124   */
1125  public void removeFromBackupSet(String name, String[] toRemove) throws IOException {
1126    if (LOG.isTraceEnabled()) {
1127      LOG.trace(
1128        " Backup set remove from : " + name + " tables [" + StringUtils.join(toRemove, " ") + "]");
1129    }
1130    String[] disjoint;
1131    String[] tables;
1132    try (Table table = connection.getTable(tableName)) {
1133      Get get = createGetForBackupSet(name);
1134      Result res = table.get(get);
1135      if (res.isEmpty()) {
1136        LOG.warn("Backup set '" + name + "' not found.");
1137        return;
1138      } else {
1139        res.advance();
1140        tables = cellValueToBackupSet(res.current());
1141        disjoint = disjoin(tables, toRemove);
1142      }
1143      if (disjoint.length > 0 && disjoint.length != tables.length) {
1144        Put put = createPutForBackupSet(name, disjoint);
1145        table.put(put);
1146      } else if (disjoint.length == tables.length) {
1147        LOG.warn("Backup set '" + name + "' does not contain tables ["
1148          + StringUtils.join(toRemove, " ") + "]");
1149      } else { // disjoint.length == 0 and tables.length >0
1150        // Delete backup set
1151        LOG.info("Backup set '" + name + "' is empty. Deleting.");
1152        deleteBackupSet(name);
1153      }
1154    }
1155  }
1156
1157  private String[] merge(String[] existingTables, String[] newTables) {
1158    Set<String> tables = new HashSet<>(Arrays.asList(existingTables));
1159    tables.addAll(Arrays.asList(newTables));
1160    return tables.toArray(new String[0]);
1161  }
1162
1163  private String[] disjoin(String[] existingTables, String[] toRemove) {
1164    Set<String> tables = new HashSet<>(Arrays.asList(existingTables));
1165    Arrays.asList(toRemove).forEach(table -> tables.remove(table));
1166    return tables.toArray(new String[0]);
1167  }
1168
1169  /**
1170   * Delete backup set
1171   * @param name set's name
1172   * @throws IOException if getting or deleting the table fails
1173   */
1174  public void deleteBackupSet(String name) throws IOException {
1175    if (LOG.isTraceEnabled()) {
1176      LOG.trace(" Backup set delete: " + name);
1177    }
1178    try (Table table = connection.getTable(tableName)) {
1179      Delete del = createDeleteForBackupSet(name);
1180      table.delete(del);
1181    }
1182  }
1183
1184  /**
1185   * Get backup system table descriptor
1186   * @return table's descriptor
1187   */
1188  public static TableDescriptor getSystemTableDescriptor(Configuration conf) {
1189    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableName(conf));
1190
1191    ColumnFamilyDescriptorBuilder colBuilder =
1192      ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY);
1193
1194    colBuilder.setMaxVersions(1);
1195    Configuration config = HBaseConfiguration.create();
1196    int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY,
1197      BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT);
1198    colBuilder.setTimeToLive(ttl);
1199
1200    ColumnFamilyDescriptor colSessionsDesc = colBuilder.build();
1201    builder.setColumnFamily(colSessionsDesc);
1202
1203    colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY);
1204    colBuilder.setTimeToLive(ttl);
1205    builder.setColumnFamily(colBuilder.build());
1206    return builder.build();
1207  }
1208
1209  public static TableName getTableName(Configuration conf) {
1210    String name = conf.get(BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_KEY,
1211      BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_DEFAULT);
1212    return TableName.valueOf(name);
1213  }
1214
1215  public static String getTableNameAsString(Configuration conf) {
1216    return getTableName(conf).getNameAsString();
1217  }
1218
1219  public static String getSnapshotName(Configuration conf) {
1220    return "snapshot_" + getTableNameAsString(conf).replace(":", "_");
1221  }
1222
1223  /**
1224   * Get backup system table descriptor
1225   * @return table's descriptor
1226   */
1227  public static TableDescriptor getSystemTableForBulkLoadedDataDescriptor(Configuration conf) {
1228    TableDescriptorBuilder builder =
1229      TableDescriptorBuilder.newBuilder(getTableNameForBulkLoadedData(conf));
1230
1231    ColumnFamilyDescriptorBuilder colBuilder =
1232      ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY);
1233    colBuilder.setMaxVersions(1);
1234    Configuration config = HBaseConfiguration.create();
1235    int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY,
1236      BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT);
1237    colBuilder.setTimeToLive(ttl);
1238    ColumnFamilyDescriptor colSessionsDesc = colBuilder.build();
1239    builder.setColumnFamily(colSessionsDesc);
1240    colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY);
1241    colBuilder.setTimeToLive(ttl);
1242    builder.setColumnFamily(colBuilder.build());
1243    return builder.build();
1244  }
1245
1246  public static TableName getTableNameForBulkLoadedData(Configuration conf) {
1247    String name = conf.get(BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_KEY,
1248      BackupRestoreConstants.BACKUP_SYSTEM_TABLE_NAME_DEFAULT) + "_bulk";
1249    return TableName.valueOf(name);
1250  }
1251
1252  /**
1253   * Creates Put operation for a given backup info object
1254   * @param context backup info
1255   * @return put operation
1256   * @throws IOException exception
1257   */
1258  private Put createPutForBackupInfo(BackupInfo context) throws IOException {
1259    Put put = new Put(rowkey(BACKUP_INFO_PREFIX, context.getBackupId()));
1260    put.addColumn(BackupSystemTable.SESSIONS_FAMILY, Bytes.toBytes("context"),
1261      context.toByteArray());
1262    return put;
1263  }
1264
1265  /**
1266   * Creates Get operation for a given backup id
1267   * @param backupId backup's ID
1268   * @return get operation
1269   * @throws IOException exception
1270   */
1271  private Get createGetForBackupInfo(String backupId) throws IOException {
1272    Get get = new Get(rowkey(BACKUP_INFO_PREFIX, backupId));
1273    get.addFamily(BackupSystemTable.SESSIONS_FAMILY);
1274    get.readVersions(1);
1275    return get;
1276  }
1277
1278  /**
1279   * Creates Delete operation for a given backup id
1280   * @param backupId backup's ID
1281   * @return delete operation
1282   */
1283  private Delete createDeleteForBackupInfo(String backupId) {
1284    Delete del = new Delete(rowkey(BACKUP_INFO_PREFIX, backupId));
1285    del.addFamily(BackupSystemTable.SESSIONS_FAMILY);
1286    return del;
1287  }
1288
1289  /**
1290   * Converts Result to BackupInfo
1291   * @param res HBase result
1292   * @return backup info instance
1293   * @throws IOException exception
1294   */
1295  private BackupInfo resultToBackupInfo(Result res) throws IOException {
1296    res.advance();
1297    Cell cell = res.current();
1298    return cellToBackupInfo(cell);
1299  }
1300
1301  /**
1302   * Creates Get to retrieve incremental backup table set from backup system table
1303   * @return get operation
1304   * @throws IOException exception
1305   */
1306  private Get createGetForIncrBackupTableSet(String backupRoot) throws IOException {
1307    Get get = new Get(rowkey(INCR_BACKUP_SET, backupRoot));
1308    get.addFamily(BackupSystemTable.META_FAMILY);
1309    get.readVersions(1);
1310    return get;
1311  }
1312
1313  /**
1314   * Creates a Get operation to retrieve the continuous backup table set from the backup system
1315   * table.
1316   * @return a Get operation for retrieving the table set
1317   */
1318  private Get createGetForContinuousBackupTableSet() throws IOException {
1319    Get get = new Get(rowkey(CONTINUOUS_BACKUP_SET));
1320    get.addFamily(BackupSystemTable.META_FAMILY);
1321    get.readVersions(1);
1322    return get;
1323  }
1324
1325  /**
1326   * Creates Put to store incremental backup table set
1327   * @param tables tables
1328   * @return put operation
1329   */
1330  private Put createPutForIncrBackupTableSet(Set<TableName> tables, String backupRoot) {
1331    Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot));
1332    for (TableName table : tables) {
1333      put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()),
1334        EMPTY_VALUE);
1335    }
1336    return put;
1337  }
1338
1339  /**
1340   * Creates a Put operation to store the continuous backup table set. Only includes tables that are
1341   * not already in the set.
1342   * @param tables         tables to add
1343   * @param existingTables tables that already have continuous backup enabled
1344   * @param startTimestamp timestamp indicating when continuous backup started
1345   * @return put operation
1346   */
1347  private Put createPutForContinuousBackupTableSet(Set<TableName> tables,
1348    Map<TableName, Long> existingTables, long startTimestamp) {
1349    Put put = new Put(rowkey(CONTINUOUS_BACKUP_SET));
1350
1351    for (TableName table : tables) {
1352      if (!existingTables.containsKey(table)) {
1353        put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()),
1354          Bytes.toBytes(startTimestamp));
1355      }
1356    }
1357
1358    return put;
1359  }
1360
1361  /**
1362   * Creates Delete for incremental backup table set
1363   * @param backupRoot backup root
1364   * @return delete operation
1365   */
1366  private Delete createDeleteForIncrBackupTableSet(String backupRoot) {
1367    Delete delete = new Delete(rowkey(INCR_BACKUP_SET, backupRoot));
1368    delete.addFamily(BackupSystemTable.META_FAMILY);
1369    return delete;
1370  }
1371
1372  /**
1373   * Creates Delete for continuous backup table set
1374   * @param tables tables to remove
1375   * @return delete operation
1376   */
1377  private Delete createDeleteForContinuousBackupTableSet(Set<TableName> tables) {
1378    Delete delete = new Delete(rowkey(CONTINUOUS_BACKUP_SET));
1379    for (TableName tableName : tables) {
1380      delete.addColumn(META_FAMILY, Bytes.toBytes(tableName.getNameAsString()));
1381    }
1382    return delete;
1383  }
1384
1385  /**
1386   * Creates Scan operation to load backup history
1387   * @param order order of the scan results
1388   * @return scan operation
1389   */
1390  private Scan createScanForBackupHistory(Order order) {
1391    Scan scan = new Scan();
1392    byte[] startRow = Bytes.toBytes(BACKUP_INFO_PREFIX);
1393    if (order == Order.NEW_TO_OLD) {
1394      byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
1395      stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
1396      scan.setReversed(true);
1397      scan.withStartRow(stopRow, false);
1398      scan.withStopRow(startRow);
1399    } else if (order == Order.OLD_TO_NEW) {
1400      scan.setStartStopRowForPrefixScan(startRow);
1401    } else {
1402      throw new IllegalArgumentException("Unsupported order: " + order);
1403    }
1404    scan.addFamily(BackupSystemTable.SESSIONS_FAMILY);
1405    scan.readVersions(1);
1406    return scan;
1407  }
1408
1409  /**
1410   * Converts cell to backup info instance.
1411   * @param current current cell
1412   * @return backup backup info instance
1413   * @throws IOException exception
1414   */
1415  private BackupInfo cellToBackupInfo(Cell current) throws IOException {
1416    byte[] data = CellUtil.cloneValue(current);
1417    return BackupInfo.fromByteArray(data);
1418  }
1419
1420  /**
1421   * Creates Put to write RS last roll log timestamp map
1422   * @param table table
1423   * @param smap  map, containing RS:ts
1424   * @return put operation
1425   */
1426  private Put createPutForWriteRegionServerLogTimestamp(TableName table, byte[] smap,
1427    String backupRoot) {
1428    Put put = new Put(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL, table.getNameAsString()));
1429    put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("log-roll-map"), smap);
1430    return put;
1431  }
1432
1433  /**
1434   * Creates Scan to load table-> { RS -> ts} map of maps
1435   * @return scan operation
1436   */
1437  private Scan createScanForReadLogTimestampMap(String backupRoot) {
1438    Scan scan = new Scan();
1439    scan.setStartStopRowForPrefixScan(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL));
1440    scan.addFamily(BackupSystemTable.META_FAMILY);
1441
1442    return scan;
1443  }
1444
1445  /**
1446   * Get table name from rowkey
1447   * @param cloneRow rowkey
1448   * @return table name
1449   */
1450  private String getTableNameForReadLogTimestampMap(byte[] cloneRow) {
1451    String s = Bytes.toString(cloneRow);
1452    int index = s.lastIndexOf(NULL);
1453    return s.substring(index + 1);
1454  }
1455
1456  /**
1457   * Creates Put to store RS last log result
1458   * @param server    server name
1459   * @param timestamp log roll result (timestamp)
1460   * @return put operation
1461   */
1462  private Put createPutForRegionServerLastLogRollResult(String server, Long timestamp,
1463    String backupRoot) {
1464    Put put = new Put(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL, server));
1465    put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("rs-log-ts"),
1466      Bytes.toBytes(timestamp));
1467    return put;
1468  }
1469
1470  /**
1471   * Creates Scan operation to load last RS log roll results
1472   * @return scan operation
1473   */
1474  private Scan createScanForReadRegionServerLastLogRollResult(String backupRoot) {
1475    Scan scan = new Scan();
1476    scan.setStartStopRowForPrefixScan(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL));
1477    scan.addFamily(BackupSystemTable.META_FAMILY);
1478    scan.readVersions(1);
1479
1480    return scan;
1481  }
1482
1483  /**
1484   * Get server's name from rowkey
1485   * @param row rowkey
1486   * @return server's name
1487   */
1488  private String getServerNameForReadRegionServerLastLogRollResult(byte[] row) {
1489    String s = Bytes.toString(row);
1490    int index = s.lastIndexOf(NULL);
1491    return s.substring(index + 1);
1492  }
1493
1494  /**
1495   * Creates Put's for bulk loads.
1496   */
1497  private static List<Put> createPutForBulkLoad(TableName table, byte[] region,
1498    Map<byte[], List<Path>> columnFamilyToHFilePaths) {
1499    List<Put> puts = new ArrayList<>();
1500    for (Map.Entry<byte[], List<Path>> entry : columnFamilyToHFilePaths.entrySet()) {
1501      for (Path path : entry.getValue()) {
1502        String file = path.toString();
1503        int lastSlash = file.lastIndexOf("/");
1504        String filename = file.substring(lastSlash + 1);
1505        Put put = new Put(rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM,
1506          Bytes.toString(region), BLK_LD_DELIM, filename));
1507        put.addColumn(BackupSystemTable.META_FAMILY, TBL_COL, table.getName());
1508        put.addColumn(BackupSystemTable.META_FAMILY, FAM_COL, entry.getKey());
1509        put.addColumn(BackupSystemTable.META_FAMILY, PATH_COL, Bytes.toBytes(file));
1510        puts.add(put);
1511        LOG.debug("Done writing bulk path {} for {} {}", file, table, Bytes.toString(region));
1512      }
1513    }
1514    return puts;
1515  }
1516
1517  public static void snapshot(Connection conn) throws IOException {
1518    try (Admin admin = conn.getAdmin()) {
1519      Configuration conf = conn.getConfiguration();
1520      admin.snapshot(BackupSystemTable.getSnapshotName(conf), BackupSystemTable.getTableName(conf));
1521    }
1522  }
1523
1524  public static void restoreFromSnapshot(Connection conn) throws IOException {
1525    Configuration conf = conn.getConfiguration();
1526    LOG.debug("Restoring " + BackupSystemTable.getTableNameAsString(conf) + " from snapshot");
1527    try (Admin admin = conn.getAdmin()) {
1528      String snapshotName = BackupSystemTable.getSnapshotName(conf);
1529      if (snapshotExists(admin, snapshotName)) {
1530        admin.restoreBackupSystemTable(snapshotName);
1531        LOG.debug("Done restoring backup system table");
1532      } else {
1533        // Snapshot does not exists, i.e completeBackup failed after
1534        // deleting backup system table snapshot
1535        // In this case we log WARN and proceed
1536        LOG.warn(
1537          "Could not restore backup system table. Snapshot " + snapshotName + " does not exists.");
1538      }
1539    }
1540  }
1541
1542  private static boolean snapshotExists(Admin admin, String snapshotName) throws IOException {
1543    List<SnapshotDescription> list = admin.listSnapshots();
1544    for (SnapshotDescription desc : list) {
1545      if (desc.getName().equals(snapshotName)) {
1546        return true;
1547      }
1548    }
1549    return false;
1550  }
1551
1552  public static boolean snapshotExists(Connection conn) throws IOException {
1553    return snapshotExists(conn.getAdmin(), getSnapshotName(conn.getConfiguration()));
1554  }
1555
1556  public static void deleteSnapshot(Connection conn) throws IOException {
1557    Configuration conf = conn.getConfiguration();
1558    LOG.debug("Deleting " + BackupSystemTable.getSnapshotName(conf) + " from the system");
1559    try (Admin admin = conn.getAdmin()) {
1560      String snapshotName = BackupSystemTable.getSnapshotName(conf);
1561      if (snapshotExists(admin, snapshotName)) {
1562        admin.deleteSnapshot(snapshotName);
1563        LOG.debug("Done deleting backup system table snapshot");
1564      } else {
1565        LOG.error("Snapshot " + snapshotName + " does not exists");
1566      }
1567    }
1568  }
1569
1570  private Put createPutForDeleteOperation(String[] backupIdList) {
1571    byte[] value = Bytes.toBytes(StringUtils.join(backupIdList, ","));
1572    Put put = new Put(DELETE_OP_ROW);
1573    put.addColumn(META_FAMILY, FAM_COL, value);
1574    return put;
1575  }
1576
1577  private Delete createDeleteForBackupDeleteOperation() {
1578    Delete delete = new Delete(DELETE_OP_ROW);
1579    delete.addFamily(META_FAMILY);
1580    return delete;
1581  }
1582
1583  private Get createGetForDeleteOperation() {
1584    Get get = new Get(DELETE_OP_ROW);
1585    get.addFamily(META_FAMILY);
1586    return get;
1587  }
1588
1589  public void startDeleteOperation(String[] backupIdList) throws IOException {
1590    if (LOG.isTraceEnabled()) {
1591      LOG.trace("Start delete operation for backups: " + StringUtils.join(backupIdList));
1592    }
1593    Put put = createPutForDeleteOperation(backupIdList);
1594    try (Table table = connection.getTable(tableName)) {
1595      table.put(put);
1596    }
1597  }
1598
1599  public void finishDeleteOperation() throws IOException {
1600    LOG.trace("Finsih delete operation for backup ids");
1601
1602    Delete delete = createDeleteForBackupDeleteOperation();
1603    try (Table table = connection.getTable(tableName)) {
1604      table.delete(delete);
1605    }
1606  }
1607
1608  public String[] getListOfBackupIdsFromDeleteOperation() throws IOException {
1609    LOG.trace("Get delete operation for backup ids");
1610
1611    Get get = createGetForDeleteOperation();
1612    try (Table table = connection.getTable(tableName)) {
1613      Result res = table.get(get);
1614      if (res.isEmpty()) {
1615        return null;
1616      }
1617      Cell cell = res.listCells().get(0);
1618      byte[] val = CellUtil.cloneValue(cell);
1619      if (val.length == 0) {
1620        return null;
1621      }
1622      return Splitter.on(',').splitToStream(new String(val, StandardCharsets.UTF_8))
1623        .toArray(String[]::new);
1624    }
1625  }
1626
1627  private Put createPutForMergeOperation(String[] backupIdList) {
1628    byte[] value = Bytes.toBytes(StringUtils.join(backupIdList, ","));
1629    Put put = new Put(MERGE_OP_ROW);
1630    put.addColumn(META_FAMILY, FAM_COL, value);
1631    return put;
1632  }
1633
1634  public boolean isMergeInProgress() throws IOException {
1635    Get get = new Get(MERGE_OP_ROW);
1636    try (Table table = connection.getTable(tableName)) {
1637      Result res = table.get(get);
1638      return !res.isEmpty();
1639    }
1640  }
1641
1642  private Put createPutForUpdateTablesForMerge(List<TableName> tables) {
1643    byte[] value = Bytes.toBytes(StringUtils.join(tables, ","));
1644    Put put = new Put(MERGE_OP_ROW);
1645    put.addColumn(META_FAMILY, PATH_COL, value);
1646    return put;
1647  }
1648
1649  private Delete createDeleteForBackupMergeOperation() {
1650    Delete delete = new Delete(MERGE_OP_ROW);
1651    delete.addFamily(META_FAMILY);
1652    return delete;
1653  }
1654
1655  private Get createGetForMergeOperation() {
1656    Get get = new Get(MERGE_OP_ROW);
1657    get.addFamily(META_FAMILY);
1658    return get;
1659  }
1660
1661  public void startMergeOperation(String[] backupIdList) throws IOException {
1662    if (LOG.isTraceEnabled()) {
1663      LOG.trace("Start merge operation for backups: " + StringUtils.join(backupIdList));
1664    }
1665    Put put = createPutForMergeOperation(backupIdList);
1666    try (Table table = connection.getTable(tableName)) {
1667      table.put(put);
1668    }
1669  }
1670
1671  public void updateProcessedTablesForMerge(List<TableName> tables) throws IOException {
1672    if (LOG.isTraceEnabled()) {
1673      LOG.trace("Update tables for merge : " + StringUtils.join(tables, ","));
1674    }
1675    Put put = createPutForUpdateTablesForMerge(tables);
1676    try (Table table = connection.getTable(tableName)) {
1677      table.put(put);
1678    }
1679  }
1680
1681  public void finishMergeOperation() throws IOException {
1682    LOG.trace("Finish merge operation for backup ids");
1683
1684    Delete delete = createDeleteForBackupMergeOperation();
1685    try (Table table = connection.getTable(tableName)) {
1686      table.delete(delete);
1687    }
1688  }
1689
1690  public String[] getListOfBackupIdsFromMergeOperation() throws IOException {
1691    LOG.trace("Get backup ids for merge operation");
1692
1693    Get get = createGetForMergeOperation();
1694    try (Table table = connection.getTable(tableName)) {
1695      Result res = table.get(get);
1696      if (res.isEmpty()) {
1697        return null;
1698      }
1699      Cell cell = res.listCells().get(0);
1700      byte[] val = CellUtil.cloneValue(cell);
1701      if (val.length == 0) {
1702        return null;
1703      }
1704      return Splitter.on(',').splitToStream(new String(val, StandardCharsets.UTF_8))
1705        .toArray(String[]::new);
1706    }
1707  }
1708
1709  /**
1710   * Creates a scan to read all registered bulk loads for the given table, or for all tables if
1711   * {@code table} is {@code null}.
1712   */
1713  static Scan createScanForOrigBulkLoadedFiles(@Nullable TableName table) {
1714    Scan scan = new Scan();
1715    byte[] startRow = table == null
1716      ? BULK_LOAD_PREFIX_BYTES
1717      : rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM);
1718    byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
1719    stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
1720    scan.withStartRow(startRow);
1721    scan.withStopRow(stopRow);
1722    scan.addFamily(BackupSystemTable.META_FAMILY);
1723    scan.readVersions(1);
1724    return scan;
1725  }
1726
1727  static String getTableNameFromOrigBulkLoadRow(String rowStr) {
1728    // format is bulk : namespace : table : region : file
1729    return Iterators.get(Splitter.onPattern(BLK_LD_DELIM).split(rowStr).iterator(), 1);
1730  }
1731
1732  static String getRegionNameFromOrigBulkLoadRow(String rowStr) {
1733    // format is bulk : namespace : table : region : file
1734    List<String> parts = Splitter.onPattern(BLK_LD_DELIM).splitToList(rowStr);
1735    Iterator<String> i = parts.iterator();
1736    int idx = 3;
1737    if (parts.size() == 4) {
1738      // the table is in default namespace
1739      idx = 2;
1740    }
1741    String region = Iterators.get(i, idx);
1742    LOG.debug("bulk row string " + rowStr + " region " + region);
1743    return region;
1744  }
1745
1746  /*
1747   * Used to query bulk loaded hfiles which have been copied by incremental backup
1748   * @param backupId the backup Id. It can be null when querying for all tables
1749   * @return the Scan object
1750   * @deprecated This method is broken if a backupId is specified - see HBASE-28715
1751   */
1752  static Scan createScanForBulkLoadedFiles(String backupId) {
1753    Scan scan = new Scan();
1754    byte[] startRow =
1755      backupId == null ? BULK_LOAD_PREFIX_BYTES : rowkey(BULK_LOAD_PREFIX, backupId + BLK_LD_DELIM);
1756    byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
1757    stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
1758    scan.withStartRow(startRow);
1759    scan.withStopRow(stopRow);
1760    scan.addFamily(BackupSystemTable.META_FAMILY);
1761    scan.readVersions(1);
1762    return scan;
1763  }
1764
1765  /**
1766   * Creates Scan operation to load backup set list
1767   * @return scan operation
1768   */
1769  private Scan createScanForBackupSetList() {
1770    Scan scan = new Scan();
1771    byte[] startRow = Bytes.toBytes(SET_KEY_PREFIX);
1772    byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
1773    stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
1774    scan.withStartRow(startRow);
1775    scan.withStopRow(stopRow);
1776    scan.addFamily(BackupSystemTable.META_FAMILY);
1777    return scan;
1778  }
1779
1780  /**
1781   * Creates Get operation to load backup set content
1782   * @return get operation
1783   */
1784  private Get createGetForBackupSet(String name) {
1785    Get get = new Get(rowkey(SET_KEY_PREFIX, name));
1786    get.addFamily(BackupSystemTable.META_FAMILY);
1787    return get;
1788  }
1789
1790  /**
1791   * Creates Delete operation to delete backup set content
1792   * @param name backup set's name
1793   * @return delete operation
1794   */
1795  private Delete createDeleteForBackupSet(String name) {
1796    Delete del = new Delete(rowkey(SET_KEY_PREFIX, name));
1797    del.addFamily(BackupSystemTable.META_FAMILY);
1798    return del;
1799  }
1800
1801  /**
1802   * Creates Put operation to update backup set content
1803   * @param name   backup set's name
1804   * @param tables list of tables
1805   * @return put operation
1806   */
1807  private Put createPutForBackupSet(String name, String[] tables) {
1808    Put put = new Put(rowkey(SET_KEY_PREFIX, name));
1809    byte[] value = convertToByteArray(tables);
1810    put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes("tables"), value);
1811    return put;
1812  }
1813
1814  private byte[] convertToByteArray(String[] tables) {
1815    return Bytes.toBytes(StringUtils.join(tables, ","));
1816  }
1817
1818  /**
1819   * Converts cell to backup set list.
1820   * @param current current cell
1821   * @return backup set as array of table names
1822   */
1823  private String[] cellValueToBackupSet(Cell current) {
1824    byte[] data = CellUtil.cloneValue(current);
1825    if (!ArrayUtils.isEmpty(data)) {
1826      return Bytes.toString(data).split(",");
1827    }
1828    return new String[0];
1829  }
1830
1831  /**
1832   * Converts cell key to backup set name.
1833   * @param current current cell
1834   * @return backup set name
1835   */
1836  private String cellKeyToBackupSetName(Cell current) {
1837    byte[] data = CellUtil.cloneRow(current);
1838    return Bytes.toString(data).substring(SET_KEY_PREFIX.length());
1839  }
1840
1841  private static byte[] rowkey(String s, String... other) {
1842    StringBuilder sb = new StringBuilder(s);
1843    for (String ss : other) {
1844      sb.append(ss);
1845    }
1846    return Bytes.toBytes(sb.toString());
1847  }
1848
1849  private static void ensureTableEnabled(Admin admin, TableName tableName) throws IOException {
1850    if (!admin.isTableEnabled(tableName)) {
1851      try {
1852        admin.enableTable(tableName);
1853      } catch (TableNotDisabledException ignored) {
1854        LOG.info("Table {} is not disabled, ignoring enable request", tableName);
1855      }
1856    }
1857  }
1858
1859  public enum Order {
1860    /**
1861     * Old backups first, most recents last. I.e. sorted by ascending backupId.
1862     */
1863    OLD_TO_NEW,
1864    /**
1865     * New backups first, oldest last. I.e. sorted by descending backupId.
1866     */
1867    NEW_TO_OLD
1868  }
1869}