001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.Iterator;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.TreeSet;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseTestingUtil;
038import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
041import org.apache.hadoop.hbase.backup.impl.BackupManager;
042import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.Connection;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeAll;
050import org.junit.jupiter.api.BeforeEach;
051import org.junit.jupiter.api.Tag;
052import org.junit.jupiter.api.Test;
053
054/**
055 * Test cases for backup system table API
056 */
057@Tag(LargeTests.TAG)
058public class TestBackupSystemTable {
059
060  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
061  protected static Configuration conf = UTIL.getConfiguration();
062  protected static SingleProcessHBaseCluster cluster;
063  protected static Connection conn;
064  protected BackupSystemTable table;
065
066  @BeforeAll
067  public static void setUp() throws Exception {
068    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
069    BackupManager.decorateMasterConfiguration(conf);
070    BackupManager.decorateRegionServerConfiguration(conf);
071    cluster = UTIL.startMiniCluster();
072    conn = UTIL.getConnection();
073  }
074
075  @BeforeEach
076  public void before() throws IOException {
077    table = new BackupSystemTable(conn);
078  }
079
080  @AfterEach
081  public void after() {
082    if (table != null) {
083      table.close();
084    }
085
086  }
087
088  @Test
089  public void testUpdateReadDeleteBackupStatus() throws IOException {
090    BackupInfo ctx = createBackupInfo();
091    table.updateBackupInfo(ctx);
092    BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
093    assertTrue(compare(ctx, readCtx));
094    // try fake backup id
095    readCtx = table.readBackupInfo("fake");
096    assertNull(readCtx);
097    // delete backup info
098    table.deleteBackupInfo(ctx.getBackupId());
099    readCtx = table.readBackupInfo(ctx.getBackupId());
100    assertNull(readCtx);
101    cleanBackupTable();
102  }
103
104  private void cleanBackupTable() throws IOException {
105    Admin admin = UTIL.getAdmin();
106    admin.disableTable(BackupSystemTable.getTableName(conf));
107    admin.truncateTable(BackupSystemTable.getTableName(conf), true);
108    if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) {
109      admin.enableTable(BackupSystemTable.getTableName(conf));
110    }
111  }
112
113  @Test
114  public void testBackupHistory() throws Exception {
115    int n = 10;
116    List<BackupInfo> list = createBackupInfoList(n);
117
118    // Load data
119    for (BackupInfo bc : list) {
120      // Make sure we set right status
121      bc.setState(BackupState.COMPLETE);
122      table.updateBackupInfo(bc);
123    }
124
125    // Reverse list for comparison
126    Collections.reverse(list);
127    List<BackupInfo> history = table.getBackupHistory();
128    assertTrue(history.size() == n);
129
130    for (int i = 0; i < n; i++) {
131      BackupInfo ctx = list.get(i);
132      BackupInfo data = history.get(i);
133      assertTrue(compare(ctx, data));
134    }
135
136    cleanBackupTable();
137
138  }
139
140  @Test
141  public void testBackupDelete() throws Exception {
142    try (BackupSystemTable table = new BackupSystemTable(conn)) {
143      int n = 10;
144      List<BackupInfo> list = createBackupInfoList(n);
145
146      // Load data
147      for (BackupInfo bc : list) {
148        // Make sure we set right status
149        bc.setState(BackupState.COMPLETE);
150        table.updateBackupInfo(bc);
151      }
152
153      // Verify exists
154      for (BackupInfo bc : list) {
155        assertNotNull(table.readBackupInfo(bc.getBackupId()));
156      }
157
158      // Delete all
159      for (BackupInfo bc : list) {
160        table.deleteBackupInfo(bc.getBackupId());
161      }
162
163      // Verify do not exists
164      for (BackupInfo bc : list) {
165        assertNull(table.readBackupInfo(bc.getBackupId()));
166      }
167
168      cleanBackupTable();
169    }
170
171  }
172
173  @Test
174  public void testRegionServerLastLogRollResults() throws IOException {
175    String[] servers = new String[] { "server1", "server2", "server3" };
176    Long[] timestamps = new Long[] { 100L, 102L, 107L };
177
178    // validate the prefix scan in readRegionServerlastLogRollResult will get the right timestamps
179    // when a backup root with the same prefix is present
180    for (int i = 0; i < servers.length; i++) {
181      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
182      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root/backup");
183    }
184
185    HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
186    assertTrue(servers.length == result.size());
187    Set<String> keys = result.keySet();
188    String[] keysAsArray = new String[keys.size()];
189    keys.toArray(keysAsArray);
190    Arrays.sort(keysAsArray);
191
192    for (int i = 0; i < keysAsArray.length; i++) {
193      assertEquals(keysAsArray[i], servers[i]);
194      Long ts1 = timestamps[i];
195      Long ts2 = result.get(keysAsArray[i]);
196      assertEquals(ts1, ts2);
197    }
198
199    cleanBackupTable();
200  }
201
202  @Test
203  public void testMixedCaseHostKeysAreReadLowercase() throws IOException {
204    table.writeRegionServerLastLogRollResult("Host-A:1", 102L, "root");
205    table.writeRegionServerLastLogRollResult("host-a:1", 100L, "root");
206    assertEquals(Map.of("host-a:1", 102L), table.readRegionServerLastLogRollResult("root"));
207
208    TableName t1 = TableName.valueOf("t1");
209    table.writeRegionServerLogTimestamp(Set.of(t1), Map.of("Host-A:1", 100L), "root");
210    assertEquals(Map.of(t1, Map.of("host-a:1", 100L)), table.readLogTimestampMap("root"));
211
212    cleanBackupTable();
213  }
214
215  @Test
216  public void testIncrementalBackupTableSet() throws IOException {
217    TreeSet<TableName> tables1 = new TreeSet<>();
218
219    tables1.add(TableName.valueOf("t1"));
220    tables1.add(TableName.valueOf("t2"));
221    tables1.add(TableName.valueOf("t3"));
222
223    TreeSet<TableName> tables2 = new TreeSet<>();
224
225    tables2.add(TableName.valueOf("t3"));
226    tables2.add(TableName.valueOf("t4"));
227    tables2.add(TableName.valueOf("t5"));
228
229    table.addIncrementalBackupTableSet(tables1, "root");
230
231    try (BackupSystemTable systemTable = new BackupSystemTable(conn)) {
232      TreeSet<TableName> res1 =
233        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
234      assertTrue(tables1.size() == res1.size());
235      Iterator<TableName> desc1 = tables1.descendingIterator();
236      Iterator<TableName> desc2 = res1.descendingIterator();
237      while (desc1.hasNext()) {
238        assertEquals(desc1.next(), desc2.next());
239      }
240      systemTable.addIncrementalBackupTableSet(tables2, "root");
241      TreeSet<TableName> res2 =
242        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
243      assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
244      tables1.addAll(tables2);
245      desc1 = tables1.descendingIterator();
246      desc2 = res2.descendingIterator();
247      while (desc1.hasNext()) {
248        assertEquals(desc1.next(), desc2.next());
249      }
250    }
251
252    cleanBackupTable();
253  }
254
255  @Test
256  public void testRegionServerLogTimestampMap() throws IOException {
257    TreeSet<TableName> tables = new TreeSet<>();
258
259    tables.add(TableName.valueOf("t1"));
260    tables.add(TableName.valueOf("t2"));
261    tables.add(TableName.valueOf("t3"));
262
263    HashMap<String, Long> rsTimestampMap = new HashMap<>();
264
265    rsTimestampMap.put("rs1:100", 100L);
266    rsTimestampMap.put("rs2:100", 101L);
267    rsTimestampMap.put("rs3:100", 103L);
268
269    // validate the prefix scan in readLogTimestampMap will get the right timestamps
270    // when a backup root with the same prefix is present
271    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
272    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root/backup");
273
274    Map<TableName, Map<String, Long>> result = table.readLogTimestampMap("root");
275
276    assertTrue(tables.size() == result.size());
277
278    for (TableName t : tables) {
279      Map<String, Long> rstm = result.get(t);
280      assertNotNull(rstm);
281      assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
282      assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
283      assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
284    }
285
286    Set<TableName> tables1 = new TreeSet<>();
287
288    tables1.add(TableName.valueOf("t3"));
289    tables1.add(TableName.valueOf("t4"));
290    tables1.add(TableName.valueOf("t5"));
291
292    HashMap<String, Long> rsTimestampMap1 = new HashMap<>();
293
294    rsTimestampMap1.put("rs1:100", 200L);
295    rsTimestampMap1.put("rs2:100", 201L);
296    rsTimestampMap1.put("rs3:100", 203L);
297
298    // validate the prefix scan in readLogTimestampMap will get the right timestamps
299    // when a backup root with the same prefix is present
300    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
301    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap, "root/backup");
302
303    result = table.readLogTimestampMap("root");
304
305    assertTrue(5 == result.size());
306
307    for (TableName t : tables) {
308      Map<String, Long> rstm = result.get(t);
309      assertNotNull(rstm);
310      if (t.equals(TableName.valueOf("t3")) == false) {
311        assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
312        assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
313        assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
314      } else {
315        assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
316        assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
317        assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
318      }
319    }
320
321    for (TableName t : tables1) {
322      Map<String, Long> rstm = result.get(t);
323      assertNotNull(rstm);
324      assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
325      assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
326      assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
327    }
328
329    cleanBackupTable();
330
331  }
332
333  /**
334   * Backup set tests
335   */
336
337  @Test
338  public void testBackupSetAddNotExists() throws IOException {
339    try (BackupSystemTable table = new BackupSystemTable(conn)) {
340
341      String[] tables = new String[] { "table1", "table2", "table3" };
342      String setName = "name";
343      table.addToBackupSet(setName, tables);
344      List<TableName> tnames = table.describeBackupSet(setName);
345      assertTrue(tnames != null);
346      assertTrue(tnames.size() == tables.length);
347      for (int i = 0; i < tnames.size(); i++) {
348        assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
349      }
350      cleanBackupTable();
351    }
352
353  }
354
355  @Test
356  public void testBackupSetAddExists() throws IOException {
357    try (BackupSystemTable table = new BackupSystemTable(conn)) {
358
359      String[] tables = new String[] { "table1", "table2", "table3" };
360      String setName = "name";
361      table.addToBackupSet(setName, tables);
362      String[] addTables = new String[] { "table4", "table5", "table6" };
363      table.addToBackupSet(setName, addTables);
364
365      Set<String> expectedTables =
366        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
367
368      List<TableName> tnames = table.describeBackupSet(setName);
369      assertTrue(tnames != null);
370      assertTrue(tnames.size() == expectedTables.size());
371      for (TableName tableName : tnames) {
372        assertTrue(expectedTables.remove(tableName.getNameAsString()));
373      }
374      cleanBackupTable();
375    }
376  }
377
378  @Test
379  public void testBackupSetAddExistsIntersects() throws IOException {
380    try (BackupSystemTable table = new BackupSystemTable(conn)) {
381
382      String[] tables = new String[] { "table1", "table2", "table3" };
383      String setName = "name";
384      table.addToBackupSet(setName, tables);
385      String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
386      table.addToBackupSet(setName, addTables);
387
388      Set<String> expectedTables =
389        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
390
391      List<TableName> tnames = table.describeBackupSet(setName);
392      assertTrue(tnames != null);
393      assertTrue(tnames.size() == expectedTables.size());
394      for (TableName tableName : tnames) {
395        assertTrue(expectedTables.remove(tableName.getNameAsString()));
396      }
397      cleanBackupTable();
398    }
399  }
400
401  @Test
402  public void testBackupSetRemoveSomeNotExists() throws IOException {
403    try (BackupSystemTable table = new BackupSystemTable(conn)) {
404
405      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
406      String setName = "name";
407      table.addToBackupSet(setName, tables);
408      String[] removeTables = new String[] { "table4", "table5", "table6" };
409      table.removeFromBackupSet(setName, removeTables);
410
411      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3"));
412
413      List<TableName> tnames = table.describeBackupSet(setName);
414      assertTrue(tnames != null);
415      assertTrue(tnames.size() == expectedTables.size());
416      for (TableName tableName : tnames) {
417        assertTrue(expectedTables.remove(tableName.getNameAsString()));
418      }
419      cleanBackupTable();
420    }
421  }
422
423  @Test
424  public void testBackupSetRemove() throws IOException {
425    try (BackupSystemTable table = new BackupSystemTable(conn)) {
426
427      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
428      String setName = "name";
429      table.addToBackupSet(setName, tables);
430      String[] removeTables = new String[] { "table4", "table3" };
431      table.removeFromBackupSet(setName, removeTables);
432
433      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2"));
434
435      List<TableName> tnames = table.describeBackupSet(setName);
436      assertTrue(tnames != null);
437      assertTrue(tnames.size() == expectedTables.size());
438      for (TableName tableName : tnames) {
439        assertTrue(expectedTables.remove(tableName.getNameAsString()));
440      }
441      cleanBackupTable();
442    }
443  }
444
445  @Test
446  public void testBackupSetDelete() throws IOException {
447    try (BackupSystemTable table = new BackupSystemTable(conn)) {
448
449      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
450      String setName = "name";
451      table.addToBackupSet(setName, tables);
452      table.deleteBackupSet(setName);
453
454      List<TableName> tnames = table.describeBackupSet(setName);
455      assertTrue(tnames == null);
456      cleanBackupTable();
457    }
458  }
459
460  @Test
461  public void testBackupSetList() throws IOException {
462    try (BackupSystemTable table = new BackupSystemTable(conn)) {
463
464      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
465      String setName1 = "name1";
466      String setName2 = "name2";
467      table.addToBackupSet(setName1, tables);
468      table.addToBackupSet(setName2, tables);
469
470      List<String> list = table.listBackupSets();
471
472      assertTrue(list.size() == 2);
473      assertTrue(list.get(0).equals(setName1));
474      assertTrue(list.get(1).equals(setName2));
475
476      cleanBackupTable();
477    }
478  }
479
480  private boolean compare(BackupInfo one, BackupInfo two) {
481    return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
482      && one.getBackupRootDir().equals(two.getBackupRootDir())
483      && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs();
484  }
485
486  private BackupInfo createBackupInfo() {
487    BackupInfo ctxt = new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
488      new TableName[] { TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
489      "/hbase/backup");
490    ctxt.setStartTs(EnvironmentEdgeManager.currentTime());
491    ctxt.setCompleteTs(EnvironmentEdgeManager.currentTime() + 1);
492    return ctxt;
493  }
494
495  private List<BackupInfo> createBackupInfoList(int size) throws InterruptedException {
496    List<BackupInfo> list = new ArrayList<>();
497    for (int i = 0; i < size; i++) {
498      list.add(createBackupInfo());
499      // XXX Why do we need this sleep?
500      Thread.sleep(10);
501    }
502    return list;
503  }
504
505  @AfterAll
506  public static void tearDown() throws IOException {
507    if (cluster != null) {
508      cluster.shutdown();
509    }
510  }
511}