001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.Iterator;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.TreeSet;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseTestingUtil;
038import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
041import org.apache.hadoop.hbase.backup.impl.BackupManager;
042import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.Connection;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.AfterEach;
049import org.junit.jupiter.api.BeforeAll;
050import org.junit.jupiter.api.BeforeEach;
051import org.junit.jupiter.api.Tag;
052import org.junit.jupiter.api.Test;
053
054/**
055 * Test cases for backup system table API
056 */
057@Tag(LargeTests.TAG)
058public class TestBackupSystemTable {
059
060  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
061  protected static Configuration conf = UTIL.getConfiguration();
062  protected static SingleProcessHBaseCluster cluster;
063  protected static Connection conn;
064  protected BackupSystemTable table;
065
066  @BeforeAll
067  public static void setUp() throws Exception {
068    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
069    BackupManager.decorateMasterConfiguration(conf);
070    BackupManager.decorateRegionServerConfiguration(conf);
071    cluster = UTIL.startMiniCluster();
072    conn = UTIL.getConnection();
073  }
074
075  @BeforeEach
076  public void before() throws IOException {
077    table = new BackupSystemTable(conn);
078  }
079
080  @AfterEach
081  public void after() {
082    if (table != null) {
083      table.close();
084    }
085
086  }
087
088  @Test
089  public void testUpdateReadDeleteBackupStatus() throws IOException {
090    BackupInfo ctx = createBackupInfo();
091    table.updateBackupInfo(ctx);
092    BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
093    assertTrue(compare(ctx, readCtx));
094    // try fake backup id
095    readCtx = table.readBackupInfo("fake");
096    assertNull(readCtx);
097    // delete backup info
098    table.deleteBackupInfo(ctx.getBackupId());
099    readCtx = table.readBackupInfo(ctx.getBackupId());
100    assertNull(readCtx);
101    cleanBackupTable();
102  }
103
104  private void cleanBackupTable() throws IOException {
105    Admin admin = UTIL.getAdmin();
106    admin.disableTable(BackupSystemTable.getTableName(conf));
107    admin.truncateTable(BackupSystemTable.getTableName(conf), true);
108    if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) {
109      admin.enableTable(BackupSystemTable.getTableName(conf));
110    }
111  }
112
113  @Test
114  public void testBackupHistory() throws Exception {
115    int n = 10;
116    List<BackupInfo> list = createBackupInfoList(n);
117
118    // Load data
119    for (BackupInfo bc : list) {
120      // Make sure we set right status
121      bc.setState(BackupState.COMPLETE);
122      table.updateBackupInfo(bc);
123    }
124
125    // Reverse list for comparison
126    Collections.reverse(list);
127    List<BackupInfo> history = table.getBackupHistory();
128    assertTrue(history.size() == n);
129
130    for (int i = 0; i < n; i++) {
131      BackupInfo ctx = list.get(i);
132      BackupInfo data = history.get(i);
133      assertTrue(compare(ctx, data));
134    }
135
136    cleanBackupTable();
137
138  }
139
140  @Test
141  public void testBackupDelete() throws Exception {
142    try (BackupSystemTable table = new BackupSystemTable(conn)) {
143      int n = 10;
144      List<BackupInfo> list = createBackupInfoList(n);
145
146      // Load data
147      for (BackupInfo bc : list) {
148        // Make sure we set right status
149        bc.setState(BackupState.COMPLETE);
150        table.updateBackupInfo(bc);
151      }
152
153      // Verify exists
154      for (BackupInfo bc : list) {
155        assertNotNull(table.readBackupInfo(bc.getBackupId()));
156      }
157
158      // Delete all
159      for (BackupInfo bc : list) {
160        table.deleteBackupInfo(bc.getBackupId());
161      }
162
163      // Verify do not exists
164      for (BackupInfo bc : list) {
165        assertNull(table.readBackupInfo(bc.getBackupId()));
166      }
167
168      cleanBackupTable();
169    }
170
171  }
172
173  @Test
174  public void testRegionServerLastLogRollResults() throws IOException {
175    String[] servers = new String[] { "server1", "server2", "server3" };
176    Long[] timestamps = new Long[] { 100L, 102L, 107L };
177
178    // validate the prefix scan in readRegionServerlastLogRollResult will get the right timestamps
179    // when a backup root with the same prefix is present
180    for (int i = 0; i < servers.length; i++) {
181      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
182      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root/backup");
183    }
184
185    HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
186    assertTrue(servers.length == result.size());
187    Set<String> keys = result.keySet();
188    String[] keysAsArray = new String[keys.size()];
189    keys.toArray(keysAsArray);
190    Arrays.sort(keysAsArray);
191
192    for (int i = 0; i < keysAsArray.length; i++) {
193      assertEquals(keysAsArray[i], servers[i]);
194      Long ts1 = timestamps[i];
195      Long ts2 = result.get(keysAsArray[i]);
196      assertEquals(ts1, ts2);
197    }
198
199    cleanBackupTable();
200  }
201
202  @Test
203  public void testIncrementalBackupTableSet() throws IOException {
204    TreeSet<TableName> tables1 = new TreeSet<>();
205
206    tables1.add(TableName.valueOf("t1"));
207    tables1.add(TableName.valueOf("t2"));
208    tables1.add(TableName.valueOf("t3"));
209
210    TreeSet<TableName> tables2 = new TreeSet<>();
211
212    tables2.add(TableName.valueOf("t3"));
213    tables2.add(TableName.valueOf("t4"));
214    tables2.add(TableName.valueOf("t5"));
215
216    table.addIncrementalBackupTableSet(tables1, "root");
217
218    try (BackupSystemTable systemTable = new BackupSystemTable(conn)) {
219      TreeSet<TableName> res1 =
220        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
221      assertTrue(tables1.size() == res1.size());
222      Iterator<TableName> desc1 = tables1.descendingIterator();
223      Iterator<TableName> desc2 = res1.descendingIterator();
224      while (desc1.hasNext()) {
225        assertEquals(desc1.next(), desc2.next());
226      }
227      systemTable.addIncrementalBackupTableSet(tables2, "root");
228      TreeSet<TableName> res2 =
229        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
230      assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
231      tables1.addAll(tables2);
232      desc1 = tables1.descendingIterator();
233      desc2 = res2.descendingIterator();
234      while (desc1.hasNext()) {
235        assertEquals(desc1.next(), desc2.next());
236      }
237    }
238
239    cleanBackupTable();
240  }
241
242  @Test
243  public void testRegionServerLogTimestampMap() throws IOException {
244    TreeSet<TableName> tables = new TreeSet<>();
245
246    tables.add(TableName.valueOf("t1"));
247    tables.add(TableName.valueOf("t2"));
248    tables.add(TableName.valueOf("t3"));
249
250    HashMap<String, Long> rsTimestampMap = new HashMap<>();
251
252    rsTimestampMap.put("rs1:100", 100L);
253    rsTimestampMap.put("rs2:100", 101L);
254    rsTimestampMap.put("rs3:100", 103L);
255
256    // validate the prefix scan in readLogTimestampMap will get the right timestamps
257    // when a backup root with the same prefix is present
258    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
259    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root/backup");
260
261    Map<TableName, Map<String, Long>> result = table.readLogTimestampMap("root");
262
263    assertTrue(tables.size() == result.size());
264
265    for (TableName t : tables) {
266      Map<String, Long> rstm = result.get(t);
267      assertNotNull(rstm);
268      assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
269      assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
270      assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
271    }
272
273    Set<TableName> tables1 = new TreeSet<>();
274
275    tables1.add(TableName.valueOf("t3"));
276    tables1.add(TableName.valueOf("t4"));
277    tables1.add(TableName.valueOf("t5"));
278
279    HashMap<String, Long> rsTimestampMap1 = new HashMap<>();
280
281    rsTimestampMap1.put("rs1:100", 200L);
282    rsTimestampMap1.put("rs2:100", 201L);
283    rsTimestampMap1.put("rs3:100", 203L);
284
285    // validate the prefix scan in readLogTimestampMap will get the right timestamps
286    // when a backup root with the same prefix is present
287    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
288    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap, "root/backup");
289
290    result = table.readLogTimestampMap("root");
291
292    assertTrue(5 == result.size());
293
294    for (TableName t : tables) {
295      Map<String, Long> rstm = result.get(t);
296      assertNotNull(rstm);
297      if (t.equals(TableName.valueOf("t3")) == false) {
298        assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
299        assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
300        assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
301      } else {
302        assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
303        assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
304        assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
305      }
306    }
307
308    for (TableName t : tables1) {
309      Map<String, Long> rstm = result.get(t);
310      assertNotNull(rstm);
311      assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
312      assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
313      assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
314    }
315
316    cleanBackupTable();
317
318  }
319
320  /**
321   * Backup set tests
322   */
323
324  @Test
325  public void testBackupSetAddNotExists() throws IOException {
326    try (BackupSystemTable table = new BackupSystemTable(conn)) {
327
328      String[] tables = new String[] { "table1", "table2", "table3" };
329      String setName = "name";
330      table.addToBackupSet(setName, tables);
331      List<TableName> tnames = table.describeBackupSet(setName);
332      assertTrue(tnames != null);
333      assertTrue(tnames.size() == tables.length);
334      for (int i = 0; i < tnames.size(); i++) {
335        assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
336      }
337      cleanBackupTable();
338    }
339
340  }
341
342  @Test
343  public void testBackupSetAddExists() throws IOException {
344    try (BackupSystemTable table = new BackupSystemTable(conn)) {
345
346      String[] tables = new String[] { "table1", "table2", "table3" };
347      String setName = "name";
348      table.addToBackupSet(setName, tables);
349      String[] addTables = new String[] { "table4", "table5", "table6" };
350      table.addToBackupSet(setName, addTables);
351
352      Set<String> expectedTables =
353        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
354
355      List<TableName> tnames = table.describeBackupSet(setName);
356      assertTrue(tnames != null);
357      assertTrue(tnames.size() == expectedTables.size());
358      for (TableName tableName : tnames) {
359        assertTrue(expectedTables.remove(tableName.getNameAsString()));
360      }
361      cleanBackupTable();
362    }
363  }
364
365  @Test
366  public void testBackupSetAddExistsIntersects() throws IOException {
367    try (BackupSystemTable table = new BackupSystemTable(conn)) {
368
369      String[] tables = new String[] { "table1", "table2", "table3" };
370      String setName = "name";
371      table.addToBackupSet(setName, tables);
372      String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
373      table.addToBackupSet(setName, addTables);
374
375      Set<String> expectedTables =
376        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
377
378      List<TableName> tnames = table.describeBackupSet(setName);
379      assertTrue(tnames != null);
380      assertTrue(tnames.size() == expectedTables.size());
381      for (TableName tableName : tnames) {
382        assertTrue(expectedTables.remove(tableName.getNameAsString()));
383      }
384      cleanBackupTable();
385    }
386  }
387
388  @Test
389  public void testBackupSetRemoveSomeNotExists() throws IOException {
390    try (BackupSystemTable table = new BackupSystemTable(conn)) {
391
392      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
393      String setName = "name";
394      table.addToBackupSet(setName, tables);
395      String[] removeTables = new String[] { "table4", "table5", "table6" };
396      table.removeFromBackupSet(setName, removeTables);
397
398      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3"));
399
400      List<TableName> tnames = table.describeBackupSet(setName);
401      assertTrue(tnames != null);
402      assertTrue(tnames.size() == expectedTables.size());
403      for (TableName tableName : tnames) {
404        assertTrue(expectedTables.remove(tableName.getNameAsString()));
405      }
406      cleanBackupTable();
407    }
408  }
409
410  @Test
411  public void testBackupSetRemove() throws IOException {
412    try (BackupSystemTable table = new BackupSystemTable(conn)) {
413
414      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
415      String setName = "name";
416      table.addToBackupSet(setName, tables);
417      String[] removeTables = new String[] { "table4", "table3" };
418      table.removeFromBackupSet(setName, removeTables);
419
420      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2"));
421
422      List<TableName> tnames = table.describeBackupSet(setName);
423      assertTrue(tnames != null);
424      assertTrue(tnames.size() == expectedTables.size());
425      for (TableName tableName : tnames) {
426        assertTrue(expectedTables.remove(tableName.getNameAsString()));
427      }
428      cleanBackupTable();
429    }
430  }
431
432  @Test
433  public void testBackupSetDelete() throws IOException {
434    try (BackupSystemTable table = new BackupSystemTable(conn)) {
435
436      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
437      String setName = "name";
438      table.addToBackupSet(setName, tables);
439      table.deleteBackupSet(setName);
440
441      List<TableName> tnames = table.describeBackupSet(setName);
442      assertTrue(tnames == null);
443      cleanBackupTable();
444    }
445  }
446
447  @Test
448  public void testBackupSetList() throws IOException {
449    try (BackupSystemTable table = new BackupSystemTable(conn)) {
450
451      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
452      String setName1 = "name1";
453      String setName2 = "name2";
454      table.addToBackupSet(setName1, tables);
455      table.addToBackupSet(setName2, tables);
456
457      List<String> list = table.listBackupSets();
458
459      assertTrue(list.size() == 2);
460      assertTrue(list.get(0).equals(setName1));
461      assertTrue(list.get(1).equals(setName2));
462
463      cleanBackupTable();
464    }
465  }
466
467  private boolean compare(BackupInfo one, BackupInfo two) {
468    return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
469      && one.getBackupRootDir().equals(two.getBackupRootDir())
470      && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs();
471  }
472
473  private BackupInfo createBackupInfo() {
474    BackupInfo ctxt = new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
475      new TableName[] { TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
476      "/hbase/backup");
477    ctxt.setStartTs(EnvironmentEdgeManager.currentTime());
478    ctxt.setCompleteTs(EnvironmentEdgeManager.currentTime() + 1);
479    return ctxt;
480  }
481
482  private List<BackupInfo> createBackupInfoList(int size) throws InterruptedException {
483    List<BackupInfo> list = new ArrayList<>();
484    for (int i = 0; i < size; i++) {
485      list.add(createBackupInfo());
486      // XXX Why do we need this sleep?
487      Thread.sleep(10);
488    }
489    return list;
490  }
491
492  @AfterAll
493  public static void tearDown() throws IOException {
494    if (cluster != null) {
495      cluster.shutdown();
496    }
497  }
498}