Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.HashSet;
031import java.util.Iterator;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.TreeSet;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
042import org.apache.hadoop.hbase.backup.impl.BackupManager;
043import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.Connection;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
048import org.junit.After;
049import org.junit.AfterClass;
050import org.junit.Before;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055
056/**
057 * Test cases for backup system table API
058 */
059@Category(MediumTests.class)
060public class TestBackupSystemTable {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestBackupSystemTable.class);
065
066  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
067  protected static Configuration conf = UTIL.getConfiguration();
068  protected static SingleProcessHBaseCluster cluster;
069  protected static Connection conn;
070  protected BackupSystemTable table;
071
072  @BeforeClass
073  public static void setUp() throws Exception {
074    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
075    BackupManager.decorateMasterConfiguration(conf);
076    BackupManager.decorateRegionServerConfiguration(conf);
077    cluster = UTIL.startMiniCluster();
078    conn = UTIL.getConnection();
079  }
080
081  @Before
082  public void before() throws IOException {
083    table = new BackupSystemTable(conn);
084  }
085
086  @After
087  public void after() {
088    if (table != null) {
089      table.close();
090    }
091
092  }
093
094  @Test
095  public void testUpdateReadDeleteBackupStatus() throws IOException {
096    BackupInfo ctx = createBackupInfo();
097    table.updateBackupInfo(ctx);
098    BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
099    assertTrue(compare(ctx, readCtx));
100    // try fake backup id
101    readCtx = table.readBackupInfo("fake");
102    assertNull(readCtx);
103    // delete backup info
104    table.deleteBackupInfo(ctx.getBackupId());
105    readCtx = table.readBackupInfo(ctx.getBackupId());
106    assertNull(readCtx);
107    cleanBackupTable();
108  }
109
110  @Test
111  public void testWriteReadBackupStartCode() throws IOException {
112    long code = 100L;
113    table.writeBackupStartCode(code, "root");
114    String readCode = table.readBackupStartCode("root");
115    assertEquals(code, Long.parseLong(readCode));
116    cleanBackupTable();
117  }
118
119  private void cleanBackupTable() throws IOException {
120    Admin admin = UTIL.getAdmin();
121    admin.disableTable(BackupSystemTable.getTableName(conf));
122    admin.truncateTable(BackupSystemTable.getTableName(conf), true);
123    if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) {
124      admin.enableTable(BackupSystemTable.getTableName(conf));
125    }
126  }
127
128  @Test
129  public void testBackupHistory() throws Exception {
130    int n = 10;
131    List<BackupInfo> list = createBackupInfoList(n);
132
133    // Load data
134    for (BackupInfo bc : list) {
135      // Make sure we set right status
136      bc.setState(BackupState.COMPLETE);
137      table.updateBackupInfo(bc);
138    }
139
140    // Reverse list for comparison
141    Collections.reverse(list);
142    List<BackupInfo> history = table.getBackupHistory();
143    assertTrue(history.size() == n);
144
145    for (int i = 0; i < n; i++) {
146      BackupInfo ctx = list.get(i);
147      BackupInfo data = history.get(i);
148      assertTrue(compare(ctx, data));
149    }
150
151    cleanBackupTable();
152
153  }
154
155  @Test
156  public void testBackupDelete() throws Exception {
157    try (BackupSystemTable table = new BackupSystemTable(conn)) {
158      int n = 10;
159      List<BackupInfo> list = createBackupInfoList(n);
160
161      // Load data
162      for (BackupInfo bc : list) {
163        // Make sure we set right status
164        bc.setState(BackupState.COMPLETE);
165        table.updateBackupInfo(bc);
166      }
167
168      // Verify exists
169      for (BackupInfo bc : list) {
170        assertNotNull(table.readBackupInfo(bc.getBackupId()));
171      }
172
173      // Delete all
174      for (BackupInfo bc : list) {
175        table.deleteBackupInfo(bc.getBackupId());
176      }
177
178      // Verify do not exists
179      for (BackupInfo bc : list) {
180        assertNull(table.readBackupInfo(bc.getBackupId()));
181      }
182
183      cleanBackupTable();
184    }
185
186  }
187
188  @Test
189  public void testRegionServerLastLogRollResults() throws IOException {
190    String[] servers = new String[] { "server1", "server2", "server3" };
191    Long[] timestamps = new Long[] { 100L, 102L, 107L };
192
193    // validate the prefix scan in readRegionServerlastLogRollResult will get the right timestamps
194    // when a backup root with the same prefix is present
195    for (int i = 0; i < servers.length; i++) {
196      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
197      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root/backup");
198    }
199
200    HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
201    assertTrue(servers.length == result.size());
202    Set<String> keys = result.keySet();
203    String[] keysAsArray = new String[keys.size()];
204    keys.toArray(keysAsArray);
205    Arrays.sort(keysAsArray);
206
207    for (int i = 0; i < keysAsArray.length; i++) {
208      assertEquals(keysAsArray[i], servers[i]);
209      Long ts1 = timestamps[i];
210      Long ts2 = result.get(keysAsArray[i]);
211      assertEquals(ts1, ts2);
212    }
213
214    cleanBackupTable();
215  }
216
217  @Test
218  public void testIncrementalBackupTableSet() throws IOException {
219    TreeSet<TableName> tables1 = new TreeSet<>();
220
221    tables1.add(TableName.valueOf("t1"));
222    tables1.add(TableName.valueOf("t2"));
223    tables1.add(TableName.valueOf("t3"));
224
225    TreeSet<TableName> tables2 = new TreeSet<>();
226
227    tables2.add(TableName.valueOf("t3"));
228    tables2.add(TableName.valueOf("t4"));
229    tables2.add(TableName.valueOf("t5"));
230
231    table.addIncrementalBackupTableSet(tables1, "root");
232
233    try (BackupSystemTable systemTable = new BackupSystemTable(conn)) {
234      TreeSet<TableName> res1 =
235        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
236      assertTrue(tables1.size() == res1.size());
237      Iterator<TableName> desc1 = tables1.descendingIterator();
238      Iterator<TableName> desc2 = res1.descendingIterator();
239      while (desc1.hasNext()) {
240        assertEquals(desc1.next(), desc2.next());
241      }
242      systemTable.addIncrementalBackupTableSet(tables2, "root");
243      TreeSet<TableName> res2 =
244        (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root");
245      assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
246      tables1.addAll(tables2);
247      desc1 = tables1.descendingIterator();
248      desc2 = res2.descendingIterator();
249      while (desc1.hasNext()) {
250        assertEquals(desc1.next(), desc2.next());
251      }
252    }
253
254    cleanBackupTable();
255  }
256
257  @Test
258  public void testRegionServerLogTimestampMap() throws IOException {
259    TreeSet<TableName> tables = new TreeSet<>();
260
261    tables.add(TableName.valueOf("t1"));
262    tables.add(TableName.valueOf("t2"));
263    tables.add(TableName.valueOf("t3"));
264
265    HashMap<String, Long> rsTimestampMap = new HashMap<>();
266
267    rsTimestampMap.put("rs1:100", 100L);
268    rsTimestampMap.put("rs2:100", 101L);
269    rsTimestampMap.put("rs3:100", 103L);
270
271    // validate the prefix scan in readLogTimestampMap will get the right timestamps
272    // when a backup root with the same prefix is present
273    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
274    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root/backup");
275
276    Map<TableName, Map<String, Long>> result = table.readLogTimestampMap("root");
277
278    assertTrue(tables.size() == result.size());
279
280    for (TableName t : tables) {
281      Map<String, Long> rstm = result.get(t);
282      assertNotNull(rstm);
283      assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
284      assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
285      assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
286    }
287
288    Set<TableName> tables1 = new TreeSet<>();
289
290    tables1.add(TableName.valueOf("t3"));
291    tables1.add(TableName.valueOf("t4"));
292    tables1.add(TableName.valueOf("t5"));
293
294    HashMap<String, Long> rsTimestampMap1 = new HashMap<>();
295
296    rsTimestampMap1.put("rs1:100", 200L);
297    rsTimestampMap1.put("rs2:100", 201L);
298    rsTimestampMap1.put("rs3:100", 203L);
299
300    // validate the prefix scan in readLogTimestampMap will get the right timestamps
301    // when a backup root with the same prefix is present
302    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
303    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap, "root/backup");
304
305    result = table.readLogTimestampMap("root");
306
307    assertTrue(5 == result.size());
308
309    for (TableName t : tables) {
310      Map<String, Long> rstm = result.get(t);
311      assertNotNull(rstm);
312      if (t.equals(TableName.valueOf("t3")) == false) {
313        assertEquals(rstm.get("rs1:100"), Long.valueOf(100L));
314        assertEquals(rstm.get("rs2:100"), Long.valueOf(101L));
315        assertEquals(rstm.get("rs3:100"), Long.valueOf(103L));
316      } else {
317        assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
318        assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
319        assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
320      }
321    }
322
323    for (TableName t : tables1) {
324      Map<String, Long> rstm = result.get(t);
325      assertNotNull(rstm);
326      assertEquals(rstm.get("rs1:100"), Long.valueOf(200L));
327      assertEquals(rstm.get("rs2:100"), Long.valueOf(201L));
328      assertEquals(rstm.get("rs3:100"), Long.valueOf(203L));
329    }
330
331    cleanBackupTable();
332
333  }
334
335  /**
336   * Backup set tests
337   */
338
339  @Test
340  public void testBackupSetAddNotExists() throws IOException {
341    try (BackupSystemTable table = new BackupSystemTable(conn)) {
342
343      String[] tables = new String[] { "table1", "table2", "table3" };
344      String setName = "name";
345      table.addToBackupSet(setName, tables);
346      List<TableName> tnames = table.describeBackupSet(setName);
347      assertTrue(tnames != null);
348      assertTrue(tnames.size() == tables.length);
349      for (int i = 0; i < tnames.size(); i++) {
350        assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
351      }
352      cleanBackupTable();
353    }
354
355  }
356
357  @Test
358  public void testBackupSetAddExists() throws IOException {
359    try (BackupSystemTable table = new BackupSystemTable(conn)) {
360
361      String[] tables = new String[] { "table1", "table2", "table3" };
362      String setName = "name";
363      table.addToBackupSet(setName, tables);
364      String[] addTables = new String[] { "table4", "table5", "table6" };
365      table.addToBackupSet(setName, addTables);
366
367      Set<String> expectedTables =
368        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
369
370      List<TableName> tnames = table.describeBackupSet(setName);
371      assertTrue(tnames != null);
372      assertTrue(tnames.size() == expectedTables.size());
373      for (TableName tableName : tnames) {
374        assertTrue(expectedTables.remove(tableName.getNameAsString()));
375      }
376      cleanBackupTable();
377    }
378  }
379
380  @Test
381  public void testBackupSetAddExistsIntersects() throws IOException {
382    try (BackupSystemTable table = new BackupSystemTable(conn)) {
383
384      String[] tables = new String[] { "table1", "table2", "table3" };
385      String setName = "name";
386      table.addToBackupSet(setName, tables);
387      String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
388      table.addToBackupSet(setName, addTables);
389
390      Set<String> expectedTables =
391        new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6"));
392
393      List<TableName> tnames = table.describeBackupSet(setName);
394      assertTrue(tnames != null);
395      assertTrue(tnames.size() == expectedTables.size());
396      for (TableName tableName : tnames) {
397        assertTrue(expectedTables.remove(tableName.getNameAsString()));
398      }
399      cleanBackupTable();
400    }
401  }
402
403  @Test
404  public void testBackupSetRemoveSomeNotExists() throws IOException {
405    try (BackupSystemTable table = new BackupSystemTable(conn)) {
406
407      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
408      String setName = "name";
409      table.addToBackupSet(setName, tables);
410      String[] removeTables = new String[] { "table4", "table5", "table6" };
411      table.removeFromBackupSet(setName, removeTables);
412
413      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3"));
414
415      List<TableName> tnames = table.describeBackupSet(setName);
416      assertTrue(tnames != null);
417      assertTrue(tnames.size() == expectedTables.size());
418      for (TableName tableName : tnames) {
419        assertTrue(expectedTables.remove(tableName.getNameAsString()));
420      }
421      cleanBackupTable();
422    }
423  }
424
425  @Test
426  public void testBackupSetRemove() throws IOException {
427    try (BackupSystemTable table = new BackupSystemTable(conn)) {
428
429      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
430      String setName = "name";
431      table.addToBackupSet(setName, tables);
432      String[] removeTables = new String[] { "table4", "table3" };
433      table.removeFromBackupSet(setName, removeTables);
434
435      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2"));
436
437      List<TableName> tnames = table.describeBackupSet(setName);
438      assertTrue(tnames != null);
439      assertTrue(tnames.size() == expectedTables.size());
440      for (TableName tableName : tnames) {
441        assertTrue(expectedTables.remove(tableName.getNameAsString()));
442      }
443      cleanBackupTable();
444    }
445  }
446
447  @Test
448  public void testBackupSetDelete() throws IOException {
449    try (BackupSystemTable table = new BackupSystemTable(conn)) {
450
451      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
452      String setName = "name";
453      table.addToBackupSet(setName, tables);
454      table.deleteBackupSet(setName);
455
456      List<TableName> tnames = table.describeBackupSet(setName);
457      assertTrue(tnames == null);
458      cleanBackupTable();
459    }
460  }
461
462  @Test
463  public void testBackupSetList() throws IOException {
464    try (BackupSystemTable table = new BackupSystemTable(conn)) {
465
466      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
467      String setName1 = "name1";
468      String setName2 = "name2";
469      table.addToBackupSet(setName1, tables);
470      table.addToBackupSet(setName2, tables);
471
472      List<String> list = table.listBackupSets();
473
474      assertTrue(list.size() == 2);
475      assertTrue(list.get(0).equals(setName1));
476      assertTrue(list.get(1).equals(setName2));
477
478      cleanBackupTable();
479    }
480  }
481
482  private boolean compare(BackupInfo one, BackupInfo two) {
483    return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
484      && one.getBackupRootDir().equals(two.getBackupRootDir())
485      && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs();
486  }
487
488  private BackupInfo createBackupInfo() {
489    BackupInfo ctxt = new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL,
490      new TableName[] { TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
491      "/hbase/backup");
492    ctxt.setStartTs(EnvironmentEdgeManager.currentTime());
493    ctxt.setCompleteTs(EnvironmentEdgeManager.currentTime() + 1);
494    return ctxt;
495  }
496
497  private List<BackupInfo> createBackupInfoList(int size) throws InterruptedException {
498    List<BackupInfo> list = new ArrayList<>();
499    for (int i = 0; i < size; i++) {
500      list.add(createBackupInfo());
501      // XXX Why do we need this sleep?
502      Thread.sleep(10);
503    }
504    return list;
505  }
506
507  @AfterClass
508  public static void tearDown() throws IOException {
509    if (cluster != null) {
510      cluster.shutdown();
511    }
512  }
513}