001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collections;
028import java.util.HashMap;
029import java.util.HashSet;
030import java.util.Iterator;
031import java.util.List;
032import java.util.Map;
033import java.util.Set;
034import java.util.TreeSet;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtil;
038import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
041import org.apache.hadoop.hbase.backup.impl.BackupManager;
042import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.Connection;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054
055/**
056 * Test cases for backup system table API
057 */
058@Category(MediumTests.class)
059public class TestBackupSystemTable {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063      HBaseClassTestRule.forClass(TestBackupSystemTable.class);
064
065  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
066  protected static Configuration conf = UTIL.getConfiguration();
067  protected static SingleProcessHBaseCluster cluster;
068  protected static Connection conn;
069  protected BackupSystemTable table;
070
071  @BeforeClass
072  public static void setUp() throws Exception {
073    conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
074    BackupManager.decorateMasterConfiguration(conf);
075    BackupManager.decorateRegionServerConfiguration(conf);
076    cluster = UTIL.startMiniCluster();
077    conn = UTIL.getConnection();
078  }
079
080  @Before
081  public void before() throws IOException {
082    table = new BackupSystemTable(conn);
083  }
084
085  @After
086  public void after() {
087    if (table != null) {
088      table.close();
089    }
090
091  }
092
093  @Test
094  public void testUpdateReadDeleteBackupStatus() throws IOException {
095    BackupInfo ctx = createBackupInfo();
096    table.updateBackupInfo(ctx);
097    BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId());
098    assertTrue(compare(ctx, readCtx));
099    // try fake backup id
100    readCtx = table.readBackupInfo("fake");
101    assertNull(readCtx);
102    // delete backup info
103    table.deleteBackupInfo(ctx.getBackupId());
104    readCtx = table.readBackupInfo(ctx.getBackupId());
105    assertNull(readCtx);
106    cleanBackupTable();
107  }
108
109  @Test
110  public void testWriteReadBackupStartCode() throws IOException {
111    Long code = 100L;
112    table.writeBackupStartCode(code, "root");
113    String readCode = table.readBackupStartCode("root");
114    assertEquals(code, new Long(Long.parseLong(readCode)));
115    cleanBackupTable();
116  }
117
118  private void cleanBackupTable() throws IOException {
119    Admin admin = UTIL.getAdmin();
120    admin.disableTable(BackupSystemTable.getTableName(conf));
121    admin.truncateTable(BackupSystemTable.getTableName(conf), true);
122    if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) {
123      admin.enableTable(BackupSystemTable.getTableName(conf));
124    }
125  }
126
127  @Test
128  public void testBackupHistory() throws IOException {
129    int n = 10;
130    List<BackupInfo> list = createBackupInfoList(n);
131
132    // Load data
133    for (BackupInfo bc : list) {
134      // Make sure we set right status
135      bc.setState(BackupState.COMPLETE);
136      table.updateBackupInfo(bc);
137    }
138
139    // Reverse list for comparison
140    Collections.reverse(list);
141    List<BackupInfo> history = table.getBackupHistory();
142    assertTrue(history.size() == n);
143
144    for (int i = 0; i < n; i++) {
145      BackupInfo ctx = list.get(i);
146      BackupInfo data = history.get(i);
147      assertTrue(compare(ctx, data));
148    }
149
150    cleanBackupTable();
151
152  }
153
154  @Test
155  public void testBackupDelete() throws IOException {
156    try (BackupSystemTable table = new BackupSystemTable(conn)) {
157      int n = 10;
158      List<BackupInfo> list = createBackupInfoList(n);
159
160      // Load data
161      for (BackupInfo bc : list) {
162        // Make sure we set right status
163        bc.setState(BackupState.COMPLETE);
164        table.updateBackupInfo(bc);
165      }
166
167      // Verify exists
168      for (BackupInfo bc : list) {
169        assertNotNull(table.readBackupInfo(bc.getBackupId()));
170      }
171
172      // Delete all
173      for (BackupInfo bc : list) {
174        table.deleteBackupInfo(bc.getBackupId());
175      }
176
177      // Verify do not exists
178      for (BackupInfo bc : list) {
179        assertNull(table.readBackupInfo(bc.getBackupId()));
180      }
181
182      cleanBackupTable();
183    }
184
185  }
186
187  @Test
188  public void testRegionServerLastLogRollResults() throws IOException {
189    String[] servers = new String[] { "server1", "server2", "server3" };
190    Long[] timestamps = new Long[] { 100L, 102L, 107L };
191
192    for (int i = 0; i < servers.length; i++) {
193      table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root");
194    }
195
196    HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root");
197    assertTrue(servers.length == result.size());
198    Set<String> keys = result.keySet();
199    String[] keysAsArray = new String[keys.size()];
200    keys.toArray(keysAsArray);
201    Arrays.sort(keysAsArray);
202
203    for (int i = 0; i < keysAsArray.length; i++) {
204      assertEquals(keysAsArray[i], servers[i]);
205      Long ts1 = timestamps[i];
206      Long ts2 = result.get(keysAsArray[i]);
207      assertEquals(ts1, ts2);
208    }
209
210    cleanBackupTable();
211  }
212
213  @Test
214  public void testIncrementalBackupTableSet() throws IOException {
215    TreeSet<TableName> tables1 = new TreeSet<>();
216
217    tables1.add(TableName.valueOf("t1"));
218    tables1.add(TableName.valueOf("t2"));
219    tables1.add(TableName.valueOf("t3"));
220
221    TreeSet<TableName> tables2 = new TreeSet<>();
222
223    tables2.add(TableName.valueOf("t3"));
224    tables2.add(TableName.valueOf("t4"));
225    tables2.add(TableName.valueOf("t5"));
226
227    table.addIncrementalBackupTableSet(tables1, "root");
228    BackupSystemTable table = new BackupSystemTable(conn);
229    TreeSet<TableName> res1 = (TreeSet<TableName>) table.getIncrementalBackupTableSet("root");
230    assertTrue(tables1.size() == res1.size());
231    Iterator<TableName> desc1 = tables1.descendingIterator();
232    Iterator<TableName> desc2 = res1.descendingIterator();
233    while (desc1.hasNext()) {
234      assertEquals(desc1.next(), desc2.next());
235    }
236
237    table.addIncrementalBackupTableSet(tables2, "root");
238    TreeSet<TableName> res2 = (TreeSet<TableName>) table.getIncrementalBackupTableSet("root");
239    assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
240
241    tables1.addAll(tables2);
242
243    desc1 = tables1.descendingIterator();
244    desc2 = res2.descendingIterator();
245
246    while (desc1.hasNext()) {
247      assertEquals(desc1.next(), desc2.next());
248    }
249    cleanBackupTable();
250
251  }
252
253  @Test
254  public void testRegionServerLogTimestampMap() throws IOException {
255    TreeSet<TableName> tables = new TreeSet<>();
256
257    tables.add(TableName.valueOf("t1"));
258    tables.add(TableName.valueOf("t2"));
259    tables.add(TableName.valueOf("t3"));
260
261    HashMap<String, Long> rsTimestampMap = new HashMap<>();
262
263    rsTimestampMap.put("rs1:100", 100L);
264    rsTimestampMap.put("rs2:100", 101L);
265    rsTimestampMap.put("rs3:100", 103L);
266
267    table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root");
268
269    Map<TableName, Map<String, Long>> result = table.readLogTimestampMap("root");
270
271    assertTrue(tables.size() == result.size());
272
273    for (TableName t : tables) {
274      Map<String, Long> rstm = result.get(t);
275      assertNotNull(rstm);
276      assertEquals(rstm.get("rs1:100"), new Long(100L));
277      assertEquals(rstm.get("rs2:100"), new Long(101L));
278      assertEquals(rstm.get("rs3:100"), new Long(103L));
279    }
280
281    Set<TableName> tables1 = new TreeSet<>();
282
283    tables1.add(TableName.valueOf("t3"));
284    tables1.add(TableName.valueOf("t4"));
285    tables1.add(TableName.valueOf("t5"));
286
287    HashMap<String, Long> rsTimestampMap1 = new HashMap<>();
288
289    rsTimestampMap1.put("rs1:100", 200L);
290    rsTimestampMap1.put("rs2:100", 201L);
291    rsTimestampMap1.put("rs3:100", 203L);
292
293    table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root");
294
295    result = table.readLogTimestampMap("root");
296
297    assertTrue(5 == result.size());
298
299    for (TableName t : tables) {
300      Map<String, Long> rstm = result.get(t);
301      assertNotNull(rstm);
302      if (t.equals(TableName.valueOf("t3")) == false) {
303        assertEquals(rstm.get("rs1:100"), new Long(100L));
304        assertEquals(rstm.get("rs2:100"), new Long(101L));
305        assertEquals(rstm.get("rs3:100"), new Long(103L));
306      } else {
307        assertEquals(rstm.get("rs1:100"), new Long(200L));
308        assertEquals(rstm.get("rs2:100"), new Long(201L));
309        assertEquals(rstm.get("rs3:100"), new Long(203L));
310      }
311    }
312
313    for (TableName t : tables1) {
314      Map<String, Long> rstm = result.get(t);
315      assertNotNull(rstm);
316      assertEquals(rstm.get("rs1:100"), new Long(200L));
317      assertEquals(rstm.get("rs2:100"), new Long(201L));
318      assertEquals(rstm.get("rs3:100"), new Long(203L));
319    }
320
321    cleanBackupTable();
322
323  }
324
325  /**
326   * Backup set tests
327   */
328
329  @Test
330  public void testBackupSetAddNotExists() throws IOException {
331    try (BackupSystemTable table = new BackupSystemTable(conn)) {
332
333      String[] tables = new String[] { "table1", "table2", "table3" };
334      String setName = "name";
335      table.addToBackupSet(setName, tables);
336      List<TableName> tnames = table.describeBackupSet(setName);
337      assertTrue(tnames != null);
338      assertTrue(tnames.size() == tables.length);
339      for (int i = 0; i < tnames.size(); i++) {
340        assertTrue(tnames.get(i).getNameAsString().equals(tables[i]));
341      }
342      cleanBackupTable();
343    }
344
345  }
346
347  @Test
348  public void testBackupSetAddExists() throws IOException {
349    try (BackupSystemTable table = new BackupSystemTable(conn)) {
350
351      String[] tables = new String[] { "table1", "table2", "table3" };
352      String setName = "name";
353      table.addToBackupSet(setName, tables);
354      String[] addTables = new String[] { "table4", "table5", "table6" };
355      table.addToBackupSet(setName, addTables);
356
357      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3",
358        "table4", "table5", "table6"));
359
360      List<TableName> tnames = table.describeBackupSet(setName);
361      assertTrue(tnames != null);
362      assertTrue(tnames.size() == expectedTables.size());
363      for (TableName tableName : tnames) {
364        assertTrue(expectedTables.remove(tableName.getNameAsString()));
365      }
366      cleanBackupTable();
367    }
368  }
369
370  @Test
371  public void testBackupSetAddExistsIntersects() throws IOException {
372    try (BackupSystemTable table = new BackupSystemTable(conn)) {
373
374      String[] tables = new String[] { "table1", "table2", "table3" };
375      String setName = "name";
376      table.addToBackupSet(setName, tables);
377      String[] addTables = new String[] { "table3", "table4", "table5", "table6" };
378      table.addToBackupSet(setName, addTables);
379
380      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3",
381        "table4", "table5", "table6"));
382
383      List<TableName> tnames = table.describeBackupSet(setName);
384      assertTrue(tnames != null);
385      assertTrue(tnames.size() == expectedTables.size());
386      for (TableName tableName : tnames) {
387        assertTrue(expectedTables.remove(tableName.getNameAsString()));
388      }
389      cleanBackupTable();
390    }
391  }
392
393  @Test
394  public void testBackupSetRemoveSomeNotExists() throws IOException {
395    try (BackupSystemTable table = new BackupSystemTable(conn)) {
396
397      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
398      String setName = "name";
399      table.addToBackupSet(setName, tables);
400      String[] removeTables = new String[] { "table4", "table5", "table6" };
401      table.removeFromBackupSet(setName, removeTables);
402
403      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3"));
404
405      List<TableName> tnames = table.describeBackupSet(setName);
406      assertTrue(tnames != null);
407      assertTrue(tnames.size() == expectedTables.size());
408      for (TableName tableName : tnames) {
409        assertTrue(expectedTables.remove(tableName.getNameAsString()));
410      }
411      cleanBackupTable();
412    }
413  }
414
415  @Test
416  public void testBackupSetRemove() throws IOException {
417    try (BackupSystemTable table = new BackupSystemTable(conn)) {
418
419      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
420      String setName = "name";
421      table.addToBackupSet(setName, tables);
422      String[] removeTables = new String[] { "table4", "table3" };
423      table.removeFromBackupSet(setName, removeTables);
424
425      Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2"));
426
427      List<TableName> tnames = table.describeBackupSet(setName);
428      assertTrue(tnames != null);
429      assertTrue(tnames.size() == expectedTables.size());
430      for (TableName tableName : tnames) {
431        assertTrue(expectedTables.remove(tableName.getNameAsString()));
432      }
433      cleanBackupTable();
434    }
435  }
436
437  @Test
438  public void testBackupSetDelete() throws IOException {
439    try (BackupSystemTable table = new BackupSystemTable(conn)) {
440
441      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
442      String setName = "name";
443      table.addToBackupSet(setName, tables);
444      table.deleteBackupSet(setName);
445
446      List<TableName> tnames = table.describeBackupSet(setName);
447      assertTrue(tnames == null);
448      cleanBackupTable();
449    }
450  }
451
452  @Test
453  public void testBackupSetList() throws IOException {
454    try (BackupSystemTable table = new BackupSystemTable(conn)) {
455
456      String[] tables = new String[] { "table1", "table2", "table3", "table4" };
457      String setName1 = "name1";
458      String setName2 = "name2";
459      table.addToBackupSet(setName1, tables);
460      table.addToBackupSet(setName2, tables);
461
462      List<String> list = table.listBackupSets();
463
464      assertTrue(list.size() == 2);
465      assertTrue(list.get(0).equals(setName1));
466      assertTrue(list.get(1).equals(setName2));
467
468      cleanBackupTable();
469    }
470  }
471
472  private boolean compare(BackupInfo one, BackupInfo two) {
473    return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
474        && one.getBackupRootDir().equals(two.getBackupRootDir())
475        && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs();
476  }
477
478  private BackupInfo createBackupInfo() {
479    BackupInfo ctxt =
480        new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL, new TableName[] {
481            TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") },
482            "/hbase/backup");
483    ctxt.setStartTs(EnvironmentEdgeManager.currentTime());
484    ctxt.setCompleteTs(EnvironmentEdgeManager.currentTime() + 1);
485    return ctxt;
486  }
487
488  private List<BackupInfo> createBackupInfoList(int size) {
489    List<BackupInfo> list = new ArrayList<>();
490    for (int i = 0; i < size; i++) {
491      list.add(createBackupInfo());
492      try {
493        Thread.sleep(10);
494      } catch (InterruptedException e) {
495        e.printStackTrace();
496      }
497    }
498    return list;
499  }
500
501  @AfterClass
502  public static void tearDown() throws IOException {
503    if (cluster != null) {
504      cluster.shutdown();
505    }
506  }
507}