001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.backup; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertNotNull; 022import static org.junit.jupiter.api.Assertions.assertNull; 023import static org.junit.jupiter.api.Assertions.assertTrue; 024 025import java.io.IOException; 026import java.util.ArrayList; 027import java.util.Arrays; 028import java.util.Collections; 029import java.util.HashMap; 030import java.util.HashSet; 031import java.util.Iterator; 032import java.util.List; 033import java.util.Map; 034import java.util.Set; 035import java.util.TreeSet; 036import org.apache.hadoop.conf.Configuration; 037import org.apache.hadoop.hbase.HBaseTestingUtil; 038import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 039import org.apache.hadoop.hbase.TableName; 040import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; 041import org.apache.hadoop.hbase.backup.impl.BackupManager; 042import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; 043import org.apache.hadoop.hbase.client.Admin; 044import org.apache.hadoop.hbase.client.Connection; 045import org.apache.hadoop.hbase.testclassification.LargeTests; 046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 047import org.junit.jupiter.api.AfterAll; 048import org.junit.jupiter.api.AfterEach; 049import org.junit.jupiter.api.BeforeAll; 050import org.junit.jupiter.api.BeforeEach; 051import org.junit.jupiter.api.Tag; 052import org.junit.jupiter.api.Test; 053 054/** 055 * Test cases for backup system table API 056 */ 057@Tag(LargeTests.TAG) 058public class TestBackupSystemTable { 059 060 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 061 protected static Configuration conf = UTIL.getConfiguration(); 062 protected static SingleProcessHBaseCluster cluster; 063 protected static Connection conn; 064 protected BackupSystemTable table; 065 066 @BeforeAll 067 public static void setUp() throws Exception { 068 conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true); 069 BackupManager.decorateMasterConfiguration(conf); 070 BackupManager.decorateRegionServerConfiguration(conf); 071 cluster = UTIL.startMiniCluster(); 072 conn = UTIL.getConnection(); 073 } 074 075 @BeforeEach 076 public void before() throws IOException { 077 table = new BackupSystemTable(conn); 078 } 079 080 @AfterEach 081 public void after() { 082 if (table != null) { 083 table.close(); 084 } 085 086 } 087 088 @Test 089 public void testUpdateReadDeleteBackupStatus() throws IOException { 090 BackupInfo ctx = createBackupInfo(); 091 table.updateBackupInfo(ctx); 092 BackupInfo readCtx = table.readBackupInfo(ctx.getBackupId()); 093 assertTrue(compare(ctx, readCtx)); 094 // try fake backup id 095 readCtx = table.readBackupInfo("fake"); 096 assertNull(readCtx); 097 // delete backup info 098 table.deleteBackupInfo(ctx.getBackupId()); 099 readCtx = table.readBackupInfo(ctx.getBackupId()); 100 assertNull(readCtx); 101 cleanBackupTable(); 102 } 103 104 private void cleanBackupTable() throws IOException { 105 Admin admin = UTIL.getAdmin(); 106 admin.disableTable(BackupSystemTable.getTableName(conf)); 107 admin.truncateTable(BackupSystemTable.getTableName(conf), true); 108 if (admin.isTableDisabled(BackupSystemTable.getTableName(conf))) { 109 admin.enableTable(BackupSystemTable.getTableName(conf)); 110 } 111 } 112 113 @Test 114 public void testBackupHistory() throws Exception { 115 int n = 10; 116 List<BackupInfo> list = createBackupInfoList(n); 117 118 // Load data 119 for (BackupInfo bc : list) { 120 // Make sure we set right status 121 bc.setState(BackupState.COMPLETE); 122 table.updateBackupInfo(bc); 123 } 124 125 // Reverse list for comparison 126 Collections.reverse(list); 127 List<BackupInfo> history = table.getBackupHistory(); 128 assertTrue(history.size() == n); 129 130 for (int i = 0; i < n; i++) { 131 BackupInfo ctx = list.get(i); 132 BackupInfo data = history.get(i); 133 assertTrue(compare(ctx, data)); 134 } 135 136 cleanBackupTable(); 137 138 } 139 140 @Test 141 public void testBackupDelete() throws Exception { 142 try (BackupSystemTable table = new BackupSystemTable(conn)) { 143 int n = 10; 144 List<BackupInfo> list = createBackupInfoList(n); 145 146 // Load data 147 for (BackupInfo bc : list) { 148 // Make sure we set right status 149 bc.setState(BackupState.COMPLETE); 150 table.updateBackupInfo(bc); 151 } 152 153 // Verify exists 154 for (BackupInfo bc : list) { 155 assertNotNull(table.readBackupInfo(bc.getBackupId())); 156 } 157 158 // Delete all 159 for (BackupInfo bc : list) { 160 table.deleteBackupInfo(bc.getBackupId()); 161 } 162 163 // Verify do not exists 164 for (BackupInfo bc : list) { 165 assertNull(table.readBackupInfo(bc.getBackupId())); 166 } 167 168 cleanBackupTable(); 169 } 170 171 } 172 173 @Test 174 public void testRegionServerLastLogRollResults() throws IOException { 175 String[] servers = new String[] { "server1", "server2", "server3" }; 176 Long[] timestamps = new Long[] { 100L, 102L, 107L }; 177 178 // validate the prefix scan in readRegionServerlastLogRollResult will get the right timestamps 179 // when a backup root with the same prefix is present 180 for (int i = 0; i < servers.length; i++) { 181 table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root"); 182 table.writeRegionServerLastLogRollResult(servers[i], timestamps[i], "root/backup"); 183 } 184 185 HashMap<String, Long> result = table.readRegionServerLastLogRollResult("root"); 186 assertTrue(servers.length == result.size()); 187 Set<String> keys = result.keySet(); 188 String[] keysAsArray = new String[keys.size()]; 189 keys.toArray(keysAsArray); 190 Arrays.sort(keysAsArray); 191 192 for (int i = 0; i < keysAsArray.length; i++) { 193 assertEquals(keysAsArray[i], servers[i]); 194 Long ts1 = timestamps[i]; 195 Long ts2 = result.get(keysAsArray[i]); 196 assertEquals(ts1, ts2); 197 } 198 199 cleanBackupTable(); 200 } 201 202 @Test 203 public void testMixedCaseHostKeysAreReadLowercase() throws IOException { 204 table.writeRegionServerLastLogRollResult("Host-A:1", 102L, "root"); 205 table.writeRegionServerLastLogRollResult("host-a:1", 100L, "root"); 206 assertEquals(Map.of("host-a:1", 102L), table.readRegionServerLastLogRollResult("root")); 207 208 TableName t1 = TableName.valueOf("t1"); 209 table.writeRegionServerLogTimestamp(Set.of(t1), Map.of("Host-A:1", 100L), "root"); 210 assertEquals(Map.of(t1, Map.of("host-a:1", 100L)), table.readLogTimestampMap("root")); 211 212 cleanBackupTable(); 213 } 214 215 @Test 216 public void testIncrementalBackupTableSet() throws IOException { 217 TreeSet<TableName> tables1 = new TreeSet<>(); 218 219 tables1.add(TableName.valueOf("t1")); 220 tables1.add(TableName.valueOf("t2")); 221 tables1.add(TableName.valueOf("t3")); 222 223 TreeSet<TableName> tables2 = new TreeSet<>(); 224 225 tables2.add(TableName.valueOf("t3")); 226 tables2.add(TableName.valueOf("t4")); 227 tables2.add(TableName.valueOf("t5")); 228 229 table.addIncrementalBackupTableSet(tables1, "root"); 230 231 try (BackupSystemTable systemTable = new BackupSystemTable(conn)) { 232 TreeSet<TableName> res1 = 233 (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root"); 234 assertTrue(tables1.size() == res1.size()); 235 Iterator<TableName> desc1 = tables1.descendingIterator(); 236 Iterator<TableName> desc2 = res1.descendingIterator(); 237 while (desc1.hasNext()) { 238 assertEquals(desc1.next(), desc2.next()); 239 } 240 systemTable.addIncrementalBackupTableSet(tables2, "root"); 241 TreeSet<TableName> res2 = 242 (TreeSet<TableName>) systemTable.getIncrementalBackupTableSet("root"); 243 assertTrue((tables2.size() + tables1.size() - 1) == res2.size()); 244 tables1.addAll(tables2); 245 desc1 = tables1.descendingIterator(); 246 desc2 = res2.descendingIterator(); 247 while (desc1.hasNext()) { 248 assertEquals(desc1.next(), desc2.next()); 249 } 250 } 251 252 cleanBackupTable(); 253 } 254 255 @Test 256 public void testRegionServerLogTimestampMap() throws IOException { 257 TreeSet<TableName> tables = new TreeSet<>(); 258 259 tables.add(TableName.valueOf("t1")); 260 tables.add(TableName.valueOf("t2")); 261 tables.add(TableName.valueOf("t3")); 262 263 HashMap<String, Long> rsTimestampMap = new HashMap<>(); 264 265 rsTimestampMap.put("rs1:100", 100L); 266 rsTimestampMap.put("rs2:100", 101L); 267 rsTimestampMap.put("rs3:100", 103L); 268 269 // validate the prefix scan in readLogTimestampMap will get the right timestamps 270 // when a backup root with the same prefix is present 271 table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root"); 272 table.writeRegionServerLogTimestamp(tables, rsTimestampMap, "root/backup"); 273 274 Map<TableName, Map<String, Long>> result = table.readLogTimestampMap("root"); 275 276 assertTrue(tables.size() == result.size()); 277 278 for (TableName t : tables) { 279 Map<String, Long> rstm = result.get(t); 280 assertNotNull(rstm); 281 assertEquals(rstm.get("rs1:100"), Long.valueOf(100L)); 282 assertEquals(rstm.get("rs2:100"), Long.valueOf(101L)); 283 assertEquals(rstm.get("rs3:100"), Long.valueOf(103L)); 284 } 285 286 Set<TableName> tables1 = new TreeSet<>(); 287 288 tables1.add(TableName.valueOf("t3")); 289 tables1.add(TableName.valueOf("t4")); 290 tables1.add(TableName.valueOf("t5")); 291 292 HashMap<String, Long> rsTimestampMap1 = new HashMap<>(); 293 294 rsTimestampMap1.put("rs1:100", 200L); 295 rsTimestampMap1.put("rs2:100", 201L); 296 rsTimestampMap1.put("rs3:100", 203L); 297 298 // validate the prefix scan in readLogTimestampMap will get the right timestamps 299 // when a backup root with the same prefix is present 300 table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1, "root"); 301 table.writeRegionServerLogTimestamp(tables1, rsTimestampMap, "root/backup"); 302 303 result = table.readLogTimestampMap("root"); 304 305 assertTrue(5 == result.size()); 306 307 for (TableName t : tables) { 308 Map<String, Long> rstm = result.get(t); 309 assertNotNull(rstm); 310 if (t.equals(TableName.valueOf("t3")) == false) { 311 assertEquals(rstm.get("rs1:100"), Long.valueOf(100L)); 312 assertEquals(rstm.get("rs2:100"), Long.valueOf(101L)); 313 assertEquals(rstm.get("rs3:100"), Long.valueOf(103L)); 314 } else { 315 assertEquals(rstm.get("rs1:100"), Long.valueOf(200L)); 316 assertEquals(rstm.get("rs2:100"), Long.valueOf(201L)); 317 assertEquals(rstm.get("rs3:100"), Long.valueOf(203L)); 318 } 319 } 320 321 for (TableName t : tables1) { 322 Map<String, Long> rstm = result.get(t); 323 assertNotNull(rstm); 324 assertEquals(rstm.get("rs1:100"), Long.valueOf(200L)); 325 assertEquals(rstm.get("rs2:100"), Long.valueOf(201L)); 326 assertEquals(rstm.get("rs3:100"), Long.valueOf(203L)); 327 } 328 329 cleanBackupTable(); 330 331 } 332 333 /** 334 * Backup set tests 335 */ 336 337 @Test 338 public void testBackupSetAddNotExists() throws IOException { 339 try (BackupSystemTable table = new BackupSystemTable(conn)) { 340 341 String[] tables = new String[] { "table1", "table2", "table3" }; 342 String setName = "name"; 343 table.addToBackupSet(setName, tables); 344 List<TableName> tnames = table.describeBackupSet(setName); 345 assertTrue(tnames != null); 346 assertTrue(tnames.size() == tables.length); 347 for (int i = 0; i < tnames.size(); i++) { 348 assertTrue(tnames.get(i).getNameAsString().equals(tables[i])); 349 } 350 cleanBackupTable(); 351 } 352 353 } 354 355 @Test 356 public void testBackupSetAddExists() throws IOException { 357 try (BackupSystemTable table = new BackupSystemTable(conn)) { 358 359 String[] tables = new String[] { "table1", "table2", "table3" }; 360 String setName = "name"; 361 table.addToBackupSet(setName, tables); 362 String[] addTables = new String[] { "table4", "table5", "table6" }; 363 table.addToBackupSet(setName, addTables); 364 365 Set<String> expectedTables = 366 new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6")); 367 368 List<TableName> tnames = table.describeBackupSet(setName); 369 assertTrue(tnames != null); 370 assertTrue(tnames.size() == expectedTables.size()); 371 for (TableName tableName : tnames) { 372 assertTrue(expectedTables.remove(tableName.getNameAsString())); 373 } 374 cleanBackupTable(); 375 } 376 } 377 378 @Test 379 public void testBackupSetAddExistsIntersects() throws IOException { 380 try (BackupSystemTable table = new BackupSystemTable(conn)) { 381 382 String[] tables = new String[] { "table1", "table2", "table3" }; 383 String setName = "name"; 384 table.addToBackupSet(setName, tables); 385 String[] addTables = new String[] { "table3", "table4", "table5", "table6" }; 386 table.addToBackupSet(setName, addTables); 387 388 Set<String> expectedTables = 389 new HashSet<>(Arrays.asList("table1", "table2", "table3", "table4", "table5", "table6")); 390 391 List<TableName> tnames = table.describeBackupSet(setName); 392 assertTrue(tnames != null); 393 assertTrue(tnames.size() == expectedTables.size()); 394 for (TableName tableName : tnames) { 395 assertTrue(expectedTables.remove(tableName.getNameAsString())); 396 } 397 cleanBackupTable(); 398 } 399 } 400 401 @Test 402 public void testBackupSetRemoveSomeNotExists() throws IOException { 403 try (BackupSystemTable table = new BackupSystemTable(conn)) { 404 405 String[] tables = new String[] { "table1", "table2", "table3", "table4" }; 406 String setName = "name"; 407 table.addToBackupSet(setName, tables); 408 String[] removeTables = new String[] { "table4", "table5", "table6" }; 409 table.removeFromBackupSet(setName, removeTables); 410 411 Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2", "table3")); 412 413 List<TableName> tnames = table.describeBackupSet(setName); 414 assertTrue(tnames != null); 415 assertTrue(tnames.size() == expectedTables.size()); 416 for (TableName tableName : tnames) { 417 assertTrue(expectedTables.remove(tableName.getNameAsString())); 418 } 419 cleanBackupTable(); 420 } 421 } 422 423 @Test 424 public void testBackupSetRemove() throws IOException { 425 try (BackupSystemTable table = new BackupSystemTable(conn)) { 426 427 String[] tables = new String[] { "table1", "table2", "table3", "table4" }; 428 String setName = "name"; 429 table.addToBackupSet(setName, tables); 430 String[] removeTables = new String[] { "table4", "table3" }; 431 table.removeFromBackupSet(setName, removeTables); 432 433 Set<String> expectedTables = new HashSet<>(Arrays.asList("table1", "table2")); 434 435 List<TableName> tnames = table.describeBackupSet(setName); 436 assertTrue(tnames != null); 437 assertTrue(tnames.size() == expectedTables.size()); 438 for (TableName tableName : tnames) { 439 assertTrue(expectedTables.remove(tableName.getNameAsString())); 440 } 441 cleanBackupTable(); 442 } 443 } 444 445 @Test 446 public void testBackupSetDelete() throws IOException { 447 try (BackupSystemTable table = new BackupSystemTable(conn)) { 448 449 String[] tables = new String[] { "table1", "table2", "table3", "table4" }; 450 String setName = "name"; 451 table.addToBackupSet(setName, tables); 452 table.deleteBackupSet(setName); 453 454 List<TableName> tnames = table.describeBackupSet(setName); 455 assertTrue(tnames == null); 456 cleanBackupTable(); 457 } 458 } 459 460 @Test 461 public void testBackupSetList() throws IOException { 462 try (BackupSystemTable table = new BackupSystemTable(conn)) { 463 464 String[] tables = new String[] { "table1", "table2", "table3", "table4" }; 465 String setName1 = "name1"; 466 String setName2 = "name2"; 467 table.addToBackupSet(setName1, tables); 468 table.addToBackupSet(setName2, tables); 469 470 List<String> list = table.listBackupSets(); 471 472 assertTrue(list.size() == 2); 473 assertTrue(list.get(0).equals(setName1)); 474 assertTrue(list.get(1).equals(setName2)); 475 476 cleanBackupTable(); 477 } 478 } 479 480 private boolean compare(BackupInfo one, BackupInfo two) { 481 return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType()) 482 && one.getBackupRootDir().equals(two.getBackupRootDir()) 483 && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs(); 484 } 485 486 private BackupInfo createBackupInfo() { 487 BackupInfo ctxt = new BackupInfo("backup_" + System.nanoTime(), BackupType.FULL, 488 new TableName[] { TableName.valueOf("t1"), TableName.valueOf("t2"), TableName.valueOf("t3") }, 489 "/hbase/backup"); 490 ctxt.setStartTs(EnvironmentEdgeManager.currentTime()); 491 ctxt.setCompleteTs(EnvironmentEdgeManager.currentTime() + 1); 492 return ctxt; 493 } 494 495 private List<BackupInfo> createBackupInfoList(int size) throws InterruptedException { 496 List<BackupInfo> list = new ArrayList<>(); 497 for (int i = 0; i < size; i++) { 498 list.add(createBackupInfo()); 499 // XXX Why do we need this sleep? 500 Thread.sleep(10); 501 } 502 return list; 503 } 504 505 @AfterAll 506 public static void tearDown() throws IOException { 507 if (cluster != null) { 508 cluster.shutdown(); 509 } 510 } 511}