001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.impl;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertThrows;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.io.IOException;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.testclassification.SmallTests;
036import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
037import org.junit.jupiter.api.AfterEach;
038import org.junit.jupiter.api.BeforeEach;
039import org.junit.jupiter.api.Tag;
040import org.junit.jupiter.api.Test;
041
042@Tag(SmallTests.TAG)
043public class TestFullTableBackupClientLogBoundaries {
044
045  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
046
047  private FileSystem fs;
048  private Path walRootDir;
049
050  @BeforeEach
051  public void setUp() throws IOException {
052    fs = TEST_UTIL.getTestFileSystem();
053    walRootDir = TEST_UTIL.getDataTestDirOnTestFS("walRoot");
054  }
055
056  @AfterEach
057  public void tearDown() throws IOException {
058    fs.delete(walRootDir, true);
059  }
060
061  @Test
062  public void testLiveServerWithoutRollResultIsCappedBelowItsOldestWAL() throws IOException {
063    ServerName joined = ServerName.valueOf("joined", 16020, 1L);
064    createWAL(walDir(joined), joined, 500);
065    createWAL(walDir(joined), joined, 600);
066    Path metaWAL =
067      new Path(walDir(joined), walName(joined, 450) + AbstractFSWALProvider.META_WAL_PROVIDER_ID);
068    fs.create(metaWAL).close();
069
070    assertEquals(Map.of("joined:16020", 499L), computeLogBoundaries(Map.of(), Set.of(joined)));
071  }
072
073  @Test
074  public void testDeadServerGetsItsNewestWAL() throws IOException {
075    ServerName dead = ServerName.valueOf("dead", 16020, 2L);
076    createWAL(new Path(walDir(dead).toString() + AbstractFSWALProvider.SPLITTING_EXT), dead, 300);
077    createWAL(new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME), dead, 350);
078
079    assertEquals(Map.of("dead:16020", 350L), computeLogBoundaries(Map.of(), Set.of()));
080  }
081
082  @Test
083  public void testRestartedServerCoversOldInstanceOnly() throws IOException {
084    ServerName oldInstance = ServerName.valueOf("restarted", 16020, 3L);
085    ServerName newInstance = ServerName.valueOf("restarted", 16020, 4L);
086    createWAL(walDir(oldInstance), oldInstance, 700);
087    createWAL(walDir(newInstance), newInstance, 800);
088
089    assertEquals(Map.of("restarted:16020", 700L),
090      computeLogBoundaries(Map.of(), Set.of(newInstance)));
091  }
092
093  @Test
094  public void testRolledServerKeepsItsRollResult() throws IOException {
095    ServerName rolled = ServerName.valueOf("rolled", 16020, 5L);
096    createWAL(walDir(rolled), rolled, 900);
097    createWAL(new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME), rolled, 950);
098
099    assertEquals(Map.of("rolled:16020", 850L),
100      computeLogBoundaries(Map.of("rolled:16020", 850L), Set.of(rolled)));
101  }
102
103  @Test
104  public void testServerRegisteringAfterTheListingGetsNoBoundary() throws IOException {
105    ServerName joined = ServerName.valueOf("joined", 16020, 6L);
106    ServerName late = ServerName.valueOf("late", 16020, 7L);
107    createWAL(walDir(joined), joined, 500);
108    Admin admin = mock(Admin.class);
109    when(admin.getRegionServers()).thenAnswer(invocation -> {
110      createWAL(walDir(late), late, 600);
111      return List.of(joined);
112    });
113
114    assertEquals(Map.of("joined:16020", 499L),
115      FullTableBackupClient.computeLogBoundaries(fs, walRootDir, Map.of(), admin));
116  }
117
118  @Test
119  public void testIncompleteLiveServerListFailsTheBackup() throws IOException {
120    ServerName rolled = ServerName.valueOf("rolled", 16020, 8L);
121    ServerName joined = ServerName.valueOf("joined", 16020, 9L);
122    createWAL(walDir(rolled), rolled, 900);
123    createWAL(walDir(joined), joined, 500);
124
125    assertThrows(IOException.class,
126      () -> computeLogBoundaries(Map.of("rolled:16020", 850L), Set.of(joined)));
127  }
128
129  private Map<String, Long> computeLogBoundaries(Map<String, Long> rolledHosts,
130    Set<ServerName> liveServers) throws IOException {
131    Admin admin = mock(Admin.class);
132    when(admin.getRegionServers()).thenReturn(liveServers);
133    return FullTableBackupClient.computeLogBoundaries(fs, walRootDir, rolledHosts, admin);
134  }
135
136  private Path walDir(ServerName serverName) {
137    return new Path(walRootDir, AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));
138  }
139
140  private void createWAL(Path dir, ServerName serverName, long ts) throws IOException {
141    fs.mkdirs(dir);
142    fs.create(new Path(dir, walName(serverName, ts))).close();
143  }
144
145  private static String walName(ServerName serverName, long ts) {
146    return serverName.toString().replace(",", "%2C") + "." + ts;
147  }
148}