001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.security.PrivilegedAction;
027import java.time.Instant;
028import java.time.ZoneId;
029import java.util.ArrayList;
030import java.util.List;
031import java.util.TimeZone;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.backup.util.BackupUtils;
039import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Addressing;
042import org.apache.hadoop.hbase.util.CommonFSUtils;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.apache.hadoop.security.UserGroupInformation;
045import org.junit.jupiter.api.BeforeAll;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.Test;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Tag(SmallTests.TAG)
052public class TestBackupUtils {
053
054  private static final Logger LOG = LoggerFactory.getLogger(TestBackupUtils.class);
055
056  protected static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
057  protected static Configuration conf = TEST_UTIL.getConfiguration();
058
059  private static FileSystem dummyFs;
060  private static Path backupRootDir;
061
062  @BeforeAll
063  public static void setUp() throws IOException {
064    dummyFs = TEST_UTIL.getTestFileSystem();
065    backupRootDir = TEST_UTIL.getDataTestDirOnTestFS("backupUT");
066  }
067
068  @Test
069  public void testGetBulkOutputDir() {
070    // Create a user who is not the current user
071    String fooUserName = "foo1234";
072    String fooGroupName = "group1";
073    UserGroupInformation ugi =
074      UserGroupInformation.createUserForTesting(fooUserName, new String[] { fooGroupName });
075    // Get user's home directory
076    Path fooHomeDirectory = ugi.doAs(new PrivilegedAction<Path>() {
077      @Override
078      public Path run() {
079        try (FileSystem fs = FileSystem.get(conf)) {
080          return fs.getHomeDirectory();
081        } catch (IOException ioe) {
082          LOG.error("Failed to get foo's home directory", ioe);
083        }
084        return null;
085      }
086    });
087
088    Path bulkOutputDir = ugi.doAs(new PrivilegedAction<Path>() {
089      @Override
090      public Path run() {
091        try {
092          return BackupUtils.getBulkOutputDir("test", conf, false);
093        } catch (IOException ioe) {
094          LOG.error("Failed to get bulk output dir path", ioe);
095        }
096        return null;
097      }
098    });
099    // Make sure the directory is in foo1234's home directory
100    assertTrue(bulkOutputDir.toString().startsWith(fooHomeDirectory.toString()));
101  }
102
103  @Test
104  public void testFilesystemWalHostNameParsing() throws IOException {
105    String[] hosts =
106      new String[] { "10.20.30.40", "127.0.0.1", "localhost", "a-region-server.domain.com" };
107
108    Path walRootDir = CommonFSUtils.getWALRootDir(conf);
109    Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
110
111    int port = 60030;
112    for (String host : hosts) {
113      ServerName serverName = ServerName.valueOf(host, port, 1234);
114
115      Path testOldWalPath = new Path(oldLogDir,
116        serverName + BackupUtils.LOGNAME_SEPARATOR + EnvironmentEdgeManager.currentTime());
117      assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port,
118        BackupUtils.parseHostFromOldLog(testOldWalPath));
119
120      Path testMasterWalPath =
121        new Path(oldLogDir, testOldWalPath.getName() + MasterRegionFactory.ARCHIVED_WAL_SUFFIX);
122      assertNull(BackupUtils.parseHostFromOldLog(testMasterWalPath));
123
124      // org.apache.hadoop.hbase.wal.BoundedGroupingStrategy does this
125      Path testOldWalWithRegionGroupingPath = new Path(oldLogDir,
126        serverName + BackupUtils.LOGNAME_SEPARATOR + serverName + BackupUtils.LOGNAME_SEPARATOR
127          + "regiongroup-0" + BackupUtils.LOGNAME_SEPARATOR + EnvironmentEdgeManager.currentTime());
128      assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port,
129        BackupUtils.parseHostFromOldLog(testOldWalWithRegionGroupingPath));
130
131      Path testOldWalInServerDirPath = new Path(new Path(oldLogDir, serverName.toString()),
132        "wal" + BackupUtils.LOGNAME_SEPARATOR + EnvironmentEdgeManager.currentTime());
133      assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port,
134        BackupUtils.parseHostFromOldLog(testOldWalInServerDirPath));
135    }
136  }
137
138  // Ensure getValidWalDirs() uses UTC timestamps regardless of what time zone the test is run in.
139  @Test
140  public void testGetValidWalDirForAllTimeZonesSingleDay() throws IOException {
141    // This UTC test time is a time when it is still "yesterday" in other time zones (such as PST)
142    List<String> walDateDirs = List.of("2026-01-23");
143    Path walDir = new Path(backupRootDir, "WALs");
144
145    // 10-minute window in UTC between start and end time
146    long startTime = Instant.parse("2026-01-23T01:00:00Z").toEpochMilli();
147    long endTime = startTime + (10 * 60 * 1000);
148
149    testGetValidWalDirs(startTime, endTime, walDir, walDateDirs, 1, walDateDirs);
150  }
151
152  // Ensure getValidWalDirs() works as expected for time ranges across multiple days for all time
153  // zones
154  @Test
155  public void testGetValidWalDirsForAllTimeZonesMultiDay() throws IOException {
156    List<String> walDateDirs = List.of("2025-12-30", "2025-12-31", "2026-01-01", "2026-01-02");
157    List<String> expectedValidWalDirs = List.of("2025-12-31", "2026-01-01");
158    Path walDir = new Path(backupRootDir, "WALs");
159
160    // 10-minute window in UTC between start and end time that spans over two days
161    long startTime = Instant.parse("2025-12-31T23:55:00Z").toEpochMilli();
162    long endTime = Instant.parse("2026-01-01T00:05:00Z").toEpochMilli();
163
164    testGetValidWalDirs(startTime, endTime, walDir, walDateDirs, 2, expectedValidWalDirs);
165  }
166
167  @Test
168  public void testGetValidWalDirExactlyMidnightUTC() throws IOException {
169    List<String> walDateDirs = List.of("2026-01-23");
170    Path walDir = new Path(backupRootDir, "WALs");
171    // This instant is UTC
172    long startAndEndTime = Instant.parse("2026-01-23T00:00:00.000Z").toEpochMilli();
173
174    testGetValidWalDirs(startAndEndTime, startAndEndTime, walDir, walDateDirs, 1, walDateDirs);
175  }
176
177  @Test
178  public void testGetValidWalDirOneMsBeforeMidnightUTC() throws IOException {
179    List<String> walDateDirs = List.of("2026-01-23");
180    Path walDir = new Path(backupRootDir, "WALs");
181    // This instant is UTC
182    long startAndEndTime = Instant.parse("2026-01-23T23:59:59.999Z").toEpochMilli();
183
184    testGetValidWalDirs(startAndEndTime, startAndEndTime, walDir, walDateDirs, 1, walDateDirs);
185  }
186
187  protected void testGetValidWalDirs(long startTime, long endTime, Path walDir,
188    List<String> availableWalDateDirs, int numExpectedValidWalDirs,
189    List<String> expectedValidWalDirs) throws IOException {
190    TimeZone defaultTimeZone = TimeZone.getDefault();
191    try {
192      // This UTC test time is a time when it is still "yesterday" in other time zones (such as PST)
193      for (String dirName : availableWalDateDirs) {
194        dummyFs.mkdirs(new Path(walDir, dirName));
195      }
196
197      // Ensure we can get valid WAL dirs regardless of the test environment's time zone
198      for (String timeZone : ZoneId.getAvailableZoneIds()) {
199        // Force test environment to use specified time zone
200        TimeZone.setDefault(TimeZone.getTimeZone(timeZone));
201
202        List<String> validWalDirs = BackupUtils.getValidWalDirs(TEST_UTIL.getConfiguration(),
203          backupRootDir, startTime, endTime);
204
205        // Verify the correct number of valid WAL dirs was found
206        assertEquals(numExpectedValidWalDirs, validWalDirs.size(),
207          "The number of valid WAL dirs should be " + numExpectedValidWalDirs + " for time zone "
208            + timeZone);
209
210        // Verify the list of valid WAL dirs is as expected
211        for (String dirName : expectedValidWalDirs) {
212          assertTrue(validWalDirs.stream().anyMatch(path -> path.endsWith("/" + dirName)),
213            "Expected " + dirName + " to be a valid WAL dir");
214        }
215
216        // Verify the list of valid WAL dirs does not contain anything expected to be invalid
217        List<String> expectedInvalidWalDirs = new ArrayList<>(availableWalDateDirs);
218        expectedInvalidWalDirs.removeAll(expectedValidWalDirs);
219        for (String dirName : expectedInvalidWalDirs) {
220          assertFalse(validWalDirs.contains(dirName),
221            "Expected " + dirName + " to NOT be a valid WAL dir");
222        }
223      }
224    } finally {
225      TimeZone.setDefault(defaultTimeZone);
226      dummyFs.delete(walDir, true);
227    }
228  }
229}