001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collections;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map.Entry;
031import java.util.Set;
032import java.util.concurrent.atomic.AtomicLong;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellScanner;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.Get;
044import org.apache.hadoop.hbase.client.Result;
045import org.apache.hadoop.hbase.client.ResultScanner;
046import org.apache.hadoop.hbase.client.Scan;
047import org.apache.hadoop.hbase.client.SnapshotDescription;
048import org.apache.hadoop.hbase.client.SnapshotType;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.client.TableDescriptor;
051import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
052import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
053import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
054import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
055import org.apache.hadoop.hbase.testclassification.MediumTests;
056import org.apache.hadoop.hbase.util.CommonFSUtils;
057import org.junit.jupiter.api.AfterAll;
058import org.junit.jupiter.api.BeforeAll;
059import org.junit.jupiter.api.BeforeEach;
060import org.junit.jupiter.api.Tag;
061import org.junit.jupiter.api.Test;
062import org.junit.jupiter.api.TestInfo;
063
064import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
065import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
066import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
067
068import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
069import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
070import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
071import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
072
073/**
074 * Test class for {@link FileArchiverNotifierImpl}.
075 */
076@Tag(MediumTests.TAG)
077public class TestFileArchiverNotifierImpl {
078  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
079  private static final AtomicLong COUNTER = new AtomicLong();
080
081  private Connection conn;
082  private Admin admin;
083  private SpaceQuotaHelperForTests helper;
084  private FileSystem fs;
085  private Configuration conf;
086  private String testName;
087
088  @BeforeAll
089  public static void setUp() throws Exception {
090    Configuration conf = TEST_UTIL.getConfiguration();
091    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
092    // Clean up the compacted files faster than normal (15s instead of 2mins)
093    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
094    // Prevent the SnapshotQuotaObserverChore from running
095    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
096    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
097    TEST_UTIL.startMiniCluster(1);
098  }
099
100  @AfterAll
101  public static void tearDown() throws Exception {
102    TEST_UTIL.shutdownMiniCluster();
103  }
104
105  @BeforeEach
106  public void setup(TestInfo testInfo) throws Exception {
107    testName = testInfo.getTestMethod().get().getName();
108    conn = TEST_UTIL.getConnection();
109    admin = TEST_UTIL.getAdmin();
110    helper = new SpaceQuotaHelperForTests(TEST_UTIL, () -> testName, COUNTER);
111    helper.removeAllQuotas(conn);
112    fs = TEST_UTIL.getTestFileSystem();
113    conf = TEST_UTIL.getConfiguration();
114  }
115
116  @Test
117  public void testSnapshotSizePersistence() throws IOException {
118    final Admin admin = TEST_UTIL.getAdmin();
119    final TableName tn = TableName.valueOf(testName);
120    if (admin.tableExists(tn)) {
121      admin.disableTable(tn);
122      admin.deleteTable(tn);
123    }
124    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn)
125      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
126    admin.createTable(desc);
127
128    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
129    List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
130    try (Table table = conn.getTable(tn)) {
131      // Writing no values will result in no records written.
132      verify(table, () -> {
133        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
134        assertEquals(0, count(table));
135      });
136
137      verify(table, () -> {
138        snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
139        snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
140        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
141        assertEquals(2, count(table));
142        assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
143        assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
144      });
145    }
146  }
147
148  @Test
149  public void testIncrementalFileArchiving() throws Exception {
150    final Admin admin = TEST_UTIL.getAdmin();
151    final TableName tn = TableName.valueOf(testName);
152    if (admin.tableExists(tn)) {
153      admin.disableTable(tn);
154      admin.deleteTable(tn);
155    }
156    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
157    final TableName tn1 = helper.createTableWithRegions(1);
158    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
159      SpaceViolationPolicy.NO_INSERTS));
160
161    // Write some data and flush it
162    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
163    admin.flush(tn1);
164
165    // Create a snapshot on the table
166    final String snapshotName1 = tn1 + "snapshot1";
167    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
168
169    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
170    long t1 = notifier.getLastFullCompute();
171    long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
172    assertEquals(0, snapshotSize, "The size of the snapshots should be zero");
173    assertTrue(t1 < notifier.getLastFullCompute(),
174      "Last compute time was not less than current compute time");
175
176    // No recently archived files and the snapshot should have no size
177    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
178
179    // Invoke the addArchivedFiles method with no files
180    notifier.addArchivedFiles(Collections.emptySet());
181
182    // The size should not have changed
183    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
184
185    notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
186
187    // The size should not have changed
188    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
189
190    // Pull one file referenced by the snapshot out of the manifest
191    Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
192    assertTrue(referencedFiles.size() >= 1, "Found snapshot referenced files: " + referencedFiles);
193    String referencedFile = Iterables.getFirst(referencedFiles, null);
194    assertNotNull(referencedFile);
195
196    // Report that a file this snapshot referenced was moved to the archive. This is a sign
197    // that the snapshot should now "own" the size of this file
198    final long fakeFileSize = 2048L;
199    notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
200
201    // Verify that the snapshot owns this file.
202    assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
203
204    // In reality, we did not actually move the file, so a "full" computation should re-set the
205    // size of the snapshot back to 0.
206    long t2 = notifier.getLastFullCompute();
207    snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
208    assertEquals(0, snapshotSize);
209    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
210    // We should also have no recently archived files after a re-computation
211    assertTrue(t2 < notifier.getLastFullCompute(),
212      "Last compute time was not less than current compute time");
213  }
214
215  @Test
216  public void testParseOldNamespaceSnapshotSize() throws Exception {
217    final Admin admin = TEST_UTIL.getAdmin();
218    final TableName fakeQuotaTableName = TableName.valueOf(testName);
219    final TableName tn = TableName.valueOf(testName + "1");
220    if (admin.tableExists(fakeQuotaTableName)) {
221      admin.disableTable(fakeQuotaTableName);
222      admin.deleteTable(fakeQuotaTableName);
223    }
224    TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName)
225      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
226      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
227    admin.createTable(desc);
228
229    final String ns = "";
230    try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
231      FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
232      // Verify no record is treated as zero
233      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
234
235      // Set an explicit value of zero
236      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
237      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
238
239      // Set a non-zero value
240      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
241      assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
242    }
243  }
244
245  private long count(Table t) throws IOException {
246    try (ResultScanner rs = t.getScanner(new Scan())) {
247      long sum = 0;
248      for (Result r : rs) {
249        while (r.advance()) {
250          sum++;
251        }
252      }
253      return sum;
254    }
255  }
256
257  private long extractSnapshotSize(Table quotaTable, TableName tn, String snapshot)
258    throws IOException {
259    Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
260    Result r = quotaTable.get(g);
261    assertNotNull(r);
262    CellScanner cs = r.cellScanner();
263    assertTrue(cs.advance());
264    Cell c = cs.current();
265    assertNotNull(c);
266    return QuotaTableUtil.extractSnapshotSize(c.getValueArray(), c.getValueOffset(),
267      c.getValueLength());
268  }
269
270  private void verify(Table t, IOThrowingRunnable test) throws IOException {
271    admin.disableTable(t.getName());
272    admin.truncateTable(t.getName(), false);
273    test.run();
274  }
275
276  @FunctionalInterface
277  private interface IOThrowingRunnable {
278    void run() throws IOException;
279  }
280
281  private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
282    HashSet<String> files = new HashSet<>();
283    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
284      CommonFSUtils.getRootDir(conf));
285    SnapshotProtos.SnapshotDescription sd =
286      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
287    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
288    // For each region referenced by the snapshot
289    for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
290      // For each column family in this region
291      for (FamilyFiles ff : rm.getFamilyFilesList()) {
292        // And each store file in that family
293        for (StoreFile sf : ff.getStoreFilesList()) {
294          files.add(sf.getName());
295        }
296      }
297    }
298    return files;
299  }
300
301  private <K, V> Entry<K, V> entry(K k, V v) {
302    return Maps.immutableEntry(k, v);
303  }
304}