001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Collections;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map.Entry;
031import java.util.Set;
032import java.util.concurrent.atomic.AtomicLong;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellScanner;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Admin;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
043import org.apache.hadoop.hbase.client.Connection;
044import org.apache.hadoop.hbase.client.Get;
045import org.apache.hadoop.hbase.client.Result;
046import org.apache.hadoop.hbase.client.ResultScanner;
047import org.apache.hadoop.hbase.client.Scan;
048import org.apache.hadoop.hbase.client.SnapshotDescription;
049import org.apache.hadoop.hbase.client.SnapshotType;
050import org.apache.hadoop.hbase.client.Table;
051import org.apache.hadoop.hbase.client.TableDescriptor;
052import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
053import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
054import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
055import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
056import org.apache.hadoop.hbase.testclassification.MediumTests;
057import org.apache.hadoop.hbase.util.CommonFSUtils;
058import org.junit.AfterClass;
059import org.junit.Before;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Rule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.junit.rules.TestName;
066
067import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
068import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
069import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
070
071import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
073import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
074import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
075
076/**
077 * Test class for {@link FileArchiverNotifierImpl}.
078 */
079@Category(MediumTests.class)
080public class TestFileArchiverNotifierImpl {
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083    HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class);
084
085  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
086  private static final AtomicLong COUNTER = new AtomicLong();
087
088  @Rule
089  public TestName testName = new TestName();
090
091  private Connection conn;
092  private Admin admin;
093  private SpaceQuotaHelperForTests helper;
094  private FileSystem fs;
095  private Configuration conf;
096
097  @BeforeClass
098  public static void setUp() throws Exception {
099    Configuration conf = TEST_UTIL.getConfiguration();
100    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
101    // Clean up the compacted files faster than normal (15s instead of 2mins)
102    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
103    // Prevent the SnapshotQuotaObserverChore from running
104    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
105    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
106    TEST_UTIL.startMiniCluster(1);
107  }
108
109  @AfterClass
110  public static void tearDown() throws Exception {
111    TEST_UTIL.shutdownMiniCluster();
112  }
113
114  @Before
115  public void setup() throws Exception {
116    conn = TEST_UTIL.getConnection();
117    admin = TEST_UTIL.getAdmin();
118    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
119    helper.removeAllQuotas(conn);
120    fs = TEST_UTIL.getTestFileSystem();
121    conf = TEST_UTIL.getConfiguration();
122  }
123
124  @Test
125  public void testSnapshotSizePersistence() throws IOException {
126    final Admin admin = TEST_UTIL.getAdmin();
127    final TableName tn = TableName.valueOf(testName.getMethodName());
128    if (admin.tableExists(tn)) {
129      admin.disableTable(tn);
130      admin.deleteTable(tn);
131    }
132    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn)
133      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
134    admin.createTable(desc);
135
136    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
137    List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
138    try (Table table = conn.getTable(tn)) {
139      // Writing no values will result in no records written.
140      verify(table, () -> {
141        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
142        assertEquals(0, count(table));
143      });
144
145      verify(table, () -> {
146        snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
147        snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
148        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
149        assertEquals(2, count(table));
150        assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
151        assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
152      });
153    }
154  }
155
156  @Test
157  public void testIncrementalFileArchiving() throws Exception {
158    final Admin admin = TEST_UTIL.getAdmin();
159    final TableName tn = TableName.valueOf(testName.getMethodName());
160    if (admin.tableExists(tn)) {
161      admin.disableTable(tn);
162      admin.deleteTable(tn);
163    }
164    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
165    final TableName tn1 = helper.createTableWithRegions(1);
166    admin.setQuota(QuotaSettingsFactory.limitTableSpace(tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE,
167      SpaceViolationPolicy.NO_INSERTS));
168
169    // Write some data and flush it
170    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
171    admin.flush(tn1);
172
173    // Create a snapshot on the table
174    final String snapshotName1 = tn1 + "snapshot1";
175    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
176
177    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
178    long t1 = notifier.getLastFullCompute();
179    long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
180    assertEquals("The size of the snapshots should be zero", 0, snapshotSize);
181    assertTrue("Last compute time was not less than current compute time",
182      t1 < notifier.getLastFullCompute());
183
184    // No recently archived files and the snapshot should have no size
185    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
186
187    // Invoke the addArchivedFiles method with no files
188    notifier.addArchivedFiles(Collections.emptySet());
189
190    // The size should not have changed
191    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
192
193    notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
194
195    // The size should not have changed
196    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
197
198    // Pull one file referenced by the snapshot out of the manifest
199    Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
200    assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1);
201    String referencedFile = Iterables.getFirst(referencedFiles, null);
202    assertNotNull(referencedFile);
203
204    // Report that a file this snapshot referenced was moved to the archive. This is a sign
205    // that the snapshot should now "own" the size of this file
206    final long fakeFileSize = 2048L;
207    notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
208
209    // Verify that the snapshot owns this file.
210    assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
211
212    // In reality, we did not actually move the file, so a "full" computation should re-set the
213    // size of the snapshot back to 0.
214    long t2 = notifier.getLastFullCompute();
215    snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
216    assertEquals(0, snapshotSize);
217    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
218    // We should also have no recently archived files after a re-computation
219    assertTrue("Last compute time was not less than current compute time",
220      t2 < notifier.getLastFullCompute());
221  }
222
223  @Test
224  public void testParseOldNamespaceSnapshotSize() throws Exception {
225    final Admin admin = TEST_UTIL.getAdmin();
226    final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName());
227    final TableName tn = TableName.valueOf(testName.getMethodName() + "1");
228    if (admin.tableExists(fakeQuotaTableName)) {
229      admin.disableTable(fakeQuotaTableName);
230      admin.deleteTable(fakeQuotaTableName);
231    }
232    TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName)
233      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
234      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
235    admin.createTable(desc);
236
237    final String ns = "";
238    try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
239      FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
240      // Verify no record is treated as zero
241      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
242
243      // Set an explicit value of zero
244      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
245      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
246
247      // Set a non-zero value
248      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
249      assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
250    }
251  }
252
253  private long count(Table t) throws IOException {
254    try (ResultScanner rs = t.getScanner(new Scan())) {
255      long sum = 0;
256      for (Result r : rs) {
257        while (r.advance()) {
258          sum++;
259        }
260      }
261      return sum;
262    }
263  }
264
265  private long extractSnapshotSize(Table quotaTable, TableName tn, String snapshot)
266    throws IOException {
267    Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
268    Result r = quotaTable.get(g);
269    assertNotNull(r);
270    CellScanner cs = r.cellScanner();
271    assertTrue(cs.advance());
272    Cell c = cs.current();
273    assertNotNull(c);
274    return QuotaTableUtil.extractSnapshotSize(c.getValueArray(), c.getValueOffset(),
275      c.getValueLength());
276  }
277
278  private void verify(Table t, IOThrowingRunnable test) throws IOException {
279    admin.disableTable(t.getName());
280    admin.truncateTable(t.getName(), false);
281    test.run();
282  }
283
284  @FunctionalInterface
285  private interface IOThrowingRunnable {
286    void run() throws IOException;
287  }
288
289  private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
290    HashSet<String> files = new HashSet<>();
291    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
292      CommonFSUtils.getRootDir(conf));
293    SnapshotProtos.SnapshotDescription sd =
294      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
295    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
296    // For each region referenced by the snapshot
297    for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
298      // For each column family in this region
299      for (FamilyFiles ff : rm.getFamilyFilesList()) {
300        // And each store file in that family
301        for (StoreFile sf : ff.getStoreFilesList()) {
302          files.add(sf.getName());
303        }
304      }
305    }
306    return files;
307  }
308
309  private <K, V> Entry<K, V> entry(K k, V v) {
310    return Maps.immutableEntry(k, v);
311  }
312}