001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to you under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.hadoop.hbase.quotas;
018
019import static org.junit.Assert.assertEquals;
020import static org.junit.Assert.assertNotNull;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map.Entry;
030import java.util.Set;
031import java.util.concurrent.atomic.AtomicLong;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.Cell;
036import org.apache.hadoop.hbase.CellScanner;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.Connection;
043import org.apache.hadoop.hbase.client.Get;
044import org.apache.hadoop.hbase.client.Result;
045import org.apache.hadoop.hbase.client.ResultScanner;
046import org.apache.hadoop.hbase.client.Scan;
047import org.apache.hadoop.hbase.client.SnapshotDescription;
048import org.apache.hadoop.hbase.client.SnapshotType;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.client.TableDescriptor;
051import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
052import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
053import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
054import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
055import org.apache.hadoop.hbase.testclassification.MediumTests;
056import org.apache.hadoop.hbase.util.CommonFSUtils;
057import org.junit.AfterClass;
058import org.junit.Before;
059import org.junit.BeforeClass;
060import org.junit.ClassRule;
061import org.junit.Rule;
062import org.junit.Test;
063import org.junit.experimental.categories.Category;
064import org.junit.rules.TestName;
065
066import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
067import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
068import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
069
070import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
071import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
073import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
074
075/**
076 * Test class for {@link FileArchiverNotifierImpl}.
077 */
078@Category(MediumTests.class)
079public class TestFileArchiverNotifierImpl {
080  @ClassRule
081  public static final HBaseClassTestRule CLASS_RULE =
082      HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class);
083
084  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
085  private static final AtomicLong COUNTER = new AtomicLong();
086
087  @Rule
088  public TestName testName = new TestName();
089
090  private Connection conn;
091  private Admin admin;
092  private SpaceQuotaHelperForTests helper;
093  private FileSystem fs;
094  private Configuration conf;
095
096  @BeforeClass
097  public static void setUp() throws Exception {
098    Configuration conf = TEST_UTIL.getConfiguration();
099    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
100    // Clean up the compacted files faster than normal (15s instead of 2mins)
101    conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
102    // Prevent the SnapshotQuotaObserverChore from running
103    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
104    conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
105    TEST_UTIL.startMiniCluster(1);
106  }
107
108  @AfterClass
109  public static void tearDown() throws Exception {
110    TEST_UTIL.shutdownMiniCluster();
111  }
112
113  @Before
114  public void setup() throws Exception {
115    conn = TEST_UTIL.getConnection();
116    admin = TEST_UTIL.getAdmin();
117    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
118    helper.removeAllQuotas(conn);
119    fs = TEST_UTIL.getTestFileSystem();
120    conf = TEST_UTIL.getConfiguration();
121  }
122
123  @Test
124  public void testSnapshotSizePersistence() throws IOException {
125    final Admin admin = TEST_UTIL.getAdmin();
126    final TableName tn = TableName.valueOf(testName.getMethodName());
127    if (admin.tableExists(tn)) {
128      admin.disableTable(tn);
129      admin.deleteTable(tn);
130    }
131    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn).addColumnFamily(
132        ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
133    admin.createTable(desc);
134
135    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
136    List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
137    try (Table table = conn.getTable(tn)) {
138      // Writing no values will result in no records written.
139      verify(table, () -> {
140        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
141        assertEquals(0, count(table));
142      });
143
144      verify(table, () -> {
145        snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
146        snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
147        notifier.persistSnapshotSizes(table, snapshotsWithSizes);
148        assertEquals(2, count(table));
149        assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
150        assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
151      });
152    }
153  }
154
155  @Test
156  public void testIncrementalFileArchiving() throws Exception {
157    final Admin admin = TEST_UTIL.getAdmin();
158    final TableName tn = TableName.valueOf(testName.getMethodName());
159    if (admin.tableExists(tn)) {
160      admin.disableTable(tn);
161      admin.deleteTable(tn);
162    }
163    final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
164    final TableName tn1 = helper.createTableWithRegions(1);
165    admin.setQuota(QuotaSettingsFactory.limitTableSpace(
166        tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
167
168    // Write some data and flush it
169    helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
170    admin.flush(tn1);
171
172    // Create a snapshot on the table
173    final String snapshotName1 = tn1 + "snapshot1";
174    admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
175
176    FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
177    long t1 = notifier.getLastFullCompute();
178    long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
179    assertEquals("The size of the snapshots should be zero", 0, snapshotSize);
180    assertTrue("Last compute time was not less than current compute time",
181        t1 < notifier.getLastFullCompute());
182
183    // No recently archived files and the snapshot should have no size
184    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
185
186    // Invoke the addArchivedFiles method with no files
187    notifier.addArchivedFiles(Collections.emptySet());
188
189    // The size should not have changed
190    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
191
192    notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
193
194    // The size should not have changed
195    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
196
197    // Pull one file referenced by the snapshot out of the manifest
198    Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
199    assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1);
200    String referencedFile = Iterables.getFirst(referencedFiles, null);
201    assertNotNull(referencedFile);
202
203    // Report that a file this snapshot referenced was moved to the archive. This is a sign
204    // that the snapshot should now "own" the size of this file
205    final long fakeFileSize = 2048L;
206    notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
207
208    // Verify that the snapshot owns this file.
209    assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
210
211    // In reality, we did not actually move the file, so a "full" computation should re-set the
212    // size of the snapshot back to 0.
213    long t2 = notifier.getLastFullCompute();
214    snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
215    assertEquals(0, snapshotSize);
216    assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
217    // We should also have no recently archived files after a re-computation
218    assertTrue("Last compute time was not less than current compute time",
219        t2 < notifier.getLastFullCompute());
220  }
221
222  @Test
223  public void testParseOldNamespaceSnapshotSize() throws Exception {
224    final Admin admin = TEST_UTIL.getAdmin();
225    final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName());
226    final TableName tn = TableName.valueOf(testName.getMethodName() + "1");
227    if (admin.tableExists(fakeQuotaTableName)) {
228      admin.disableTable(fakeQuotaTableName);
229      admin.deleteTable(fakeQuotaTableName);
230    }
231    TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName).addColumnFamily(
232        ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
233        .addColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
234    admin.createTable(desc);
235
236    final String ns = "";
237    try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
238      FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
239      // Verify no record is treated as zero
240      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
241
242      // Set an explicit value of zero
243      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
244      assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
245
246      // Set a non-zero value
247      fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
248      assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
249    }
250  }
251
252  private long count(Table t) throws IOException {
253    try (ResultScanner rs = t.getScanner(new Scan())) {
254      long sum = 0;
255      for (Result r : rs) {
256        while (r.advance()) {
257          sum++;
258        }
259      }
260      return sum;
261    }
262  }
263
264  private long extractSnapshotSize(
265      Table quotaTable, TableName tn, String snapshot) throws IOException {
266    Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
267    Result r = quotaTable.get(g);
268    assertNotNull(r);
269    CellScanner cs = r.cellScanner();
270    assertTrue(cs.advance());
271    Cell c = cs.current();
272    assertNotNull(c);
273    return QuotaTableUtil.extractSnapshotSize(
274        c.getValueArray(), c.getValueOffset(), c.getValueLength());
275  }
276
277  private void verify(Table t, IOThrowingRunnable test) throws IOException {
278    admin.disableTable(t.getName());
279    admin.truncateTable(t.getName(), false);
280    test.run();
281  }
282
283  @FunctionalInterface
284  private interface IOThrowingRunnable {
285    void run() throws IOException;
286  }
287
288  private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
289    HashSet<String> files = new HashSet<>();
290    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
291        snapshotName, CommonFSUtils.getRootDir(conf));
292    SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
293        fs, snapshotDir);
294    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
295    // For each region referenced by the snapshot
296    for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
297      // For each column family in this region
298      for (FamilyFiles ff : rm.getFamilyFilesList()) {
299        // And each store file in that family
300        for (StoreFile sf : ff.getStoreFilesList()) {
301          files.add(sf.getName());
302        }
303      }
304    }
305    return files;
306  }
307
308  private <K,V> Entry<K,V> entry(K k, V v) {
309    return Maps.immutableEntry(k, v);
310  }
311}