Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.FileNotFoundException;
028import java.io.IOException;
029import java.util.Arrays;
030import java.util.Comparator;
031import java.util.Map;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FSDataInputStream;
034import org.apache.hadoop.fs.FSDataOutputStream;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableDescriptors;
042import org.apache.hadoop.hbase.TableExistsException;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.TableDescriptor;
046import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
047import org.apache.hadoop.hbase.exceptions.DeserializationException;
048import org.apache.hadoop.hbase.testclassification.MediumTests;
049import org.apache.hadoop.hbase.testclassification.MiscTests;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * Tests for {@link FSTableDescriptors}.
060 */
061// Do not support to be executed in he same JVM as other tests
062@Category({MiscTests.class, MediumTests.class})
063public class TestFSTableDescriptors {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067      HBaseClassTestRule.forClass(TestFSTableDescriptors.class);
068
069  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
070  private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class);
071
072  @Rule
073  public TestName name = new TestName();
074
075  @Test (expected=IllegalArgumentException.class)
076  public void testRegexAgainstOldStyleTableInfo() {
077    Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
078    int i = FSTableDescriptors.getTableInfoSequenceId(p);
079    assertEquals(0, i);
080    // Assert it won't eat garbage -- that it fails
081    p = new Path("/tmp", "abc");
082    FSTableDescriptors.getTableInfoSequenceId(p);
083  }
084
085  @Test
086  public void testCreateAndUpdate() throws IOException {
087    Path testdir = UTIL.getDataTestDir(name.getMethodName());
088    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
089    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
090    FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
091    assertTrue(fstd.createTableDescriptor(htd));
092    assertFalse(fstd.createTableDescriptor(htd));
093    FileStatus [] statuses = fs.listStatus(testdir);
094    assertTrue("statuses.length="+statuses.length, statuses.length == 1);
095    for (int i = 0; i < 10; i++) {
096      fstd.updateTableDescriptor(htd);
097    }
098    statuses = fs.listStatus(testdir);
099    assertTrue(statuses.length == 1);
100    Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
101    statuses = fs.listStatus(tmpTableDir);
102    assertTrue(statuses.length == 0);
103  }
104
105  @Test
106  public void testSequenceIdAdvancesOnTableInfo() throws IOException {
107    Path testdir = UTIL.getDataTestDir(name.getMethodName());
108    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
109    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
110    FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
111    Path p0 = fstd.updateTableDescriptor(htd);
112    int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
113    Path p1 = fstd.updateTableDescriptor(htd);
114    // Assert we cleaned up the old file.
115    assertTrue(!fs.exists(p0));
116    int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
117    assertTrue(i1 == i0 + 1);
118    Path p2 = fstd.updateTableDescriptor(htd);
119    // Assert we cleaned up the old file.
120    assertTrue(!fs.exists(p1));
121    int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
122    assertTrue(i2 == i1 + 1);
123    Path p3 = fstd.updateTableDescriptor(htd);
124    // Assert we cleaned up the old file.
125    assertTrue(!fs.exists(p2));
126    int i3 = FSTableDescriptors.getTableInfoSequenceId(p3);
127    assertTrue(i3 == i2 + 1);
128    TableDescriptor descriptor = fstd.get(htd.getTableName());
129    assertEquals(descriptor, htd);
130  }
131
132  @Test
133  public void testFormatTableInfoSequenceId() {
134    Path p0 = assertWriteAndReadSequenceId(0);
135    // Assert p0 has format we expect.
136    StringBuilder sb = new StringBuilder();
137    for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
138      sb.append("0");
139    }
140    assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
141      p0.getName());
142    // Check a few more.
143    Path p2 = assertWriteAndReadSequenceId(2);
144    Path p10000 = assertWriteAndReadSequenceId(10000);
145    // Get a .tablinfo that has no sequenceid suffix.
146    Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
147    FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
148    FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
149    FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
150    FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
151    Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
152    assertTrue(comparator.compare(fs, fs0) > 0);
153    assertTrue(comparator.compare(fs0, fs2) > 0);
154    assertTrue(comparator.compare(fs2, fs10000) > 0);
155  }
156
157  private Path assertWriteAndReadSequenceId(final int i) {
158    Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
159    int ii = FSTableDescriptors.getTableInfoSequenceId(p);
160    assertEquals(i, ii);
161    return p;
162  }
163
164  @Test
165  public void testRemoves() throws IOException {
166    final String name = this.name.getMethodName();
167    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
168    // Cleanup old tests if any detrius laying around.
169    Path rootdir = new Path(UTIL.getDataTestDir(), name);
170    TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
171    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
172    htds.add(htd);
173    assertNotNull(htds.remove(htd.getTableName()));
174    assertNull(htds.remove(htd.getTableName()));
175  }
176
177  @Test public void testReadingHTDFromFS() throws IOException {
178    final String name = this.name.getMethodName();
179    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
180    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
181    Path rootdir = UTIL.getDataTestDir(name);
182    FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
183    fstd.createTableDescriptor(htd);
184    TableDescriptor td2 =
185      FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
186    assertTrue(htd.equals(td2));
187  }
188
189  @Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException {
190    final String name = this.name.getMethodName();
191    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
192    Path rootdir = UTIL.getDataTestDir(name);
193    FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
194    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
195    Path descriptorFile = fstd.updateTableDescriptor(htd);
196    try (FSDataOutputStream out = fs.create(descriptorFile, true)) {
197      out.write(TableDescriptorBuilder.toByteArray(htd));
198    }
199    FSTableDescriptors fstd2 = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
200    TableDescriptor td2 = fstd2.get(htd.getTableName());
201    assertEquals(htd, td2);
202    FileStatus descriptorFile2 =
203        FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName()));
204    byte[] buffer = TableDescriptorBuilder.toByteArray(htd);
205    try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) {
206      in.readFully(buffer);
207    }
208    TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer);
209    assertEquals(htd, td3);
210  }
211
212  @Test public void testTableDescriptors()
213  throws IOException, InterruptedException {
214    final String name = this.name.getMethodName();
215    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
216    // Cleanup old tests if any debris laying around.
217    Path rootdir = new Path(UTIL.getDataTestDir(), name);
218    FSTableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir) {
219      @Override
220      public TableDescriptor get(TableName tablename)
221          throws TableExistsException, FileNotFoundException, IOException {
222        LOG.info(tablename + ", cachehits=" + this.cachehits);
223        return super.get(tablename);
224      }
225    };
226    final int count = 10;
227    // Write out table infos.
228    for (int i = 0; i < count; i++) {
229      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
230    }
231
232    for (int i = 0; i < count; i++) {
233      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
234    }
235    for (int i = 0; i < count; i++) {
236      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
237    }
238    // Update the table infos
239    for (int i = 0; i < count; i++) {
240      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
241      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
242      htds.updateTableDescriptor(builder.build());
243    }
244    // Wait a while so mod time we write is for sure different.
245    Thread.sleep(100);
246    for (int i = 0; i < count; i++) {
247      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
248    }
249    for (int i = 0; i < count; i++) {
250      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
251    }
252    assertEquals(count * 4, htds.invocations);
253    assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
254      htds.cachehits >= (count * 2));
255  }
256
257  @Test
258  public void testTableDescriptorsNoCache()
259    throws IOException, InterruptedException {
260    final String name = this.name.getMethodName();
261    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
262    // Cleanup old tests if any debris laying around.
263    Path rootdir = new Path(UTIL.getDataTestDir(), name);
264    FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir,
265      false, false);
266    final int count = 10;
267    // Write out table infos.
268    for (int i = 0; i < count; i++) {
269      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
270    }
271
272    for (int i = 0; i < 2 * count; i++) {
273      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i % 2)));
274    }
275    // Update the table infos
276    for (int i = 0; i < count; i++) {
277      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
278      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
279      htds.updateTableDescriptor(builder.build());
280    }
281    for (int i = 0; i < count; i++) {
282      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i)));
283      assertTrue("Column Family " + i + " missing",
284                 htds.get(TableName.valueOf(name + i)).hasColumnFamily(Bytes.toBytes("" + i)));
285    }
286    assertEquals(count * 4, htds.invocations);
287    assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits);
288  }
289
290  @Test
291  public void testGetAll()
292    throws IOException, InterruptedException {
293    final String name = "testGetAll";
294    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
295    // Cleanup old tests if any debris laying around.
296    Path rootdir = new Path(UTIL.getDataTestDir(), name);
297    FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
298    final int count = 4;
299    // Write out table infos.
300    for (int i = 0; i < count; i++) {
301      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
302    }
303    // add hbase:meta
304    htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
305
306    assertEquals("getAll() didn't return all TableDescriptors, expected: " +
307                   (count + 1) + " got: " + htds.getAll().size(),
308                 count + 1, htds.getAll().size());
309
310  }
311
312  @Test
313  public void testCacheConsistency()
314    throws IOException, InterruptedException {
315    final String name = this.name.getMethodName();
316    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
317    // Cleanup old tests if any debris laying around.
318    Path rootdir = new Path(UTIL.getDataTestDir(), name);
319    FSTableDescriptors chtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
320    FSTableDescriptors nonchtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs,
321      rootdir, false, false);
322
323    final int count = 10;
324    // Write out table infos via non-cached FSTableDescriptors
325    for (int i = 0; i < count; i++) {
326      nonchtds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
327    }
328
329    // Calls to getAll() won't increase the cache counter, do per table.
330    for (int i = 0; i < count; i++) {
331      assertTrue(chtds.get(TableName.valueOf(name + i)) !=  null);
332    }
333
334    assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
335
336    // add a new entry for hbase:meta
337    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build();
338    nonchtds.createTableDescriptor(htd);
339
340    // hbase:meta will only increase the cachehit by 1
341    assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
342
343    for (Map.Entry<String, TableDescriptor> entry: nonchtds.getAll().entrySet()) {
344      String t = (String) entry.getKey();
345      TableDescriptor nchtd = entry.getValue();
346      assertTrue("expected " + htd.toString() +
347                   " got: " + chtds.get(TableName.valueOf(t)).toString(),
348                 (nchtd.equals(chtds.get(TableName.valueOf(t)))));
349    }
350  }
351
352  @Test
353  public void testNoSuchTable() throws IOException {
354    final String name = "testNoSuchTable";
355    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
356    // Cleanup old tests if any detrius laying around.
357    Path rootdir = new Path(UTIL.getDataTestDir(), name);
358    TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
359    assertNull("There shouldn't be any HTD for this table",
360      htds.get(TableName.valueOf("NoSuchTable")));
361  }
362
363  @Test
364  public void testUpdates() throws IOException {
365    final String name = "testUpdates";
366    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
367    // Cleanup old tests if any detrius laying around.
368    Path rootdir = new Path(UTIL.getDataTestDir(), name);
369    TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
370    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
371    htds.add(htd);
372    htds.add(htd);
373    htds.add(htd);
374  }
375
376  @Test
377  public void testTableInfoFileStatusComparator() {
378    FileStatus bare =
379      new FileStatus(0, false, 0, 0, -1,
380        new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
381    FileStatus future =
382      new FileStatus(0, false, 0, 0, -1,
383        new Path("/tmp/tablinfo." + System.currentTimeMillis()));
384    FileStatus farFuture =
385      new FileStatus(0, false, 0, 0, -1,
386        new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
387    FileStatus [] alist = {bare, future, farFuture};
388    FileStatus [] blist = {bare, farFuture, future};
389    FileStatus [] clist = {farFuture, bare, future};
390    Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
391    Arrays.sort(alist, c);
392    Arrays.sort(blist, c);
393    Arrays.sort(clist, c);
394    // Now assert all sorted same in way we want.
395    for (int i = 0; i < alist.length; i++) {
396      assertTrue(alist[i].equals(blist[i]));
397      assertTrue(blist[i].equals(clist[i]));
398      assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
399    }
400  }
401
402  @Test
403  public void testReadingInvalidDirectoryFromFS() throws IOException {
404    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
405    try {
406      new FSTableDescriptors(UTIL.getConfiguration(), fs,
407          FSUtils.getRootDir(UTIL.getConfiguration()))
408          .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
409      fail("Shouldn't be able to read a table descriptor for the archive directory.");
410    } catch (Exception e) {
411      LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
412          + e.getMessage());
413    }
414  }
415
416  @Test
417  public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
418    Path testdir = UTIL.getDataTestDir(name.getMethodName());
419    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
420    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
421    FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
422    assertTrue(fstd.createTableDescriptor(htd));
423    assertFalse(fstd.createTableDescriptor(htd));
424    htd = TableDescriptorBuilder.newBuilder(htd)
425            .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"))
426            .build();
427    assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
428    Path tableDir = fstd.getTableDir(htd.getTableName());
429    Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
430    FileStatus[] statuses = fs.listStatus(tmpTableDir);
431    assertTrue(statuses.length == 0);
432
433    assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
434  }
435
436  private static class FSTableDescriptorsTest extends FSTableDescriptors {
437
438    public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir)
439      throws IOException {
440      this(conf, fs, rootdir, false, true);
441    }
442
443    public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir,
444      boolean fsreadonly, boolean usecache) throws IOException {
445      super(conf, fs, rootdir, fsreadonly, usecache);
446    }
447
448    @Override
449    public TableDescriptor get(TableName tablename)
450      throws TableExistsException, FileNotFoundException, IOException {
451      LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
452                 " TableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
453      return super.get(tablename);
454    }
455  }
456}
457