001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.util.Arrays;
029import java.util.Comparator;
030import java.util.Map;
031import org.apache.hadoop.fs.FSDataInputStream;
032import org.apache.hadoop.fs.FSDataOutputStream;
033import org.apache.hadoop.fs.FileStatus;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.TableDescriptors;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
044import org.apache.hadoop.hbase.exceptions.DeserializationException;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.apache.hadoop.hbase.testclassification.MiscTests;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Tests for {@link FSTableDescriptors}.
057 */
058// Do not support to be executed in he same JVM as other tests
059@Category({MiscTests.class, MediumTests.class})
060public class TestFSTableDescriptors {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestFSTableDescriptors.class);
065
066  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
067  private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class);
068
069  @Rule
070  public TestName name = new TestName();
071
072  @Test (expected=IllegalArgumentException.class)
073  public void testRegexAgainstOldStyleTableInfo() {
074    Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
075    int i = FSTableDescriptors.getTableInfoSequenceId(p);
076    assertEquals(0, i);
077    // Assert it won't eat garbage -- that it fails
078    p = new Path("/tmp", "abc");
079    FSTableDescriptors.getTableInfoSequenceId(p);
080  }
081
082  @Test
083  public void testCreateAndUpdate() throws IOException {
084    Path testdir = UTIL.getDataTestDir(name.getMethodName());
085    TableDescriptor htd =
086      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
087    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
088    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
089    assertTrue(fstd.createTableDescriptor(htd));
090    assertFalse(fstd.createTableDescriptor(htd));
091    FileStatus[] statuses = fs.listStatus(testdir);
092    assertTrue("statuses.length=" + statuses.length, statuses.length == 1);
093    for (int i = 0; i < 10; i++) {
094      fstd.update(htd);
095    }
096    statuses = fs.listStatus(testdir);
097    assertTrue(statuses.length == 1);
098    Path tmpTableDir = new Path(CommonFSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
099    statuses = fs.listStatus(tmpTableDir);
100    assertTrue(statuses.length == 0);
101  }
102
103  @Test
104  public void testSequenceIdAdvancesOnTableInfo() throws IOException {
105    Path testdir = UTIL.getDataTestDir(name.getMethodName());
106    TableDescriptor htd =
107      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
108    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
109    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
110    Path p0 = fstd.updateTableDescriptor(htd);
111    int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
112    Path p1 = fstd.updateTableDescriptor(htd);
113    // Assert we cleaned up the old file.
114    assertTrue(!fs.exists(p0));
115    int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
116    assertTrue(i1 == i0 + 1);
117    Path p2 = fstd.updateTableDescriptor(htd);
118    // Assert we cleaned up the old file.
119    assertTrue(!fs.exists(p1));
120    int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
121    assertTrue(i2 == i1 + 1);
122    Path p3 = fstd.updateTableDescriptor(htd);
123    // Assert we cleaned up the old file.
124    assertTrue(!fs.exists(p2));
125    int i3 = FSTableDescriptors.getTableInfoSequenceId(p3);
126    assertTrue(i3 == i2 + 1);
127    TableDescriptor descriptor = fstd.get(htd.getTableName());
128    assertEquals(descriptor, htd);
129  }
130
131  @Test
132  public void testFormatTableInfoSequenceId() {
133    Path p0 = assertWriteAndReadSequenceId(0);
134    // Assert p0 has format we expect.
135    StringBuilder sb = new StringBuilder();
136    for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
137      sb.append("0");
138    }
139    assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
140      p0.getName());
141    // Check a few more.
142    Path p2 = assertWriteAndReadSequenceId(2);
143    Path p10000 = assertWriteAndReadSequenceId(10000);
144    // Get a .tablinfo that has no sequenceid suffix.
145    Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
146    FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
147    FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
148    FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
149    FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
150    Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
151    assertTrue(comparator.compare(fs, fs0) > 0);
152    assertTrue(comparator.compare(fs0, fs2) > 0);
153    assertTrue(comparator.compare(fs2, fs10000) > 0);
154  }
155
156  private Path assertWriteAndReadSequenceId(final int i) {
157    Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
158    int ii = FSTableDescriptors.getTableInfoSequenceId(p);
159    assertEquals(i, ii);
160    return p;
161  }
162
163  @Test
164  public void testRemoves() throws IOException {
165    final String name = this.name.getMethodName();
166    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
167    // Cleanup old tests if any detrius laying around.
168    Path rootdir = new Path(UTIL.getDataTestDir(), name);
169    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
170    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
171    htds.update(htd);
172    assertNotNull(htds.remove(htd.getTableName()));
173    assertNull(htds.remove(htd.getTableName()));
174  }
175
176  @Test public void testReadingHTDFromFS() throws IOException {
177    final String name = this.name.getMethodName();
178    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
179    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
180    Path rootdir = UTIL.getDataTestDir(name);
181    FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
182    fstd.createTableDescriptor(htd);
183    TableDescriptor td2 =
184      FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
185    assertTrue(htd.equals(td2));
186  }
187
188  @Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException {
189    final String name = this.name.getMethodName();
190    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
191    Path rootdir = UTIL.getDataTestDir(name);
192    FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
193    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
194    Path descriptorFile = fstd.updateTableDescriptor(htd);
195    try (FSDataOutputStream out = fs.create(descriptorFile, true)) {
196      out.write(TableDescriptorBuilder.toByteArray(htd));
197    }
198    FSTableDescriptors fstd2 = new FSTableDescriptors(fs, rootdir);
199    TableDescriptor td2 = fstd2.get(htd.getTableName());
200    assertEquals(htd, td2);
201    FileStatus descriptorFile2 =
202        FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName()));
203    byte[] buffer = TableDescriptorBuilder.toByteArray(htd);
204    try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) {
205      in.readFully(buffer);
206    }
207    TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer);
208    assertEquals(htd, td3);
209  }
210
211  @Test public void testTableDescriptors()
212  throws IOException, InterruptedException {
213    final String name = this.name.getMethodName();
214    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
215    // Cleanup old tests if any debris laying around.
216    Path rootdir = new Path(UTIL.getDataTestDir(), name);
217    FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
218      @Override
219      public TableDescriptor get(TableName tablename) {
220        LOG.info(tablename + ", cachehits=" + this.cachehits);
221        return super.get(tablename);
222      }
223    };
224    final int count = 10;
225    // Write out table infos.
226    for (int i = 0; i < count; i++) {
227      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
228    }
229
230    for (int i = 0; i < count; i++) {
231      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
232    }
233    for (int i = 0; i < count; i++) {
234      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
235    }
236    // Update the table infos
237    for (int i = 0; i < count; i++) {
238      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
239      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
240      htds.update(builder.build());
241    }
242    // Wait a while so mod time we write is for sure different.
243    Thread.sleep(100);
244    for (int i = 0; i < count; i++) {
245      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
246    }
247    for (int i = 0; i < count; i++) {
248      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
249    }
250    assertEquals(count * 4, htds.invocations);
251    assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
252      htds.cachehits >= (count * 2));
253  }
254
255  @Test
256  public void testTableDescriptorsNoCache()
257    throws IOException, InterruptedException {
258    final String name = this.name.getMethodName();
259    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
260    // Cleanup old tests if any debris laying around.
261    Path rootdir = new Path(UTIL.getDataTestDir(), name);
262    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir, false);
263    final int count = 10;
264    // Write out table infos.
265    for (int i = 0; i < count; i++) {
266      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
267    }
268
269    for (int i = 0; i < 2 * count; i++) {
270      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i % 2)));
271    }
272    // Update the table infos
273    for (int i = 0; i < count; i++) {
274      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
275      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
276      htds.update(builder.build());
277    }
278    for (int i = 0; i < count; i++) {
279      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i)));
280      assertTrue("Column Family " + i + " missing",
281                 htds.get(TableName.valueOf(name + i)).hasColumnFamily(Bytes.toBytes("" + i)));
282    }
283    assertEquals(count * 4, htds.invocations);
284    assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits);
285  }
286
287  @Test
288  public void testGetAll()
289    throws IOException, InterruptedException {
290    final String name = "testGetAll";
291    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
292    // Cleanup old tests if any debris laying around.
293    Path rootdir = new Path(UTIL.getDataTestDir(), name);
294    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
295    final int count = 4;
296    // Write out table infos.
297    for (int i = 0; i < count; i++) {
298      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
299    }
300    // add hbase:meta
301    htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
302
303    assertEquals("getAll() didn't return all TableDescriptors, expected: " +
304                   (count + 1) + " got: " + htds.getAll().size(),
305                 count + 1, htds.getAll().size());
306  }
307
308  @Test
309  public void testGetAllOrdering() throws Exception {
310    final String name = "testGetAllOrdering";
311    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
312    Path rootDir = new Path(UTIL.getDataTestDir(), name);
313    FSTableDescriptors tds = new FSTableDescriptorsTest(fs, rootDir);
314
315    String[] tableNames = new String[] { "foo", "bar", "foo:bar", "bar:foo" };
316    for (String tableName : tableNames) {
317      tds.createTableDescriptor(
318          TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)).build());
319    }
320
321    Map<String, TableDescriptor> tables = tds.getAll();
322    // Remove hbase:meta from list. It shows up now since we  made it dynamic. The schema
323    // is written into the fs by the FSTableDescriptors constructor now where before it
324    // didn't.
325    tables.remove(TableName.META_TABLE_NAME.getNameAsString());
326    assertEquals(4, tables.size());
327
328
329    String[] tableNamesOrdered =
330        new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" };
331    int i = 0;
332    for (Map.Entry<String, TableDescriptor> entry : tables.entrySet()) {
333      assertEquals(tableNamesOrdered[i], entry.getKey());
334      assertEquals(tableNamesOrdered[i],
335          entry.getValue().getTableName().getNameWithNamespaceInclAsString());
336      i++;
337    }
338  }
339
340  @Test
341  public void testCacheConsistency()
342    throws IOException, InterruptedException {
343    final String name = this.name.getMethodName();
344    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
345    // Cleanup old tests if any debris laying around.
346    Path rootdir = new Path(UTIL.getDataTestDir(), name);
347    FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir);
348    FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs, rootdir, false);
349
350    final int count = 10;
351    // Write out table infos via non-cached FSTableDescriptors
352    for (int i = 0; i < count; i++) {
353      nonchtds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
354    }
355
356    // Calls to getAll() won't increase the cache counter, do per table.
357    for (int i = 0; i < count; i++) {
358      assertTrue(chtds.get(TableName.valueOf(name + i)) !=  null);
359    }
360
361    assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
362
363    // add a new entry for random table name.
364    TableName random = TableName.valueOf("random");
365    TableDescriptor htd = TableDescriptorBuilder.newBuilder(random).build();
366    nonchtds.createTableDescriptor(htd);
367
368    // random will only increase the cachehit by 1
369    assertEquals(nonchtds.getAll().size(), chtds.getAll().size() + 1);
370
371    for (Map.Entry<String, TableDescriptor> entry : chtds.getAll().entrySet()) {
372      String t = (String) entry.getKey();
373      TableDescriptor nchtd = entry.getValue();
374      assertTrue(
375        "expected " + htd.toString() + " got: " + chtds.get(TableName.valueOf(t)).toString(),
376        (nchtd.equals(chtds.get(TableName.valueOf(t)))));
377    }
378    // this is by design, for FSTableDescriptor with cache enabled, once we have done a full scan
379    // and load all the table descriptors to cache, we will not go to file system again, as the only
380    // way to update table descriptor is to through us so we can cache it when updating.
381    assertNotNull(nonchtds.get(random));
382    assertNull(chtds.get(random));
383  }
384
385  @Test
386  public void testNoSuchTable() throws IOException {
387    final String name = "testNoSuchTable";
388    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
389    // Cleanup old tests if any detrius laying around.
390    Path rootdir = new Path(UTIL.getDataTestDir(), name);
391    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
392    assertNull("There shouldn't be any HTD for this table",
393      htds.get(TableName.valueOf("NoSuchTable")));
394  }
395
396  @Test
397  public void testUpdates() throws IOException {
398    final String name = "testUpdates";
399    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
400    // Cleanup old tests if any detrius laying around.
401    Path rootdir = new Path(UTIL.getDataTestDir(), name);
402    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
403    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
404    htds.update(htd);
405    htds.update(htd);
406    htds.update(htd);
407  }
408
409  @Test
410  public void testTableInfoFileStatusComparator() {
411    FileStatus bare =
412      new FileStatus(0, false, 0, 0, -1,
413        new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
414    FileStatus future =
415      new FileStatus(0, false, 0, 0, -1,
416        new Path("/tmp/tablinfo." + System.currentTimeMillis()));
417    FileStatus farFuture =
418      new FileStatus(0, false, 0, 0, -1,
419        new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
420    FileStatus [] alist = {bare, future, farFuture};
421    FileStatus [] blist = {bare, farFuture, future};
422    FileStatus [] clist = {farFuture, bare, future};
423    Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
424    Arrays.sort(alist, c);
425    Arrays.sort(blist, c);
426    Arrays.sort(clist, c);
427    // Now assert all sorted same in way we want.
428    for (int i = 0; i < alist.length; i++) {
429      assertTrue(alist[i].equals(blist[i]));
430      assertTrue(blist[i].equals(clist[i]));
431      assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
432    }
433  }
434
435  @Test
436  public void testReadingInvalidDirectoryFromFS() throws IOException {
437    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
438    try {
439      new FSTableDescriptors(fs, CommonFSUtils.getRootDir(UTIL.getConfiguration()))
440          .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
441      fail("Shouldn't be able to read a table descriptor for the archive directory.");
442    } catch (Exception e) {
443      LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
444          + e.getMessage());
445    }
446  }
447
448  @Test
449  public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
450    Path testdir = UTIL.getDataTestDir(name.getMethodName());
451    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
452    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
453    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
454    assertTrue(fstd.createTableDescriptor(htd));
455    assertFalse(fstd.createTableDescriptor(htd));
456    htd = TableDescriptorBuilder.newBuilder(htd)
457            .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"))
458            .build();
459    assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
460    Path tableDir = fstd.getTableDir(htd.getTableName());
461    Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
462    FileStatus[] statuses = fs.listStatus(tmpTableDir);
463    assertTrue(statuses.length == 0);
464
465    assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
466  }
467
468  private static class FSTableDescriptorsTest extends FSTableDescriptors {
469
470    public FSTableDescriptorsTest(FileSystem fs, Path rootdir) {
471      this(fs, rootdir, true);
472    }
473
474    public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean usecache) {
475      super(fs, rootdir, false, usecache);
476    }
477
478    @Override
479    public TableDescriptor get(TableName tablename) {
480      LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
481                 " TableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
482      return super.get(tablename);
483    }
484  }
485}
486