Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.FileNotFoundException;
028import java.io.IOException;
029import java.util.Arrays;
030import java.util.Comparator;
031import java.util.Map;
032import org.apache.hadoop.fs.FSDataInputStream;
033import org.apache.hadoop.fs.FSDataOutputStream;
034import org.apache.hadoop.fs.FileStatus;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HBaseTestingUtility;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.TableDescriptors;
041import org.apache.hadoop.hbase.TableExistsException;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
046import org.apache.hadoop.hbase.exceptions.DeserializationException;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.testclassification.MiscTests;
049import org.junit.ClassRule;
050import org.junit.Rule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.rules.TestName;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057/**
058 * Tests for {@link FSTableDescriptors}.
059 */
060// Do not support to be executed in he same JVM as other tests
061@Category({MiscTests.class, MediumTests.class})
062public class TestFSTableDescriptors {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestFSTableDescriptors.class);
067
068  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
069  private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class);
070
071  @Rule
072  public TestName name = new TestName();
073
074  @Test (expected=IllegalArgumentException.class)
075  public void testRegexAgainstOldStyleTableInfo() {
076    Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
077    int i = FSTableDescriptors.getTableInfoSequenceId(p);
078    assertEquals(0, i);
079    // Assert it won't eat garbage -- that it fails
080    p = new Path("/tmp", "abc");
081    FSTableDescriptors.getTableInfoSequenceId(p);
082  }
083
084  @Test
085  public void testCreateAndUpdate() throws IOException {
086    Path testdir = UTIL.getDataTestDir(name.getMethodName());
087    TableDescriptor htd =
088      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
089    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
090    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
091    assertTrue(fstd.createTableDescriptor(htd));
092    assertFalse(fstd.createTableDescriptor(htd));
093    FileStatus[] statuses = fs.listStatus(testdir);
094    assertTrue("statuses.length=" + statuses.length, statuses.length == 1);
095    for (int i = 0; i < 10; i++) {
096      fstd.updateTableDescriptor(htd);
097    }
098    statuses = fs.listStatus(testdir);
099    assertTrue(statuses.length == 1);
100    Path tmpTableDir = new Path(CommonFSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
101    statuses = fs.listStatus(tmpTableDir);
102    assertTrue(statuses.length == 0);
103  }
104
105  @Test
106  public void testSequenceIdAdvancesOnTableInfo() throws IOException {
107    Path testdir = UTIL.getDataTestDir(name.getMethodName());
108    TableDescriptor htd =
109      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
110    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
111    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
112    Path p0 = fstd.updateTableDescriptor(htd);
113    int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
114    Path p1 = fstd.updateTableDescriptor(htd);
115    // Assert we cleaned up the old file.
116    assertTrue(!fs.exists(p0));
117    int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
118    assertTrue(i1 == i0 + 1);
119    Path p2 = fstd.updateTableDescriptor(htd);
120    // Assert we cleaned up the old file.
121    assertTrue(!fs.exists(p1));
122    int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
123    assertTrue(i2 == i1 + 1);
124    Path p3 = fstd.updateTableDescriptor(htd);
125    // Assert we cleaned up the old file.
126    assertTrue(!fs.exists(p2));
127    int i3 = FSTableDescriptors.getTableInfoSequenceId(p3);
128    assertTrue(i3 == i2 + 1);
129    TableDescriptor descriptor = fstd.get(htd.getTableName());
130    assertEquals(descriptor, htd);
131  }
132
133  @Test
134  public void testFormatTableInfoSequenceId() {
135    Path p0 = assertWriteAndReadSequenceId(0);
136    // Assert p0 has format we expect.
137    StringBuilder sb = new StringBuilder();
138    for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
139      sb.append("0");
140    }
141    assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
142      p0.getName());
143    // Check a few more.
144    Path p2 = assertWriteAndReadSequenceId(2);
145    Path p10000 = assertWriteAndReadSequenceId(10000);
146    // Get a .tablinfo that has no sequenceid suffix.
147    Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
148    FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
149    FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
150    FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
151    FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
152    Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
153    assertTrue(comparator.compare(fs, fs0) > 0);
154    assertTrue(comparator.compare(fs0, fs2) > 0);
155    assertTrue(comparator.compare(fs2, fs10000) > 0);
156  }
157
158  private Path assertWriteAndReadSequenceId(final int i) {
159    Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
160    int ii = FSTableDescriptors.getTableInfoSequenceId(p);
161    assertEquals(i, ii);
162    return p;
163  }
164
165  @Test
166  public void testRemoves() throws IOException {
167    final String name = this.name.getMethodName();
168    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
169    // Cleanup old tests if any detrius laying around.
170    Path rootdir = new Path(UTIL.getDataTestDir(), name);
171    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
172    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
173    htds.update(htd);
174    assertNotNull(htds.remove(htd.getTableName()));
175    assertNull(htds.remove(htd.getTableName()));
176  }
177
178  @Test public void testReadingHTDFromFS() throws IOException {
179    final String name = this.name.getMethodName();
180    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
181    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
182    Path rootdir = UTIL.getDataTestDir(name);
183    FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
184    fstd.createTableDescriptor(htd);
185    TableDescriptor td2 =
186      FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
187    assertTrue(htd.equals(td2));
188  }
189
190  @Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException {
191    final String name = this.name.getMethodName();
192    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
193    Path rootdir = UTIL.getDataTestDir(name);
194    FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
195    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
196    Path descriptorFile = fstd.updateTableDescriptor(htd);
197    try (FSDataOutputStream out = fs.create(descriptorFile, true)) {
198      out.write(TableDescriptorBuilder.toByteArray(htd));
199    }
200    FSTableDescriptors fstd2 = new FSTableDescriptors(fs, rootdir);
201    TableDescriptor td2 = fstd2.get(htd.getTableName());
202    assertEquals(htd, td2);
203    FileStatus descriptorFile2 =
204        FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName()));
205    byte[] buffer = TableDescriptorBuilder.toByteArray(htd);
206    try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) {
207      in.readFully(buffer);
208    }
209    TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer);
210    assertEquals(htd, td3);
211  }
212
213  @Test public void testTableDescriptors()
214  throws IOException, InterruptedException {
215    final String name = this.name.getMethodName();
216    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
217    // Cleanup old tests if any debris laying around.
218    Path rootdir = new Path(UTIL.getDataTestDir(), name);
219    FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
220      @Override
221      public TableDescriptor get(TableName tablename)
222          throws TableExistsException, FileNotFoundException, IOException {
223        LOG.info(tablename + ", cachehits=" + this.cachehits);
224        return super.get(tablename);
225      }
226    };
227    final int count = 10;
228    // Write out table infos.
229    for (int i = 0; i < count; i++) {
230      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
231    }
232
233    for (int i = 0; i < count; i++) {
234      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
235    }
236    for (int i = 0; i < count; i++) {
237      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
238    }
239    // Update the table infos
240    for (int i = 0; i < count; i++) {
241      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
242      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
243      htds.updateTableDescriptor(builder.build());
244    }
245    // Wait a while so mod time we write is for sure different.
246    Thread.sleep(100);
247    for (int i = 0; i < count; i++) {
248      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
249    }
250    for (int i = 0; i < count; i++) {
251      assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
252    }
253    assertEquals(count * 4, htds.invocations);
254    assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
255      htds.cachehits >= (count * 2));
256  }
257
258  @Test
259  public void testTableDescriptorsNoCache()
260    throws IOException, InterruptedException {
261    final String name = this.name.getMethodName();
262    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
263    // Cleanup old tests if any debris laying around.
264    Path rootdir = new Path(UTIL.getDataTestDir(), name);
265    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir, false);
266    final int count = 10;
267    // Write out table infos.
268    for (int i = 0; i < count; i++) {
269      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
270    }
271
272    for (int i = 0; i < 2 * count; i++) {
273      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i % 2)));
274    }
275    // Update the table infos
276    for (int i = 0; i < count; i++) {
277      TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i));
278      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
279      htds.updateTableDescriptor(builder.build());
280    }
281    for (int i = 0; i < count; i++) {
282      assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i)));
283      assertTrue("Column Family " + i + " missing",
284                 htds.get(TableName.valueOf(name + i)).hasColumnFamily(Bytes.toBytes("" + i)));
285    }
286    assertEquals(count * 4, htds.invocations);
287    assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits);
288  }
289
290  @Test
291  public void testGetAll()
292    throws IOException, InterruptedException {
293    final String name = "testGetAll";
294    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
295    // Cleanup old tests if any debris laying around.
296    Path rootdir = new Path(UTIL.getDataTestDir(), name);
297    FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
298    final int count = 4;
299    // Write out table infos.
300    for (int i = 0; i < count; i++) {
301      htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
302    }
303    // add hbase:meta
304    htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
305
306    assertEquals("getAll() didn't return all TableDescriptors, expected: " +
307                   (count + 1) + " got: " + htds.getAll().size(),
308                 count + 1, htds.getAll().size());
309  }
310
311  @Test
312  public void testGetAllOrdering() throws Exception {
313    final String name = "testGetAllOrdering";
314    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
315    Path rootDir = new Path(UTIL.getDataTestDir(), name);
316    FSTableDescriptors tds = new FSTableDescriptorsTest(fs, rootDir);
317
318    String[] tableNames = new String[] { "foo", "bar", "foo:bar", "bar:foo" };
319    for (String tableName : tableNames) {
320      tds.createTableDescriptor(
321          TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)).build());
322    }
323
324    Map<String, TableDescriptor> tables = tds.getAll();
325    // Remove hbase:meta from list. It shows up now since we  made it dynamic. The schema
326    // is written into the fs by the FSTableDescriptors constructor now where before it
327    // didn't.
328    tables.remove(TableName.META_TABLE_NAME.getNameAsString());
329    assertEquals(4, tables.size());
330
331
332    String[] tableNamesOrdered =
333        new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" };
334    int i = 0;
335    for (Map.Entry<String, TableDescriptor> entry : tables.entrySet()) {
336      assertEquals(tableNamesOrdered[i], entry.getKey());
337      assertEquals(tableNamesOrdered[i],
338          entry.getValue().getTableName().getNameWithNamespaceInclAsString());
339      i++;
340    }
341  }
342
343  @Test
344  public void testCacheConsistency()
345    throws IOException, InterruptedException {
346    final String name = this.name.getMethodName();
347    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
348    // Cleanup old tests if any debris laying around.
349    Path rootdir = new Path(UTIL.getDataTestDir(), name);
350    FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir);
351    FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs, rootdir, false);
352
353    final int count = 10;
354    // Write out table infos via non-cached FSTableDescriptors
355    for (int i = 0; i < count; i++) {
356      nonchtds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
357    }
358
359    // Calls to getAll() won't increase the cache counter, do per table.
360    for (int i = 0; i < count; i++) {
361      assertTrue(chtds.get(TableName.valueOf(name + i)) !=  null);
362    }
363
364    assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
365
366    // add a new entry for random table name.
367    TableName random = TableName.valueOf("random");
368    TableDescriptor htd = TableDescriptorBuilder.newBuilder(random).build();
369    nonchtds.createTableDescriptor(htd);
370
371    // random will only increase the cachehit by 1
372    assertEquals(nonchtds.getAll().size(), chtds.getAll().size() + 1);
373
374    for (Map.Entry<String, TableDescriptor> entry: nonchtds.getAll().entrySet()) {
375      String t = (String) entry.getKey();
376      TableDescriptor nchtd = entry.getValue();
377      assertTrue("expected " + htd.toString() +
378                   " got: " + chtds.get(TableName.valueOf(t)).toString(),
379                 (nchtd.equals(chtds.get(TableName.valueOf(t)))));
380    }
381  }
382
383  @Test
384  public void testNoSuchTable() throws IOException {
385    final String name = "testNoSuchTable";
386    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
387    // Cleanup old tests if any detrius laying around.
388    Path rootdir = new Path(UTIL.getDataTestDir(), name);
389    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
390    assertNull("There shouldn't be any HTD for this table",
391      htds.get(TableName.valueOf("NoSuchTable")));
392  }
393
394  @Test
395  public void testUpdates() throws IOException {
396    final String name = "testUpdates";
397    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
398    // Cleanup old tests if any detrius laying around.
399    Path rootdir = new Path(UTIL.getDataTestDir(), name);
400    TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
401    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
402    htds.update(htd);
403    htds.update(htd);
404    htds.update(htd);
405  }
406
407  @Test
408  public void testTableInfoFileStatusComparator() {
409    FileStatus bare =
410      new FileStatus(0, false, 0, 0, -1,
411        new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
412    FileStatus future =
413      new FileStatus(0, false, 0, 0, -1,
414        new Path("/tmp/tablinfo." + System.currentTimeMillis()));
415    FileStatus farFuture =
416      new FileStatus(0, false, 0, 0, -1,
417        new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
418    FileStatus [] alist = {bare, future, farFuture};
419    FileStatus [] blist = {bare, farFuture, future};
420    FileStatus [] clist = {farFuture, bare, future};
421    Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
422    Arrays.sort(alist, c);
423    Arrays.sort(blist, c);
424    Arrays.sort(clist, c);
425    // Now assert all sorted same in way we want.
426    for (int i = 0; i < alist.length; i++) {
427      assertTrue(alist[i].equals(blist[i]));
428      assertTrue(blist[i].equals(clist[i]));
429      assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
430    }
431  }
432
433  @Test
434  public void testReadingInvalidDirectoryFromFS() throws IOException {
435    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
436    try {
437      new FSTableDescriptors(fs, CommonFSUtils.getRootDir(UTIL.getConfiguration()))
438          .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
439      fail("Shouldn't be able to read a table descriptor for the archive directory.");
440    } catch (Exception e) {
441      LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
442          + e.getMessage());
443    }
444  }
445
446  @Test
447  public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
448    Path testdir = UTIL.getDataTestDir(name.getMethodName());
449    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
450    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
451    FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
452    assertTrue(fstd.createTableDescriptor(htd));
453    assertFalse(fstd.createTableDescriptor(htd));
454    htd = TableDescriptorBuilder.newBuilder(htd)
455            .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"))
456            .build();
457    assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
458    Path tableDir = fstd.getTableDir(htd.getTableName());
459    Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
460    FileStatus[] statuses = fs.listStatus(tmpTableDir);
461    assertTrue(statuses.length == 0);
462
463    assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
464  }
465
466  private static class FSTableDescriptorsTest extends FSTableDescriptors {
467
468    public FSTableDescriptorsTest(FileSystem fs, Path rootdir) {
469      this(fs, rootdir, true);
470    }
471
472    public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean usecache) {
473      super(fs, rootdir, false, usecache);
474    }
475
476    @Override
477    public TableDescriptor get(TableName tablename)
478      throws TableExistsException, FileNotFoundException, IOException {
479      LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
480                 " TableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
481      return super.get(tablename);
482    }
483  }
484}
485