Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.http;
019
020import static org.apache.hadoop.hbase.client.hamcrest.BytesMatchers.bytesAsStringBinary;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.hamcrest.Matchers.allOf;
023import static org.hamcrest.Matchers.contains;
024import static org.hamcrest.Matchers.equalTo;
025import static org.hamcrest.Matchers.hasProperty;
026import static org.hamcrest.Matchers.startsWith;
027import static org.mockito.Mockito.mock;
028import static org.mockito.Mockito.when;
029
030import java.util.List;
031import java.util.Objects;
032import java.util.concurrent.CompletableFuture;
033import java.util.stream.Collectors;
034import javax.servlet.http.HttpServletRequest;
035import org.apache.hadoop.hbase.ConnectionExtension;
036import org.apache.hadoop.hbase.MiniClusterExtension;
037import org.apache.hadoop.hbase.NamespaceDescriptor;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.AsyncAdmin;
040import org.apache.hadoop.hbase.client.AsyncConnection;
041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
043import org.apache.hadoop.hbase.client.TableDescriptor;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.master.RegionState;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.util.RegionSplitter;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052import org.junit.jupiter.api.TestInfo;
053import org.junit.jupiter.api.extension.RegisterExtension;
054
055import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils;
056
057/**
058 * Cluster-backed correctness tests for the functionality provided by {@link MetaBrowser}.
059 */
060@Tag(MasterTests.TAG)
061@Tag(MediumTests.TAG)
062public class TestMetaBrowser {
063
064  @RegisterExtension
065  private static final MiniClusterExtension miniClusterExtension =
066    MiniClusterExtension.newBuilder().build();
067
068  @RegisterExtension
069  private static final ConnectionExtension connectionExtension =
070    ConnectionExtension.createAsyncConnectionExtension(miniClusterExtension::createAsyncConnection);
071
072  private AsyncConnection connection;
073  private AsyncAdmin admin;
074
075  @BeforeEach
076  public void before() {
077    connection = connectionExtension.getAsyncConnection();
078    admin = connection.getAdmin();
079    clearTablesAndNamespaces().join();
080  }
081
082  private CompletableFuture<Void> clearTablesAndNamespaces() {
083    return admin.listTableNames(false).thenApply(tableNames -> tableNames.stream()
084      .map(
085        tableName -> disableIfEnabled(tableName).thenCompose(_void -> admin.deleteTable(tableName)))
086      .toArray(CompletableFuture[]::new)).thenCompose(CompletableFuture::allOf)
087      .thenCompose(_void -> admin.listNamespaceDescriptors())
088      .thenApply(nds -> nds.stream().map(NamespaceDescriptor::getName)
089        .filter(name -> !Objects.equals(name, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()))
090        .filter(name -> !Objects.equals(name, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()))
091        .collect(Collectors.toList()))
092      .thenCompose(namespaces -> CompletableFuture
093        .allOf(namespaces.stream().map(admin::deleteNamespace).toArray(CompletableFuture[]::new)));
094  }
095
096  private CompletableFuture<Void> disableIfEnabled(final TableName tableName) {
097    return admin.isTableEnabled(tableName)
098      .thenCompose(isEnabled -> isEnabled
099        ? admin.disableTable(tableName)
100        : CompletableFuture.completedFuture(null));
101  }
102
103  @Test
104  public void noFilters() {
105    final String namespaceName = "noFilters";
106    final TableName a = TableName.valueOf("a");
107    final TableName b = TableName.valueOf(namespaceName, "b");
108
109    CompletableFuture
110      .allOf(createTable(a), createNamespace(namespaceName).thenCompose(_void -> createTable(b, 2)))
111      .join();
112
113    final HttpServletRequest request = new MockRequestBuilder().build();
114    final List<RegionReplicaInfo> rows;
115    try (final MetaBrowser.Results results = new MetaBrowser(connection, request).getResults()) {
116      rows = IterableUtils.toList(results);
117    }
118    assertThat(rows,
119      contains(hasProperty("row", bytesAsStringBinary(startsWith(a + ",,"))),
120        hasProperty("row", bytesAsStringBinary(startsWith(b + ",,"))),
121        hasProperty("row", bytesAsStringBinary(startsWith(b + ",80000000")))));
122  }
123
124  @Test
125  public void limit(TestInfo testInfo) {
126    final String tableName = testInfo.getTestMethod().get().getName();
127    createTable(TableName.valueOf(tableName), 8).join();
128
129    final HttpServletRequest request = new MockRequestBuilder().setLimit(5).build();
130    final List<RegionReplicaInfo> rows;
131    try (final MetaBrowser.Results results = new MetaBrowser(connection, request).getResults()) {
132      rows = IterableUtils.toList(results);
133    }
134    assertThat(rows,
135      contains(hasProperty("row", bytesAsStringBinary(startsWith(tableName + ",,"))),
136        hasProperty("row", bytesAsStringBinary(startsWith(tableName + ",20000000"))),
137        hasProperty("row", bytesAsStringBinary(startsWith(tableName + ",40000000"))),
138        hasProperty("row", bytesAsStringBinary(startsWith(tableName + ",60000000"))),
139        hasProperty("row", bytesAsStringBinary(startsWith(tableName + ",80000000")))));
140  }
141
142  @Test
143  public void regionStateFilter(TestInfo testInfo) {
144    final String namespaceName = testInfo.getTestMethod().get().getName();
145    final TableName foo = TableName.valueOf(namespaceName, "foo");
146    final TableName bar = TableName.valueOf(namespaceName, "bar");
147
148    createNamespace(namespaceName)
149      .thenCompose(_void1 -> CompletableFuture.allOf(
150        createTable(foo, 2).thenCompose(_void2 -> admin.disableTable(foo)), createTable(bar, 2)))
151      .join();
152
153    final HttpServletRequest request = new MockRequestBuilder().setLimit(10_000)
154      .setRegionState(RegionState.State.OPEN).setTable(namespaceName).build();
155    final List<RegionReplicaInfo> rows;
156    try (final MetaBrowser.Results results = new MetaBrowser(connection, request).getResults()) {
157      rows = IterableUtils.toList(results);
158    }
159    assertThat(rows,
160      contains(hasProperty("row", bytesAsStringBinary(startsWith(bar.toString() + ",,"))),
161        hasProperty("row", bytesAsStringBinary(startsWith(bar.toString() + ",80000000")))));
162  }
163
164  @Test
165  public void scanTableFilter(TestInfo testInfo) {
166    final String namespaceName = testInfo.getTestMethod().get().getName();
167    final TableName a = TableName.valueOf("a");
168    final TableName b = TableName.valueOf(namespaceName, "b");
169
170    CompletableFuture
171      .allOf(createTable(a), createNamespace(namespaceName).thenCompose(_void -> createTable(b, 2)))
172      .join();
173
174    final HttpServletRequest request = new MockRequestBuilder().setTable(namespaceName).build();
175    final List<RegionReplicaInfo> rows;
176    try (final MetaBrowser.Results results = new MetaBrowser(connection, request).getResults()) {
177      rows = IterableUtils.toList(results);
178    }
179    assertThat(rows, contains(hasProperty("row", bytesAsStringBinary(startsWith(b + ",,"))),
180      hasProperty("row", bytesAsStringBinary(startsWith(b + ",80000000")))));
181  }
182
183  @Test
184  public void paginateWithReplicas(TestInfo testInfo) {
185    final String namespaceName = testInfo.getTestMethod().get().getName();
186    final TableName a = TableName.valueOf("a");
187    final TableName b = TableName.valueOf(namespaceName, "b");
188
189    CompletableFuture.allOf(createTableWithReplicas(a, 2),
190      createNamespace(namespaceName).thenCompose(_void -> createTable(b, 2))).join();
191
192    final HttpServletRequest request1 = new MockRequestBuilder().setLimit(2).build();
193    final List<RegionReplicaInfo> rows1;
194    try (final MetaBrowser.Results results = new MetaBrowser(connection, request1).getResults()) {
195      rows1 = IterableUtils.toList(results);
196    }
197    assertThat(rows1,
198      contains(
199        allOf(hasProperty("regionName", bytesAsStringBinary(startsWith(a + ",,"))),
200          hasProperty("replicaId", equalTo(0))),
201        allOf(hasProperty("regionName", bytesAsStringBinary(startsWith(a + ",,"))),
202          hasProperty("replicaId", equalTo(1)))));
203
204    final HttpServletRequest request2 = new MockRequestBuilder().setLimit(2)
205      .setStart(MetaBrowser.buildStartParamFrom(rows1.get(rows1.size() - 1).getRow())).build();
206    final List<RegionReplicaInfo> rows2;
207    try (final MetaBrowser.Results results = new MetaBrowser(connection, request2).getResults()) {
208      rows2 = IterableUtils.toList(results);
209    }
210    assertThat(rows2, contains(hasProperty("row", bytesAsStringBinary(startsWith(b + ",,"))),
211      hasProperty("row", bytesAsStringBinary(startsWith(b + ",80000000")))));
212  }
213
214  @Test
215  public void paginateWithTableFilter(TestInfo testInfo) {
216    final String namespaceName = testInfo.getTestMethod().get().getName();
217    final TableName a = TableName.valueOf("a");
218    final TableName b = TableName.valueOf(namespaceName, "b");
219
220    CompletableFuture
221      .allOf(createTable(a), createNamespace(namespaceName).thenCompose(_void -> createTable(b, 5)))
222      .join();
223
224    final HttpServletRequest request1 =
225      new MockRequestBuilder().setLimit(2).setTable(namespaceName).build();
226    final List<RegionReplicaInfo> rows1;
227    try (final MetaBrowser.Results results = new MetaBrowser(connection, request1).getResults()) {
228      rows1 = IterableUtils.toList(results);
229    }
230    assertThat(rows1, contains(hasProperty("row", bytesAsStringBinary(startsWith(b + ",,"))),
231      hasProperty("row", bytesAsStringBinary(startsWith(b + ",33333333")))));
232
233    final HttpServletRequest request2 = new MockRequestBuilder().setLimit(2).setTable(namespaceName)
234      .setStart(MetaBrowser.buildStartParamFrom(rows1.get(rows1.size() - 1).getRow())).build();
235    final List<RegionReplicaInfo> rows2;
236    try (final MetaBrowser.Results results = new MetaBrowser(connection, request2).getResults()) {
237      rows2 = IterableUtils.toList(results);
238    }
239    assertThat(rows2, contains(hasProperty("row", bytesAsStringBinary(startsWith(b + ",66666666"))),
240      hasProperty("row", bytesAsStringBinary(startsWith(b + ",99999999")))));
241
242    final HttpServletRequest request3 = new MockRequestBuilder().setLimit(2).setTable(namespaceName)
243      .setStart(MetaBrowser.buildStartParamFrom(rows2.get(rows2.size() - 1).getRow())).build();
244    final List<RegionReplicaInfo> rows3;
245    try (final MetaBrowser.Results results = new MetaBrowser(connection, request3).getResults()) {
246      rows3 = IterableUtils.toList(results);
247    }
248    assertThat(rows3,
249      contains(hasProperty("row", bytesAsStringBinary(startsWith(b + ",cccccccc")))));
250  }
251
252  private ColumnFamilyDescriptor columnFamilyDescriptor() {
253    return ColumnFamilyDescriptorBuilder.of("f1");
254  }
255
256  private TableDescriptor tableDescriptor(final TableName tableName) {
257    return TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(columnFamilyDescriptor())
258      .build();
259  }
260
261  private TableDescriptor tableDescriptor(final TableName tableName, final int replicaCount) {
262    return TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(replicaCount)
263      .setColumnFamily(columnFamilyDescriptor()).build();
264  }
265
266  private CompletableFuture<Void> createTable(final TableName tableName) {
267    return admin.createTable(tableDescriptor(tableName));
268  }
269
270  private CompletableFuture<Void> createTable(final TableName tableName, final int splitCount) {
271    return admin.createTable(tableDescriptor(tableName),
272      new RegionSplitter.HexStringSplit().split(splitCount));
273  }
274
275  private CompletableFuture<Void> createTableWithReplicas(final TableName tableName,
276    final int replicaCount) {
277    return admin.createTable(tableDescriptor(tableName, replicaCount));
278  }
279
280  private CompletableFuture<Void> createNamespace(final String namespace) {
281    final NamespaceDescriptor descriptor = NamespaceDescriptor.create(namespace).build();
282    return admin.createNamespace(descriptor);
283  }
284
285  /**
286   * Helper for mocking an {@link HttpServletRequest} relevant to the test.
287   */
288  static class MockRequestBuilder {
289
290    private String limit = null;
291    private String regionState = null;
292    private String start = null;
293    private String table = null;
294
295    public MockRequestBuilder setLimit(final int value) {
296      this.limit = Integer.toString(value);
297      return this;
298    }
299
300    public MockRequestBuilder setLimit(final String value) {
301      this.limit = value;
302      return this;
303    }
304
305    public MockRequestBuilder setRegionState(final RegionState.State value) {
306      this.regionState = value.toString();
307      return this;
308    }
309
310    public MockRequestBuilder setRegionState(final String value) {
311      this.regionState = value;
312      return this;
313    }
314
315    public MockRequestBuilder setStart(final String value) {
316      this.start = value;
317      return this;
318    }
319
320    public MockRequestBuilder setTable(final String value) {
321      this.table = value;
322      return this;
323    }
324
325    public HttpServletRequest build() {
326      final HttpServletRequest request = mock(HttpServletRequest.class);
327      when(request.getRequestURI()).thenReturn("/table.jsp");
328      when(request.getParameter("name")).thenReturn("hbase%3Ameta");
329
330      when(request.getParameter("scan_limit")).thenReturn(limit);
331      when(request.getParameter("scan_region_state")).thenReturn(regionState);
332      when(request.getParameter("scan_start")).thenReturn(start);
333      when(request.getParameter("scan_table")).thenReturn(table);
334
335      return request;
336    }
337  }
338}