001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasName;
021import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasParentSpanId;
022import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasStatusWithCode;
023import static org.hamcrest.MatcherAssert.assertThat;
024import static org.hamcrest.Matchers.allOf;
025import static org.hamcrest.Matchers.containsString;
026import static org.hamcrest.Matchers.endsWith;
027import static org.hamcrest.Matchers.hasItem;
028import static org.hamcrest.Matchers.isOneOf;
029import static org.hamcrest.Matchers.startsWith;
030
031import io.opentelemetry.api.trace.StatusCode;
032import io.opentelemetry.sdk.trace.data.SpanData;
033import java.util.List;
034import java.util.function.Supplier;
035import org.apache.hadoop.hbase.client.trace.StringTraceRenderer;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.testclassification.RegionServerTests;
038import org.apache.hadoop.hbase.trace.OpenTelemetryClassRule;
039import org.hamcrest.Matcher;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.junit.rules.ExternalResource;
044import org.junit.rules.RuleChain;
045import org.junit.rules.TestRule;
046import org.junit.runners.model.Statement;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050/**
051 * Test that sundry operations internal to the region server are traced as expected.
052 */
053@Category({ MediumTests.class, RegionServerTests.class })
054public class TestServerInternalsTracing {
055  private static final Logger LOG = LoggerFactory.getLogger(TestServerInternalsTracing.class);
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE =
059    HBaseClassTestRule.forClass(TestServerInternalsTracing.class);
060
061  private static final String NO_PARENT_ID = "0000000000000000";
062  private static List<SpanData> spans;
063
064  /**
065   * Wait for the underlying cluster to come up -- defined by meta being available.
066   */
067  private static class Setup extends ExternalResource {
068    private final Supplier<HBaseTestingUtility> testingUtilSupplier;
069
070    public Setup(final Supplier<HBaseTestingUtility> testingUtilSupplier) {
071      this.testingUtilSupplier = testingUtilSupplier;
072    }
073
074    @Override
075    protected void before() throws Throwable {
076      final HBaseTestingUtility testingUtil = testingUtilSupplier.get();
077      testingUtil.waitTableAvailable(TableName.META_TABLE_NAME);
078    }
079  }
080
081  private static class Noop extends Statement {
082    @Override
083    public void evaluate() throws Throwable {
084    }
085  }
086
087  @ClassRule
088  public static TestRule classRule = (base, description) -> new Statement() {
089    @Override
090    public void evaluate() throws Throwable {
091      // setup and tear down the cluster, collecting all the spans produced in the process.
092      final OpenTelemetryClassRule otelClassRule = OpenTelemetryClassRule.create();
093      final MiniClusterRule miniClusterRule = MiniClusterRule.newBuilder().build();
094      final Setup setup = new Setup(miniClusterRule::getTestingUtility);
095      final TestRule clusterRule =
096        RuleChain.outerRule(otelClassRule).around(miniClusterRule).around(setup);
097      clusterRule.apply(new Noop(), description).evaluate();
098      spans = otelClassRule.getSpans();
099      if (LOG.isDebugEnabled()) {
100        StringTraceRenderer renderer = new StringTraceRenderer(spans);
101        renderer.render(LOG::debug);
102      }
103      base.evaluate();
104    }
105  };
106
107  @Test
108  public void testHMasterConstructor() {
109    final Matcher<SpanData> masterConstructorMatcher = allOf(hasName("HMaster.cxtor"),
110      hasParentSpanId(NO_PARENT_ID), hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
111    assertThat("there should be a span from the HMaster constructor.", spans,
112      hasItem(masterConstructorMatcher));
113    final SpanData masterConstructorSpan = spans.stream().filter(masterConstructorMatcher::matches)
114      .findAny().orElseThrow(AssertionError::new);
115    assertThat("the HMaster constructor span should show zookeeper interaction.", spans, hasItem(
116      allOf(hasName(startsWith("RecoverableZookeeper.")), hasParentSpanId(masterConstructorSpan))));
117  }
118
119  @Test
120  public void testHMasterBecomeActiveMaster() {
121    final Matcher<SpanData> masterBecomeActiveMasterMatcher =
122      allOf(hasName("HMaster.becomeActiveMaster"), hasParentSpanId(NO_PARENT_ID),
123        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
124    assertThat("there should be a span from the HMaster.becomeActiveMaster.", spans,
125      hasItem(masterBecomeActiveMasterMatcher));
126    final SpanData masterBecomeActiveMasterSpan = spans.stream()
127      .filter(masterBecomeActiveMasterMatcher::matches).findAny().orElseThrow(AssertionError::new);
128    assertThat("the HMaster.becomeActiveMaster span should show zookeeper interaction.", spans,
129      hasItem(allOf(hasName(startsWith("RecoverableZookeeper.")),
130        hasParentSpanId(masterBecomeActiveMasterSpan))));
131    assertThat("the HMaster.becomeActiveMaster span should show Region interaction.", spans,
132      hasItem(
133        allOf(hasName(startsWith("Region.")), hasParentSpanId(masterBecomeActiveMasterSpan))));
134    assertThat("the HMaster.becomeActiveMaster span should show RegionScanner interaction.", spans,
135      hasItem(allOf(hasName(startsWith("RegionScanner.")),
136        hasParentSpanId(masterBecomeActiveMasterSpan))));
137    assertThat("the HMaster.becomeActiveMaster span should show hbase:meta interaction.", spans,
138      hasItem(allOf(hasName(containsString("hbase:meta")),
139        hasParentSpanId(masterBecomeActiveMasterSpan))));
140    assertThat("the HMaster.becomeActiveMaster span should show WAL interaction.", spans,
141      hasItem(allOf(hasName(startsWith("WAL.")), hasParentSpanId(masterBecomeActiveMasterSpan))));
142  }
143
144  @Test
145  public void testZKWatcherHMaster() {
146    final Matcher<SpanData> mZKWatcherMatcher = allOf(hasName(startsWith("ZKWatcher-master")),
147      hasParentSpanId(NO_PARENT_ID), hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
148    assertThat("there should be a span from the ZKWatcher running in the HMaster.", spans,
149      hasItem(mZKWatcherMatcher));
150    final SpanData mZKWatcherSpan =
151      spans.stream().filter(mZKWatcherMatcher::matches).findAny().orElseThrow(AssertionError::new);
152    assertThat("the ZKWatcher running in the HMaster span should invoke processEvent.", spans,
153      hasItem(allOf(hasName(containsString("processEvent")), hasParentSpanId(mZKWatcherSpan))));
154  }
155
156  @Test
157  public void testHMasterShutdown() {
158    final Matcher<SpanData> masterShutdownMatcher = allOf(hasName("HMaster.shutdown"),
159      hasParentSpanId(NO_PARENT_ID), hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
160    assertThat("there should be a span from the HMaster.shutdown.", spans,
161      hasItem(masterShutdownMatcher));
162    final SpanData masterShutdownSpan = spans.stream().filter(masterShutdownMatcher::matches)
163      .findAny().orElseThrow(AssertionError::new);
164    assertThat("the HMaster.shutdown span should show zookeeper interaction.", spans, hasItem(
165      allOf(hasName(startsWith("RecoverableZookeeper.")), hasParentSpanId(masterShutdownSpan))));
166    assertThat(
167      "the HMaster.shutdown span should show ShortCircuitingClusterConnection interaction.", spans,
168      hasItem(allOf(hasName(startsWith("ShortCircuitingClusterConnection.")),
169        hasParentSpanId(masterShutdownSpan))));
170  }
171
172  @Test
173  public void testHMasterExitingMainLoop() {
174    final Matcher<SpanData> masterExitingMainLoopMatcher =
175      allOf(hasName("HMaster exiting main loop"), hasParentSpanId(NO_PARENT_ID),
176        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
177    assertThat("there should be a span from the HMaster exiting main loop.", spans,
178      hasItem(masterExitingMainLoopMatcher));
179    final SpanData masterExitingMainLoopSpan = spans.stream()
180      .filter(masterExitingMainLoopMatcher::matches).findAny().orElseThrow(AssertionError::new);
181    assertThat("the HMaster exiting main loop span should show HTable interaction.", spans,
182      hasItem(allOf(hasName(startsWith("HTable.")), hasParentSpanId(masterExitingMainLoopSpan))));
183  }
184
185  @Test
186  public void testTryRegionServerReport() {
187    final Matcher<SpanData> tryRegionServerReportMatcher =
188      allOf(hasName("HRegionServer.tryRegionServerReport"), hasParentSpanId(NO_PARENT_ID),
189        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
190    assertThat("there should be a span for the region server sending a report.", spans,
191      hasItem(tryRegionServerReportMatcher));
192    final SpanData tryRegionServerReportSpan = spans.stream()
193      .filter(tryRegionServerReportMatcher::matches).findAny().orElseThrow(AssertionError::new);
194    assertThat(
195      "the region server report span should have an invocation of the RegionServerReport RPC.",
196      spans, hasItem(allOf(hasName(endsWith("RegionServerStatusService/RegionServerReport")),
197        hasParentSpanId(tryRegionServerReportSpan))));
198  }
199
200  @Test
201  public void testHRegionServerStartup() {
202    final Matcher<SpanData> regionServerStartupMatcher = allOf(hasName("HRegionServer.startup"),
203      hasParentSpanId(NO_PARENT_ID), hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
204    assertThat("there should be a span from the HRegionServer startup procedure.", spans,
205      hasItem(regionServerStartupMatcher));
206    final SpanData regionServerStartupSpan = spans.stream()
207      .filter(regionServerStartupMatcher::matches).findAny().orElseThrow(AssertionError::new);
208    assertThat("the HRegionServer startup procedure span should show zookeeper interaction.", spans,
209      hasItem(allOf(hasName(startsWith("RecoverableZookeeper.")),
210        hasParentSpanId(regionServerStartupSpan))));
211  }
212
213  @Test
214  public void testHRegionServerConstructor() {
215    final Matcher<SpanData> rsConstructorMatcher = allOf(hasName("HRegionServer.cxtor"),
216      hasParentSpanId(NO_PARENT_ID), hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
217    assertThat("there should be a span from the HRegionServer constructor.", spans,
218      hasItem(rsConstructorMatcher));
219    final SpanData rsConstructorSpan = spans.stream().filter(rsConstructorMatcher::matches)
220      .findAny().orElseThrow(AssertionError::new);
221    assertThat("the HRegionServer constructor span should show zookeeper interaction.", spans,
222      hasItem(
223        allOf(hasName(startsWith("RecoverableZookeeper.")), hasParentSpanId(rsConstructorSpan))));
224    assertThat("the HRegionServer constructor span should invoke the MasterAddressTracker.", spans,
225      hasItem(
226        allOf(hasName(startsWith("MasterAddressTracker.")), hasParentSpanId(rsConstructorSpan))));
227  }
228
229  @Test
230  public void testHRegionServerPreRegistrationInitialization() {
231    final Matcher<SpanData> rsPreRegistrationInitializationMatcher =
232      allOf(hasName("HRegionServer.preRegistrationInitialization"), hasParentSpanId(NO_PARENT_ID),
233        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
234    assertThat("there should be a span from the HRegionServer preRegistrationInitialization.",
235      spans, hasItem(rsPreRegistrationInitializationMatcher));
236    final SpanData rsPreRegistrationInitializationSpan =
237      spans.stream().filter(rsPreRegistrationInitializationMatcher::matches).findAny()
238        .orElseThrow(AssertionError::new);
239    assertThat(
240      "the HRegionServer preRegistrationInitialization span should show zookeeper interaction.",
241      spans, hasItem(allOf(hasName(startsWith("RecoverableZookeeper.")),
242        hasParentSpanId(rsPreRegistrationInitializationSpan))));
243  }
244
245  @Test
246  public void testHRegionServerRegisterWithMaster() {
247    final Matcher<SpanData> rsRegisterWithMasterMatcher =
248      allOf(hasName("HRegionServer.registerWithMaster"), hasParentSpanId(NO_PARENT_ID),
249        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
250    assertThat("there should be a span from the HRegionServer registerWithMaster.", spans,
251      hasItem(rsRegisterWithMasterMatcher));
252    final SpanData rsRegisterWithMasterSpan = spans.stream()
253      .filter(rsRegisterWithMasterMatcher::matches).findAny().orElseThrow(AssertionError::new);
254    assertThat("the HRegionServer registerWithMaster span should show zookeeper interaction.",
255      spans, hasItem(allOf(hasName(startsWith("RecoverableZookeeper.")),
256        hasParentSpanId(rsRegisterWithMasterSpan))));
257    assertThat(
258      "the HRegionServer registerWithMaster span should have an invocation of the"
259        + " RegionServerStartup RPC.",
260      spans, hasItem(allOf(hasName(endsWith("RegionServerStatusService/RegionServerStartup")),
261        hasParentSpanId(rsRegisterWithMasterSpan))));
262  }
263
264  @Test
265  public void testZKWatcherRegionServer() {
266    final Matcher<SpanData> rsZKWatcherMatcher =
267      allOf(hasName(startsWith("ZKWatcher-regionserver")), hasParentSpanId(NO_PARENT_ID),
268        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
269    assertThat("there should be a span from the ZKWatcher running in the HRegionServer.", spans,
270      hasItem(rsZKWatcherMatcher));
271    final SpanData rsZKWatcherSpan =
272      spans.stream().filter(rsZKWatcherMatcher::matches).findAny().orElseThrow(AssertionError::new);
273    assertThat("the ZKWatcher running in the HRegionServer span should invoke processEvent.", spans,
274      hasItem(allOf(hasName(containsString("processEvent")), hasParentSpanId(rsZKWatcherSpan))));
275  }
276
277  @Test
278  public void testHRegionServerExitingMainLoop() {
279    final Matcher<SpanData> rsExitingMainLoopMatcher =
280      allOf(hasName("HRegionServer exiting main loop"), hasParentSpanId(NO_PARENT_ID),
281        hasStatusWithCode(isOneOf(StatusCode.OK, StatusCode.ERROR)));
282    assertThat("there should be a span from the HRegionServer exiting main loop.", spans,
283      hasItem(rsExitingMainLoopMatcher));
284    final SpanData rsExitingMainLoopSpan = spans.stream().filter(rsExitingMainLoopMatcher::matches)
285      .findAny().orElseThrow(AssertionError::new);
286    assertThat("the HRegionServer exiting main loop span should show zookeeper interaction.", spans,
287      hasItem(allOf(hasName(startsWith("RecoverableZookeeper.")),
288        hasParentSpanId(rsExitingMainLoopSpan))));
289    assertThat(
290      "the HRegionServer exiting main loop span should show "
291        + "ShortCircuitingClusterConnection interaction.",
292      spans, hasItem(allOf(hasName(startsWith("ShortCircuitingClusterConnection.")),
293        hasParentSpanId(rsExitingMainLoopSpan))));
294    assertThat("the HRegionServer exiting main loop span should invoke CloseMetaHandler.", spans,
295      hasItem(allOf(hasName("CloseMetaHandler"), hasParentSpanId(rsExitingMainLoopSpan))));
296  }
297}