001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.File;
026import java.io.IOException;
027import java.util.Arrays;
028import java.util.HashMap;
029import java.util.HashSet;
030import java.util.Map;
031import java.util.Optional;
032import java.util.Set;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.Coprocessor;
037import org.apache.hadoop.hbase.CoprocessorEnvironment;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.MiniHBaseCluster;
043import org.apache.hadoop.hbase.RegionMetrics;
044import org.apache.hadoop.hbase.ServerMetrics;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.Admin;
048import org.apache.hadoop.hbase.regionserver.HRegion;
049import org.apache.hadoop.hbase.regionserver.Region;
050import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
051import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
052import org.apache.hadoop.hbase.testclassification.MediumTests;
053import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
054import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
055import org.apache.hadoop.hdfs.MiniDFSCluster;
056import org.junit.AfterClass;
057import org.junit.BeforeClass;
058import org.junit.ClassRule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064/**
065 * Test coprocessors class loading.
066 */
067@Category({CoprocessorTests.class, MediumTests.class})
068public class TestClassLoading {
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071      HBaseClassTestRule.forClass(TestClassLoading.class);
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestClassLoading.class);
074  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
075
076  public static class TestMasterCoprocessor implements MasterCoprocessor, MasterObserver {
077    @Override
078    public Optional<MasterObserver> getMasterObserver() {
079      return Optional.of(this);
080    }
081  }
082
083  private static MiniDFSCluster cluster;
084
085  static final TableName tableName = TableName.valueOf("TestClassLoading");
086  static final String cpName1 = "TestCP1";
087  static final String cpName2 = "TestCP2";
088  static final String cpName3 = "TestCP3";
089  static final String cpName4 = "TestCP4";
090  static final String cpName5 = "TestCP5";
091  static final String cpName6 = "TestCP6";
092
093  private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
094  // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
095  private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
096  private static Class<?> regionServerCoprocessor = SampleRegionWALCoprocessor.class;
097  private static Class<?> masterCoprocessor = TestMasterCoprocessor.class;
098
099  private static final String[] regionServerSystemCoprocessors =
100      new String[]{ regionServerCoprocessor.getSimpleName() };
101
102  private static final String[] masterRegionServerSystemCoprocessors = new String[] {
103      regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
104      regionServerCoprocessor.getSimpleName() };
105
106  @BeforeClass
107  public static void setUpBeforeClass() throws Exception {
108    Configuration conf = TEST_UTIL.getConfiguration();
109
110    // regionCoprocessor1 will be loaded on all regionservers, since it is
111    // loaded for any tables (user or meta).
112    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
113        regionCoprocessor1.getName());
114
115    // regionCoprocessor2 will be loaded only on regionservers that serve a
116    // user table region. Therefore, if there are no user tables loaded,
117    // this coprocessor will not be loaded on any regionserver.
118    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
119        regionCoprocessor2.getName());
120
121    conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
122        regionServerCoprocessor.getName());
123    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
124        masterCoprocessor.getName());
125    TEST_UTIL.startMiniCluster(1);
126    cluster = TEST_UTIL.getDFSCluster();
127  }
128
129  @AfterClass
130  public static void tearDownAfterClass() throws Exception {
131    TEST_UTIL.shutdownMiniCluster();
132  }
133
134  static File buildCoprocessorJar(String className) throws Exception {
135    String code =
136        "import org.apache.hadoop.hbase.coprocessor.*;" +
137            "public class " + className + " implements RegionCoprocessor {}";
138    return ClassLoaderTestHelper.buildJar(
139      TEST_UTIL.getDataTestDir().toString(), className, code);
140  }
141
142  @Test
143  // HBASE-3516: Test CP Class loading from HDFS
144  public void testClassLoadingFromHDFS() throws Exception {
145    FileSystem fs = cluster.getFileSystem();
146
147    File jarFile1 = buildCoprocessorJar(cpName1);
148    File jarFile2 = buildCoprocessorJar(cpName2);
149
150    // copy the jars into dfs
151    fs.copyFromLocalFile(new Path(jarFile1.getPath()),
152      new Path(fs.getUri().toString() + Path.SEPARATOR));
153    String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
154      jarFile1.getName();
155    Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
156    assertTrue("Copy jar file to HDFS failed.",
157      fs.exists(pathOnHDFS1));
158    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
159
160    fs.copyFromLocalFile(new Path(jarFile2.getPath()),
161        new Path(fs.getUri().toString() + Path.SEPARATOR));
162    String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
163      jarFile2.getName();
164    Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
165    assertTrue("Copy jar file to HDFS failed.",
166      fs.exists(pathOnHDFS2));
167    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
168
169    // create a table that references the coprocessors
170    HTableDescriptor htd = new HTableDescriptor(tableName);
171    htd.addFamily(new HColumnDescriptor("test"));
172      // without configuration values
173    htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
174      "|" + Coprocessor.PRIORITY_USER);
175      // with configuration values
176    htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
177      "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
178    Admin admin = TEST_UTIL.getAdmin();
179    if (admin.tableExists(tableName)) {
180      if (admin.isTableEnabled(tableName)) {
181        admin.disableTable(tableName);
182      }
183      admin.deleteTable(tableName);
184    }
185    CoprocessorClassLoader.clearCache();
186    byte[] startKey = {10, 63};
187    byte[] endKey = {12, 43};
188    admin.createTable(htd, startKey, endKey, 4);
189    waitForTable(htd.getTableName());
190
191    // verify that the coprocessors were loaded
192    boolean foundTableRegion=false;
193    boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
194    Map<Region, Set<ClassLoader>> regionsActiveClassLoaders = new HashMap<>();
195    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
196    for (HRegion region:
197        hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
198      if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
199        foundTableRegion = true;
200        CoprocessorEnvironment env;
201        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
202        found1 = found1 && (env != null);
203        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
204        found2 = found2 && (env != null);
205        if (env != null) {
206          Configuration conf = env.getConfiguration();
207          found2_k1 = found2_k1 && (conf.get("k1") != null);
208          found2_k2 = found2_k2 && (conf.get("k2") != null);
209          found2_k3 = found2_k3 && (conf.get("k3") != null);
210        } else {
211          found2_k1 = false;
212          found2_k2 = false;
213          found2_k3 = false;
214        }
215        regionsActiveClassLoaders
216            .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
217      }
218    }
219
220    assertTrue("No region was found for table " + tableName, foundTableRegion);
221    assertTrue("Class " + cpName1 + " was missing on a region", found1);
222    assertTrue("Class " + cpName2 + " was missing on a region", found2);
223    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
224    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
225    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
226    // check if CP classloaders are cached
227    assertNotNull(jarFileOnHDFS1 + " was not cached",
228      CoprocessorClassLoader.getIfCached(pathOnHDFS1));
229    assertNotNull(jarFileOnHDFS2 + " was not cached",
230      CoprocessorClassLoader.getIfCached(pathOnHDFS2));
231    //two external jar used, should be one classloader per jar
232    assertEquals("The number of cached classloaders should be equal to the number" +
233      " of external jar files",
234      2, CoprocessorClassLoader.getAllCached().size());
235    //check if region active classloaders are shared across all RS regions
236    Set<ClassLoader> externalClassLoaders = new HashSet<>(
237      CoprocessorClassLoader.getAllCached());
238    for (Map.Entry<Region, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
239      assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
240        + " ClassLoader Cache:" + externalClassLoaders
241        + " Region ClassLoaders:" + regionCP.getValue(),
242        externalClassLoaders.containsAll(regionCP.getValue()));
243    }
244  }
245
246  private String getLocalPath(File file) {
247    return new Path(file.toURI()).toString();
248  }
249
250  @Test
251  // HBASE-3516: Test CP Class loading from local file system
252  public void testClassLoadingFromLocalFS() throws Exception {
253    File jarFile = buildCoprocessorJar(cpName3);
254
255    // create a table that references the jar
256    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
257    htd.addFamily(new HColumnDescriptor("test"));
258    htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
259      Coprocessor.PRIORITY_USER);
260    Admin admin = TEST_UTIL.getAdmin();
261    admin.createTable(htd);
262    waitForTable(htd.getTableName());
263
264    // verify that the coprocessor was loaded
265    boolean found = false;
266    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
267    for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
268      if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
269        found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
270      }
271    }
272    assertTrue("Class " + cpName3 + " was missing on a region", found);
273  }
274
275  @Test
276  // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
277  public void testPrivateClassLoader() throws Exception {
278    File jarFile = buildCoprocessorJar(cpName4);
279
280    // create a table that references the jar
281    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
282    htd.addFamily(new HColumnDescriptor("test"));
283    htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
284      Coprocessor.PRIORITY_USER);
285    Admin admin = TEST_UTIL.getAdmin();
286    admin.createTable(htd);
287    waitForTable(htd.getTableName());
288
289    // verify that the coprocessor was loaded correctly
290    boolean found = false;
291    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
292    for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
293      if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) {
294        Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
295        if (cp != null) {
296          found = true;
297          assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
298            cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
299        }
300      }
301    }
302    assertTrue("Class " + cpName4 + " was missing on a region", found);
303  }
304
305  @Test
306  // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
307  // less strict
308  public void testHBase3810() throws Exception {
309    // allowed value pattern: [path] | class name | [priority] | [key values]
310
311    File jarFile1 = buildCoprocessorJar(cpName1);
312    File jarFile2 = buildCoprocessorJar(cpName2);
313    File jarFile5 = buildCoprocessorJar(cpName5);
314    File jarFile6 = buildCoprocessorJar(cpName6);
315
316    String cpKey1 = "COPROCESSOR$1";
317    String cpKey2 = " Coprocessor$2 ";
318    String cpKey3 = " coprocessor$03 ";
319
320    String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
321        Coprocessor.PRIORITY_USER;
322    String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
323    // load from default class loader
324    String cpValue3 =
325        " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
326
327    // create a table that references the jar
328    HTableDescriptor htd = new HTableDescriptor(tableName);
329    htd.addFamily(new HColumnDescriptor("test"));
330
331    // add 3 coprocessors by setting htd attributes directly.
332    htd.setValue(cpKey1, cpValue1);
333    htd.setValue(cpKey2, cpValue2);
334    htd.setValue(cpKey3, cpValue3);
335
336    // add 2 coprocessor by using new htd.setCoprocessor() api
337    htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
338        Coprocessor.PRIORITY_USER, null);
339    Map<String, String> kvs = new HashMap<>();
340    kvs.put("k1", "v1");
341    kvs.put("k2", "v2");
342    kvs.put("k3", "v3");
343    htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
344        Coprocessor.PRIORITY_USER, kvs);
345
346    Admin admin = TEST_UTIL.getAdmin();
347    if (admin.tableExists(tableName)) {
348      if (admin.isTableEnabled(tableName)) {
349        admin.disableTable(tableName);
350      }
351      admin.deleteTable(tableName);
352    }
353    admin.createTable(htd);
354    waitForTable(htd.getTableName());
355
356    // verify that the coprocessor was loaded
357    boolean found_2 = false, found_1 = false, found_3 = false,
358        found_5 = false, found_6 = false;
359    boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
360        found6_k4 = false;
361
362    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
363    for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
364      if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
365        found_1 = found_1 ||
366            (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
367        found_2 = found_2 ||
368            (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
369        found_3 = found_3 ||
370            (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
371                != null);
372        found_5 = found_5 ||
373            (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
374
375        CoprocessorEnvironment env =
376            region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
377        if (env != null) {
378          found_6 = true;
379          Configuration conf = env.getConfiguration();
380          found6_k1 = conf.get("k1") != null;
381          found6_k2 = conf.get("k2") != null;
382          found6_k3 = conf.get("k3") != null;
383        }
384      }
385    }
386
387    assertTrue("Class " + cpName1 + " was missing on a region", found_1);
388    assertTrue("Class " + cpName2 + " was missing on a region", found_2);
389    assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
390    assertTrue("Class " + cpName5 + " was missing on a region", found_5);
391    assertTrue("Class " + cpName6 + " was missing on a region", found_6);
392
393    assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
394    assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
395    assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
396    assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
397  }
398
399  @Test
400  public void testClassLoadingFromLibDirInJar() throws Exception {
401    loadingClassFromLibDirInJar("/lib/");
402  }
403
404  @Test
405  public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
406    loadingClassFromLibDirInJar("lib/");
407  }
408
409  void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
410    FileSystem fs = cluster.getFileSystem();
411
412    File innerJarFile1 = buildCoprocessorJar(cpName1);
413    File innerJarFile2 = buildCoprocessorJar(cpName2);
414    File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
415
416    ClassLoaderTestHelper.addJarFilesToJar(
417      outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
418
419    // copy the jars into dfs
420    fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
421      new Path(fs.getUri().toString() + Path.SEPARATOR));
422    String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
423      outerJarFile.getName();
424    assertTrue("Copy jar file to HDFS failed.",
425      fs.exists(new Path(jarFileOnHDFS)));
426    LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
427
428    // create a table that references the coprocessors
429    HTableDescriptor htd = new HTableDescriptor(tableName);
430    htd.addFamily(new HColumnDescriptor("test"));
431      // without configuration values
432    htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
433      "|" + Coprocessor.PRIORITY_USER);
434      // with configuration values
435    htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
436      "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
437    Admin admin = TEST_UTIL.getAdmin();
438    if (admin.tableExists(tableName)) {
439      if (admin.isTableEnabled(tableName)) {
440        admin.disableTable(tableName);
441      }
442      admin.deleteTable(tableName);
443    }
444    admin.createTable(htd);
445    waitForTable(htd.getTableName());
446
447    // verify that the coprocessors were loaded
448    boolean found1 = false, found2 = false, found2_k1 = false,
449        found2_k2 = false, found2_k3 = false;
450    MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
451    for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
452      if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
453        CoprocessorEnvironment env;
454        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
455        if (env != null) {
456          found1 = true;
457        }
458        env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
459        if (env != null) {
460          found2 = true;
461          Configuration conf = env.getConfiguration();
462          found2_k1 = conf.get("k1") != null;
463          found2_k2 = conf.get("k2") != null;
464          found2_k3 = conf.get("k3") != null;
465        }
466      }
467    }
468    assertTrue("Class " + cpName1 + " was missing on a region", found1);
469    assertTrue("Class " + cpName2 + " was missing on a region", found2);
470    assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
471    assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
472    assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
473  }
474
475  @Test
476  public void testRegionServerCoprocessorsReported() throws Exception {
477    // This was a test for HBASE-4070.
478    // We are removing coprocessors from region load in HBASE-5258.
479    // Therefore, this test now only checks system coprocessors.
480    assertAllRegionServers(null);
481  }
482
483  /**
484   * return the subset of all regionservers
485   * (actually returns set of ServerLoads)
486   * which host some region in a given table.
487   * used by assertAllRegionServers() below to
488   * test reporting of loaded coprocessors.
489   * @param tableName : given table.
490   * @return subset of all servers.
491   */
492  Map<ServerName, ServerMetrics> serversForTable(String tableName) {
493    Map<ServerName, ServerMetrics> serverLoadHashMap = new HashMap<>();
494    for(Map.Entry<ServerName, ServerMetrics> server:
495        TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
496            getOnlineServers().entrySet()) {
497      for(Map.Entry<byte[], RegionMetrics> region:
498          server.getValue().getRegionMetrics().entrySet()) {
499        if (region.getValue().getNameAsString().equals(tableName)) {
500          // this server hosts a region of tableName: add this server..
501          serverLoadHashMap.put(server.getKey(),server.getValue());
502          // .. and skip the rest of the regions that it hosts.
503          break;
504        }
505      }
506    }
507    return serverLoadHashMap;
508  }
509
510  void assertAllRegionServers(String tableName) throws InterruptedException {
511    Map<ServerName, ServerMetrics> servers;
512    boolean success = false;
513    String[] expectedCoprocessors = regionServerSystemCoprocessors;
514    if (tableName == null) {
515      // if no tableName specified, use all servers.
516      servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
517    } else {
518      servers = serversForTable(tableName);
519    }
520    for (int i = 0; i < 5; i++) {
521      boolean any_failed = false;
522      for(Map.Entry<ServerName, ServerMetrics> server: servers.entrySet()) {
523        String[] actualCoprocessors =
524          server.getValue().getCoprocessorNames().stream().toArray(size -> new String[size]);
525        if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
526          LOG.debug("failed comparison: actual: " +
527              Arrays.toString(actualCoprocessors) +
528              " ; expected: " + Arrays.toString(expectedCoprocessors));
529          any_failed = true;
530          expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
531          break;
532        }
533        expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
534      }
535      if (any_failed == false) {
536        success = true;
537        break;
538      }
539      LOG.debug("retrying after failed comparison: " + i);
540      Thread.sleep(1000);
541    }
542    assertTrue(success);
543  }
544
545  private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
546    if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
547      expectedCoprocessors = masterRegionServerSystemCoprocessors;
548    } else {
549      expectedCoprocessors = regionServerSystemCoprocessors;
550    }
551    return expectedCoprocessors;
552  }
553
554  @Test
555  public void testMasterCoprocessorsReported() {
556    // HBASE 4070: Improve region server metrics to report loaded coprocessors
557    // to master: verify that the master is reporting the correct set of
558    // loaded coprocessors.
559    final String loadedMasterCoprocessorsVerify =
560        "[" + masterCoprocessor.getSimpleName() + "]";
561    String loadedMasterCoprocessors =
562        java.util.Arrays.toString(
563            TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessors());
564    assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
565  }
566
567  private void waitForTable(TableName name) throws InterruptedException, IOException {
568    // First wait until all regions are online
569    TEST_UTIL.waitTableEnabled(name);
570    // Now wait a bit longer for the coprocessor hosts to load the CPs
571    Thread.sleep(1000);
572  }
573}