001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.wal; 019 020import static org.junit.Assert.assertEquals; 021 022import com.google.protobuf.ServiceException; 023import java.io.IOException; 024import java.util.List; 025import java.util.Random; 026import java.util.SortedMap; 027import java.util.TreeMap; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.TableName; 031import org.apache.hadoop.hbase.client.Delete; 032import org.apache.hadoop.hbase.client.Put; 033import org.apache.hadoop.hbase.client.Table; 034import org.apache.hadoop.hbase.master.HMaster; 035import org.apache.hadoop.hbase.regionserver.HRegionServer; 036import org.apache.hadoop.hbase.regionserver.Region; 037import org.apache.hadoop.hbase.testclassification.MediumTests; 038import org.apache.hadoop.hbase.testclassification.RegionServerTests; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.junit.After; 041import org.junit.Before; 042import org.junit.ClassRule; 043import org.junit.Test; 044import org.junit.experimental.categories.Category; 045 046import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 047 048import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; 049import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest; 050import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest; 051 052@Category({RegionServerTests.class, MediumTests.class}) 053public class TestWALFiltering { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestWALFiltering.class); 058 059 private static final int NUM_MASTERS = 1; 060 private static final int NUM_RS = 4; 061 062 private static final TableName TABLE_NAME = 063 TableName.valueOf("TestWALFiltering"); 064 private static final byte[] CF1 = Bytes.toBytes("MyCF1"); 065 private static final byte[] CF2 = Bytes.toBytes("MyCF2"); 066 private static final byte[][] FAMILIES = { CF1, CF2 }; 067 068 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 069 070 @Before 071 public void setUp() throws Exception { 072 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); 073 fillTable(); 074 } 075 076 @After 077 public void tearDown() throws Exception { 078 TEST_UTIL.shutdownMiniCluster(); 079 } 080 081 private void fillTable() throws IOException, InterruptedException { 082 Table table = TEST_UTIL.createTable(TABLE_NAME, FAMILIES, 3, 083 Bytes.toBytes("row0"), Bytes.toBytes("row99"), NUM_RS); 084 Random rand = new Random(19387129L); 085 for (int iStoreFile = 0; iStoreFile < 4; ++iStoreFile) { 086 for (int iRow = 0; iRow < 100; ++iRow) { 087 final byte[] row = Bytes.toBytes("row" + iRow); 088 Put put = new Put(row); 089 Delete del = new Delete(row); 090 for (int iCol = 0; iCol < 10; ++iCol) { 091 final byte[] cf = rand.nextBoolean() ? CF1 : CF2; 092 final long ts = Math.abs(rand.nextInt()); 093 final byte[] qual = Bytes.toBytes("col" + iCol); 094 if (rand.nextBoolean()) { 095 final byte[] value = Bytes.toBytes("value_for_row_" + iRow + 096 "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" + 097 ts + "_random_" + rand.nextLong()); 098 put.addColumn(cf, qual, ts, value); 099 } else if (rand.nextDouble() < 0.8) { 100 del.addColumn(cf, qual, ts); 101 } else { 102 del.addColumn(cf, qual, ts); 103 } 104 } 105 table.put(put); 106 table.delete(del); 107 } 108 } 109 TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); 110 } 111 112 @Test 113 public void testFlushedSequenceIdsSentToHMaster() 114 throws IOException, InterruptedException, 115 org.apache.hbase.thirdparty.com.google.protobuf.ServiceException, ServiceException { 116 SortedMap<byte[], Long> allFlushedSequenceIds = new TreeMap<>(Bytes.BYTES_COMPARATOR); 117 for (int i = 0; i < NUM_RS; ++i) { 118 flushAllRegions(i); 119 } 120 Thread.sleep(10000); 121 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); 122 for (int i = 0; i < NUM_RS; ++i) { 123 for (byte[] regionName : getRegionsByServer(i)) { 124 if (allFlushedSequenceIds.containsKey(regionName)) { 125 GetLastFlushedSequenceIdRequest req = 126 RequestConverter.buildGetLastFlushedSequenceIdRequest(regionName); 127 128 assertEquals((long)allFlushedSequenceIds.get(regionName), 129 master.getMasterRpcServices().getLastFlushedSequenceId( 130 null, req).getLastFlushedSequenceId()); 131 } 132 } 133 } 134 } 135 136 private List<byte[]> getRegionsByServer(int rsId) throws IOException { 137 List<byte[]> regionNames = Lists.newArrayList(); 138 HRegionServer hrs = getRegionServer(rsId); 139 for (Region r : hrs.getRegions(TABLE_NAME)) { 140 regionNames.add(r.getRegionInfo().getRegionName()); 141 } 142 return regionNames; 143 } 144 145 private HRegionServer getRegionServer(int rsId) { 146 return TEST_UTIL.getMiniHBaseCluster().getRegionServer(rsId); 147 } 148 149 private void flushAllRegions(int rsId) 150 throws ServiceException, 151 org.apache.hbase.thirdparty.com.google.protobuf.ServiceException, IOException { 152 HRegionServer hrs = getRegionServer(rsId); 153 for (byte[] regionName : getRegionsByServer(rsId)) { 154 FlushRegionRequest request = 155 RequestConverter.buildFlushRegionRequest(regionName); 156 hrs.getRSRpcServices().flushRegion(null, request); 157 } 158 } 159 160}