001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.replication.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertNull;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.util.ArrayList;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import java.util.NavigableMap;
029import java.util.TreeMap;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseConfiguration;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.RegionInfoBuilder;
037import org.apache.hadoop.hbase.testclassification.ReplicationTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.wal.WALEdit;
041import org.apache.hadoop.hbase.wal.WALKeyImpl;
042import org.junit.jupiter.api.Tag;
043import org.junit.jupiter.api.Test;
044
045import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
046
047import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
048import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
049import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
050
051@Tag(ReplicationTests.TAG)
052@Tag(SmallTests.TAG)
053public class TestReplicationWALEdits {
054
055  private static final Configuration CONF = HBaseConfiguration.create();
056
057  private static final TableName TABLE_NAME = TableName.valueOf("test");
058
059  private static final byte[] F1 = Bytes.toBytes("f1");
060
061  private static final byte[] F2 = Bytes.toBytes("f2");
062
063  private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
064
065  /**
066   * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
067   * compaction WALEdit.
068   */
069  @Test
070  public void testCompactionWALEdits() throws Exception {
071    TableName tableName = TableName.valueOf("testCompactionWALEdits");
072    WALProtos.CompactionDescriptor compactionDescriptor =
073      WALProtos.CompactionDescriptor.getDefaultInstance();
074    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(HConstants.EMPTY_START_ROW)
075      .setEndKey(HConstants.EMPTY_END_ROW).build();
076    WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
077    ReplicationSourceWALActionListener.scopeWALEdits(new WALKeyImpl(), edit, CONF);
078  }
079
080  private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
081    // 1. Create store files for the families
082    Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
083    Map<String, Long> storeFilesSize = new HashMap<>(1);
084    List<Path> p = new ArrayList<>(1);
085    Path hfilePath1 = new Path(Bytes.toString(F1));
086    p.add(hfilePath1);
087    storeFilesSize.put(hfilePath1.getName(), 0L);
088    storeFiles.put(F1, p);
089    scope.put(F1, 1);
090    p = new ArrayList<>(1);
091    Path hfilePath2 = new Path(Bytes.toString(F2));
092    p.add(hfilePath2);
093    storeFilesSize.put(hfilePath2.getName(), 0L);
094    storeFiles.put(F2, p);
095    // 2. Create bulk load descriptor
096    BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(RI.getTable(),
097      UnsafeByteOperations.unsafeWrap(RI.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
098
099    // 3. create bulk load wal edit event
100    WALEdit logEdit = WALEdit.createBulkLoadEvent(RI, desc);
101    return logEdit;
102  }
103
104  @Test
105  public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
106    NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
107    // 1. Get the bulk load wal edit event
108    WALEdit logEdit = getBulkLoadWALEdit(scope);
109    // 2. Create wal key
110    WALKeyImpl logKey = new WALKeyImpl(scope);
111
112    // 3. Get the scopes for the key
113    ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, CONF);
114
115    // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
116    assertNull(logKey.getReplicationScopes(),
117      "No bulk load entries scope should be added if bulk load replication is disabled.");
118  }
119
120  @Test
121  public void testBulkLoadWALEdits() throws Exception {
122    // 1. Get the bulk load wal edit event
123    NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
124    WALEdit logEdit = getBulkLoadWALEdit(scope);
125    // 2. Create wal key
126    WALKeyImpl logKey = new WALKeyImpl(scope);
127    // 3. Enable bulk load hfile replication
128    Configuration bulkLoadConf = HBaseConfiguration.create(CONF);
129    bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
130
131    // 4. Get the scopes for the key
132    ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, bulkLoadConf);
133
134    NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
135    // Assert family with replication scope global is present in the key scopes
136    assertTrue(scopes.containsKey(F1),
137      "This family scope is set to global, should be part of replication key scopes.");
138    // Assert family with replication scope local is not present in the key scopes
139    assertFalse(scopes.containsKey(F2),
140      "This family scope is set to local, should not be part of replication key scopes");
141  }
142}