001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Set;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HColumnDescriptor;
034import org.apache.hadoop.hbase.HRegionInfo;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.KeyValueTestUtil;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Durability;
040import org.apache.hadoop.hbase.client.Put;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.RegionServerTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053@Category({RegionServerTests.class, MediumTests.class})
054public class TestColumnSeeking {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058      HBaseClassTestRule.forClass(TestColumnSeeking.class);
059
060  @Rule public TestName name = new TestName();
061  private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
062
063  private static final Logger LOG = LoggerFactory.getLogger(TestColumnSeeking.class);
064
065  @SuppressWarnings("unchecked")
066  @Test
067  public void testDuplicateVersions() throws IOException {
068    String family = "Family";
069    byte[] familyBytes = Bytes.toBytes("Family");
070    TableName table = TableName.valueOf(name.getMethodName());
071
072    HColumnDescriptor hcd =
073        new HColumnDescriptor(familyBytes).setMaxVersions(1000);
074    hcd.setMaxVersions(3);
075    HTableDescriptor htd = new HTableDescriptor(table);
076    htd.addFamily(hcd);
077    HRegionInfo info = new HRegionInfo(table, null, null, false);
078    // Set this so that the archiver writes to the temp dir as well.
079    HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
080    try {
081      List<String> rows = generateRandomWords(10, "row");
082      List<String> allColumns = generateRandomWords(10, "column");
083      List<String> values = generateRandomWords(100, "value");
084
085      long maxTimestamp = 2;
086      double selectPercent = 0.5;
087      int numberOfTests = 5;
088      double flushPercentage = 0.2;
089      double minorPercentage = 0.2;
090      double majorPercentage = 0.2;
091      double putPercentage = 0.2;
092
093      HashMap<String, KeyValue> allKVMap = new HashMap<>();
094
095      HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
096      ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
097
098      for (int i = 0; i < numberOfTests; i++) {
099        kvMaps[i] = new HashMap<>();
100        columnLists[i] = new ArrayList<>();
101        for (String column : allColumns) {
102          if (Math.random() < selectPercent) {
103            columnLists[i].add(column);
104          }
105        }
106      }
107
108      for (String value : values) {
109        for (String row : rows) {
110          Put p = new Put(Bytes.toBytes(row));
111          p.setDurability(Durability.SKIP_WAL);
112          for (String column : allColumns) {
113            for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
114              KeyValue kv =
115                  KeyValueTestUtil.create(row, family, column, timestamp, value);
116              if (Math.random() < putPercentage) {
117                p.add(kv);
118                allKVMap.put(kv.getKeyString(), kv);
119                for (int i = 0; i < numberOfTests; i++) {
120                  if (columnLists[i].contains(column)) {
121                    kvMaps[i].put(kv.getKeyString(), kv);
122                  }
123                }
124              }
125            }
126          }
127          region.put(p);
128          if (Math.random() < flushPercentage) {
129            LOG.info("Flushing... ");
130            region.flush(true);
131          }
132
133          if (Math.random() < minorPercentage) {
134            LOG.info("Minor compacting... ");
135            region.compact(false);
136          }
137
138          if (Math.random() < majorPercentage) {
139            LOG.info("Major compacting... ");
140            region.compact(true);
141          }
142        }
143      }
144
145      for (int i = 0; i < numberOfTests + 1; i++) {
146        Collection<KeyValue> kvSet;
147        Scan scan = new Scan();
148        scan.setMaxVersions();
149        if (i < numberOfTests) {
150          if (columnLists[i].isEmpty()) continue; // HBASE-7700
151          kvSet = kvMaps[i].values();
152          for (String column : columnLists[i]) {
153            scan.addColumn(familyBytes, Bytes.toBytes(column));
154          }
155          LOG.info("ExplicitColumns scanner");
156          LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
157              + kvSet.size());
158        } else {
159          kvSet = allKVMap.values();
160          LOG.info("Wildcard scanner");
161          LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
162
163        }
164        InternalScanner scanner = region.getScanner(scan);
165        List<Cell> results = new ArrayList<>();
166        while (scanner.next(results))
167          ;
168        assertEquals(kvSet.size(), results.size());
169        assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
170      }
171    } finally {
172      HBaseTestingUtility.closeRegionAndWAL(region);
173    }
174
175    HBaseTestingUtility.closeRegionAndWAL(region);
176  }
177
178  @SuppressWarnings("unchecked")
179  @Test
180  public void testReseeking() throws IOException {
181    String family = "Family";
182    byte[] familyBytes = Bytes.toBytes("Family");
183    TableName table = TableName.valueOf(name.getMethodName());
184
185    HTableDescriptor htd = new HTableDescriptor(table);
186    HColumnDescriptor hcd = new HColumnDescriptor(family);
187    hcd.setMaxVersions(3);
188    htd.addFamily(hcd);
189
190    HRegionInfo info = new HRegionInfo(table, null, null, false);
191    HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
192
193    List<String> rows = generateRandomWords(10, "row");
194    List<String> allColumns = generateRandomWords(100, "column");
195
196    long maxTimestamp = 2;
197    double selectPercent = 0.5;
198    int numberOfTests = 5;
199    double flushPercentage = 0.2;
200    double minorPercentage = 0.2;
201    double majorPercentage = 0.2;
202    double putPercentage = 0.2;
203
204    HashMap<String, KeyValue> allKVMap = new HashMap<>();
205
206    HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
207    ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
208    String valueString = "Value";
209
210    for (int i = 0; i < numberOfTests; i++) {
211      kvMaps[i] = new HashMap<>();
212      columnLists[i] = new ArrayList<>();
213      for (String column : allColumns) {
214        if (Math.random() < selectPercent) {
215          columnLists[i].add(column);
216        }
217      }
218    }
219
220    for (String row : rows) {
221      Put p = new Put(Bytes.toBytes(row));
222      p.setDurability(Durability.SKIP_WAL);
223      for (String column : allColumns) {
224        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
225          KeyValue kv =
226              KeyValueTestUtil.create(row, family, column, timestamp,
227                  valueString);
228          if (Math.random() < putPercentage) {
229            p.add(kv);
230            allKVMap.put(kv.getKeyString(), kv);
231            for (int i = 0; i < numberOfTests; i++) {
232              if (columnLists[i].contains(column)) {
233                kvMaps[i].put(kv.getKeyString(), kv);
234              }
235            }
236          }
237
238        }
239      }
240      region.put(p);
241      if (Math.random() < flushPercentage) {
242        LOG.info("Flushing... ");
243        region.flush(true);
244      }
245
246      if (Math.random() < minorPercentage) {
247        LOG.info("Minor compacting... ");
248        region.compact(false);
249      }
250
251      if (Math.random() < majorPercentage) {
252        LOG.info("Major compacting... ");
253        region.compact(true);
254      }
255    }
256
257    for (int i = 0; i < numberOfTests + 1; i++) {
258      Collection<KeyValue> kvSet;
259      Scan scan = new Scan();
260      scan.setMaxVersions();
261      if (i < numberOfTests) {
262        if (columnLists[i].isEmpty()) continue; // HBASE-7700
263        kvSet = kvMaps[i].values();
264        for (String column : columnLists[i]) {
265          scan.addColumn(familyBytes, Bytes.toBytes(column));
266        }
267        LOG.info("ExplicitColumns scanner");
268        LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
269            + kvSet.size());
270      } else {
271        kvSet = allKVMap.values();
272        LOG.info("Wildcard scanner");
273        LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
274
275      }
276      InternalScanner scanner = region.getScanner(scan);
277      List<Cell> results = new ArrayList<>();
278      while (scanner.next(results))
279        ;
280      assertEquals(kvSet.size(), results.size());
281      assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
282    }
283
284    HBaseTestingUtility.closeRegionAndWAL(region);
285  }
286
287  List<String> generateRandomWords(int numberOfWords, String suffix) {
288    Set<String> wordSet = new HashSet<>();
289    for (int i = 0; i < numberOfWords; i++) {
290      int lengthOfWords = (int) (Math.random() * 5) + 1;
291      char[] wordChar = new char[lengthOfWords];
292      for (int j = 0; j < wordChar.length; j++) {
293        wordChar[j] = (char) (Math.random() * 26 + 97);
294      }
295      String word;
296      if (suffix == null) {
297        word = new String(wordChar);
298      } else {
299        word = new String(wordChar) + suffix;
300      }
301      wordSet.add(word);
302    }
303    List<String> wordList = new ArrayList<>(wordSet);
304    return wordList;
305  }
306
307}
308