001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Set;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.HColumnDescriptor;
034import org.apache.hadoop.hbase.HRegionInfo;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.KeyValue;
037import org.apache.hadoop.hbase.KeyValueTestUtil;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Durability;
040import org.apache.hadoop.hbase.client.Put;
041import org.apache.hadoop.hbase.client.Scan;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.RegionServerTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053@Category({ RegionServerTests.class, MediumTests.class })
054public class TestColumnSeeking {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestColumnSeeking.class);
059
060  @Rule
061  public TestName name = new TestName();
062  private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
063
064  private static final Logger LOG = LoggerFactory.getLogger(TestColumnSeeking.class);
065
066  @SuppressWarnings("unchecked")
067  @Test
068  public void testDuplicateVersions() throws IOException {
069    String family = "Family";
070    byte[] familyBytes = Bytes.toBytes("Family");
071    TableName table = TableName.valueOf(name.getMethodName());
072
073    HColumnDescriptor hcd = new HColumnDescriptor(familyBytes).setMaxVersions(1000);
074    hcd.setMaxVersions(3);
075    HTableDescriptor htd = new HTableDescriptor(table);
076    htd.addFamily(hcd);
077    HRegionInfo info = new HRegionInfo(table, null, null, false);
078    // Set this so that the archiver writes to the temp dir as well.
079    HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
080    try {
081      List<String> rows = generateRandomWords(10, "row");
082      List<String> allColumns = generateRandomWords(10, "column");
083      List<String> values = generateRandomWords(100, "value");
084
085      long maxTimestamp = 2;
086      double selectPercent = 0.5;
087      int numberOfTests = 5;
088      double flushPercentage = 0.2;
089      double minorPercentage = 0.2;
090      double majorPercentage = 0.2;
091      double putPercentage = 0.2;
092
093      HashMap<String, KeyValue> allKVMap = new HashMap<>();
094
095      HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
096      ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
097
098      for (int i = 0; i < numberOfTests; i++) {
099        kvMaps[i] = new HashMap<>();
100        columnLists[i] = new ArrayList<>();
101        for (String column : allColumns) {
102          if (Math.random() < selectPercent) {
103            columnLists[i].add(column);
104          }
105        }
106      }
107
108      for (String value : values) {
109        for (String row : rows) {
110          Put p = new Put(Bytes.toBytes(row));
111          p.setDurability(Durability.SKIP_WAL);
112          for (String column : allColumns) {
113            for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
114              KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, value);
115              if (Math.random() < putPercentage) {
116                p.add(kv);
117                allKVMap.put(kv.getKeyString(), kv);
118                for (int i = 0; i < numberOfTests; i++) {
119                  if (columnLists[i].contains(column)) {
120                    kvMaps[i].put(kv.getKeyString(), kv);
121                  }
122                }
123              }
124            }
125          }
126          region.put(p);
127          if (Math.random() < flushPercentage) {
128            LOG.info("Flushing... ");
129            region.flush(true);
130          }
131
132          if (Math.random() < minorPercentage) {
133            LOG.info("Minor compacting... ");
134            region.compact(false);
135          }
136
137          if (Math.random() < majorPercentage) {
138            LOG.info("Major compacting... ");
139            region.compact(true);
140          }
141        }
142      }
143
144      for (int i = 0; i < numberOfTests + 1; i++) {
145        Collection<KeyValue> kvSet;
146        Scan scan = new Scan();
147        scan.setMaxVersions();
148        if (i < numberOfTests) {
149          if (columnLists[i].isEmpty()) continue; // HBASE-7700
150          kvSet = kvMaps[i].values();
151          for (String column : columnLists[i]) {
152            scan.addColumn(familyBytes, Bytes.toBytes(column));
153          }
154          LOG.info("ExplicitColumns scanner");
155          LOG.info("Columns: " + columnLists[i].size() + "  Keys: " + kvSet.size());
156        } else {
157          kvSet = allKVMap.values();
158          LOG.info("Wildcard scanner");
159          LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
160
161        }
162        InternalScanner scanner = region.getScanner(scan);
163        List<Cell> results = new ArrayList<>();
164        while (scanner.next(results))
165          ;
166        assertEquals(kvSet.size(), results.size());
167        assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
168      }
169    } finally {
170      HBaseTestingUtility.closeRegionAndWAL(region);
171    }
172
173    HBaseTestingUtility.closeRegionAndWAL(region);
174  }
175
176  @SuppressWarnings("unchecked")
177  @Test
178  public void testReseeking() throws IOException {
179    String family = "Family";
180    byte[] familyBytes = Bytes.toBytes("Family");
181    TableName table = TableName.valueOf(name.getMethodName());
182
183    HTableDescriptor htd = new HTableDescriptor(table);
184    HColumnDescriptor hcd = new HColumnDescriptor(family);
185    hcd.setMaxVersions(3);
186    htd.addFamily(hcd);
187
188    HRegionInfo info = new HRegionInfo(table, null, null, false);
189    HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
190
191    List<String> rows = generateRandomWords(10, "row");
192    List<String> allColumns = generateRandomWords(100, "column");
193
194    long maxTimestamp = 2;
195    double selectPercent = 0.5;
196    int numberOfTests = 5;
197    double flushPercentage = 0.2;
198    double minorPercentage = 0.2;
199    double majorPercentage = 0.2;
200    double putPercentage = 0.2;
201
202    HashMap<String, KeyValue> allKVMap = new HashMap<>();
203
204    HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
205    ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
206    String valueString = "Value";
207
208    for (int i = 0; i < numberOfTests; i++) {
209      kvMaps[i] = new HashMap<>();
210      columnLists[i] = new ArrayList<>();
211      for (String column : allColumns) {
212        if (Math.random() < selectPercent) {
213          columnLists[i].add(column);
214        }
215      }
216    }
217
218    for (String row : rows) {
219      Put p = new Put(Bytes.toBytes(row));
220      p.setDurability(Durability.SKIP_WAL);
221      for (String column : allColumns) {
222        for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
223          KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
224          if (Math.random() < putPercentage) {
225            p.add(kv);
226            allKVMap.put(kv.getKeyString(), kv);
227            for (int i = 0; i < numberOfTests; i++) {
228              if (columnLists[i].contains(column)) {
229                kvMaps[i].put(kv.getKeyString(), kv);
230              }
231            }
232          }
233
234        }
235      }
236      region.put(p);
237      if (Math.random() < flushPercentage) {
238        LOG.info("Flushing... ");
239        region.flush(true);
240      }
241
242      if (Math.random() < minorPercentage) {
243        LOG.info("Minor compacting... ");
244        region.compact(false);
245      }
246
247      if (Math.random() < majorPercentage) {
248        LOG.info("Major compacting... ");
249        region.compact(true);
250      }
251    }
252
253    for (int i = 0; i < numberOfTests + 1; i++) {
254      Collection<KeyValue> kvSet;
255      Scan scan = new Scan();
256      scan.setMaxVersions();
257      if (i < numberOfTests) {
258        if (columnLists[i].isEmpty()) continue; // HBASE-7700
259        kvSet = kvMaps[i].values();
260        for (String column : columnLists[i]) {
261          scan.addColumn(familyBytes, Bytes.toBytes(column));
262        }
263        LOG.info("ExplicitColumns scanner");
264        LOG.info("Columns: " + columnLists[i].size() + "  Keys: " + kvSet.size());
265      } else {
266        kvSet = allKVMap.values();
267        LOG.info("Wildcard scanner");
268        LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
269
270      }
271      InternalScanner scanner = region.getScanner(scan);
272      List<Cell> results = new ArrayList<>();
273      while (scanner.next(results))
274        ;
275      assertEquals(kvSet.size(), results.size());
276      assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
277    }
278
279    HBaseTestingUtility.closeRegionAndWAL(region);
280  }
281
282  List<String> generateRandomWords(int numberOfWords, String suffix) {
283    Set<String> wordSet = new HashSet<>();
284    for (int i = 0; i < numberOfWords; i++) {
285      int lengthOfWords = (int) (Math.random() * 5) + 1;
286      char[] wordChar = new char[lengthOfWords];
287      for (int j = 0; j < wordChar.length; j++) {
288        wordChar[j] = (char) (Math.random() * 26 + 97);
289      }
290      String word;
291      if (suffix == null) {
292        word = new String(wordChar);
293      } else {
294        word = new String(wordChar) + suffix;
295      }
296      wordSet.add(word);
297    }
298    List<String> wordList = new ArrayList<>(wordSet);
299    return wordList;
300  }
301
302}