001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import com.codahale.metrics.Histogram; 027import com.codahale.metrics.Snapshot; 028import com.codahale.metrics.UniformReservoir; 029import com.fasterxml.jackson.databind.ObjectMapper; 030import java.io.BufferedReader; 031import java.io.ByteArrayInputStream; 032import java.io.IOException; 033import java.io.InputStreamReader; 034import java.lang.reflect.Constructor; 035import java.lang.reflect.InvocationTargetException; 036import java.util.LinkedList; 037import java.util.NoSuchElementException; 038import java.util.Queue; 039import java.util.Random; 040import org.apache.hadoop.fs.FSDataInputStream; 041import org.apache.hadoop.fs.FileSystem; 042import org.apache.hadoop.fs.Path; 043import org.apache.hadoop.hbase.PerformanceEvaluation.RandomReadTest; 044import org.apache.hadoop.hbase.PerformanceEvaluation.TestOptions; 045import org.apache.hadoop.hbase.regionserver.CompactingMemStore; 046import org.apache.hadoop.hbase.testclassification.MiscTests; 047import org.apache.hadoop.hbase.testclassification.SmallTests; 048import org.junit.ClassRule; 049import org.junit.Ignore; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052 053@Category({MiscTests.class, SmallTests.class}) 054public class TestPerformanceEvaluation { 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestPerformanceEvaluation.class); 058 059 private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); 060 061 @Test 062 public void testDefaultInMemoryCompaction() { 063 PerformanceEvaluation.TestOptions defaultOpts = 064 new PerformanceEvaluation.TestOptions(); 065 assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT, 066 defaultOpts.getInMemoryCompaction().toString()); 067 HTableDescriptor htd = PerformanceEvaluation.getTableDescriptor(defaultOpts); 068 for (HColumnDescriptor hcd: htd.getFamilies()) { 069 assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT, 070 hcd.getInMemoryCompaction().toString()); 071 } 072 } 073 074 @Test 075 public void testSerialization() throws IOException { 076 PerformanceEvaluation.TestOptions options = new PerformanceEvaluation.TestOptions(); 077 assertFalse(options.isAutoFlush()); 078 options.setAutoFlush(true); 079 ObjectMapper mapper = new ObjectMapper(); 080 String optionsString = mapper.writeValueAsString(options); 081 PerformanceEvaluation.TestOptions optionsDeserialized = 082 mapper.readValue(optionsString, PerformanceEvaluation.TestOptions.class); 083 assertTrue(optionsDeserialized.isAutoFlush()); 084 } 085 086 /** 087 * Exercise the mr spec writing. Simple assertions to make sure it is basically working. 088 */ 089 @Ignore @Test 090 public void testWriteInputFile() throws IOException { 091 TestOptions opts = new PerformanceEvaluation.TestOptions(); 092 final int clients = 10; 093 opts.setNumClientThreads(clients); 094 opts.setPerClientRunRows(10); 095 Path dir = 096 PerformanceEvaluation.writeInputFile(HTU.getConfiguration(), opts, HTU.getDataTestDir()); 097 FileSystem fs = FileSystem.get(HTU.getConfiguration()); 098 Path p = new Path(dir, PerformanceEvaluation.JOB_INPUT_FILENAME); 099 long len = fs.getFileStatus(p).getLen(); 100 assertTrue(len > 0); 101 byte[] content = new byte[(int) len]; 102 try (FSDataInputStream dis = fs.open(p)) { 103 dis.readFully(content); 104 BufferedReader br = 105 new BufferedReader(new InputStreamReader(new ByteArrayInputStream(content))); 106 int count = 0; 107 while (br.readLine() != null) { 108 count++; 109 } 110 assertEquals(clients, count); 111 } 112 } 113 114 @Test 115 public void testSizeCalculation() { 116 TestOptions opts = new PerformanceEvaluation.TestOptions(); 117 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 118 int rows = opts.getPerClientRunRows(); 119 // Default row count 120 final int defaultPerClientRunRows = 1024 * 1024; 121 assertEquals(defaultPerClientRunRows, rows); 122 // If size is 2G, then twice the row count. 123 opts.setSize(2.0f); 124 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 125 assertEquals(defaultPerClientRunRows * 2, opts.getPerClientRunRows()); 126 // If two clients, then they get half the rows each. 127 opts.setNumClientThreads(2); 128 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 129 assertEquals(defaultPerClientRunRows, opts.getPerClientRunRows()); 130 // What if valueSize is 'random'? Then half of the valueSize so twice the rows. 131 opts.valueRandom = true; 132 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 133 assertEquals(defaultPerClientRunRows * 2, opts.getPerClientRunRows()); 134 } 135 136 @Test 137 public void testRandomReadCalculation() { 138 TestOptions opts = new PerformanceEvaluation.TestOptions(); 139 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 140 int rows = opts.getPerClientRunRows(); 141 // Default row count 142 final int defaultPerClientRunRows = 1024 * 1024; 143 assertEquals(defaultPerClientRunRows, rows); 144 // If size is 2G, then twice the row count. 145 opts.setSize(2.0f); 146 opts.setPerClientRunRows(1000); 147 opts.setCmdName(PerformanceEvaluation.RANDOM_READ); 148 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 149 assertEquals(1000, opts.getPerClientRunRows()); 150 // If two clients, then they get half the rows each. 151 opts.setNumClientThreads(2); 152 opts = PerformanceEvaluation.calculateRowsAndSize(opts); 153 assertEquals(1000, opts.getPerClientRunRows()); 154 Random random = new Random(); 155 // assuming we will get one before this loop expires 156 boolean foundValue = false; 157 for (int i = 0; i < 10000000; i++) { 158 int randomRow = PerformanceEvaluation.generateRandomRow(random, opts.totalRows); 159 if (randomRow > 1000) { 160 foundValue = true; 161 break; 162 } 163 } 164 assertTrue("We need to get a value more than 1000", foundValue); 165 } 166 167 @Test 168 public void testZipfian() throws NoSuchMethodException, SecurityException, InstantiationException, 169 IllegalAccessException, IllegalArgumentException, InvocationTargetException { 170 TestOptions opts = new PerformanceEvaluation.TestOptions(); 171 opts.setValueZipf(true); 172 final int valueSize = 1024; 173 opts.setValueSize(valueSize); 174 RandomReadTest rrt = new RandomReadTest(null, opts, null); 175 Constructor<?> ctor = 176 Histogram.class.getDeclaredConstructor(com.codahale.metrics.Reservoir.class); 177 ctor.setAccessible(true); 178 Histogram histogram = (Histogram)ctor.newInstance(new UniformReservoir(1024 * 500)); 179 for (int i = 0; i < 100; i++) { 180 histogram.update(rrt.getValueLength(null)); 181 } 182 Snapshot snapshot = histogram.getSnapshot(); 183 double stddev = snapshot.getStdDev(); 184 assertTrue(stddev != 0 && stddev != 1.0); 185 assertTrue(snapshot.getStdDev() != 0); 186 double median = snapshot.getMedian(); 187 assertTrue(median != 0 && median != 1 && median != valueSize); 188 } 189 190 @Test 191 public void testSetBufferSizeOption() { 192 TestOptions opts = new PerformanceEvaluation.TestOptions(); 193 long bufferSize = opts.getBufferSize(); 194 assertEquals(bufferSize, 2L * 1024L * 1024L); 195 opts.setBufferSize(64L * 1024L); 196 bufferSize = opts.getBufferSize(); 197 assertEquals(bufferSize, 64L * 1024L); 198 } 199 200 @Test 201 public void testParseOptsWithThreads() { 202 Queue<String> opts = new LinkedList<>(); 203 String cmdName = "sequentialWrite"; 204 int threads = 1; 205 opts.offer(cmdName); 206 opts.offer(String.valueOf(threads)); 207 PerformanceEvaluation.TestOptions options = PerformanceEvaluation.parseOpts(opts); 208 assertNotNull(options); 209 assertNotNull(options.getCmdName()); 210 assertEquals(cmdName, options.getCmdName()); 211 assertEquals(threads, options.getNumClientThreads()); 212 } 213 214 @Test 215 public void testParseOptsWrongThreads() { 216 Queue<String> opts = new LinkedList<>(); 217 String cmdName = "sequentialWrite"; 218 opts.offer(cmdName); 219 opts.offer("qq"); 220 try { 221 PerformanceEvaluation.parseOpts(opts); 222 } catch (IllegalArgumentException e) { 223 System.out.println(e.getMessage()); 224 assertEquals("Command " + cmdName + " does not have threads number", e.getMessage()); 225 assertTrue(e.getCause() instanceof NumberFormatException); 226 } 227 } 228 229 @Test 230 public void testParseOptsNoThreads() { 231 Queue<String> opts = new LinkedList<>(); 232 String cmdName = "sequentialWrite"; 233 try { 234 PerformanceEvaluation.parseOpts(opts); 235 } catch (IllegalArgumentException e) { 236 System.out.println(e.getMessage()); 237 assertEquals("Command " + cmdName + " does not have threads number", e.getMessage()); 238 assertTrue(e.getCause() instanceof NoSuchElementException); 239 } 240 } 241 242 @Test 243 public void testParseOptsMultiPuts() { 244 Queue<String> opts = new LinkedList<>(); 245 String cmdName = "sequentialWrite"; 246 opts.offer("--multiPut=10"); 247 opts.offer(cmdName); 248 opts.offer("64"); 249 PerformanceEvaluation.TestOptions options = null; 250 try { 251 options = PerformanceEvaluation.parseOpts(opts); 252 fail("should fail"); 253 } catch (IllegalArgumentException e) { 254 System.out.println(e.getMessage()); 255 } 256 ((LinkedList<String>) opts).offerFirst("--multiPut=10"); 257 ((LinkedList<String>) opts).offerFirst("--autoFlush=true"); 258 options = PerformanceEvaluation.parseOpts(opts); 259 assertNotNull(options); 260 assertNotNull(options.getCmdName()); 261 assertEquals(cmdName, options.getCmdName()); 262 assertEquals(10, options.getMultiPut()); 263 } 264 265 @Test 266 public void testParseOptsConnCount() { 267 Queue<String> opts = new LinkedList<>(); 268 String cmdName = "sequentialWrite"; 269 opts.offer("--oneCon=true"); 270 opts.offer("--connCount=10"); 271 opts.offer(cmdName); 272 opts.offer("64"); 273 PerformanceEvaluation.TestOptions options = null; 274 try { 275 options = PerformanceEvaluation.parseOpts(opts); 276 fail("should fail"); 277 } catch (IllegalArgumentException e) { 278 System.out.println(e.getMessage()); 279 } 280 ((LinkedList<String>) opts).offerFirst("--connCount=10"); 281 options = PerformanceEvaluation.parseOpts(opts); 282 assertNotNull(options); 283 assertNotNull(options.getCmdName()); 284 assertEquals(cmdName, options.getCmdName()); 285 assertEquals(10, options.getConnCount()); 286 } 287}