1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.util;
18
19 import java.nio.ByteBuffer;
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Random;
26
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.Tag;
30 import org.apache.hadoop.io.WritableUtils;
31
32 import com.google.common.primitives.Bytes;
33
34
35
36
37
38 @edu.umd.cs.findbugs.annotations.SuppressWarnings(
39 value="RV_ABSOLUTE_VALUE_OF_RANDOM_INT",
40 justification="Should probably fix")
41 @InterfaceAudience.Private
42 public class RedundantKVGenerator {
43
44 static byte[] DEFAULT_COMMON_PREFIX = new byte[0];
45 static int DEFAULT_NUMBER_OF_ROW_PREFIXES = 10;
46 static int DEFAULT_AVERAGE_PREFIX_LENGTH = 6;
47 static int DEFAULT_PREFIX_LENGTH_VARIANCE = 3;
48 static int DEFAULT_AVERAGE_SUFFIX_LENGTH = 3;
49 static int DEFAULT_SUFFIX_LENGTH_VARIANCE = 3;
50 static int DEFAULT_NUMBER_OF_ROW = 500;
51
52
53 static float DEFAULT_CHANCE_FOR_SAME_QUALIFIER = 0.5f;
54 static float DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER = 0.4f;
55 static int DEFAULT_AVERAGE_QUALIFIER_LENGTH = 9;
56 static int DEFAULT_QUALIFIER_LENGTH_VARIANCE = 3;
57
58 static int DEFAULT_COLUMN_FAMILY_LENGTH = 9;
59 static int DEFAULT_VALUE_LENGTH = 8;
60 static float DEFAULT_CHANCE_FOR_ZERO_VALUE = 0.5f;
61
62 static int DEFAULT_BASE_TIMESTAMP_DIVIDE = 1000000;
63 static int DEFAULT_TIMESTAMP_DIFF_SIZE = 100000000;
64
65
66
67
68 public RedundantKVGenerator() {
69 this(new Random(42L),
70 DEFAULT_NUMBER_OF_ROW_PREFIXES,
71 DEFAULT_AVERAGE_PREFIX_LENGTH,
72 DEFAULT_PREFIX_LENGTH_VARIANCE,
73 DEFAULT_AVERAGE_SUFFIX_LENGTH,
74 DEFAULT_SUFFIX_LENGTH_VARIANCE,
75 DEFAULT_NUMBER_OF_ROW,
76
77 DEFAULT_CHANCE_FOR_SAME_QUALIFIER,
78 DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER,
79 DEFAULT_AVERAGE_QUALIFIER_LENGTH,
80 DEFAULT_QUALIFIER_LENGTH_VARIANCE,
81
82 DEFAULT_COLUMN_FAMILY_LENGTH,
83 DEFAULT_VALUE_LENGTH,
84 DEFAULT_CHANCE_FOR_ZERO_VALUE,
85
86 DEFAULT_BASE_TIMESTAMP_DIVIDE,
87 DEFAULT_TIMESTAMP_DIFF_SIZE
88 );
89 }
90
91
92
93
94
95
96 public RedundantKVGenerator(Random randomizer,
97 int numberOfRowPrefixes,
98 int averagePrefixLength,
99 int prefixLengthVariance,
100 int averageSuffixLength,
101 int suffixLengthVariance,
102 int numberOfRows,
103
104 float chanceForSameQualifier,
105 float chanceForSimiliarQualifier,
106 int averageQualifierLength,
107 int qualifierLengthVariance,
108
109 int columnFamilyLength,
110 int valueLength,
111 float chanceForZeroValue,
112
113 int baseTimestampDivide,
114 int timestampDiffSize
115 ) {
116 this.randomizer = randomizer;
117
118 this.commonPrefix = DEFAULT_COMMON_PREFIX;
119 this.numberOfRowPrefixes = numberOfRowPrefixes;
120 this.averagePrefixLength = averagePrefixLength;
121 this.prefixLengthVariance = prefixLengthVariance;
122 this.averageSuffixLength = averageSuffixLength;
123 this.suffixLengthVariance = suffixLengthVariance;
124 this.numberOfRows = numberOfRows;
125
126 this.chanceForSameQualifier = chanceForSameQualifier;
127 this.chanceForSimilarQualifier = chanceForSimiliarQualifier;
128 this.averageQualifierLength = averageQualifierLength;
129 this.qualifierLengthVariance = qualifierLengthVariance;
130
131 this.columnFamilyLength = columnFamilyLength;
132 this.valueLength = valueLength;
133 this.chanceForZeroValue = chanceForZeroValue;
134
135 this.baseTimestampDivide = baseTimestampDivide;
136 this.timestampDiffSize = timestampDiffSize;
137 }
138
139
140 private Random randomizer;
141
142
143 private byte[] commonPrefix;
144 private int numberOfRowPrefixes;
145 private int averagePrefixLength = 6;
146 private int prefixLengthVariance = 3;
147 private int averageSuffixLength = 3;
148 private int suffixLengthVariance = 3;
149 private int numberOfRows = 500;
150
151
152 private byte[] family;
153
154
155 private float chanceForSameQualifier = 0.5f;
156 private float chanceForSimilarQualifier = 0.4f;
157 private int averageQualifierLength = 9;
158 private int qualifierLengthVariance = 3;
159
160 private int columnFamilyLength = 9;
161 private int valueLength = 8;
162 private float chanceForZeroValue = 0.5f;
163
164 private int baseTimestampDivide = 1000000;
165 private int timestampDiffSize = 100000000;
166
167 private List<byte[]> generateRows() {
168
169 List<byte[]> prefixes = new ArrayList<byte[]>();
170 prefixes.add(new byte[0]);
171 for (int i = 1; i < numberOfRowPrefixes; ++i) {
172 int prefixLength = averagePrefixLength;
173 prefixLength += randomizer.nextInt(2 * prefixLengthVariance + 1) -
174 prefixLengthVariance;
175 byte[] newPrefix = new byte[prefixLength];
176 randomizer.nextBytes(newPrefix);
177 byte[] newPrefixWithCommon = newPrefix;
178 prefixes.add(newPrefixWithCommon);
179 }
180
181
182 List<byte[]> rows = new ArrayList<byte[]>();
183 for (int i = 0; i < numberOfRows; ++i) {
184 int suffixLength = averageSuffixLength;
185 suffixLength += randomizer.nextInt(2 * suffixLengthVariance + 1) -
186 suffixLengthVariance;
187 int randomPrefix = randomizer.nextInt(prefixes.size());
188 byte[] row = new byte[prefixes.get(randomPrefix).length +
189 suffixLength];
190 byte[] rowWithCommonPrefix = Bytes.concat(commonPrefix, row);
191 rows.add(rowWithCommonPrefix);
192 }
193
194 return rows;
195 }
196
197
198
199
200
201
202 public List<KeyValue> generateTestKeyValues(int howMany) {
203 return generateTestKeyValues(howMany, false);
204 }
205
206
207
208
209
210 public List<KeyValue> generateTestKeyValues(int howMany, boolean useTags) {
211 List<KeyValue> result = new ArrayList<KeyValue>();
212
213 List<byte[]> rows = generateRows();
214 Map<Integer, List<byte[]>> rowsToQualifier = new HashMap<Integer, List<byte[]>>();
215
216 if(family==null){
217 family = new byte[columnFamilyLength];
218 randomizer.nextBytes(family);
219 }
220
221 long baseTimestamp = Math.abs(randomizer.nextInt()) / baseTimestampDivide;
222
223 byte[] value = new byte[valueLength];
224
225 for (int i = 0; i < howMany; ++i) {
226 long timestamp = baseTimestamp;
227 if(timestampDiffSize > 0){
228 timestamp += randomizer.nextInt(timestampDiffSize);
229 }
230 Integer rowId = randomizer.nextInt(rows.size());
231 byte[] row = rows.get(rowId);
232
233
234
235 byte[] qualifier;
236 float qualifierChance = randomizer.nextFloat();
237 if (!rowsToQualifier.containsKey(rowId)
238 || qualifierChance > chanceForSameQualifier + chanceForSimilarQualifier) {
239 int qualifierLength = averageQualifierLength;
240 qualifierLength += randomizer.nextInt(2 * qualifierLengthVariance + 1)
241 - qualifierLengthVariance;
242 qualifier = new byte[qualifierLength];
243 randomizer.nextBytes(qualifier);
244
245
246 if (!rowsToQualifier.containsKey(rowId)) {
247 rowsToQualifier.put(rowId, new ArrayList<byte[]>());
248 }
249 rowsToQualifier.get(rowId).add(qualifier);
250 } else if (qualifierChance > chanceForSameQualifier) {
251
252 List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
253 byte[] originalQualifier = previousQualifiers.get(randomizer.nextInt(previousQualifiers
254 .size()));
255
256 qualifier = new byte[originalQualifier.length];
257 int commonPrefix = randomizer.nextInt(qualifier.length);
258 System.arraycopy(originalQualifier, 0, qualifier, 0, commonPrefix);
259 for (int j = commonPrefix; j < qualifier.length; ++j) {
260 qualifier[j] = (byte) (randomizer.nextInt() & 0xff);
261 }
262
263 rowsToQualifier.get(rowId).add(qualifier);
264 } else {
265
266 List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
267 qualifier = previousQualifiers.get(randomizer.nextInt(previousQualifiers.size()));
268 }
269
270 if (randomizer.nextFloat() < chanceForZeroValue) {
271 for (int j = 0; j < value.length; ++j) {
272 value[j] = (byte) 0;
273 }
274 } else {
275 randomizer.nextBytes(value);
276 }
277
278 if (useTags) {
279 result.add(new KeyValue(row, family, qualifier, timestamp, value, new Tag[] { new Tag(
280 (byte) 1, "value1") }));
281 } else {
282 result.add(new KeyValue(row, family, qualifier, timestamp, value));
283 }
284 }
285
286 Collections.sort(result, KeyValue.COMPARATOR);
287
288 return result;
289 }
290
291
292
293
294
295
296 public static ByteBuffer convertKvToByteBuffer(List<KeyValue> keyValues,
297 boolean includesMemstoreTS) {
298 int totalSize = 0;
299 for (KeyValue kv : keyValues) {
300 totalSize += kv.getLength();
301 if (includesMemstoreTS) {
302 totalSize += WritableUtils.getVIntSize(kv.getMvccVersion());
303 }
304 }
305
306 ByteBuffer result = ByteBuffer.allocate(totalSize);
307 for (KeyValue kv : keyValues) {
308 result.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
309 if (includesMemstoreTS) {
310 ByteBufferUtils.writeVLong(result, kv.getMvccVersion());
311 }
312 }
313 return result;
314 }
315
316
317
318 public RedundantKVGenerator setCommonPrefix(byte[] prefix){
319 this.commonPrefix = prefix;
320 return this;
321 }
322
323 public RedundantKVGenerator setRandomizer(Random randomizer) {
324 this.randomizer = randomizer;
325 return this;
326 }
327
328 public RedundantKVGenerator setNumberOfRowPrefixes(int numberOfRowPrefixes) {
329 this.numberOfRowPrefixes = numberOfRowPrefixes;
330 return this;
331 }
332
333 public RedundantKVGenerator setAveragePrefixLength(int averagePrefixLength) {
334 this.averagePrefixLength = averagePrefixLength;
335 return this;
336 }
337
338 public RedundantKVGenerator setPrefixLengthVariance(int prefixLengthVariance) {
339 this.prefixLengthVariance = prefixLengthVariance;
340 return this;
341 }
342
343 public RedundantKVGenerator setAverageSuffixLength(int averageSuffixLength) {
344 this.averageSuffixLength = averageSuffixLength;
345 return this;
346 }
347
348 public RedundantKVGenerator setSuffixLengthVariance(int suffixLengthVariance) {
349 this.suffixLengthVariance = suffixLengthVariance;
350 return this;
351 }
352
353 public RedundantKVGenerator setNumberOfRows(int numberOfRows) {
354 this.numberOfRows = numberOfRows;
355 return this;
356 }
357
358 public RedundantKVGenerator setChanceForSameQualifier(float chanceForSameQualifier) {
359 this.chanceForSameQualifier = chanceForSameQualifier;
360 return this;
361 }
362
363 public RedundantKVGenerator setChanceForSimilarQualifier(float chanceForSimiliarQualifier) {
364 this.chanceForSimilarQualifier = chanceForSimiliarQualifier;
365 return this;
366 }
367
368 public RedundantKVGenerator setAverageQualifierLength(int averageQualifierLength) {
369 this.averageQualifierLength = averageQualifierLength;
370 return this;
371 }
372
373 public RedundantKVGenerator setQualifierLengthVariance(int qualifierLengthVariance) {
374 this.qualifierLengthVariance = qualifierLengthVariance;
375 return this;
376 }
377
378 public RedundantKVGenerator setColumnFamilyLength(int columnFamilyLength) {
379 this.columnFamilyLength = columnFamilyLength;
380 return this;
381 }
382
383 public RedundantKVGenerator setFamily(byte[] family) {
384 this.family = family;
385 this.columnFamilyLength = family.length;
386 return this;
387 }
388
389 public RedundantKVGenerator setValueLength(int valueLength) {
390 this.valueLength = valueLength;
391 return this;
392 }
393
394 public RedundantKVGenerator setChanceForZeroValue(float chanceForZeroValue) {
395 this.chanceForZeroValue = chanceForZeroValue;
396 return this;
397 }
398
399 public RedundantKVGenerator setBaseTimestampDivide(int baseTimestampDivide) {
400 this.baseTimestampDivide = baseTimestampDivide;
401 return this;
402 }
403
404 public RedundantKVGenerator setTimestampDiffSize(int timestampDiffSize) {
405 this.timestampDiffSize = timestampDiffSize;
406 return this;
407 }
408
409 }