1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Mockito.doThrow;
40 import static org.mockito.Mockito.mock;
41 import static org.mockito.Mockito.never;
42 import static org.mockito.Mockito.spy;
43 import static org.mockito.Mockito.times;
44 import static org.mockito.Mockito.verify;
45 import static org.mockito.Mockito.when;
46
47 import java.io.IOException;
48 import java.io.InterruptedIOException;
49 import java.security.PrivilegedExceptionAction;
50 import java.util.ArrayList;
51 import java.util.Arrays;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.HashMap;
55 import java.util.List;
56 import java.util.Map;
57 import java.util.NavigableMap;
58 import java.util.TreeMap;
59 import java.util.UUID;
60 import java.util.concurrent.Callable;
61 import java.util.concurrent.CountDownLatch;
62 import java.util.concurrent.ExecutorService;
63 import java.util.concurrent.Executors;
64 import java.util.concurrent.Future;
65 import java.util.concurrent.TimeUnit;
66 import java.util.concurrent.atomic.AtomicBoolean;
67 import java.util.concurrent.atomic.AtomicInteger;
68 import java.util.concurrent.atomic.AtomicReference;
69
70 import org.apache.commons.lang.RandomStringUtils;
71 import org.apache.commons.logging.Log;
72 import org.apache.commons.logging.LogFactory;
73 import org.apache.hadoop.conf.Configuration;
74 import org.apache.hadoop.fs.FSDataOutputStream;
75 import org.apache.hadoop.fs.FileStatus;
76 import org.apache.hadoop.fs.FileSystem;
77 import org.apache.hadoop.fs.Path;
78 import org.apache.hadoop.hbase.Cell;
79 import org.apache.hadoop.hbase.CellComparator;
80 import org.apache.hadoop.hbase.CellUtil;
81 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
82 import org.apache.hadoop.hbase.DroppedSnapshotException;
83 import org.apache.hadoop.hbase.HBaseConfiguration;
84 import org.apache.hadoop.hbase.HBaseTestCase;
85 import org.apache.hadoop.hbase.HBaseTestingUtility;
86 import org.apache.hadoop.hbase.HColumnDescriptor;
87 import org.apache.hadoop.hbase.HConstants;
88 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
89 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
90 import org.apache.hadoop.hbase.HRegionInfo;
91 import org.apache.hadoop.hbase.HTableDescriptor;
92 import org.apache.hadoop.hbase.KeyValue;
93 import org.apache.hadoop.hbase.KeyValueUtil;
94 import org.apache.hadoop.hbase.MiniHBaseCluster;
95 import org.apache.hadoop.hbase.MultithreadedTestUtil;
96 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
97 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
98 import org.apache.hadoop.hbase.NotServingRegionException;
99 import org.apache.hadoop.hbase.RegionTooBusyException;
100 import org.apache.hadoop.hbase.ServerName;
101 import org.apache.hadoop.hbase.TableName;
102 import org.apache.hadoop.hbase.Tag;
103 import org.apache.hadoop.hbase.TagType;
104 import org.apache.hadoop.hbase.Waiter;
105 import org.apache.hadoop.hbase.client.Append;
106 import org.apache.hadoop.hbase.client.Delete;
107 import org.apache.hadoop.hbase.client.Durability;
108 import org.apache.hadoop.hbase.client.Get;
109 import org.apache.hadoop.hbase.client.Increment;
110 import org.apache.hadoop.hbase.client.Mutation;
111 import org.apache.hadoop.hbase.client.Put;
112 import org.apache.hadoop.hbase.client.Result;
113 import org.apache.hadoop.hbase.client.RowMutations;
114 import org.apache.hadoop.hbase.client.Scan;
115 import org.apache.hadoop.hbase.client.Table;
116 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
117 import org.apache.hadoop.hbase.filter.BinaryComparator;
118 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
119 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
120 import org.apache.hadoop.hbase.filter.Filter;
121 import org.apache.hadoop.hbase.filter.FilterBase;
122 import org.apache.hadoop.hbase.filter.FilterList;
123 import org.apache.hadoop.hbase.filter.NullComparator;
124 import org.apache.hadoop.hbase.filter.PrefixFilter;
125 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
126 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
127 import org.apache.hadoop.hbase.io.hfile.HFile;
128 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
129 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
130 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
131 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
136 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
137 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
138 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
139 import org.apache.hadoop.hbase.regionserver.Region.RowLock;
140 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
141 import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler;
142 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
143 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
144 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
145 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
146 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
147 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
148 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
149 import org.apache.hadoop.hbase.security.User;
150 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
151 import org.apache.hadoop.hbase.testclassification.MediumTests;
152 import org.apache.hadoop.hbase.util.Bytes;
153 import org.apache.hadoop.hbase.util.ClassSize;
154 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
155 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
156 import org.apache.hadoop.hbase.util.FSUtils;
157 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
158 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
159 import org.apache.hadoop.hbase.util.PairOfSameType;
160 import org.apache.hadoop.hbase.util.Threads;
161 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
162 import org.apache.hadoop.hbase.wal.FaultyFSLog;
163 import org.apache.hadoop.hbase.wal.WAL;
164 import org.apache.hadoop.hbase.wal.WALFactory;
165 import org.apache.hadoop.hbase.wal.WALKey;
166 import org.apache.hadoop.hbase.wal.WALProvider;
167 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
168 import org.apache.hadoop.hbase.wal.WALSplitter;
169 import org.junit.After;
170 import org.junit.Assert;
171 import org.junit.Before;
172 import org.junit.Rule;
173 import org.junit.Test;
174 import org.junit.experimental.categories.Category;
175 import org.junit.rules.TestName;
176 import org.mockito.ArgumentCaptor;
177 import org.mockito.ArgumentMatcher;
178 import org.mockito.Mockito;
179 import org.mockito.invocation.InvocationOnMock;
180 import org.mockito.stubbing.Answer;
181
182 import com.google.common.collect.ImmutableList;
183 import com.google.common.collect.Lists;
184 import com.google.common.collect.Maps;
185 import com.google.protobuf.ByteString;
186
187
188
189
190
191
192
193 @Category(MediumTests.class)
194 @SuppressWarnings("deprecation")
195 public class TestHRegion {
196
197
198 private static final Log LOG = LogFactory.getLog(TestHRegion.class);
199 @Rule public TestName name = new TestName();
200
201 private static final String COLUMN_FAMILY = "MyCF";
202 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
203
204 HRegion region = null;
205
206 private static HBaseTestingUtility TEST_UTIL;
207 public static Configuration CONF ;
208 private String dir;
209 private static FileSystem FILESYSTEM;
210 private final int MAX_VERSIONS = 2;
211
212
213 protected byte[] tableName;
214 protected String method;
215 protected final byte[] qual1 = Bytes.toBytes("qual1");
216 protected final byte[] qual2 = Bytes.toBytes("qual2");
217 protected final byte[] qual3 = Bytes.toBytes("qual3");
218 protected final byte[] value1 = Bytes.toBytes("value1");
219 protected final byte[] value2 = Bytes.toBytes("value2");
220 protected final byte[] row = Bytes.toBytes("rowA");
221 protected final byte[] row2 = Bytes.toBytes("rowB");
222
223 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
224 .getInstance(MetricsAssertHelper.class);
225
226 @Before
227 public void setup() throws IOException {
228 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
229 FILESYSTEM = TEST_UTIL.getTestFileSystem();
230 CONF = TEST_UTIL.getConfiguration();
231 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
232 method = name.getMethodName();
233 tableName = Bytes.toBytes(name.getMethodName());
234 }
235
236 @After
237 public void tearDown() throws Exception {
238
239 HBaseTestingUtility.closeRegionAndWAL(region);
240 EnvironmentEdgeManagerTestHelper.reset();
241 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
242 TEST_UTIL.cleanupTestDir();
243 }
244
245 String getName() {
246 return name.getMethodName();
247 }
248
249
250
251
252
253 @Test (timeout = 100000)
254 public void testSequenceId() throws IOException {
255 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
256 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
257
258 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
259 HBaseTestingUtility.closeRegionAndWAL(this.region);
260 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
261 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
262
263 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
264 byte [] value = Bytes.toBytes(name.getMethodName());
265
266 Put put = new Put(value);
267 put.addColumn(COLUMN_FAMILY_BYTES, null, value);
268 region.put(put);
269
270 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
271 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
272 region.flush(true);
273 long max = region.getMaxFlushedSeqId();
274 HBaseTestingUtility.closeRegionAndWAL(this.region);
275 assertEquals(max, region.getMaxFlushedSeqId());
276 this.region = null;
277 }
278
279
280
281
282
283
284
285
286
287
288
289 @Test (timeout=60000)
290 public void testCloseCarryingSnapshot() throws IOException {
291 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
292 Store store = region.getStore(COLUMN_FAMILY_BYTES);
293
294 byte [] value = Bytes.toBytes(name.getMethodName());
295
296 Put put = new Put(value);
297 put.add(COLUMN_FAMILY_BYTES, null, value);
298
299 region.put(put);
300 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
301 storeFlushCtx.prepare();
302
303 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
304 region.put(put);
305
306 HBaseTestingUtility.closeRegionAndWAL(region);
307 assertEquals(0, region.getMemstoreSize());
308 region = null;
309 }
310
311
312
313
314
315 @Test (timeout=60000)
316 public void testMemstoreSnapshotSize() throws IOException {
317 class MyFaultyFSLog extends FaultyFSLog {
318 StoreFlushContext storeFlushCtx;
319 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
320 throws IOException {
321 super(fs, rootDir, logName, conf);
322 }
323
324 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
325 this.storeFlushCtx = storeFlushCtx;
326 }
327
328 @Override
329 public void sync(long txid) throws IOException {
330 storeFlushCtx.prepare();
331 super.sync(txid);
332 }
333 }
334
335 FileSystem fs = FileSystem.get(CONF);
336 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
337 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
338 region = initHRegion(tableName, null, null, name.getMethodName(),
339 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
340
341 Store store = region.getStore(COLUMN_FAMILY_BYTES);
342
343 byte [] value = Bytes.toBytes(name.getMethodName());
344 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
345
346 Put put = new Put(value);
347 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
348 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
349
350 boolean threwIOE = false;
351 try {
352 region.put(put);
353 } catch (IOException ioe) {
354 threwIOE = true;
355 } finally {
356 assertTrue("The regionserver should have thrown an exception", threwIOE);
357 }
358 long sz = store.getFlushableSize();
359 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
360 }
361
362
363
364
365 @Test
366 public void testMemstoreSizeWithFlushCanceling() throws IOException {
367 FileSystem fs = FileSystem.get(CONF);
368 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
369 FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
370 region = initHRegion(tableName, null, null, name.getMethodName(),
371 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
372 Store store = region.getStore(COLUMN_FAMILY_BYTES);
373 assertEquals(0, region.getMemstoreSize());
374
375
376 byte [] value = Bytes.toBytes(name.getMethodName());
377 Put put = new Put(value);
378 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
379 region.put(put);
380 long onePutSize = region.getMemstoreSize();
381 assertTrue(onePutSize > 0);
382 region.flush(true);
383 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
384 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
385
386
387 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
388 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
389 when(mockedCPHost.preFlush(Mockito.isA(HStore.class), Mockito.isA(InternalScanner.class))).
390 thenReturn(null);
391 region.setCoprocessorHost(mockedCPHost);
392 region.put(put);
393 region.flush(true);
394 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
395 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
396
397
398 region.setCoprocessorHost(normalCPHost);
399 region.flush(true);
400 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
401 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
402 }
403
404 @Test
405 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
406 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
407 FileSystem fs = FileSystem.get(CONF);
408 Path rootDir = new Path(dir + testName);
409 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
410 region = initHRegion(tableName, null, null, name.getMethodName(),
411 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
412 Store store = region.getStore(COLUMN_FAMILY_BYTES);
413 assertEquals(0, region.getMemstoreSize());
414
415
416 byte [] value = Bytes.toBytes(name.getMethodName());
417 Put put = new Put(value);
418 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
419 region.put(put);
420 long onePutSize = region.getMemstoreSize();
421 assertTrue(onePutSize > 0);
422
423 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
424 doThrow(new IOException())
425 .when(mockedCPHost).postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>>any());
426 region.setCoprocessorHost(mockedCPHost);
427
428 put = new Put(value);
429 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
430 try {
431 region.put(put);
432 fail("Should have failed with IOException");
433 } catch (IOException expected) {
434 }
435 long expectedSize = onePutSize * 2 - ClassSize.ARRAY;
436 assertEquals("memstoreSize should be incremented", expectedSize, region.getMemstoreSize());
437 assertEquals("flushable size should be incremented", expectedSize, store.getFlushableSize());
438
439 region.setCoprocessorHost(null);
440 }
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457 @Test (timeout=60000)
458 public void testFlushSizeAccounting() throws Exception {
459 final Configuration conf = HBaseConfiguration.create(CONF);
460
461 conf.setInt("hbase.hstore.flush.retries.number", 1);
462 final User user =
463 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
464
465 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
466 user.runAs(new PrivilegedExceptionAction<Object>() {
467 @Override
468 public Object run() throws Exception {
469
470 FileSystem fs = FileSystem.get(conf);
471 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
472 FaultyFileSystem ffs = (FaultyFileSystem)fs;
473 HRegion region = null;
474 try {
475
476 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
477 long size = region.getMemstoreSize();
478 Assert.assertEquals(0, size);
479
480 Put p1 = new Put(row);
481 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
482 region.put(p1);
483 final long sizeOfOnePut = region.getMemstoreSize();
484
485 try {
486 LOG.info("Flushing");
487 region.flush(true);
488 Assert.fail("Didn't bubble up IOE!");
489 } catch (DroppedSnapshotException dse) {
490
491 region.closing.set(false);
492 } catch (Exception e) {
493
494 region.closing.set(false);
495 }
496
497 ffs.fault.set(false);
498
499 try {
500 region.getWAL().rollWriter(true);
501 } catch (Exception e) {
502 int x = 0;
503 }
504
505 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize());
506
507
508 Put p2 = new Put(row);
509 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
510 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
511 region.put(p2);
512 long expectedSize = sizeOfOnePut * 3- ClassSize.ARRAY;
513 Assert.assertEquals(expectedSize, region.getMemstoreSize());
514
515
516
517 region.flush(true);
518
519 Assert.assertEquals(sizeOfOnePut * 2 - ClassSize.ARRAY, region.getMemstoreSize());
520 } catch (Exception e) {
521 int x = 0;
522 } finally {
523 HRegion.closeHRegion(region);
524 }
525 return null;
526 }
527 });
528 FileSystem.closeAllForUGI(user.getUGI());
529 }
530
531 @Test (timeout=60000)
532 public void testCloseWithFailingFlush() throws Exception {
533 final Configuration conf = HBaseConfiguration.create(CONF);
534
535 conf.setInt("hbase.hstore.flush.retries.number", 1);
536 final User user =
537 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
538
539 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
540 user.runAs(new PrivilegedExceptionAction<Object>() {
541 @Override
542 public Object run() throws Exception {
543
544 FileSystem fs = FileSystem.get(conf);
545 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
546 FaultyFileSystem ffs = (FaultyFileSystem)fs;
547 HRegion region = null;
548 try {
549
550 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
551 long size = region.getMemstoreSize();
552 Assert.assertEquals(0, size);
553
554 Put p1 = new Put(row);
555 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
556 region.put(p1);
557
558 Store store = region.getStore(COLUMN_FAMILY_BYTES);
559 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
560 storeFlushCtx.prepare();
561
562 Put p2 = new Put(row);
563 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
564 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
565 region.put(p2);
566
567 HBaseTestingUtility.closeRegionAndWAL(region);
568 region = null;
569 fail();
570 } catch (IOException dse) {
571
572 LOG.info("Expected DroppedSnapshotException");
573 } finally {
574
575 ffs.fault.set(false);
576 if (region != null) {
577 region.getWAL().rollWriter(true);
578 HRegion.closeHRegion(region);
579 }
580 }
581 return null;
582 }
583 });
584 FileSystem.closeAllForUGI(user.getUGI());
585 }
586
587 @Test
588 public void testCompactionAffectedByScanners() throws Exception {
589 byte[] family = Bytes.toBytes("family");
590 this.region = initHRegion(tableName, method, CONF, family);
591
592 Put put = new Put(Bytes.toBytes("r1"));
593 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
594 region.put(put);
595 region.flush(true);
596
597 Scan scan = new Scan();
598 scan.setMaxVersions(3);
599
600 RegionScanner scanner1 = region.getScanner(scan);
601
602 Delete delete = new Delete(Bytes.toBytes("r1"));
603 region.delete(delete);
604 region.flush(true);
605
606
607 RegionScanner scanner2 = region.getScanner(scan);
608
609 List<Cell> results = new ArrayList<Cell>();
610
611 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
612
613
614 region.compact(true);
615
616
617 RegionScanner scanner3 = region.getScanner(scan);
618
619
620 scanner1.next(results);
621 System.out.println(results);
622 assertEquals(1, results.size());
623
624 results.clear();
625 scanner2.next(results);
626 System.out.println(results);
627 assertEquals(0, results.size());
628
629 results.clear();
630 scanner3.next(results);
631 System.out.println(results);
632 assertEquals(0, results.size());
633 }
634
635 @Test
636 public void testToShowNPEOnRegionScannerReseek() throws Exception {
637 byte[] family = Bytes.toBytes("family");
638 this.region = initHRegion(tableName, method, CONF, family);
639
640 Put put = new Put(Bytes.toBytes("r1"));
641 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
642 region.put(put);
643 put = new Put(Bytes.toBytes("r2"));
644 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
645 region.put(put);
646 region.flush(true);
647
648 Scan scan = new Scan();
649 scan.setMaxVersions(3);
650
651 RegionScanner scanner1 = region.getScanner(scan);
652
653 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
654
655 region.compact(true);
656
657 scanner1.reseek(Bytes.toBytes("r2"));
658 List<Cell> results = new ArrayList<Cell>();
659 scanner1.next(results);
660 Cell keyValue = results.get(0);
661 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
662 scanner1.close();
663 }
664
665 @Test
666 public void testSkipRecoveredEditsReplay() throws Exception {
667 String method = "testSkipRecoveredEditsReplay";
668 TableName tableName = TableName.valueOf(method);
669 byte[] family = Bytes.toBytes("family");
670 this.region = initHRegion(tableName, method, CONF, family);
671 final WALFactory wals = new WALFactory(CONF, null, method);
672 try {
673 Path regiondir = region.getRegionFileSystem().getRegionDir();
674 FileSystem fs = region.getRegionFileSystem().getFileSystem();
675 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
676
677 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
678
679 long maxSeqId = 1050;
680 long minSeqId = 1000;
681
682 for (long i = minSeqId; i <= maxSeqId; i += 10) {
683 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
684 fs.create(recoveredEdits);
685 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
686
687 long time = System.nanoTime();
688 WALEdit edit = new WALEdit();
689 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
690 .toBytes(i)));
691 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
692 HConstants.DEFAULT_CLUSTER_ID), edit));
693
694 writer.close();
695 }
696 MonitoredTask status = TaskMonitor.get().createStatus(method);
697 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
698 for (Store store : region.getStores()) {
699 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
700 }
701 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
702 assertEquals(maxSeqId, seqId);
703 region.getMVCC().advanceTo(seqId);
704 Get get = new Get(row);
705 Result result = region.get(get);
706 for (long i = minSeqId; i <= maxSeqId; i += 10) {
707 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
708 assertEquals(1, kvs.size());
709 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
710 }
711 } finally {
712 HRegion.closeHRegion(this.region);
713 this.region = null;
714 wals.close();
715 }
716 }
717
718 @Test
719 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
720 String method = "testSkipRecoveredEditsReplaySomeIgnored";
721 TableName tableName = TableName.valueOf(method);
722 byte[] family = Bytes.toBytes("family");
723 this.region = initHRegion(tableName, method, CONF, family);
724 final WALFactory wals = new WALFactory(CONF, null, method);
725 try {
726 Path regiondir = region.getRegionFileSystem().getRegionDir();
727 FileSystem fs = region.getRegionFileSystem().getFileSystem();
728 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
729
730 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
731
732 long maxSeqId = 1050;
733 long minSeqId = 1000;
734
735 for (long i = minSeqId; i <= maxSeqId; i += 10) {
736 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
737 fs.create(recoveredEdits);
738 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
739
740 long time = System.nanoTime();
741 WALEdit edit = new WALEdit();
742 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
743 .toBytes(i)));
744 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
745 HConstants.DEFAULT_CLUSTER_ID), edit));
746
747 writer.close();
748 }
749 long recoverSeqId = 1030;
750 MonitoredTask status = TaskMonitor.get().createStatus(method);
751 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
752 for (Store store : region.getStores()) {
753 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
754 }
755 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
756 assertEquals(maxSeqId, seqId);
757 region.getMVCC().advanceTo(seqId);
758 Get get = new Get(row);
759 Result result = region.get(get);
760 for (long i = minSeqId; i <= maxSeqId; i += 10) {
761 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
762 if (i < recoverSeqId) {
763 assertEquals(0, kvs.size());
764 } else {
765 assertEquals(1, kvs.size());
766 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
767 }
768 }
769 } finally {
770 HRegion.closeHRegion(this.region);
771 this.region = null;
772 wals.close();
773 }
774 }
775
776 @Test
777 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
778 byte[] family = Bytes.toBytes("family");
779 this.region = initHRegion(tableName, method, CONF, family);
780 Path regiondir = region.getRegionFileSystem().getRegionDir();
781 FileSystem fs = region.getRegionFileSystem().getFileSystem();
782
783 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
784 for (int i = 1000; i < 1050; i += 10) {
785 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
786 FSDataOutputStream dos = fs.create(recoveredEdits);
787 dos.writeInt(i);
788 dos.close();
789 }
790 long minSeqId = 2000;
791 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
792 FSDataOutputStream dos = fs.create(recoveredEdits);
793 dos.close();
794
795 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
796 for (Store store : region.getStores()) {
797 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
798 }
799 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null);
800 assertEquals(minSeqId, seqId);
801 }
802
803 @Test
804 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
805 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
806 TableName tableName = TableName.valueOf(method);
807 byte[] family = Bytes.toBytes("family");
808 this.region = initHRegion(tableName, method, CONF, family);
809 final WALFactory wals = new WALFactory(CONF, null, method);
810 try {
811 Path regiondir = region.getRegionFileSystem().getRegionDir();
812 FileSystem fs = region.getRegionFileSystem().getFileSystem();
813 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
814 byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
815
816 assertEquals(0, region.getStoreFileList(columns).size());
817
818 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
819
820 long maxSeqId = 1050;
821 long minSeqId = 1000;
822
823 for (long i = minSeqId; i <= maxSeqId; i += 10) {
824 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
825 fs.create(recoveredEdits);
826 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
827
828 long time = System.nanoTime();
829 WALEdit edit = null;
830 if (i == maxSeqId) {
831 edit = WALEdit.createCompaction(region.getRegionInfo(),
832 CompactionDescriptor.newBuilder()
833 .setTableName(ByteString.copyFrom(tableName.getName()))
834 .setFamilyName(ByteString.copyFrom(regionName))
835 .setEncodedRegionName(ByteString.copyFrom(regionName))
836 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
837 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
838 .build());
839 } else {
840 edit = new WALEdit();
841 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
842 .toBytes(i)));
843 }
844 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
845 HConstants.DEFAULT_CLUSTER_ID), edit));
846 writer.close();
847 }
848
849 long recoverSeqId = 1030;
850 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
851 MonitoredTask status = TaskMonitor.get().createStatus(method);
852 for (Store store : region.getStores()) {
853 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
854 }
855 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
856 assertEquals(maxSeqId, seqId);
857
858
859 assertEquals(1, region.getStoreFileList(columns).size());
860
861 } finally {
862 HRegion.closeHRegion(this.region);
863 this.region = null;
864 wals.close();
865 }
866 }
867
868 @Test
869 public void testRecoveredEditsReplayCompaction() throws Exception {
870 testRecoveredEditsReplayCompaction(false);
871 testRecoveredEditsReplayCompaction(true);
872 }
873 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
874 String method = name.getMethodName();
875 TableName tableName = TableName.valueOf(method);
876 byte[] family = Bytes.toBytes("family");
877 this.region = initHRegion(tableName, method, CONF, family);
878 final WALFactory wals = new WALFactory(CONF, null, method);
879 try {
880 Path regiondir = region.getRegionFileSystem().getRegionDir();
881 FileSystem fs = region.getRegionFileSystem().getFileSystem();
882 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
883
884 long maxSeqId = 3;
885 long minSeqId = 0;
886
887 for (long i = minSeqId; i < maxSeqId; i++) {
888 Put put = new Put(Bytes.toBytes(i));
889 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
890 region.put(put);
891 region.flush(true);
892 }
893
894
895 assertEquals(3, region.getStore(family).getStorefilesCount());
896 List<Path> storeFiles = new ArrayList<Path>(3);
897 for (StoreFile sf : region.getStore(family).getStorefiles()) {
898 storeFiles.add(sf.getPath());
899 }
900
901
902 CONF.setBoolean("hbase.hstore.compaction.complete", false);
903 region.compactStores();
904
905
906 assertEquals(3, region.getStore(family).getStorefilesCount());
907
908
909 Path tmpDir = region.getRegionFileSystem().getTempDir();
910 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
911 String errorMsg = "Expected to find 1 file in the region temp directory "
912 + "from the compaction, could not find any";
913 assertNotNull(errorMsg, files);
914 assertEquals(errorMsg, 1, files.length);
915
916 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
917 files[0].getPath());
918
919 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
920 byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
921 for (int i=0; i < encodedNameAsBytes.length; i++) {
922
923 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
924 }
925
926 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
927 .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
928 storeFiles, Lists.newArrayList(newFile),
929 region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
930
931 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
932 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
933
934 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
935
936 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
937 fs.create(recoveredEdits);
938 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
939
940 long time = System.nanoTime();
941
942 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
943 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
944 compactionDescriptor)));
945 writer.close();
946
947
948 region.getTableDesc();
949 region.getRegionInfo();
950 HBaseTestingUtility.closeRegionAndWAL(this.region);
951 try {
952 region = HRegion.openHRegion(region, null);
953 } catch (WrongRegionException wre) {
954 fail("Matching encoded region name should not have produced WrongRegionException");
955 }
956
957
958 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
959 for (StoreFile sf : sfs) {
960 LOG.info(sf.getPath());
961 }
962 if (!mismatchedRegionName) {
963 assertEquals(1, region.getStore(family).getStorefilesCount());
964 }
965 files = FSUtils.listStatus(fs, tmpDir);
966 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
967
968 for (long i = minSeqId; i < maxSeqId; i++) {
969 Get get = new Get(Bytes.toBytes(i));
970 Result result = region.get(get);
971 byte[] value = result.getValue(family, Bytes.toBytes(i));
972 assertArrayEquals(Bytes.toBytes(i), value);
973 }
974 } finally {
975 HRegion.closeHRegion(this.region);
976 this.region = null;
977 wals.close();
978 }
979 }
980
981 @Test
982 public void testFlushMarkers() throws Exception {
983
984 String method = name.getMethodName();
985 TableName tableName = TableName.valueOf(method);
986 byte[] family = Bytes.toBytes("family");
987 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
988 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
989 FSUtils.setRootDir(walConf, logDir);
990 final WALFactory wals = new WALFactory(walConf, null, method);
991 final WAL wal = wals.getWAL(tableName.getName(), tableName.getNamespace());
992
993 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
994 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
995 try {
996 Path regiondir = region.getRegionFileSystem().getRegionDir();
997 FileSystem fs = region.getRegionFileSystem().getFileSystem();
998 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
999
1000 long maxSeqId = 3;
1001 long minSeqId = 0;
1002
1003 for (long i = minSeqId; i < maxSeqId; i++) {
1004 Put put = new Put(Bytes.toBytes(i));
1005 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1006 region.put(put);
1007 region.flush(true);
1008 }
1009
1010
1011 assertEquals(3, region.getStore(family).getStorefilesCount());
1012 List<String> storeFiles = new ArrayList<String>(3);
1013 for (StoreFile sf : region.getStore(family).getStorefiles()) {
1014 storeFiles.add(sf.getPath().getName());
1015 }
1016
1017
1018 wal.shutdown();
1019 WAL.Reader reader = WALFactory.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
1020 TEST_UTIL.getConfiguration());
1021 try {
1022 List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
1023 long lastFlushSeqId = -1;
1024 while (true) {
1025 WAL.Entry entry = reader.next();
1026 if (entry == null) {
1027 break;
1028 }
1029 Cell cell = entry.getEdit().getCells().get(0);
1030 if (WALEdit.isMetaEditFamily(cell)) {
1031 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
1032 assertNotNull(flushDesc);
1033 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
1034 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1035 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
1036 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1037 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
1038 }
1039 lastFlushSeqId = flushDesc.getFlushSequenceNumber();
1040 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
1041 assertEquals(1, flushDesc.getStoreFlushesCount());
1042 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
1043 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
1044 assertEquals("family", storeFlushDesc.getStoreHomeDir());
1045 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1046 assertEquals(0, storeFlushDesc.getFlushOutputCount());
1047 } else {
1048 assertEquals(1, storeFlushDesc.getFlushOutputCount());
1049 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
1050 }
1051
1052 flushDescriptors.add(entry);
1053 }
1054 }
1055
1056 assertEquals(3 * 2, flushDescriptors.size());
1057
1058
1059
1060 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
1061
1062 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1063 fs.create(recoveredEdits);
1064 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1065
1066 for (WAL.Entry entry : flushDescriptors) {
1067 writer.append(entry);
1068 }
1069 writer.close();
1070 } finally {
1071 if (null != reader) {
1072 try {
1073 reader.close();
1074 } catch (IOException exception) {
1075 LOG.warn("Problem closing wal: " + exception.getMessage());
1076 LOG.debug("exception details", exception);
1077 }
1078 }
1079 }
1080
1081
1082 HBaseTestingUtility.closeRegionAndWAL(this.region);
1083 region = HRegion.openHRegion(region, null);
1084
1085
1086 for (long i = minSeqId; i < maxSeqId; i++) {
1087 Get get = new Get(Bytes.toBytes(i));
1088 Result result = region.get(get);
1089 byte[] value = result.getValue(family, Bytes.toBytes(i));
1090 assertArrayEquals(Bytes.toBytes(i), value);
1091 }
1092 } finally {
1093 HBaseTestingUtility.closeRegionAndWAL(this.region);
1094 this.region = null;
1095 wals.close();
1096 }
1097 }
1098
1099 class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
1100 volatile FlushAction[] actions;
1101 public IsFlushWALMarker(FlushAction... actions) {
1102 this.actions = actions;
1103 }
1104 @Override
1105 public boolean matches(Object edit) {
1106 List<Cell> cells = ((WALEdit)edit).getCells();
1107 if (cells.isEmpty()) {
1108 return false;
1109 }
1110 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1111 FlushDescriptor desc = null;
1112 try {
1113 desc = WALEdit.getFlushDescriptor(cells.get(0));
1114 } catch (IOException e) {
1115 LOG.warn(e);
1116 return false;
1117 }
1118 if (desc != null) {
1119 for (FlushAction action : actions) {
1120 if (desc.getAction() == action) {
1121 return true;
1122 }
1123 }
1124 }
1125 }
1126 return false;
1127 }
1128 public IsFlushWALMarker set(FlushAction... actions) {
1129 this.actions = actions;
1130 return this;
1131 }
1132 }
1133
1134 @Test (timeout=60000)
1135 public void testFlushMarkersWALFail() throws Exception {
1136
1137 String method = name.getMethodName();
1138 TableName tableName = TableName.valueOf(method);
1139 byte[] family = Bytes.toBytes("family");
1140
1141
1142 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1143
1144 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1145 FSUtils.setRootDir(walConf, logDir);
1146
1147
1148 class FailAppendFlushMarkerWAL extends FSHLog {
1149 volatile FlushAction [] flushActions = null;
1150
1151 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1152 throws IOException {
1153 super(fs, root, logDir, conf);
1154 }
1155
1156 @Override
1157 protected Writer createWriterInstance(Path path) throws IOException {
1158 final Writer w = super.createWriterInstance(path);
1159 return new Writer() {
1160 @Override
1161 public void close() throws IOException {
1162 w.close();
1163 }
1164
1165 @Override
1166 public void sync() throws IOException {
1167 w.sync();
1168 }
1169
1170 @Override
1171 public void append(Entry entry) throws IOException {
1172 List<Cell> cells = entry.getEdit().getCells();
1173 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1174 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1175 if (desc != null) {
1176 for (FlushAction flushAction: flushActions) {
1177 if (desc.getAction().equals(flushAction)) {
1178 throw new IOException("Failed to append flush marker! " + flushAction);
1179 }
1180 }
1181 }
1182 }
1183 w.append(entry);
1184 }
1185
1186 @Override
1187 public long getLength() throws IOException {
1188 return w.getLength();
1189 }
1190 };
1191 }
1192 }
1193 FailAppendFlushMarkerWAL wal =
1194 new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1195 getName(), walConf);
1196 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1197 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1198 int i = 0;
1199 Put put = new Put(Bytes.toBytes(i));
1200 put.setDurability(Durability.SKIP_WAL);
1201 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1202 region.put(put);
1203
1204
1205 wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH};
1206
1207
1208 try {
1209 region.flush(true);
1210 fail("This should have thrown exception");
1211 } catch (DroppedSnapshotException unexpected) {
1212
1213 throw unexpected;
1214 } catch (IOException expected) {
1215
1216 }
1217
1218
1219 region.close(true);
1220 wal.close();
1221
1222
1223 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH};
1224 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1225 getName(), walConf);
1226 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1227 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1228 region.put(put);
1229
1230
1231
1232
1233 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH};
1234
1235 try {
1236 region.flush(true);
1237 fail("This should have thrown exception");
1238 } catch (DroppedSnapshotException expected) {
1239
1240
1241 } catch (IOException unexpected) {
1242 throw unexpected;
1243 }
1244 }
1245
1246 @Test
1247 public void testGetWhileRegionClose() throws IOException {
1248 TableName tableName = TableName.valueOf(name.getMethodName());
1249 Configuration hc = initSplit();
1250 int numRows = 100;
1251 byte[][] families = { fam1, fam2, fam3 };
1252
1253
1254 String method = name.getMethodName();
1255 this.region = initHRegion(tableName, method, hc, families);
1256
1257 final int startRow = 100;
1258 putData(startRow, numRows, qual1, families);
1259 putData(startRow, numRows, qual2, families);
1260 putData(startRow, numRows, qual3, families);
1261 final AtomicBoolean done = new AtomicBoolean(false);
1262 final AtomicInteger gets = new AtomicInteger(0);
1263 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1264 try {
1265
1266 for (int i = 0; i < threads.length / 2; i++) {
1267 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1268 threads[i].setDaemon(true);
1269 threads[i].start();
1270 }
1271
1272
1273 this.region.closing.set(true);
1274 for (int i = threads.length / 2; i < threads.length; i++) {
1275 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1276 threads[i].setDaemon(true);
1277 threads[i].start();
1278 }
1279 } finally {
1280 if (this.region != null) {
1281 HBaseTestingUtility.closeRegionAndWAL(this.region);
1282 this.region = null;
1283 }
1284 }
1285 done.set(true);
1286 for (GetTillDoneOrException t : threads) {
1287 try {
1288 t.join();
1289 } catch (InterruptedException e) {
1290 e.printStackTrace();
1291 }
1292 if (t.e != null) {
1293 LOG.info("Exception=" + t.e);
1294 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1295 }
1296 }
1297 }
1298
1299
1300
1301
1302
1303 class GetTillDoneOrException extends Thread {
1304 private final Get g;
1305 private final AtomicBoolean done;
1306 private final AtomicInteger count;
1307 private Exception e;
1308
1309 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1310 super("getter." + i);
1311 this.g = new Get(r);
1312 this.done = d;
1313 this.count = c;
1314 }
1315
1316 @Override
1317 public void run() {
1318 while (!this.done.get()) {
1319 try {
1320 assertTrue(region.get(g).size() > 0);
1321 this.count.incrementAndGet();
1322 } catch (Exception e) {
1323 this.e = e;
1324 break;
1325 }
1326 }
1327 }
1328 }
1329
1330
1331
1332
1333 @Test
1334 public void testWeirdCacheBehaviour() throws Exception {
1335 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1336 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1337 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1338 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1339 String value = "this is the value";
1340 String value2 = "this is some other value";
1341 String keyPrefix1 = "prefix1";
1342 String keyPrefix2 = "prefix2";
1343 String keyPrefix3 = "prefix3";
1344 putRows(this.region, 3, value, keyPrefix1);
1345 putRows(this.region, 3, value, keyPrefix2);
1346 putRows(this.region, 3, value, keyPrefix3);
1347 putRows(this.region, 3, value2, keyPrefix1);
1348 putRows(this.region, 3, value2, keyPrefix2);
1349 putRows(this.region, 3, value2, keyPrefix3);
1350 System.out.println("Checking values for key: " + keyPrefix1);
1351 assertEquals("Got back incorrect number of rows from scan", 3,
1352 getNumberOfRows(keyPrefix1, value2, this.region));
1353 System.out.println("Checking values for key: " + keyPrefix2);
1354 assertEquals("Got back incorrect number of rows from scan", 3,
1355 getNumberOfRows(keyPrefix2, value2, this.region));
1356 System.out.println("Checking values for key: " + keyPrefix3);
1357 assertEquals("Got back incorrect number of rows from scan", 3,
1358 getNumberOfRows(keyPrefix3, value2, this.region));
1359 deleteColumns(this.region, value2, keyPrefix1);
1360 deleteColumns(this.region, value2, keyPrefix2);
1361 deleteColumns(this.region, value2, keyPrefix3);
1362 System.out.println("Starting important checks.....");
1363 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1364 getNumberOfRows(keyPrefix1, value2, this.region));
1365 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1366 getNumberOfRows(keyPrefix2, value2, this.region));
1367 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1368 getNumberOfRows(keyPrefix3, value2, this.region));
1369 }
1370
1371 @Test
1372 public void testAppendWithReadOnlyTable() throws Exception {
1373 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1374 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1375 boolean exceptionCaught = false;
1376 Append append = new Append(Bytes.toBytes("somerow"));
1377 append.setDurability(Durability.SKIP_WAL);
1378 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1379 Bytes.toBytes("somevalue"));
1380 try {
1381 region.append(append);
1382 } catch (IOException e) {
1383 exceptionCaught = true;
1384 }
1385 assertTrue(exceptionCaught);
1386 }
1387
1388 @Test
1389 public void testIncrWithReadOnlyTable() throws Exception {
1390 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1391 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1392 boolean exceptionCaught = false;
1393 Increment inc = new Increment(Bytes.toBytes("somerow"));
1394 inc.setDurability(Durability.SKIP_WAL);
1395 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1396 try {
1397 region.increment(inc);
1398 } catch (IOException e) {
1399 exceptionCaught = true;
1400 }
1401 assertTrue(exceptionCaught);
1402 }
1403
1404 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1405 InternalScanner scanner = buildScanner(keyPrefix, value, r);
1406 int count = 0;
1407 boolean more = false;
1408 List<Cell> results = new ArrayList<Cell>();
1409 do {
1410 more = scanner.next(results);
1411 if (results != null && !results.isEmpty())
1412 count++;
1413 else
1414 break;
1415 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1416 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1417 r.delete(delete);
1418 results.clear();
1419 } while (more);
1420 assertEquals("Did not perform correct number of deletes", 3, count);
1421 }
1422
1423 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1424 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1425 int numberOfResults = 0;
1426 List<Cell> results = new ArrayList<Cell>();
1427 boolean more = false;
1428 do {
1429 more = resultScanner.next(results);
1430 if (results != null && !results.isEmpty())
1431 numberOfResults++;
1432 else
1433 break;
1434 for (Cell kv : results) {
1435 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1436 }
1437 results.clear();
1438 } while (more);
1439 return numberOfResults;
1440 }
1441
1442 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1443 throws IOException {
1444
1445 FilterList allFilters = new FilterList();
1446 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1447
1448 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1449 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1450 filter.setFilterIfMissing(true);
1451 allFilters.addFilter(filter);
1452 Scan scan = new Scan();
1453 scan.addFamily(Bytes.toBytes("trans-blob"));
1454 scan.addFamily(Bytes.toBytes("trans-type"));
1455 scan.addFamily(Bytes.toBytes("trans-date"));
1456 scan.addFamily(Bytes.toBytes("trans-tags"));
1457 scan.addFamily(Bytes.toBytes("trans-group"));
1458 scan.setFilter(allFilters);
1459 return r.getScanner(scan);
1460 }
1461
1462 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1463 for (int i = 0; i < numRows; i++) {
1464 String row = key + "_" + i
1465 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1466 Put put = new Put(Bytes.toBytes(row));
1467 put.setDurability(Durability.SKIP_WAL);
1468 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1469 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1470 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1471 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1472 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1473 r.put(put);
1474 }
1475 }
1476
1477 @Test
1478 public void testFamilyWithAndWithoutColon() throws Exception {
1479 byte[] b = Bytes.toBytes(getName());
1480 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1481 this.region = initHRegion(b, getName(), CONF, cf);
1482 Put p = new Put(b);
1483 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1484 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1485 boolean exception = false;
1486 try {
1487 this.region.put(p);
1488 } catch (NoSuchColumnFamilyException e) {
1489 exception = true;
1490 }
1491 assertTrue(exception);
1492 }
1493
1494 @Test
1495 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1496 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1497 byte[] qual = Bytes.toBytes("qual");
1498 byte[] val = Bytes.toBytes("val");
1499 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1500 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1501 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1502 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1503
1504 LOG.info("First a batch put with all valid puts");
1505 final Put[] puts = new Put[10];
1506 for (int i = 0; i < 10; i++) {
1507 puts[i] = new Put(Bytes.toBytes("row_" + i));
1508 puts[i].add(cf, qual, val);
1509 }
1510
1511 OperationStatus[] codes = this.region.batchMutate(puts);
1512 assertEquals(10, codes.length);
1513 for (int i = 0; i < 10; i++) {
1514 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1515 }
1516 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1517
1518 LOG.info("Next a batch put with one invalid family");
1519 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1520 codes = this.region.batchMutate(puts);
1521 assertEquals(10, codes.length);
1522 for (int i = 0; i < 10; i++) {
1523 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1524 codes[i].getOperationStatusCode());
1525 }
1526
1527 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1528 }
1529
1530 @Test
1531 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1532 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1533 byte[] qual = Bytes.toBytes("qual");
1534 byte[] val = Bytes.toBytes("val");
1535 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1536 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1537 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1538 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1539
1540 final Put[] puts = new Put[10];
1541 for (int i = 0; i < 10; i++) {
1542 puts[i] = new Put(Bytes.toBytes("row_" + i));
1543 puts[i].add(cf, qual, val);
1544 }
1545 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1546
1547 LOG.info("batchPut will have to break into four batches to avoid row locks");
1548 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1549 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
1550 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
1551 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
1552
1553
1554 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1555 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
1556 final CountDownLatch startingPuts = new CountDownLatch(1);
1557 final CountDownLatch startingClose = new CountDownLatch(1);
1558 TestThread putter = new TestThread(ctx) {
1559 @Override
1560 public void doWork() throws IOException {
1561 startingPuts.countDown();
1562 retFromThread.set(region.batchMutate(puts));
1563 }
1564 };
1565 LOG.info("...starting put thread while holding locks");
1566 ctx.addThread(putter);
1567 ctx.startThreads();
1568
1569
1570
1571
1572
1573
1574 Thread regionCloseThread = new TestThread(ctx) {
1575 @Override
1576 public void doWork() {
1577 try {
1578 startingPuts.await();
1579
1580
1581 Thread.sleep(10);
1582 startingClose.countDown();
1583 HBaseTestingUtility.closeRegionAndWAL(region);
1584 region = null;
1585 } catch (IOException e) {
1586 throw new RuntimeException(e);
1587 } catch (InterruptedException e) {
1588 throw new RuntimeException(e);
1589 }
1590 }
1591 };
1592 regionCloseThread.start();
1593
1594 startingClose.await();
1595 startingPuts.await();
1596 Thread.sleep(100);
1597 LOG.info("...releasing row lock 1, which should let put thread continue");
1598 rowLock1.release();
1599 rowLock2.release();
1600 rowLock3.release();
1601 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1602
1603 LOG.info("...joining on put thread");
1604 ctx.stop();
1605 regionCloseThread.join();
1606
1607 OperationStatus[] codes = retFromThread.get();
1608 for (int i = 0; i < codes.length; i++) {
1609 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1610 codes[i].getOperationStatusCode());
1611 }
1612 rowLock4.release();
1613 }
1614
1615 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1616 throws InterruptedException {
1617 long startWait = System.currentTimeMillis();
1618 long currentCount;
1619 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1620 Thread.sleep(100);
1621 if (System.currentTimeMillis() - startWait > 10000) {
1622 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1623 expectedCount, currentCount));
1624 }
1625 }
1626 }
1627
1628 @Test
1629 public void testBatchPutWithTsSlop() throws Exception {
1630 byte[] b = Bytes.toBytes(getName());
1631 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1632 byte[] qual = Bytes.toBytes("qual");
1633 byte[] val = Bytes.toBytes("val");
1634
1635
1636 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1637 this.region = initHRegion(b, getName(), CONF, cf);
1638
1639 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1640 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1641 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1642
1643 final Put[] puts = new Put[10];
1644 for (int i = 0; i < 10; i++) {
1645 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1646 puts[i].add(cf, qual, val);
1647 }
1648
1649 OperationStatus[] codes = this.region.batchMutate(puts);
1650 assertEquals(10, codes.length);
1651 for (int i = 0; i < 10; i++) {
1652 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1653 }
1654 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1655 }
1656
1657
1658
1659
1660 @Test
1661 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1662 byte[] row1 = Bytes.toBytes("row1");
1663 byte[] fam1 = Bytes.toBytes("fam1");
1664 byte[] qf1 = Bytes.toBytes("qualifier");
1665 byte[] emptyVal = new byte[] {};
1666 byte[] val1 = Bytes.toBytes("value1");
1667 byte[] val2 = Bytes.toBytes("value2");
1668
1669
1670 String method = this.getName();
1671 this.region = initHRegion(tableName, method, CONF, fam1);
1672
1673 Put put = new Put(row1);
1674 put.add(fam1, qf1, emptyVal);
1675
1676
1677 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1678 emptyVal), put, true);
1679 assertTrue(res);
1680
1681
1682 put = new Put(row1);
1683 put.add(fam1, qf1, val1);
1684
1685
1686 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1687 put, true);
1688 assertTrue(res);
1689
1690
1691 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1692 put, true);
1693 assertFalse(res);
1694
1695 Delete delete = new Delete(row1);
1696 delete.deleteColumn(fam1, qf1);
1697 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1698 delete, true);
1699 assertFalse(res);
1700
1701 put = new Put(row1);
1702 put.add(fam1, qf1, val2);
1703
1704 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1705 put, true);
1706 assertTrue(res);
1707
1708
1709 delete = new Delete(row1);
1710 delete.deleteColumn(fam1, qf1);
1711 delete.deleteColumn(fam1, qf1);
1712 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1713 delete, true);
1714 assertTrue(res);
1715
1716 delete = new Delete(row1);
1717 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1718 delete, true);
1719 assertTrue(res);
1720
1721
1722 put = new Put(row1);
1723 put.add(fam1, qf1, val1);
1724
1725 res = region
1726 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1727 assertTrue(res);
1728 }
1729
1730 @Test
1731 public void testCheckAndMutate_WithWrongValue() throws IOException {
1732 byte[] row1 = Bytes.toBytes("row1");
1733 byte[] fam1 = Bytes.toBytes("fam1");
1734 byte[] qf1 = Bytes.toBytes("qualifier");
1735 byte[] val1 = Bytes.toBytes("value1");
1736 byte[] val2 = Bytes.toBytes("value2");
1737
1738
1739 String method = this.getName();
1740 this.region = initHRegion(tableName, method, CONF, fam1);
1741
1742 Put put = new Put(row1);
1743 put.add(fam1, qf1, val1);
1744 region.put(put);
1745
1746
1747 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1748 val2), put, true);
1749 assertFalse(res);
1750
1751
1752 Delete delete = new Delete(row1);
1753 delete.deleteFamily(fam1);
1754 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1755 put, true);
1756 assertFalse(res);
1757 }
1758
1759 @Test
1760 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1761 byte[] row1 = Bytes.toBytes("row1");
1762 byte[] fam1 = Bytes.toBytes("fam1");
1763 byte[] qf1 = Bytes.toBytes("qualifier");
1764 byte[] val1 = Bytes.toBytes("value1");
1765
1766
1767 String method = this.getName();
1768 this.region = initHRegion(tableName, method, CONF, fam1);
1769
1770 Put put = new Put(row1);
1771 put.add(fam1, qf1, val1);
1772 region.put(put);
1773
1774
1775 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1776 val1), put, true);
1777 assertTrue( res);
1778
1779
1780 Delete delete = new Delete(row1);
1781 delete.deleteColumn(fam1, qf1);
1782 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1783 delete, true);
1784 assertTrue(res);
1785 }
1786
1787 @Test
1788 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1789 byte[] row1 = Bytes.toBytes("row1");
1790 byte[] fam1 = Bytes.toBytes("fam1");
1791 byte[] qf1 = Bytes.toBytes("qualifier");
1792 byte[] val1 = Bytes.toBytes("value1");
1793 byte[] val2 = Bytes.toBytes("value2");
1794 byte[] val3 = Bytes.toBytes("value3");
1795 byte[] val4 = Bytes.toBytes("value4");
1796
1797
1798 String method = this.getName();
1799 this.region = initHRegion(tableName, method, CONF, fam1);
1800
1801 Put put = new Put(row1);
1802 put.add(fam1, qf1, val3);
1803 region.put(put);
1804
1805
1806 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1807 new BinaryComparator(val3), put, true);
1808 assertEquals(false, res);
1809
1810
1811 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1812 new BinaryComparator(val4), put, true);
1813 assertEquals(false, res);
1814
1815
1816
1817 put = new Put(row1);
1818 put.add(fam1, qf1, val2);
1819 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1820 new BinaryComparator(val2), put, true);
1821 assertEquals(true, res);
1822
1823
1824 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1825 new BinaryComparator(val3), put, true);
1826 assertEquals(false, res);
1827
1828
1829
1830 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1831 new BinaryComparator(val2), put, true);
1832 assertEquals(true, res);
1833
1834
1835
1836 put = new Put(row1);
1837 put.add(fam1, qf1, val3);
1838 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1839 new BinaryComparator(val1), put, true);
1840 assertEquals(true, res);
1841
1842
1843 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1844 new BinaryComparator(val3), put, true);
1845 assertEquals(false, res);
1846
1847
1848 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1849 new BinaryComparator(val2), put, true);
1850 assertEquals(false, res);
1851
1852
1853
1854 put = new Put(row1);
1855 put.add(fam1, qf1, val2);
1856 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1857 new BinaryComparator(val4), put, true);
1858 assertEquals(true, res);
1859
1860
1861 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1862 new BinaryComparator(val1), put, true);
1863 assertEquals(false, res);
1864
1865
1866
1867 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1868 new BinaryComparator(val2), put, true);
1869 assertEquals(true, res);
1870
1871
1872 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1873 new BinaryComparator(val3), put, true);
1874 assertEquals(true, res);
1875 }
1876
1877 @Test
1878 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1879 byte[] row1 = Bytes.toBytes("row1");
1880 byte[] fam1 = Bytes.toBytes("fam1");
1881 byte[] fam2 = Bytes.toBytes("fam2");
1882 byte[] qf1 = Bytes.toBytes("qualifier");
1883 byte[] val1 = Bytes.toBytes("value1");
1884 byte[] val2 = Bytes.toBytes("value2");
1885
1886 byte[][] families = { fam1, fam2 };
1887
1888
1889 String method = this.getName();
1890 this.region = initHRegion(tableName, method, CONF, families);
1891
1892 Put put = new Put(row1);
1893 put.add(fam1, qf1, val1);
1894 region.put(put);
1895
1896
1897 long ts = System.currentTimeMillis();
1898 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1899 put = new Put(row1);
1900 put.add(kv);
1901
1902
1903 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1904 val1), put, true);
1905 assertEquals(true, res);
1906
1907 Get get = new Get(row1);
1908 get.addColumn(fam2, qf1);
1909 Cell[] actual = region.get(get).rawCells();
1910
1911 Cell[] expected = { kv };
1912
1913 assertEquals(expected.length, actual.length);
1914 for (int i = 0; i < actual.length; i++) {
1915 assertEquals(expected[i], actual[i]);
1916 }
1917 }
1918
1919 @Test
1920 public void testCheckAndPut_wrongRowInPut() throws IOException {
1921 TableName tableName = TableName.valueOf(name.getMethodName());
1922 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1923 Put put = new Put(row2);
1924 put.add(fam1, qual1, value1);
1925 try {
1926 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1927 new BinaryComparator(value2), put, false);
1928 fail();
1929 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1930
1931 }
1932 }
1933
1934 @Test
1935 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1936 byte[] row1 = Bytes.toBytes("row1");
1937 byte[] fam1 = Bytes.toBytes("fam1");
1938 byte[] fam2 = Bytes.toBytes("fam2");
1939 byte[] qf1 = Bytes.toBytes("qualifier1");
1940 byte[] qf2 = Bytes.toBytes("qualifier2");
1941 byte[] qf3 = Bytes.toBytes("qualifier3");
1942 byte[] val1 = Bytes.toBytes("value1");
1943 byte[] val2 = Bytes.toBytes("value2");
1944 byte[] val3 = Bytes.toBytes("value3");
1945 byte[] emptyVal = new byte[] {};
1946
1947 byte[][] families = { fam1, fam2 };
1948
1949
1950 String method = this.getName();
1951 this.region = initHRegion(tableName, method, CONF, families);
1952
1953 Put put = new Put(row1);
1954 put.add(fam1, qf1, val1);
1955 region.put(put);
1956 Threads.sleep(2);
1957
1958 put = new Put(row1);
1959 put.add(fam1, qf1, val2);
1960 put.add(fam2, qf1, val3);
1961 put.add(fam2, qf2, val2);
1962 put.add(fam2, qf3, val1);
1963 put.add(fam1, qf3, val1);
1964 region.put(put);
1965
1966
1967 Delete delete = new Delete(row1);
1968 delete.deleteColumn(fam1, qf1);
1969 delete.deleteColumn(fam2, qf1);
1970 delete.deleteColumn(fam1, qf3);
1971 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1972 val2), delete, true);
1973 assertEquals(true, res);
1974
1975 Get get = new Get(row1);
1976 get.addColumn(fam1, qf1);
1977 get.addColumn(fam1, qf3);
1978 get.addColumn(fam2, qf2);
1979 Result r = region.get(get);
1980 assertEquals(2, r.size());
1981 assertArrayEquals(val1, r.getValue(fam1, qf1));
1982 assertArrayEquals(val2, r.getValue(fam2, qf2));
1983
1984
1985 delete = new Delete(row1);
1986 delete.deleteFamily(fam2);
1987 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1988 delete, true);
1989 assertEquals(true, res);
1990
1991 get = new Get(row1);
1992 r = region.get(get);
1993 assertEquals(1, r.size());
1994 assertArrayEquals(val1, r.getValue(fam1, qf1));
1995
1996
1997 delete = new Delete(row1);
1998 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1999 delete, true);
2000 assertEquals(true, res);
2001 get = new Get(row1);
2002 r = region.get(get);
2003 assertEquals(0, r.size());
2004 }
2005
2006
2007
2008
2009 @Test
2010 public void testDelete_multiDeleteColumn() throws IOException {
2011 byte[] row1 = Bytes.toBytes("row1");
2012 byte[] fam1 = Bytes.toBytes("fam1");
2013 byte[] qual = Bytes.toBytes("qualifier");
2014 byte[] value = Bytes.toBytes("value");
2015
2016 Put put = new Put(row1);
2017 put.add(fam1, qual, 1, value);
2018 put.add(fam1, qual, 2, value);
2019
2020 String method = this.getName();
2021 this.region = initHRegion(tableName, method, CONF, fam1);
2022 region.put(put);
2023
2024
2025 Delete delete = new Delete(row1);
2026 delete.deleteColumn(fam1, qual);
2027 delete.deleteColumn(fam1, qual);
2028 region.delete(delete);
2029
2030 Get get = new Get(row1);
2031 get.addFamily(fam1);
2032 Result r = region.get(get);
2033 assertEquals(0, r.size());
2034 }
2035
2036 @Test
2037 public void testDelete_CheckFamily() throws IOException {
2038 byte[] row1 = Bytes.toBytes("row1");
2039 byte[] fam1 = Bytes.toBytes("fam1");
2040 byte[] fam2 = Bytes.toBytes("fam2");
2041 byte[] fam3 = Bytes.toBytes("fam3");
2042 byte[] fam4 = Bytes.toBytes("fam4");
2043
2044
2045 String method = this.getName();
2046 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
2047 List<Cell> kvs = new ArrayList<Cell>();
2048 kvs.add(new KeyValue(row1, fam4, null, null));
2049
2050
2051 byte[] family = fam2;
2052 try {
2053 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2054 Bytes.BYTES_COMPARATOR);
2055 deleteMap.put(family, kvs);
2056 region.delete(deleteMap, Durability.SYNC_WAL);
2057 } catch (Exception e) {
2058 assertTrue("Family " + new String(family) + " does not exist", false);
2059 }
2060
2061
2062 boolean ok = false;
2063 family = fam4;
2064 try {
2065 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2066 Bytes.BYTES_COMPARATOR);
2067 deleteMap.put(family, kvs);
2068 region.delete(deleteMap, Durability.SYNC_WAL);
2069 } catch (Exception e) {
2070 ok = true;
2071 }
2072 assertTrue("Family " + new String(family) + " does exist", ok);
2073 }
2074
2075 @Test
2076 public void testDelete_mixed() throws IOException, InterruptedException {
2077 byte[] fam = Bytes.toBytes("info");
2078 byte[][] families = { fam };
2079 String method = this.getName();
2080 this.region = initHRegion(tableName, method, CONF, families);
2081 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2082
2083 byte[] row = Bytes.toBytes("table_name");
2084
2085 byte[] serverinfo = Bytes.toBytes("serverinfo");
2086 byte[] splitA = Bytes.toBytes("splitA");
2087 byte[] splitB = Bytes.toBytes("splitB");
2088
2089
2090 Put put = new Put(row);
2091 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2092 region.put(put);
2093
2094 put = new Put(row);
2095 put.add(fam, splitB, Bytes.toBytes("reference_B"));
2096 region.put(put);
2097
2098 put = new Put(row);
2099 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
2100 region.put(put);
2101
2102
2103 Delete delete = new Delete(row);
2104 delete.deleteColumns(fam, splitA);
2105 region.delete(delete);
2106
2107
2108 Get get = new Get(row).addColumn(fam, serverinfo);
2109 Result result = region.get(get);
2110 assertEquals(1, result.size());
2111
2112 get = new Get(row).addColumn(fam, splitA);
2113 result = region.get(get);
2114 assertEquals(0, result.size());
2115
2116 get = new Get(row).addColumn(fam, splitB);
2117 result = region.get(get);
2118 assertEquals(1, result.size());
2119
2120
2121 put = new Put(row);
2122 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2123 region.put(put);
2124 get = new Get(row);
2125 result = region.get(get);
2126 assertEquals(3, result.size());
2127
2128
2129 delete = new Delete(row);
2130 region.delete(delete);
2131 assertEquals(0, region.get(get).size());
2132
2133 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2134 result = region.get(get);
2135 assertEquals(1, result.size());
2136 }
2137
2138 @Test
2139 public void testDeleteRowWithFutureTs() throws IOException {
2140 byte[] fam = Bytes.toBytes("info");
2141 byte[][] families = { fam };
2142 String method = this.getName();
2143 this.region = initHRegion(tableName, method, CONF, families);
2144 byte[] row = Bytes.toBytes("table_name");
2145
2146 byte[] serverinfo = Bytes.toBytes("serverinfo");
2147
2148
2149 Put put = new Put(row);
2150 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2151 region.put(put);
2152
2153
2154 Delete delete = new Delete(row);
2155 region.delete(delete);
2156
2157
2158 Get get = new Get(row).addColumn(fam, serverinfo);
2159 Result result = region.get(get);
2160 assertEquals(1, result.size());
2161
2162
2163 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2164 region.delete(delete);
2165
2166
2167 get = new Get(row).addColumn(fam, serverinfo);
2168 result = region.get(get);
2169 assertEquals(0, result.size());
2170 }
2171
2172
2173
2174
2175
2176 @Test
2177 public void testPutWithLatestTS() throws IOException {
2178 byte[] fam = Bytes.toBytes("info");
2179 byte[][] families = { fam };
2180 String method = this.getName();
2181 this.region = initHRegion(tableName, method, CONF, families);
2182 byte[] row = Bytes.toBytes("row1");
2183
2184 byte[] qual = Bytes.toBytes("qual");
2185
2186
2187 Put put = new Put(row);
2188 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2189 region.put(put);
2190
2191
2192 Get get = new Get(row).addColumn(fam, qual);
2193 Result result = region.get(get);
2194 assertEquals(1, result.size());
2195 Cell kv = result.rawCells()[0];
2196 LOG.info("Got: " + kv);
2197 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2198 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2199
2200
2201
2202 row = Bytes.toBytes("row2");
2203 put = new Put(row);
2204 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2205 region.put(put);
2206
2207
2208 get = new Get(row).addColumn(fam, qual);
2209 result = region.get(get);
2210 assertEquals(1, result.size());
2211 kv = result.rawCells()[0];
2212 LOG.info("Got: " + kv);
2213 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2214 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2215 }
2216
2217
2218
2219
2220
2221
2222 @Test
2223 public void testPutWithTsSlop() throws IOException {
2224 byte[] fam = Bytes.toBytes("info");
2225 byte[][] families = { fam };
2226 String method = this.getName();
2227
2228
2229 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2230 this.region = initHRegion(tableName, method, CONF, families);
2231 boolean caughtExcep = false;
2232 try {
2233
2234 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2235
2236 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2237 Bytes.toBytes("value")));
2238 fail("Expected IOE for TS out of configured timerange");
2239 } catch (FailedSanityCheckException ioe) {
2240 LOG.debug("Received expected exception", ioe);
2241 caughtExcep = true;
2242 }
2243 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2244 }
2245
2246 @Test
2247 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2248 byte[] fam1 = Bytes.toBytes("columnA");
2249 byte[] fam2 = Bytes.toBytes("columnB");
2250 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2251 byte[] rowA = Bytes.toBytes("rowA");
2252 byte[] rowB = Bytes.toBytes("rowB");
2253
2254 byte[] value = Bytes.toBytes("value");
2255
2256 Delete delete = new Delete(rowA);
2257 delete.deleteFamily(fam1);
2258
2259 region.delete(delete);
2260
2261
2262 Put put = new Put(rowA);
2263 put.add(fam2, null, value);
2264 region.put(put);
2265
2266 put = new Put(rowB);
2267 put.add(fam1, null, value);
2268 put.add(fam2, null, value);
2269 region.put(put);
2270
2271 Scan scan = new Scan();
2272 scan.addFamily(fam1).addFamily(fam2);
2273 InternalScanner s = region.getScanner(scan);
2274 List<Cell> results = new ArrayList<Cell>();
2275 s.next(results);
2276 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2277
2278 results.clear();
2279 s.next(results);
2280 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2281 }
2282
2283 @Test
2284 public void testDataInMemoryWithoutWAL() throws IOException {
2285 FileSystem fs = FileSystem.get(CONF);
2286 Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
2287 FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
2288 region = initHRegion(tableName, null, null, name.getMethodName(),
2289 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
2290
2291 Cell originalCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1,
2292 System.currentTimeMillis(), KeyValue.Type.Put.getCode(), value1);
2293 final long originalSize = KeyValueUtil.length(originalCell);
2294
2295 Cell addCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1,
2296 System.currentTimeMillis(), KeyValue.Type.Put.getCode(), Bytes.toBytes("xxxxxxxxxx"));
2297 final long addSize = KeyValueUtil.length(addCell);
2298
2299 LOG.info("originalSize:" + originalSize
2300 + ", addSize:" + addSize);
2301
2302
2303
2304
2305 testDataInMemoryWithoutWAL(region,
2306 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
2307 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL),
2308 originalSize + addSize);
2309
2310
2311 testDataInMemoryWithoutWAL(region,
2312 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
2313 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL),
2314 originalSize + addSize);
2315
2316
2317 testDataInMemoryWithoutWAL(region,
2318 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
2319 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL),
2320 0);
2321
2322
2323 testDataInMemoryWithoutWAL(region,
2324 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
2325 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL),
2326 0);
2327 }
2328
2329 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut,
2330 final Put addPut, long delta) throws IOException {
2331 final long initSize = region.getDataInMemoryWithoutWAL();
2332
2333 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
2334 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
2335 Answer<Boolean> answer = new Answer<Boolean>() {
2336 @Override
2337 public Boolean answer(InvocationOnMock invocation) throws Throwable {
2338 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgumentAt(0,
2339 MiniBatchOperationInProgress.class);
2340 mb.addOperationsFromCP(0, new Mutation[]{addPut});
2341 return false;
2342 }
2343 };
2344 when(mockedCPHost.preBatchMutate(Mockito.isA(MiniBatchOperationInProgress.class)))
2345 .then(answer);
2346 region.setCoprocessorHost(mockedCPHost);
2347 region.put(originalPut);
2348 region.setCoprocessorHost(normalCPHost);
2349 final long finalSize = region.getDataInMemoryWithoutWAL();
2350 assertEquals("finalSize:" + finalSize + ", initSize:"
2351 + initSize + ", delta:" + delta,finalSize, initSize + delta);
2352 }
2353
2354 @Test
2355 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2356 Delete delete = new Delete(row);
2357 delete.deleteColumns(fam1, qual1);
2358 doTestDelete_AndPostInsert(delete);
2359 }
2360
2361 @Test
2362 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2363 Delete delete = new Delete(row);
2364 delete.deleteFamily(fam1);
2365 doTestDelete_AndPostInsert(delete);
2366 }
2367
2368 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2369 TableName tableName = TableName.valueOf(name.getMethodName());
2370 this.region = initHRegion(tableName, getName(), CONF, fam1);
2371 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2372 Put put = new Put(row);
2373 put.add(fam1, qual1, value1);
2374 region.put(put);
2375
2376
2377 region.delete(delete);
2378
2379
2380 put = new Put(row);
2381 put.add(fam1, qual1, value2);
2382 region.put(put);
2383
2384
2385 Get get = new Get(row);
2386 get.addColumn(fam1, qual1);
2387
2388 Result r = region.get(get);
2389 assertEquals(1, r.size());
2390 assertArrayEquals(value2, r.getValue(fam1, qual1));
2391
2392
2393 Scan scan = new Scan(row);
2394 scan.addColumn(fam1, qual1);
2395 InternalScanner s = region.getScanner(scan);
2396
2397 List<Cell> results = new ArrayList<Cell>();
2398 assertEquals(false, s.next(results));
2399 assertEquals(1, results.size());
2400 Cell kv = results.get(0);
2401
2402 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2403 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2404 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2405 assertArrayEquals(row, CellUtil.cloneRow(kv));
2406 }
2407
2408 @Test
2409 public void testDelete_CheckTimestampUpdated() throws IOException {
2410 TableName tableName = TableName.valueOf(name.getMethodName());
2411 byte[] row1 = Bytes.toBytes("row1");
2412 byte[] col1 = Bytes.toBytes("col1");
2413 byte[] col2 = Bytes.toBytes("col2");
2414 byte[] col3 = Bytes.toBytes("col3");
2415
2416
2417 String method = this.getName();
2418 this.region = initHRegion(tableName, method, CONF, fam1);
2419
2420 List<Cell> kvs = new ArrayList<Cell>();
2421 kvs.add(new KeyValue(row1, fam1, col1, null));
2422 kvs.add(new KeyValue(row1, fam1, col2, null));
2423 kvs.add(new KeyValue(row1, fam1, col3, null));
2424
2425 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2426 Bytes.BYTES_COMPARATOR);
2427 deleteMap.put(fam1, kvs);
2428 region.delete(deleteMap, Durability.SYNC_WAL);
2429
2430
2431
2432 long now = System.currentTimeMillis();
2433 DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2434 Cell firstCell = memstore.activeSection.getCellSkipListSet().first();
2435 assertTrue(firstCell.getTimestamp() <= now);
2436 now = firstCell.getTimestamp();
2437 for (Cell cell : memstore.activeSection.getCellSkipListSet()) {
2438 assertTrue(cell.getTimestamp() <= now);
2439 now = cell.getTimestamp();
2440 }
2441 }
2442
2443
2444
2445
2446 @Test
2447 public void testGet_FamilyChecker() throws IOException {
2448 byte[] row1 = Bytes.toBytes("row1");
2449 byte[] fam1 = Bytes.toBytes("fam1");
2450 byte[] fam2 = Bytes.toBytes("False");
2451 byte[] col1 = Bytes.toBytes("col1");
2452
2453
2454 String method = this.getName();
2455 this.region = initHRegion(tableName, method, CONF, fam1);
2456 Get get = new Get(row1);
2457 get.addColumn(fam2, col1);
2458
2459
2460 try {
2461 region.get(get);
2462 fail("Expecting DoNotRetryIOException in get but did not get any");
2463 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2464 LOG.info("Got expected DoNotRetryIOException successfully");
2465 }
2466 }
2467
2468 @Test
2469 public void testGet_Basic() throws IOException {
2470 byte[] row1 = Bytes.toBytes("row1");
2471 byte[] fam1 = Bytes.toBytes("fam1");
2472 byte[] col1 = Bytes.toBytes("col1");
2473 byte[] col2 = Bytes.toBytes("col2");
2474 byte[] col3 = Bytes.toBytes("col3");
2475 byte[] col4 = Bytes.toBytes("col4");
2476 byte[] col5 = Bytes.toBytes("col5");
2477
2478
2479 String method = this.getName();
2480 this.region = initHRegion(tableName, method, CONF, fam1);
2481
2482 Put put = new Put(row1);
2483 put.add(fam1, col1, null);
2484 put.add(fam1, col2, null);
2485 put.add(fam1, col3, null);
2486 put.add(fam1, col4, null);
2487 put.add(fam1, col5, null);
2488 region.put(put);
2489
2490 Get get = new Get(row1);
2491 get.addColumn(fam1, col2);
2492 get.addColumn(fam1, col4);
2493
2494 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2495 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2496 KeyValue[] expected = { kv1, kv2 };
2497
2498
2499 Result res = region.get(get);
2500 assertEquals(expected.length, res.size());
2501 for (int i = 0; i < res.size(); i++) {
2502 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2503 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2504 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2505 }
2506
2507
2508 Get g = new Get(row1);
2509 final int count = 2;
2510 g.setFilter(new ColumnCountGetFilter(count));
2511 res = region.get(g);
2512 assertEquals(count, res.size());
2513 }
2514
2515 @Test
2516 public void testGet_Empty() throws IOException {
2517 byte[] row = Bytes.toBytes("row");
2518 byte[] fam = Bytes.toBytes("fam");
2519
2520 String method = this.getName();
2521 this.region = initHRegion(tableName, method, CONF, fam);
2522 Get get = new Get(row);
2523 get.addFamily(fam);
2524 Result r = region.get(get);
2525
2526 assertTrue(r.isEmpty());
2527 }
2528
2529
2530
2531
2532 @Test
2533 public void testMerge() throws IOException {
2534 byte[][] families = { fam1, fam2, fam3 };
2535 Configuration hc = initSplit();
2536
2537 String method = this.getName();
2538 this.region = initHRegion(tableName, method, hc, families);
2539 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2540 region.flush(true);
2541 region.compactStores();
2542 region.waitForFlushesAndCompactions();
2543 for(Store s:region.getStores()) {
2544 s.closeAndArchiveCompactedFiles();
2545 }
2546 byte[] splitRow = region.checkSplit();
2547 assertNotNull(splitRow);
2548 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2549 HRegion[] subregions = splitRegion(region, splitRow);
2550 try {
2551
2552 for (int i = 0; i < subregions.length; i++) {
2553 HRegion.openHRegion(subregions[i], null);
2554 subregions[i].compactStores();
2555 subregions[i].waitForFlushesAndCompactions();
2556 for(Store s:subregions[i].getStores()) {
2557 s.closeAndArchiveCompactedFiles();
2558 }
2559 }
2560 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2561 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2562 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2563 long startTime = System.currentTimeMillis();
2564 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2565 LOG.info("Merge regions elapsed time: "
2566 + ((System.currentTimeMillis() - startTime) / 1000.0));
2567 FILESYSTEM.delete(oldRegion1, true);
2568 FILESYSTEM.delete(oldRegion2, true);
2569 FILESYSTEM.delete(oldRegionPath, true);
2570 LOG.info("splitAndMerge completed.");
2571 } finally {
2572 for (int i = 0; i < subregions.length; i++) {
2573 try {
2574 HRegion.closeHRegion(subregions[i]);
2575 } catch (IOException e) {
2576
2577 }
2578 }
2579 }
2580 }
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2591 PairOfSameType<Region> result = null;
2592 SplitTransactionImpl st = new SplitTransactionImpl(parent, midkey);
2593
2594
2595 if (!st.prepare()) {
2596 parent.clearSplit();
2597 return null;
2598 }
2599 try {
2600 result = st.execute(null, null);
2601 } catch (IOException ioe) {
2602 try {
2603 LOG.info("Running rollback of failed split of " +
2604 parent.getRegionInfo().getRegionNameAsString() + "; " + ioe.getMessage());
2605 st.rollback(null, null);
2606 LOG.info("Successful rollback of failed split of " +
2607 parent.getRegionInfo().getRegionNameAsString());
2608 return null;
2609 } catch (RuntimeException e) {
2610
2611 LOG.info("Failed rollback of failed split of " +
2612 parent.getRegionInfo().getRegionNameAsString() + " -- aborting server", e);
2613 }
2614 }
2615 finally {
2616 parent.clearSplit();
2617 }
2618
2619 assertNotNull(result);
2620 return new HRegion[] { (HRegion)result.getFirst(), (HRegion)result.getSecond() };
2621 }
2622
2623
2624
2625
2626 @Test
2627 public void testGetScanner_WithOkFamilies() throws IOException {
2628 byte[] fam1 = Bytes.toBytes("fam1");
2629 byte[] fam2 = Bytes.toBytes("fam2");
2630
2631 byte[][] families = { fam1, fam2 };
2632
2633
2634 String method = this.getName();
2635 this.region = initHRegion(tableName, method, CONF, families);
2636 Scan scan = new Scan();
2637 scan.addFamily(fam1);
2638 scan.addFamily(fam2);
2639 try {
2640 region.getScanner(scan);
2641 } catch (Exception e) {
2642 assertTrue("Families could not be found in Region", false);
2643 }
2644 }
2645
2646 @Test
2647 public void testGetScanner_WithNotOkFamilies() throws IOException {
2648 byte[] fam1 = Bytes.toBytes("fam1");
2649 byte[] fam2 = Bytes.toBytes("fam2");
2650
2651 byte[][] families = { fam1 };
2652
2653
2654 String method = this.getName();
2655 this.region = initHRegion(tableName, method, CONF, families);
2656 Scan scan = new Scan();
2657 scan.addFamily(fam2);
2658 boolean ok = false;
2659 try {
2660 region.getScanner(scan);
2661 } catch (Exception e) {
2662 ok = true;
2663 }
2664 assertTrue("Families could not be found in Region", ok);
2665 }
2666
2667 @Test
2668 public void testGetScanner_WithNoFamilies() throws IOException {
2669 byte[] row1 = Bytes.toBytes("row1");
2670 byte[] fam1 = Bytes.toBytes("fam1");
2671 byte[] fam2 = Bytes.toBytes("fam2");
2672 byte[] fam3 = Bytes.toBytes("fam3");
2673 byte[] fam4 = Bytes.toBytes("fam4");
2674
2675 byte[][] families = { fam1, fam2, fam3, fam4 };
2676
2677
2678 String method = this.getName();
2679 this.region = initHRegion(tableName, method, CONF, families);
2680
2681 Put put = new Put(row1);
2682 put.add(fam1, null, null);
2683 put.add(fam2, null, null);
2684 put.add(fam3, null, null);
2685 put.add(fam4, null, null);
2686 region.put(put);
2687
2688 Scan scan = null;
2689 HRegion.RegionScannerImpl is = null;
2690
2691
2692
2693
2694 scan = new Scan();
2695 scan.addFamily(fam2);
2696 scan.addFamily(fam4);
2697 is = (RegionScannerImpl) region.getScanner(scan);
2698 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2699
2700 scan = new Scan();
2701 is = (RegionScannerImpl) region.getScanner(scan);
2702 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2703 }
2704
2705
2706
2707
2708
2709
2710 @Test
2711 public void testGetScanner_WithRegionClosed() throws IOException {
2712 byte[] fam1 = Bytes.toBytes("fam1");
2713 byte[] fam2 = Bytes.toBytes("fam2");
2714
2715 byte[][] families = { fam1, fam2 };
2716
2717
2718 String method = this.getName();
2719 try {
2720 this.region = initHRegion(tableName, method, CONF, families);
2721 } catch (IOException e) {
2722 e.printStackTrace();
2723 fail("Got IOException during initHRegion, " + e.getMessage());
2724 }
2725 region.closed.set(true);
2726 try {
2727 region.getScanner(new Scan());
2728 fail("Expected to get an exception during getScanner on a region that is closed");
2729 } catch (NotServingRegionException e) {
2730
2731 } catch (IOException e) {
2732 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2733 + e.getMessage());
2734 }
2735 }
2736
2737 @Test
2738 public void testRegionScanner_Next() throws IOException {
2739 byte[] row1 = Bytes.toBytes("row1");
2740 byte[] row2 = Bytes.toBytes("row2");
2741 byte[] fam1 = Bytes.toBytes("fam1");
2742 byte[] fam2 = Bytes.toBytes("fam2");
2743 byte[] fam3 = Bytes.toBytes("fam3");
2744 byte[] fam4 = Bytes.toBytes("fam4");
2745
2746 byte[][] families = { fam1, fam2, fam3, fam4 };
2747 long ts = System.currentTimeMillis();
2748
2749
2750 String method = this.getName();
2751 this.region = initHRegion(tableName, method, CONF, families);
2752
2753 Put put = null;
2754 put = new Put(row1);
2755 put.add(fam1, (byte[]) null, ts, null);
2756 put.add(fam2, (byte[]) null, ts, null);
2757 put.add(fam3, (byte[]) null, ts, null);
2758 put.add(fam4, (byte[]) null, ts, null);
2759 region.put(put);
2760
2761 put = new Put(row2);
2762 put.add(fam1, (byte[]) null, ts, null);
2763 put.add(fam2, (byte[]) null, ts, null);
2764 put.add(fam3, (byte[]) null, ts, null);
2765 put.add(fam4, (byte[]) null, ts, null);
2766 region.put(put);
2767
2768 Scan scan = new Scan();
2769 scan.addFamily(fam2);
2770 scan.addFamily(fam4);
2771 InternalScanner is = region.getScanner(scan);
2772
2773 List<Cell> res = null;
2774
2775
2776 List<Cell> expected1 = new ArrayList<Cell>();
2777 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2778 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2779
2780 res = new ArrayList<Cell>();
2781 is.next(res);
2782 for (int i = 0; i < res.size(); i++) {
2783 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2784 }
2785
2786
2787 List<Cell> expected2 = new ArrayList<Cell>();
2788 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2789 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2790
2791 res = new ArrayList<Cell>();
2792 is.next(res);
2793 for (int i = 0; i < res.size(); i++) {
2794 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2795 }
2796 }
2797
2798 @Test
2799 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2800 byte[] row1 = Bytes.toBytes("row1");
2801 byte[] qf1 = Bytes.toBytes("qualifier1");
2802 byte[] qf2 = Bytes.toBytes("qualifier2");
2803 byte[] fam1 = Bytes.toBytes("fam1");
2804 byte[][] families = { fam1 };
2805
2806 long ts1 = System.currentTimeMillis();
2807 long ts2 = ts1 + 1;
2808 long ts3 = ts1 + 2;
2809
2810
2811 String method = this.getName();
2812 this.region = initHRegion(tableName, method, CONF, families);
2813
2814 Put put = null;
2815 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2816 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2817 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2818
2819 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2820 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2821 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2822
2823 put = new Put(row1);
2824 put.add(kv13);
2825 put.add(kv12);
2826 put.add(kv11);
2827 put.add(kv23);
2828 put.add(kv22);
2829 put.add(kv21);
2830 region.put(put);
2831
2832
2833 List<Cell> expected = new ArrayList<Cell>();
2834 expected.add(kv13);
2835 expected.add(kv12);
2836
2837 Scan scan = new Scan(row1);
2838 scan.addColumn(fam1, qf1);
2839 scan.setMaxVersions(MAX_VERSIONS);
2840 List<Cell> actual = new ArrayList<Cell>();
2841 InternalScanner scanner = region.getScanner(scan);
2842
2843 boolean hasNext = scanner.next(actual);
2844 assertEquals(false, hasNext);
2845
2846
2847 for (int i = 0; i < expected.size(); i++) {
2848 assertEquals(expected.get(i), actual.get(i));
2849 }
2850 }
2851
2852 @Test
2853 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2854 byte[] row1 = Bytes.toBytes("row1");
2855 byte[] qf1 = Bytes.toBytes("qualifier1");
2856 byte[] qf2 = Bytes.toBytes("qualifier2");
2857 byte[] fam1 = Bytes.toBytes("fam1");
2858 byte[][] families = { fam1 };
2859
2860 long ts1 = 1;
2861 long ts2 = ts1 + 1;
2862 long ts3 = ts1 + 2;
2863
2864
2865 String method = this.getName();
2866 this.region = initHRegion(tableName, method, CONF, families);
2867
2868 Put put = null;
2869 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2870 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2871 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2872
2873 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2874 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2875 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2876
2877 put = new Put(row1);
2878 put.add(kv13);
2879 put.add(kv12);
2880 put.add(kv11);
2881 put.add(kv23);
2882 put.add(kv22);
2883 put.add(kv21);
2884 region.put(put);
2885 region.flush(true);
2886
2887
2888 List<Cell> expected = new ArrayList<Cell>();
2889 expected.add(kv13);
2890 expected.add(kv12);
2891 expected.add(kv23);
2892 expected.add(kv22);
2893
2894 Scan scan = new Scan(row1);
2895 scan.addColumn(fam1, qf1);
2896 scan.addColumn(fam1, qf2);
2897 scan.setMaxVersions(MAX_VERSIONS);
2898 List<Cell> actual = new ArrayList<Cell>();
2899 InternalScanner scanner = region.getScanner(scan);
2900
2901 boolean hasNext = scanner.next(actual);
2902 assertEquals(false, hasNext);
2903
2904
2905 for (int i = 0; i < expected.size(); i++) {
2906 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2907 }
2908 }
2909
2910 @Test
2911 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2912 byte[] row1 = Bytes.toBytes("row1");
2913 byte[] fam1 = Bytes.toBytes("fam1");
2914 byte[][] families = { fam1 };
2915 byte[] qf1 = Bytes.toBytes("qualifier1");
2916 byte[] qf2 = Bytes.toBytes("qualifier2");
2917
2918 long ts1 = 1;
2919 long ts2 = ts1 + 1;
2920 long ts3 = ts1 + 2;
2921 long ts4 = ts1 + 3;
2922
2923
2924 String method = this.getName();
2925 this.region = initHRegion(tableName, method, CONF, families);
2926
2927 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2928 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2929 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2930 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2931
2932 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2933 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2934 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2935 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2936
2937 Put put = null;
2938 put = new Put(row1);
2939 put.add(kv14);
2940 put.add(kv24);
2941 region.put(put);
2942 region.flush(true);
2943
2944 put = new Put(row1);
2945 put.add(kv23);
2946 put.add(kv13);
2947 region.put(put);
2948 region.flush(true);
2949
2950 put = new Put(row1);
2951 put.add(kv22);
2952 put.add(kv12);
2953 region.put(put);
2954 region.flush(true);
2955
2956 put = new Put(row1);
2957 put.add(kv21);
2958 put.add(kv11);
2959 region.put(put);
2960
2961
2962 List<Cell> expected = new ArrayList<Cell>();
2963 expected.add(kv14);
2964 expected.add(kv13);
2965 expected.add(kv12);
2966 expected.add(kv24);
2967 expected.add(kv23);
2968 expected.add(kv22);
2969
2970 Scan scan = new Scan(row1);
2971 scan.addColumn(fam1, qf1);
2972 scan.addColumn(fam1, qf2);
2973 int versions = 3;
2974 scan.setMaxVersions(versions);
2975 List<Cell> actual = new ArrayList<Cell>();
2976 InternalScanner scanner = region.getScanner(scan);
2977
2978 boolean hasNext = scanner.next(actual);
2979 assertEquals(false, hasNext);
2980
2981
2982 for (int i = 0; i < expected.size(); i++) {
2983 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2984 }
2985 }
2986
2987 @Test
2988 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2989 byte[] row1 = Bytes.toBytes("row1");
2990 byte[] qf1 = Bytes.toBytes("qualifier1");
2991 byte[] qf2 = Bytes.toBytes("qualifier2");
2992 byte[] fam1 = Bytes.toBytes("fam1");
2993 byte[][] families = { fam1 };
2994
2995 long ts1 = System.currentTimeMillis();
2996 long ts2 = ts1 + 1;
2997 long ts3 = ts1 + 2;
2998
2999
3000 String method = this.getName();
3001 this.region = initHRegion(tableName, method, CONF, families);
3002
3003 Put put = null;
3004 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3005 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3006 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3007
3008 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3009 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3010 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3011
3012 put = new Put(row1);
3013 put.add(kv13);
3014 put.add(kv12);
3015 put.add(kv11);
3016 put.add(kv23);
3017 put.add(kv22);
3018 put.add(kv21);
3019 region.put(put);
3020
3021
3022 List<Cell> expected = new ArrayList<Cell>();
3023 expected.add(kv13);
3024 expected.add(kv12);
3025 expected.add(kv23);
3026 expected.add(kv22);
3027
3028 Scan scan = new Scan(row1);
3029 scan.addFamily(fam1);
3030 scan.setMaxVersions(MAX_VERSIONS);
3031 List<Cell> actual = new ArrayList<Cell>();
3032 InternalScanner scanner = region.getScanner(scan);
3033
3034 boolean hasNext = scanner.next(actual);
3035 assertEquals(false, hasNext);
3036
3037
3038 for (int i = 0; i < expected.size(); i++) {
3039 assertEquals(expected.get(i), actual.get(i));
3040 }
3041 }
3042
3043 @Test
3044 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
3045 byte[] row1 = Bytes.toBytes("row1");
3046 byte[] qf1 = Bytes.toBytes("qualifier1");
3047 byte[] qf2 = Bytes.toBytes("qualifier2");
3048 byte[] fam1 = Bytes.toBytes("fam1");
3049
3050 long ts1 = 1;
3051 long ts2 = ts1 + 1;
3052 long ts3 = ts1 + 2;
3053
3054
3055 String method = this.getName();
3056 this.region = initHRegion(tableName, method, CONF, fam1);
3057
3058 Put put = null;
3059 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3060 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3061 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3062
3063 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3064 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3065 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3066
3067 put = new Put(row1);
3068 put.add(kv13);
3069 put.add(kv12);
3070 put.add(kv11);
3071 put.add(kv23);
3072 put.add(kv22);
3073 put.add(kv21);
3074 region.put(put);
3075 region.flush(true);
3076
3077
3078 List<Cell> expected = new ArrayList<Cell>();
3079 expected.add(kv13);
3080 expected.add(kv12);
3081 expected.add(kv23);
3082 expected.add(kv22);
3083
3084 Scan scan = new Scan(row1);
3085 scan.addFamily(fam1);
3086 scan.setMaxVersions(MAX_VERSIONS);
3087 List<Cell> actual = new ArrayList<Cell>();
3088 InternalScanner scanner = region.getScanner(scan);
3089
3090 boolean hasNext = scanner.next(actual);
3091 assertEquals(false, hasNext);
3092
3093
3094 for (int i = 0; i < expected.size(); i++) {
3095 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3096 }
3097 }
3098
3099 @Test
3100 public void testScanner_StopRow1542() throws IOException {
3101 byte[] family = Bytes.toBytes("testFamily");
3102 this.region = initHRegion(tableName, getName(), CONF, family);
3103 byte[] row1 = Bytes.toBytes("row111");
3104 byte[] row2 = Bytes.toBytes("row222");
3105 byte[] row3 = Bytes.toBytes("row333");
3106 byte[] row4 = Bytes.toBytes("row444");
3107 byte[] row5 = Bytes.toBytes("row555");
3108
3109 byte[] col1 = Bytes.toBytes("Pub111");
3110 byte[] col2 = Bytes.toBytes("Pub222");
3111
3112 Put put = new Put(row1);
3113 put.add(family, col1, Bytes.toBytes(10L));
3114 region.put(put);
3115
3116 put = new Put(row2);
3117 put.add(family, col1, Bytes.toBytes(15L));
3118 region.put(put);
3119
3120 put = new Put(row3);
3121 put.add(family, col2, Bytes.toBytes(20L));
3122 region.put(put);
3123
3124 put = new Put(row4);
3125 put.add(family, col2, Bytes.toBytes(30L));
3126 region.put(put);
3127
3128 put = new Put(row5);
3129 put.add(family, col1, Bytes.toBytes(40L));
3130 region.put(put);
3131
3132 Scan scan = new Scan(row3, row4);
3133 scan.setMaxVersions();
3134 scan.addColumn(family, col1);
3135 InternalScanner s = region.getScanner(scan);
3136
3137 List<Cell> results = new ArrayList<Cell>();
3138 assertEquals(false, s.next(results));
3139 assertEquals(0, results.size());
3140 }
3141
3142 @Test
3143 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3144 byte[] row1 = Bytes.toBytes("row1");
3145 byte[] fam1 = Bytes.toBytes("fam1");
3146 byte[] qf1 = Bytes.toBytes("qualifier1");
3147 byte[] qf2 = Bytes.toBytes("quateslifier2");
3148
3149 long ts1 = 1;
3150 long ts2 = ts1 + 1;
3151 long ts3 = ts1 + 2;
3152 long ts4 = ts1 + 3;
3153
3154
3155 String method = this.getName();
3156 this.region = initHRegion(tableName, method, CONF, fam1);
3157
3158 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3159 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3160 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3161 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3162
3163 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3164 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3165 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3166 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3167
3168 Put put = null;
3169 put = new Put(row1);
3170 put.add(kv14);
3171 put.add(kv24);
3172 region.put(put);
3173 region.flush(true);
3174
3175 put = new Put(row1);
3176 put.add(kv23);
3177 put.add(kv13);
3178 region.put(put);
3179 region.flush(true);
3180
3181 put = new Put(row1);
3182 put.add(kv22);
3183 put.add(kv12);
3184 region.put(put);
3185 region.flush(true);
3186
3187 put = new Put(row1);
3188 put.add(kv21);
3189 put.add(kv11);
3190 region.put(put);
3191
3192
3193 List<KeyValue> expected = new ArrayList<KeyValue>();
3194 expected.add(kv14);
3195 expected.add(kv13);
3196 expected.add(kv12);
3197 expected.add(kv24);
3198 expected.add(kv23);
3199 expected.add(kv22);
3200
3201 Scan scan = new Scan(row1);
3202 int versions = 3;
3203 scan.setMaxVersions(versions);
3204 List<Cell> actual = new ArrayList<Cell>();
3205 InternalScanner scanner = region.getScanner(scan);
3206
3207 boolean hasNext = scanner.next(actual);
3208 assertEquals(false, hasNext);
3209
3210
3211 for (int i = 0; i < expected.size(); i++) {
3212 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3213 }
3214 }
3215
3216
3217
3218
3219
3220
3221
3222 @Test
3223 public void testScanner_JoinedScanners() throws IOException {
3224 byte[] cf_essential = Bytes.toBytes("essential");
3225 byte[] cf_joined = Bytes.toBytes("joined");
3226 byte[] cf_alpha = Bytes.toBytes("alpha");
3227 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3228 byte[] row1 = Bytes.toBytes("row1");
3229 byte[] row2 = Bytes.toBytes("row2");
3230 byte[] row3 = Bytes.toBytes("row3");
3231
3232 byte[] col_normal = Bytes.toBytes("d");
3233 byte[] col_alpha = Bytes.toBytes("a");
3234
3235 byte[] filtered_val = Bytes.toBytes(3);
3236
3237 Put put = new Put(row1);
3238 put.add(cf_essential, col_normal, Bytes.toBytes(1));
3239 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3240 region.put(put);
3241
3242 put = new Put(row2);
3243 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3244 put.add(cf_joined, col_normal, Bytes.toBytes(2));
3245 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3246 region.put(put);
3247
3248 put = new Put(row3);
3249 put.add(cf_essential, col_normal, filtered_val);
3250 put.add(cf_joined, col_normal, filtered_val);
3251 region.put(put);
3252
3253
3254
3255
3256
3257 Scan scan = new Scan();
3258 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3259 CompareOp.NOT_EQUAL, filtered_val);
3260 scan.setFilter(filter);
3261 scan.setLoadColumnFamiliesOnDemand(true);
3262 InternalScanner s = region.getScanner(scan);
3263
3264 List<Cell> results = new ArrayList<Cell>();
3265 assertTrue(s.next(results));
3266 assertEquals(results.size(), 1);
3267 results.clear();
3268
3269 assertTrue(s.next(results));
3270 assertEquals(results.size(), 3);
3271 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3272 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3273 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3274 results.clear();
3275
3276 assertFalse(s.next(results));
3277 assertEquals(results.size(), 0);
3278 }
3279
3280
3281
3282
3283
3284
3285 @Test
3286 public void testScanner_JoinedScannersWithLimits() throws IOException {
3287 final byte[] cf_first = Bytes.toBytes("first");
3288 final byte[] cf_second = Bytes.toBytes("second");
3289
3290 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3291 final byte[] col_a = Bytes.toBytes("a");
3292 final byte[] col_b = Bytes.toBytes("b");
3293
3294 Put put;
3295
3296 for (int i = 0; i < 10; i++) {
3297 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3298 put.add(cf_first, col_a, Bytes.toBytes(i));
3299 if (i < 5) {
3300 put.add(cf_first, col_b, Bytes.toBytes(i));
3301 put.add(cf_second, col_a, Bytes.toBytes(i));
3302 put.add(cf_second, col_b, Bytes.toBytes(i));
3303 }
3304 region.put(put);
3305 }
3306
3307 Scan scan = new Scan();
3308 scan.setLoadColumnFamiliesOnDemand(true);
3309 Filter bogusFilter = new FilterBase() {
3310 @Override
3311 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3312 return ReturnCode.INCLUDE;
3313 }
3314 @Override
3315 public boolean isFamilyEssential(byte[] name) {
3316 return Bytes.equals(name, cf_first);
3317 }
3318 };
3319
3320 scan.setFilter(bogusFilter);
3321 InternalScanner s = region.getScanner(scan);
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352 List<Cell> results = new ArrayList<Cell>();
3353 int index = 0;
3354 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
3355 while (true) {
3356 boolean more = s.next(results, scannerContext);
3357 if ((index >> 1) < 5) {
3358 if (index % 2 == 0)
3359 assertEquals(results.size(), 3);
3360 else
3361 assertEquals(results.size(), 1);
3362 } else
3363 assertEquals(results.size(), 1);
3364 results.clear();
3365 index++;
3366 if (!more)
3367 break;
3368 }
3369 }
3370
3371
3372
3373
3374
3375
3376 @Test
3377 public void testLongQualifier() throws Exception {
3378 String method = name.getMethodName();
3379 TableName tableName = TableName.valueOf(method);
3380 byte[] family = Bytes.toBytes("family");
3381 this.region = initHRegion(tableName, method, CONF, family);
3382 byte[] q = new byte[Short.MAX_VALUE+2];
3383 Arrays.fill(q, 0, q.length-1, (byte)42);
3384 for (byte i=0; i<10; i++) {
3385 Put p = new Put(Bytes.toBytes("row"));
3386
3387 q[q.length-1]=i;
3388 p.addColumn(family, q, q);
3389 region.put(p);
3390 }
3391 region.flush(false);
3392 }
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402 @Test
3403 public void testBasicSplit() throws Exception {
3404 byte[][] families = { fam1, fam2, fam3 };
3405
3406 Configuration hc = initSplit();
3407
3408 String method = this.getName();
3409 this.region = initHRegion(tableName, method, hc, families);
3410
3411 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3412 region.flush(true);
3413 region.compactStores();
3414 byte[] splitRow = region.checkSplit();
3415 assertNotNull(splitRow);
3416 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3417 HRegion[] regions = splitRegion(region, splitRow);
3418 try {
3419
3420
3421
3422 for (int i = 0; i < regions.length; i++) {
3423 regions[i] = HRegion.openHRegion(regions[i], null);
3424 }
3425
3426
3427 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3428 assertGet(regions[1], fam3, splitRow);
3429
3430 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3431 assertScan(regions[1], fam3, splitRow);
3432
3433 for (int i = 0; i < regions.length; i++) {
3434
3435
3436 for (int j = 0; j < 2; j++) {
3437 HBaseTestCase.addContent(regions[i], fam3);
3438 }
3439 HBaseTestCase.addContent(regions[i], fam2);
3440 HBaseTestCase.addContent(regions[i], fam1);
3441 regions[i].flush(true);
3442 }
3443
3444 byte[][] midkeys = new byte[regions.length][];
3445
3446 for (int i = 0; i < regions.length; i++) {
3447 regions[i].compactStores();
3448 midkeys[i] = regions[i].checkSplit();
3449 }
3450
3451 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3452
3453
3454 for (int i = 0; i < regions.length; i++) {
3455 HRegion[] rs = null;
3456 if (midkeys[i] != null) {
3457 rs = splitRegion(regions[i], midkeys[i]);
3458 for (int j = 0; j < rs.length; j++) {
3459 sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()),
3460 HRegion.openHRegion(rs[j], null));
3461 }
3462 }
3463 }
3464 LOG.info("Made 4 regions");
3465
3466
3467 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3468 byte[] b = Bytes.toBytes(START_KEY);
3469 for (HRegion r : sortedMap.values()) {
3470 assertGet(r, fam3, b);
3471 b[0] += interval;
3472 }
3473 } finally {
3474 for (int i = 0; i < regions.length; i++) {
3475 try {
3476 regions[i].close();
3477 } catch (IOException e) {
3478
3479 }
3480 }
3481 }
3482 }
3483
3484 @Test
3485 public void testSplitRegion() throws IOException {
3486 byte[] qualifier = Bytes.toBytes("qualifier");
3487 Configuration hc = initSplit();
3488 int numRows = 10;
3489 byte[][] families = { fam1, fam3 };
3490
3491
3492 String method = this.getName();
3493 this.region = initHRegion(tableName, method, hc, families);
3494
3495
3496 int startRow = 100;
3497 putData(startRow, numRows, qualifier, families);
3498 int splitRow = startRow + numRows;
3499 putData(splitRow, numRows, qualifier, families);
3500 region.flush(true);
3501
3502 HRegion[] regions = null;
3503 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3504
3505 for (int i = 0; i < regions.length; i++) {
3506 regions[i] = HRegion.openHRegion(regions[i], null);
3507 }
3508
3509 assertEquals(2, regions.length);
3510
3511
3512
3513 verifyData(regions[0], startRow, numRows, qualifier, families);
3514 verifyData(regions[1], splitRow, numRows, qualifier, families);
3515 }
3516
3517 @Test
3518 public void testClearForceSplit() throws IOException {
3519 byte[] qualifier = Bytes.toBytes("qualifier");
3520 Configuration hc = initSplit();
3521 int numRows = 10;
3522 byte[][] families = { fam1, fam3 };
3523
3524
3525 String method = this.getName();
3526 this.region = initHRegion(tableName, method, hc, families);
3527
3528
3529 int startRow = 100;
3530 putData(startRow, numRows, qualifier, families);
3531 int splitRow = startRow + numRows;
3532 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3533 putData(splitRow, numRows, qualifier, families);
3534 region.flush(true);
3535
3536 HRegion[] regions = null;
3537
3538 region.forceSplit(splitRowBytes);
3539 assertTrue(region.shouldForceSplit());
3540
3541 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3542
3543
3544 HStore storeMock = Mockito.mock(HStore.class);
3545 when(storeMock.hasReferences()).thenReturn(true);
3546 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3547 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3548 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3549 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3550 assertTrue(region.hasReferences());
3551
3552
3553 regions = splitRegion(region, splitRowBytes);
3554 assertNull(regions);
3555
3556
3557 assertFalse(region.shouldForceSplit());
3558
3559
3560 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3561 assertFalse(region.hasReferences());
3562
3563
3564 regions = splitRegion(region, splitRowBytes);
3565
3566
3567 for (int i = 0; i < regions.length; i++) {
3568 regions[i] = HRegion.openHRegion(regions[i], null);
3569 }
3570
3571 assertEquals(2, regions.length);
3572
3573
3574
3575 verifyData(regions[0], startRow, numRows, qualifier, families);
3576 verifyData(regions[1], splitRow, numRows, qualifier, families);
3577 }
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589 @Test
3590 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3591 byte[] family = Bytes.toBytes("family");
3592 int numRows = 1000;
3593 int flushAndScanInterval = 10;
3594 int compactInterval = 10 * flushAndScanInterval;
3595
3596 String method = "testFlushCacheWhileScanning";
3597 this.region = initHRegion(tableName, method, CONF, family);
3598 FlushThread flushThread = new FlushThread();
3599 try {
3600 flushThread.start();
3601
3602 Scan scan = new Scan();
3603 scan.addFamily(family);
3604 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3605 new BinaryComparator(Bytes.toBytes(5L))));
3606
3607 int expectedCount = 0;
3608 List<Cell> res = new ArrayList<Cell>();
3609
3610 boolean toggle = true;
3611 for (long i = 0; i < numRows; i++) {
3612 Put put = new Put(Bytes.toBytes(i));
3613 put.setDurability(Durability.SKIP_WAL);
3614 put.add(family, qual1, Bytes.toBytes(i % 10));
3615 region.put(put);
3616
3617 if (i != 0 && i % compactInterval == 0) {
3618 LOG.debug("iteration = " + i+ " ts="+System.currentTimeMillis());
3619 region.compact(true);
3620 }
3621
3622 if (i % 10 == 5L) {
3623 expectedCount++;
3624 }
3625
3626 if (i != 0 && i % flushAndScanInterval == 0) {
3627 res.clear();
3628 InternalScanner scanner = region.getScanner(scan);
3629 if (toggle) {
3630 flushThread.flush();
3631 }
3632 while (scanner.next(res))
3633 ;
3634 if (!toggle) {
3635 flushThread.flush();
3636 }
3637 assertEquals("toggle="+toggle+"i=" + i + " ts="+System.currentTimeMillis(),
3638 expectedCount, res.size());
3639 toggle = !toggle;
3640 }
3641 }
3642
3643 } finally {
3644 try {
3645 flushThread.done();
3646 flushThread.join();
3647 flushThread.checkNoError();
3648 } catch (InterruptedException ie) {
3649 LOG.warn("Caught exception when joining with flushThread", ie);
3650 }
3651 HRegion.closeHRegion(this.region);
3652 this.region = null;
3653 }
3654 }
3655
3656 protected class FlushThread extends Thread {
3657 private volatile boolean done;
3658 private Throwable error = null;
3659
3660 FlushThread() {
3661 super("FlushThread");
3662 }
3663
3664 public void done() {
3665 done = true;
3666 synchronized (this) {
3667 interrupt();
3668 }
3669 }
3670
3671 public void checkNoError() {
3672 if (error != null) {
3673 assertNull(error);
3674 }
3675 }
3676
3677 @Override
3678 public void run() {
3679 done = false;
3680 while (!done) {
3681 synchronized (this) {
3682 try {
3683 wait();
3684 } catch (InterruptedException ignored) {
3685 if (done) {
3686 break;
3687 }
3688 }
3689 }
3690 try {
3691 region.flush(true);
3692 } catch (IOException e) {
3693 if (!done) {
3694 LOG.error("Error while flushing cache", e);
3695 error = e;
3696 }
3697 break;
3698 } catch (Throwable t) {
3699 LOG.error("Uncaught exception", t);
3700 throw t;
3701 }
3702 }
3703 }
3704
3705 public void flush() {
3706 synchronized (this) {
3707 notify();
3708 }
3709 }
3710 }
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721 @Test
3722 public void testWritesWhileScanning() throws IOException, InterruptedException {
3723 int testCount = 100;
3724 int numRows = 1;
3725 int numFamilies = 10;
3726 int numQualifiers = 100;
3727 int flushInterval = 7;
3728 int compactInterval = 5 * flushInterval;
3729 byte[][] families = new byte[numFamilies][];
3730 for (int i = 0; i < numFamilies; i++) {
3731 families[i] = Bytes.toBytes("family" + i);
3732 }
3733 byte[][] qualifiers = new byte[numQualifiers][];
3734 for (int i = 0; i < numQualifiers; i++) {
3735 qualifiers[i] = Bytes.toBytes("qual" + i);
3736 }
3737
3738 String method = "testWritesWhileScanning";
3739 this.region = initHRegion(tableName, method, CONF, families);
3740 FlushThread flushThread = new FlushThread();
3741 PutThread putThread = new PutThread(numRows, families, qualifiers);
3742 try {
3743 putThread.start();
3744 putThread.waitForFirstPut();
3745
3746 flushThread.start();
3747
3748 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3749
3750 int expectedCount = numFamilies * numQualifiers;
3751 List<Cell> res = new ArrayList<Cell>();
3752
3753 long prevTimestamp = 0L;
3754 for (int i = 0; i < testCount; i++) {
3755
3756 if (i != 0 && i % compactInterval == 0) {
3757 region.compact(true);
3758 for (Store store : region.getStores()) {
3759 store.closeAndArchiveCompactedFiles();
3760 }
3761 }
3762
3763 if (i != 0 && i % flushInterval == 0) {
3764 flushThread.flush();
3765 }
3766
3767 boolean previousEmpty = res.isEmpty();
3768 res.clear();
3769 InternalScanner scanner = region.getScanner(scan);
3770 while (scanner.next(res))
3771 ;
3772 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3773 assertEquals("i=" + i, expectedCount, res.size());
3774 long timestamp = res.get(0).getTimestamp();
3775 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3776 timestamp >= prevTimestamp);
3777 prevTimestamp = timestamp;
3778 }
3779 }
3780 putThread.done();
3781 region.flush(true);
3782
3783 } finally {
3784 try {
3785 flushThread.done();
3786 flushThread.join();
3787 flushThread.checkNoError();
3788
3789 putThread.join();
3790 putThread.checkNoError();
3791 } catch (InterruptedException ie) {
3792 LOG.warn("Caught exception when joining with flushThread", ie);
3793 }
3794
3795 try {
3796 LOG.info("Before close: " + this.region.getMVCC());
3797 HRegion.closeHRegion(this.region);
3798 } catch (DroppedSnapshotException dse) {
3799
3800
3801
3802
3803 }
3804 this.region = null;
3805 }
3806 }
3807
3808 protected class PutThread extends Thread {
3809 private boolean done;
3810 private int numPutsFinished = 0;
3811
3812 private Throwable error = null;
3813 private int numRows;
3814 private byte[][] families;
3815 private byte[][] qualifiers;
3816
3817 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3818 super("PutThread");
3819 this.numRows = numRows;
3820 this.families = families;
3821 this.qualifiers = qualifiers;
3822 }
3823
3824
3825
3826
3827 public void waitForFirstPut() throws InterruptedException {
3828
3829 while (numPutsFinished == 0) {
3830 checkNoError();
3831 Thread.sleep(50);
3832 }
3833 }
3834
3835 public void done() {
3836 done = true;
3837 synchronized (this) {
3838 interrupt();
3839 }
3840 }
3841
3842 public void checkNoError() {
3843 if (error != null) {
3844 assertNull(error);
3845 }
3846 }
3847
3848 @Override
3849 public void run() {
3850 done = false;
3851 while (!done) {
3852 try {
3853 for (int r = 0; r < numRows; r++) {
3854 byte[] row = Bytes.toBytes("row" + r);
3855 Put put = new Put(row);
3856 put.setDurability(Durability.SKIP_WAL);
3857 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3858 for (byte[] family : families) {
3859 for (byte[] qualifier : qualifiers) {
3860 put.add(family, qualifier, (long) numPutsFinished, value);
3861 }
3862 }
3863 region.put(put);
3864 numPutsFinished++;
3865 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3866 System.out.println("put iteration = " + numPutsFinished);
3867 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3868 region.delete(delete);
3869 }
3870 numPutsFinished++;
3871 }
3872 } catch (InterruptedIOException e) {
3873
3874 LOG.info("Interrupted", e);
3875 } catch (IOException e) {
3876 LOG.error("Error while putting records", e);
3877 error = e;
3878 break;
3879 }
3880 }
3881
3882 }
3883
3884 }
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895 @Test
3896 public void testWritesWhileGetting() throws Exception {
3897 int testCount = 100;
3898 int numRows = 1;
3899 int numFamilies = 10;
3900 int numQualifiers = 100;
3901 int compactInterval = 100;
3902 byte[][] families = new byte[numFamilies][];
3903 for (int i = 0; i < numFamilies; i++) {
3904 families[i] = Bytes.toBytes("family" + i);
3905 }
3906 byte[][] qualifiers = new byte[numQualifiers][];
3907 for (int i = 0; i < numQualifiers; i++) {
3908 qualifiers[i] = Bytes.toBytes("qual" + i);
3909 }
3910
3911
3912 String method = "testWritesWhileGetting";
3913
3914
3915
3916
3917
3918 Configuration conf = HBaseConfiguration.create(CONF);
3919 conf.setInt("hbase.hstore.compaction.min", 1);
3920 conf.setInt("hbase.hstore.compaction.max", 1000);
3921 this.region = initHRegion(tableName, method, conf, families);
3922 PutThread putThread = null;
3923 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3924 try {
3925 putThread = new PutThread(numRows, families, qualifiers);
3926 putThread.start();
3927 putThread.waitForFirstPut();
3928
3929
3930 ctx.addThread(new RepeatingTestThread(ctx) {
3931
3932 @Override
3933 public void doAnAction() throws Exception {
3934 region.flush(true);
3935
3936
3937 region.compact(false);
3938 for (Store store : region.getStores()) {
3939 store.closeAndArchiveCompactedFiles();
3940 }
3941 }
3942 });
3943 ctx.startThreads();
3944
3945 Get get = new Get(Bytes.toBytes("row0"));
3946 Result result = null;
3947
3948 int expectedCount = numFamilies * numQualifiers;
3949
3950 long prevTimestamp = 0L;
3951 for (int i = 0; i < testCount; i++) {
3952 LOG.info("testWritesWhileGetting verify turn " + i);
3953 boolean previousEmpty = result == null || result.isEmpty();
3954 result = region.get(get);
3955 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3956 assertEquals("i=" + i, expectedCount, result.size());
3957
3958
3959 long timestamp = 0;
3960 for (Cell kv : result.rawCells()) {
3961 if (CellUtil.matchingFamily(kv, families[0])
3962 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3963 timestamp = kv.getTimestamp();
3964 }
3965 }
3966 assertTrue(timestamp >= prevTimestamp);
3967 prevTimestamp = timestamp;
3968 Cell previousKV = null;
3969
3970 for (Cell kv : result.rawCells()) {
3971 byte[] thisValue = CellUtil.cloneValue(kv);
3972 if (previousKV != null) {
3973 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3974 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3975 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3976 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3977 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3978 }
3979 }
3980 previousKV = kv;
3981 }
3982 }
3983 }
3984 } finally {
3985 if (putThread != null)
3986 putThread.done();
3987
3988 region.flush(true);
3989
3990 if (putThread != null) {
3991 putThread.join();
3992 putThread.checkNoError();
3993 }
3994
3995 ctx.stop();
3996 HRegion.closeHRegion(this.region);
3997 this.region = null;
3998 }
3999 }
4000
4001 @Test
4002 public void testHolesInMeta() throws Exception {
4003 byte[] family = Bytes.toBytes("family");
4004 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
4005 false, family);
4006 byte[] rowNotServed = Bytes.toBytes("a");
4007 Get g = new Get(rowNotServed);
4008 try {
4009 region.get(g);
4010 fail();
4011 } catch (WrongRegionException x) {
4012
4013 }
4014 byte[] row = Bytes.toBytes("y");
4015 g = new Get(row);
4016 region.get(g);
4017 }
4018
4019 @Test
4020 public void testIndexesScanWithOneDeletedRow() throws IOException {
4021 byte[] family = Bytes.toBytes("family");
4022
4023
4024 String method = "testIndexesScanWithOneDeletedRow";
4025 this.region = initHRegion(tableName, method, CONF, family);
4026 Put put = new Put(Bytes.toBytes(1L));
4027 put.add(family, qual1, 1L, Bytes.toBytes(1L));
4028 region.put(put);
4029
4030 region.flush(true);
4031
4032 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
4033 region.delete(delete);
4034
4035 put = new Put(Bytes.toBytes(2L));
4036 put.add(family, qual1, 2L, Bytes.toBytes(2L));
4037 region.put(put);
4038
4039 Scan idxScan = new Scan();
4040 idxScan.addFamily(family);
4041 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
4042 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
4043 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
4044 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
4045 InternalScanner scanner = region.getScanner(idxScan);
4046 List<Cell> res = new ArrayList<>();
4047
4048 while (scanner.next(res)) {
4049
4050 }
4051 assertEquals(1L, res.size());
4052 }
4053
4054
4055
4056
4057 @Test
4058 public void testBloomFilterSize() throws IOException {
4059 byte[] fam1 = Bytes.toBytes("fam1");
4060 byte[] qf1 = Bytes.toBytes("col");
4061 byte[] val1 = Bytes.toBytes("value1");
4062
4063 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
4064 .setBloomFilterType(BloomType.ROWCOL);
4065
4066 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4067 htd.addFamily(hcd);
4068 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4069 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4070 int num_unique_rows = 10;
4071 int duplicate_multiplier = 2;
4072 int num_storefiles = 4;
4073
4074 int version = 0;
4075 for (int f = 0; f < num_storefiles; f++) {
4076 for (int i = 0; i < duplicate_multiplier; i++) {
4077 for (int j = 0; j < num_unique_rows; j++) {
4078 Put put = new Put(Bytes.toBytes("row" + j));
4079 put.setDurability(Durability.SKIP_WAL);
4080 put.add(fam1, qf1, version++, val1);
4081 region.put(put);
4082 }
4083 }
4084 region.flush(true);
4085 }
4086
4087 HStore store = (HStore) region.getStore(fam1);
4088 Collection<StoreFile> storeFiles = store.getStorefiles();
4089 for (StoreFile storefile : storeFiles) {
4090 StoreFile.Reader reader = storefile.getReader();
4091 reader.loadFileInfo();
4092 reader.loadBloomfilter();
4093 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4094 assertEquals(num_unique_rows, reader.getFilterEntries());
4095 }
4096
4097 region.compact(true);
4098
4099
4100 storeFiles = store.getStorefiles();
4101 for (StoreFile storefile : storeFiles) {
4102 StoreFile.Reader reader = storefile.getReader();
4103 reader.loadFileInfo();
4104 reader.loadBloomfilter();
4105 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4106 assertEquals(num_unique_rows, reader.getFilterEntries());
4107 }
4108 }
4109
4110 @Test
4111 public void testAllColumnsWithBloomFilter() throws IOException {
4112 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4113 byte[] FAMILY = Bytes.toBytes("family");
4114
4115
4116 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4117 .setBloomFilterType(BloomType.ROWCOL);
4118 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4119 htd.addFamily(hcd);
4120 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4121 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4122
4123 byte row[] = Bytes.toBytes("row:" + 0);
4124 byte column[] = Bytes.toBytes("column:" + 0);
4125 Put put = new Put(row);
4126 put.setDurability(Durability.SKIP_WAL);
4127 for (long idx = 1; idx <= 4; idx++) {
4128 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4129 }
4130 region.put(put);
4131
4132
4133 region.flush(true);
4134
4135
4136 Get get = new Get(row);
4137 get.setMaxVersions();
4138 Cell[] kvs = region.get(get).rawCells();
4139
4140
4141 assertEquals(4, kvs.length);
4142 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4143 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4144 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4145 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4146 }
4147
4148
4149
4150
4151
4152
4153 @Test
4154 public void testDeleteRowWithBloomFilter() throws IOException {
4155 byte[] familyName = Bytes.toBytes("familyName");
4156
4157
4158 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4159 .setBloomFilterType(BloomType.ROWCOL);
4160
4161 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4162 htd.addFamily(hcd);
4163 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4164 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4165
4166 byte row[] = Bytes.toBytes("row1");
4167 byte col[] = Bytes.toBytes("col1");
4168
4169 Put put = new Put(row);
4170 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4171 region.put(put);
4172 region.flush(true);
4173
4174 Delete del = new Delete(row);
4175 region.delete(del);
4176 region.flush(true);
4177
4178
4179 Get get = new Get(row);
4180 get.addColumn(familyName, col);
4181
4182 Cell[] keyValues = region.get(get).rawCells();
4183 assertEquals(0,keyValues.length);
4184 }
4185
4186 @Test
4187 public void testgetHDFSBlocksDistribution() throws Exception {
4188 HBaseTestingUtility htu = new HBaseTestingUtility();
4189
4190
4191
4192
4193
4194
4195
4196 htu.getConfiguration().setInt("dfs.replication", 2);
4197
4198
4199 MiniHBaseCluster cluster = null;
4200 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4201 int regionServersCount = 3;
4202
4203 try {
4204 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4205 byte[][] families = { fam1, fam2 };
4206 Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4207
4208
4209 byte row[] = Bytes.toBytes("row1");
4210 byte col[] = Bytes.toBytes("col1");
4211
4212 Put put = new Put(row);
4213 put.add(fam1, col, 1, Bytes.toBytes("test1"));
4214 put.add(fam2, col, 1, Bytes.toBytes("test2"));
4215 ht.put(put);
4216
4217 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4218 .get(0);
4219 firstRegion.flush(true);
4220 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4221
4222
4223
4224
4225
4226 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4227 StringBuilder sb = new StringBuilder();
4228 for (String host: blocksDistribution1.getTopHosts()) {
4229 if (sb.length() > 0) sb.append(", ");
4230 sb.append(host);
4231 sb.append("=");
4232 sb.append(blocksDistribution1.getWeight(host));
4233 }
4234
4235 String topHost = blocksDistribution1.getTopHosts().get(0);
4236 long topHostWeight = blocksDistribution1.getWeight(topHost);
4237 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4238 topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4239 LOG.info(msg);
4240 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4241
4242
4243
4244 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4245 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4246 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4247
4248 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4249
4250 ht.close();
4251 } finally {
4252 if (cluster != null) {
4253 htu.shutdownMiniCluster();
4254 }
4255 }
4256 }
4257
4258
4259
4260
4261
4262
4263
4264 @Test
4265 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4266 TableName tableName = TableName.valueOf(name.getMethodName());
4267 HRegionInfo info = null;
4268 try {
4269 FileSystem fs = Mockito.mock(FileSystem.class);
4270 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4271 HTableDescriptor htd = new HTableDescriptor(tableName);
4272 htd.addFamily(new HColumnDescriptor("cf"));
4273 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4274 HConstants.EMPTY_BYTE_ARRAY, false);
4275 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4276 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4277
4278 region.initialize();
4279 fail("Region initialization should fail due to IOException");
4280 } catch (IOException io) {
4281 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4282 for (MonitoredTask monitoredTask : tasks) {
4283 if (!(monitoredTask instanceof MonitoredRPCHandler)
4284 && monitoredTask.getDescription().contains(region.toString())) {
4285 assertTrue("Region state should be ABORTED.",
4286 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4287 break;
4288 }
4289 }
4290 }
4291 }
4292
4293
4294
4295
4296
4297 @Test
4298 public void testRegionInfoFileCreation() throws IOException {
4299 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4300
4301 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4302 htd.addFamily(new HColumnDescriptor("cf"));
4303
4304 HRegionInfo hri = new HRegionInfo(htd.getTableName());
4305
4306
4307 region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4308
4309 Path regionDir = region.getRegionFileSystem().getRegionDir();
4310 FileSystem fs = region.getRegionFileSystem().getFileSystem();
4311 HRegion.closeHRegion(region);
4312
4313 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4314
4315
4316 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4317 fs.exists(regionInfoFile));
4318
4319
4320 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4321 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4322 HRegion.closeHRegion(region);
4323
4324
4325 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4326 fs.exists(regionInfoFile));
4327
4328
4329 fs.delete(regionInfoFile, true);
4330 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4331 fs.exists(regionInfoFile));
4332
4333 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4334
4335 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4336 HRegion.closeHRegion(region);
4337
4338
4339 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4340 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4341
4342 region = null;
4343 }
4344
4345
4346
4347
4348 private static class Incrementer implements Runnable {
4349 private HRegion region;
4350 private final static byte[] incRow = Bytes.toBytes("incRow");
4351 private final static byte[] family = Bytes.toBytes("family");
4352 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4353 private final static long ONE = 1l;
4354 private int incCounter;
4355
4356 public Incrementer(HRegion region, int incCounter) {
4357 this.region = region;
4358 this.incCounter = incCounter;
4359 }
4360
4361 @Override
4362 public void run() {
4363 int count = 0;
4364 while (count < incCounter) {
4365 Increment inc = new Increment(incRow);
4366 inc.addColumn(family, qualifier, ONE);
4367 count++;
4368 try {
4369 region.increment(inc);
4370 } catch (IOException e) {
4371 e.printStackTrace();
4372 break;
4373 }
4374 }
4375 }
4376 }
4377
4378
4379
4380
4381
4382 @Test
4383 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4384 byte[] family = Incrementer.family;
4385 this.region = initHRegion(tableName, method, CONF, family);
4386 final HRegion region = this.region;
4387 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4388 Runnable flusher = new Runnable() {
4389 @Override
4390 public void run() {
4391 while (!incrementDone.get()) {
4392 try {
4393 region.flush(true);
4394 } catch (Exception e) {
4395 e.printStackTrace();
4396 }
4397 }
4398 }
4399 };
4400
4401
4402 int threadNum = 20;
4403 int incCounter = 100;
4404 long expected = threadNum * incCounter;
4405 Thread[] incrementers = new Thread[threadNum];
4406 Thread flushThread = new Thread(flusher);
4407 for (int i = 0; i < threadNum; i++) {
4408 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4409 incrementers[i].start();
4410 }
4411 flushThread.start();
4412 for (int i = 0; i < threadNum; i++) {
4413 incrementers[i].join();
4414 }
4415
4416 incrementDone.set(true);
4417 flushThread.join();
4418
4419 Get get = new Get(Incrementer.incRow);
4420 get.addColumn(Incrementer.family, Incrementer.qualifier);
4421 get.setMaxVersions(1);
4422 Result res = this.region.get(get);
4423 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4424
4425
4426 assertEquals(kvs.size(), 1);
4427 Cell kv = kvs.get(0);
4428 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4429 }
4430
4431
4432
4433
4434 private static class Appender implements Runnable {
4435 private HRegion region;
4436 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4437 private final static byte[] family = Bytes.toBytes("family");
4438 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4439 private final static byte[] CHAR = Bytes.toBytes("a");
4440 private int appendCounter;
4441
4442 public Appender(HRegion region, int appendCounter) {
4443 this.region = region;
4444 this.appendCounter = appendCounter;
4445 }
4446
4447 @Override
4448 public void run() {
4449 int count = 0;
4450 while (count < appendCounter) {
4451 Append app = new Append(appendRow);
4452 app.add(family, qualifier, CHAR);
4453 count++;
4454 try {
4455 region.append(app);
4456 } catch (IOException e) {
4457 e.printStackTrace();
4458 break;
4459 }
4460 }
4461 }
4462 }
4463
4464
4465
4466
4467
4468 @Test
4469 public void testParallelAppendWithMemStoreFlush() throws Exception {
4470 byte[] family = Appender.family;
4471 this.region = initHRegion(tableName, method, CONF, family);
4472 final HRegion region = this.region;
4473 final AtomicBoolean appendDone = new AtomicBoolean(false);
4474 Runnable flusher = new Runnable() {
4475 @Override
4476 public void run() {
4477 while (!appendDone.get()) {
4478 try {
4479 region.flush(true);
4480 } catch (Exception e) {
4481 e.printStackTrace();
4482 }
4483 }
4484 }
4485 };
4486
4487
4488
4489 int threadNum = 20;
4490 int appendCounter = 100;
4491 byte[] expected = new byte[threadNum * appendCounter];
4492 for (int i = 0; i < threadNum * appendCounter; i++) {
4493 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4494 }
4495 Thread[] appenders = new Thread[threadNum];
4496 Thread flushThread = new Thread(flusher);
4497 for (int i = 0; i < threadNum; i++) {
4498 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4499 appenders[i].start();
4500 }
4501 flushThread.start();
4502 for (int i = 0; i < threadNum; i++) {
4503 appenders[i].join();
4504 }
4505
4506 appendDone.set(true);
4507 flushThread.join();
4508
4509 Get get = new Get(Appender.appendRow);
4510 get.addColumn(Appender.family, Appender.qualifier);
4511 get.setMaxVersions(1);
4512 Result res = this.region.get(get);
4513 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4514
4515
4516 assertEquals(kvs.size(), 1);
4517 Cell kv = kvs.get(0);
4518 byte[] appendResult = new byte[kv.getValueLength()];
4519 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4520 assertArrayEquals(expected, appendResult);
4521 }
4522
4523
4524
4525
4526
4527 @Test
4528 public void testPutWithMemStoreFlush() throws Exception {
4529 byte[] family = Bytes.toBytes("family");
4530 ;
4531 byte[] qualifier = Bytes.toBytes("qualifier");
4532 byte[] row = Bytes.toBytes("putRow");
4533 byte[] value = null;
4534 this.region = initHRegion(tableName, method, CONF, family);
4535 Put put = null;
4536 Get get = null;
4537 List<Cell> kvs = null;
4538 Result res = null;
4539
4540 put = new Put(row);
4541 value = Bytes.toBytes("value0");
4542 put.add(family, qualifier, 1234567l, value);
4543 region.put(put);
4544 get = new Get(row);
4545 get.addColumn(family, qualifier);
4546 get.setMaxVersions();
4547 res = this.region.get(get);
4548 kvs = res.getColumnCells(family, qualifier);
4549 assertEquals(1, kvs.size());
4550 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4551
4552 region.flush(true);
4553 get = new Get(row);
4554 get.addColumn(family, qualifier);
4555 get.setMaxVersions();
4556 res = this.region.get(get);
4557 kvs = res.getColumnCells(family, qualifier);
4558 assertEquals(1, kvs.size());
4559 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4560
4561 put = new Put(row);
4562 value = Bytes.toBytes("value1");
4563 put.add(family, qualifier, 1234567l, value);
4564 region.put(put);
4565 get = new Get(row);
4566 get.addColumn(family, qualifier);
4567 get.setMaxVersions();
4568 res = this.region.get(get);
4569 kvs = res.getColumnCells(family, qualifier);
4570 assertEquals(1, kvs.size());
4571 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4572
4573 region.flush(true);
4574 get = new Get(row);
4575 get.addColumn(family, qualifier);
4576 get.setMaxVersions();
4577 res = this.region.get(get);
4578 kvs = res.getColumnCells(family, qualifier);
4579 assertEquals(1, kvs.size());
4580 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4581 }
4582
4583 @Test
4584 public void testDurability() throws Exception {
4585 String method = "testDurability";
4586
4587
4588
4589
4590
4591 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4592 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4593 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4594
4595 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4596 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4597 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4598
4599 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4600 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4601
4602 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4603 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4604
4605 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4606 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4607 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4608
4609
4610 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4611 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4612 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4613 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4614 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4615 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4616
4617 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4618 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4619 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4620 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4621 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4622 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4623
4624
4625 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4626 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4627 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4628 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4629 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4630 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4631
4632 }
4633
4634 private void durabilityTest(String method, Durability tableDurability,
4635 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4636 final boolean expectSyncFromLogSyncer) throws Exception {
4637 Configuration conf = HBaseConfiguration.create(CONF);
4638 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4639 TableName tableName = TableName.valueOf(method);
4640 byte[] family = Bytes.toBytes("family");
4641 Path logDir = new Path(new Path(dir + method), "log");
4642 final Configuration walConf = new Configuration(conf);
4643 FSUtils.setRootDir(walConf, logDir);
4644 final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4645 final WAL wal = spy(wals.getWAL(tableName.getName(), tableName.getNamespace()));
4646 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4647 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4648 new byte[][] { family });
4649
4650 Put put = new Put(Bytes.toBytes("r1"));
4651 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4652 put.setDurability(mutationDurability);
4653 region.put(put);
4654
4655
4656 verify(wal, expectAppend ? times(1) : never())
4657 .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4658 (WALEdit)any(), Mockito.anyBoolean());
4659
4660
4661 if (expectSync || expectSyncFromLogSyncer) {
4662 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4663 @Override
4664 public boolean evaluate() throws Exception {
4665 try {
4666 if (expectSync) {
4667 verify(wal, times(1)).sync(anyLong());
4668 } else if (expectSyncFromLogSyncer) {
4669 verify(wal, times(1)).sync();
4670 }
4671 } catch (Throwable ignore) {
4672 }
4673 return true;
4674 }
4675 });
4676 } else {
4677
4678 verify(wal, never()).sync();
4679 }
4680
4681 HRegion.closeHRegion(this.region);
4682 this.region = null;
4683 }
4684
4685 @Test
4686 public void testRegionReplicaSecondary() throws IOException {
4687
4688
4689 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4690 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4691
4692 byte[][] families = new byte[][] {
4693 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4694 };
4695 byte[] cq = Bytes.toBytes("cq");
4696 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4697 for (byte[] family : families) {
4698 htd.addFamily(new HColumnDescriptor(family));
4699 }
4700
4701 long time = System.currentTimeMillis();
4702 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4703 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4704 false, time, 0);
4705 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4706 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4707 false, time, 1);
4708
4709 HRegion primaryRegion = null, secondaryRegion = null;
4710
4711 try {
4712 primaryRegion = HRegion.createHRegion(primaryHri,
4713 rootDir, TEST_UTIL.getConfiguration(), htd);
4714
4715
4716 putData(primaryRegion, 0, 1000, cq, families);
4717
4718
4719 primaryRegion.flush(true);
4720
4721
4722 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4723
4724 verifyData(secondaryRegion, 0, 1000, cq, families);
4725 } finally {
4726 if (primaryRegion != null) {
4727 HRegion.closeHRegion(primaryRegion);
4728 }
4729 if (secondaryRegion != null) {
4730 HRegion.closeHRegion(secondaryRegion);
4731 }
4732 }
4733 }
4734
4735 @Test
4736 public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4737
4738
4739 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4740 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4741
4742 byte[][] families = new byte[][] {
4743 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4744 };
4745 byte[] cq = Bytes.toBytes("cq");
4746 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4747 for (byte[] family : families) {
4748 htd.addFamily(new HColumnDescriptor(family));
4749 }
4750
4751 long time = System.currentTimeMillis();
4752 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4753 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4754 false, time, 0);
4755 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4756 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4757 false, time, 1);
4758
4759 HRegion primaryRegion = null, secondaryRegion = null;
4760
4761 try {
4762 primaryRegion = HRegion.createHRegion(primaryHri,
4763 rootDir, TEST_UTIL.getConfiguration(), htd);
4764
4765
4766 putData(primaryRegion, 0, 1000, cq, families);
4767
4768
4769 primaryRegion.flush(true);
4770
4771
4772 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4773
4774 try {
4775 putData(secondaryRegion, 0, 1000, cq, families);
4776 fail("Should have thrown exception");
4777 } catch (IOException ex) {
4778
4779 }
4780 } finally {
4781 if (primaryRegion != null) {
4782 HRegion.closeHRegion(primaryRegion);
4783 }
4784 if (secondaryRegion != null) {
4785 HRegion.closeHRegion(secondaryRegion);
4786 }
4787 }
4788 }
4789
4790 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
4791 Configuration confForWAL = new Configuration(conf);
4792 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
4793 return new WALFactory(confForWAL,
4794 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
4795 "hregion-" + RandomStringUtils.randomNumeric(8));
4796 }
4797
4798 @Test
4799 public void testCompactionFromPrimary() throws IOException {
4800 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4801 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4802
4803 byte[][] families = new byte[][] {
4804 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4805 };
4806 byte[] cq = Bytes.toBytes("cq");
4807 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4808 for (byte[] family : families) {
4809 htd.addFamily(new HColumnDescriptor(family));
4810 }
4811
4812 long time = System.currentTimeMillis();
4813 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4814 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4815 false, time, 0);
4816 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4817 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4818 false, time, 1);
4819
4820 HRegion primaryRegion = null, secondaryRegion = null;
4821
4822 try {
4823 primaryRegion = HRegion.createHRegion(primaryHri,
4824 rootDir, TEST_UTIL.getConfiguration(), htd);
4825
4826
4827 putData(primaryRegion, 0, 1000, cq, families);
4828
4829
4830 primaryRegion.flush(true);
4831
4832
4833 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4834
4835
4836 Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
4837 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
4838 Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
4839 Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
4840
4841 verifyData(secondaryRegion, 0, 1000, cq, families);
4842 } finally {
4843 if (primaryRegion != null) {
4844 HRegion.closeHRegion(primaryRegion);
4845 }
4846 if (secondaryRegion != null) {
4847 HRegion.closeHRegion(secondaryRegion);
4848 }
4849 }
4850 }
4851
4852 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4853 putData(this.region, startRow, numRows, qf, families);
4854 }
4855
4856 private void putData(HRegion region,
4857 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4858 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
4859 }
4860
4861 static void putData(HRegion region, Durability durability,
4862 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4863 for (int i = startRow; i < startRow + numRows; i++) {
4864 Put put = new Put(Bytes.toBytes("" + i));
4865 put.setDurability(durability);
4866 for (byte[] family : families) {
4867 put.add(family, qf, null);
4868 }
4869 region.put(put);
4870 }
4871 }
4872
4873 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4874 throws IOException {
4875 for (int i = startRow; i < startRow + numRows; i++) {
4876 byte[] row = Bytes.toBytes("" + i);
4877 Get get = new Get(row);
4878 for (byte[] family : families) {
4879 get.addColumn(family, qf);
4880 }
4881 Result result = newReg.get(get);
4882 Cell[] raw = result.rawCells();
4883 assertEquals(families.length, result.size());
4884 for (int j = 0; j < families.length; j++) {
4885 assertTrue(CellUtil.matchingRow(raw[j], row));
4886 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4887 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4888 }
4889 }
4890 }
4891
4892 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4893
4894 Get get = new Get(k).addFamily(family).setMaxVersions();
4895 Cell[] results = r.get(get).rawCells();
4896 for (int j = 0; j < results.length; j++) {
4897 byte[] tmp = CellUtil.cloneValue(results[j]);
4898
4899 assertTrue(Bytes.equals(k, tmp));
4900 }
4901 }
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4915 throws IOException {
4916 byte[][] families = { fs };
4917 Scan scan = new Scan();
4918 for (int i = 0; i < families.length; i++)
4919 scan.addFamily(families[i]);
4920 InternalScanner s = r.getScanner(scan);
4921 try {
4922 List<Cell> curVals = new ArrayList<Cell>();
4923 boolean first = true;
4924 OUTER_LOOP: while (s.next(curVals)) {
4925 for (Cell kv : curVals) {
4926 byte[] val = CellUtil.cloneValue(kv);
4927 byte[] curval = val;
4928 if (first) {
4929 first = false;
4930 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4931 } else {
4932
4933 break OUTER_LOOP;
4934 }
4935 }
4936 }
4937 } finally {
4938 s.close();
4939 }
4940 }
4941
4942
4943
4944
4945
4946 @Test
4947 public void testFlushResult() throws IOException {
4948 String method = name.getMethodName();
4949 byte[] tableName = Bytes.toBytes(method);
4950 byte[] family = Bytes.toBytes("family");
4951
4952 this.region = initHRegion(tableName, method, family);
4953
4954
4955 HRegion.FlushResult fr = region.flush(true);
4956 assertFalse(fr.isFlushSucceeded());
4957 assertFalse(fr.isCompactionNeeded());
4958
4959
4960 for (int i = 0; i < 2; i++) {
4961 Put put = new Put(tableName).add(family, family, tableName);
4962 region.put(put);
4963 fr = region.flush(true);
4964 assertTrue(fr.isFlushSucceeded());
4965 assertFalse(fr.isCompactionNeeded());
4966 }
4967
4968
4969 for (int i = 0; i < 2; i++) {
4970 Put put = new Put(tableName).add(family, family, tableName);
4971 region.put(put);
4972 fr = region.flush(true);
4973 assertTrue(fr.isFlushSucceeded());
4974 assertTrue(fr.isCompactionNeeded());
4975 }
4976 }
4977
4978 private Configuration initSplit() {
4979
4980 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4981
4982
4983 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4984
4985 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4986
4987
4988 CONF.setLong("hbase.client.pause", 15 * 1000);
4989
4990
4991
4992 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4993 return CONF;
4994 }
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5006 byte[]... families) throws IOException {
5007 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
5008 }
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5020 byte[]... families) throws IOException {
5021 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
5022 }
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
5035 boolean isReadOnly, byte[]... families) throws IOException {
5036 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
5037 }
5038
5039 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5040 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
5041 throws IOException {
5042 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
5043 Durability.SYNC_WAL, null, families);
5044 }
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
5059 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
5060 WAL wal, byte[]... families) throws IOException {
5061 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
5062 isReadOnly, durability, wal, families);
5063 }
5064
5065
5066
5067
5068
5069 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
5070 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5071 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5072 Bytes.toString(CellUtil.cloneRow(kv)));
5073 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5074 Bytes.toString(CellUtil.cloneFamily(kv)));
5075 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5076 Bytes.toString(CellUtil.cloneQualifier(kv)));
5077 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5078 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5079 Bytes.toString(CellUtil.cloneValue(kv)));
5080 }
5081
5082 @Test (timeout=60000)
5083 public void testReverseScanner_FromMemStore_SingleCF_Normal()
5084 throws IOException {
5085 byte[] rowC = Bytes.toBytes("rowC");
5086 byte[] rowA = Bytes.toBytes("rowA");
5087 byte[] rowB = Bytes.toBytes("rowB");
5088 byte[] cf = Bytes.toBytes("CF");
5089 byte[][] families = { cf };
5090 byte[] col = Bytes.toBytes("C");
5091 long ts = 1;
5092 String method = this.getName();
5093 this.region = initHRegion(tableName, method, families);
5094 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5095 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5096 null);
5097 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5098 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5099 Put put = null;
5100 put = new Put(rowC);
5101 put.add(kv1);
5102 put.add(kv11);
5103 region.put(put);
5104 put = new Put(rowA);
5105 put.add(kv2);
5106 region.put(put);
5107 put = new Put(rowB);
5108 put.add(kv3);
5109 region.put(put);
5110
5111 Scan scan = new Scan(rowC);
5112 scan.setMaxVersions(5);
5113 scan.setReversed(true);
5114 InternalScanner scanner = region.getScanner(scan);
5115 List<Cell> currRow = new ArrayList<Cell>();
5116 boolean hasNext = scanner.next(currRow);
5117 assertEquals(2, currRow.size());
5118 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5119 assertTrue(hasNext);
5120 currRow.clear();
5121 hasNext = scanner.next(currRow);
5122 assertEquals(1, currRow.size());
5123 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5124 assertTrue(hasNext);
5125 currRow.clear();
5126 hasNext = scanner.next(currRow);
5127 assertEquals(1, currRow.size());
5128 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5129 assertFalse(hasNext);
5130 scanner.close();
5131 }
5132
5133 @Test (timeout=60000)
5134 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5135 throws IOException {
5136 byte[] rowC = Bytes.toBytes("rowC");
5137 byte[] rowA = Bytes.toBytes("rowA");
5138 byte[] rowB = Bytes.toBytes("rowB");
5139 byte[] rowD = Bytes.toBytes("rowD");
5140 byte[] cf = Bytes.toBytes("CF");
5141 byte[][] families = { cf };
5142 byte[] col = Bytes.toBytes("C");
5143 long ts = 1;
5144 String method = this.getName();
5145 this.region = initHRegion(tableName, method, families);
5146 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5147 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5148 null);
5149 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5150 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5151 Put put = null;
5152 put = new Put(rowC);
5153 put.add(kv1);
5154 put.add(kv11);
5155 region.put(put);
5156 put = new Put(rowA);
5157 put.add(kv2);
5158 region.put(put);
5159 put = new Put(rowB);
5160 put.add(kv3);
5161 region.put(put);
5162
5163 Scan scan = new Scan(rowD);
5164 List<Cell> currRow = new ArrayList<Cell>();
5165 scan.setReversed(true);
5166 scan.setMaxVersions(5);
5167 InternalScanner scanner = region.getScanner(scan);
5168 boolean hasNext = scanner.next(currRow);
5169 assertEquals(2, currRow.size());
5170 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5171 assertTrue(hasNext);
5172 currRow.clear();
5173 hasNext = scanner.next(currRow);
5174 assertEquals(1, currRow.size());
5175 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5176 assertTrue(hasNext);
5177 currRow.clear();
5178 hasNext = scanner.next(currRow);
5179 assertEquals(1, currRow.size());
5180 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5181 assertFalse(hasNext);
5182 scanner.close();
5183 }
5184
5185 @Test (timeout=60000)
5186 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5187 throws IOException {
5188 byte[] rowC = Bytes.toBytes("rowC");
5189 byte[] rowA = Bytes.toBytes("rowA");
5190 byte[] rowB = Bytes.toBytes("rowB");
5191 byte[] cf = Bytes.toBytes("CF");
5192 byte[][] families = { cf };
5193 byte[] col = Bytes.toBytes("C");
5194 long ts = 1;
5195 String method = this.getName();
5196 this.region = initHRegion(tableName, method, families);
5197 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5198 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5199 null);
5200 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5201 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5202 Put put = null;
5203 put = new Put(rowC);
5204 put.add(kv1);
5205 put.add(kv11);
5206 region.put(put);
5207 put = new Put(rowA);
5208 put.add(kv2);
5209 region.put(put);
5210 put = new Put(rowB);
5211 put.add(kv3);
5212 region.put(put);
5213 Scan scan = new Scan();
5214 List<Cell> currRow = new ArrayList<Cell>();
5215 scan.setReversed(true);
5216 InternalScanner scanner = region.getScanner(scan);
5217 boolean hasNext = scanner.next(currRow);
5218 assertEquals(1, currRow.size());
5219 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5220 assertTrue(hasNext);
5221 currRow.clear();
5222 hasNext = scanner.next(currRow);
5223 assertEquals(1, currRow.size());
5224 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5225 assertTrue(hasNext);
5226 currRow.clear();
5227 hasNext = scanner.next(currRow);
5228 assertEquals(1, currRow.size());
5229 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5230 assertFalse(hasNext);
5231 scanner.close();
5232 }
5233
5234 @Test (timeout=60000)
5235 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5236
5237 byte[] rowA = Bytes.toBytes("rowA");
5238 byte[] rowB = Bytes.toBytes("rowB");
5239 byte[] rowC = Bytes.toBytes("rowC");
5240 byte[] rowD = Bytes.toBytes("rowD");
5241 byte[] rowE = Bytes.toBytes("rowE");
5242 byte[] cf = Bytes.toBytes("CF");
5243 byte[][] families = { cf };
5244 byte[] col1 = Bytes.toBytes("col1");
5245 byte[] col2 = Bytes.toBytes("col2");
5246 long ts = 1;
5247 String method = this.getName();
5248 this.region = initHRegion(tableName, method, families);
5249 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5250 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5251 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5252 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5253 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5254 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5255 Put put = null;
5256 put = new Put(rowA);
5257 put.add(kv1);
5258 region.put(put);
5259 put = new Put(rowB);
5260 put.add(kv2);
5261 region.put(put);
5262 put = new Put(rowC);
5263 put.add(kv3);
5264 region.put(put);
5265 put = new Put(rowD);
5266 put.add(kv4_1);
5267 region.put(put);
5268 put = new Put(rowD);
5269 put.add(kv4_2);
5270 region.put(put);
5271 put = new Put(rowE);
5272 put.add(kv5);
5273 region.put(put);
5274 region.flush(true);
5275 Scan scan = new Scan(rowD, rowA);
5276 scan.addColumn(families[0], col1);
5277 scan.setReversed(true);
5278 List<Cell> currRow = new ArrayList<Cell>();
5279 InternalScanner scanner = region.getScanner(scan);
5280 boolean hasNext = scanner.next(currRow);
5281 assertEquals(1, currRow.size());
5282 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5283 assertTrue(hasNext);
5284 currRow.clear();
5285 hasNext = scanner.next(currRow);
5286 assertEquals(1, currRow.size());
5287 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5288 assertTrue(hasNext);
5289 currRow.clear();
5290 hasNext = scanner.next(currRow);
5291 assertEquals(1, currRow.size());
5292 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5293 assertFalse(hasNext);
5294 scanner.close();
5295
5296 scan = new Scan(rowD, rowA);
5297 scan.addColumn(families[0], col2);
5298 scan.setReversed(true);
5299 currRow.clear();
5300 scanner = region.getScanner(scan);
5301 hasNext = scanner.next(currRow);
5302 assertEquals(1, currRow.size());
5303 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5304 scanner.close();
5305 }
5306
5307 @Test (timeout=60000)
5308 public void testReverseScanner_smaller_blocksize() throws IOException {
5309
5310 byte[] rowA = Bytes.toBytes("rowA");
5311 byte[] rowB = Bytes.toBytes("rowB");
5312 byte[] rowC = Bytes.toBytes("rowC");
5313 byte[] rowD = Bytes.toBytes("rowD");
5314 byte[] rowE = Bytes.toBytes("rowE");
5315 byte[] cf = Bytes.toBytes("CF");
5316 byte[][] families = { cf };
5317 byte[] col1 = Bytes.toBytes("col1");
5318 byte[] col2 = Bytes.toBytes("col2");
5319 long ts = 1;
5320 String method = this.getName();
5321 HBaseConfiguration config = new HBaseConfiguration();
5322 config.setInt("test.block.size", 1);
5323 this.region = initHRegion(tableName, method, config, families);
5324 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5325 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5326 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5327 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5328 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5329 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5330 Put put = null;
5331 put = new Put(rowA);
5332 put.add(kv1);
5333 region.put(put);
5334 put = new Put(rowB);
5335 put.add(kv2);
5336 region.put(put);
5337 put = new Put(rowC);
5338 put.add(kv3);
5339 region.put(put);
5340 put = new Put(rowD);
5341 put.add(kv4_1);
5342 region.put(put);
5343 put = new Put(rowD);
5344 put.add(kv4_2);
5345 region.put(put);
5346 put = new Put(rowE);
5347 put.add(kv5);
5348 region.put(put);
5349 region.flush(true);
5350 Scan scan = new Scan(rowD, rowA);
5351 scan.addColumn(families[0], col1);
5352 scan.setReversed(true);
5353 List<Cell> currRow = new ArrayList<Cell>();
5354 InternalScanner scanner = region.getScanner(scan);
5355 boolean hasNext = scanner.next(currRow);
5356 assertEquals(1, currRow.size());
5357 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5358 assertTrue(hasNext);
5359 currRow.clear();
5360 hasNext = scanner.next(currRow);
5361 assertEquals(1, currRow.size());
5362 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5363 assertTrue(hasNext);
5364 currRow.clear();
5365 hasNext = scanner.next(currRow);
5366 assertEquals(1, currRow.size());
5367 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5368 assertFalse(hasNext);
5369 scanner.close();
5370
5371 scan = new Scan(rowD, rowA);
5372 scan.addColumn(families[0], col2);
5373 scan.setReversed(true);
5374 currRow.clear();
5375 scanner = region.getScanner(scan);
5376 hasNext = scanner.next(currRow);
5377 assertEquals(1, currRow.size());
5378 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5379 scanner.close();
5380 }
5381
5382 @Test (timeout=60000)
5383 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5384 throws IOException {
5385 byte[] row0 = Bytes.toBytes("row0");
5386 byte[] row1 = Bytes.toBytes("row1");
5387 byte[] row2 = Bytes.toBytes("row2");
5388 byte[] row3 = Bytes.toBytes("row3");
5389 byte[] row4 = Bytes.toBytes("row4");
5390 byte[] row5 = Bytes.toBytes("row5");
5391 byte[] cf1 = Bytes.toBytes("CF1");
5392 byte[] cf2 = Bytes.toBytes("CF2");
5393 byte[] cf3 = Bytes.toBytes("CF3");
5394 byte[][] families = { cf1, cf2, cf3 };
5395 byte[] col = Bytes.toBytes("C");
5396 long ts = 1;
5397 String method = this.getName();
5398 HBaseConfiguration conf = new HBaseConfiguration();
5399
5400 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5401 this.region = initHRegion(tableName, method, conf, families);
5402
5403 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5404 null);
5405 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5406 null);
5407 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5408 KeyValue.Type.Put, null);
5409 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5410 null);
5411 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5412 null);
5413 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5414 null);
5415 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5416 KeyValue.Type.Put, null);
5417 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5418 null);
5419 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5420 KeyValue.Type.Put, null);
5421 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5422 null);
5423 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5424 null);
5425 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5426 KeyValue.Type.Put, null);
5427 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5428 null);
5429 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5430 KeyValue.Type.Put, null);
5431 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5432 null);
5433 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5434 null);
5435
5436 Put put = null;
5437 put = new Put(row1);
5438 put.add(kv1_2_1);
5439 region.put(put);
5440 put = new Put(row2);
5441 put.add(kv2_4_1);
5442 region.put(put);
5443 put = new Put(row4);
5444 put.add(kv4_5_4);
5445 put.add(kv4_5_5);
5446 region.put(put);
5447 region.flush(true);
5448
5449 put = new Put(row4);
5450 put.add(kv4_5_1);
5451 put.add(kv4_5_3);
5452 region.put(put);
5453 put = new Put(row1);
5454 put.add(kv1_2_2);
5455 region.put(put);
5456 put = new Put(row2);
5457 put.add(kv2_4_4);
5458 region.put(put);
5459 region.flush(true);
5460
5461 put = new Put(row4);
5462 put.add(kv4_5_2);
5463 region.put(put);
5464 put = new Put(row2);
5465 put.add(kv2_4_2);
5466 put.add(kv2_4_3);
5467 region.put(put);
5468 put = new Put(row3);
5469 put.add(kv3_2_2);
5470 region.put(put);
5471 region.flush(true);
5472
5473
5474 put = new Put(row0);
5475 put.add(kv0_1_1);
5476 region.put(put);
5477 put = new Put(row3);
5478 put.add(kv3_2_1);
5479 region.put(put);
5480 put = new Put(row5);
5481 put.add(kv5_2_1);
5482 put.add(kv5_2_2);
5483 region.put(put);
5484
5485 Scan scan = new Scan(row4);
5486 scan.setMaxVersions(5);
5487 scan.setBatch(3);
5488 scan.setReversed(true);
5489 InternalScanner scanner = region.getScanner(scan);
5490 List<Cell> currRow = new ArrayList<Cell>();
5491 boolean hasNext = false;
5492
5493
5494
5495 hasNext = scanner.next(currRow);
5496 assertEquals(3, currRow.size());
5497 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5498 assertTrue(hasNext);
5499 currRow.clear();
5500 hasNext = scanner.next(currRow);
5501 assertEquals(2, currRow.size());
5502 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5503 assertTrue(hasNext);
5504
5505 currRow.clear();
5506 hasNext = scanner.next(currRow);
5507 assertEquals(2, currRow.size());
5508 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5509 assertTrue(hasNext);
5510
5511
5512 currRow.clear();
5513 hasNext = scanner.next(currRow);
5514 assertEquals(3, currRow.size());
5515 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5516 assertTrue(hasNext);
5517 currRow.clear();
5518 hasNext = scanner.next(currRow);
5519 assertEquals(1, currRow.size());
5520 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5521 assertTrue(hasNext);
5522
5523 currRow.clear();
5524 hasNext = scanner.next(currRow);
5525 assertEquals(2, currRow.size());
5526 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5527 assertTrue(hasNext);
5528
5529 currRow.clear();
5530 hasNext = scanner.next(currRow);
5531 assertEquals(1, currRow.size());
5532 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5533 assertFalse(hasNext);
5534
5535 scanner.close();
5536 }
5537
5538 @Test (timeout=60000)
5539 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5540 throws IOException {
5541 byte[] row1 = Bytes.toBytes("row1");
5542 byte[] row2 = Bytes.toBytes("row2");
5543 byte[] row3 = Bytes.toBytes("row3");
5544 byte[] row4 = Bytes.toBytes("row4");
5545 byte[] cf1 = Bytes.toBytes("CF1");
5546 byte[] cf2 = Bytes.toBytes("CF2");
5547 byte[] cf3 = Bytes.toBytes("CF3");
5548 byte[] cf4 = Bytes.toBytes("CF4");
5549 byte[][] families = { cf1, cf2, cf3, cf4 };
5550 byte[] col = Bytes.toBytes("C");
5551 long ts = 1;
5552 String method = this.getName();
5553 HBaseConfiguration conf = new HBaseConfiguration();
5554
5555 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5556 this.region = initHRegion(tableName, method, conf, families);
5557 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5558 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5559 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5560 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5561
5562 Put put = new Put(row1);
5563 put.add(kv1);
5564 region.put(put);
5565 region.flush(true);
5566
5567 put = new Put(row2);
5568 put.add(kv2);
5569 region.put(put);
5570 region.flush(true);
5571
5572 put = new Put(row3);
5573 put.add(kv3);
5574 region.put(put);
5575 region.flush(true);
5576
5577 put = new Put(row4);
5578 put.add(kv4);
5579 region.put(put);
5580
5581 Scan scan = new Scan(row4);
5582 scan.setReversed(true);
5583 scan.setBatch(10);
5584 InternalScanner scanner = region.getScanner(scan);
5585 List<Cell> currRow = new ArrayList<Cell>();
5586 boolean hasNext = scanner.next(currRow);
5587 assertEquals(1, currRow.size());
5588 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5589 assertTrue(hasNext);
5590 currRow.clear();
5591 hasNext = scanner.next(currRow);
5592 assertEquals(1, currRow.size());
5593 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5594 assertTrue(hasNext);
5595 currRow.clear();
5596 hasNext = scanner.next(currRow);
5597 assertEquals(1, currRow.size());
5598 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5599 assertTrue(hasNext);
5600 currRow.clear();
5601 hasNext = scanner.next(currRow);
5602 assertEquals(1, currRow.size());
5603 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5604 assertFalse(hasNext);
5605 }
5606
5607
5608
5609
5610 @Test (timeout = 60000)
5611 public void testReverseScanner_StackOverflow() throws IOException {
5612 byte[] cf1 = Bytes.toBytes("CF1");
5613 byte[][] families = {cf1};
5614 byte[] col = Bytes.toBytes("C");
5615 String method = this.getName();
5616 HBaseConfiguration conf = new HBaseConfiguration();
5617 this.region = initHRegion(tableName, method, conf, families);
5618
5619 Put put = new Put(Bytes.toBytes("19998"));
5620 put.add(cf1, col, Bytes.toBytes("val"));
5621 region.put(put);
5622 region.flushcache(true, true);
5623 Put put2 = new Put(Bytes.toBytes("19997"));
5624 put2.add(cf1, col, Bytes.toBytes("val"));
5625 region.put(put2);
5626
5627 Scan scan = new Scan(Bytes.toBytes("19998"));
5628 scan.setReversed(true);
5629 InternalScanner scanner = region.getScanner(scan);
5630
5631
5632
5633 for (int i = 10000; i < 20000; i++) {
5634 Put p = new Put(Bytes.toBytes("" + i));
5635 p.add(cf1, col, Bytes.toBytes("" + i));
5636 region.put(p);
5637 }
5638 region.flushcache(true, true);
5639
5640
5641
5642 for (int i = 10000; i < 20000; i++) {
5643 Put p = new Put(Bytes.toBytes("" + i));
5644 p.add(cf1, col, Bytes.toBytes("" + i));
5645 region.put(p);
5646 }
5647
5648 List<Cell> currRow = new ArrayList<>();
5649 boolean hasNext;
5650 do {
5651 hasNext = scanner.next(currRow);
5652 } while (hasNext);
5653 assertEquals(2, currRow.size());
5654 assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5655 assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5656 }
5657
5658 @Test (timeout=60000)
5659 public void testSplitRegionWithReverseScan() throws IOException {
5660 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5661 byte [] qualifier = Bytes.toBytes("qualifier");
5662 Configuration hc = initSplit();
5663 int numRows = 3;
5664 byte [][] families = {fam1};
5665
5666
5667 String method = this.getName();
5668 this.region = initHRegion(tableName, method, hc, families);
5669
5670
5671 int startRow = 100;
5672 putData(startRow, numRows, qualifier, families);
5673 int splitRow = startRow + numRows;
5674 putData(splitRow, numRows, qualifier, families);
5675 region.flush(true);
5676
5677 HRegion [] regions = null;
5678 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5679
5680 for (int i = 0; i < regions.length; i++) {
5681 regions[i] = HRegion.openHRegion(regions[i], null);
5682 }
5683
5684 assertEquals(2, regions.length);
5685
5686
5687
5688 verifyData(regions[0], startRow, numRows, qualifier, families);
5689 verifyData(regions[1], splitRow, numRows, qualifier, families);
5690
5691
5692 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5693 scan.setReversed(true);
5694 InternalScanner scanner = regions[1].getScanner(scan);
5695 List<Cell> currRow = new ArrayList<Cell>();
5696 boolean more = false;
5697 int verify = startRow + 2 * numRows - 1;
5698 do {
5699 more = scanner.next(currRow);
5700 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5701 verify--;
5702 currRow.clear();
5703 } while(more);
5704 assertEquals(verify, startRow + numRows - 1);
5705 scanner.close();
5706
5707 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5708 scan.setReversed(true);
5709 scanner = regions[1].getScanner(scan);
5710 verify = startRow + 2 * numRows - 1;
5711 do {
5712 more = scanner.next(currRow);
5713 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5714 verify--;
5715 currRow.clear();
5716 } while(more);
5717 assertEquals(verify, startRow + numRows - 1);
5718 scanner.close();
5719
5720 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5721 scan.setReversed(true);
5722 scanner = regions[0].getScanner(scan);
5723 verify = startRow + numRows - 1;
5724 do {
5725 more = scanner.next(currRow);
5726 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5727 verify--;
5728 currRow.clear();
5729 } while(more);
5730 assertEquals(verify, 99);
5731 scanner.close();
5732
5733 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5734 scan.setReversed(true);
5735 scanner = regions[0].getScanner(scan);
5736 verify = startRow + numRows - 1;
5737 do {
5738 more = scanner.next(currRow);
5739 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5740 verify--;
5741 currRow.clear();
5742 } while(more);
5743 assertEquals(verify, startRow - 1);
5744 scanner.close();
5745 }
5746
5747 @Test
5748 public void testWriteRequestsCounter() throws IOException {
5749 byte[] fam = Bytes.toBytes("info");
5750 byte[][] families = { fam };
5751 this.region = initHRegion(tableName, method, CONF, families);
5752
5753 Assert.assertEquals(0L, region.getWriteRequestsCount());
5754
5755 Put put = new Put(row);
5756 put.add(fam, fam, fam);
5757
5758 Assert.assertEquals(0L, region.getWriteRequestsCount());
5759 region.put(put);
5760 Assert.assertEquals(1L, region.getWriteRequestsCount());
5761 region.put(put);
5762 Assert.assertEquals(2L, region.getWriteRequestsCount());
5763 region.put(put);
5764 Assert.assertEquals(3L, region.getWriteRequestsCount());
5765
5766 region.delete(new Delete(row));
5767 Assert.assertEquals(4L, region.getWriteRequestsCount());
5768 }
5769
5770 @Test
5771 public void testOpenRegionWrittenToWAL() throws Exception {
5772 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5773 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5774
5775 HTableDescriptor htd
5776 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5777 htd.addFamily(new HColumnDescriptor(fam1));
5778 htd.addFamily(new HColumnDescriptor(fam2));
5779
5780 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5781 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5782
5783
5784 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(),
5785 htd);
5786 assertNotNull(region);
5787
5788
5789 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5790 region.flush(true);
5791 HBaseTestingUtility.closeRegionAndWAL(region);
5792
5793 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5794
5795
5796 WAL wal = mockWAL();
5797 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5798
5799 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5800 TEST_UTIL.getConfiguration(), rss, null);
5801
5802 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5803 , editCaptor.capture(), anyBoolean());
5804
5805 WALEdit edit = editCaptor.getValue();
5806 assertNotNull(edit);
5807 assertNotNull(edit.getCells());
5808 assertEquals(1, edit.getCells().size());
5809 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5810 assertNotNull(desc);
5811
5812 LOG.info("RegionEventDescriptor from WAL: " + desc);
5813
5814 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5815 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5816 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5817 hri.getEncodedNameAsBytes()));
5818 assertTrue(desc.getLogSequenceNumber() > 0);
5819 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5820 assertEquals(2, desc.getStoresCount());
5821
5822 StoreDescriptor store = desc.getStores(0);
5823 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5824 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5825 assertEquals(1, store.getStoreFileCount());
5826 assertFalse(store.getStoreFile(0).contains("/"));
5827
5828 store = desc.getStores(1);
5829 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5830 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5831 assertEquals(0, store.getStoreFileCount());
5832 }
5833
5834
5835 static class HRegionWithSeqId extends HRegion {
5836 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
5837 final Configuration confParam, final HRegionInfo regionInfo,
5838 final HTableDescriptor htd, final RegionServerServices rsServices) {
5839 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
5840 }
5841 @Override
5842 protected long getNextSequenceId(WAL wal) throws IOException {
5843 return 42;
5844 }
5845 }
5846
5847 @Test
5848 public void testFlushedFileWithNoTags() throws Exception {
5849 String method = "testFlushedFileWithNoTags";
5850 HTableDescriptor htd = new HTableDescriptor(tableName);
5851 htd.addFamily(new HColumnDescriptor(fam1));
5852 region = initHRegion(Bytes.toBytes(method), method, TEST_UTIL.getConfiguration(), fam1);
5853 Put put = new Put(Bytes.toBytes("a-b-0-0"));
5854 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
5855 region.put(put);
5856 region.flush(true);
5857 Store store = region.getStore(fam1);
5858 Collection<StoreFile> storefiles = store.getStorefiles();
5859 for (StoreFile sf : storefiles) {
5860 assertFalse("Tags should not be present "
5861 ,sf.getReader().getHFileReader().getFileContext().isIncludesTags());
5862 }
5863 }
5864
5865 @Test
5866 public void testOpenRegionWrittenToWALForLogReplay() throws Exception {
5867
5868 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWALForLogReplay",
5869 100, 42);
5870 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5871
5872 HTableDescriptor htd
5873 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWALForLogReplay"));
5874 htd.addFamily(new HColumnDescriptor(fam1));
5875 htd.addFamily(new HColumnDescriptor(fam2));
5876
5877 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5878 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5879
5880
5881 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5882 .getConfiguration(), htd);
5883 assertNotNull(region);
5884
5885
5886 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5887 region.flush(true);
5888 HRegion.closeHRegion(region);
5889
5890 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5891
5892
5893 WAL wal = mockWAL();
5894 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5895
5896
5897 HashMap<String, Region> recoveringRegions = Maps.newHashMap();
5898 recoveringRegions.put(region.getRegionInfo().getEncodedName(), null);
5899 when(rss.getRecoveringRegions()).thenReturn(recoveringRegions);
5900
5901 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5902 conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName());
5903 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5904 conf, rss, null);
5905
5906
5907
5908 verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5909 , editCaptor.capture(), anyBoolean());
5910
5911
5912 new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo")
5913 .prepare().process();
5914
5915
5916 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5917 , editCaptor.capture(), anyBoolean());
5918
5919 WALEdit edit = editCaptor.getValue();
5920 assertNotNull(edit);
5921 assertNotNull(edit.getCells());
5922 assertEquals(1, edit.getCells().size());
5923 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5924 assertNotNull(desc);
5925
5926 LOG.info("RegionEventDescriptor from WAL: " + desc);
5927
5928 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5929 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5930 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5931 hri.getEncodedNameAsBytes()));
5932 assertTrue(desc.getLogSequenceNumber() > 0);
5933 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5934 assertEquals(2, desc.getStoresCount());
5935
5936 StoreDescriptor store = desc.getStores(0);
5937 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5938 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5939 assertEquals(1, store.getStoreFileCount());
5940 assertFalse(store.getStoreFile(0).contains("/"));
5941
5942 store = desc.getStores(1);
5943 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5944 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5945 assertEquals(0, store.getStoreFileCount());
5946 }
5947
5948
5949
5950
5951
5952
5953
5954 private WAL mockWAL() throws IOException {
5955 WAL wal = mock(WAL.class);
5956 Mockito.when(wal.append((HTableDescriptor)Mockito.any(), (HRegionInfo)Mockito.any(),
5957 (WALKey)Mockito.any(), (WALEdit)Mockito.any(), Mockito.anyBoolean())).
5958 thenAnswer(new Answer<Long>() {
5959 @Override
5960 public Long answer(InvocationOnMock invocation) throws Throwable {
5961 WALKey key = invocation.getArgumentAt(2, WALKey.class);
5962 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
5963 key.setWriteEntry(we);
5964 return 1L;
5965 }
5966
5967 });
5968 return wal;
5969 }
5970
5971 @Test
5972 public void testCloseRegionWrittenToWAL() throws Exception {
5973
5974 Path rootDir = new Path(dir + name.getMethodName());
5975 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5976
5977 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
5978 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5979
5980 HTableDescriptor htd
5981 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5982 htd.addFamily(new HColumnDescriptor(fam1));
5983 htd.addFamily(new HColumnDescriptor(fam2));
5984
5985 final HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5986 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5987
5988 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5989
5990
5991 WAL wal = mockWAL();
5992 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5993
5994
5995
5996 region = HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri));
5997 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5998 TEST_UTIL.getConfiguration(), rss, null);
5999
6000
6001 region.close(false);
6002
6003
6004 verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
6005 editCaptor.capture(), anyBoolean());
6006
6007 WALEdit edit = editCaptor.getAllValues().get(1);
6008 assertNotNull(edit);
6009 assertNotNull(edit.getCells());
6010 assertEquals(1, edit.getCells().size());
6011 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6012 assertNotNull(desc);
6013
6014 LOG.info("RegionEventDescriptor from WAL: " + desc);
6015
6016 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
6017 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
6018 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
6019 hri.getEncodedNameAsBytes()));
6020 assertTrue(desc.getLogSequenceNumber() > 0);
6021 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6022 assertEquals(2, desc.getStoresCount());
6023
6024 StoreDescriptor store = desc.getStores(0);
6025 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6026 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6027 assertEquals(0, store.getStoreFileCount());
6028
6029 store = desc.getStores(1);
6030 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6031 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6032 assertEquals(0, store.getStoreFileCount());
6033 }
6034
6035
6036
6037
6038 @Test (timeout=24000)
6039 public void testRegionTooBusy() throws IOException {
6040 String method = "testRegionTooBusy";
6041 byte[] tableName = Bytes.toBytes(method);
6042 byte[] family = Bytes.toBytes("family");
6043 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
6044 HRegion.DEFAULT_BUSY_WAIT_DURATION);
6045 CONF.setLong("hbase.busy.wait.duration", 1000);
6046 region = initHRegion(tableName, method, CONF, family);
6047 final AtomicBoolean stopped = new AtomicBoolean(true);
6048 Thread t = new Thread(new Runnable() {
6049 @Override
6050 public void run() {
6051 try {
6052 region.lock.writeLock().lock();
6053 stopped.set(false);
6054 while (!stopped.get()) {
6055 Thread.sleep(100);
6056 }
6057 } catch (InterruptedException ie) {
6058 } finally {
6059 region.lock.writeLock().unlock();
6060 }
6061 }
6062 });
6063 t.start();
6064 Get get = new Get(row);
6065 try {
6066 while (stopped.get()) {
6067 Thread.sleep(100);
6068 }
6069 region.get(get);
6070 fail("Should throw RegionTooBusyException");
6071 } catch (InterruptedException ie) {
6072 fail("test interrupted");
6073 } catch (RegionTooBusyException e) {
6074
6075 } finally {
6076 stopped.set(true);
6077 try {
6078 t.join();
6079 } catch (Throwable e) {
6080 }
6081
6082 HRegion.closeHRegion(region);
6083 region = null;
6084 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
6085 }
6086 }
6087
6088 @Test
6089 public void testCellTTLs() throws IOException {
6090 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6091 EnvironmentEdgeManager.injectEdge(edge);
6092
6093 final byte[] row = Bytes.toBytes("testRow");
6094 final byte[] q1 = Bytes.toBytes("q1");
6095 final byte[] q2 = Bytes.toBytes("q2");
6096 final byte[] q3 = Bytes.toBytes("q3");
6097 final byte[] q4 = Bytes.toBytes("q4");
6098
6099 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
6100 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
6101 hcd.setTimeToLive(10);
6102 htd.addFamily(hcd);
6103
6104 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6105 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6106
6107 region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
6108 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
6109 TEST_UTIL.getDataTestDir(), conf, htd);
6110 assertNotNull(region);
6111 long now = EnvironmentEdgeManager.currentTime();
6112
6113 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
6114 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6115
6116 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6117
6118 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6119
6120 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6121 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6122
6123 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6124
6125 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6126
6127
6128 region.flush(true);
6129
6130
6131 Result r = region.get(new Get(row));
6132 assertNotNull(r.getValue(fam1, q1));
6133 assertNotNull(r.getValue(fam1, q2));
6134 assertNotNull(r.getValue(fam1, q3));
6135 assertNotNull(r.getValue(fam1, q4));
6136
6137
6138 edge.incrementTime(5000);
6139
6140 r = region.get(new Get(row));
6141 assertNull(r.getValue(fam1, q1));
6142 assertNotNull(r.getValue(fam1, q2));
6143 assertNotNull(r.getValue(fam1, q3));
6144 assertNotNull(r.getValue(fam1, q4));
6145
6146
6147 edge.incrementTime(5000);
6148
6149 r = region.get(new Get(row));
6150 assertNull(r.getValue(fam1, q1));
6151 assertNull(r.getValue(fam1, q2));
6152 assertNotNull(r.getValue(fam1, q3));
6153 assertNotNull(r.getValue(fam1, q4));
6154
6155
6156 edge.incrementTime(5000);
6157
6158 r = region.get(new Get(row));
6159 assertNull(r.getValue(fam1, q1));
6160 assertNull(r.getValue(fam1, q2));
6161 assertNull(r.getValue(fam1, q3));
6162 assertNotNull(r.getValue(fam1, q4));
6163
6164
6165 edge.incrementTime(10000);
6166
6167 r = region.get(new Get(row));
6168 assertNull(r.getValue(fam1, q1));
6169 assertNull(r.getValue(fam1, q2));
6170 assertNull(r.getValue(fam1, q3));
6171 assertNull(r.getValue(fam1, q4));
6172
6173
6174
6175
6176 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
6177 r = region.get(new Get(row));
6178 byte[] val = r.getValue(fam1, q1);
6179 assertNotNull(val);
6180 assertEquals(Bytes.toLong(val), 1L);
6181
6182
6183 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6184 incr.setTTL(5000);
6185 region.increment(incr);
6186
6187
6188 r = region.get(new Get(row));
6189 val = r.getValue(fam1, q1);
6190 assertNotNull(val);
6191 assertEquals(Bytes.toLong(val), 2L);
6192
6193
6194 edge.incrementTime(5000);
6195
6196
6197 r = region.get(new Get(row));
6198 val = r.getValue(fam1, q1);
6199 assertNotNull(val);
6200 assertEquals(Bytes.toLong(val), 1L);
6201
6202
6203 edge.incrementTime(5000);
6204
6205
6206 r = region.get(new Get(row));
6207 assertNull(r.getValue(fam1, q1));
6208 }
6209
6210 @Test
6211 public void testIncrementTimestampsAreMonotonic() throws IOException {
6212 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6213 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6214 EnvironmentEdgeManager.injectEdge(edge);
6215
6216 edge.setValue(10);
6217 Increment inc = new Increment(row);
6218 inc.setDurability(Durability.SKIP_WAL);
6219 inc.addColumn(fam1, qual1, 1L);
6220 region.increment(inc);
6221
6222 Result result = region.get(new Get(row));
6223 Cell c = result.getColumnLatestCell(fam1, qual1);
6224 assertNotNull(c);
6225 assertEquals(c.getTimestamp(), 10L);
6226
6227 edge.setValue(1);
6228 region.increment(inc);
6229 result = region.get(new Get(row));
6230 c = result.getColumnLatestCell(fam1, qual1);
6231 assertEquals(c.getTimestamp(), 11L);
6232 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
6233 }
6234
6235 @Test
6236 public void testAppendTimestampsAreMonotonic() throws IOException {
6237 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6238 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6239 EnvironmentEdgeManager.injectEdge(edge);
6240
6241 edge.setValue(10);
6242 Append a = new Append(row);
6243 a.setDurability(Durability.SKIP_WAL);
6244 a.add(fam1, qual1, qual1);
6245 region.append(a);
6246
6247 Result result = region.get(new Get(row));
6248 Cell c = result.getColumnLatestCell(fam1, qual1);
6249 assertNotNull(c);
6250 assertEquals(c.getTimestamp(), 10L);
6251
6252 edge.setValue(1);
6253 region.append(a);
6254 result = region.get(new Get(row));
6255 c = result.getColumnLatestCell(fam1, qual1);
6256 assertEquals(c.getTimestamp(), 11L);
6257
6258 byte[] expected = new byte[qual1.length*2];
6259 System.arraycopy(qual1, 0, expected, 0, qual1.length);
6260 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
6261
6262 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6263 expected, 0, expected.length));
6264 }
6265
6266 @Test
6267 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
6268 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6269 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6270 EnvironmentEdgeManager.injectEdge(edge);
6271
6272 edge.setValue(10);
6273 Put p = new Put(row);
6274 p.setDurability(Durability.SKIP_WAL);
6275 p.addColumn(fam1, qual1, qual1);
6276 region.put(p);
6277
6278 Result result = region.get(new Get(row));
6279 Cell c = result.getColumnLatestCell(fam1, qual1);
6280 assertNotNull(c);
6281 assertEquals(c.getTimestamp(), 10L);
6282
6283 edge.setValue(1);
6284 p = new Put(row);
6285 p.setDurability(Durability.SKIP_WAL);
6286 p.addColumn(fam1, qual1, qual2);
6287 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
6288 result = region.get(new Get(row));
6289 c = result.getColumnLatestCell(fam1, qual1);
6290 assertEquals(c.getTimestamp(), 10L);
6291
6292 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6293 qual2, 0, qual2.length));
6294 }
6295
6296 @Test(timeout = 60000)
6297 public void testBatchMutateWithWrongRegionException() throws Exception {
6298 final byte[] a = Bytes.toBytes("a");
6299 final byte[] b = Bytes.toBytes("b");
6300 final byte[] c = Bytes.toBytes("c");
6301
6302 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
6303 CONF.setInt("hbase.rowlock.wait.duration", 1000);
6304 region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
6305
6306 Mutation[] mutations = new Mutation[] {
6307 new Put(a).addImmutable(fam1, null, null),
6308 new Put(c).addImmutable(fam1, null, null),
6309 new Put(b).addImmutable(fam1, null, null),
6310 };
6311
6312 OperationStatus[] status = region.batchMutate(mutations);
6313 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6314 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
6315 assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6316
6317
6318
6319 final CountDownLatch obtainedRowLock = new CountDownLatch(1);
6320 ExecutorService exec = Executors.newFixedThreadPool(2);
6321 Future<Void> f1 = exec.submit(new Callable<Void>() {
6322 @Override
6323 public Void call() throws Exception {
6324 LOG.info("Acquiring row lock");
6325 RowLock rl = region.getRowLock(b);
6326 obtainedRowLock.countDown();
6327 LOG.info("Waiting for 5 seconds before releasing lock");
6328 Threads.sleep(5000);
6329 LOG.info("Releasing row lock");
6330 rl.release();
6331 return null;
6332 }
6333 });
6334 obtainedRowLock.await(30, TimeUnit.SECONDS);
6335
6336 Future<Void> f2 = exec.submit(new Callable<Void>() {
6337 @Override
6338 public Void call() throws Exception {
6339 Mutation[] mutations = new Mutation[] {
6340 new Put(a).addImmutable(fam1, null, null),
6341 new Put(b).addImmutable(fam1, null, null),
6342 };
6343
6344
6345 OperationStatus[] status = region.batchMutate(mutations);
6346 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6347 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6348 return null;
6349 }
6350 });
6351
6352 f1.get();
6353 f2.get();
6354
6355 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
6356 }
6357
6358 @Test
6359 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
6360 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6361 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6362 EnvironmentEdgeManager.injectEdge(edge);
6363
6364 edge.setValue(10);
6365 Put p = new Put(row);
6366 p.setDurability(Durability.SKIP_WAL);
6367 p.addColumn(fam1, qual1, qual1);
6368 region.put(p);
6369
6370 Result result = region.get(new Get(row));
6371 Cell c = result.getColumnLatestCell(fam1, qual1);
6372 assertNotNull(c);
6373 assertEquals(c.getTimestamp(), 10L);
6374
6375 edge.setValue(1);
6376 p = new Put(row);
6377 p.setDurability(Durability.SKIP_WAL);
6378 p.addColumn(fam1, qual1, qual2);
6379 RowMutations rm = new RowMutations(row);
6380 rm.add(p);
6381 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
6382 rm, false);
6383 result = region.get(new Get(row));
6384 c = result.getColumnLatestCell(fam1, qual1);
6385 assertEquals(c.getTimestamp(), 10L);
6386
6387 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6388 qual2, 0, qual2.length));
6389 }
6390
6391 static HRegion initHRegion(byte[] tableName, String callingMethod,
6392 byte[]... families) throws IOException {
6393 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6394 families);
6395 }
6396
6397
6398
6399
6400
6401 @Test
6402 public void testWritesWhileRollWriter() throws IOException {
6403 int testCount = 10;
6404 int numRows = 1024;
6405 int numFamilies = 2;
6406 int numQualifiers = 2;
6407 final byte[][] families = new byte[numFamilies][];
6408 for (int i = 0; i < numFamilies; i++) {
6409 families[i] = Bytes.toBytes("family" + i);
6410 }
6411 final byte[][] qualifiers = new byte[numQualifiers][];
6412 for (int i = 0; i < numQualifiers; i++) {
6413 qualifiers[i] = Bytes.toBytes("qual" + i);
6414 }
6415
6416 String method = "testWritesWhileRollWriter";
6417 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2);
6418 this.region = initHRegion(tableName, method, CONF, families);
6419 try {
6420 List<Thread> threads = new ArrayList<Thread>();
6421 for (int i = 0; i < numRows; i++) {
6422 final int count = i;
6423 Thread t = new Thread(new Runnable() {
6424
6425 @Override
6426 public void run() {
6427 byte[] row = Bytes.toBytes("row" + count);
6428 Put put = new Put(row);
6429 put.setDurability(Durability.SYNC_WAL);
6430 byte[] value = Bytes.toBytes(String.valueOf(count));
6431 for (byte[] family : families) {
6432 for (byte[] qualifier : qualifiers) {
6433 put.addColumn(family, qualifier, (long) count, value);
6434 }
6435 }
6436 try {
6437 region.put(put);
6438 } catch (IOException e) {
6439 throw new RuntimeException(e);
6440 }
6441 }
6442 });
6443 threads.add(t);
6444 }
6445 for (Thread t : threads) {
6446 t.start();
6447 }
6448
6449 for (int i = 0; i < testCount; i++) {
6450 region.getWAL().rollWriter();
6451 Thread.yield();
6452 }
6453 } finally {
6454 try {
6455 HBaseTestingUtility.closeRegionAndWAL(this.region);
6456 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024);
6457 } catch (DroppedSnapshotException dse) {
6458
6459
6460
6461
6462 }
6463 this.region = null;
6464 }
6465 }
6466
6467 @Test
6468 public void testMutateRow_WriteRequestCount() throws Exception {
6469 byte[] row1 = Bytes.toBytes("row1");
6470 byte[] fam1 = Bytes.toBytes("fam1");
6471 byte[] qf1 = Bytes.toBytes("qualifier");
6472 byte[] val1 = Bytes.toBytes("value1");
6473
6474 RowMutations rm = new RowMutations(row1);
6475 Put put = new Put(row1);
6476 put.addColumn(fam1, qf1, val1);
6477 rm.add(put);
6478
6479 this.region = initHRegion(tableName, method, CONF, fam1);
6480 long wrcBeforeMutate = this.region.writeRequestsCount.get();
6481 this.region.mutateRow(rm);
6482 long wrcAfterMutate = this.region.writeRequestsCount.get();
6483 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate);
6484 }
6485
6486 @Test
6487 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception {
6488 byte[] cf1 = Bytes.toBytes("CF1");
6489 byte[][] families = { cf1 };
6490 byte[] col = Bytes.toBytes("C");
6491 String method = this.getName();
6492 HBaseConfiguration conf = new HBaseConfiguration();
6493 this.region = initHRegion(tableName, method, conf, families);
6494
6495 Put put = new Put(Bytes.toBytes("19996"));
6496 put.addColumn(cf1, col, Bytes.toBytes("val"));
6497 region.put(put);
6498 Put put2 = new Put(Bytes.toBytes("19995"));
6499 put2.addColumn(cf1, col, Bytes.toBytes("val"));
6500 region.put(put2);
6501
6502
6503 Scan scan = new Scan(Bytes.toBytes("19996"));
6504 scan.setReversed(true);
6505 RegionScanner scanner = region.getScanner(scan);
6506
6507
6508 region.flushcache(true, true);
6509
6510
6511
6512 for (int i = 10000; i < 20000; i++) {
6513 Put p = new Put(Bytes.toBytes("" + i));
6514 p.addColumn(cf1, col, Bytes.toBytes("" + i));
6515 region.put(p);
6516 }
6517
6518 List<Cell> currRow = new ArrayList<>();
6519 boolean hasNext;
6520 boolean assertDone = false;
6521 do {
6522 hasNext = scanner.next(currRow);
6523
6524
6525 if (!assertDone) {
6526 StoreScanner current =
6527 (StoreScanner) (((RegionScannerImpl) scanner).storeHeap).getCurrentForTesting();
6528 List<KeyValueScanner> scanners = current.getAllScannersForTesting();
6529 assertEquals("There should be only one scanner the store file scanner", 1,
6530 scanners.size());
6531 assertDone = true;
6532 }
6533 } while (hasNext);
6534
6535 assertEquals(2, currRow.size());
6536 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(),
6537 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6538 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(),
6539 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6540 }
6541
6542 @Test
6543 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception {
6544 byte[] cf1 = Bytes.toBytes("CF1");
6545 byte[][] families = { cf1 };
6546 byte[] col = Bytes.toBytes("C");
6547
6548 HBaseConfiguration conf = new HBaseConfiguration();
6549 this.region = initHRegion(tableName, method, conf, families);
6550
6551 Put put = new Put(Bytes.toBytes("199996"));
6552 put.addColumn(cf1, col, Bytes.toBytes("val"));
6553 region.put(put);
6554 Put put2 = new Put(Bytes.toBytes("199995"));
6555 put2.addColumn(cf1, col, Bytes.toBytes("val"));
6556 region.put(put2);
6557
6558
6559 Scan scan = new Scan(Bytes.toBytes("199996"));
6560 scan.setReversed(true);
6561 RegionScanner scanner = region.getScanner(scan);
6562
6563
6564 for (int i = 100000; i < 200000; i++) {
6565 Put p = new Put(Bytes.toBytes("" + i));
6566 p.addColumn(cf1, col, Bytes.toBytes("" + i));
6567 region.put(p);
6568 }
6569 List<Cell> currRow = new ArrayList<>();
6570 boolean hasNext;
6571 do {
6572 hasNext = scanner.next(currRow);
6573 } while (hasNext);
6574
6575 assertEquals(2, currRow.size());
6576 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(),
6577 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6578 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(),
6579 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6580 }
6581 }