1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21
22 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
23 import static org.apache.hadoop.hbase.HBaseTestingUtility.FIRST_CHAR;
24 import static org.apache.hadoop.hbase.HBaseTestingUtility.LAST_CHAR;
25 import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
26 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
27 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
28 import static org.apache.hadoop.hbase.HBaseTestingUtility.fam3;
29 import static org.junit.Assert.assertArrayEquals;
30 import static org.junit.Assert.assertEquals;
31 import static org.junit.Assert.assertFalse;
32 import static org.junit.Assert.assertNotNull;
33 import static org.junit.Assert.assertNull;
34 import static org.junit.Assert.assertTrue;
35 import static org.junit.Assert.fail;
36 import static org.mockito.Matchers.any;
37 import static org.mockito.Matchers.anyBoolean;
38 import static org.mockito.Matchers.anyLong;
39 import static org.mockito.Mockito.doThrow;
40 import static org.mockito.Mockito.mock;
41 import static org.mockito.Mockito.never;
42 import static org.mockito.Mockito.spy;
43 import static org.mockito.Mockito.times;
44 import static org.mockito.Mockito.verify;
45 import static org.mockito.Mockito.when;
46
47 import java.io.IOException;
48 import java.io.InterruptedIOException;
49 import java.security.PrivilegedExceptionAction;
50 import java.util.ArrayList;
51 import java.util.Arrays;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.HashMap;
55 import java.util.List;
56 import java.util.Map;
57 import java.util.NavigableMap;
58 import java.util.TreeMap;
59 import java.util.UUID;
60 import java.util.concurrent.Callable;
61 import java.util.concurrent.CountDownLatch;
62 import java.util.concurrent.ExecutorService;
63 import java.util.concurrent.Executors;
64 import java.util.concurrent.Future;
65 import java.util.concurrent.TimeUnit;
66 import java.util.concurrent.atomic.AtomicBoolean;
67 import java.util.concurrent.atomic.AtomicInteger;
68 import java.util.concurrent.atomic.AtomicReference;
69
70 import org.apache.commons.lang.RandomStringUtils;
71 import org.apache.commons.logging.Log;
72 import org.apache.commons.logging.LogFactory;
73 import org.apache.hadoop.conf.Configuration;
74 import org.apache.hadoop.fs.FSDataOutputStream;
75 import org.apache.hadoop.fs.FileStatus;
76 import org.apache.hadoop.fs.FileSystem;
77 import org.apache.hadoop.fs.Path;
78 import org.apache.hadoop.hbase.Cell;
79 import org.apache.hadoop.hbase.CellComparator;
80 import org.apache.hadoop.hbase.CellUtil;
81 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
82 import org.apache.hadoop.hbase.DroppedSnapshotException;
83 import org.apache.hadoop.hbase.HBaseConfiguration;
84 import org.apache.hadoop.hbase.HBaseTestCase;
85 import org.apache.hadoop.hbase.HBaseTestingUtility;
86 import org.apache.hadoop.hbase.HColumnDescriptor;
87 import org.apache.hadoop.hbase.HConstants;
88 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
89 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
90 import org.apache.hadoop.hbase.HRegionInfo;
91 import org.apache.hadoop.hbase.HTableDescriptor;
92 import org.apache.hadoop.hbase.KeyValue;
93 import org.apache.hadoop.hbase.MiniHBaseCluster;
94 import org.apache.hadoop.hbase.MultithreadedTestUtil;
95 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
96 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
97 import org.apache.hadoop.hbase.NotServingRegionException;
98 import org.apache.hadoop.hbase.RegionTooBusyException;
99 import org.apache.hadoop.hbase.ServerName;
100 import org.apache.hadoop.hbase.TableName;
101 import org.apache.hadoop.hbase.Tag;
102 import org.apache.hadoop.hbase.TagType;
103 import org.apache.hadoop.hbase.Waiter;
104 import org.apache.hadoop.hbase.client.Append;
105 import org.apache.hadoop.hbase.client.Delete;
106 import org.apache.hadoop.hbase.client.Durability;
107 import org.apache.hadoop.hbase.client.Get;
108 import org.apache.hadoop.hbase.client.Increment;
109 import org.apache.hadoop.hbase.client.Mutation;
110 import org.apache.hadoop.hbase.client.Put;
111 import org.apache.hadoop.hbase.client.Result;
112 import org.apache.hadoop.hbase.client.RowMutations;
113 import org.apache.hadoop.hbase.client.Scan;
114 import org.apache.hadoop.hbase.client.Table;
115 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
116 import org.apache.hadoop.hbase.filter.BinaryComparator;
117 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
118 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
119 import org.apache.hadoop.hbase.filter.Filter;
120 import org.apache.hadoop.hbase.filter.FilterBase;
121 import org.apache.hadoop.hbase.filter.FilterList;
122 import org.apache.hadoop.hbase.filter.NullComparator;
123 import org.apache.hadoop.hbase.filter.PrefixFilter;
124 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
125 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
126 import org.apache.hadoop.hbase.io.hfile.HFile;
127 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
128 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
129 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
130 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
136 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
137 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
138 import org.apache.hadoop.hbase.regionserver.Region.RowLock;
139 import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem;
140 import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler;
141 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
142 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
143 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
144 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
145 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
146 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
147 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
148 import org.apache.hadoop.hbase.security.User;
149 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
150 import org.apache.hadoop.hbase.testclassification.MediumTests;
151 import org.apache.hadoop.hbase.util.Bytes;
152 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
153 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
154 import org.apache.hadoop.hbase.util.FSUtils;
155 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
156 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
157 import org.apache.hadoop.hbase.util.PairOfSameType;
158 import org.apache.hadoop.hbase.util.Threads;
159 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
160 import org.apache.hadoop.hbase.wal.FaultyFSLog;
161 import org.apache.hadoop.hbase.wal.WAL;
162 import org.apache.hadoop.hbase.wal.WALFactory;
163 import org.apache.hadoop.hbase.wal.WALKey;
164 import org.apache.hadoop.hbase.wal.WALProvider;
165 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
166 import org.apache.hadoop.hbase.wal.WALSplitter;
167 import org.junit.After;
168 import org.junit.Assert;
169 import org.junit.Before;
170 import org.junit.Rule;
171 import org.junit.Test;
172 import org.junit.experimental.categories.Category;
173 import org.junit.rules.TestName;
174 import org.mockito.ArgumentCaptor;
175 import org.mockito.ArgumentMatcher;
176 import org.mockito.Mockito;
177 import org.mockito.invocation.InvocationOnMock;
178 import org.mockito.stubbing.Answer;
179
180 import com.google.common.collect.ImmutableList;
181 import com.google.common.collect.Lists;
182 import com.google.common.collect.Maps;
183 import com.google.protobuf.ByteString;
184
185
186
187
188
189
190
191 @Category(MediumTests.class)
192 @SuppressWarnings("deprecation")
193 public class TestHRegion {
194
195
196 private static final Log LOG = LogFactory.getLog(TestHRegion.class);
197 @Rule public TestName name = new TestName();
198
199 private static final String COLUMN_FAMILY = "MyCF";
200 private static final byte [] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
201
202 HRegion region = null;
203
204 private static HBaseTestingUtility TEST_UTIL;
205 public static Configuration CONF ;
206 private String dir;
207 private static FileSystem FILESYSTEM;
208 private final int MAX_VERSIONS = 2;
209
210
211 protected byte[] tableName;
212 protected String method;
213 protected final byte[] qual1 = Bytes.toBytes("qual1");
214 protected final byte[] qual2 = Bytes.toBytes("qual2");
215 protected final byte[] qual3 = Bytes.toBytes("qual3");
216 protected final byte[] value1 = Bytes.toBytes("value1");
217 protected final byte[] value2 = Bytes.toBytes("value2");
218 protected final byte[] row = Bytes.toBytes("rowA");
219 protected final byte[] row2 = Bytes.toBytes("rowB");
220
221 protected final MetricsAssertHelper metricsAssertHelper = CompatibilitySingletonFactory
222 .getInstance(MetricsAssertHelper.class);
223
224 @Before
225 public void setup() throws IOException {
226 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
227 FILESYSTEM = TEST_UTIL.getTestFileSystem();
228 CONF = TEST_UTIL.getConfiguration();
229 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
230 method = name.getMethodName();
231 tableName = Bytes.toBytes(name.getMethodName());
232 }
233
234 @After
235 public void tearDown() throws Exception {
236
237 HBaseTestingUtility.closeRegionAndWAL(region);
238 EnvironmentEdgeManagerTestHelper.reset();
239 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
240 TEST_UTIL.cleanupTestDir();
241 }
242
243 String getName() {
244 return name.getMethodName();
245 }
246
247
248
249
250
251 @Test (timeout = 100000)
252 public void testSequenceId() throws IOException {
253 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
254 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
255
256 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
257 HBaseTestingUtility.closeRegionAndWAL(this.region);
258 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
259 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
260
261 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
262 byte [] value = Bytes.toBytes(name.getMethodName());
263
264 Put put = new Put(value);
265 put.addColumn(COLUMN_FAMILY_BYTES, null, value);
266 region.put(put);
267
268 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
269 assertEquals(0, (long)region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
270 region.flush(true);
271 long max = region.getMaxFlushedSeqId();
272 HBaseTestingUtility.closeRegionAndWAL(this.region);
273 assertEquals(max, region.getMaxFlushedSeqId());
274 this.region = null;
275 }
276
277
278
279
280
281
282
283
284
285
286
287 @Test (timeout=60000)
288 public void testCloseCarryingSnapshot() throws IOException {
289 region = initHRegion(tableName, name.getMethodName(), CONF, COLUMN_FAMILY_BYTES);
290 Store store = region.getStore(COLUMN_FAMILY_BYTES);
291
292 byte [] value = Bytes.toBytes(name.getMethodName());
293
294 Put put = new Put(value);
295 put.add(COLUMN_FAMILY_BYTES, null, value);
296
297 region.put(put);
298 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
299 storeFlushCtx.prepare();
300
301 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
302 region.put(put);
303
304 HBaseTestingUtility.closeRegionAndWAL(region);
305 assertEquals(0, region.getMemstoreSize());
306 region = null;
307 }
308
309
310
311
312
313 @Test (timeout=60000)
314 public void testMemstoreSnapshotSize() throws IOException {
315 class MyFaultyFSLog extends FaultyFSLog {
316 StoreFlushContext storeFlushCtx;
317 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
318 throws IOException {
319 super(fs, rootDir, logName, conf);
320 }
321
322 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
323 this.storeFlushCtx = storeFlushCtx;
324 }
325
326 @Override
327 public void sync(long txid) throws IOException {
328 storeFlushCtx.prepare();
329 super.sync(txid);
330 }
331 }
332
333 FileSystem fs = FileSystem.get(CONF);
334 Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
335 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
336 region = initHRegion(tableName, null, null, name.getMethodName(),
337 CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES);
338
339 Store store = region.getStore(COLUMN_FAMILY_BYTES);
340
341 byte [] value = Bytes.toBytes(name.getMethodName());
342 faultyLog.setStoreFlushCtx(store.createFlushContext(12345));
343
344 Put put = new Put(value);
345 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
346 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
347
348 boolean threwIOE = false;
349 try {
350 region.put(put);
351 } catch (IOException ioe) {
352 threwIOE = true;
353 } finally {
354 assertTrue("The regionserver should have thrown an exception", threwIOE);
355 }
356 long sz = store.getFlushableSize();
357 assertTrue("flushable size should be zero, but it is " + sz, sz == 0);
358 }
359
360
361
362
363 @Test
364 public void testMemstoreSizeWithFlushCanceling() throws IOException {
365 FileSystem fs = FileSystem.get(CONF);
366 Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
367 FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
368 region = initHRegion(tableName, null, null, name.getMethodName(),
369 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
370 Store store = region.getStore(COLUMN_FAMILY_BYTES);
371 assertEquals(0, region.getMemstoreSize());
372
373
374 byte [] value = Bytes.toBytes(name.getMethodName());
375 Put put = new Put(value);
376 put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
377 region.put(put);
378 long onePutSize = region.getMemstoreSize();
379 assertTrue(onePutSize > 0);
380 region.flush(true);
381 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
382 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
383
384
385 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
386 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
387 when(mockedCPHost.preFlush(Mockito.isA(HStore.class), Mockito.isA(InternalScanner.class))).
388 thenReturn(null);
389 region.setCoprocessorHost(mockedCPHost);
390 region.put(put);
391 region.flush(true);
392 assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
393 assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());
394
395
396 region.setCoprocessorHost(normalCPHost);
397 region.flush(true);
398 assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
399 assertEquals("flushable size should be zero", 0, store.getFlushableSize());
400 }
401
402 @Test
403 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
404 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
405 FileSystem fs = FileSystem.get(CONF);
406 Path rootDir = new Path(dir + testName);
407 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
408 region = initHRegion(tableName, null, null, name.getMethodName(),
409 CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
410 Store store = region.getStore(COLUMN_FAMILY_BYTES);
411 assertEquals(0, region.getMemstoreSize());
412
413
414 byte [] value = Bytes.toBytes(name.getMethodName());
415 Put put = new Put(value);
416 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
417 region.put(put);
418 long onePutSize = region.getMemstoreSize();
419 assertTrue(onePutSize > 0);
420
421 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
422 doThrow(new IOException())
423 .when(mockedCPHost).postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>>any());
424 region.setCoprocessorHost(mockedCPHost);
425
426 put = new Put(value);
427 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
428 try {
429 region.put(put);
430 fail("Should have failed with IOException");
431 } catch (IOException expected) {
432 }
433 assertEquals("memstoreSize should be incremented", onePutSize * 2, region.getMemstoreSize());
434 assertEquals("flushable size should be incremented", onePutSize * 2, store.getFlushableSize());
435
436 region.setCoprocessorHost(null);
437 }
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454 @Test (timeout=60000)
455 public void testFlushSizeAccounting() throws Exception {
456 final Configuration conf = HBaseConfiguration.create(CONF);
457
458 conf.setInt("hbase.hstore.flush.retries.number", 1);
459 final User user =
460 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
461
462 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
463 user.runAs(new PrivilegedExceptionAction<Object>() {
464 @Override
465 public Object run() throws Exception {
466
467 FileSystem fs = FileSystem.get(conf);
468 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
469 FaultyFileSystem ffs = (FaultyFileSystem)fs;
470 HRegion region = null;
471 try {
472
473 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
474 long size = region.getMemstoreSize();
475 Assert.assertEquals(0, size);
476
477 Put p1 = new Put(row);
478 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
479 region.put(p1);
480 final long sizeOfOnePut = region.getMemstoreSize();
481
482 try {
483 LOG.info("Flushing");
484 region.flush(true);
485 Assert.fail("Didn't bubble up IOE!");
486 } catch (DroppedSnapshotException dse) {
487
488 region.closing.set(false);
489 } catch (Exception e) {
490
491 region.closing.set(false);
492 }
493
494 ffs.fault.set(false);
495
496 try {
497 region.getWAL().rollWriter(true);
498 } catch (Exception e) {
499 int x = 0;
500 }
501
502 Assert.assertEquals(sizeOfOnePut, region.getMemstoreSize());
503
504
505 Put p2 = new Put(row);
506 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
507 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
508 region.put(p2);
509 Assert.assertEquals(sizeOfOnePut * 3, region.getMemstoreSize());
510
511
512
513 region.flush(true);
514
515 Assert.assertEquals(sizeOfOnePut * 2, region.getMemstoreSize());
516 } catch (Exception e) {
517 int x = 0;
518 } finally {
519 HRegion.closeHRegion(region);
520 }
521 return null;
522 }
523 });
524 FileSystem.closeAllForUGI(user.getUGI());
525 }
526
527 @Test (timeout=60000)
528 public void testCloseWithFailingFlush() throws Exception {
529 final Configuration conf = HBaseConfiguration.create(CONF);
530
531 conf.setInt("hbase.hstore.flush.retries.number", 1);
532 final User user =
533 User.createUserForTesting(conf, this.name.getMethodName(), new String[]{"foo"});
534
535 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
536 user.runAs(new PrivilegedExceptionAction<Object>() {
537 @Override
538 public Object run() throws Exception {
539
540 FileSystem fs = FileSystem.get(conf);
541 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
542 FaultyFileSystem ffs = (FaultyFileSystem)fs;
543 HRegion region = null;
544 try {
545
546 region = initHRegion(tableName, name.getMethodName(), conf, COLUMN_FAMILY_BYTES);
547 long size = region.getMemstoreSize();
548 Assert.assertEquals(0, size);
549
550 Put p1 = new Put(row);
551 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[])null));
552 region.put(p1);
553
554 Store store = region.getStore(COLUMN_FAMILY_BYTES);
555 StoreFlushContext storeFlushCtx = store.createFlushContext(12345);
556 storeFlushCtx.prepare();
557
558 Put p2 = new Put(row);
559 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[])null));
560 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[])null));
561 region.put(p2);
562
563 HBaseTestingUtility.closeRegionAndWAL(region);
564 region = null;
565 fail();
566 } catch (IOException dse) {
567
568 LOG.info("Expected DroppedSnapshotException");
569 } finally {
570
571 ffs.fault.set(false);
572 region.getWAL().rollWriter(true);
573 HRegion.closeHRegion(region);
574 }
575 return null;
576 }
577 });
578 FileSystem.closeAllForUGI(user.getUGI());
579 }
580
581 @Test
582 public void testCompactionAffectedByScanners() throws Exception {
583 byte[] family = Bytes.toBytes("family");
584 this.region = initHRegion(tableName, method, CONF, family);
585
586 Put put = new Put(Bytes.toBytes("r1"));
587 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
588 region.put(put);
589 region.flush(true);
590
591 Scan scan = new Scan();
592 scan.setMaxVersions(3);
593
594 RegionScanner scanner1 = region.getScanner(scan);
595
596 Delete delete = new Delete(Bytes.toBytes("r1"));
597 region.delete(delete);
598 region.flush(true);
599
600
601 RegionScanner scanner2 = region.getScanner(scan);
602
603 List<Cell> results = new ArrayList<Cell>();
604
605 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
606
607
608 region.compact(true);
609
610
611 RegionScanner scanner3 = region.getScanner(scan);
612
613
614 scanner1.next(results);
615 System.out.println(results);
616 assertEquals(1, results.size());
617
618 results.clear();
619 scanner2.next(results);
620 System.out.println(results);
621 assertEquals(0, results.size());
622
623 results.clear();
624 scanner3.next(results);
625 System.out.println(results);
626 assertEquals(0, results.size());
627 }
628
629 @Test
630 public void testToShowNPEOnRegionScannerReseek() throws Exception {
631 byte[] family = Bytes.toBytes("family");
632 this.region = initHRegion(tableName, method, CONF, family);
633
634 Put put = new Put(Bytes.toBytes("r1"));
635 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
636 region.put(put);
637 put = new Put(Bytes.toBytes("r2"));
638 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
639 region.put(put);
640 region.flush(true);
641
642 Scan scan = new Scan();
643 scan.setMaxVersions(3);
644
645 RegionScanner scanner1 = region.getScanner(scan);
646
647 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
648
649 region.compact(true);
650
651 scanner1.reseek(Bytes.toBytes("r2"));
652 List<Cell> results = new ArrayList<Cell>();
653 scanner1.next(results);
654 Cell keyValue = results.get(0);
655 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
656 scanner1.close();
657 }
658
659 @Test
660 public void testSkipRecoveredEditsReplay() throws Exception {
661 String method = "testSkipRecoveredEditsReplay";
662 TableName tableName = TableName.valueOf(method);
663 byte[] family = Bytes.toBytes("family");
664 this.region = initHRegion(tableName, method, CONF, family);
665 final WALFactory wals = new WALFactory(CONF, null, method);
666 try {
667 Path regiondir = region.getRegionFileSystem().getRegionDir();
668 FileSystem fs = region.getRegionFileSystem().getFileSystem();
669 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
670
671 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
672
673 long maxSeqId = 1050;
674 long minSeqId = 1000;
675
676 for (long i = minSeqId; i <= maxSeqId; i += 10) {
677 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
678 fs.create(recoveredEdits);
679 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
680
681 long time = System.nanoTime();
682 WALEdit edit = new WALEdit();
683 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
684 .toBytes(i)));
685 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
686 HConstants.DEFAULT_CLUSTER_ID), edit));
687
688 writer.close();
689 }
690 MonitoredTask status = TaskMonitor.get().createStatus(method);
691 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
692 for (Store store : region.getStores()) {
693 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
694 }
695 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
696 assertEquals(maxSeqId, seqId);
697 region.getMVCC().advanceTo(seqId);
698 Get get = new Get(row);
699 Result result = region.get(get);
700 for (long i = minSeqId; i <= maxSeqId; i += 10) {
701 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
702 assertEquals(1, kvs.size());
703 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
704 }
705 } finally {
706 HRegion.closeHRegion(this.region);
707 this.region = null;
708 wals.close();
709 }
710 }
711
712 @Test
713 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
714 String method = "testSkipRecoveredEditsReplaySomeIgnored";
715 TableName tableName = TableName.valueOf(method);
716 byte[] family = Bytes.toBytes("family");
717 this.region = initHRegion(tableName, method, CONF, family);
718 final WALFactory wals = new WALFactory(CONF, null, method);
719 try {
720 Path regiondir = region.getRegionFileSystem().getRegionDir();
721 FileSystem fs = region.getRegionFileSystem().getFileSystem();
722 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
723
724 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
725
726 long maxSeqId = 1050;
727 long minSeqId = 1000;
728
729 for (long i = minSeqId; i <= maxSeqId; i += 10) {
730 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
731 fs.create(recoveredEdits);
732 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
733
734 long time = System.nanoTime();
735 WALEdit edit = new WALEdit();
736 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
737 .toBytes(i)));
738 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
739 HConstants.DEFAULT_CLUSTER_ID), edit));
740
741 writer.close();
742 }
743 long recoverSeqId = 1030;
744 MonitoredTask status = TaskMonitor.get().createStatus(method);
745 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
746 for (Store store : region.getStores()) {
747 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
748 }
749 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
750 assertEquals(maxSeqId, seqId);
751 region.getMVCC().advanceTo(seqId);
752 Get get = new Get(row);
753 Result result = region.get(get);
754 for (long i = minSeqId; i <= maxSeqId; i += 10) {
755 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
756 if (i < recoverSeqId) {
757 assertEquals(0, kvs.size());
758 } else {
759 assertEquals(1, kvs.size());
760 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
761 }
762 }
763 } finally {
764 HRegion.closeHRegion(this.region);
765 this.region = null;
766 wals.close();
767 }
768 }
769
770 @Test
771 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
772 byte[] family = Bytes.toBytes("family");
773 this.region = initHRegion(tableName, method, CONF, family);
774 Path regiondir = region.getRegionFileSystem().getRegionDir();
775 FileSystem fs = region.getRegionFileSystem().getFileSystem();
776
777 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
778 for (int i = 1000; i < 1050; i += 10) {
779 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
780 FSDataOutputStream dos = fs.create(recoveredEdits);
781 dos.writeInt(i);
782 dos.close();
783 }
784 long minSeqId = 2000;
785 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
786 FSDataOutputStream dos = fs.create(recoveredEdits);
787 dos.close();
788
789 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
790 for (Store store : region.getStores()) {
791 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
792 }
793 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null);
794 assertEquals(minSeqId, seqId);
795 }
796
797 @Test
798 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
799 String method = "testSkipRecoveredEditsReplayTheLastFileIgnored";
800 TableName tableName = TableName.valueOf(method);
801 byte[] family = Bytes.toBytes("family");
802 this.region = initHRegion(tableName, method, CONF, family);
803 final WALFactory wals = new WALFactory(CONF, null, method);
804 try {
805 Path regiondir = region.getRegionFileSystem().getRegionDir();
806 FileSystem fs = region.getRegionFileSystem().getFileSystem();
807 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
808 byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
809
810 assertEquals(0, region.getStoreFileList(columns).size());
811
812 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
813
814 long maxSeqId = 1050;
815 long minSeqId = 1000;
816
817 for (long i = minSeqId; i <= maxSeqId; i += 10) {
818 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
819 fs.create(recoveredEdits);
820 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
821
822 long time = System.nanoTime();
823 WALEdit edit = null;
824 if (i == maxSeqId) {
825 edit = WALEdit.createCompaction(region.getRegionInfo(),
826 CompactionDescriptor.newBuilder()
827 .setTableName(ByteString.copyFrom(tableName.getName()))
828 .setFamilyName(ByteString.copyFrom(regionName))
829 .setEncodedRegionName(ByteString.copyFrom(regionName))
830 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
831 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName()))
832 .build());
833 } else {
834 edit = new WALEdit();
835 edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
836 .toBytes(i)));
837 }
838 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, i, time,
839 HConstants.DEFAULT_CLUSTER_ID), edit));
840 writer.close();
841 }
842
843 long recoverSeqId = 1030;
844 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
845 MonitoredTask status = TaskMonitor.get().createStatus(method);
846 for (Store store : region.getStores()) {
847 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
848 }
849 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
850 assertEquals(maxSeqId, seqId);
851
852
853 assertEquals(1, region.getStoreFileList(columns).size());
854
855 } finally {
856 HRegion.closeHRegion(this.region);
857 this.region = null;
858 wals.close();
859 }
860 }
861
862 @Test
863 public void testRecoveredEditsReplayCompaction() throws Exception {
864 testRecoveredEditsReplayCompaction(false);
865 testRecoveredEditsReplayCompaction(true);
866 }
867 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
868 String method = name.getMethodName();
869 TableName tableName = TableName.valueOf(method);
870 byte[] family = Bytes.toBytes("family");
871 this.region = initHRegion(tableName, method, CONF, family);
872 final WALFactory wals = new WALFactory(CONF, null, method);
873 try {
874 Path regiondir = region.getRegionFileSystem().getRegionDir();
875 FileSystem fs = region.getRegionFileSystem().getFileSystem();
876 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
877
878 long maxSeqId = 3;
879 long minSeqId = 0;
880
881 for (long i = minSeqId; i < maxSeqId; i++) {
882 Put put = new Put(Bytes.toBytes(i));
883 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
884 region.put(put);
885 region.flush(true);
886 }
887
888
889 assertEquals(3, region.getStore(family).getStorefilesCount());
890 List<Path> storeFiles = new ArrayList<Path>(3);
891 for (StoreFile sf : region.getStore(family).getStorefiles()) {
892 storeFiles.add(sf.getPath());
893 }
894
895
896 CONF.setBoolean("hbase.hstore.compaction.complete", false);
897 region.compactStores();
898
899
900 assertEquals(3, region.getStore(family).getStorefilesCount());
901
902
903 Path tmpDir = region.getRegionFileSystem().getTempDir();
904 FileStatus[] files = FSUtils.listStatus(fs, tmpDir);
905 String errorMsg = "Expected to find 1 file in the region temp directory "
906 + "from the compaction, could not find any";
907 assertNotNull(errorMsg, files);
908 assertEquals(errorMsg, 1, files.length);
909
910 Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
911 files[0].getPath());
912
913 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
914 byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
915 for (int i=0; i < encodedNameAsBytes.length; i++) {
916
917 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
918 }
919
920 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
921 .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
922 storeFiles, Lists.newArrayList(newFile),
923 region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
924
925 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
926 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
927
928 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
929
930 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
931 fs.create(recoveredEdits);
932 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
933
934 long time = System.nanoTime();
935
936 writer.append(new WAL.Entry(new HLogKey(regionName, tableName, 10, time,
937 HConstants.DEFAULT_CLUSTER_ID), WALEdit.createCompaction(region.getRegionInfo(),
938 compactionDescriptor)));
939 writer.close();
940
941
942 region.getTableDesc();
943 region.getRegionInfo();
944 HBaseTestingUtility.closeRegionAndWAL(this.region);
945 try {
946 region = HRegion.openHRegion(region, null);
947 } catch (WrongRegionException wre) {
948 fail("Matching encoded region name should not have produced WrongRegionException");
949 }
950
951
952 Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
953 for (StoreFile sf : sfs) {
954 LOG.info(sf.getPath());
955 }
956 if (!mismatchedRegionName) {
957 assertEquals(1, region.getStore(family).getStorefilesCount());
958 }
959 files = FSUtils.listStatus(fs, tmpDir);
960 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
961
962 for (long i = minSeqId; i < maxSeqId; i++) {
963 Get get = new Get(Bytes.toBytes(i));
964 Result result = region.get(get);
965 byte[] value = result.getValue(family, Bytes.toBytes(i));
966 assertArrayEquals(Bytes.toBytes(i), value);
967 }
968 } finally {
969 HRegion.closeHRegion(this.region);
970 this.region = null;
971 wals.close();
972 }
973 }
974
975 @Test
976 public void testFlushMarkers() throws Exception {
977
978 String method = name.getMethodName();
979 TableName tableName = TableName.valueOf(method);
980 byte[] family = Bytes.toBytes("family");
981 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
982 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
983 FSUtils.setRootDir(walConf, logDir);
984 final WALFactory wals = new WALFactory(walConf, null, method);
985 final WAL wal = wals.getWAL(tableName.getName(), tableName.getNamespace());
986
987 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
988 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
989 try {
990 Path regiondir = region.getRegionFileSystem().getRegionDir();
991 FileSystem fs = region.getRegionFileSystem().getFileSystem();
992 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
993
994 long maxSeqId = 3;
995 long minSeqId = 0;
996
997 for (long i = minSeqId; i < maxSeqId; i++) {
998 Put put = new Put(Bytes.toBytes(i));
999 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1000 region.put(put);
1001 region.flush(true);
1002 }
1003
1004
1005 assertEquals(3, region.getStore(family).getStorefilesCount());
1006 List<String> storeFiles = new ArrayList<String>(3);
1007 for (StoreFile sf : region.getStore(family).getStorefiles()) {
1008 storeFiles.add(sf.getPath().getName());
1009 }
1010
1011
1012 wal.shutdown();
1013 WAL.Reader reader = WALFactory.createReader(fs, DefaultWALProvider.getCurrentFileName(wal),
1014 TEST_UTIL.getConfiguration());
1015 try {
1016 List<WAL.Entry> flushDescriptors = new ArrayList<WAL.Entry>();
1017 long lastFlushSeqId = -1;
1018 while (true) {
1019 WAL.Entry entry = reader.next();
1020 if (entry == null) {
1021 break;
1022 }
1023 Cell cell = entry.getEdit().getCells().get(0);
1024 if (WALEdit.isMetaEditFamily(cell)) {
1025 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
1026 assertNotNull(flushDesc);
1027 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
1028 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1029 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
1030 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1031 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
1032 }
1033 lastFlushSeqId = flushDesc.getFlushSequenceNumber();
1034 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
1035 assertEquals(1, flushDesc.getStoreFlushesCount());
1036 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
1037 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
1038 assertEquals("family", storeFlushDesc.getStoreHomeDir());
1039 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1040 assertEquals(0, storeFlushDesc.getFlushOutputCount());
1041 } else {
1042 assertEquals(1, storeFlushDesc.getFlushOutputCount());
1043 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
1044 }
1045
1046 flushDescriptors.add(entry);
1047 }
1048 }
1049
1050 assertEquals(3 * 2, flushDescriptors.size());
1051
1052
1053
1054 Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
1055
1056 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1057 fs.create(recoveredEdits);
1058 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1059
1060 for (WAL.Entry entry : flushDescriptors) {
1061 writer.append(entry);
1062 }
1063 writer.close();
1064 } finally {
1065 if (null != reader) {
1066 try {
1067 reader.close();
1068 } catch (IOException exception) {
1069 LOG.warn("Problem closing wal: " + exception.getMessage());
1070 LOG.debug("exception details", exception);
1071 }
1072 }
1073 }
1074
1075
1076 HBaseTestingUtility.closeRegionAndWAL(this.region);
1077 region = HRegion.openHRegion(region, null);
1078
1079
1080 for (long i = minSeqId; i < maxSeqId; i++) {
1081 Get get = new Get(Bytes.toBytes(i));
1082 Result result = region.get(get);
1083 byte[] value = result.getValue(family, Bytes.toBytes(i));
1084 assertArrayEquals(Bytes.toBytes(i), value);
1085 }
1086 } finally {
1087 HBaseTestingUtility.closeRegionAndWAL(this.region);
1088 this.region = null;
1089 wals.close();
1090 }
1091 }
1092
1093 class IsFlushWALMarker extends ArgumentMatcher<WALEdit> {
1094 volatile FlushAction[] actions;
1095 public IsFlushWALMarker(FlushAction... actions) {
1096 this.actions = actions;
1097 }
1098 @Override
1099 public boolean matches(Object edit) {
1100 List<Cell> cells = ((WALEdit)edit).getCells();
1101 if (cells.isEmpty()) {
1102 return false;
1103 }
1104 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1105 FlushDescriptor desc = null;
1106 try {
1107 desc = WALEdit.getFlushDescriptor(cells.get(0));
1108 } catch (IOException e) {
1109 LOG.warn(e);
1110 return false;
1111 }
1112 if (desc != null) {
1113 for (FlushAction action : actions) {
1114 if (desc.getAction() == action) {
1115 return true;
1116 }
1117 }
1118 }
1119 }
1120 return false;
1121 }
1122 public IsFlushWALMarker set(FlushAction... actions) {
1123 this.actions = actions;
1124 return this;
1125 }
1126 }
1127
1128 @Test (timeout=60000)
1129 public void testFlushMarkersWALFail() throws Exception {
1130
1131 String method = name.getMethodName();
1132 TableName tableName = TableName.valueOf(method);
1133 byte[] family = Bytes.toBytes("family");
1134
1135
1136 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1137
1138 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1139 FSUtils.setRootDir(walConf, logDir);
1140
1141
1142 class FailAppendFlushMarkerWAL extends FSHLog {
1143 volatile FlushAction [] flushActions = null;
1144
1145 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1146 throws IOException {
1147 super(fs, root, logDir, conf);
1148 }
1149
1150 @Override
1151 protected Writer createWriterInstance(Path path) throws IOException {
1152 final Writer w = super.createWriterInstance(path);
1153 return new Writer() {
1154 @Override
1155 public void close() throws IOException {
1156 w.close();
1157 }
1158
1159 @Override
1160 public void sync() throws IOException {
1161 w.sync();
1162 }
1163
1164 @Override
1165 public void append(Entry entry) throws IOException {
1166 List<Cell> cells = entry.getEdit().getCells();
1167 if (WALEdit.isMetaEditFamily(cells.get(0))) {
1168 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1169 if (desc != null) {
1170 for (FlushAction flushAction: flushActions) {
1171 if (desc.getAction().equals(flushAction)) {
1172 throw new IOException("Failed to append flush marker! " + flushAction);
1173 }
1174 }
1175 }
1176 }
1177 w.append(entry);
1178 }
1179
1180 @Override
1181 public long getLength() throws IOException {
1182 return w.getLength();
1183 }
1184 };
1185 }
1186 }
1187 FailAppendFlushMarkerWAL wal =
1188 new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1189 getName(), walConf);
1190 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1191 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1192 int i = 0;
1193 Put put = new Put(Bytes.toBytes(i));
1194 put.setDurability(Durability.SKIP_WAL);
1195 put.add(family, Bytes.toBytes(i), Bytes.toBytes(i));
1196 region.put(put);
1197
1198
1199 wal.flushActions = new FlushAction [] {FlushAction.START_FLUSH};
1200
1201
1202 try {
1203 region.flush(true);
1204 fail("This should have thrown exception");
1205 } catch (DroppedSnapshotException unexpected) {
1206
1207 throw unexpected;
1208 } catch (IOException expected) {
1209
1210 }
1211
1212
1213 region.close(true);
1214 wal.close();
1215
1216
1217 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH};
1218 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
1219 getName(), walConf);
1220 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
1221 HConstants.EMPTY_END_ROW, method, CONF, false, Durability.USE_DEFAULT, wal, family);
1222 region.put(put);
1223
1224
1225
1226
1227 wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH};
1228
1229 try {
1230 region.flush(true);
1231 fail("This should have thrown exception");
1232 } catch (DroppedSnapshotException expected) {
1233
1234
1235 } catch (IOException unexpected) {
1236 throw unexpected;
1237 }
1238 }
1239
1240 @Test
1241 public void testGetWhileRegionClose() throws IOException {
1242 TableName tableName = TableName.valueOf(name.getMethodName());
1243 Configuration hc = initSplit();
1244 int numRows = 100;
1245 byte[][] families = { fam1, fam2, fam3 };
1246
1247
1248 String method = name.getMethodName();
1249 this.region = initHRegion(tableName, method, hc, families);
1250
1251 final int startRow = 100;
1252 putData(startRow, numRows, qual1, families);
1253 putData(startRow, numRows, qual2, families);
1254 putData(startRow, numRows, qual3, families);
1255 final AtomicBoolean done = new AtomicBoolean(false);
1256 final AtomicInteger gets = new AtomicInteger(0);
1257 GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1258 try {
1259
1260 for (int i = 0; i < threads.length / 2; i++) {
1261 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1262 threads[i].setDaemon(true);
1263 threads[i].start();
1264 }
1265
1266
1267 this.region.closing.set(true);
1268 for (int i = threads.length / 2; i < threads.length; i++) {
1269 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1270 threads[i].setDaemon(true);
1271 threads[i].start();
1272 }
1273 } finally {
1274 if (this.region != null) {
1275 HBaseTestingUtility.closeRegionAndWAL(this.region);
1276 this.region = null;
1277 }
1278 }
1279 done.set(true);
1280 for (GetTillDoneOrException t : threads) {
1281 try {
1282 t.join();
1283 } catch (InterruptedException e) {
1284 e.printStackTrace();
1285 }
1286 if (t.e != null) {
1287 LOG.info("Exception=" + t.e);
1288 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1289 }
1290 }
1291 }
1292
1293
1294
1295
1296
1297 class GetTillDoneOrException extends Thread {
1298 private final Get g;
1299 private final AtomicBoolean done;
1300 private final AtomicInteger count;
1301 private Exception e;
1302
1303 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, final AtomicInteger c) {
1304 super("getter." + i);
1305 this.g = new Get(r);
1306 this.done = d;
1307 this.count = c;
1308 }
1309
1310 @Override
1311 public void run() {
1312 while (!this.done.get()) {
1313 try {
1314 assertTrue(region.get(g).size() > 0);
1315 this.count.incrementAndGet();
1316 } catch (Exception e) {
1317 this.e = e;
1318 break;
1319 }
1320 }
1321 }
1322 }
1323
1324
1325
1326
1327 @Test
1328 public void testWeirdCacheBehaviour() throws Exception {
1329 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
1330 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1331 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1332 this.region = initHRegion(TABLE, getName(), CONF, FAMILIES);
1333 String value = "this is the value";
1334 String value2 = "this is some other value";
1335 String keyPrefix1 = "prefix1";
1336 String keyPrefix2 = "prefix2";
1337 String keyPrefix3 = "prefix3";
1338 putRows(this.region, 3, value, keyPrefix1);
1339 putRows(this.region, 3, value, keyPrefix2);
1340 putRows(this.region, 3, value, keyPrefix3);
1341 putRows(this.region, 3, value2, keyPrefix1);
1342 putRows(this.region, 3, value2, keyPrefix2);
1343 putRows(this.region, 3, value2, keyPrefix3);
1344 System.out.println("Checking values for key: " + keyPrefix1);
1345 assertEquals("Got back incorrect number of rows from scan", 3,
1346 getNumberOfRows(keyPrefix1, value2, this.region));
1347 System.out.println("Checking values for key: " + keyPrefix2);
1348 assertEquals("Got back incorrect number of rows from scan", 3,
1349 getNumberOfRows(keyPrefix2, value2, this.region));
1350 System.out.println("Checking values for key: " + keyPrefix3);
1351 assertEquals("Got back incorrect number of rows from scan", 3,
1352 getNumberOfRows(keyPrefix3, value2, this.region));
1353 deleteColumns(this.region, value2, keyPrefix1);
1354 deleteColumns(this.region, value2, keyPrefix2);
1355 deleteColumns(this.region, value2, keyPrefix3);
1356 System.out.println("Starting important checks.....");
1357 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1358 getNumberOfRows(keyPrefix1, value2, this.region));
1359 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1360 getNumberOfRows(keyPrefix2, value2, this.region));
1361 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1362 getNumberOfRows(keyPrefix3, value2, this.region));
1363 }
1364
1365 @Test
1366 public void testAppendWithReadOnlyTable() throws Exception {
1367 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1368 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1369 boolean exceptionCaught = false;
1370 Append append = new Append(Bytes.toBytes("somerow"));
1371 append.setDurability(Durability.SKIP_WAL);
1372 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1373 Bytes.toBytes("somevalue"));
1374 try {
1375 region.append(append);
1376 } catch (IOException e) {
1377 exceptionCaught = true;
1378 }
1379 assertTrue(exceptionCaught);
1380 }
1381
1382 @Test
1383 public void testIncrWithReadOnlyTable() throws Exception {
1384 byte[] TABLE = Bytes.toBytes("readOnlyTable");
1385 this.region = initHRegion(TABLE, getName(), CONF, true, Bytes.toBytes("somefamily"));
1386 boolean exceptionCaught = false;
1387 Increment inc = new Increment(Bytes.toBytes("somerow"));
1388 inc.setDurability(Durability.SKIP_WAL);
1389 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1390 try {
1391 region.increment(inc);
1392 } catch (IOException e) {
1393 exceptionCaught = true;
1394 }
1395 assertTrue(exceptionCaught);
1396 }
1397
1398 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1399 InternalScanner scanner = buildScanner(keyPrefix, value, r);
1400 int count = 0;
1401 boolean more = false;
1402 List<Cell> results = new ArrayList<Cell>();
1403 do {
1404 more = scanner.next(results);
1405 if (results != null && !results.isEmpty())
1406 count++;
1407 else
1408 break;
1409 Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1410 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1411 r.delete(delete);
1412 results.clear();
1413 } while (more);
1414 assertEquals("Did not perform correct number of deletes", 3, count);
1415 }
1416
1417 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1418 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1419 int numberOfResults = 0;
1420 List<Cell> results = new ArrayList<Cell>();
1421 boolean more = false;
1422 do {
1423 more = resultScanner.next(results);
1424 if (results != null && !results.isEmpty())
1425 numberOfResults++;
1426 else
1427 break;
1428 for (Cell kv : results) {
1429 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1430 }
1431 results.clear();
1432 } while (more);
1433 return numberOfResults;
1434 }
1435
1436 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1437 throws IOException {
1438
1439 FilterList allFilters = new FilterList();
1440 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1441
1442 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1443 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
1444 filter.setFilterIfMissing(true);
1445 allFilters.addFilter(filter);
1446 Scan scan = new Scan();
1447 scan.addFamily(Bytes.toBytes("trans-blob"));
1448 scan.addFamily(Bytes.toBytes("trans-type"));
1449 scan.addFamily(Bytes.toBytes("trans-date"));
1450 scan.addFamily(Bytes.toBytes("trans-tags"));
1451 scan.addFamily(Bytes.toBytes("trans-group"));
1452 scan.setFilter(allFilters);
1453 return r.getScanner(scan);
1454 }
1455
1456 private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1457 for (int i = 0; i < numRows; i++) {
1458 String row = key + "_" + i
1459 System.out.println(String.format("Saving row: %s, with value %s", row, value));
1460 Put put = new Put(Bytes.toBytes(row));
1461 put.setDurability(Durability.SKIP_WAL);
1462 put.add(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1463 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1464 put.add(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1465 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1466 put.add(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1467 r.put(put);
1468 }
1469 }
1470
1471 @Test
1472 public void testFamilyWithAndWithoutColon() throws Exception {
1473 byte[] b = Bytes.toBytes(getName());
1474 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1475 this.region = initHRegion(b, getName(), CONF, cf);
1476 Put p = new Put(b);
1477 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1478 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
1479 boolean exception = false;
1480 try {
1481 this.region.put(p);
1482 } catch (NoSuchColumnFamilyException e) {
1483 exception = true;
1484 }
1485 assertTrue(exception);
1486 }
1487
1488 @Test
1489 public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1490 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1491 byte[] qual = Bytes.toBytes("qual");
1492 byte[] val = Bytes.toBytes("val");
1493 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1494 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1495 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1496 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1497
1498 LOG.info("First a batch put with all valid puts");
1499 final Put[] puts = new Put[10];
1500 for (int i = 0; i < 10; i++) {
1501 puts[i] = new Put(Bytes.toBytes("row_" + i));
1502 puts[i].add(cf, qual, val);
1503 }
1504
1505 OperationStatus[] codes = this.region.batchMutate(puts);
1506 assertEquals(10, codes.length);
1507 for (int i = 0; i < 10; i++) {
1508 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1509 }
1510 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1511
1512 LOG.info("Next a batch put with one invalid family");
1513 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1514 codes = this.region.batchMutate(puts);
1515 assertEquals(10, codes.length);
1516 for (int i = 0; i < 10; i++) {
1517 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1518 codes[i].getOperationStatusCode());
1519 }
1520
1521 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1522 }
1523
1524 @Test
1525 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1526 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1527 byte[] qual = Bytes.toBytes("qual");
1528 byte[] val = Bytes.toBytes("val");
1529 this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
1530 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1531 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1532 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1533
1534 final Put[] puts = new Put[10];
1535 for (int i = 0; i < 10; i++) {
1536 puts[i] = new Put(Bytes.toBytes("row_" + i));
1537 puts[i].add(cf, qual, val);
1538 }
1539 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
1540
1541 LOG.info("batchPut will have to break into four batches to avoid row locks");
1542 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1543 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
1544 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
1545 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
1546
1547
1548 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1549 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
1550 final CountDownLatch startingPuts = new CountDownLatch(1);
1551 final CountDownLatch startingClose = new CountDownLatch(1);
1552 TestThread putter = new TestThread(ctx) {
1553 @Override
1554 public void doWork() throws IOException {
1555 startingPuts.countDown();
1556 retFromThread.set(region.batchMutate(puts));
1557 }
1558 };
1559 LOG.info("...starting put thread while holding locks");
1560 ctx.addThread(putter);
1561 ctx.startThreads();
1562
1563
1564
1565
1566
1567
1568 Thread regionCloseThread = new TestThread(ctx) {
1569 @Override
1570 public void doWork() {
1571 try {
1572 startingPuts.await();
1573
1574
1575 Thread.sleep(10);
1576 startingClose.countDown();
1577 HBaseTestingUtility.closeRegionAndWAL(region);
1578 region = null;
1579 } catch (IOException e) {
1580 throw new RuntimeException(e);
1581 } catch (InterruptedException e) {
1582 throw new RuntimeException(e);
1583 }
1584 }
1585 };
1586 regionCloseThread.start();
1587
1588 startingClose.await();
1589 startingPuts.await();
1590 Thread.sleep(100);
1591 LOG.info("...releasing row lock 1, which should let put thread continue");
1592 rowLock1.release();
1593 rowLock2.release();
1594 rowLock3.release();
1595 waitForCounter(source, "syncTimeNumOps", syncs + 1);
1596
1597 LOG.info("...joining on put thread");
1598 ctx.stop();
1599 regionCloseThread.join();
1600
1601 OperationStatus[] codes = retFromThread.get();
1602 for (int i = 0; i < codes.length; i++) {
1603 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1604 codes[i].getOperationStatusCode());
1605 }
1606 rowLock4.release();
1607 }
1608
1609 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1610 throws InterruptedException {
1611 long startWait = System.currentTimeMillis();
1612 long currentCount;
1613 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1614 Thread.sleep(100);
1615 if (System.currentTimeMillis() - startWait > 10000) {
1616 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1617 expectedCount, currentCount));
1618 }
1619 }
1620 }
1621
1622 @Test
1623 public void testBatchPutWithTsSlop() throws Exception {
1624 byte[] b = Bytes.toBytes(getName());
1625 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1626 byte[] qual = Bytes.toBytes("qual");
1627 byte[] val = Bytes.toBytes("val");
1628
1629
1630 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1631 this.region = initHRegion(b, getName(), CONF, cf);
1632
1633 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1634 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1635 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1636
1637 final Put[] puts = new Put[10];
1638 for (int i = 0; i < 10; i++) {
1639 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
1640 puts[i].add(cf, qual, val);
1641 }
1642
1643 OperationStatus[] codes = this.region.batchMutate(puts);
1644 assertEquals(10, codes.length);
1645 for (int i = 0; i < 10; i++) {
1646 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1647 }
1648 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1649 }
1650
1651
1652
1653
1654 @Test
1655 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1656 byte[] row1 = Bytes.toBytes("row1");
1657 byte[] fam1 = Bytes.toBytes("fam1");
1658 byte[] qf1 = Bytes.toBytes("qualifier");
1659 byte[] emptyVal = new byte[] {};
1660 byte[] val1 = Bytes.toBytes("value1");
1661 byte[] val2 = Bytes.toBytes("value2");
1662
1663
1664 String method = this.getName();
1665 this.region = initHRegion(tableName, method, CONF, fam1);
1666
1667 Put put = new Put(row1);
1668 put.add(fam1, qf1, emptyVal);
1669
1670
1671 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1672 emptyVal), put, true);
1673 assertTrue(res);
1674
1675
1676 put = new Put(row1);
1677 put.add(fam1, qf1, val1);
1678
1679
1680 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1681 put, true);
1682 assertTrue(res);
1683
1684
1685 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1686 put, true);
1687 assertFalse(res);
1688
1689 Delete delete = new Delete(row1);
1690 delete.deleteColumn(fam1, qf1);
1691 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1692 delete, true);
1693 assertFalse(res);
1694
1695 put = new Put(row1);
1696 put.add(fam1, qf1, val2);
1697
1698 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1699 put, true);
1700 assertTrue(res);
1701
1702
1703 delete = new Delete(row1);
1704 delete.deleteColumn(fam1, qf1);
1705 delete.deleteColumn(fam1, qf1);
1706 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1707 delete, true);
1708 assertTrue(res);
1709
1710 delete = new Delete(row1);
1711 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1712 delete, true);
1713 assertTrue(res);
1714
1715
1716 put = new Put(row1);
1717 put.add(fam1, qf1, val1);
1718
1719 res = region
1720 .checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new NullComparator(), put, true);
1721 assertTrue(res);
1722 }
1723
1724 @Test
1725 public void testCheckAndMutate_WithWrongValue() throws IOException {
1726 byte[] row1 = Bytes.toBytes("row1");
1727 byte[] fam1 = Bytes.toBytes("fam1");
1728 byte[] qf1 = Bytes.toBytes("qualifier");
1729 byte[] val1 = Bytes.toBytes("value1");
1730 byte[] val2 = Bytes.toBytes("value2");
1731
1732
1733 String method = this.getName();
1734 this.region = initHRegion(tableName, method, CONF, fam1);
1735
1736 Put put = new Put(row1);
1737 put.add(fam1, qf1, val1);
1738 region.put(put);
1739
1740
1741 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1742 val2), put, true);
1743 assertFalse(res);
1744
1745
1746 Delete delete = new Delete(row1);
1747 delete.deleteFamily(fam1);
1748 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val2),
1749 put, true);
1750 assertFalse(res);
1751 }
1752
1753 @Test
1754 public void testCheckAndMutate_WithCorrectValue() throws IOException {
1755 byte[] row1 = Bytes.toBytes("row1");
1756 byte[] fam1 = Bytes.toBytes("fam1");
1757 byte[] qf1 = Bytes.toBytes("qualifier");
1758 byte[] val1 = Bytes.toBytes("value1");
1759
1760
1761 String method = this.getName();
1762 this.region = initHRegion(tableName, method, CONF, fam1);
1763
1764 Put put = new Put(row1);
1765 put.add(fam1, qf1, val1);
1766 region.put(put);
1767
1768
1769 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1770 val1), put, true);
1771 assertTrue( res);
1772
1773
1774 Delete delete = new Delete(row1);
1775 delete.deleteColumn(fam1, qf1);
1776 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1777 delete, true);
1778 assertTrue(res);
1779 }
1780
1781 @Test
1782 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1783 byte[] row1 = Bytes.toBytes("row1");
1784 byte[] fam1 = Bytes.toBytes("fam1");
1785 byte[] qf1 = Bytes.toBytes("qualifier");
1786 byte[] val1 = Bytes.toBytes("value1");
1787 byte[] val2 = Bytes.toBytes("value2");
1788 byte[] val3 = Bytes.toBytes("value3");
1789 byte[] val4 = Bytes.toBytes("value4");
1790
1791
1792 String method = this.getName();
1793 this.region = initHRegion(tableName, method, CONF, fam1);
1794
1795 Put put = new Put(row1);
1796 put.add(fam1, qf1, val3);
1797 region.put(put);
1798
1799
1800 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1801 new BinaryComparator(val3), put, true);
1802 assertEquals(false, res);
1803
1804
1805 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1806 new BinaryComparator(val4), put, true);
1807 assertEquals(false, res);
1808
1809
1810
1811 put = new Put(row1);
1812 put.add(fam1, qf1, val2);
1813 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS,
1814 new BinaryComparator(val2), put, true);
1815 assertEquals(true, res);
1816
1817
1818 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1819 new BinaryComparator(val3), put, true);
1820 assertEquals(false, res);
1821
1822
1823
1824 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1825 new BinaryComparator(val2), put, true);
1826 assertEquals(true, res);
1827
1828
1829
1830 put = new Put(row1);
1831 put.add(fam1, qf1, val3);
1832 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.LESS_OR_EQUAL,
1833 new BinaryComparator(val1), put, true);
1834 assertEquals(true, res);
1835
1836
1837 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1838 new BinaryComparator(val3), put, true);
1839 assertEquals(false, res);
1840
1841
1842 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1843 new BinaryComparator(val2), put, true);
1844 assertEquals(false, res);
1845
1846
1847
1848 put = new Put(row1);
1849 put.add(fam1, qf1, val2);
1850 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER,
1851 new BinaryComparator(val4), put, true);
1852 assertEquals(true, res);
1853
1854
1855 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1856 new BinaryComparator(val1), put, true);
1857 assertEquals(false, res);
1858
1859
1860
1861 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1862 new BinaryComparator(val2), put, true);
1863 assertEquals(true, res);
1864
1865
1866 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.GREATER_OR_EQUAL,
1867 new BinaryComparator(val3), put, true);
1868 assertEquals(true, res);
1869 }
1870
1871 @Test
1872 public void testCheckAndPut_ThatPutWasWritten() throws IOException {
1873 byte[] row1 = Bytes.toBytes("row1");
1874 byte[] fam1 = Bytes.toBytes("fam1");
1875 byte[] fam2 = Bytes.toBytes("fam2");
1876 byte[] qf1 = Bytes.toBytes("qualifier");
1877 byte[] val1 = Bytes.toBytes("value1");
1878 byte[] val2 = Bytes.toBytes("value2");
1879
1880 byte[][] families = { fam1, fam2 };
1881
1882
1883 String method = this.getName();
1884 this.region = initHRegion(tableName, method, CONF, families);
1885
1886 Put put = new Put(row1);
1887 put.add(fam1, qf1, val1);
1888 region.put(put);
1889
1890
1891 long ts = System.currentTimeMillis();
1892 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
1893 put = new Put(row1);
1894 put.add(kv);
1895
1896
1897 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1898 val1), put, true);
1899 assertEquals(true, res);
1900
1901 Get get = new Get(row1);
1902 get.addColumn(fam2, qf1);
1903 Cell[] actual = region.get(get).rawCells();
1904
1905 Cell[] expected = { kv };
1906
1907 assertEquals(expected.length, actual.length);
1908 for (int i = 0; i < actual.length; i++) {
1909 assertEquals(expected[i], actual[i]);
1910 }
1911 }
1912
1913 @Test
1914 public void testCheckAndPut_wrongRowInPut() throws IOException {
1915 TableName tableName = TableName.valueOf(name.getMethodName());
1916 this.region = initHRegion(tableName, this.getName(), CONF, COLUMNS);
1917 Put put = new Put(row2);
1918 put.add(fam1, qual1, value1);
1919 try {
1920 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1921 new BinaryComparator(value2), put, false);
1922 fail();
1923 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
1924
1925 }
1926 }
1927
1928 @Test
1929 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
1930 byte[] row1 = Bytes.toBytes("row1");
1931 byte[] fam1 = Bytes.toBytes("fam1");
1932 byte[] fam2 = Bytes.toBytes("fam2");
1933 byte[] qf1 = Bytes.toBytes("qualifier1");
1934 byte[] qf2 = Bytes.toBytes("qualifier2");
1935 byte[] qf3 = Bytes.toBytes("qualifier3");
1936 byte[] val1 = Bytes.toBytes("value1");
1937 byte[] val2 = Bytes.toBytes("value2");
1938 byte[] val3 = Bytes.toBytes("value3");
1939 byte[] emptyVal = new byte[] {};
1940
1941 byte[][] families = { fam1, fam2 };
1942
1943
1944 String method = this.getName();
1945 this.region = initHRegion(tableName, method, CONF, families);
1946
1947 Put put = new Put(row1);
1948 put.add(fam1, qf1, val1);
1949 region.put(put);
1950 Threads.sleep(2);
1951
1952 put = new Put(row1);
1953 put.add(fam1, qf1, val2);
1954 put.add(fam2, qf1, val3);
1955 put.add(fam2, qf2, val2);
1956 put.add(fam2, qf3, val1);
1957 put.add(fam1, qf3, val1);
1958 region.put(put);
1959
1960
1961 Delete delete = new Delete(row1);
1962 delete.deleteColumn(fam1, qf1);
1963 delete.deleteColumn(fam2, qf1);
1964 delete.deleteColumn(fam1, qf3);
1965 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
1966 val2), delete, true);
1967 assertEquals(true, res);
1968
1969 Get get = new Get(row1);
1970 get.addColumn(fam1, qf1);
1971 get.addColumn(fam1, qf3);
1972 get.addColumn(fam2, qf2);
1973 Result r = region.get(get);
1974 assertEquals(2, r.size());
1975 assertArrayEquals(val1, r.getValue(fam1, qf1));
1976 assertArrayEquals(val2, r.getValue(fam2, qf2));
1977
1978
1979 delete = new Delete(row1);
1980 delete.deleteFamily(fam2);
1981 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL, new BinaryComparator(emptyVal),
1982 delete, true);
1983 assertEquals(true, res);
1984
1985 get = new Get(row1);
1986 r = region.get(get);
1987 assertEquals(1, r.size());
1988 assertArrayEquals(val1, r.getValue(fam1, qf1));
1989
1990
1991 delete = new Delete(row1);
1992 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(val1),
1993 delete, true);
1994 assertEquals(true, res);
1995 get = new Get(row1);
1996 r = region.get(get);
1997 assertEquals(0, r.size());
1998 }
1999
2000
2001
2002
2003 @Test
2004 public void testDelete_multiDeleteColumn() throws IOException {
2005 byte[] row1 = Bytes.toBytes("row1");
2006 byte[] fam1 = Bytes.toBytes("fam1");
2007 byte[] qual = Bytes.toBytes("qualifier");
2008 byte[] value = Bytes.toBytes("value");
2009
2010 Put put = new Put(row1);
2011 put.add(fam1, qual, 1, value);
2012 put.add(fam1, qual, 2, value);
2013
2014 String method = this.getName();
2015 this.region = initHRegion(tableName, method, CONF, fam1);
2016 region.put(put);
2017
2018
2019 Delete delete = new Delete(row1);
2020 delete.deleteColumn(fam1, qual);
2021 delete.deleteColumn(fam1, qual);
2022 region.delete(delete);
2023
2024 Get get = new Get(row1);
2025 get.addFamily(fam1);
2026 Result r = region.get(get);
2027 assertEquals(0, r.size());
2028 }
2029
2030 @Test
2031 public void testDelete_CheckFamily() throws IOException {
2032 byte[] row1 = Bytes.toBytes("row1");
2033 byte[] fam1 = Bytes.toBytes("fam1");
2034 byte[] fam2 = Bytes.toBytes("fam2");
2035 byte[] fam3 = Bytes.toBytes("fam3");
2036 byte[] fam4 = Bytes.toBytes("fam4");
2037
2038
2039 String method = this.getName();
2040 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
2041 List<Cell> kvs = new ArrayList<Cell>();
2042 kvs.add(new KeyValue(row1, fam4, null, null));
2043
2044
2045 byte[] family = fam2;
2046 try {
2047 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2048 Bytes.BYTES_COMPARATOR);
2049 deleteMap.put(family, kvs);
2050 region.delete(deleteMap, Durability.SYNC_WAL);
2051 } catch (Exception e) {
2052 assertTrue("Family " + new String(family) + " does not exist", false);
2053 }
2054
2055
2056 boolean ok = false;
2057 family = fam4;
2058 try {
2059 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2060 Bytes.BYTES_COMPARATOR);
2061 deleteMap.put(family, kvs);
2062 region.delete(deleteMap, Durability.SYNC_WAL);
2063 } catch (Exception e) {
2064 ok = true;
2065 }
2066 assertTrue("Family " + new String(family) + " does exist", ok);
2067 }
2068
2069 @Test
2070 public void testDelete_mixed() throws IOException, InterruptedException {
2071 byte[] fam = Bytes.toBytes("info");
2072 byte[][] families = { fam };
2073 String method = this.getName();
2074 this.region = initHRegion(tableName, method, CONF, families);
2075 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2076
2077 byte[] row = Bytes.toBytes("table_name");
2078
2079 byte[] serverinfo = Bytes.toBytes("serverinfo");
2080 byte[] splitA = Bytes.toBytes("splitA");
2081 byte[] splitB = Bytes.toBytes("splitB");
2082
2083
2084 Put put = new Put(row);
2085 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2086 region.put(put);
2087
2088 put = new Put(row);
2089 put.add(fam, splitB, Bytes.toBytes("reference_B"));
2090 region.put(put);
2091
2092 put = new Put(row);
2093 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
2094 region.put(put);
2095
2096
2097 Delete delete = new Delete(row);
2098 delete.deleteColumns(fam, splitA);
2099 region.delete(delete);
2100
2101
2102 Get get = new Get(row).addColumn(fam, serverinfo);
2103 Result result = region.get(get);
2104 assertEquals(1, result.size());
2105
2106 get = new Get(row).addColumn(fam, splitA);
2107 result = region.get(get);
2108 assertEquals(0, result.size());
2109
2110 get = new Get(row).addColumn(fam, splitB);
2111 result = region.get(get);
2112 assertEquals(1, result.size());
2113
2114
2115 put = new Put(row);
2116 put.add(fam, splitA, Bytes.toBytes("reference_A"));
2117 region.put(put);
2118 get = new Get(row);
2119 result = region.get(get);
2120 assertEquals(3, result.size());
2121
2122
2123 delete = new Delete(row);
2124 region.delete(delete);
2125 assertEquals(0, region.get(get).size());
2126
2127 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
2128 result = region.get(get);
2129 assertEquals(1, result.size());
2130 }
2131
2132 @Test
2133 public void testDeleteRowWithFutureTs() throws IOException {
2134 byte[] fam = Bytes.toBytes("info");
2135 byte[][] families = { fam };
2136 String method = this.getName();
2137 this.region = initHRegion(tableName, method, CONF, families);
2138 byte[] row = Bytes.toBytes("table_name");
2139
2140 byte[] serverinfo = Bytes.toBytes("serverinfo");
2141
2142
2143 Put put = new Put(row);
2144 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
2145 region.put(put);
2146
2147
2148 Delete delete = new Delete(row);
2149 region.delete(delete);
2150
2151
2152 Get get = new Get(row).addColumn(fam, serverinfo);
2153 Result result = region.get(get);
2154 assertEquals(1, result.size());
2155
2156
2157 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
2158 region.delete(delete);
2159
2160
2161 get = new Get(row).addColumn(fam, serverinfo);
2162 result = region.get(get);
2163 assertEquals(0, result.size());
2164 }
2165
2166
2167
2168
2169
2170 @Test
2171 public void testPutWithLatestTS() throws IOException {
2172 byte[] fam = Bytes.toBytes("info");
2173 byte[][] families = { fam };
2174 String method = this.getName();
2175 this.region = initHRegion(tableName, method, CONF, families);
2176 byte[] row = Bytes.toBytes("row1");
2177
2178 byte[] qual = Bytes.toBytes("qual");
2179
2180
2181 Put put = new Put(row);
2182 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2183 region.put(put);
2184
2185
2186 Get get = new Get(row).addColumn(fam, qual);
2187 Result result = region.get(get);
2188 assertEquals(1, result.size());
2189 Cell kv = result.rawCells()[0];
2190 LOG.info("Got: " + kv);
2191 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2192 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2193
2194
2195
2196 row = Bytes.toBytes("row2");
2197 put = new Put(row);
2198 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
2199 region.put(put);
2200
2201
2202 get = new Get(row).addColumn(fam, qual);
2203 result = region.get(get);
2204 assertEquals(1, result.size());
2205 kv = result.rawCells()[0];
2206 LOG.info("Got: " + kv);
2207 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
2208 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
2209 }
2210
2211
2212
2213
2214
2215
2216 @Test
2217 public void testPutWithTsSlop() throws IOException {
2218 byte[] fam = Bytes.toBytes("info");
2219 byte[][] families = { fam };
2220 String method = this.getName();
2221
2222
2223 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
2224 this.region = initHRegion(tableName, method, CONF, families);
2225 boolean caughtExcep = false;
2226 try {
2227
2228 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
2229
2230 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), System.currentTimeMillis() + 2000,
2231 Bytes.toBytes("value")));
2232 fail("Expected IOE for TS out of configured timerange");
2233 } catch (FailedSanityCheckException ioe) {
2234 LOG.debug("Received expected exception", ioe);
2235 caughtExcep = true;
2236 }
2237 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
2238 }
2239
2240 @Test
2241 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
2242 byte[] fam1 = Bytes.toBytes("columnA");
2243 byte[] fam2 = Bytes.toBytes("columnB");
2244 this.region = initHRegion(tableName, getName(), CONF, fam1, fam2);
2245 byte[] rowA = Bytes.toBytes("rowA");
2246 byte[] rowB = Bytes.toBytes("rowB");
2247
2248 byte[] value = Bytes.toBytes("value");
2249
2250 Delete delete = new Delete(rowA);
2251 delete.deleteFamily(fam1);
2252
2253 region.delete(delete);
2254
2255
2256 Put put = new Put(rowA);
2257 put.add(fam2, null, value);
2258 region.put(put);
2259
2260 put = new Put(rowB);
2261 put.add(fam1, null, value);
2262 put.add(fam2, null, value);
2263 region.put(put);
2264
2265 Scan scan = new Scan();
2266 scan.addFamily(fam1).addFamily(fam2);
2267 InternalScanner s = region.getScanner(scan);
2268 List<Cell> results = new ArrayList<Cell>();
2269 s.next(results);
2270 assertTrue(CellUtil.matchingRow(results.get(0), rowA));
2271
2272 results.clear();
2273 s.next(results);
2274 assertTrue(CellUtil.matchingRow(results.get(0), rowB));
2275 }
2276
2277 @Test
2278 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
2279 Delete delete = new Delete(row);
2280 delete.deleteColumns(fam1, qual1);
2281 doTestDelete_AndPostInsert(delete);
2282 }
2283
2284 @Test
2285 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
2286 Delete delete = new Delete(row);
2287 delete.deleteFamily(fam1);
2288 doTestDelete_AndPostInsert(delete);
2289 }
2290
2291 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
2292 TableName tableName = TableName.valueOf(name.getMethodName());
2293 this.region = initHRegion(tableName, getName(), CONF, fam1);
2294 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2295 Put put = new Put(row);
2296 put.add(fam1, qual1, value1);
2297 region.put(put);
2298
2299
2300 region.delete(delete);
2301
2302
2303 put = new Put(row);
2304 put.add(fam1, qual1, value2);
2305 region.put(put);
2306
2307
2308 Get get = new Get(row);
2309 get.addColumn(fam1, qual1);
2310
2311 Result r = region.get(get);
2312 assertEquals(1, r.size());
2313 assertArrayEquals(value2, r.getValue(fam1, qual1));
2314
2315
2316 Scan scan = new Scan(row);
2317 scan.addColumn(fam1, qual1);
2318 InternalScanner s = region.getScanner(scan);
2319
2320 List<Cell> results = new ArrayList<Cell>();
2321 assertEquals(false, s.next(results));
2322 assertEquals(1, results.size());
2323 Cell kv = results.get(0);
2324
2325 assertArrayEquals(value2, CellUtil.cloneValue(kv));
2326 assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
2327 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
2328 assertArrayEquals(row, CellUtil.cloneRow(kv));
2329 }
2330
2331 @Test
2332 public void testDelete_CheckTimestampUpdated() throws IOException {
2333 TableName tableName = TableName.valueOf(name.getMethodName());
2334 byte[] row1 = Bytes.toBytes("row1");
2335 byte[] col1 = Bytes.toBytes("col1");
2336 byte[] col2 = Bytes.toBytes("col2");
2337 byte[] col3 = Bytes.toBytes("col3");
2338
2339
2340 String method = this.getName();
2341 this.region = initHRegion(tableName, method, CONF, fam1);
2342
2343 List<Cell> kvs = new ArrayList<Cell>();
2344 kvs.add(new KeyValue(row1, fam1, col1, null));
2345 kvs.add(new KeyValue(row1, fam1, col2, null));
2346 kvs.add(new KeyValue(row1, fam1, col3, null));
2347
2348 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<byte[], List<Cell>>(
2349 Bytes.BYTES_COMPARATOR);
2350 deleteMap.put(fam1, kvs);
2351 region.delete(deleteMap, Durability.SYNC_WAL);
2352
2353
2354
2355 long now = System.currentTimeMillis();
2356 DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
2357 Cell firstCell = memstore.activeSection.getCellSkipListSet().first();
2358 assertTrue(firstCell.getTimestamp() <= now);
2359 now = firstCell.getTimestamp();
2360 for (Cell cell : memstore.activeSection.getCellSkipListSet()) {
2361 assertTrue(cell.getTimestamp() <= now);
2362 now = cell.getTimestamp();
2363 }
2364 }
2365
2366
2367
2368
2369 @Test
2370 public void testGet_FamilyChecker() throws IOException {
2371 byte[] row1 = Bytes.toBytes("row1");
2372 byte[] fam1 = Bytes.toBytes("fam1");
2373 byte[] fam2 = Bytes.toBytes("False");
2374 byte[] col1 = Bytes.toBytes("col1");
2375
2376
2377 String method = this.getName();
2378 this.region = initHRegion(tableName, method, CONF, fam1);
2379 Get get = new Get(row1);
2380 get.addColumn(fam2, col1);
2381
2382
2383 try {
2384 region.get(get);
2385 fail("Expecting DoNotRetryIOException in get but did not get any");
2386 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
2387 LOG.info("Got expected DoNotRetryIOException successfully");
2388 }
2389 }
2390
2391 @Test
2392 public void testGet_Basic() throws IOException {
2393 byte[] row1 = Bytes.toBytes("row1");
2394 byte[] fam1 = Bytes.toBytes("fam1");
2395 byte[] col1 = Bytes.toBytes("col1");
2396 byte[] col2 = Bytes.toBytes("col2");
2397 byte[] col3 = Bytes.toBytes("col3");
2398 byte[] col4 = Bytes.toBytes("col4");
2399 byte[] col5 = Bytes.toBytes("col5");
2400
2401
2402 String method = this.getName();
2403 this.region = initHRegion(tableName, method, CONF, fam1);
2404
2405 Put put = new Put(row1);
2406 put.add(fam1, col1, null);
2407 put.add(fam1, col2, null);
2408 put.add(fam1, col3, null);
2409 put.add(fam1, col4, null);
2410 put.add(fam1, col5, null);
2411 region.put(put);
2412
2413 Get get = new Get(row1);
2414 get.addColumn(fam1, col2);
2415 get.addColumn(fam1, col4);
2416
2417 KeyValue kv1 = new KeyValue(row1, fam1, col2);
2418 KeyValue kv2 = new KeyValue(row1, fam1, col4);
2419 KeyValue[] expected = { kv1, kv2 };
2420
2421
2422 Result res = region.get(get);
2423 assertEquals(expected.length, res.size());
2424 for (int i = 0; i < res.size(); i++) {
2425 assertTrue(CellUtil.matchingRow(expected[i], res.rawCells()[i]));
2426 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
2427 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
2428 }
2429
2430
2431 Get g = new Get(row1);
2432 final int count = 2;
2433 g.setFilter(new ColumnCountGetFilter(count));
2434 res = region.get(g);
2435 assertEquals(count, res.size());
2436 }
2437
2438 @Test
2439 public void testGet_Empty() throws IOException {
2440 byte[] row = Bytes.toBytes("row");
2441 byte[] fam = Bytes.toBytes("fam");
2442
2443 String method = this.getName();
2444 this.region = initHRegion(tableName, method, CONF, fam);
2445 Get get = new Get(row);
2446 get.addFamily(fam);
2447 Result r = region.get(get);
2448
2449 assertTrue(r.isEmpty());
2450 }
2451
2452
2453
2454
2455 @Test
2456 public void testMerge() throws IOException {
2457 byte[][] families = { fam1, fam2, fam3 };
2458 Configuration hc = initSplit();
2459
2460 String method = this.getName();
2461 this.region = initHRegion(tableName, method, hc, families);
2462 LOG.info("" + HBaseTestCase.addContent(region, fam3));
2463 region.flush(true);
2464 region.compactStores();
2465 region.waitForFlushesAndCompactions();
2466 for(Store s:region.getStores()) {
2467 s.closeAndArchiveCompactedFiles();
2468 }
2469 byte[] splitRow = region.checkSplit();
2470 assertNotNull(splitRow);
2471 LOG.info("SplitRow: " + Bytes.toString(splitRow));
2472 HRegion[] subregions = splitRegion(region, splitRow);
2473 try {
2474
2475 for (int i = 0; i < subregions.length; i++) {
2476 HRegion.openHRegion(subregions[i], null);
2477 subregions[i].compactStores();
2478 subregions[i].waitForFlushesAndCompactions();
2479 for(Store s:subregions[i].getStores()) {
2480 s.closeAndArchiveCompactedFiles();
2481 }
2482 }
2483 Path oldRegionPath = region.getRegionFileSystem().getRegionDir();
2484 Path oldRegion1 = subregions[0].getRegionFileSystem().getRegionDir();
2485 Path oldRegion2 = subregions[1].getRegionFileSystem().getRegionDir();
2486 long startTime = System.currentTimeMillis();
2487 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
2488 LOG.info("Merge regions elapsed time: "
2489 + ((System.currentTimeMillis() - startTime) / 1000.0));
2490 FILESYSTEM.delete(oldRegion1, true);
2491 FILESYSTEM.delete(oldRegion2, true);
2492 FILESYSTEM.delete(oldRegionPath, true);
2493 LOG.info("splitAndMerge completed.");
2494 } finally {
2495 for (int i = 0; i < subregions.length; i++) {
2496 try {
2497 HRegion.closeHRegion(subregions[i]);
2498 } catch (IOException e) {
2499
2500 }
2501 }
2502 }
2503 }
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513 HRegion[] splitRegion(final HRegion parent, final byte[] midkey) throws IOException {
2514 PairOfSameType<Region> result = null;
2515 SplitTransactionImpl st = new SplitTransactionImpl(parent, midkey);
2516
2517
2518 if (!st.prepare()) {
2519 parent.clearSplit();
2520 return null;
2521 }
2522 try {
2523 result = st.execute(null, null);
2524 } catch (IOException ioe) {
2525 try {
2526 LOG.info("Running rollback of failed split of " +
2527 parent.getRegionInfo().getRegionNameAsString() + "; " + ioe.getMessage());
2528 st.rollback(null, null);
2529 LOG.info("Successful rollback of failed split of " +
2530 parent.getRegionInfo().getRegionNameAsString());
2531 return null;
2532 } catch (RuntimeException e) {
2533
2534 LOG.info("Failed rollback of failed split of " +
2535 parent.getRegionInfo().getRegionNameAsString() + " -- aborting server", e);
2536 }
2537 }
2538 finally {
2539 parent.clearSplit();
2540 }
2541 return new HRegion[] { (HRegion)result.getFirst(), (HRegion)result.getSecond() };
2542 }
2543
2544
2545
2546
2547 @Test
2548 public void testGetScanner_WithOkFamilies() throws IOException {
2549 byte[] fam1 = Bytes.toBytes("fam1");
2550 byte[] fam2 = Bytes.toBytes("fam2");
2551
2552 byte[][] families = { fam1, fam2 };
2553
2554
2555 String method = this.getName();
2556 this.region = initHRegion(tableName, method, CONF, families);
2557 Scan scan = new Scan();
2558 scan.addFamily(fam1);
2559 scan.addFamily(fam2);
2560 try {
2561 region.getScanner(scan);
2562 } catch (Exception e) {
2563 assertTrue("Families could not be found in Region", false);
2564 }
2565 }
2566
2567 @Test
2568 public void testGetScanner_WithNotOkFamilies() throws IOException {
2569 byte[] fam1 = Bytes.toBytes("fam1");
2570 byte[] fam2 = Bytes.toBytes("fam2");
2571
2572 byte[][] families = { fam1 };
2573
2574
2575 String method = this.getName();
2576 this.region = initHRegion(tableName, method, CONF, families);
2577 Scan scan = new Scan();
2578 scan.addFamily(fam2);
2579 boolean ok = false;
2580 try {
2581 region.getScanner(scan);
2582 } catch (Exception e) {
2583 ok = true;
2584 }
2585 assertTrue("Families could not be found in Region", ok);
2586 }
2587
2588 @Test
2589 public void testGetScanner_WithNoFamilies() throws IOException {
2590 byte[] row1 = Bytes.toBytes("row1");
2591 byte[] fam1 = Bytes.toBytes("fam1");
2592 byte[] fam2 = Bytes.toBytes("fam2");
2593 byte[] fam3 = Bytes.toBytes("fam3");
2594 byte[] fam4 = Bytes.toBytes("fam4");
2595
2596 byte[][] families = { fam1, fam2, fam3, fam4 };
2597
2598
2599 String method = this.getName();
2600 this.region = initHRegion(tableName, method, CONF, families);
2601
2602 Put put = new Put(row1);
2603 put.add(fam1, null, null);
2604 put.add(fam2, null, null);
2605 put.add(fam3, null, null);
2606 put.add(fam4, null, null);
2607 region.put(put);
2608
2609 Scan scan = null;
2610 HRegion.RegionScannerImpl is = null;
2611
2612
2613
2614
2615 scan = new Scan();
2616 scan.addFamily(fam2);
2617 scan.addFamily(fam4);
2618 is = (RegionScannerImpl) region.getScanner(scan);
2619 assertEquals(1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2620
2621 scan = new Scan();
2622 is = (RegionScannerImpl) region.getScanner(scan);
2623 assertEquals(families.length - 1, ((RegionScannerImpl) is).storeHeap.getHeap().size());
2624 }
2625
2626
2627
2628
2629
2630
2631 @Test
2632 public void testGetScanner_WithRegionClosed() throws IOException {
2633 byte[] fam1 = Bytes.toBytes("fam1");
2634 byte[] fam2 = Bytes.toBytes("fam2");
2635
2636 byte[][] families = { fam1, fam2 };
2637
2638
2639 String method = this.getName();
2640 try {
2641 this.region = initHRegion(tableName, method, CONF, families);
2642 } catch (IOException e) {
2643 e.printStackTrace();
2644 fail("Got IOException during initHRegion, " + e.getMessage());
2645 }
2646 region.closed.set(true);
2647 try {
2648 region.getScanner(null);
2649 fail("Expected to get an exception during getScanner on a region that is closed");
2650 } catch (NotServingRegionException e) {
2651
2652 } catch (IOException e) {
2653 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
2654 + e.getMessage());
2655 }
2656 }
2657
2658 @Test
2659 public void testRegionScanner_Next() throws IOException {
2660 byte[] row1 = Bytes.toBytes("row1");
2661 byte[] row2 = Bytes.toBytes("row2");
2662 byte[] fam1 = Bytes.toBytes("fam1");
2663 byte[] fam2 = Bytes.toBytes("fam2");
2664 byte[] fam3 = Bytes.toBytes("fam3");
2665 byte[] fam4 = Bytes.toBytes("fam4");
2666
2667 byte[][] families = { fam1, fam2, fam3, fam4 };
2668 long ts = System.currentTimeMillis();
2669
2670
2671 String method = this.getName();
2672 this.region = initHRegion(tableName, method, CONF, families);
2673
2674 Put put = null;
2675 put = new Put(row1);
2676 put.add(fam1, (byte[]) null, ts, null);
2677 put.add(fam2, (byte[]) null, ts, null);
2678 put.add(fam3, (byte[]) null, ts, null);
2679 put.add(fam4, (byte[]) null, ts, null);
2680 region.put(put);
2681
2682 put = new Put(row2);
2683 put.add(fam1, (byte[]) null, ts, null);
2684 put.add(fam2, (byte[]) null, ts, null);
2685 put.add(fam3, (byte[]) null, ts, null);
2686 put.add(fam4, (byte[]) null, ts, null);
2687 region.put(put);
2688
2689 Scan scan = new Scan();
2690 scan.addFamily(fam2);
2691 scan.addFamily(fam4);
2692 InternalScanner is = region.getScanner(scan);
2693
2694 List<Cell> res = null;
2695
2696
2697 List<Cell> expected1 = new ArrayList<Cell>();
2698 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2699 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2700
2701 res = new ArrayList<Cell>();
2702 is.next(res);
2703 for (int i = 0; i < res.size(); i++) {
2704 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
2705 }
2706
2707
2708 List<Cell> expected2 = new ArrayList<Cell>();
2709 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2710 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2711
2712 res = new ArrayList<Cell>();
2713 is.next(res);
2714 for (int i = 0; i < res.size(); i++) {
2715 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
2716 }
2717 }
2718
2719 @Test
2720 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
2721 byte[] row1 = Bytes.toBytes("row1");
2722 byte[] qf1 = Bytes.toBytes("qualifier1");
2723 byte[] qf2 = Bytes.toBytes("qualifier2");
2724 byte[] fam1 = Bytes.toBytes("fam1");
2725 byte[][] families = { fam1 };
2726
2727 long ts1 = System.currentTimeMillis();
2728 long ts2 = ts1 + 1;
2729 long ts3 = ts1 + 2;
2730
2731
2732 String method = this.getName();
2733 this.region = initHRegion(tableName, method, CONF, families);
2734
2735 Put put = null;
2736 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2737 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2738 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2739
2740 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2741 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2742 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2743
2744 put = new Put(row1);
2745 put.add(kv13);
2746 put.add(kv12);
2747 put.add(kv11);
2748 put.add(kv23);
2749 put.add(kv22);
2750 put.add(kv21);
2751 region.put(put);
2752
2753
2754 List<Cell> expected = new ArrayList<Cell>();
2755 expected.add(kv13);
2756 expected.add(kv12);
2757
2758 Scan scan = new Scan(row1);
2759 scan.addColumn(fam1, qf1);
2760 scan.setMaxVersions(MAX_VERSIONS);
2761 List<Cell> actual = new ArrayList<Cell>();
2762 InternalScanner scanner = region.getScanner(scan);
2763
2764 boolean hasNext = scanner.next(actual);
2765 assertEquals(false, hasNext);
2766
2767
2768 for (int i = 0; i < expected.size(); i++) {
2769 assertEquals(expected.get(i), actual.get(i));
2770 }
2771 }
2772
2773 @Test
2774 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
2775 byte[] row1 = Bytes.toBytes("row1");
2776 byte[] qf1 = Bytes.toBytes("qualifier1");
2777 byte[] qf2 = Bytes.toBytes("qualifier2");
2778 byte[] fam1 = Bytes.toBytes("fam1");
2779 byte[][] families = { fam1 };
2780
2781 long ts1 = 1;
2782 long ts2 = ts1 + 1;
2783 long ts3 = ts1 + 2;
2784
2785
2786 String method = this.getName();
2787 this.region = initHRegion(tableName, method, CONF, families);
2788
2789 Put put = null;
2790 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2791 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2792 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2793
2794 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2795 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2796 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2797
2798 put = new Put(row1);
2799 put.add(kv13);
2800 put.add(kv12);
2801 put.add(kv11);
2802 put.add(kv23);
2803 put.add(kv22);
2804 put.add(kv21);
2805 region.put(put);
2806 region.flush(true);
2807
2808
2809 List<Cell> expected = new ArrayList<Cell>();
2810 expected.add(kv13);
2811 expected.add(kv12);
2812 expected.add(kv23);
2813 expected.add(kv22);
2814
2815 Scan scan = new Scan(row1);
2816 scan.addColumn(fam1, qf1);
2817 scan.addColumn(fam1, qf2);
2818 scan.setMaxVersions(MAX_VERSIONS);
2819 List<Cell> actual = new ArrayList<Cell>();
2820 InternalScanner scanner = region.getScanner(scan);
2821
2822 boolean hasNext = scanner.next(actual);
2823 assertEquals(false, hasNext);
2824
2825
2826 for (int i = 0; i < expected.size(); i++) {
2827 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2828 }
2829 }
2830
2831 @Test
2832 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() throws IOException {
2833 byte[] row1 = Bytes.toBytes("row1");
2834 byte[] fam1 = Bytes.toBytes("fam1");
2835 byte[][] families = { fam1 };
2836 byte[] qf1 = Bytes.toBytes("qualifier1");
2837 byte[] qf2 = Bytes.toBytes("qualifier2");
2838
2839 long ts1 = 1;
2840 long ts2 = ts1 + 1;
2841 long ts3 = ts1 + 2;
2842 long ts4 = ts1 + 3;
2843
2844
2845 String method = this.getName();
2846 this.region = initHRegion(tableName, method, CONF, families);
2847
2848 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2849 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2850 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2851 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2852
2853 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2854 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2855 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2856 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2857
2858 Put put = null;
2859 put = new Put(row1);
2860 put.add(kv14);
2861 put.add(kv24);
2862 region.put(put);
2863 region.flush(true);
2864
2865 put = new Put(row1);
2866 put.add(kv23);
2867 put.add(kv13);
2868 region.put(put);
2869 region.flush(true);
2870
2871 put = new Put(row1);
2872 put.add(kv22);
2873 put.add(kv12);
2874 region.put(put);
2875 region.flush(true);
2876
2877 put = new Put(row1);
2878 put.add(kv21);
2879 put.add(kv11);
2880 region.put(put);
2881
2882
2883 List<Cell> expected = new ArrayList<Cell>();
2884 expected.add(kv14);
2885 expected.add(kv13);
2886 expected.add(kv12);
2887 expected.add(kv24);
2888 expected.add(kv23);
2889 expected.add(kv22);
2890
2891 Scan scan = new Scan(row1);
2892 scan.addColumn(fam1, qf1);
2893 scan.addColumn(fam1, qf2);
2894 int versions = 3;
2895 scan.setMaxVersions(versions);
2896 List<Cell> actual = new ArrayList<Cell>();
2897 InternalScanner scanner = region.getScanner(scan);
2898
2899 boolean hasNext = scanner.next(actual);
2900 assertEquals(false, hasNext);
2901
2902
2903 for (int i = 0; i < expected.size(); i++) {
2904 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
2905 }
2906 }
2907
2908 @Test
2909 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
2910 byte[] row1 = Bytes.toBytes("row1");
2911 byte[] qf1 = Bytes.toBytes("qualifier1");
2912 byte[] qf2 = Bytes.toBytes("qualifier2");
2913 byte[] fam1 = Bytes.toBytes("fam1");
2914 byte[][] families = { fam1 };
2915
2916 long ts1 = System.currentTimeMillis();
2917 long ts2 = ts1 + 1;
2918 long ts3 = ts1 + 2;
2919
2920
2921 String method = this.getName();
2922 this.region = initHRegion(tableName, method, CONF, families);
2923
2924 Put put = null;
2925 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2926 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2927 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2928
2929 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2930 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2931 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2932
2933 put = new Put(row1);
2934 put.add(kv13);
2935 put.add(kv12);
2936 put.add(kv11);
2937 put.add(kv23);
2938 put.add(kv22);
2939 put.add(kv21);
2940 region.put(put);
2941
2942
2943 List<Cell> expected = new ArrayList<Cell>();
2944 expected.add(kv13);
2945 expected.add(kv12);
2946 expected.add(kv23);
2947 expected.add(kv22);
2948
2949 Scan scan = new Scan(row1);
2950 scan.addFamily(fam1);
2951 scan.setMaxVersions(MAX_VERSIONS);
2952 List<Cell> actual = new ArrayList<Cell>();
2953 InternalScanner scanner = region.getScanner(scan);
2954
2955 boolean hasNext = scanner.next(actual);
2956 assertEquals(false, hasNext);
2957
2958
2959 for (int i = 0; i < expected.size(); i++) {
2960 assertEquals(expected.get(i), actual.get(i));
2961 }
2962 }
2963
2964 @Test
2965 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
2966 byte[] row1 = Bytes.toBytes("row1");
2967 byte[] qf1 = Bytes.toBytes("qualifier1");
2968 byte[] qf2 = Bytes.toBytes("qualifier2");
2969 byte[] fam1 = Bytes.toBytes("fam1");
2970
2971 long ts1 = 1;
2972 long ts2 = ts1 + 1;
2973 long ts3 = ts1 + 2;
2974
2975
2976 String method = this.getName();
2977 this.region = initHRegion(tableName, method, CONF, fam1);
2978
2979 Put put = null;
2980 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2981 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2982 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2983
2984 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2985 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2986 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2987
2988 put = new Put(row1);
2989 put.add(kv13);
2990 put.add(kv12);
2991 put.add(kv11);
2992 put.add(kv23);
2993 put.add(kv22);
2994 put.add(kv21);
2995 region.put(put);
2996 region.flush(true);
2997
2998
2999 List<Cell> expected = new ArrayList<Cell>();
3000 expected.add(kv13);
3001 expected.add(kv12);
3002 expected.add(kv23);
3003 expected.add(kv22);
3004
3005 Scan scan = new Scan(row1);
3006 scan.addFamily(fam1);
3007 scan.setMaxVersions(MAX_VERSIONS);
3008 List<Cell> actual = new ArrayList<Cell>();
3009 InternalScanner scanner = region.getScanner(scan);
3010
3011 boolean hasNext = scanner.next(actual);
3012 assertEquals(false, hasNext);
3013
3014
3015 for (int i = 0; i < expected.size(); i++) {
3016 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3017 }
3018 }
3019
3020 @Test
3021 public void testScanner_StopRow1542() throws IOException {
3022 byte[] family = Bytes.toBytes("testFamily");
3023 this.region = initHRegion(tableName, getName(), CONF, family);
3024 byte[] row1 = Bytes.toBytes("row111");
3025 byte[] row2 = Bytes.toBytes("row222");
3026 byte[] row3 = Bytes.toBytes("row333");
3027 byte[] row4 = Bytes.toBytes("row444");
3028 byte[] row5 = Bytes.toBytes("row555");
3029
3030 byte[] col1 = Bytes.toBytes("Pub111");
3031 byte[] col2 = Bytes.toBytes("Pub222");
3032
3033 Put put = new Put(row1);
3034 put.add(family, col1, Bytes.toBytes(10L));
3035 region.put(put);
3036
3037 put = new Put(row2);
3038 put.add(family, col1, Bytes.toBytes(15L));
3039 region.put(put);
3040
3041 put = new Put(row3);
3042 put.add(family, col2, Bytes.toBytes(20L));
3043 region.put(put);
3044
3045 put = new Put(row4);
3046 put.add(family, col2, Bytes.toBytes(30L));
3047 region.put(put);
3048
3049 put = new Put(row5);
3050 put.add(family, col1, Bytes.toBytes(40L));
3051 region.put(put);
3052
3053 Scan scan = new Scan(row3, row4);
3054 scan.setMaxVersions();
3055 scan.addColumn(family, col1);
3056 InternalScanner s = region.getScanner(scan);
3057
3058 List<Cell> results = new ArrayList<Cell>();
3059 assertEquals(false, s.next(results));
3060 assertEquals(0, results.size());
3061 }
3062
3063 @Test
3064 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
3065 byte[] row1 = Bytes.toBytes("row1");
3066 byte[] fam1 = Bytes.toBytes("fam1");
3067 byte[] qf1 = Bytes.toBytes("qualifier1");
3068 byte[] qf2 = Bytes.toBytes("quateslifier2");
3069
3070 long ts1 = 1;
3071 long ts2 = ts1 + 1;
3072 long ts3 = ts1 + 2;
3073 long ts4 = ts1 + 3;
3074
3075
3076 String method = this.getName();
3077 this.region = initHRegion(tableName, method, CONF, fam1);
3078
3079 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3080 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3081 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3082 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3083
3084 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3085 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3086 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3087 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3088
3089 Put put = null;
3090 put = new Put(row1);
3091 put.add(kv14);
3092 put.add(kv24);
3093 region.put(put);
3094 region.flush(true);
3095
3096 put = new Put(row1);
3097 put.add(kv23);
3098 put.add(kv13);
3099 region.put(put);
3100 region.flush(true);
3101
3102 put = new Put(row1);
3103 put.add(kv22);
3104 put.add(kv12);
3105 region.put(put);
3106 region.flush(true);
3107
3108 put = new Put(row1);
3109 put.add(kv21);
3110 put.add(kv11);
3111 region.put(put);
3112
3113
3114 List<KeyValue> expected = new ArrayList<KeyValue>();
3115 expected.add(kv14);
3116 expected.add(kv13);
3117 expected.add(kv12);
3118 expected.add(kv24);
3119 expected.add(kv23);
3120 expected.add(kv22);
3121
3122 Scan scan = new Scan(row1);
3123 int versions = 3;
3124 scan.setMaxVersions(versions);
3125 List<Cell> actual = new ArrayList<Cell>();
3126 InternalScanner scanner = region.getScanner(scan);
3127
3128 boolean hasNext = scanner.next(actual);
3129 assertEquals(false, hasNext);
3130
3131
3132 for (int i = 0; i < expected.size(); i++) {
3133 assertTrue(CellComparator.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3134 }
3135 }
3136
3137
3138
3139
3140
3141
3142
3143 @Test
3144 public void testScanner_JoinedScanners() throws IOException {
3145 byte[] cf_essential = Bytes.toBytes("essential");
3146 byte[] cf_joined = Bytes.toBytes("joined");
3147 byte[] cf_alpha = Bytes.toBytes("alpha");
3148 this.region = initHRegion(tableName, getName(), CONF, cf_essential, cf_joined, cf_alpha);
3149 byte[] row1 = Bytes.toBytes("row1");
3150 byte[] row2 = Bytes.toBytes("row2");
3151 byte[] row3 = Bytes.toBytes("row3");
3152
3153 byte[] col_normal = Bytes.toBytes("d");
3154 byte[] col_alpha = Bytes.toBytes("a");
3155
3156 byte[] filtered_val = Bytes.toBytes(3);
3157
3158 Put put = new Put(row1);
3159 put.add(cf_essential, col_normal, Bytes.toBytes(1));
3160 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
3161 region.put(put);
3162
3163 put = new Put(row2);
3164 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
3165 put.add(cf_joined, col_normal, Bytes.toBytes(2));
3166 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
3167 region.put(put);
3168
3169 put = new Put(row3);
3170 put.add(cf_essential, col_normal, filtered_val);
3171 put.add(cf_joined, col_normal, filtered_val);
3172 region.put(put);
3173
3174
3175
3176
3177
3178 Scan scan = new Scan();
3179 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
3180 CompareOp.NOT_EQUAL, filtered_val);
3181 scan.setFilter(filter);
3182 scan.setLoadColumnFamiliesOnDemand(true);
3183 InternalScanner s = region.getScanner(scan);
3184
3185 List<Cell> results = new ArrayList<Cell>();
3186 assertTrue(s.next(results));
3187 assertEquals(results.size(), 1);
3188 results.clear();
3189
3190 assertTrue(s.next(results));
3191 assertEquals(results.size(), 3);
3192 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
3193 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
3194 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
3195 results.clear();
3196
3197 assertFalse(s.next(results));
3198 assertEquals(results.size(), 0);
3199 }
3200
3201
3202
3203
3204
3205
3206 @Test
3207 public void testScanner_JoinedScannersWithLimits() throws IOException {
3208 final byte[] cf_first = Bytes.toBytes("first");
3209 final byte[] cf_second = Bytes.toBytes("second");
3210
3211 this.region = initHRegion(tableName, getName(), CONF, cf_first, cf_second);
3212 final byte[] col_a = Bytes.toBytes("a");
3213 final byte[] col_b = Bytes.toBytes("b");
3214
3215 Put put;
3216
3217 for (int i = 0; i < 10; i++) {
3218 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
3219 put.add(cf_first, col_a, Bytes.toBytes(i));
3220 if (i < 5) {
3221 put.add(cf_first, col_b, Bytes.toBytes(i));
3222 put.add(cf_second, col_a, Bytes.toBytes(i));
3223 put.add(cf_second, col_b, Bytes.toBytes(i));
3224 }
3225 region.put(put);
3226 }
3227
3228 Scan scan = new Scan();
3229 scan.setLoadColumnFamiliesOnDemand(true);
3230 Filter bogusFilter = new FilterBase() {
3231 @Override
3232 public ReturnCode filterKeyValue(Cell ignored) throws IOException {
3233 return ReturnCode.INCLUDE;
3234 }
3235 @Override
3236 public boolean isFamilyEssential(byte[] name) {
3237 return Bytes.equals(name, cf_first);
3238 }
3239 };
3240
3241 scan.setFilter(bogusFilter);
3242 InternalScanner s = region.getScanner(scan);
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273 List<Cell> results = new ArrayList<Cell>();
3274 int index = 0;
3275 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
3276 while (true) {
3277 boolean more = s.next(results, scannerContext);
3278 if ((index >> 1) < 5) {
3279 if (index % 2 == 0)
3280 assertEquals(results.size(), 3);
3281 else
3282 assertEquals(results.size(), 1);
3283 } else
3284 assertEquals(results.size(), 1);
3285 results.clear();
3286 index++;
3287 if (!more)
3288 break;
3289 }
3290 }
3291
3292
3293
3294
3295
3296
3297 @Test
3298 public void testLongQualifier() throws Exception {
3299 String method = name.getMethodName();
3300 TableName tableName = TableName.valueOf(method);
3301 byte[] family = Bytes.toBytes("family");
3302 this.region = initHRegion(tableName, method, CONF, family);
3303 byte[] q = new byte[Short.MAX_VALUE+2];
3304 Arrays.fill(q, 0, q.length-1, (byte)42);
3305 for (byte i=0; i<10; i++) {
3306 Put p = new Put(Bytes.toBytes("row"));
3307
3308 q[q.length-1]=i;
3309 p.addColumn(family, q, q);
3310 region.put(p);
3311 }
3312 region.flush(false);
3313 }
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323 @Test
3324 public void testBasicSplit() throws Exception {
3325 byte[][] families = { fam1, fam2, fam3 };
3326
3327 Configuration hc = initSplit();
3328
3329 String method = this.getName();
3330 this.region = initHRegion(tableName, method, hc, families);
3331
3332 LOG.info("" + HBaseTestCase.addContent(region, fam3));
3333 region.flush(true);
3334 region.compactStores();
3335 byte[] splitRow = region.checkSplit();
3336 assertNotNull(splitRow);
3337 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3338 HRegion[] regions = splitRegion(region, splitRow);
3339 try {
3340
3341
3342
3343 for (int i = 0; i < regions.length; i++) {
3344 regions[i] = HRegion.openHRegion(regions[i], null);
3345 }
3346
3347
3348 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3349 assertGet(regions[1], fam3, splitRow);
3350
3351 assertScan(regions[0], fam3, Bytes.toBytes(START_KEY));
3352 assertScan(regions[1], fam3, splitRow);
3353
3354 for (int i = 0; i < regions.length; i++) {
3355
3356
3357 for (int j = 0; j < 2; j++) {
3358 HBaseTestCase.addContent(regions[i], fam3);
3359 }
3360 HBaseTestCase.addContent(regions[i], fam2);
3361 HBaseTestCase.addContent(regions[i], fam1);
3362 regions[i].flush(true);
3363 }
3364
3365 byte[][] midkeys = new byte[regions.length][];
3366
3367 for (int i = 0; i < regions.length; i++) {
3368 regions[i].compactStores();
3369 midkeys[i] = regions[i].checkSplit();
3370 }
3371
3372 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3373
3374
3375 for (int i = 0; i < regions.length; i++) {
3376 HRegion[] rs = null;
3377 if (midkeys[i] != null) {
3378 rs = splitRegion(regions[i], midkeys[i]);
3379 for (int j = 0; j < rs.length; j++) {
3380 sortedMap.put(Bytes.toString(rs[j].getRegionInfo().getRegionName()),
3381 HRegion.openHRegion(rs[j], null));
3382 }
3383 }
3384 }
3385 LOG.info("Made 4 regions");
3386
3387
3388 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3389 byte[] b = Bytes.toBytes(START_KEY);
3390 for (HRegion r : sortedMap.values()) {
3391 assertGet(r, fam3, b);
3392 b[0] += interval;
3393 }
3394 } finally {
3395 for (int i = 0; i < regions.length; i++) {
3396 try {
3397 regions[i].close();
3398 } catch (IOException e) {
3399
3400 }
3401 }
3402 }
3403 }
3404
3405 @Test
3406 public void testSplitRegion() throws IOException {
3407 byte[] qualifier = Bytes.toBytes("qualifier");
3408 Configuration hc = initSplit();
3409 int numRows = 10;
3410 byte[][] families = { fam1, fam3 };
3411
3412
3413 String method = this.getName();
3414 this.region = initHRegion(tableName, method, hc, families);
3415
3416
3417 int startRow = 100;
3418 putData(startRow, numRows, qualifier, families);
3419 int splitRow = startRow + numRows;
3420 putData(splitRow, numRows, qualifier, families);
3421 region.flush(true);
3422
3423 HRegion[] regions = null;
3424 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3425
3426 for (int i = 0; i < regions.length; i++) {
3427 regions[i] = HRegion.openHRegion(regions[i], null);
3428 }
3429
3430 assertEquals(2, regions.length);
3431
3432
3433
3434 verifyData(regions[0], startRow, numRows, qualifier, families);
3435 verifyData(regions[1], splitRow, numRows, qualifier, families);
3436 }
3437
3438 @Test
3439 public void testClearForceSplit() throws IOException {
3440 byte[] qualifier = Bytes.toBytes("qualifier");
3441 Configuration hc = initSplit();
3442 int numRows = 10;
3443 byte[][] families = { fam1, fam3 };
3444
3445
3446 String method = this.getName();
3447 this.region = initHRegion(tableName, method, hc, families);
3448
3449
3450 int startRow = 100;
3451 putData(startRow, numRows, qualifier, families);
3452 int splitRow = startRow + numRows;
3453 byte[] splitRowBytes = Bytes.toBytes("" + splitRow);
3454 putData(splitRow, numRows, qualifier, families);
3455 region.flush(true);
3456
3457 HRegion[] regions = null;
3458
3459 region.forceSplit(splitRowBytes);
3460 assertTrue(region.shouldForceSplit());
3461
3462 assertTrue(Bytes.equals(splitRowBytes, region.checkSplit()));
3463
3464
3465 HStore storeMock = Mockito.mock(HStore.class);
3466 when(storeMock.hasReferences()).thenReturn(true);
3467 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
3468 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
3469 when(storeMock.getColumnFamilyName()).thenReturn("cf");
3470 region.stores.put(Bytes.toBytes(storeMock.getColumnFamilyName()), storeMock);
3471 assertTrue(region.hasReferences());
3472
3473
3474 regions = splitRegion(region, splitRowBytes);
3475 assertNull(regions);
3476
3477
3478 assertFalse(region.shouldForceSplit());
3479
3480
3481 region.stores.remove(Bytes.toBytes(storeMock.getColumnFamilyName()));
3482 assertFalse(region.hasReferences());
3483
3484
3485 regions = splitRegion(region, splitRowBytes);
3486
3487
3488 for (int i = 0; i < regions.length; i++) {
3489 regions[i] = HRegion.openHRegion(regions[i], null);
3490 }
3491
3492 assertEquals(2, regions.length);
3493
3494
3495
3496 verifyData(regions[0], startRow, numRows, qualifier, families);
3497 verifyData(regions[1], splitRow, numRows, qualifier, families);
3498 }
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510 @Test
3511 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3512 byte[] family = Bytes.toBytes("family");
3513 int numRows = 1000;
3514 int flushAndScanInterval = 10;
3515 int compactInterval = 10 * flushAndScanInterval;
3516
3517 String method = "testFlushCacheWhileScanning";
3518 this.region = initHRegion(tableName, method, CONF, family);
3519 try {
3520 FlushThread flushThread = new FlushThread();
3521 flushThread.start();
3522
3523 Scan scan = new Scan();
3524 scan.addFamily(family);
3525 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOp.EQUAL,
3526 new BinaryComparator(Bytes.toBytes(5L))));
3527
3528 int expectedCount = 0;
3529 List<Cell> res = new ArrayList<Cell>();
3530
3531 boolean toggle = true;
3532 for (long i = 0; i < numRows; i++) {
3533 Put put = new Put(Bytes.toBytes(i));
3534 put.setDurability(Durability.SKIP_WAL);
3535 put.add(family, qual1, Bytes.toBytes(i % 10));
3536 region.put(put);
3537
3538 if (i != 0 && i % compactInterval == 0) {
3539
3540 region.compact(true);
3541 }
3542
3543 if (i % 10 == 5L) {
3544 expectedCount++;
3545 }
3546
3547 if (i != 0 && i % flushAndScanInterval == 0) {
3548 res.clear();
3549 InternalScanner scanner = region.getScanner(scan);
3550 if (toggle) {
3551 flushThread.flush();
3552 }
3553 while (scanner.next(res))
3554 ;
3555 if (!toggle) {
3556 flushThread.flush();
3557 }
3558 assertEquals("i=" + i, expectedCount, res.size());
3559 toggle = !toggle;
3560 }
3561 }
3562
3563 flushThread.done();
3564 flushThread.join();
3565 flushThread.checkNoError();
3566 } finally {
3567 HRegion.closeHRegion(this.region);
3568 this.region = null;
3569 }
3570 }
3571
3572 protected class FlushThread extends Thread {
3573 private volatile boolean done;
3574 private Throwable error = null;
3575
3576 FlushThread() {
3577 super("FlushThread");
3578 }
3579
3580 public void done() {
3581 done = true;
3582 synchronized (this) {
3583 interrupt();
3584 }
3585 }
3586
3587 public void checkNoError() {
3588 if (error != null) {
3589 assertNull(error);
3590 }
3591 }
3592
3593 @Override
3594 public void run() {
3595 done = false;
3596 while (!done) {
3597 synchronized (this) {
3598 try {
3599 wait();
3600 } catch (InterruptedException ignored) {
3601 if (done) {
3602 break;
3603 }
3604 }
3605 }
3606 try {
3607 region.flush(true);
3608 } catch (IOException e) {
3609 if (!done) {
3610 LOG.error("Error while flushing cache", e);
3611 error = e;
3612 }
3613 break;
3614 } catch (Throwable t) {
3615 LOG.error("Uncaught exception", t);
3616 throw t;
3617 }
3618 }
3619 }
3620
3621 public void flush() {
3622 synchronized (this) {
3623 notify();
3624 }
3625 }
3626 }
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637 @Test
3638 public void testWritesWhileScanning() throws IOException, InterruptedException {
3639 int testCount = 100;
3640 int numRows = 1;
3641 int numFamilies = 10;
3642 int numQualifiers = 100;
3643 int flushInterval = 7;
3644 int compactInterval = 5 * flushInterval;
3645 byte[][] families = new byte[numFamilies][];
3646 for (int i = 0; i < numFamilies; i++) {
3647 families[i] = Bytes.toBytes("family" + i);
3648 }
3649 byte[][] qualifiers = new byte[numQualifiers][];
3650 for (int i = 0; i < numQualifiers; i++) {
3651 qualifiers[i] = Bytes.toBytes("qual" + i);
3652 }
3653
3654 String method = "testWritesWhileScanning";
3655 this.region = initHRegion(tableName, method, CONF, families);
3656 try {
3657 PutThread putThread = new PutThread(numRows, families, qualifiers);
3658 putThread.start();
3659 putThread.waitForFirstPut();
3660
3661 FlushThread flushThread = new FlushThread();
3662 flushThread.start();
3663
3664 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3665
3666 int expectedCount = numFamilies * numQualifiers;
3667 List<Cell> res = new ArrayList<Cell>();
3668
3669 long prevTimestamp = 0L;
3670 for (int i = 0; i < testCount; i++) {
3671
3672 if (i != 0 && i % compactInterval == 0) {
3673 region.compact(true);
3674 for (Store store : region.getStores()) {
3675 store.closeAndArchiveCompactedFiles();
3676 }
3677 }
3678
3679 if (i != 0 && i % flushInterval == 0) {
3680 flushThread.flush();
3681 }
3682
3683 boolean previousEmpty = res.isEmpty();
3684 res.clear();
3685 InternalScanner scanner = region.getScanner(scan);
3686 while (scanner.next(res))
3687 ;
3688 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3689 assertEquals("i=" + i, expectedCount, res.size());
3690 long timestamp = res.get(0).getTimestamp();
3691 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3692 timestamp >= prevTimestamp);
3693 prevTimestamp = timestamp;
3694 }
3695 }
3696
3697 putThread.done();
3698
3699 region.flush(true);
3700
3701 putThread.join();
3702 putThread.checkNoError();
3703
3704 flushThread.done();
3705 flushThread.join();
3706 flushThread.checkNoError();
3707 } finally {
3708 try {
3709 LOG.info("Before close: " + this.region.getMVCC());
3710 HRegion.closeHRegion(this.region);
3711 } catch (DroppedSnapshotException dse) {
3712
3713
3714
3715
3716 }
3717 this.region = null;
3718 }
3719 }
3720
3721 protected class PutThread extends Thread {
3722 private volatile boolean done;
3723 private volatile int numPutsFinished = 0;
3724
3725 private Throwable error = null;
3726 private int numRows;
3727 private byte[][] families;
3728 private byte[][] qualifiers;
3729
3730 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
3731 super("PutThread");
3732 this.numRows = numRows;
3733 this.families = families;
3734 this.qualifiers = qualifiers;
3735 }
3736
3737
3738
3739
3740 public void waitForFirstPut() throws InterruptedException {
3741
3742 while (numPutsFinished == 0) {
3743 checkNoError();
3744 Thread.sleep(50);
3745 }
3746 }
3747
3748 public void done() {
3749 done = true;
3750 synchronized (this) {
3751 interrupt();
3752 }
3753 }
3754
3755 public void checkNoError() {
3756 if (error != null) {
3757 assertNull(error);
3758 }
3759 }
3760
3761 @Override
3762 public void run() {
3763 done = false;
3764 while (!done) {
3765 try {
3766 for (int r = 0; r < numRows; r++) {
3767 byte[] row = Bytes.toBytes("row" + r);
3768 Put put = new Put(row);
3769 put.setDurability(Durability.SKIP_WAL);
3770 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3771 for (byte[] family : families) {
3772 for (byte[] qualifier : qualifiers) {
3773 put.add(family, qualifier, (long) numPutsFinished, value);
3774 }
3775 }
3776 region.put(put);
3777 numPutsFinished++;
3778 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3779 System.out.println("put iteration = " + numPutsFinished);
3780 Delete delete = new Delete(row, (long) numPutsFinished - 30);
3781 region.delete(delete);
3782 }
3783 numPutsFinished++;
3784 }
3785 } catch (InterruptedIOException e) {
3786
3787 LOG.info("Interrupted", e);
3788 } catch (IOException e) {
3789 LOG.error("Error while putting records", e);
3790 error = e;
3791 break;
3792 }
3793 }
3794
3795 }
3796
3797 }
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808 @Test
3809 public void testWritesWhileGetting() throws Exception {
3810 int testCount = 100;
3811 int numRows = 1;
3812 int numFamilies = 10;
3813 int numQualifiers = 100;
3814 int compactInterval = 100;
3815 byte[][] families = new byte[numFamilies][];
3816 for (int i = 0; i < numFamilies; i++) {
3817 families[i] = Bytes.toBytes("family" + i);
3818 }
3819 byte[][] qualifiers = new byte[numQualifiers][];
3820 for (int i = 0; i < numQualifiers; i++) {
3821 qualifiers[i] = Bytes.toBytes("qual" + i);
3822 }
3823
3824
3825 String method = "testWritesWhileGetting";
3826
3827
3828
3829
3830
3831 Configuration conf = HBaseConfiguration.create(CONF);
3832 conf.setInt("hbase.hstore.compaction.min", 1);
3833 conf.setInt("hbase.hstore.compaction.max", 1000);
3834 this.region = initHRegion(tableName, method, conf, families);
3835 PutThread putThread = null;
3836 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
3837 try {
3838 putThread = new PutThread(numRows, families, qualifiers);
3839 putThread.start();
3840 putThread.waitForFirstPut();
3841
3842
3843 ctx.addThread(new RepeatingTestThread(ctx) {
3844
3845 @Override
3846 public void doAnAction() throws Exception {
3847 region.flush(true);
3848
3849
3850 region.compact(false);
3851 for (Store store : region.getStores()) {
3852 store.closeAndArchiveCompactedFiles();
3853 }
3854 }
3855 });
3856 ctx.startThreads();
3857
3858 Get get = new Get(Bytes.toBytes("row0"));
3859 Result result = null;
3860
3861 int expectedCount = numFamilies * numQualifiers;
3862
3863 long prevTimestamp = 0L;
3864 for (int i = 0; i < testCount; i++) {
3865 LOG.info("testWritesWhileGetting verify turn " + i);
3866 boolean previousEmpty = result == null || result.isEmpty();
3867 result = region.get(get);
3868 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3869 assertEquals("i=" + i, expectedCount, result.size());
3870
3871
3872 long timestamp = 0;
3873 for (Cell kv : result.rawCells()) {
3874 if (CellUtil.matchingFamily(kv, families[0])
3875 && CellUtil.matchingQualifier(kv, qualifiers[0])) {
3876 timestamp = kv.getTimestamp();
3877 }
3878 }
3879 assertTrue(timestamp >= prevTimestamp);
3880 prevTimestamp = timestamp;
3881 Cell previousKV = null;
3882
3883 for (Cell kv : result.rawCells()) {
3884 byte[] thisValue = CellUtil.cloneValue(kv);
3885 if (previousKV != null) {
3886 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
3887 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
3888 + "(memStoreTS:" + previousKV.getMvccVersion() + ")" + ", New KV: " + kv
3889 + "(memStoreTS:" + kv.getMvccVersion() + ")");
3890 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
3891 }
3892 }
3893 previousKV = kv;
3894 }
3895 }
3896 }
3897 } finally {
3898 if (putThread != null)
3899 putThread.done();
3900
3901 region.flush(true);
3902
3903 if (putThread != null) {
3904 putThread.join();
3905 putThread.checkNoError();
3906 }
3907
3908 ctx.stop();
3909 HRegion.closeHRegion(this.region);
3910 this.region = null;
3911 }
3912 }
3913
3914 @Test
3915 public void testHolesInMeta() throws Exception {
3916 byte[] family = Bytes.toBytes("family");
3917 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF,
3918 false, family);
3919 byte[] rowNotServed = Bytes.toBytes("a");
3920 Get g = new Get(rowNotServed);
3921 try {
3922 region.get(g);
3923 fail();
3924 } catch (WrongRegionException x) {
3925
3926 }
3927 byte[] row = Bytes.toBytes("y");
3928 g = new Get(row);
3929 region.get(g);
3930 }
3931
3932 @Test
3933 public void testIndexesScanWithOneDeletedRow() throws IOException {
3934 byte[] family = Bytes.toBytes("family");
3935
3936
3937 String method = "testIndexesScanWithOneDeletedRow";
3938 this.region = initHRegion(tableName, method, CONF, family);
3939 Put put = new Put(Bytes.toBytes(1L));
3940 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3941 region.put(put);
3942
3943 region.flush(true);
3944
3945 Delete delete = new Delete(Bytes.toBytes(1L), 1L);
3946 region.delete(delete);
3947
3948 put = new Put(Bytes.toBytes(2L));
3949 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3950 region.put(put);
3951
3952 Scan idxScan = new Scan();
3953 idxScan.addFamily(family);
3954 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList(
3955 new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL,
3956 new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1,
3957 CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L))))));
3958 InternalScanner scanner = region.getScanner(idxScan);
3959 List<Cell> res = new ArrayList<>();
3960
3961 while (scanner.next(res)) {
3962
3963 }
3964 assertEquals(1L, res.size());
3965 }
3966
3967
3968
3969
3970 @Test
3971 public void testBloomFilterSize() throws IOException {
3972 byte[] fam1 = Bytes.toBytes("fam1");
3973 byte[] qf1 = Bytes.toBytes("col");
3974 byte[] val1 = Bytes.toBytes("value1");
3975
3976 HColumnDescriptor hcd = new HColumnDescriptor(fam1).setMaxVersions(Integer.MAX_VALUE)
3977 .setBloomFilterType(BloomType.ROWCOL);
3978
3979 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
3980 htd.addFamily(hcd);
3981 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
3982 this.region = TEST_UTIL.createLocalHRegion(info, htd);
3983 int num_unique_rows = 10;
3984 int duplicate_multiplier = 2;
3985 int num_storefiles = 4;
3986
3987 int version = 0;
3988 for (int f = 0; f < num_storefiles; f++) {
3989 for (int i = 0; i < duplicate_multiplier; i++) {
3990 for (int j = 0; j < num_unique_rows; j++) {
3991 Put put = new Put(Bytes.toBytes("row" + j));
3992 put.setDurability(Durability.SKIP_WAL);
3993 put.add(fam1, qf1, version++, val1);
3994 region.put(put);
3995 }
3996 }
3997 region.flush(true);
3998 }
3999
4000 HStore store = (HStore) region.getStore(fam1);
4001 Collection<StoreFile> storeFiles = store.getStorefiles();
4002 for (StoreFile storefile : storeFiles) {
4003 StoreFile.Reader reader = storefile.getReader();
4004 reader.loadFileInfo();
4005 reader.loadBloomfilter();
4006 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4007 assertEquals(num_unique_rows, reader.getFilterEntries());
4008 }
4009
4010 region.compact(true);
4011
4012
4013 storeFiles = store.getStorefiles();
4014 for (StoreFile storefile : storeFiles) {
4015 StoreFile.Reader reader = storefile.getReader();
4016 reader.loadFileInfo();
4017 reader.loadBloomfilter();
4018 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4019 assertEquals(num_unique_rows, reader.getFilterEntries());
4020 }
4021 }
4022
4023 @Test
4024 public void testAllColumnsWithBloomFilter() throws IOException {
4025 byte[] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
4026 byte[] FAMILY = Bytes.toBytes("family");
4027
4028
4029 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(Integer.MAX_VALUE)
4030 .setBloomFilterType(BloomType.ROWCOL);
4031 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
4032 htd.addFamily(hcd);
4033 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4034 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4035
4036 byte row[] = Bytes.toBytes("row:" + 0);
4037 byte column[] = Bytes.toBytes("column:" + 0);
4038 Put put = new Put(row);
4039 put.setDurability(Durability.SKIP_WAL);
4040 for (long idx = 1; idx <= 4; idx++) {
4041 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4042 }
4043 region.put(put);
4044
4045
4046 region.flush(true);
4047
4048
4049 Get get = new Get(row);
4050 get.setMaxVersions();
4051 Cell[] kvs = region.get(get).rawCells();
4052
4053
4054 assertEquals(4, kvs.length);
4055 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4056 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4057 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4058 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4059 }
4060
4061
4062
4063
4064
4065
4066 @Test
4067 public void testDeleteRowWithBloomFilter() throws IOException {
4068 byte[] familyName = Bytes.toBytes("familyName");
4069
4070
4071 HColumnDescriptor hcd = new HColumnDescriptor(familyName).setMaxVersions(Integer.MAX_VALUE)
4072 .setBloomFilterType(BloomType.ROWCOL);
4073
4074 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
4075 htd.addFamily(hcd);
4076 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
4077 this.region = TEST_UTIL.createLocalHRegion(info, htd);
4078
4079 byte row[] = Bytes.toBytes("row1");
4080 byte col[] = Bytes.toBytes("col1");
4081
4082 Put put = new Put(row);
4083 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
4084 region.put(put);
4085 region.flush(true);
4086
4087 Delete del = new Delete(row);
4088 region.delete(del);
4089 region.flush(true);
4090
4091
4092 Get get = new Get(row);
4093 get.addColumn(familyName, col);
4094
4095 Cell[] keyValues = region.get(get).rawCells();
4096 assertEquals(0,keyValues.length);
4097 }
4098
4099 @Test
4100 public void testgetHDFSBlocksDistribution() throws Exception {
4101 HBaseTestingUtility htu = new HBaseTestingUtility();
4102
4103
4104
4105
4106
4107
4108
4109 htu.getConfiguration().setInt("dfs.replication", 2);
4110
4111
4112 MiniHBaseCluster cluster = null;
4113 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
4114 int regionServersCount = 3;
4115
4116 try {
4117 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
4118 byte[][] families = { fam1, fam2 };
4119 Table ht = htu.createTable(Bytes.toBytes(this.getName()), families);
4120
4121
4122 byte row[] = Bytes.toBytes("row1");
4123 byte col[] = Bytes.toBytes("col1");
4124
4125 Put put = new Put(row);
4126 put.add(fam1, col, 1, Bytes.toBytes("test1"));
4127 put.add(fam2, col, 1, Bytes.toBytes("test2"));
4128 ht.put(put);
4129
4130 HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName()))
4131 .get(0);
4132 firstRegion.flush(true);
4133 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
4134
4135
4136
4137
4138
4139 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
4140 StringBuilder sb = new StringBuilder();
4141 for (String host: blocksDistribution1.getTopHosts()) {
4142 if (sb.length() > 0) sb.append(", ");
4143 sb.append(host);
4144 sb.append("=");
4145 sb.append(blocksDistribution1.getWeight(host));
4146 }
4147
4148 String topHost = blocksDistribution1.getTopHosts().get(0);
4149 long topHostWeight = blocksDistribution1.getWeight(topHost);
4150 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" +
4151 topHostWeight + ", topHost=" + topHost + "; " + sb.toString();
4152 LOG.info(msg);
4153 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
4154
4155
4156
4157 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
4158 htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo());
4159 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
4160
4161 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
4162
4163 ht.close();
4164 } finally {
4165 if (cluster != null) {
4166 htu.shutdownMiniCluster();
4167 }
4168 }
4169 }
4170
4171
4172
4173
4174
4175
4176
4177 @Test
4178 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
4179 TableName tableName = TableName.valueOf(name.getMethodName());
4180 HRegionInfo info = null;
4181 try {
4182 FileSystem fs = Mockito.mock(FileSystem.class);
4183 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
4184 HTableDescriptor htd = new HTableDescriptor(tableName);
4185 htd.addFamily(new HColumnDescriptor("cf"));
4186 info = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY,
4187 HConstants.EMPTY_BYTE_ARRAY, false);
4188 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
4189 region = HRegion.newHRegion(path, null, fs, CONF, info, htd, null);
4190
4191 region.initialize();
4192 fail("Region initialization should fail due to IOException");
4193 } catch (IOException io) {
4194 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
4195 for (MonitoredTask monitoredTask : tasks) {
4196 if (!(monitoredTask instanceof MonitoredRPCHandler)
4197 && monitoredTask.getDescription().contains(region.toString())) {
4198 assertTrue("Region state should be ABORTED.",
4199 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
4200 break;
4201 }
4202 }
4203 }
4204 }
4205
4206
4207
4208
4209
4210 @Test
4211 public void testRegionInfoFileCreation() throws IOException {
4212 Path rootDir = new Path(dir + "testRegionInfoFileCreation");
4213
4214 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testtb"));
4215 htd.addFamily(new HColumnDescriptor("cf"));
4216
4217 HRegionInfo hri = new HRegionInfo(htd.getTableName());
4218
4219
4220 region = HRegion.createHRegion(hri, rootDir, CONF, htd, null, false, true);
4221
4222 Path regionDir = region.getRegionFileSystem().getRegionDir();
4223 FileSystem fs = region.getRegionFileSystem().getFileSystem();
4224 HRegion.closeHRegion(region);
4225
4226 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
4227
4228
4229 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4230 fs.exists(regionInfoFile));
4231
4232
4233 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4234 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4235 HRegion.closeHRegion(region);
4236
4237
4238 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4239 fs.exists(regionInfoFile));
4240
4241
4242 fs.delete(regionInfoFile, true);
4243 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
4244 fs.exists(regionInfoFile));
4245
4246 region = HRegion.openHRegion(rootDir, hri, htd, null, CONF);
4247
4248 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
4249 HRegion.closeHRegion(region);
4250
4251
4252 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
4253 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
4254
4255 region = null;
4256 }
4257
4258
4259
4260
4261 private static class Incrementer implements Runnable {
4262 private HRegion region;
4263 private final static byte[] incRow = Bytes.toBytes("incRow");
4264 private final static byte[] family = Bytes.toBytes("family");
4265 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4266 private final static long ONE = 1l;
4267 private int incCounter;
4268
4269 public Incrementer(HRegion region, int incCounter) {
4270 this.region = region;
4271 this.incCounter = incCounter;
4272 }
4273
4274 @Override
4275 public void run() {
4276 int count = 0;
4277 while (count < incCounter) {
4278 Increment inc = new Increment(incRow);
4279 inc.addColumn(family, qualifier, ONE);
4280 count++;
4281 try {
4282 region.increment(inc);
4283 } catch (IOException e) {
4284 e.printStackTrace();
4285 break;
4286 }
4287 }
4288 }
4289 }
4290
4291
4292
4293
4294
4295 @Test
4296 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4297 byte[] family = Incrementer.family;
4298 this.region = initHRegion(tableName, method, CONF, family);
4299 final HRegion region = this.region;
4300 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4301 Runnable flusher = new Runnable() {
4302 @Override
4303 public void run() {
4304 while (!incrementDone.get()) {
4305 try {
4306 region.flush(true);
4307 } catch (Exception e) {
4308 e.printStackTrace();
4309 }
4310 }
4311 }
4312 };
4313
4314
4315 int threadNum = 20;
4316 int incCounter = 100;
4317 long expected = threadNum * incCounter;
4318 Thread[] incrementers = new Thread[threadNum];
4319 Thread flushThread = new Thread(flusher);
4320 for (int i = 0; i < threadNum; i++) {
4321 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4322 incrementers[i].start();
4323 }
4324 flushThread.start();
4325 for (int i = 0; i < threadNum; i++) {
4326 incrementers[i].join();
4327 }
4328
4329 incrementDone.set(true);
4330 flushThread.join();
4331
4332 Get get = new Get(Incrementer.incRow);
4333 get.addColumn(Incrementer.family, Incrementer.qualifier);
4334 get.setMaxVersions(1);
4335 Result res = this.region.get(get);
4336 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
4337
4338
4339 assertEquals(kvs.size(), 1);
4340 Cell kv = kvs.get(0);
4341 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
4342 }
4343
4344
4345
4346
4347 private static class Appender implements Runnable {
4348 private HRegion region;
4349 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4350 private final static byte[] family = Bytes.toBytes("family");
4351 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4352 private final static byte[] CHAR = Bytes.toBytes("a");
4353 private int appendCounter;
4354
4355 public Appender(HRegion region, int appendCounter) {
4356 this.region = region;
4357 this.appendCounter = appendCounter;
4358 }
4359
4360 @Override
4361 public void run() {
4362 int count = 0;
4363 while (count < appendCounter) {
4364 Append app = new Append(appendRow);
4365 app.add(family, qualifier, CHAR);
4366 count++;
4367 try {
4368 region.append(app);
4369 } catch (IOException e) {
4370 e.printStackTrace();
4371 break;
4372 }
4373 }
4374 }
4375 }
4376
4377
4378
4379
4380
4381 @Test
4382 public void testParallelAppendWithMemStoreFlush() throws Exception {
4383 byte[] family = Appender.family;
4384 this.region = initHRegion(tableName, method, CONF, family);
4385 final HRegion region = this.region;
4386 final AtomicBoolean appendDone = new AtomicBoolean(false);
4387 Runnable flusher = new Runnable() {
4388 @Override
4389 public void run() {
4390 while (!appendDone.get()) {
4391 try {
4392 region.flush(true);
4393 } catch (Exception e) {
4394 e.printStackTrace();
4395 }
4396 }
4397 }
4398 };
4399
4400
4401
4402 int threadNum = 20;
4403 int appendCounter = 100;
4404 byte[] expected = new byte[threadNum * appendCounter];
4405 for (int i = 0; i < threadNum * appendCounter; i++) {
4406 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4407 }
4408 Thread[] appenders = new Thread[threadNum];
4409 Thread flushThread = new Thread(flusher);
4410 for (int i = 0; i < threadNum; i++) {
4411 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4412 appenders[i].start();
4413 }
4414 flushThread.start();
4415 for (int i = 0; i < threadNum; i++) {
4416 appenders[i].join();
4417 }
4418
4419 appendDone.set(true);
4420 flushThread.join();
4421
4422 Get get = new Get(Appender.appendRow);
4423 get.addColumn(Appender.family, Appender.qualifier);
4424 get.setMaxVersions(1);
4425 Result res = this.region.get(get);
4426 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
4427
4428
4429 assertEquals(kvs.size(), 1);
4430 Cell kv = kvs.get(0);
4431 byte[] appendResult = new byte[kv.getValueLength()];
4432 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4433 assertArrayEquals(expected, appendResult);
4434 }
4435
4436
4437
4438
4439
4440 @Test
4441 public void testPutWithMemStoreFlush() throws Exception {
4442 byte[] family = Bytes.toBytes("family");
4443 ;
4444 byte[] qualifier = Bytes.toBytes("qualifier");
4445 byte[] row = Bytes.toBytes("putRow");
4446 byte[] value = null;
4447 this.region = initHRegion(tableName, method, CONF, family);
4448 Put put = null;
4449 Get get = null;
4450 List<Cell> kvs = null;
4451 Result res = null;
4452
4453 put = new Put(row);
4454 value = Bytes.toBytes("value0");
4455 put.add(family, qualifier, 1234567l, value);
4456 region.put(put);
4457 get = new Get(row);
4458 get.addColumn(family, qualifier);
4459 get.setMaxVersions();
4460 res = this.region.get(get);
4461 kvs = res.getColumnCells(family, qualifier);
4462 assertEquals(1, kvs.size());
4463 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4464
4465 region.flush(true);
4466 get = new Get(row);
4467 get.addColumn(family, qualifier);
4468 get.setMaxVersions();
4469 res = this.region.get(get);
4470 kvs = res.getColumnCells(family, qualifier);
4471 assertEquals(1, kvs.size());
4472 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
4473
4474 put = new Put(row);
4475 value = Bytes.toBytes("value1");
4476 put.add(family, qualifier, 1234567l, value);
4477 region.put(put);
4478 get = new Get(row);
4479 get.addColumn(family, qualifier);
4480 get.setMaxVersions();
4481 res = this.region.get(get);
4482 kvs = res.getColumnCells(family, qualifier);
4483 assertEquals(1, kvs.size());
4484 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4485
4486 region.flush(true);
4487 get = new Get(row);
4488 get.addColumn(family, qualifier);
4489 get.setMaxVersions();
4490 res = this.region.get(get);
4491 kvs = res.getColumnCells(family, qualifier);
4492 assertEquals(1, kvs.size());
4493 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
4494 }
4495
4496 @Test
4497 public void testDurability() throws Exception {
4498 String method = "testDurability";
4499
4500
4501
4502
4503
4504 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4505 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4506 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4507
4508 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4509 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4510 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
4511
4512 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
4513 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4514
4515 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
4516 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
4517
4518 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
4519 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
4520 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
4521
4522
4523 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4524 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4525 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4526 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
4527 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
4528 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4529
4530 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4531 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4532 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4533 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
4534 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
4535 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
4536
4537
4538 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4539 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4540 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, true, false, false);
4541 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, true, false, false);
4542 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, true, false, false);
4543 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, true, false, false);
4544
4545 }
4546
4547 private void durabilityTest(String method, Durability tableDurability,
4548 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
4549 final boolean expectSyncFromLogSyncer) throws Exception {
4550 Configuration conf = HBaseConfiguration.create(CONF);
4551 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
4552 TableName tableName = TableName.valueOf(method);
4553 byte[] family = Bytes.toBytes("family");
4554 Path logDir = new Path(new Path(dir + method), "log");
4555 final Configuration walConf = new Configuration(conf);
4556 FSUtils.setRootDir(walConf, logDir);
4557 final WALFactory wals = new WALFactory(walConf, null, UUID.randomUUID().toString());
4558 final WAL wal = spy(wals.getWAL(tableName.getName(), tableName.getNamespace()));
4559 this.region = initHRegion(tableName.getName(), HConstants.EMPTY_START_ROW,
4560 HConstants.EMPTY_END_ROW, method, conf, false, tableDurability, wal,
4561 new byte[][] { family });
4562
4563 Put put = new Put(Bytes.toBytes("r1"));
4564 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
4565 put.setDurability(mutationDurability);
4566 region.put(put);
4567
4568
4569 verify(wal, expectAppend ? times(1) : never())
4570 .append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
4571 (WALEdit)any(), Mockito.anyBoolean());
4572
4573
4574 if (expectSync || expectSyncFromLogSyncer) {
4575 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
4576 @Override
4577 public boolean evaluate() throws Exception {
4578 try {
4579 if (expectSync) {
4580 verify(wal, times(1)).sync(anyLong());
4581 } else if (expectSyncFromLogSyncer) {
4582 verify(wal, times(1)).sync();
4583 }
4584 } catch (Throwable ignore) {
4585 }
4586 return true;
4587 }
4588 });
4589 } else {
4590
4591 verify(wal, never()).sync();
4592 }
4593
4594 HRegion.closeHRegion(this.region);
4595 this.region = null;
4596 }
4597
4598 @Test
4599 public void testRegionReplicaSecondary() throws IOException {
4600
4601
4602 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4603 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4604
4605 byte[][] families = new byte[][] {
4606 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4607 };
4608 byte[] cq = Bytes.toBytes("cq");
4609 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4610 for (byte[] family : families) {
4611 htd.addFamily(new HColumnDescriptor(family));
4612 }
4613
4614 long time = System.currentTimeMillis();
4615 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4616 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4617 false, time, 0);
4618 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4619 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4620 false, time, 1);
4621
4622 HRegion primaryRegion = null, secondaryRegion = null;
4623
4624 try {
4625 primaryRegion = HRegion.createHRegion(primaryHri,
4626 rootDir, TEST_UTIL.getConfiguration(), htd);
4627
4628
4629 putData(primaryRegion, 0, 1000, cq, families);
4630
4631
4632 primaryRegion.flush(true);
4633
4634
4635 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4636
4637 verifyData(secondaryRegion, 0, 1000, cq, families);
4638 } finally {
4639 if (primaryRegion != null) {
4640 HRegion.closeHRegion(primaryRegion);
4641 }
4642 if (secondaryRegion != null) {
4643 HRegion.closeHRegion(secondaryRegion);
4644 }
4645 }
4646 }
4647
4648 @Test
4649 public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
4650
4651
4652 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4653 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4654
4655 byte[][] families = new byte[][] {
4656 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4657 };
4658 byte[] cq = Bytes.toBytes("cq");
4659 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4660 for (byte[] family : families) {
4661 htd.addFamily(new HColumnDescriptor(family));
4662 }
4663
4664 long time = System.currentTimeMillis();
4665 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4666 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4667 false, time, 0);
4668 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4669 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4670 false, time, 1);
4671
4672 HRegion primaryRegion = null, secondaryRegion = null;
4673
4674 try {
4675 primaryRegion = HRegion.createHRegion(primaryHri,
4676 rootDir, TEST_UTIL.getConfiguration(), htd);
4677
4678
4679 putData(primaryRegion, 0, 1000, cq, families);
4680
4681
4682 primaryRegion.flush(true);
4683
4684
4685 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4686
4687 try {
4688 putData(secondaryRegion, 0, 1000, cq, families);
4689 fail("Should have thrown exception");
4690 } catch (IOException ex) {
4691
4692 }
4693 } finally {
4694 if (primaryRegion != null) {
4695 HRegion.closeHRegion(primaryRegion);
4696 }
4697 if (secondaryRegion != null) {
4698 HRegion.closeHRegion(secondaryRegion);
4699 }
4700 }
4701 }
4702
4703 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
4704 Configuration confForWAL = new Configuration(conf);
4705 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
4706 return new WALFactory(confForWAL,
4707 Collections.<WALActionsListener>singletonList(new MetricsWAL()),
4708 "hregion-" + RandomStringUtils.randomNumeric(8));
4709 }
4710
4711 @Test
4712 public void testCompactionFromPrimary() throws IOException {
4713 Path rootDir = new Path(dir + "testRegionReplicaSecondary");
4714 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
4715
4716 byte[][] families = new byte[][] {
4717 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
4718 };
4719 byte[] cq = Bytes.toBytes("cq");
4720 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
4721 for (byte[] family : families) {
4722 htd.addFamily(new HColumnDescriptor(family));
4723 }
4724
4725 long time = System.currentTimeMillis();
4726 HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
4727 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4728 false, time, 0);
4729 HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
4730 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
4731 false, time, 1);
4732
4733 HRegion primaryRegion = null, secondaryRegion = null;
4734
4735 try {
4736 primaryRegion = HRegion.createHRegion(primaryHri,
4737 rootDir, TEST_UTIL.getConfiguration(), htd);
4738
4739
4740 putData(primaryRegion, 0, 1000, cq, families);
4741
4742
4743 primaryRegion.flush(true);
4744
4745
4746 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);
4747
4748
4749 Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
4750 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
4751 Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
4752 Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);
4753
4754 verifyData(secondaryRegion, 0, 1000, cq, families);
4755 } finally {
4756 if (primaryRegion != null) {
4757 HRegion.closeHRegion(primaryRegion);
4758 }
4759 if (secondaryRegion != null) {
4760 HRegion.closeHRegion(secondaryRegion);
4761 }
4762 }
4763 }
4764
4765 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4766 putData(this.region, startRow, numRows, qf, families);
4767 }
4768
4769 private void putData(HRegion region,
4770 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4771 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
4772 }
4773
4774 static void putData(HRegion region, Durability durability,
4775 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
4776 for (int i = startRow; i < startRow + numRows; i++) {
4777 Put put = new Put(Bytes.toBytes("" + i));
4778 put.setDurability(durability);
4779 for (byte[] family : families) {
4780 put.add(family, qf, null);
4781 }
4782 region.put(put);
4783 }
4784 }
4785
4786 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
4787 throws IOException {
4788 for (int i = startRow; i < startRow + numRows; i++) {
4789 byte[] row = Bytes.toBytes("" + i);
4790 Get get = new Get(row);
4791 for (byte[] family : families) {
4792 get.addColumn(family, qf);
4793 }
4794 Result result = newReg.get(get);
4795 Cell[] raw = result.rawCells();
4796 assertEquals(families.length, result.size());
4797 for (int j = 0; j < families.length; j++) {
4798 assertTrue(CellUtil.matchingRow(raw[j], row));
4799 assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
4800 assertTrue(CellUtil.matchingQualifier(raw[j], qf));
4801 }
4802 }
4803 }
4804
4805 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
4806
4807 Get get = new Get(k).addFamily(family).setMaxVersions();
4808 Cell[] results = r.get(get).rawCells();
4809 for (int j = 0; j < results.length; j++) {
4810 byte[] tmp = CellUtil.cloneValue(results[j]);
4811
4812 assertTrue(Bytes.equals(k, tmp));
4813 }
4814 }
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827 private void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
4828 throws IOException {
4829 byte[][] families = { fs };
4830 Scan scan = new Scan();
4831 for (int i = 0; i < families.length; i++)
4832 scan.addFamily(families[i]);
4833 InternalScanner s = r.getScanner(scan);
4834 try {
4835 List<Cell> curVals = new ArrayList<Cell>();
4836 boolean first = true;
4837 OUTER_LOOP: while (s.next(curVals)) {
4838 for (Cell kv : curVals) {
4839 byte[] val = CellUtil.cloneValue(kv);
4840 byte[] curval = val;
4841 if (first) {
4842 first = false;
4843 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4844 } else {
4845
4846 break OUTER_LOOP;
4847 }
4848 }
4849 }
4850 } finally {
4851 s.close();
4852 }
4853 }
4854
4855
4856
4857
4858
4859 @Test
4860 public void testFlushResult() throws IOException {
4861 String method = name.getMethodName();
4862 byte[] tableName = Bytes.toBytes(method);
4863 byte[] family = Bytes.toBytes("family");
4864
4865 this.region = initHRegion(tableName, method, family);
4866
4867
4868 HRegion.FlushResult fr = region.flush(true);
4869 assertFalse(fr.isFlushSucceeded());
4870 assertFalse(fr.isCompactionNeeded());
4871
4872
4873 for (int i = 0; i < 2; i++) {
4874 Put put = new Put(tableName).add(family, family, tableName);
4875 region.put(put);
4876 fr = region.flush(true);
4877 assertTrue(fr.isFlushSucceeded());
4878 assertFalse(fr.isCompactionNeeded());
4879 }
4880
4881
4882 for (int i = 0; i < 2; i++) {
4883 Put put = new Put(tableName).add(family, family, tableName);
4884 region.put(put);
4885 fr = region.flush(true);
4886 assertTrue(fr.isFlushSucceeded());
4887 assertTrue(fr.isCompactionNeeded());
4888 }
4889 }
4890
4891 private Configuration initSplit() {
4892
4893 CONF.setInt("hbase.hstore.compactionThreshold", 2);
4894
4895
4896 CONF.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4897
4898 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
4899
4900
4901 CONF.setLong("hbase.client.pause", 15 * 1000);
4902
4903
4904
4905 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4906 return CONF;
4907 }
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918 public static HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
4919 byte[]... families) throws IOException {
4920 return initHRegion(tableName.getName(), null, null, callingMethod, conf, false, families);
4921 }
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4933 byte[]... families) throws IOException {
4934 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4935 }
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947 public static HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
4948 boolean isReadOnly, byte[]... families) throws IOException {
4949 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4950 }
4951
4952 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4953 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4954 throws IOException {
4955 return initHRegion(tableName, startKey, stopKey, callingMethod, conf, isReadOnly,
4956 Durability.SYNC_WAL, null, families);
4957 }
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971 public static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4972 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
4973 WAL wal, byte[]... families) throws IOException {
4974 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
4975 isReadOnly, durability, wal, families);
4976 }
4977
4978
4979
4980
4981
4982 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
4983 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4984 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
4985 Bytes.toString(CellUtil.cloneRow(kv)));
4986 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
4987 Bytes.toString(CellUtil.cloneFamily(kv)));
4988 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
4989 Bytes.toString(CellUtil.cloneQualifier(kv)));
4990 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
4991 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
4992 Bytes.toString(CellUtil.cloneValue(kv)));
4993 }
4994
4995 @Test (timeout=60000)
4996 public void testReverseScanner_FromMemStore_SingleCF_Normal()
4997 throws IOException {
4998 byte[] rowC = Bytes.toBytes("rowC");
4999 byte[] rowA = Bytes.toBytes("rowA");
5000 byte[] rowB = Bytes.toBytes("rowB");
5001 byte[] cf = Bytes.toBytes("CF");
5002 byte[][] families = { cf };
5003 byte[] col = Bytes.toBytes("C");
5004 long ts = 1;
5005 String method = this.getName();
5006 this.region = initHRegion(tableName, method, families);
5007 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5008 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5009 null);
5010 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5011 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5012 Put put = null;
5013 put = new Put(rowC);
5014 put.add(kv1);
5015 put.add(kv11);
5016 region.put(put);
5017 put = new Put(rowA);
5018 put.add(kv2);
5019 region.put(put);
5020 put = new Put(rowB);
5021 put.add(kv3);
5022 region.put(put);
5023
5024 Scan scan = new Scan(rowC);
5025 scan.setMaxVersions(5);
5026 scan.setReversed(true);
5027 InternalScanner scanner = region.getScanner(scan);
5028 List<Cell> currRow = new ArrayList<Cell>();
5029 boolean hasNext = scanner.next(currRow);
5030 assertEquals(2, currRow.size());
5031 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5032 assertTrue(hasNext);
5033 currRow.clear();
5034 hasNext = scanner.next(currRow);
5035 assertEquals(1, currRow.size());
5036 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5037 assertTrue(hasNext);
5038 currRow.clear();
5039 hasNext = scanner.next(currRow);
5040 assertEquals(1, currRow.size());
5041 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5042 assertFalse(hasNext);
5043 scanner.close();
5044 }
5045
5046 @Test (timeout=60000)
5047 public void testReverseScanner_FromMemStore_SingleCF_LargerKey()
5048 throws IOException {
5049 byte[] rowC = Bytes.toBytes("rowC");
5050 byte[] rowA = Bytes.toBytes("rowA");
5051 byte[] rowB = Bytes.toBytes("rowB");
5052 byte[] rowD = Bytes.toBytes("rowD");
5053 byte[] cf = Bytes.toBytes("CF");
5054 byte[][] families = { cf };
5055 byte[] col = Bytes.toBytes("C");
5056 long ts = 1;
5057 String method = this.getName();
5058 this.region = initHRegion(tableName, method, families);
5059 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5060 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5061 null);
5062 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5063 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5064 Put put = null;
5065 put = new Put(rowC);
5066 put.add(kv1);
5067 put.add(kv11);
5068 region.put(put);
5069 put = new Put(rowA);
5070 put.add(kv2);
5071 region.put(put);
5072 put = new Put(rowB);
5073 put.add(kv3);
5074 region.put(put);
5075
5076 Scan scan = new Scan(rowD);
5077 List<Cell> currRow = new ArrayList<Cell>();
5078 scan.setReversed(true);
5079 scan.setMaxVersions(5);
5080 InternalScanner scanner = region.getScanner(scan);
5081 boolean hasNext = scanner.next(currRow);
5082 assertEquals(2, currRow.size());
5083 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5084 assertTrue(hasNext);
5085 currRow.clear();
5086 hasNext = scanner.next(currRow);
5087 assertEquals(1, currRow.size());
5088 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5089 assertTrue(hasNext);
5090 currRow.clear();
5091 hasNext = scanner.next(currRow);
5092 assertEquals(1, currRow.size());
5093 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5094 assertFalse(hasNext);
5095 scanner.close();
5096 }
5097
5098 @Test (timeout=60000)
5099 public void testReverseScanner_FromMemStore_SingleCF_FullScan()
5100 throws IOException {
5101 byte[] rowC = Bytes.toBytes("rowC");
5102 byte[] rowA = Bytes.toBytes("rowA");
5103 byte[] rowB = Bytes.toBytes("rowB");
5104 byte[] cf = Bytes.toBytes("CF");
5105 byte[][] families = { cf };
5106 byte[] col = Bytes.toBytes("C");
5107 long ts = 1;
5108 String method = this.getName();
5109 this.region = initHRegion(tableName, method, families);
5110 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5111 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put,
5112 null);
5113 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5114 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5115 Put put = null;
5116 put = new Put(rowC);
5117 put.add(kv1);
5118 put.add(kv11);
5119 region.put(put);
5120 put = new Put(rowA);
5121 put.add(kv2);
5122 region.put(put);
5123 put = new Put(rowB);
5124 put.add(kv3);
5125 region.put(put);
5126 Scan scan = new Scan();
5127 List<Cell> currRow = new ArrayList<Cell>();
5128 scan.setReversed(true);
5129 InternalScanner scanner = region.getScanner(scan);
5130 boolean hasNext = scanner.next(currRow);
5131 assertEquals(1, currRow.size());
5132 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5133 assertTrue(hasNext);
5134 currRow.clear();
5135 hasNext = scanner.next(currRow);
5136 assertEquals(1, currRow.size());
5137 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5138 assertTrue(hasNext);
5139 currRow.clear();
5140 hasNext = scanner.next(currRow);
5141 assertEquals(1, currRow.size());
5142 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowA));
5143 assertFalse(hasNext);
5144 scanner.close();
5145 }
5146
5147 @Test (timeout=60000)
5148 public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
5149
5150 byte[] rowA = Bytes.toBytes("rowA");
5151 byte[] rowB = Bytes.toBytes("rowB");
5152 byte[] rowC = Bytes.toBytes("rowC");
5153 byte[] rowD = Bytes.toBytes("rowD");
5154 byte[] rowE = Bytes.toBytes("rowE");
5155 byte[] cf = Bytes.toBytes("CF");
5156 byte[][] families = { cf };
5157 byte[] col1 = Bytes.toBytes("col1");
5158 byte[] col2 = Bytes.toBytes("col2");
5159 long ts = 1;
5160 String method = this.getName();
5161 this.region = initHRegion(tableName, method, families);
5162 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5163 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5164 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5165 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5166 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5167 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5168 Put put = null;
5169 put = new Put(rowA);
5170 put.add(kv1);
5171 region.put(put);
5172 put = new Put(rowB);
5173 put.add(kv2);
5174 region.put(put);
5175 put = new Put(rowC);
5176 put.add(kv3);
5177 region.put(put);
5178 put = new Put(rowD);
5179 put.add(kv4_1);
5180 region.put(put);
5181 put = new Put(rowD);
5182 put.add(kv4_2);
5183 region.put(put);
5184 put = new Put(rowE);
5185 put.add(kv5);
5186 region.put(put);
5187 region.flush(true);
5188 Scan scan = new Scan(rowD, rowA);
5189 scan.addColumn(families[0], col1);
5190 scan.setReversed(true);
5191 List<Cell> currRow = new ArrayList<Cell>();
5192 InternalScanner scanner = region.getScanner(scan);
5193 boolean hasNext = scanner.next(currRow);
5194 assertEquals(1, currRow.size());
5195 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5196 assertTrue(hasNext);
5197 currRow.clear();
5198 hasNext = scanner.next(currRow);
5199 assertEquals(1, currRow.size());
5200 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5201 assertTrue(hasNext);
5202 currRow.clear();
5203 hasNext = scanner.next(currRow);
5204 assertEquals(1, currRow.size());
5205 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5206 assertFalse(hasNext);
5207 scanner.close();
5208
5209 scan = new Scan(rowD, rowA);
5210 scan.addColumn(families[0], col2);
5211 scan.setReversed(true);
5212 currRow.clear();
5213 scanner = region.getScanner(scan);
5214 hasNext = scanner.next(currRow);
5215 assertEquals(1, currRow.size());
5216 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5217 scanner.close();
5218 }
5219
5220 @Test (timeout=60000)
5221 public void testReverseScanner_smaller_blocksize() throws IOException {
5222
5223 byte[] rowA = Bytes.toBytes("rowA");
5224 byte[] rowB = Bytes.toBytes("rowB");
5225 byte[] rowC = Bytes.toBytes("rowC");
5226 byte[] rowD = Bytes.toBytes("rowD");
5227 byte[] rowE = Bytes.toBytes("rowE");
5228 byte[] cf = Bytes.toBytes("CF");
5229 byte[][] families = { cf };
5230 byte[] col1 = Bytes.toBytes("col1");
5231 byte[] col2 = Bytes.toBytes("col2");
5232 long ts = 1;
5233 String method = this.getName();
5234 HBaseConfiguration config = new HBaseConfiguration();
5235 config.setInt("test.block.size", 1);
5236 this.region = initHRegion(tableName, method, config, families);
5237 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
5238 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
5239 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
5240 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
5241 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
5242 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
5243 Put put = null;
5244 put = new Put(rowA);
5245 put.add(kv1);
5246 region.put(put);
5247 put = new Put(rowB);
5248 put.add(kv2);
5249 region.put(put);
5250 put = new Put(rowC);
5251 put.add(kv3);
5252 region.put(put);
5253 put = new Put(rowD);
5254 put.add(kv4_1);
5255 region.put(put);
5256 put = new Put(rowD);
5257 put.add(kv4_2);
5258 region.put(put);
5259 put = new Put(rowE);
5260 put.add(kv5);
5261 region.put(put);
5262 region.flush(true);
5263 Scan scan = new Scan(rowD, rowA);
5264 scan.addColumn(families[0], col1);
5265 scan.setReversed(true);
5266 List<Cell> currRow = new ArrayList<Cell>();
5267 InternalScanner scanner = region.getScanner(scan);
5268 boolean hasNext = scanner.next(currRow);
5269 assertEquals(1, currRow.size());
5270 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5271 assertTrue(hasNext);
5272 currRow.clear();
5273 hasNext = scanner.next(currRow);
5274 assertEquals(1, currRow.size());
5275 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowC));
5276 assertTrue(hasNext);
5277 currRow.clear();
5278 hasNext = scanner.next(currRow);
5279 assertEquals(1, currRow.size());
5280 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowB));
5281 assertFalse(hasNext);
5282 scanner.close();
5283
5284 scan = new Scan(rowD, rowA);
5285 scan.addColumn(families[0], col2);
5286 scan.setReversed(true);
5287 currRow.clear();
5288 scanner = region.getScanner(scan);
5289 hasNext = scanner.next(currRow);
5290 assertEquals(1, currRow.size());
5291 assertTrue(Bytes.equals(currRow.get(0).getRow(), rowD));
5292 scanner.close();
5293 }
5294
5295 @Test (timeout=60000)
5296 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1()
5297 throws IOException {
5298 byte[] row0 = Bytes.toBytes("row0");
5299 byte[] row1 = Bytes.toBytes("row1");
5300 byte[] row2 = Bytes.toBytes("row2");
5301 byte[] row3 = Bytes.toBytes("row3");
5302 byte[] row4 = Bytes.toBytes("row4");
5303 byte[] row5 = Bytes.toBytes("row5");
5304 byte[] cf1 = Bytes.toBytes("CF1");
5305 byte[] cf2 = Bytes.toBytes("CF2");
5306 byte[] cf3 = Bytes.toBytes("CF3");
5307 byte[][] families = { cf1, cf2, cf3 };
5308 byte[] col = Bytes.toBytes("C");
5309 long ts = 1;
5310 String method = this.getName();
5311 HBaseConfiguration conf = new HBaseConfiguration();
5312
5313 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5314 this.region = initHRegion(tableName, method, conf, families);
5315
5316 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put,
5317 null);
5318 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put,
5319 null);
5320 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1,
5321 KeyValue.Type.Put, null);
5322 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put,
5323 null);
5324 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put,
5325 null);
5326 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put,
5327 null);
5328 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4,
5329 KeyValue.Type.Put, null);
5330 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put,
5331 null);
5332 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4,
5333 KeyValue.Type.Put, null);
5334 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put,
5335 null);
5336 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put,
5337 null);
5338 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5,
5339 KeyValue.Type.Put, null);
5340 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put,
5341 null);
5342 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3,
5343 KeyValue.Type.Put, null);
5344 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put,
5345 null);
5346 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put,
5347 null);
5348
5349 Put put = null;
5350 put = new Put(row1);
5351 put.add(kv1_2_1);
5352 region.put(put);
5353 put = new Put(row2);
5354 put.add(kv2_4_1);
5355 region.put(put);
5356 put = new Put(row4);
5357 put.add(kv4_5_4);
5358 put.add(kv4_5_5);
5359 region.put(put);
5360 region.flush(true);
5361
5362 put = new Put(row4);
5363 put.add(kv4_5_1);
5364 put.add(kv4_5_3);
5365 region.put(put);
5366 put = new Put(row1);
5367 put.add(kv1_2_2);
5368 region.put(put);
5369 put = new Put(row2);
5370 put.add(kv2_4_4);
5371 region.put(put);
5372 region.flush(true);
5373
5374 put = new Put(row4);
5375 put.add(kv4_5_2);
5376 region.put(put);
5377 put = new Put(row2);
5378 put.add(kv2_4_2);
5379 put.add(kv2_4_3);
5380 region.put(put);
5381 put = new Put(row3);
5382 put.add(kv3_2_2);
5383 region.put(put);
5384 region.flush(true);
5385
5386
5387 put = new Put(row0);
5388 put.add(kv0_1_1);
5389 region.put(put);
5390 put = new Put(row3);
5391 put.add(kv3_2_1);
5392 region.put(put);
5393 put = new Put(row5);
5394 put.add(kv5_2_1);
5395 put.add(kv5_2_2);
5396 region.put(put);
5397
5398 Scan scan = new Scan(row4);
5399 scan.setMaxVersions(5);
5400 scan.setBatch(3);
5401 scan.setReversed(true);
5402 InternalScanner scanner = region.getScanner(scan);
5403 List<Cell> currRow = new ArrayList<Cell>();
5404 boolean hasNext = false;
5405
5406
5407
5408 hasNext = scanner.next(currRow);
5409 assertEquals(3, currRow.size());
5410 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5411 assertTrue(hasNext);
5412 currRow.clear();
5413 hasNext = scanner.next(currRow);
5414 assertEquals(2, currRow.size());
5415 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5416 assertTrue(hasNext);
5417
5418 currRow.clear();
5419 hasNext = scanner.next(currRow);
5420 assertEquals(2, currRow.size());
5421 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5422 assertTrue(hasNext);
5423
5424
5425 currRow.clear();
5426 hasNext = scanner.next(currRow);
5427 assertEquals(3, currRow.size());
5428 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5429 assertTrue(hasNext);
5430 currRow.clear();
5431 hasNext = scanner.next(currRow);
5432 assertEquals(1, currRow.size());
5433 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5434 assertTrue(hasNext);
5435
5436 currRow.clear();
5437 hasNext = scanner.next(currRow);
5438 assertEquals(2, currRow.size());
5439 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5440 assertTrue(hasNext);
5441
5442 currRow.clear();
5443 hasNext = scanner.next(currRow);
5444 assertEquals(1, currRow.size());
5445 assertTrue(Bytes.equals(currRow.get(0).getRow(), row0));
5446 assertFalse(hasNext);
5447
5448 scanner.close();
5449 }
5450
5451 @Test (timeout=60000)
5452 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2()
5453 throws IOException {
5454 byte[] row1 = Bytes.toBytes("row1");
5455 byte[] row2 = Bytes.toBytes("row2");
5456 byte[] row3 = Bytes.toBytes("row3");
5457 byte[] row4 = Bytes.toBytes("row4");
5458 byte[] cf1 = Bytes.toBytes("CF1");
5459 byte[] cf2 = Bytes.toBytes("CF2");
5460 byte[] cf3 = Bytes.toBytes("CF3");
5461 byte[] cf4 = Bytes.toBytes("CF4");
5462 byte[][] families = { cf1, cf2, cf3, cf4 };
5463 byte[] col = Bytes.toBytes("C");
5464 long ts = 1;
5465 String method = this.getName();
5466 HBaseConfiguration conf = new HBaseConfiguration();
5467
5468 conf.setInt("hbase.hstore.compactionThreshold", 10000);
5469 this.region = initHRegion(tableName, method, conf, families);
5470 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
5471 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
5472 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
5473 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
5474
5475 Put put = new Put(row1);
5476 put.add(kv1);
5477 region.put(put);
5478 region.flush(true);
5479
5480 put = new Put(row2);
5481 put.add(kv2);
5482 region.put(put);
5483 region.flush(true);
5484
5485 put = new Put(row3);
5486 put.add(kv3);
5487 region.put(put);
5488 region.flush(true);
5489
5490 put = new Put(row4);
5491 put.add(kv4);
5492 region.put(put);
5493
5494 Scan scan = new Scan(row4);
5495 scan.setReversed(true);
5496 scan.setBatch(10);
5497 InternalScanner scanner = region.getScanner(scan);
5498 List<Cell> currRow = new ArrayList<Cell>();
5499 boolean hasNext = scanner.next(currRow);
5500 assertEquals(1, currRow.size());
5501 assertTrue(Bytes.equals(currRow.get(0).getRow(), row4));
5502 assertTrue(hasNext);
5503 currRow.clear();
5504 hasNext = scanner.next(currRow);
5505 assertEquals(1, currRow.size());
5506 assertTrue(Bytes.equals(currRow.get(0).getRow(), row3));
5507 assertTrue(hasNext);
5508 currRow.clear();
5509 hasNext = scanner.next(currRow);
5510 assertEquals(1, currRow.size());
5511 assertTrue(Bytes.equals(currRow.get(0).getRow(), row2));
5512 assertTrue(hasNext);
5513 currRow.clear();
5514 hasNext = scanner.next(currRow);
5515 assertEquals(1, currRow.size());
5516 assertTrue(Bytes.equals(currRow.get(0).getRow(), row1));
5517 assertFalse(hasNext);
5518 }
5519
5520
5521
5522
5523 @Test (timeout = 60000)
5524 public void testReverseScanner_StackOverflow() throws IOException {
5525 byte[] cf1 = Bytes.toBytes("CF1");
5526 byte[][] families = {cf1};
5527 byte[] col = Bytes.toBytes("C");
5528 String method = this.getName();
5529 HBaseConfiguration conf = new HBaseConfiguration();
5530 this.region = initHRegion(tableName, method, conf, families);
5531
5532 Put put = new Put(Bytes.toBytes("19998"));
5533 put.add(cf1, col, Bytes.toBytes("val"));
5534 region.put(put);
5535 region.flushcache(true, true);
5536 Put put2 = new Put(Bytes.toBytes("19997"));
5537 put2.add(cf1, col, Bytes.toBytes("val"));
5538 region.put(put2);
5539
5540 Scan scan = new Scan(Bytes.toBytes("19998"));
5541 scan.setReversed(true);
5542 InternalScanner scanner = region.getScanner(scan);
5543
5544
5545
5546 for (int i = 10000; i < 20000; i++) {
5547 Put p = new Put(Bytes.toBytes("" + i));
5548 p.add(cf1, col, Bytes.toBytes("" + i));
5549 region.put(p);
5550 }
5551 region.flushcache(true, true);
5552
5553
5554
5555 for (int i = 10000; i < 20000; i++) {
5556 Put p = new Put(Bytes.toBytes("" + i));
5557 p.add(cf1, col, Bytes.toBytes("" + i));
5558 region.put(p);
5559 }
5560
5561 List<Cell> currRow = new ArrayList<>();
5562 boolean hasNext;
5563 do {
5564 hasNext = scanner.next(currRow);
5565 } while (hasNext);
5566 assertEquals(2, currRow.size());
5567 assertArrayEquals(Bytes.toBytes("19998"), currRow.get(0).getRow());
5568 assertArrayEquals(Bytes.toBytes("19997"), currRow.get(1).getRow());
5569 }
5570
5571 @Test (timeout=60000)
5572 public void testSplitRegionWithReverseScan() throws IOException {
5573 byte [] tableName = Bytes.toBytes("testSplitRegionWithReverseScan");
5574 byte [] qualifier = Bytes.toBytes("qualifier");
5575 Configuration hc = initSplit();
5576 int numRows = 3;
5577 byte [][] families = {fam1};
5578
5579
5580 String method = this.getName();
5581 this.region = initHRegion(tableName, method, hc, families);
5582
5583
5584 int startRow = 100;
5585 putData(startRow, numRows, qualifier, families);
5586 int splitRow = startRow + numRows;
5587 putData(splitRow, numRows, qualifier, families);
5588 region.flush(true);
5589
5590 HRegion [] regions = null;
5591 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
5592
5593 for (int i = 0; i < regions.length; i++) {
5594 regions[i] = HRegion.openHRegion(regions[i], null);
5595 }
5596
5597 assertEquals(2, regions.length);
5598
5599
5600
5601 verifyData(regions[0], startRow, numRows, qualifier, families);
5602 verifyData(regions[1], splitRow, numRows, qualifier, families);
5603
5604
5605 Scan scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 10 * numRows)));
5606 scan.setReversed(true);
5607 InternalScanner scanner = regions[1].getScanner(scan);
5608 List<Cell> currRow = new ArrayList<Cell>();
5609 boolean more = false;
5610 int verify = startRow + 2 * numRows - 1;
5611 do {
5612 more = scanner.next(currRow);
5613 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5614 verify--;
5615 currRow.clear();
5616 } while(more);
5617 assertEquals(verify, startRow + numRows - 1);
5618 scanner.close();
5619
5620 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + 2 * numRows - 1)));
5621 scan.setReversed(true);
5622 scanner = regions[1].getScanner(scan);
5623 verify = startRow + 2 * numRows - 1;
5624 do {
5625 more = scanner.next(currRow);
5626 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5627 verify--;
5628 currRow.clear();
5629 } while(more);
5630 assertEquals(verify, startRow + numRows - 1);
5631 scanner.close();
5632
5633 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows)));
5634 scan.setReversed(true);
5635 scanner = regions[0].getScanner(scan);
5636 verify = startRow + numRows - 1;
5637 do {
5638 more = scanner.next(currRow);
5639 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5640 verify--;
5641 currRow.clear();
5642 } while(more);
5643 assertEquals(verify, 99);
5644 scanner.close();
5645
5646 scan = new Scan(Bytes.toBytes(String.valueOf(startRow + numRows - 1)));
5647 scan.setReversed(true);
5648 scanner = regions[0].getScanner(scan);
5649 verify = startRow + numRows - 1;
5650 do {
5651 more = scanner.next(currRow);
5652 assertEquals(Bytes.toString(currRow.get(0).getRow()), verify + "");
5653 verify--;
5654 currRow.clear();
5655 } while(more);
5656 assertEquals(verify, startRow - 1);
5657 scanner.close();
5658 }
5659
5660 @Test
5661 public void testWriteRequestsCounter() throws IOException {
5662 byte[] fam = Bytes.toBytes("info");
5663 byte[][] families = { fam };
5664 this.region = initHRegion(tableName, method, CONF, families);
5665
5666 Assert.assertEquals(0L, region.getWriteRequestsCount());
5667
5668 Put put = new Put(row);
5669 put.add(fam, fam, fam);
5670
5671 Assert.assertEquals(0L, region.getWriteRequestsCount());
5672 region.put(put);
5673 Assert.assertEquals(1L, region.getWriteRequestsCount());
5674 region.put(put);
5675 Assert.assertEquals(2L, region.getWriteRequestsCount());
5676 region.put(put);
5677 Assert.assertEquals(3L, region.getWriteRequestsCount());
5678
5679 region.delete(new Delete(row));
5680 Assert.assertEquals(4L, region.getWriteRequestsCount());
5681 }
5682
5683 @Test
5684 public void testOpenRegionWrittenToWAL() throws Exception {
5685 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWAL", 100, 42);
5686 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5687
5688 HTableDescriptor htd
5689 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5690 htd.addFamily(new HColumnDescriptor(fam1));
5691 htd.addFamily(new HColumnDescriptor(fam2));
5692
5693 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5694 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5695
5696
5697 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(),
5698 htd);
5699 assertNotNull(region);
5700
5701
5702 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5703 region.flush(true);
5704 HBaseTestingUtility.closeRegionAndWAL(region);
5705
5706 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5707
5708
5709 WAL wal = mockWAL();
5710 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5711
5712 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5713 TEST_UTIL.getConfiguration(), rss, null);
5714
5715 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5716 , editCaptor.capture(), anyBoolean());
5717
5718 WALEdit edit = editCaptor.getValue();
5719 assertNotNull(edit);
5720 assertNotNull(edit.getCells());
5721 assertEquals(1, edit.getCells().size());
5722 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5723 assertNotNull(desc);
5724
5725 LOG.info("RegionEventDescriptor from WAL: " + desc);
5726
5727 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5728 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5729 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5730 hri.getEncodedNameAsBytes()));
5731 assertTrue(desc.getLogSequenceNumber() > 0);
5732 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5733 assertEquals(2, desc.getStoresCount());
5734
5735 StoreDescriptor store = desc.getStores(0);
5736 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5737 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5738 assertEquals(1, store.getStoreFileCount());
5739 assertFalse(store.getStoreFile(0).contains("/"));
5740
5741 store = desc.getStores(1);
5742 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5743 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5744 assertEquals(0, store.getStoreFileCount());
5745 }
5746
5747
5748 static class HRegionWithSeqId extends HRegion {
5749 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
5750 final Configuration confParam, final HRegionInfo regionInfo,
5751 final HTableDescriptor htd, final RegionServerServices rsServices) {
5752 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
5753 }
5754 @Override
5755 protected long getNextSequenceId(WAL wal) throws IOException {
5756 return 42;
5757 }
5758 }
5759
5760 @Test
5761 public void testFlushedFileWithNoTags() throws Exception {
5762 String method = "testFlushedFileWithNoTags";
5763 HTableDescriptor htd = new HTableDescriptor(tableName);
5764 htd.addFamily(new HColumnDescriptor(fam1));
5765 region = initHRegion(Bytes.toBytes(method), method, TEST_UTIL.getConfiguration(), fam1);
5766 Put put = new Put(Bytes.toBytes("a-b-0-0"));
5767 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
5768 region.put(put);
5769 region.flush(true);
5770 Store store = region.getStore(fam1);
5771 Collection<StoreFile> storefiles = store.getStorefiles();
5772 for (StoreFile sf : storefiles) {
5773 assertFalse("Tags should not be present "
5774 ,sf.getReader().getHFileReader().getFileContext().isIncludesTags());
5775 }
5776 }
5777
5778 @Test
5779 public void testOpenRegionWrittenToWALForLogReplay() throws Exception {
5780
5781 final ServerName serverName = ServerName.valueOf("testOpenRegionWrittenToWALForLogReplay",
5782 100, 42);
5783 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5784
5785 HTableDescriptor htd
5786 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWALForLogReplay"));
5787 htd.addFamily(new HColumnDescriptor(fam1));
5788 htd.addFamily(new HColumnDescriptor(fam2));
5789
5790 HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5791 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5792
5793
5794 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL
5795 .getConfiguration(), htd);
5796 assertNotNull(region);
5797
5798
5799 region.put(new Put(Bytes.toBytes("a")).add(fam1, fam1, fam1));
5800 region.flush(true);
5801 HRegion.closeHRegion(region);
5802
5803 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5804
5805
5806 WAL wal = mockWAL();
5807 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5808
5809
5810 HashMap<String, Region> recoveringRegions = Maps.newHashMap();
5811 recoveringRegions.put(region.getRegionInfo().getEncodedName(), null);
5812 when(rss.getRecoveringRegions()).thenReturn(recoveringRegions);
5813
5814 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
5815 conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName());
5816 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5817 conf, rss, null);
5818
5819
5820
5821 verify(wal, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5822 , editCaptor.capture(), anyBoolean());
5823
5824
5825 new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo")
5826 .prepare().process();
5827
5828
5829 verify(wal, times(1)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any()
5830 , editCaptor.capture(), anyBoolean());
5831
5832 WALEdit edit = editCaptor.getValue();
5833 assertNotNull(edit);
5834 assertNotNull(edit.getCells());
5835 assertEquals(1, edit.getCells().size());
5836 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5837 assertNotNull(desc);
5838
5839 LOG.info("RegionEventDescriptor from WAL: " + desc);
5840
5841 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
5842 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5843 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5844 hri.getEncodedNameAsBytes()));
5845 assertTrue(desc.getLogSequenceNumber() > 0);
5846 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5847 assertEquals(2, desc.getStoresCount());
5848
5849 StoreDescriptor store = desc.getStores(0);
5850 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5851 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5852 assertEquals(1, store.getStoreFileCount());
5853 assertFalse(store.getStoreFile(0).contains("/"));
5854
5855 store = desc.getStores(1);
5856 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5857 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5858 assertEquals(0, store.getStoreFileCount());
5859 }
5860
5861
5862
5863
5864
5865
5866
5867 private WAL mockWAL() throws IOException {
5868 WAL wal = mock(WAL.class);
5869 Mockito.when(wal.append((HTableDescriptor)Mockito.any(), (HRegionInfo)Mockito.any(),
5870 (WALKey)Mockito.any(), (WALEdit)Mockito.any(), Mockito.anyBoolean())).
5871 thenAnswer(new Answer<Long>() {
5872 @Override
5873 public Long answer(InvocationOnMock invocation) throws Throwable {
5874 WALKey key = invocation.getArgumentAt(2, WALKey.class);
5875 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
5876 key.setWriteEntry(we);
5877 return 1L;
5878 }
5879
5880 });
5881 return wal;
5882 }
5883
5884 @Test
5885 public void testCloseRegionWrittenToWAL() throws Exception {
5886
5887 Path rootDir = new Path(dir + name.getMethodName());
5888 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5889
5890 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
5891 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
5892
5893 HTableDescriptor htd
5894 = new HTableDescriptor(TableName.valueOf("testOpenRegionWrittenToWAL"));
5895 htd.addFamily(new HColumnDescriptor(fam1));
5896 htd.addFamily(new HColumnDescriptor(fam2));
5897
5898 final HRegionInfo hri = new HRegionInfo(htd.getTableName(),
5899 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY);
5900
5901 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
5902
5903
5904 WAL wal = mockWAL();
5905 when(rss.getWAL((HRegionInfo) any())).thenReturn(wal);
5906
5907
5908
5909 region = HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri));
5910 region = HRegion.openHRegion(hri, htd, rss.getWAL(hri),
5911 TEST_UTIL.getConfiguration(), rss, null);
5912
5913
5914 region.close(false);
5915
5916
5917 verify(wal, times(2)).append((HTableDescriptor)any(), (HRegionInfo)any(), (WALKey)any(),
5918 editCaptor.capture(), anyBoolean());
5919
5920 WALEdit edit = editCaptor.getAllValues().get(1);
5921 assertNotNull(edit);
5922 assertNotNull(edit.getCells());
5923 assertEquals(1, edit.getCells().size());
5924 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
5925 assertNotNull(desc);
5926
5927 LOG.info("RegionEventDescriptor from WAL: " + desc);
5928
5929 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
5930 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getName()));
5931 assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(),
5932 hri.getEncodedNameAsBytes()));
5933 assertTrue(desc.getLogSequenceNumber() > 0);
5934 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
5935 assertEquals(2, desc.getStoresCount());
5936
5937 StoreDescriptor store = desc.getStores(0);
5938 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
5939 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
5940 assertEquals(0, store.getStoreFileCount());
5941
5942 store = desc.getStores(1);
5943 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
5944 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
5945 assertEquals(0, store.getStoreFileCount());
5946 }
5947
5948
5949
5950
5951 @Test (timeout=24000)
5952 public void testRegionTooBusy() throws IOException {
5953 String method = "testRegionTooBusy";
5954 byte[] tableName = Bytes.toBytes(method);
5955 byte[] family = Bytes.toBytes("family");
5956 long defaultBusyWaitDuration = CONF.getLong("hbase.busy.wait.duration",
5957 HRegion.DEFAULT_BUSY_WAIT_DURATION);
5958 CONF.setLong("hbase.busy.wait.duration", 1000);
5959 region = initHRegion(tableName, method, CONF, family);
5960 final AtomicBoolean stopped = new AtomicBoolean(true);
5961 Thread t = new Thread(new Runnable() {
5962 @Override
5963 public void run() {
5964 try {
5965 region.lock.writeLock().lock();
5966 stopped.set(false);
5967 while (!stopped.get()) {
5968 Thread.sleep(100);
5969 }
5970 } catch (InterruptedException ie) {
5971 } finally {
5972 region.lock.writeLock().unlock();
5973 }
5974 }
5975 });
5976 t.start();
5977 Get get = new Get(row);
5978 try {
5979 while (stopped.get()) {
5980 Thread.sleep(100);
5981 }
5982 region.get(get);
5983 fail("Should throw RegionTooBusyException");
5984 } catch (InterruptedException ie) {
5985 fail("test interrupted");
5986 } catch (RegionTooBusyException e) {
5987
5988 } finally {
5989 stopped.set(true);
5990 try {
5991 t.join();
5992 } catch (Throwable e) {
5993 }
5994
5995 HRegion.closeHRegion(region);
5996 region = null;
5997 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
5998 }
5999 }
6000
6001 @Test
6002 public void testCellTTLs() throws IOException {
6003 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6004 EnvironmentEdgeManager.injectEdge(edge);
6005
6006 final byte[] row = Bytes.toBytes("testRow");
6007 final byte[] q1 = Bytes.toBytes("q1");
6008 final byte[] q2 = Bytes.toBytes("q2");
6009 final byte[] q3 = Bytes.toBytes("q3");
6010 final byte[] q4 = Bytes.toBytes("q4");
6011
6012 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCellTTLs"));
6013 HColumnDescriptor hcd = new HColumnDescriptor(fam1);
6014 hcd.setTimeToLive(10);
6015 htd.addFamily(hcd);
6016
6017 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6018 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6019
6020 region = HRegion.createHRegion(new HRegionInfo(htd.getTableName(),
6021 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
6022 TEST_UTIL.getDataTestDir(), conf, htd);
6023 assertNotNull(region);
6024 long now = EnvironmentEdgeManager.currentTime();
6025
6026 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now,
6027 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6028
6029 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6030
6031 region.put(new Put(row).add(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6032
6033 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6034 HConstants.EMPTY_BYTE_ARRAY, new Tag[] {
6035
6036 new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) } )));
6037
6038 region.put(new Put(row).add(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6039
6040
6041 region.flush(true);
6042
6043
6044 Result r = region.get(new Get(row));
6045 assertNotNull(r.getValue(fam1, q1));
6046 assertNotNull(r.getValue(fam1, q2));
6047 assertNotNull(r.getValue(fam1, q3));
6048 assertNotNull(r.getValue(fam1, q4));
6049
6050
6051 edge.incrementTime(5000);
6052
6053 r = region.get(new Get(row));
6054 assertNull(r.getValue(fam1, q1));
6055 assertNotNull(r.getValue(fam1, q2));
6056 assertNotNull(r.getValue(fam1, q3));
6057 assertNotNull(r.getValue(fam1, q4));
6058
6059
6060 edge.incrementTime(5000);
6061
6062 r = region.get(new Get(row));
6063 assertNull(r.getValue(fam1, q1));
6064 assertNull(r.getValue(fam1, q2));
6065 assertNotNull(r.getValue(fam1, q3));
6066 assertNotNull(r.getValue(fam1, q4));
6067
6068
6069 edge.incrementTime(5000);
6070
6071 r = region.get(new Get(row));
6072 assertNull(r.getValue(fam1, q1));
6073 assertNull(r.getValue(fam1, q2));
6074 assertNull(r.getValue(fam1, q3));
6075 assertNotNull(r.getValue(fam1, q4));
6076
6077
6078 edge.incrementTime(10000);
6079
6080 r = region.get(new Get(row));
6081 assertNull(r.getValue(fam1, q1));
6082 assertNull(r.getValue(fam1, q2));
6083 assertNull(r.getValue(fam1, q3));
6084 assertNull(r.getValue(fam1, q4));
6085
6086
6087
6088
6089 region.put(new Put(row).add(fam1, q1, Bytes.toBytes(1L)));
6090 r = region.get(new Get(row));
6091 byte[] val = r.getValue(fam1, q1);
6092 assertNotNull(val);
6093 assertEquals(Bytes.toLong(val), 1L);
6094
6095
6096 Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6097 incr.setTTL(5000);
6098 region.increment(incr);
6099
6100
6101 r = region.get(new Get(row));
6102 val = r.getValue(fam1, q1);
6103 assertNotNull(val);
6104 assertEquals(Bytes.toLong(val), 2L);
6105
6106
6107 edge.incrementTime(5000);
6108
6109
6110 r = region.get(new Get(row));
6111 val = r.getValue(fam1, q1);
6112 assertNotNull(val);
6113 assertEquals(Bytes.toLong(val), 1L);
6114
6115
6116 edge.incrementTime(5000);
6117
6118
6119 r = region.get(new Get(row));
6120 assertNull(r.getValue(fam1, q1));
6121 }
6122
6123 @Test
6124 public void testIncrementTimestampsAreMonotonic() throws IOException {
6125 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6126 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6127 EnvironmentEdgeManager.injectEdge(edge);
6128
6129 edge.setValue(10);
6130 Increment inc = new Increment(row);
6131 inc.setDurability(Durability.SKIP_WAL);
6132 inc.addColumn(fam1, qual1, 1L);
6133 region.increment(inc);
6134
6135 Result result = region.get(new Get(row));
6136 Cell c = result.getColumnLatestCell(fam1, qual1);
6137 assertNotNull(c);
6138 assertEquals(c.getTimestamp(), 10L);
6139
6140 edge.setValue(1);
6141 region.increment(inc);
6142 result = region.get(new Get(row));
6143 c = result.getColumnLatestCell(fam1, qual1);
6144 assertEquals(c.getTimestamp(), 11L);
6145 assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
6146 }
6147
6148 @Test
6149 public void testAppendTimestampsAreMonotonic() throws IOException {
6150 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6151 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6152 EnvironmentEdgeManager.injectEdge(edge);
6153
6154 edge.setValue(10);
6155 Append a = new Append(row);
6156 a.setDurability(Durability.SKIP_WAL);
6157 a.add(fam1, qual1, qual1);
6158 region.append(a);
6159
6160 Result result = region.get(new Get(row));
6161 Cell c = result.getColumnLatestCell(fam1, qual1);
6162 assertNotNull(c);
6163 assertEquals(c.getTimestamp(), 10L);
6164
6165 edge.setValue(1);
6166 region.append(a);
6167 result = region.get(new Get(row));
6168 c = result.getColumnLatestCell(fam1, qual1);
6169 assertEquals(c.getTimestamp(), 11L);
6170
6171 byte[] expected = new byte[qual1.length*2];
6172 System.arraycopy(qual1, 0, expected, 0, qual1.length);
6173 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
6174
6175 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6176 expected, 0, expected.length));
6177 }
6178
6179 @Test
6180 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
6181 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6182 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6183 EnvironmentEdgeManager.injectEdge(edge);
6184
6185 edge.setValue(10);
6186 Put p = new Put(row);
6187 p.setDurability(Durability.SKIP_WAL);
6188 p.addColumn(fam1, qual1, qual1);
6189 region.put(p);
6190
6191 Result result = region.get(new Get(row));
6192 Cell c = result.getColumnLatestCell(fam1, qual1);
6193 assertNotNull(c);
6194 assertEquals(c.getTimestamp(), 10L);
6195
6196 edge.setValue(1);
6197 p = new Put(row);
6198 p.setDurability(Durability.SKIP_WAL);
6199 p.addColumn(fam1, qual1, qual2);
6200 region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1), p, false);
6201 result = region.get(new Get(row));
6202 c = result.getColumnLatestCell(fam1, qual1);
6203 assertEquals(c.getTimestamp(), 10L);
6204
6205 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6206 qual2, 0, qual2.length));
6207 }
6208
6209 @Test(timeout = 60000)
6210 public void testBatchMutateWithWrongRegionException() throws Exception {
6211 final byte[] a = Bytes.toBytes("a");
6212 final byte[] b = Bytes.toBytes("b");
6213 final byte[] c = Bytes.toBytes("c");
6214
6215 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
6216 CONF.setInt("hbase.rowlock.wait.duration", 1000);
6217 region = initHRegion(tableName, a, c, name.getMethodName(), CONF, false, fam1);
6218
6219 Mutation[] mutations = new Mutation[] {
6220 new Put(a).addImmutable(fam1, null, null),
6221 new Put(c).addImmutable(fam1, null, null),
6222 new Put(b).addImmutable(fam1, null, null),
6223 };
6224
6225 OperationStatus[] status = region.batchMutate(mutations);
6226 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6227 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
6228 assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6229
6230
6231
6232 final CountDownLatch obtainedRowLock = new CountDownLatch(1);
6233 ExecutorService exec = Executors.newFixedThreadPool(2);
6234 Future<Void> f1 = exec.submit(new Callable<Void>() {
6235 @Override
6236 public Void call() throws Exception {
6237 LOG.info("Acquiring row lock");
6238 RowLock rl = region.getRowLock(b);
6239 obtainedRowLock.countDown();
6240 LOG.info("Waiting for 5 seconds before releasing lock");
6241 Threads.sleep(5000);
6242 LOG.info("Releasing row lock");
6243 rl.release();
6244 return null;
6245 }
6246 });
6247 obtainedRowLock.await(30, TimeUnit.SECONDS);
6248
6249 Future<Void> f2 = exec.submit(new Callable<Void>() {
6250 @Override
6251 public Void call() throws Exception {
6252 Mutation[] mutations = new Mutation[] {
6253 new Put(a).addImmutable(fam1, null, null),
6254 new Put(b).addImmutable(fam1, null, null),
6255 };
6256
6257
6258 OperationStatus[] status = region.batchMutate(mutations);
6259 assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6260 assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
6261 return null;
6262 }
6263 });
6264
6265 f1.get();
6266 f2.get();
6267
6268 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
6269 }
6270
6271 @Test
6272 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
6273 region = initHRegion(tableName, name.getMethodName(), CONF, fam1);
6274 ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6275 EnvironmentEdgeManager.injectEdge(edge);
6276
6277 edge.setValue(10);
6278 Put p = new Put(row);
6279 p.setDurability(Durability.SKIP_WAL);
6280 p.addColumn(fam1, qual1, qual1);
6281 region.put(p);
6282
6283 Result result = region.get(new Get(row));
6284 Cell c = result.getColumnLatestCell(fam1, qual1);
6285 assertNotNull(c);
6286 assertEquals(c.getTimestamp(), 10L);
6287
6288 edge.setValue(1);
6289 p = new Put(row);
6290 p.setDurability(Durability.SKIP_WAL);
6291 p.addColumn(fam1, qual1, qual2);
6292 RowMutations rm = new RowMutations(row);
6293 rm.add(p);
6294 region.checkAndRowMutate(row, fam1, qual1, CompareOp.EQUAL, new BinaryComparator(qual1),
6295 rm, false);
6296 result = region.get(new Get(row));
6297 c = result.getColumnLatestCell(fam1, qual1);
6298 assertEquals(c.getTimestamp(), 10L);
6299
6300 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
6301 qual2, 0, qual2.length));
6302 }
6303
6304 static HRegion initHRegion(byte[] tableName, String callingMethod,
6305 byte[]... families) throws IOException {
6306 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(),
6307 families);
6308 }
6309
6310
6311
6312
6313
6314 @Test
6315 public void testWritesWhileRollWriter() throws IOException {
6316 int testCount = 10;
6317 int numRows = 1024;
6318 int numFamilies = 2;
6319 int numQualifiers = 2;
6320 final byte[][] families = new byte[numFamilies][];
6321 for (int i = 0; i < numFamilies; i++) {
6322 families[i] = Bytes.toBytes("family" + i);
6323 }
6324 final byte[][] qualifiers = new byte[numQualifiers][];
6325 for (int i = 0; i < numQualifiers; i++) {
6326 qualifiers[i] = Bytes.toBytes("qual" + i);
6327 }
6328
6329 String method = "testWritesWhileRollWriter";
6330 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2);
6331 this.region = initHRegion(tableName, method, CONF, families);
6332 try {
6333 List<Thread> threads = new ArrayList<Thread>();
6334 for (int i = 0; i < numRows; i++) {
6335 final int count = i;
6336 Thread t = new Thread(new Runnable() {
6337
6338 @Override
6339 public void run() {
6340 byte[] row = Bytes.toBytes("row" + count);
6341 Put put = new Put(row);
6342 put.setDurability(Durability.SYNC_WAL);
6343 byte[] value = Bytes.toBytes(String.valueOf(count));
6344 for (byte[] family : families) {
6345 for (byte[] qualifier : qualifiers) {
6346 put.addColumn(family, qualifier, (long) count, value);
6347 }
6348 }
6349 try {
6350 region.put(put);
6351 } catch (IOException e) {
6352 throw new RuntimeException(e);
6353 }
6354 }
6355 });
6356 threads.add(t);
6357 }
6358 for (Thread t : threads) {
6359 t.start();
6360 }
6361
6362 for (int i = 0; i < testCount; i++) {
6363 region.getWAL().rollWriter();
6364 Thread.yield();
6365 }
6366 } finally {
6367 try {
6368 HBaseTestingUtility.closeRegionAndWAL(this.region);
6369 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024);
6370 } catch (DroppedSnapshotException dse) {
6371
6372
6373
6374
6375 }
6376 this.region = null;
6377 }
6378 }
6379
6380 @Test
6381 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception {
6382 byte[] cf1 = Bytes.toBytes("CF1");
6383 byte[][] families = { cf1 };
6384 byte[] col = Bytes.toBytes("C");
6385 String method = this.getName();
6386 HBaseConfiguration conf = new HBaseConfiguration();
6387 this.region = initHRegion(tableName, method, conf, families);
6388
6389 Put put = new Put(Bytes.toBytes("19996"));
6390 put.addColumn(cf1, col, Bytes.toBytes("val"));
6391 region.put(put);
6392 Put put2 = new Put(Bytes.toBytes("19995"));
6393 put2.addColumn(cf1, col, Bytes.toBytes("val"));
6394 region.put(put2);
6395
6396
6397 Scan scan = new Scan(Bytes.toBytes("19996"));
6398 scan.setReversed(true);
6399 RegionScanner scanner = region.getScanner(scan);
6400
6401
6402 region.flushcache(true, true);
6403
6404
6405
6406 for (int i = 10000; i < 20000; i++) {
6407 Put p = new Put(Bytes.toBytes("" + i));
6408 p.addColumn(cf1, col, Bytes.toBytes("" + i));
6409 region.put(p);
6410 }
6411
6412 List<Cell> currRow = new ArrayList<>();
6413 boolean hasNext;
6414 boolean assertDone = false;
6415 do {
6416 hasNext = scanner.next(currRow);
6417
6418
6419 if (!assertDone) {
6420 StoreScanner current =
6421 (StoreScanner) (((RegionScannerImpl) scanner).storeHeap).getCurrentForTesting();
6422 List<KeyValueScanner> scanners = current.getAllScannersForTesting();
6423 assertEquals("There should be only one scanner the store file scanner", 1,
6424 scanners.size());
6425 assertDone = true;
6426 }
6427 } while (hasNext);
6428
6429 assertEquals(2, currRow.size());
6430 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(),
6431 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6432 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(),
6433 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6434 }
6435
6436 @Test
6437 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception {
6438 byte[] cf1 = Bytes.toBytes("CF1");
6439 byte[][] families = { cf1 };
6440 byte[] col = Bytes.toBytes("C");
6441
6442 HBaseConfiguration conf = new HBaseConfiguration();
6443 this.region = initHRegion(tableName, method, conf, families);
6444
6445 Put put = new Put(Bytes.toBytes("199996"));
6446 put.addColumn(cf1, col, Bytes.toBytes("val"));
6447 region.put(put);
6448 Put put2 = new Put(Bytes.toBytes("199995"));
6449 put2.addColumn(cf1, col, Bytes.toBytes("val"));
6450 region.put(put2);
6451
6452
6453 Scan scan = new Scan(Bytes.toBytes("199996"));
6454 scan.setReversed(true);
6455 RegionScanner scanner = region.getScanner(scan);
6456
6457
6458 for (int i = 100000; i < 200000; i++) {
6459 Put p = new Put(Bytes.toBytes("" + i));
6460 p.addColumn(cf1, col, Bytes.toBytes("" + i));
6461 region.put(p);
6462 }
6463 List<Cell> currRow = new ArrayList<>();
6464 boolean hasNext;
6465 do {
6466 hasNext = scanner.next(currRow);
6467 } while (hasNext);
6468
6469 assertEquals(2, currRow.size());
6470 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(),
6471 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6472 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(),
6473 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6474 }
6475
6476 @Test
6477 public void testMutateRow_WriteRequestCount() throws Exception {
6478 byte[] row1 = Bytes.toBytes("row1");
6479 byte[] fam1 = Bytes.toBytes("fam1");
6480 byte[] qf1 = Bytes.toBytes("qualifier");
6481 byte[] val1 = Bytes.toBytes("value1");
6482
6483 RowMutations rm = new RowMutations(row1);
6484 Put put = new Put(row1);
6485 put.addColumn(fam1, qf1, val1);
6486 rm.add(put);
6487
6488 this.region = initHRegion(tableName, method, CONF, fam1);
6489 try {
6490 long wrcBeforeMutate = this.region.writeRequestsCount.get();
6491 this.region.mutateRow(rm);
6492 long wrcAfterMutate = this.region.writeRequestsCount.get();
6493 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate);
6494 } finally {
6495 HBaseTestingUtility.closeRegionAndWAL(this.region);
6496 this.region = null;
6497 }
6498 }
6499 }